1
0

Generalizing notification system

This commit is contained in:
Jon Michael Aanes 2024-06-02 23:14:19 +02:00
parent dfaa2d387c
commit be8f727c74
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
3 changed files with 80 additions and 18 deletions

View File

@ -2,6 +2,7 @@ import argparse
import logging
import personal_data.main
from personal_data.notification import NotificationType
def parse_arguments():
@ -16,6 +17,7 @@ def parse_arguments():
)
parser.add_argument('--cookiejar', action='store_true')
parser.add_argument('--email', action='store_true', dest='send_email_notification')
parser.add_argument('--loud-sound', action='store_true', dest='trigger_loud_and_annoying_sound')
return parser.parse_args()
@ -24,10 +26,18 @@ def main():
logging.getLogger('personal_data').setLevel('INFO')
args = parse_arguments()
scraper_filter = frozenset(args.fetchers)
# Determine notification sounds
notification_types = set()
if args.send_email_notification:
notification_types.add(NotificationType.EMAIL)
if args.trigger_loud_and_annoying_sound:
notification_types.add(NotificationType.LOUD_SOUND)
personal_data.main.main(
scraper_filter,
use_cookiejar=args.cookiejar,
send_email_notification=args.send_email_notification,
notification_types = frozenset(notification_types),
)

View File

@ -28,8 +28,7 @@ except ImportError:
import personal_data.data
import personal_data.fetchers
from . import mailgun
from . import notification
CSV_DIALECT = 'one_true_dialect'
csv.register_dialect(CSV_DIALECT, lineterminator='\n', skipinitialspace=True)
@ -206,17 +205,6 @@ def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
return session
def send_notification(
session: requests.Session,
scraper_name: str,
latest_dict: frozendict,
):
body = ['A new update has occured for ', scraper_name, '\n']
for k, v in latest_dict.items():
body.append(f'{k}: {v}\n')
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
def available_scrapers() -> list[type[personal_data.data.Scraper]]:
subclasses = []
class_queue = [personal_data.data.Scraper]
@ -237,7 +225,7 @@ def main(
scraper_filter: frozenset[str],
*,
use_cookiejar: bool,
send_email_notification: bool = False,
notification_types: frozenset[notification.NotificationType] = frozenset(),
) -> None:
if use_cookiejar:
cookiejar = browsercookie.firefox()
@ -246,7 +234,7 @@ def main(
cookiejar = []
logger.warning('No cookiejar is used')
if not send_email_notification:
if len(notification_types) == 0:
logger.info('Email not enabled: Notifications will not be sent!')
for scraper_cls in available_scrapers():
@ -275,7 +263,10 @@ def main(
)
logger.info('Scraper done: %s', scraper.dataset_name)
if status['extended'] and send_email_notification:
send_notification(session, scraper_cls.__name__, status['dicts'][-1])
if status['extended']:
notification.send_notifications(session,
scraper_cls.__name__,
status['dicts'][-1],
notification_types)
del scraper, session

View File

@ -0,0 +1,61 @@
import csv
import datetime
import decimal
import inspect
import io
import logging
from collections.abc import Iterable, Mapping, Sequence
from decimal import Decimal
import requests
import requests_cache
from frozendict import frozendict
import personal_data.data
import personal_data.fetchers
from . import mailgun
import enum
logger = logging.getLogger(__name__)
class NotificationType(enum.Enum):
EMAIL = 1
LOUD_SOUND = 2
def send_email_notification(
session: requests.Session,
scraper_name: str,
latest_dict: frozendict,
) -> None:
body = ['A new update has occured for ', scraper_name, '\n']
for k, v in latest_dict.items():
body.append(f'{k}: {v}\n')
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
def play_loud_and_annoying_sound(
session: requests.Session,
scraper_name: str,
latest_dict: frozendict,
) -> None:
pass
NOTIFICATION_TYPE_TO_NOTIFIER = {
NotificationType.EMAIL: send_email_notification,
NotificationType.LOUD_SOUND: play_loud_and_annoying_sound,
}
def send_notifications(
session: requests.Session,
scraper_name: str,
latest_dict: frozendict,
notification_types: set[NotificationType],
) -> None:
for notification_type in notification_types:
NOTIFICATION_TYPE_TO_NOTIFIER[notification_type](session, scraper_name, latest_dict)