Generalizing notification system
This commit is contained in:
parent
dfaa2d387c
commit
be8f727c74
|
@ -2,6 +2,7 @@ import argparse
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import personal_data.main
|
import personal_data.main
|
||||||
|
from personal_data.notification import NotificationType
|
||||||
|
|
||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
|
@ -16,6 +17,7 @@ def parse_arguments():
|
||||||
)
|
)
|
||||||
parser.add_argument('--cookiejar', action='store_true')
|
parser.add_argument('--cookiejar', action='store_true')
|
||||||
parser.add_argument('--email', action='store_true', dest='send_email_notification')
|
parser.add_argument('--email', action='store_true', dest='send_email_notification')
|
||||||
|
parser.add_argument('--loud-sound', action='store_true', dest='trigger_loud_and_annoying_sound')
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
@ -24,10 +26,18 @@ def main():
|
||||||
logging.getLogger('personal_data').setLevel('INFO')
|
logging.getLogger('personal_data').setLevel('INFO')
|
||||||
args = parse_arguments()
|
args = parse_arguments()
|
||||||
scraper_filter = frozenset(args.fetchers)
|
scraper_filter = frozenset(args.fetchers)
|
||||||
|
|
||||||
|
# Determine notification sounds
|
||||||
|
notification_types = set()
|
||||||
|
if args.send_email_notification:
|
||||||
|
notification_types.add(NotificationType.EMAIL)
|
||||||
|
if args.trigger_loud_and_annoying_sound:
|
||||||
|
notification_types.add(NotificationType.LOUD_SOUND)
|
||||||
|
|
||||||
personal_data.main.main(
|
personal_data.main.main(
|
||||||
scraper_filter,
|
scraper_filter,
|
||||||
use_cookiejar=args.cookiejar,
|
use_cookiejar=args.cookiejar,
|
||||||
send_email_notification=args.send_email_notification,
|
notification_types = frozenset(notification_types),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -28,8 +28,7 @@ except ImportError:
|
||||||
|
|
||||||
import personal_data.data
|
import personal_data.data
|
||||||
import personal_data.fetchers
|
import personal_data.fetchers
|
||||||
|
from . import notification
|
||||||
from . import mailgun
|
|
||||||
|
|
||||||
CSV_DIALECT = 'one_true_dialect'
|
CSV_DIALECT = 'one_true_dialect'
|
||||||
csv.register_dialect(CSV_DIALECT, lineterminator='\n', skipinitialspace=True)
|
csv.register_dialect(CSV_DIALECT, lineterminator='\n', skipinitialspace=True)
|
||||||
|
@ -206,17 +205,6 @@ def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
|
||||||
return session
|
return session
|
||||||
|
|
||||||
|
|
||||||
def send_notification(
|
|
||||||
session: requests.Session,
|
|
||||||
scraper_name: str,
|
|
||||||
latest_dict: frozendict,
|
|
||||||
):
|
|
||||||
body = ['A new update has occured for ', scraper_name, '\n']
|
|
||||||
for k, v in latest_dict.items():
|
|
||||||
body.append(f'{k}: {v}\n')
|
|
||||||
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
|
|
||||||
|
|
||||||
|
|
||||||
def available_scrapers() -> list[type[personal_data.data.Scraper]]:
|
def available_scrapers() -> list[type[personal_data.data.Scraper]]:
|
||||||
subclasses = []
|
subclasses = []
|
||||||
class_queue = [personal_data.data.Scraper]
|
class_queue = [personal_data.data.Scraper]
|
||||||
|
@ -237,7 +225,7 @@ def main(
|
||||||
scraper_filter: frozenset[str],
|
scraper_filter: frozenset[str],
|
||||||
*,
|
*,
|
||||||
use_cookiejar: bool,
|
use_cookiejar: bool,
|
||||||
send_email_notification: bool = False,
|
notification_types: frozenset[notification.NotificationType] = frozenset(),
|
||||||
) -> None:
|
) -> None:
|
||||||
if use_cookiejar:
|
if use_cookiejar:
|
||||||
cookiejar = browsercookie.firefox()
|
cookiejar = browsercookie.firefox()
|
||||||
|
@ -246,7 +234,7 @@ def main(
|
||||||
cookiejar = []
|
cookiejar = []
|
||||||
logger.warning('No cookiejar is used')
|
logger.warning('No cookiejar is used')
|
||||||
|
|
||||||
if not send_email_notification:
|
if len(notification_types) == 0:
|
||||||
logger.info('Email not enabled: Notifications will not be sent!')
|
logger.info('Email not enabled: Notifications will not be sent!')
|
||||||
|
|
||||||
for scraper_cls in available_scrapers():
|
for scraper_cls in available_scrapers():
|
||||||
|
@ -275,7 +263,10 @@ def main(
|
||||||
)
|
)
|
||||||
logger.info('Scraper done: %s', scraper.dataset_name)
|
logger.info('Scraper done: %s', scraper.dataset_name)
|
||||||
|
|
||||||
if status['extended'] and send_email_notification:
|
if status['extended']:
|
||||||
send_notification(session, scraper_cls.__name__, status['dicts'][-1])
|
notification.send_notifications(session,
|
||||||
|
scraper_cls.__name__,
|
||||||
|
status['dicts'][-1],
|
||||||
|
notification_types)
|
||||||
|
|
||||||
del scraper, session
|
del scraper, session
|
||||||
|
|
61
personal_data/notification.py
Normal file
61
personal_data/notification.py
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
import csv
|
||||||
|
import datetime
|
||||||
|
import decimal
|
||||||
|
import inspect
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import requests_cache
|
||||||
|
from frozendict import frozendict
|
||||||
|
|
||||||
|
import personal_data.data
|
||||||
|
import personal_data.fetchers
|
||||||
|
|
||||||
|
from . import mailgun
|
||||||
|
|
||||||
|
import enum
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class NotificationType(enum.Enum):
|
||||||
|
EMAIL = 1
|
||||||
|
LOUD_SOUND = 2
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def send_email_notification(
|
||||||
|
session: requests.Session,
|
||||||
|
scraper_name: str,
|
||||||
|
latest_dict: frozendict,
|
||||||
|
) -> None:
|
||||||
|
body = ['A new update has occured for ', scraper_name, '\n']
|
||||||
|
for k, v in latest_dict.items():
|
||||||
|
body.append(f'{k}: {v}\n')
|
||||||
|
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
|
||||||
|
|
||||||
|
|
||||||
|
def play_loud_and_annoying_sound(
|
||||||
|
session: requests.Session,
|
||||||
|
scraper_name: str,
|
||||||
|
latest_dict: frozendict,
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
NOTIFICATION_TYPE_TO_NOTIFIER = {
|
||||||
|
NotificationType.EMAIL: send_email_notification,
|
||||||
|
NotificationType.LOUD_SOUND: play_loud_and_annoying_sound,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def send_notifications(
|
||||||
|
session: requests.Session,
|
||||||
|
scraper_name: str,
|
||||||
|
latest_dict: frozendict,
|
||||||
|
notification_types: set[NotificationType],
|
||||||
|
) -> None:
|
||||||
|
for notification_type in notification_types:
|
||||||
|
NOTIFICATION_TYPE_TO_NOTIFIER[notification_type](session, scraper_name, latest_dict)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user