Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Logging

Browse Source
This commit is contained in:
Jon Michael Aanes 2024-04-17 00:45:15 +02:00
parent 20244735aa
commit e3b23637a0
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
3 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
personal_data/__main__.py
View File

@ -1,5 +1,6 @@
import personal_data.main import personal_data.main
import argparse import argparse
import logging
def parse_arguments(): def parse_arguments():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
@ -7,6 +8,8 @@ def parse_arguments():
return parser.parse_args() return parser.parse_args()
def main(): def main():
logging.basicConfig()
logging.getLogger('personal_data').setLevel('INFO')
args = parse_arguments() args = parse_arguments()
scraper_filter = frozenset(args.fetchers) scraper_filter = frozenset(args.fetchers)
personal_data.main.main(scraper_filter) personal_data.main.main(scraper_filter)

7
personal_data/mailgun.py
View File

@ -1,7 +1,9 @@
import requests import requests
import personal_data.secrets as secrets import personal_data.secrets as secrets
import logging
logger = logging.getLogger(__name__)
MAILGUN_API_ENDPOINT = 'https://api.mailgun.net/v3/{mailgun_domain}/messages' MAILGUN_API_ENDPOINT = 'https://api.mailgun.net/v3/{mailgun_domain}/messages'
FROM_MAIL_NAME = 'Personal Scrapers' FROM_MAIL_NAME = 'Personal Scrapers'
@ -14,6 +16,8 @@ def send_email(session: requests.Session, subject: str, text: str):
assert subject != '' assert subject != ''
assert text != '' assert text != ''
logger.info('Sending email using mailgun!')
data = { data = {
'from': f'{FROM_MAIL_NAME} <{FROM_MAIL_USERNAME}@{secrets.MAILGUN_DOMAIN}>', 'from': f'{FROM_MAIL_NAME} <{FROM_MAIL_USERNAME}@{secrets.MAILGUN_DOMAIN}>',
'to': [secrets.MAILGUN_RECIPIENT], 'to': [secrets.MAILGUN_RECIPIENT],
@ -27,5 +31,6 @@ def send_email(session: requests.Session, subject: str, text: str):
data=data, data=data,
) )
response.raise_for_status() response.raise_for_status()
logger.info('Email sent!')
return response return response

16
personal_data/main.py
View File

@ -127,7 +127,7 @@ def extend_csv_file(
with open(filename, 'w') as csvfile: with open(filename, 'w') as csvfile:
csvfile.write(output_csv) csvfile.write(output_csv)
del csvfile del csvfile
logger.warning( logger.info(
'Extended CSV "%s" from %d to %d lines', 'Extended CSV "%s" from %d to %d lines',
filename, filename,
original_num_dicts, original_num_dicts,
@ -162,19 +162,21 @@ def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
return session return session
def send_notification(session: requests.Session, scraper_name: str, latest_dict: frozendict): def send_notification(session: requests.Session, scraper_name: str, latest_dict: frozendict):
maingun.send_email(session, f'Updated {scraper_name}', repr(latest_dict)) body = ['A new update has occured for ', scraper_name, '\n']
for k, v in latest_dict.items():
body.append(f'{k}: {v}\n')
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
def main(scraper_filter: frozenset[str]): def main(scraper_filter: frozenset[str]):
cookiejar = browsercookie.firefox() cookiejar = browsercookie.firefox()
logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar)) logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
for scraper_cls in personal_data.data.Scraper.__subclasses__(): for scraper_cls in personal_data.data.Scraper.__subclasses__():
session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape()) session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())
scraper = scraper_cls(session) scraper = scraper_cls(session)
if scraper_cls.__name__ not in scraper_filter: if scraper_cls.__name__ not in scraper_filter:
continue continue
logger.warning( logger.info(
'Running %s, appending to "%s"', 'Running %s, appending to "%s"',
scraper_cls.__name__, scraper_cls.__name__,
scraper.dataset_name, scraper.dataset_name,
@ -193,10 +195,10 @@ def main(scraper_filter: frozenset[str]):
deduplicate_mode=scraper.deduplicate_mode, deduplicate_mode=scraper.deduplicate_mode,
deduplicate_ignore_columns=scraper.deduplicate_ignore_columns, deduplicate_ignore_columns=scraper.deduplicate_ignore_columns,
) )
logger.warning('Scraper done: %s', scraper.dataset_name) logger.info('Scraper done: %s', scraper.dataset_name)
if status['extended']: if status['extended']:
print('Extended') print('Extended')
send_notification(scraper_cls.__name__, status['dicts'][-1]) send_notification(session, scraper_cls.__name__, status['dicts'][-1])
del scraper, session del scraper, session