Logging
This commit is contained in:
parent
20244735aa
commit
e3b23637a0
|
@ -1,5 +1,6 @@
|
||||||
import personal_data.main
|
import personal_data.main
|
||||||
import argparse
|
import argparse
|
||||||
|
import logging
|
||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
|
@ -7,6 +8,8 @@ def parse_arguments():
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
logging.basicConfig()
|
||||||
|
logging.getLogger('personal_data').setLevel('INFO')
|
||||||
args = parse_arguments()
|
args = parse_arguments()
|
||||||
scraper_filter = frozenset(args.fetchers)
|
scraper_filter = frozenset(args.fetchers)
|
||||||
personal_data.main.main(scraper_filter)
|
personal_data.main.main(scraper_filter)
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
import personal_data.secrets as secrets
|
import personal_data.secrets as secrets
|
||||||
|
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
MAILGUN_API_ENDPOINT = 'https://api.mailgun.net/v3/{mailgun_domain}/messages'
|
MAILGUN_API_ENDPOINT = 'https://api.mailgun.net/v3/{mailgun_domain}/messages'
|
||||||
|
|
||||||
FROM_MAIL_NAME = 'Personal Scrapers'
|
FROM_MAIL_NAME = 'Personal Scrapers'
|
||||||
|
@ -14,6 +16,8 @@ def send_email(session: requests.Session, subject: str, text: str):
|
||||||
assert subject != ''
|
assert subject != ''
|
||||||
assert text != ''
|
assert text != ''
|
||||||
|
|
||||||
|
logger.info('Sending email using mailgun!')
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
'from': f'{FROM_MAIL_NAME} <{FROM_MAIL_USERNAME}@{secrets.MAILGUN_DOMAIN}>',
|
'from': f'{FROM_MAIL_NAME} <{FROM_MAIL_USERNAME}@{secrets.MAILGUN_DOMAIN}>',
|
||||||
'to': [secrets.MAILGUN_RECIPIENT],
|
'to': [secrets.MAILGUN_RECIPIENT],
|
||||||
|
@ -27,5 +31,6 @@ def send_email(session: requests.Session, subject: str, text: str):
|
||||||
data=data,
|
data=data,
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
logger.info('Email sent!')
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
|
@ -127,7 +127,7 @@ def extend_csv_file(
|
||||||
with open(filename, 'w') as csvfile:
|
with open(filename, 'w') as csvfile:
|
||||||
csvfile.write(output_csv)
|
csvfile.write(output_csv)
|
||||||
del csvfile
|
del csvfile
|
||||||
logger.warning(
|
logger.info(
|
||||||
'Extended CSV "%s" from %d to %d lines',
|
'Extended CSV "%s" from %d to %d lines',
|
||||||
filename,
|
filename,
|
||||||
original_num_dicts,
|
original_num_dicts,
|
||||||
|
@ -162,19 +162,21 @@ def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
|
||||||
return session
|
return session
|
||||||
|
|
||||||
def send_notification(session: requests.Session, scraper_name: str, latest_dict: frozendict):
|
def send_notification(session: requests.Session, scraper_name: str, latest_dict: frozendict):
|
||||||
maingun.send_email(session, f'Updated {scraper_name}', repr(latest_dict))
|
body = ['A new update has occured for ', scraper_name, '\n']
|
||||||
|
for k, v in latest_dict.items():
|
||||||
|
body.append(f'{k}: {v}\n')
|
||||||
|
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
|
||||||
|
|
||||||
def main(scraper_filter: frozenset[str]):
|
def main(scraper_filter: frozenset[str]):
|
||||||
cookiejar = browsercookie.firefox()
|
cookiejar = browsercookie.firefox()
|
||||||
logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar))
|
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
|
||||||
|
|
||||||
for scraper_cls in personal_data.data.Scraper.__subclasses__():
|
for scraper_cls in personal_data.data.Scraper.__subclasses__():
|
||||||
session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())
|
session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())
|
||||||
scraper = scraper_cls(session)
|
scraper = scraper_cls(session)
|
||||||
if scraper_cls.__name__ not in scraper_filter:
|
if scraper_cls.__name__ not in scraper_filter:
|
||||||
continue
|
continue
|
||||||
logger.warning(
|
logger.info(
|
||||||
'Running %s, appending to "%s"',
|
'Running %s, appending to "%s"',
|
||||||
scraper_cls.__name__,
|
scraper_cls.__name__,
|
||||||
scraper.dataset_name,
|
scraper.dataset_name,
|
||||||
|
@ -193,10 +195,10 @@ def main(scraper_filter: frozenset[str]):
|
||||||
deduplicate_mode=scraper.deduplicate_mode,
|
deduplicate_mode=scraper.deduplicate_mode,
|
||||||
deduplicate_ignore_columns=scraper.deduplicate_ignore_columns,
|
deduplicate_ignore_columns=scraper.deduplicate_ignore_columns,
|
||||||
)
|
)
|
||||||
logger.warning('Scraper done: %s', scraper.dataset_name)
|
logger.info('Scraper done: %s', scraper.dataset_name)
|
||||||
|
|
||||||
if status['extended']:
|
if status['extended']:
|
||||||
print('Extended')
|
print('Extended')
|
||||||
send_notification(scraper_cls.__name__, status['dicts'][-1])
|
send_notification(session, scraper_cls.__name__, status['dicts'][-1])
|
||||||
|
|
||||||
del scraper, session
|
del scraper, session
|
||||||
|
|
Loading…
Reference in New Issue
Block a user