From fdb6e2d1801546cb54886a85255c01fabf29586b Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Thu, 18 Apr 2024 00:13:56 +0200 Subject: [PATCH] Cookiejar is configurable --- personal_data/__main__.py | 3 ++- personal_data/main.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/personal_data/__main__.py b/personal_data/__main__.py index 7454c06..7a5fb6d 100644 --- a/personal_data/__main__.py +++ b/personal_data/__main__.py @@ -5,6 +5,7 @@ import logging def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument('fetchers', metavar='FETCHER', type=str, nargs='+') + parser.add_argument('--cookiejar', action = 'store_true') return parser.parse_args() def main(): @@ -12,7 +13,7 @@ def main(): logging.getLogger('personal_data').setLevel('INFO') args = parse_arguments() scraper_filter = frozenset(args.fetchers) - personal_data.main.main(scraper_filter) + personal_data.main.main(scraper_filter, use_cookiejar = args.cookiejar) if __name__ == '__main__': main() diff --git a/personal_data/main.py b/personal_data/main.py index 8542962..1516be9 100644 --- a/personal_data/main.py +++ b/personal_data/main.py @@ -171,9 +171,13 @@ def send_notification(session: requests.Session, scraper_name: str, latest_dict: body.append(f'{k}: {v}\n') mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body)) -def main(scraper_filter: frozenset[str]): - cookiejar = browsercookie.firefox() - logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar)) +def main(scraper_filter: frozenset[str], use_cookiejar: bool): + if use_cookiejar: + cookiejar = browsercookie.firefox() + logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar)) + else: + cookiejar = [] + logger.warning('No cookiejar is used') for scraper_cls in personal_data.data.Scraper.__subclasses__(): session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())