From 87d798fa8f12a77d8fe6855c652e6ae8234cb100 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Mon, 1 Apr 2024 01:09:31 +0200 Subject: [PATCH] Ignore cfscrape failure on start --- personal_data/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/personal_data/__init__.py b/personal_data/__init__.py index 64f74e6..fdc90bc 100644 --- a/personal_data/__init__.py +++ b/personal_data/__init__.py @@ -4,11 +4,15 @@ import io import logging import browsercookie -import cfscrape import requests import requests_cache from frozendict import frozendict +try: + import cfscrape +except Exception: + cfscrape = None + logger = logging.getLogger(__name__) import personal_data.data @@ -115,7 +119,7 @@ STANDARD_HEADERS = { } -def get_session(with_cfscrape: bool, cookiejar) -> requests.Session: +def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session: assert isinstance(with_cfscrape, bool) if with_cfscrape: session = cfscrape.create_scraper() @@ -131,7 +135,7 @@ def main(): logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar)) for scraper_cls in personal_data.data.Scraper.__subclasses__(): - session = get_session(scraper_cls.requires_cfscrape(), cookiejar) + session = get_session(cookiejar, with_cfscrape = scraper_cls.requires_cfscrape()) scraper = scraper_cls(session) logger.warning( 'Running %s, appending to "%s"',