Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Ignore cfscrape failure on start
Some checks failed
Python Package / Package (push) Failing after 21s
Details

Browse Source
This commit is contained in:
Jon Michael Aanes 2024-04-01 01:09:31 +02:00
parent 01933423cf
commit 87d798fa8f
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
personal_data/__init__.py
View File

@ -4,11 +4,15 @@ import io
import logging import logging
import browsercookie import browsercookie
import cfscrape
import requests import requests
import requests_cache import requests_cache
from frozendict import frozendict from frozendict import frozendict
try:
import cfscrape
except Exception:
cfscrape = None
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
import personal_data.data import personal_data.data
@ -115,7 +119,7 @@ STANDARD_HEADERS = {
} }
def get_session(with_cfscrape: bool, cookiejar) -> requests.Session: def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
assert isinstance(with_cfscrape, bool) assert isinstance(with_cfscrape, bool)
if with_cfscrape: if with_cfscrape:
session = cfscrape.create_scraper() session = cfscrape.create_scraper()
@ -131,7 +135,7 @@ def main():
logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar)) logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar))
for scraper_cls in personal_data.data.Scraper.__subclasses__(): for scraper_cls in personal_data.data.Scraper.__subclasses__():
session = get_session(scraper_cls.requires_cfscrape(), cookiejar) session = get_session(cookiejar, with_cfscrape = scraper_cls.requires_cfscrape())
scraper = scraper_cls(session) scraper = scraper_cls(session)
logger.warning( logger.warning(
'Running %s, appending to "%s"', 'Running %s, appending to "%s"',