Ignore cfscrape failure on start
Some checks failed
Python Package / Package (push) Failing after 21s
Some checks failed
Python Package / Package (push) Failing after 21s
This commit is contained in:
parent
01933423cf
commit
87d798fa8f
|
@ -4,11 +4,15 @@ import io
|
|||
import logging
|
||||
|
||||
import browsercookie
|
||||
import cfscrape
|
||||
import requests
|
||||
import requests_cache
|
||||
from frozendict import frozendict
|
||||
|
||||
try:
|
||||
import cfscrape
|
||||
except Exception:
|
||||
cfscrape = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import personal_data.data
|
||||
|
@ -115,7 +119,7 @@ STANDARD_HEADERS = {
|
|||
}
|
||||
|
||||
|
||||
def get_session(with_cfscrape: bool, cookiejar) -> requests.Session:
|
||||
def get_session(cookiejar, *, with_cfscrape: bool) -> requests.Session:
|
||||
assert isinstance(with_cfscrape, bool)
|
||||
if with_cfscrape:
|
||||
session = cfscrape.create_scraper()
|
||||
|
@ -131,7 +135,7 @@ def main():
|
|||
logger.warning('Got cookiejar from firefox: %s cookies', len(cookiejar))
|
||||
|
||||
for scraper_cls in personal_data.data.Scraper.__subclasses__():
|
||||
session = get_session(scraper_cls.requires_cfscrape(), cookiejar)
|
||||
session = get_session(cookiejar, with_cfscrape = scraper_cls.requires_cfscrape())
|
||||
scraper = scraper_cls(session)
|
||||
logger.warning(
|
||||
'Running %s, appending to "%s"',
|
||||
|
|
Loading…
Reference in New Issue
Block a user