Cookiejar is configurable
This commit is contained in:
parent
553f6fe7d4
commit
fdb6e2d180
|
@ -5,6 +5,7 @@ import logging
|
|||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('fetchers', metavar='FETCHER', type=str, nargs='+')
|
||||
parser.add_argument('--cookiejar', action = 'store_true')
|
||||
return parser.parse_args()
|
||||
|
||||
def main():
|
||||
|
@ -12,7 +13,7 @@ def main():
|
|||
logging.getLogger('personal_data').setLevel('INFO')
|
||||
args = parse_arguments()
|
||||
scraper_filter = frozenset(args.fetchers)
|
||||
personal_data.main.main(scraper_filter)
|
||||
personal_data.main.main(scraper_filter, use_cookiejar = args.cookiejar)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
@ -171,9 +171,13 @@ def send_notification(session: requests.Session, scraper_name: str, latest_dict:
|
|||
body.append(f'{k}: {v}\n')
|
||||
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
|
||||
|
||||
def main(scraper_filter: frozenset[str]):
|
||||
cookiejar = browsercookie.firefox()
|
||||
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
|
||||
def main(scraper_filter: frozenset[str], use_cookiejar: bool):
|
||||
if use_cookiejar:
|
||||
cookiejar = browsercookie.firefox()
|
||||
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
|
||||
else:
|
||||
cookiejar = []
|
||||
logger.warning('No cookiejar is used')
|
||||
|
||||
for scraper_cls in personal_data.data.Scraper.__subclasses__():
|
||||
session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())
|
||||
|
|
Loading…
Reference in New Issue
Block a user