Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Cookiejar is configurable
Some checks failed
Build container / Package-Python (push) Failing after 25s
Details
Build container / Package-Container (push) Successful in 1m30s
Details

Browse Source
This commit is contained in:
Jon Michael Aanes 2024-04-18 00:13:56 +02:00
parent 553f6fe7d4
commit fdb6e2d180
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
2 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
personal_data/__main__.py
View File

@ -5,6 +5,7 @@ import logging
def parse_arguments(): def parse_arguments():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('fetchers', metavar='FETCHER', type=str, nargs='+') parser.add_argument('fetchers', metavar='FETCHER', type=str, nargs='+')
parser.add_argument('--cookiejar', action = 'store_true')
return parser.parse_args() return parser.parse_args()
def main(): def main():
@ -12,7 +13,7 @@ def main():
logging.getLogger('personal_data').setLevel('INFO') logging.getLogger('personal_data').setLevel('INFO')
args = parse_arguments() args = parse_arguments()
scraper_filter = frozenset(args.fetchers) scraper_filter = frozenset(args.fetchers)
personal_data.main.main(scraper_filter) personal_data.main.main(scraper_filter, use_cookiejar = args.cookiejar)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

10
personal_data/main.py
View File

@ -171,9 +171,13 @@ def send_notification(session: requests.Session, scraper_name: str, latest_dict:
body.append(f'{k}: {v}\n') body.append(f'{k}: {v}\n')
mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body)) mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body))
def main(scraper_filter: frozenset[str]): def main(scraper_filter: frozenset[str], use_cookiejar: bool):
cookiejar = browsercookie.firefox() if use_cookiejar:
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar)) cookiejar = browsercookie.firefox()
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
else:
cookiejar = []
logger.warning('No cookiejar is used')
for scraper_cls in personal_data.data.Scraper.__subclasses__(): for scraper_cls in personal_data.data.Scraper.__subclasses__():
session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape()) session = get_session(cookiejar, with_cfscrape=scraper_cls.requires_cfscrape())