diff --git a/personal_data/__main__.py b/personal_data/__main__.py index bf6cdc2..351158e 100644 --- a/personal_data/__main__.py +++ b/personal_data/__main__.py @@ -7,11 +7,15 @@ import personal_data.main def parse_arguments(): available_scraper_names = personal_data.main.available_scraper_names() parser = argparse.ArgumentParser() - parser.add_argument('fetchers', metavar='FETCHER', type=str, nargs='+', - choices=available_scraper_names) + parser.add_argument( + 'fetchers', + metavar='FETCHER', + type=str, + nargs='+', + choices=available_scraper_names, + ) parser.add_argument('--cookiejar', action='store_true') - parser.add_argument('--email', action='store_true', - dest='send_email_notification') + parser.add_argument('--email', action='store_true', dest='send_email_notification') return parser.parse_args() @@ -20,8 +24,11 @@ def main(): logging.getLogger('personal_data').setLevel('INFO') args = parse_arguments() scraper_filter = frozenset(args.fetchers) - personal_data.main.main(scraper_filter, use_cookiejar=args.cookiejar, - send_email_notification = args.send_email_notification) + personal_data.main.main( + scraper_filter, + use_cookiejar=args.cookiejar, + send_email_notification=args.send_email_notification, + ) if __name__ == '__main__': diff --git a/personal_data/fetchers/psnprofiles.py b/personal_data/fetchers/psnprofiles.py index 3a08b95..1a7ae64 100644 --- a/personal_data/fetchers/psnprofiles.py +++ b/personal_data/fetchers/psnprofiles.py @@ -1,5 +1,4 @@ import dataclasses -import datetime import logging import re from collections.abc import Iterator @@ -16,6 +15,7 @@ logger = logging.getLogger(__name__) URL_PROFILE = 'https://psnprofiles.com/{psn_id}' URL_USER_GAME_TROPHIES = 'https://psnprofiles.com/trophies/{game_id}/{psn_id}' + def game_psnprofiles_id_from_url(relative_url: str) -> int: m = re.match(r'/(?:trophy|trophies)/(\d+)\-(?:[\w-]+)(/[\w-]*)?', relative_url) result = m.group(1) @@ -117,7 +117,9 @@ class PsnProfilesScraper(Scraper): if len(small_infos) > 2: time_played_div = small_infos[2] time_played_div.sup.extract() - time_played = personal_data.parse_util.parse_date(time_played_div.get_text()) + time_played = personal_data.parse_util.parse_date( + time_played_div.get_text(), + ) else: time_played = None diff --git a/personal_data/main.py b/personal_data/main.py index bf12c26..e239d0e 100644 --- a/personal_data/main.py +++ b/personal_data/main.py @@ -1,5 +1,4 @@ import csv -from collections.abc import Iterator import datetime import io import logging @@ -177,15 +176,21 @@ def send_notification( body.append(f'{k}: {v}\n') mailgun.send_email(session, f'Updated {scraper_name}', ''.join(body)) + def available_scrapers() -> list[type[personal_data.data.Scraper]]: return personal_data.data.Scraper.__subclasses__() + def available_scraper_names() -> list[str]: return [scraper_cls.__name__ for scraper_cls in available_scrapers()] -def main(scraper_filter: frozenset[str], *, use_cookiejar: bool, - send_email_notification: bool = False) -> None: +def main( + scraper_filter: frozenset[str], + *, + use_cookiejar: bool, + send_email_notification: bool = False, +) -> None: if use_cookiejar: cookiejar = browsercookie.firefox() logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar)) diff --git a/personal_data/parse_util.py b/personal_data/parse_util.py index c619f2a..8fa0312 100644 --- a/personal_data/parse_util.py +++ b/personal_data/parse_util.py @@ -28,14 +28,17 @@ def parse_duration(text: str) -> datetime.timedelta: unit = DATETIME_UNITS[unit] return unit * num + def parse_response_datetime(response) -> datetime.datetime: return datetime.datetime.strptime(response.headers['Date'], FORMAT_DATE_HEADER) + def parse_time(text: str) -> datetime.datetime: text = text.replace('\n', ' ') text = text.strip() return datetime.datetime.strptime(text, '%d %b %Y %I:%M:%S %p') + def parse_date(text: str) -> datetime.date: return datetime.datetime.strptime( text.strip(),