diff --git a/personal_data/fetchers/psnprofiles.py b/personal_data/fetchers/psnprofiles.py index 378ed5f..fc3ba63 100644 --- a/personal_data/fetchers/psnprofiles.py +++ b/personal_data/fetchers/psnprofiles.py @@ -32,7 +32,11 @@ MAX_NUMBER_GAMES_TO_PARSE = 10000 @dataclasses.dataclass(frozen=True) class PsnProfiles(Scraper): - """Downloads all trophies for the given user.""" + """Downloads all trophies for the given user. + + Individual game pages are cached for a period between 1 to 30 days, + depending upon how recently you played them. + """ dataset_name = 'games_played' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS @@ -49,7 +53,8 @@ class PsnProfiles(Scraper): logger.info('Found %d games from overview', len(games_rows)) for idx, (game_id, game_name) in enumerate(reversed(games_ids.items())): - yield from self._scrape_game_trophies(game_id, game_name) + cache_duration = datetime.timedelta(days=min(idx+1, 30)) + yield from self._scrape_game_trophies(game_id, game_name, cache_duration) del game_id if idx >= MAX_NUMBER_GAMES_TO_PARSE: break @@ -141,6 +146,7 @@ class PsnProfiles(Scraper): self, psnprofiles_id: int, game_name: str, + cache_duration: datetime.timedelta, ) -> Iterator[dict]: assert isinstance(psnprofiles_id, int), psnprofiles_id assert isinstance(game_name, str), game_name @@ -151,7 +157,7 @@ class PsnProfiles(Scraper): psn_id=secrets.PLAYSTATION_PSN_ID, game_id=psnprofiles_id, ) - response = self.session.get(url) + response = self.session.get(url, expire_after=cache_duration) response.raise_for_status() # Parse data