Set cache duration based on index in list.

personal_data/fetchers/psnprofiles.py

@@ -32,7 +32,11 @@ MAX_NUMBER_GAMES_TO_PARSE = 10000
 
 @dataclasses.dataclass(frozen=True)
 class PsnProfiles(Scraper):
-    """Downloads all trophies for the given user."""
+    """Downloads all trophies for the given user.
+
+    Individual game pages are cached for a period between 1 to 30 days,
+    depending upon how recently you played them.
+    """
 
     dataset_name = 'games_played'
     deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
@@ -49,7 +53,8 @@ class PsnProfiles(Scraper):
         logger.info('Found %d games from overview', len(games_rows))
 
         for idx, (game_id, game_name) in enumerate(reversed(games_ids.items())):
-            yield from self._scrape_game_trophies(game_id, game_name)
+            cache_duration = datetime.timedelta(days=min(idx+1, 30))
+            yield from self._scrape_game_trophies(game_id, game_name, cache_duration)
             del game_id
             if idx >= MAX_NUMBER_GAMES_TO_PARSE:
                 break
@@ -141,6 +146,7 @@ class PsnProfiles(Scraper):
         self,
         psnprofiles_id: int,
         game_name: str,
+        cache_duration: datetime.timedelta,
     ) -> Iterator[dict]:
         assert isinstance(psnprofiles_id, int), psnprofiles_id
         assert isinstance(game_name, str), game_name
@@ -151,7 +157,7 @@ class PsnProfiles(Scraper):
             psn_id=secrets.PLAYSTATION_PSN_ID,
             game_id=psnprofiles_id,
         )
-        response = self.session.get(url)
+        response = self.session.get(url, expire_after=cache_duration)
         response.raise_for_status()
 
         # Parse data