Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Max count on scraped
Some checks failed
Python Package / Package (push) Failing after 27s
Details

Browse Source
This commit is contained in:
Jon Michael Aanes 2024-04-06 18:59:18 +02:00
parent 1dfc67d741
commit be1daa79d1
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
personal_data/fetchers/psnprofiles.py
View File

@ -42,6 +42,8 @@ def parse_time(text: str) -> datetime.datetime:
assert parse_time('06 Apr 2024 06:11:42 PM') assert parse_time('06 Apr 2024 06:11:42 PM')
assert parse_time('26 Mar 2024 7:07:01 PM') assert parse_time('26 Mar 2024 7:07:01 PM')
MAX_GAME_ITERATIONS = 10
@dataclasses.dataclass(frozen=True) @dataclasses.dataclass(frozen=True)
class PsnProfilesScraper(Scraper): class PsnProfilesScraper(Scraper):
dataset_name = 'games_played_playstation' dataset_name = 'games_played_playstation'
@ -53,12 +55,16 @@ class PsnProfilesScraper(Scraper):
def scrape(self): def scrape(self):
games_rows = list(self.scrape_games_overview()) games_rows = list(self.scrape_games_overview())
games_ids = {row['psnprofiles.game_id']: ['game.name'] for row in games_rows} games_ids = {row['psnprofiles.game_id']: row['game.name'] for row in games_rows}
yield from games_rows yield from games_rows
idx = 0
for game_id, game_name in games_ids.items(): for game_id, game_name in games_ids.items():
yield from self.scrape_game_trophies(game_id, game_name) yield from self.scrape_game_trophies(game_id, game_name)
del game_id del game_id
idx += 1
if idx >= MAX_GAME_ITERATIONS:
break
def scrape_games_overview(self) -> Iterator[dict]: def scrape_games_overview(self) -> Iterator[dict]:
# Request to get overview # Request to get overview
@ -149,7 +155,10 @@ class PsnProfilesScraper(Scraper):
d['me.last_played_time'] = time_played d['me.last_played_time'] = time_played
yield d yield d
def scrape_game_trophies(self, psnprofiles_id: str, game_name: str) -> Iterator[dict]: def scrape_game_trophies(self, psnprofiles_id: int, game_name: str) -> Iterator[dict]:
assert isinstance(psnprofiles_id, int), psnprofiles_id
assert isinstance(game_name, str), game_name
logger.info('Getting Game Trophies %s', psnprofiles_id) logger.info('Getting Game Trophies %s', psnprofiles_id)
url = URL_USER_GAME_TROPHIES.format(psn_id=secrets.PLAYSTATION_PSN_ID, url = URL_USER_GAME_TROPHIES.format(psn_id=secrets.PLAYSTATION_PSN_ID,