From ce63ad2d531d1a864f00bd74d50f847555cced28 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Fri, 25 Oct 2024 21:47:44 +0200 Subject: [PATCH] Merged all dataset outputs --- obsidian_import/__init__.py | 2 +- personal_data/fetchers/ffxiv_lodestone.py | 8 +++++--- personal_data/fetchers/psnprofiles.py | 2 +- personal_data/fetchers/steam_community.py | 11 +++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/obsidian_import/__init__.py b/obsidian_import/__init__.py index 0a9e7dc..519c8bd 100644 --- a/obsidian_import/__init__.py +++ b/obsidian_import/__init__.py @@ -213,7 +213,7 @@ def import_watched_series_csv_from_file(vault: ObsidianVault) -> int: def import_played_games_csv_from_file(vault: ObsidianVault) -> int: - data_path = Path('output/games_played_playstation.csv') + data_path = Path('output/games_played.csv') return import_activity_sample_csv_from_file( vault, data_path, diff --git a/personal_data/fetchers/ffxiv_lodestone.py b/personal_data/fetchers/ffxiv_lodestone.py index 255e1a4..d518bd9 100644 --- a/personal_data/fetchers/ffxiv_lodestone.py +++ b/personal_data/fetchers/ffxiv_lodestone.py @@ -18,12 +18,13 @@ URL_PROFILE_MOUNTS = ( 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/mount/' ) -FORMAT_DATE_HEADER = '%d/%m/%YYYY' +FFXIV_ARR_NAME = 'Final Fantasy XIV: A Realm Reborn' +FFXIV_ARR_RELEASE_DATE = datetime.date(2013,8,27) @dataclasses.dataclass(frozen=True) class LodestoneAchievementScraper(Scraper): - dataset_name = 'games_played_playstation' + dataset_name = 'games_played' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS def scrape(self): @@ -67,7 +68,8 @@ class LodestoneAchievementScraper(Scraper): trophy_icon = trophy_icon.src yield { - 'game.name': 'Final Fantasy XIV: A Realm Reborn', + 'game.name': FFXIV_ARR_NAME, + 'game.release_date': FFXIV_ARR_RELEASE_DATE, 'me.last_played_time': time_acquired, # Trophy Data 'trophy.name': trophy_name, diff --git a/personal_data/fetchers/psnprofiles.py b/personal_data/fetchers/psnprofiles.py index a8f0234..5fc16a9 100644 --- a/personal_data/fetchers/psnprofiles.py +++ b/personal_data/fetchers/psnprofiles.py @@ -34,7 +34,7 @@ MAX_NUMBER_GAMES_TO_PARSE = 10000 class PsnProfilesScraper(Scraper): """Downloads all trophies for the given user.""" - dataset_name = 'games_played_playstation' + dataset_name = 'games_played' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS @staticmethod diff --git a/personal_data/fetchers/steam_community.py b/personal_data/fetchers/steam_community.py index 852f95b..cc7b24c 100644 --- a/personal_data/fetchers/steam_community.py +++ b/personal_data/fetchers/steam_community.py @@ -24,12 +24,14 @@ FORMAT_DATE_HEADER = '%d/%m/%YYYY' class SteamAchievementScraper(Scraper): """Downloads all trophies for the given user.""" - dataset_name = 'games_played_TODO' + dataset_name = 'games_played' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS def scrape(self) -> Iterator[dict[str, Any]]: - username = secrets.STEAM_USERNAME - for appid in self._determine_appids_from_recent_activity(username): + username: str = secrets.STEAM_USERNAME + appids = list(self._determine_appids_from_recent_activity(username)) + logger.info('Found %d Steam Apps', len(appids)) + for appid in appids: yield from self._scrape_app_achievements(username, appid) def _determine_appids_from_recent_activity(self, username: str) -> Iterator[int]: @@ -75,7 +77,7 @@ class SteamAchievementScraper(Scraper): for entry in soup.select('.achieveRow'): trophy_name: str = entry.select_one('h3').get_text() trophy_desc: str = entry.select_one('h5').get_text() - trophy_icon: str = entry.select_one('img').src + trophy_icon: str = entry.select_one('img')['src'] time_acquired_html: str = entry.select_one('.achieveUnlockTime') if time_acquired_html is None: @@ -87,6 +89,7 @@ class SteamAchievementScraper(Scraper): yield { 'game.name': game_name, + #'game.release_date': None, 'me.last_played_time': time_acquired, # Trophy Data 'trophy.name': trophy_name,