diff --git a/README.md b/README.md index d1a8687..caaf3e9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # Personal Data Fetcher Systems This is a collection of small fetchers for personal data spread around the internet. @@ -10,6 +9,12 @@ Supported fetchers: - Partisia Blockchain BYOC and MPC trackers - Playstation achievements by way of PSN Profiles. +## Usage + +```sh +python -m personal_data +``` + ## Ideas for more fetchers - [ ] YouTube (Music): Liked videos with title and URL. diff --git a/personal_data/_version.py b/personal_data/_version.py index df9144c..10939f0 100644 --- a/personal_data/_version.py +++ b/personal_data/_version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.1.2' diff --git a/personal_data/fetchers/playstation.py b/personal_data/fetchers/playstation.py deleted file mode 100644 index da73d84..0000000 --- a/personal_data/fetchers/playstation.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) - -URL_RECENTLY_PLAYED_HTML = 'https://library.playstation.com/recently-played' -URL_RECENTLY_PLAYED_API = 'https://web.np.playstation.com/api/graphql/v1/op?operationName=getUserGameList&variables=%7B%22limit%22%3A50%2C%22categories%22%3A%22ps4_game%2Cps5_native_game%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%22e0136f81d7d1fb6be58238c574e9a46e1c0cc2f7f6977a08a5a46f224523a004%22%7D%7D' - - -def scrape_played_last(session): - # Initial request to trigger cookie. - logger.warning('Trying to trigger initial cookie usage') - - response = session.get(URL_RECENTLY_PLAYED_HTML, cookies=session.cookies) - response.raise_for_status() - - # Now trigger API call. - logger.warning('Trying to fetch data from API') - headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0', - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.5', - 'content-type': 'application/json', - 'X-PSN-App-Ver': 'my-playstation/0.1.0-20230720235210-hotfix-1-g1e9f07ff-1e9f07ff247eafea4d9d9236f73863cb9cd5d3e8', - 'X-PSN-Correlation-Id': 'bc7a39b1-a99c-478b-9494-d2fddf189875', - 'apollographql-client-name': 'my-playstation', - 'apollographql-client-version': '0.1.0-20230720235210-hotfix-1-g1e9f07ff', - 'X-PSN-Request-Id': '8ad64653-d8b5-4941-b565-b5536c9853df', - 'Referer': 'https://library.playstation.com/', - 'Origin': 'https://library.playstation.com', - } - result = session.get(URL_RECENTLY_PLAYED_API, headers=headers) - result.raise_for_status() - - print(result.json()) - games_data = result.json()['data']['gameLibraryTitlesRetrieve']['games'] - print(games_data) - for game_data in games_data: - yield { - # Important fields - 'game.name': game_data['name'], - 'me.last_played_time': game_data['lastPlayedDateTime'], - 'playstation.product_id': game_data['productId'], - # Secondary fields - 'playstation.concept_id': game_data['conceptId'], - 'playstation.title_id': game_data['titleId'], - 'playstation.entitlement_id': game_data['entitlementId'], - 'me.acquired_by_playstation_membership': game_data['membership'], - 'game.platform': game_data['platform'], - 'game.icon': game_data['image']['url'], - } - - -""" -SCRAPERS = [ - Scraper(scrape_played_last, 'games_played_playstation', - deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS) -] -""" diff --git a/personal_data/main.py b/personal_data/main.py index e239d0e..53833bd 100644 --- a/personal_data/main.py +++ b/personal_data/main.py @@ -3,23 +3,29 @@ import datetime import io import logging -import browsercookie import requests import requests_cache from frozendict import frozendict +logger = logging.getLogger(__name__) + try: import cfscrape except ImportError: cfscrape = None + logger.warning('cfscrape not installed: Certain fetchers might not work') + +try: + import browsercookie +except ImportError: + logger.warning('browsercookie not installed: Certain fetchers might not work') + browsercookie = None -logger = logging.getLogger(__name__) import personal_data.data import personal_data.fetchers.crunchyroll import personal_data.fetchers.ffxiv_lodestone import personal_data.fetchers.partisia_blockchain -import personal_data.fetchers.playstation import personal_data.fetchers.psnprofiles from personal_data import mailgun