Initial with Playstation

__main__.py

@@ -0,0 +1,53 @@
+
+import requests
+import requests_cache
+import csv
+import datetime
+import io
+from frozendict import frozendict
+
+import playstation
+
+def determine_scrapers():
+    scrapers = []
+    scrapers += playstation.SCRAPERS
+    return scrapers
+
+def extend_csv_file(filename, new_dicts , deduplicate = False):
+
+    dicts = []
+    with open(filename, 'r') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            dicts .append(frozendict(row))
+        del csvfile
+
+    dicts += [frozendict(d) for d in new_dicts]
+    del new_dicts
+    if deduplicate:
+        dicts = list(set(dicts))
+
+    fieldnames = dicts[0].keys()
+
+    csvfile_in_memory = io.StringIO()
+    writer = csv.DictWriter(csvfile_in_memory, fieldnames=fieldnames)
+    writer.writeheader()
+    for d in dicts:
+        writer.writerow(d)
+    output_csv = csvfile_in_memory.getvalue()
+    del writer, csvfile_in_memory
+
+    with open(filename, 'w') as csvfile:
+        csvfile.write(output_csv)
+        del csvfile
+
+def main():
+    session = requests_cache.CachedSession('web_cache')
+    for scraper in determine_scrapers():
+        result_rows = list(scraper.scraper(session))
+        extend_csv_file('output/'+scraper.dataset_name, result_rows,
+                        deduplicate = scraper.deduplicate)
+
+if __name__ == '__main__':
+    main()
+

data.py

@@ -0,0 +1,10 @@
+
+import dataclasses
+
+@dataclasses.dataclass
+class Scraper:
+    scraper: object # TODO: Callable
+    dataset_name: str
+    deduplicate: bool
+    dataset_format: str = 'list-of-dicts'
+

playstation.py

@@ -0,0 +1,55 @@
+
+from data import Scraper
+import secrets
+
+def scrape_played_last(session):
+    url = "https://web.np.playstation.com/api/graphql/v1/op?operationName=getUserGameList&variables=%7B%22limit%22%3A50%2C%22categories%22%3A%22ps4_game%2Cps5_native_game%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%22e0136f81d7d1fb6be58238c574e9a46e1c0cc2f7f6977a08a5a46f224523a004%22%7D%7D"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0",
+        "Accept": "application/json",
+        "Accept-Language": "en-US,en;q=0.5",
+        "content-type": "application/json",
+        "X-PSN-App-Ver": "my-playstation/0.1.0-20230720235210-hotfix-1-g1e9f07ff-1e9f07ff247eafea4d9d9236f73863cb9cd5d3e8",
+        "X-PSN-Correlation-Id": "bc7a39b1-a99c-478b-9494-d2fddf189875",
+        "apollographql-client-name": "my-playstation",
+        "apollographql-client-version": "0.1.0-20230720235210-hotfix-1-g1e9f07ff",
+        "X-PSN-Request-Id": "8ad64653-d8b5-4941-b565-b5536c9853df",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-site",
+        "Pragma": "no-cache",
+        "Cache-Control": "no-cache",
+        "Cookie": secrets.PLAYSTATION_COM_COOKIES,
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Referer': 'https://library.playstation.com/',
+        'Origin': 'https://library.playstation.com',
+        'DNT': '1',
+        'Connection': 'keep-alive',
+        'TE': 'trailers',
+    }
+    result = session.get(url, headers = headers)
+    result.raise_for_status()
+
+    print(result.json())
+    games_data = result.json()['data']['gameLibraryTitlesRetrieve']['games']
+    print(games_data)
+    for game_data in games_data:
+        yield {
+                # Important fields
+                'game.name': game_data['name'],
+                'me.last_played_time': game_data['lastPlayedDateTime'],
+                'playstation.product_id': game_data['productId'],
+
+                # Secondary fields
+                'playstation.concept_id': game_data['conceptId'],
+                'playstation.title_id': game_data['titleId'],
+                'playstation.entitlement_id': game_data['entitlementId'],
+                'me.acquired_by_playstation_membership': game_data['membership'],
+                'game.platform': game_data['platform'],
+                'game.icon': game_data['image']['url'],
+        }
+
+SCRAPERS = [
+        Scraper(scrape_played_last, 'games_played_playstation', deduplicate = True)
+]
+