Initial with Playstation
This commit is contained in:
commit
8b96802dd9
53
__main__.py
Normal file
53
__main__.py
Normal file
|
@ -0,0 +1,53 @@
|
|||
|
||||
import requests
|
||||
import requests_cache
|
||||
import csv
|
||||
import datetime
|
||||
import io
|
||||
from frozendict import frozendict
|
||||
|
||||
import playstation
|
||||
|
||||
def determine_scrapers():
|
||||
scrapers = []
|
||||
scrapers += playstation.SCRAPERS
|
||||
return scrapers
|
||||
|
||||
def extend_csv_file(filename, new_dicts , deduplicate = False):
|
||||
|
||||
dicts = []
|
||||
with open(filename, 'r') as csvfile:
|
||||
reader = csv.DictReader(csvfile)
|
||||
for row in reader:
|
||||
dicts .append(frozendict(row))
|
||||
del csvfile
|
||||
|
||||
dicts += [frozendict(d) for d in new_dicts]
|
||||
del new_dicts
|
||||
if deduplicate:
|
||||
dicts = list(set(dicts))
|
||||
|
||||
fieldnames = dicts[0].keys()
|
||||
|
||||
csvfile_in_memory = io.StringIO()
|
||||
writer = csv.DictWriter(csvfile_in_memory, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for d in dicts:
|
||||
writer.writerow(d)
|
||||
output_csv = csvfile_in_memory.getvalue()
|
||||
del writer, csvfile_in_memory
|
||||
|
||||
with open(filename, 'w') as csvfile:
|
||||
csvfile.write(output_csv)
|
||||
del csvfile
|
||||
|
||||
def main():
|
||||
session = requests_cache.CachedSession('web_cache')
|
||||
for scraper in determine_scrapers():
|
||||
result_rows = list(scraper.scraper(session))
|
||||
extend_csv_file('output/'+scraper.dataset_name, result_rows,
|
||||
deduplicate = scraper.deduplicate)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
10
data.py
Normal file
10
data.py
Normal file
|
@ -0,0 +1,10 @@
|
|||
|
||||
import dataclasses
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Scraper:
|
||||
scraper: object # TODO: Callable
|
||||
dataset_name: str
|
||||
deduplicate: bool
|
||||
dataset_format: str = 'list-of-dicts'
|
||||
|
55
playstation.py
Normal file
55
playstation.py
Normal file
|
@ -0,0 +1,55 @@
|
|||
|
||||
from data import Scraper
|
||||
import secrets
|
||||
|
||||
def scrape_played_last(session):
|
||||
url = "https://web.np.playstation.com/api/graphql/v1/op?operationName=getUserGameList&variables=%7B%22limit%22%3A50%2C%22categories%22%3A%22ps4_game%2Cps5_native_game%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%22e0136f81d7d1fb6be58238c574e9a46e1c0cc2f7f6977a08a5a46f224523a004%22%7D%7D"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0",
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"content-type": "application/json",
|
||||
"X-PSN-App-Ver": "my-playstation/0.1.0-20230720235210-hotfix-1-g1e9f07ff-1e9f07ff247eafea4d9d9236f73863cb9cd5d3e8",
|
||||
"X-PSN-Correlation-Id": "bc7a39b1-a99c-478b-9494-d2fddf189875",
|
||||
"apollographql-client-name": "my-playstation",
|
||||
"apollographql-client-version": "0.1.0-20230720235210-hotfix-1-g1e9f07ff",
|
||||
"X-PSN-Request-Id": "8ad64653-d8b5-4941-b565-b5536c9853df",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"Pragma": "no-cache",
|
||||
"Cache-Control": "no-cache",
|
||||
"Cookie": secrets.PLAYSTATION_COM_COOKIES,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Referer': 'https://library.playstation.com/',
|
||||
'Origin': 'https://library.playstation.com',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'TE': 'trailers',
|
||||
}
|
||||
result = session.get(url, headers = headers)
|
||||
result.raise_for_status()
|
||||
|
||||
print(result.json())
|
||||
games_data = result.json()['data']['gameLibraryTitlesRetrieve']['games']
|
||||
print(games_data)
|
||||
for game_data in games_data:
|
||||
yield {
|
||||
# Important fields
|
||||
'game.name': game_data['name'],
|
||||
'me.last_played_time': game_data['lastPlayedDateTime'],
|
||||
'playstation.product_id': game_data['productId'],
|
||||
|
||||
# Secondary fields
|
||||
'playstation.concept_id': game_data['conceptId'],
|
||||
'playstation.title_id': game_data['titleId'],
|
||||
'playstation.entitlement_id': game_data['entitlementId'],
|
||||
'me.acquired_by_playstation_membership': game_data['membership'],
|
||||
'game.platform': game_data['platform'],
|
||||
'game.icon': game_data['image']['url'],
|
||||
}
|
||||
|
||||
SCRAPERS = [
|
||||
Scraper(scrape_played_last, 'games_played_playstation', deduplicate = True)
|
||||
]
|
||||
|
Loading…
Reference in New Issue
Block a user