import secrets import functools import re import logging import bs4 import datetime from personal_data.data import Scraper, DeduplicateMode import personal_data.html_util import personal_data.parse_util logger = logging.getLogger(__name__) URL_PROFILE = 'https://psnprofiles.com/{psn_id}' FORMAT_DATE_HEADER = '%a, %d %b %Y %H:%M:%S GMT' FORMAT_DAY_MONTH_YEAR = '%d %B %Y' def game_psnprofiles_id_from_url(relative_url: str) -> int: m = re.match(r'/(?:trophy|trophies)/(\d+)\-(?:[\w-]+)(/[\w-]*)?', relative_url) result = m.group(1) return int(result) assert game_psnprofiles_id_from_url('/trophies/21045-theatrhythm-final-bar-line') assert game_psnprofiles_id_from_url('/trophies/21045-theatrhythm-final-bar-line/') assert game_psnprofiles_id_from_url('/trophies/21045-theatrhythm-final-bar-line/Jmaanes') assert game_psnprofiles_id_from_url('/trophy/21045-theatrhythm-final-bar-line/19-seasoned-hunter') def scrape_personal_page(session): # Request to get watch history logger.info('Getting Watchlist') url = URL_PROFILE.format(psn_id = secrets.PLAYSTATION_PSN_ID) response = session.get(url) response.raise_for_status() NOW = datetime.datetime.strptime(response.headers['Date'], FORMAT_DATE_HEADER) # Parse data soup = bs4.BeautifulSoup(response.content, 'lxml') soup = personal_data.html_util.normalize_soup_slightly(soup, classes = False) # Recent trophies. soup_recent_tropies = soup.select('ul#recent-trophies > li') assert len(soup_recent_tropies) > 0, url for row in soup_recent_tropies: cells = row.select_one('.info .box td').find_all('div') trophy_name = cells[0].get_text().strip() trophy_desc = cells[1].get_text().strip() game_name = cells[2].a.extract().get_text().strip() psnprofiles_id = game_psnprofiles_id_from_url(cells[0].find('a')['href']) trophy_icon = row.find(class_='icon').find('img')['src'] gotten_at = cells[2].get_text().strip().removesuffix(' in').removesuffix(' ago') gotten_at = personal_data.parse_util.parse_duration(gotten_at) time_acquired = NOW - gotten_at yield { 'game.name' : game_name, 'me.last_played_time': time_acquired, # Trophy Data 'trophy.name': trophy_name, 'trophy.desc': trophy_desc, 'trophy.icon': trophy_icon, 'psnprofiles.game_id': psnprofiles_id, } del row, cells, time_acquired # Games table table_rows = soup.find(id = 'gamesTable').find_all('tr') assert len(table_rows) > 0, url for row in table_rows: cells = row.find_all('td') # Check for pagination if re.match(r'show \d+ more games', cells[0].get_text().strip(), re.IGNORECASE): break game_name = cells[1].find(class_ = 'title').get_text() psnprofiles_id = game_psnprofiles_id_from_url(cells[0].find('a')['href']) game_icon = cells[0].find('img')['src'] game_name = row.select_one('.title').get_text() game_platform = row.select_one('.platform').get_text() small_infos = cells[1].find_all('div') if len(small_infos) > 2: time_played_div = small_infos[2] time_played_div.sup.extract() time_played = datetime.datetime.strptime(time_played_div.get_text().strip(), FORMAT_DAY_MONTH_YEAR).date() else: time_played = None d = { # Important fields 'game.name': game_name, # Secondary fields 'game.platform': game_platform, 'game.icon': game_icon, 'psnprofiles.game_id': psnprofiles_id, } if time_played: d['me.last_played_time'] = time_played yield d SCRAPERS = [ Scraper(scrape_personal_page, 'games_played_playstation', deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS) ]