import dataclasses import datetime import logging import re import secrets import bs4 import personal_data.html_util import personal_data.parse_util from personal_data.data import DeduplicateMode, Scraper logger = logging.getLogger(__name__) URL_PROFILE_ACHIEVEMENTS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/achievement/?page={page_idx}' URL_PROFILE_MINIONS = ( 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/minion/' ) URL_PROFILE_MOUNTS = ( 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/mount/' ) FORMAT_DATE_HEADER = '%d/%m/%YYYY' @dataclasses.dataclass(frozen=True) class LodestoneAchievementScraper(Scraper): dataset_name = 'games_played_playstation' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS def scrape(self): for page_idx in range(1, 13 + 1): # TODO: Automatically determine url = URL_PROFILE_ACHIEVEMENTS.format( character_id=secrets.FFXIV_CHARACTER_ID, page_idx=page_idx, ) response = self.session.get(url) response.raise_for_status() NOW = personal_data.parse_util.response_datetime(response) # Parse data soup = bs4.BeautifulSoup(response.content, 'lxml') soup = personal_data.html_util.normalize_soup_slightly( soup, classes=False, scripts=False, ) # print(soup) for entry in soup.select('.ldst__achievement ul li.entry'): time_acquired = str(entry.script.text).strip() time_acquired = re.search( r'ldst_strftime\((\d+)\s*,', time_acquired, ).group(1) time_acquired = int(time_acquired) time_acquired = datetime.datetime.fromtimestamp(time_acquired) trophy_desc = ( entry.select_one('.entry__activity__txt').get_text().strip() ) trophy_name = re.match( r'^.*achievement "([^"]+)" earned!$', trophy_desc, ).group(1) trophy_icon = entry.select_one('.entry__achievement__frame img') trophy_icon = trophy_icon.src yield { 'game.name': 'Final Fantasy XIV: A Realm Reborn', 'me.last_played_time': time_acquired, # Trophy Data 'trophy.name': trophy_name, 'trophy.desc': trophy_desc, 'trophy.icon': trophy_icon, }