personal-data/personal_data/fetchers/ffxiv_lodestone.py

import dataclasses
import datetime
import logging
import re
import bs4

from ..data import DeduplicateMode, Scraper
from .. import secrets, parse_util, html_util

logger = logging.getLogger(__name__)

URL_PROFILE_ACHIEVEMENTS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/achievement/?page={page_idx}'
URL_PROFILE_MINIONS = (
    'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/minion/'
)
URL_PROFILE_MOUNTS = (
    'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/mount/'
)

FORMAT_DATE_HEADER = '%d/%m/%YYYY'


@dataclasses.dataclass(frozen=True)
class LodestoneAchievementScraper(Scraper):
    dataset_name = 'games_played_playstation'
    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS

    def scrape(self):
        for page_idx in range(1, 13 + 1):  # TODO: Automatically determine
            url = URL_PROFILE_ACHIEVEMENTS.format(
                character_id=secrets.FFXIV_CHARACTER_ID,
                page_idx=page_idx,
            )
            response = self.session.get(url)
            response.raise_for_status()

            NOW = parse_util.parse_response_datetime(response)

            # Parse data
            soup = bs4.BeautifulSoup(response.content, 'lxml')
            soup = html_util.normalize_soup_slightly(
                soup,
                classes=False,
                scripts=False,
            )

            for entry in soup.select('.ldst__achievement ul li.entry'):
                time_acquired = str(entry.script.text).strip()
                time_acquired = re.search(
                    r'ldst_strftime\((\d+)\s*,',
                    time_acquired,
                ).group(1)
                time_acquired = int(time_acquired)
                time_acquired = datetime.datetime.fromtimestamp(time_acquired,tz=datetime.UTC)
                trophy_desc = (
                    entry.select_one('.entry__activity__txt').get_text().strip()
                )
                trophy_name = re.match(
                    r'^.*achievement "([^"]+)" earned!$',
                    trophy_desc,
                ).group(1)
                trophy_icon = entry.select_one('.entry__achievement__frame img')
                trophy_icon = trophy_icon.src

                yield {
                    'game.name': 'Final Fantasy XIV: A Realm Reborn',
                    'me.last_played_time': time_acquired,
                    # Trophy Data
                    'trophy.name': trophy_name,
                    'trophy.desc': trophy_desc,
                    'trophy.icon': trophy_icon,
                }