1
0
personal-data/personal_data/fetchers/ffxiv_lodestone.py

78 lines
2.6 KiB
Python
Raw Normal View History

2024-03-03 16:25:34 +00:00
import dataclasses
2024-03-31 22:55:55 +00:00
import datetime
2024-03-03 16:25:34 +00:00
import logging
2024-03-31 22:55:55 +00:00
import re
import secrets
2024-03-03 16:25:34 +00:00
import bs4
import personal_data.html_util
import personal_data.parse_util
2024-03-31 22:55:55 +00:00
from personal_data.data import DeduplicateMode, Scraper
2024-03-03 16:25:34 +00:00
logger = logging.getLogger(__name__)
URL_PROFILE_ACHIEVEMENTS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/achievement/?page={page_idx}'
2024-03-31 22:55:55 +00:00
URL_PROFILE_MINIONS = (
'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/minion/'
)
URL_PROFILE_MOUNTS = (
'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/mount/'
)
2024-03-03 16:25:34 +00:00
FORMAT_DATE_HEADER = '%d/%m/%YYYY'
2024-03-31 22:55:55 +00:00
@dataclasses.dataclass(frozen=True)
2024-03-03 16:25:34 +00:00
class LodestoneAchievementScraper(Scraper):
dataset_name = 'games_played_playstation'
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
def scrape(self):
2024-03-31 22:55:55 +00:00
for page_idx in range(1, 13 + 1): # TODO: Automatically determine
url = URL_PROFILE_ACHIEVEMENTS.format(
character_id=secrets.FFXIV_CHARACTER_ID,
page_idx=page_idx,
)
2024-03-03 16:25:34 +00:00
response = self.session.get(url)
response.raise_for_status()
NOW = personal_data.parse_util.response_datetime(response)
# Parse data
soup = bs4.BeautifulSoup(response.content, 'lxml')
2024-03-31 22:55:55 +00:00
soup = personal_data.html_util.normalize_soup_slightly(
soup,
classes=False,
scripts=False,
)
2024-03-03 16:25:34 +00:00
2024-03-31 22:55:55 +00:00
# print(soup)
2024-03-03 16:25:34 +00:00
for entry in soup.select('.ldst__achievement ul li.entry'):
time_acquired = str(entry.script.text).strip()
2024-03-31 22:55:55 +00:00
time_acquired = re.search(
r'ldst_strftime\((\d+)\s*,',
time_acquired,
).group(1)
2024-03-03 16:25:34 +00:00
time_acquired = int(time_acquired)
time_acquired = datetime.datetime.fromtimestamp(time_acquired)
2024-03-31 22:55:55 +00:00
trophy_desc = (
entry.select_one('.entry__activity__txt').get_text().strip()
)
trophy_name = re.match(
r'^.*achievement "([^"]+)" earned!$',
trophy_desc,
).group(1)
2024-03-03 16:25:34 +00:00
trophy_icon = entry.select_one('.entry__achievement__frame img')
trophy_icon = trophy_icon.src
yield {
2024-03-31 22:55:55 +00:00
'game.name': 'Final Fantasy XIV: A Realm Reborn',
2024-03-03 16:25:34 +00:00
'me.last_played_time': time_acquired,
# Trophy Data
'trophy.name': trophy_name,
'trophy.desc': trophy_desc,
'trophy.icon': trophy_icon,
}