59 lines
2.3 KiB
Python
59 lines
2.3 KiB
Python
|
import secrets
|
||
|
|
import functools
|
||
|
|
import dataclasses
|
||
|
|
import re
|
||
|
|
import logging
|
||
|
|
import bs4
|
||
|
|
import datetime
|
||
|
|
|
||
|
|
from personal_data.data import Scraper, DeduplicateMode
|
||
|
|
import personal_data.html_util
|
||
|
|
import personal_data.parse_util
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
URL_PROFILE_ACHIEVEMENTS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/achievement/?page={page_idx}'
|
||
|
|
URL_PROFILE_MINIONS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/minion/'
|
||
|
|
URL_PROFILE_MOUNTS = 'https://eu.finalfantasyxiv.com/lodestone/character/{character_id}/mount/'
|
||
|
|
|
||
|
|
FORMAT_DATE_HEADER = '%d/%m/%YYYY'
|
||
|
|
|
||
|
|
@dataclasses.dataclass(frozen = True)
|
||
|
|
class LodestoneAchievementScraper(Scraper):
|
||
|
|
dataset_name = 'games_played_playstation'
|
||
|
|
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
||
|
|
|
||
|
|
def scrape(self):
|
||
|
|
for page_idx in range(1, 13+1): # TODO: Automatically determine
|
||
|
|
url = URL_PROFILE_ACHIEVEMENTS.format(character_id = secrets.FFXIV_CHARACTER_ID, page_idx = page_idx)
|
||
|
|
response = self.session.get(url)
|
||
|
|
response.raise_for_status()
|
||
|
|
|
||
|
|
NOW = personal_data.parse_util.response_datetime(response)
|
||
|
|
|
||
|
|
# Parse data
|
||
|
|
soup = bs4.BeautifulSoup(response.content, 'lxml')
|
||
|
|
soup = personal_data.html_util.normalize_soup_slightly(soup, classes = False, scripts = False)
|
||
|
|
|
||
|
|
#print(soup)
|
||
|
|
|
||
|
|
for entry in soup.select('.ldst__achievement ul li.entry'):
|
||
|
|
time_acquired = str(entry.script.text).strip()
|
||
|
|
time_acquired = re.search(r'ldst_strftime\((\d+)\s*,', time_acquired).group(1)
|
||
|
|
time_acquired = int(time_acquired)
|
||
|
|
time_acquired = datetime.datetime.fromtimestamp(time_acquired)
|
||
|
|
trophy_desc = entry.select_one('.entry__activity__txt').get_text().strip()
|
||
|
|
trophy_name = re.match(r'^.*achievement "([^"]+)" earned!$', trophy_desc).group(1)
|
||
|
|
trophy_icon = entry.select_one('.entry__achievement__frame img')
|
||
|
|
trophy_icon = trophy_icon.src
|
||
|
|
|
||
|
|
yield {
|
||
|
|
'game.name' : 'Final Fantasy XIV',
|
||
|
|
'me.last_played_time': time_acquired,
|
||
|
|
|
||
|
|
# Trophy Data
|
||
|
|
'trophy.name': trophy_name,
|
||
|
|
'trophy.desc': trophy_desc,
|
||
|
|
'trophy.icon': trophy_icon,
|
||
|
|
}
|