Compare commits
3 Commits
c6fd5bc5cc
...
784d55d1a0
Author | SHA1 | Date | |
---|---|---|---|
784d55d1a0 | |||
a6b059b25e | |||
263b0b2843 |
|
@ -1 +1 @@
|
||||||
__version__ = '0.1.41'
|
__version__ = '0.1.42'
|
||||||
|
|
70
personal_data/fetchers/steam_community.py
Normal file
70
personal_data/fetchers/steam_community.py
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
import dataclasses
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import bs4
|
||||||
|
from typing import Any
|
||||||
|
from collections.abc import Iterator
|
||||||
|
|
||||||
|
from ..data import DeduplicateMode, Scraper
|
||||||
|
from .. import secrets, parse_util, html_util
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
URL_SITE_ROOT = 'https://steamcommunity.com/'
|
||||||
|
|
||||||
|
URL_GAME_ACHIVEMENTS = URL_SITE_ROOT+'id/{username}/stats/appid/{appid}'
|
||||||
|
|
||||||
|
FORMAT_DATE_HEADER = '%d/%m/%YYYY'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class SteamAchievementScraper(Scraper):
|
||||||
|
dataset_name = 'games_played_TODO'
|
||||||
|
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
||||||
|
|
||||||
|
def scrape(self) -> Iterator[dict[str, Any]]:
|
||||||
|
yield from self.scrape_app(105600)
|
||||||
|
|
||||||
|
def scrape_app(self, appid: int) -> Iterator[dict[str, Any]]:
|
||||||
|
url = URL_GAME_ACHIVEMENTS.format(
|
||||||
|
username=secrets.STEAM_USERNAME,
|
||||||
|
appid=appid,
|
||||||
|
)
|
||||||
|
response = self.session.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
NOW = parse_util.parse_response_datetime(response)
|
||||||
|
|
||||||
|
# Parse data
|
||||||
|
soup = bs4.BeautifulSoup(response.content, 'lxml')
|
||||||
|
|
||||||
|
game_name: str = re.match(r'Steam Community :: (.+) :: Jmaa', soup.head.title.get_text()).group(1)
|
||||||
|
|
||||||
|
soup = html_util.normalize_soup_slightly(
|
||||||
|
soup,
|
||||||
|
classes=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
for entry in soup.select('.achieveRow'):
|
||||||
|
trophy_name: str = entry.select_one('h3').get_text()
|
||||||
|
trophy_desc: str = entry.select_one('h5').get_text()
|
||||||
|
trophy_icon: str = entry.select_one('img').src
|
||||||
|
|
||||||
|
time_acquired_html: str = entry.select_one('.achieveUnlockTime')
|
||||||
|
if time_acquired_html is None:
|
||||||
|
continue
|
||||||
|
time_acquired_text: str = time_acquired_html.get_text().strip().removeprefix('Unlocked ')
|
||||||
|
time_acquired: datetime.datetime = parse_util.parse_time(time_acquired_text)
|
||||||
|
|
||||||
|
yield {
|
||||||
|
'game.name': game_name,
|
||||||
|
'me.last_played_time': time_acquired,
|
||||||
|
# Trophy Data
|
||||||
|
'trophy.name': trophy_name,
|
||||||
|
'trophy.desc': trophy_desc,
|
||||||
|
'trophy.icon': trophy_icon,
|
||||||
|
}
|
||||||
|
|
||||||
|
del entry, time_acquired
|
|
@ -36,14 +36,26 @@ def parse_response_datetime(response) -> datetime.datetime:
|
||||||
).replace(tzinfo=datetime.UTC)
|
).replace(tzinfo=datetime.UTC)
|
||||||
|
|
||||||
|
|
||||||
LOCAL_TIMEZONE = datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo
|
LOCAL_TIMEZONE = datetime.datetime.now(datetime.UTC).astimezone().tzinfo
|
||||||
|
|
||||||
|
|
||||||
|
def try_parse(text:str, fmt:str) -> datetime.datetime | None:
|
||||||
|
try:
|
||||||
|
time = datetime.datetime.strptime(text, fmt)
|
||||||
|
if time.tzinfo is None:
|
||||||
|
time = time.replace(tzinfo=LOCAL_TIMEZONE)
|
||||||
|
except:
|
||||||
|
time = None
|
||||||
|
return time
|
||||||
|
|
||||||
|
|
||||||
def parse_time(text: str) -> datetime.datetime:
|
def parse_time(text: str) -> datetime.datetime:
|
||||||
text = text.replace('\n', ' ')
|
text = text.replace('\n', ' ')
|
||||||
text = text.strip()
|
text = text.strip()
|
||||||
time = datetime.datetime.strptime(text, '%d %b %Y %I:%M:%S %p')
|
|
||||||
time = time.replace(tzinfo=LOCAL_TIMEZONE)
|
time = try_parse(text, '%d %b %Y %I:%M:%S %p')
|
||||||
|
time = time or try_parse(text, '%d %b, %Y @ %I:%M%p')
|
||||||
|
|
||||||
assert time.tzinfo is not None, time
|
assert time.tzinfo is not None, time
|
||||||
return time
|
return time
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,9 @@ PLAYSTATION_PSN_ID = load_secret('PLAYSTATION_PSN_ID')
|
||||||
# Partisia Blockchain
|
# Partisia Blockchain
|
||||||
PBC_ACCOUNT_ADDRESS = load_secret('PBC_ACCOUNT_ADDRESS')
|
PBC_ACCOUNT_ADDRESS = load_secret('PBC_ACCOUNT_ADDRESS')
|
||||||
|
|
||||||
|
# Steam
|
||||||
|
STEAM_USERNAME = load_secret('STEAM_USERNAME')
|
||||||
|
|
||||||
# Kucoin
|
# Kucoin
|
||||||
KUCOIN_KEY = load_secret('KUCOIN_KEY')
|
KUCOIN_KEY = load_secret('KUCOIN_KEY')
|
||||||
KUCOIN_SECRET = load_secret('KUCOIN_SECRET')
|
KUCOIN_SECRET = load_secret('KUCOIN_SECRET')
|
||||||
|
|
|
@ -7,6 +7,7 @@ lint.ignore = [
|
||||||
'Q003', 'D205', # Format conflict
|
'Q003', 'D205', # Format conflict
|
||||||
|
|
||||||
'TCH', # Microoptimization at the cost of readability
|
'TCH', # Microoptimization at the cost of readability
|
||||||
|
'TID252', # I like relative imports
|
||||||
|
|
||||||
'D407', 'D413', # Weird documentation stuff
|
'D407', 'D413', # Weird documentation stuff
|
||||||
|
|
||||||
|
|
|
@ -60,7 +60,7 @@ LIST_2 = [
|
||||||
27,
|
27,
|
||||||
31,
|
31,
|
||||||
134506,
|
134506,
|
||||||
tzinfo=datetime.timezone.utc,
|
tzinfo=datetime.UTC,
|
||||||
),
|
),
|
||||||
'weight.kg': Decimal('73.6'),
|
'weight.kg': Decimal('73.6'),
|
||||||
},
|
},
|
||||||
|
@ -75,7 +75,7 @@ LIST_2 = [
|
||||||
36,
|
36,
|
||||||
9,
|
9,
|
||||||
590355,
|
590355,
|
||||||
tzinfo=datetime.timezone.utc,
|
tzinfo=datetime.UTC,
|
||||||
),
|
),
|
||||||
'weight.kg': Decimal('74.7'),
|
'weight.kg': Decimal('74.7'),
|
||||||
},
|
},
|
||||||
|
|
|
@ -8,7 +8,7 @@ from personal_data.main import to_value
|
||||||
PARSE_MAPPINGS = [
|
PARSE_MAPPINGS = [
|
||||||
(
|
(
|
||||||
'2024-04-28 21:35:40+00:00',
|
'2024-04-28 21:35:40+00:00',
|
||||||
datetime.datetime(2024, 4, 28, 21, 35, 40, tzinfo=datetime.timezone.utc),
|
datetime.datetime(2024, 4, 28, 21, 35, 40, tzinfo=datetime.UTC),
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'0003791e9f5f3691b8bbbe0d12a7ae9c3f2e89db38',
|
'0003791e9f5f3691b8bbbe0d12a7ae9c3f2e89db38',
|
||||||
|
|
Loading…
Reference in New Issue
Block a user