Compare commits
3 Commits
090b854f66
...
c6fd5bc5cc
Author | SHA1 | Date | |
---|---|---|---|
c6fd5bc5cc | |||
43cc7eceaf | |||
0057693f28 |
|
@ -1 +1 @@
|
||||||
__version__ = '0.1.40'
|
__version__ = '0.1.41'
|
||||||
|
|
|
@ -2,13 +2,10 @@ import dataclasses
|
||||||
import datetime
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import secrets
|
|
||||||
|
|
||||||
import bs4
|
import bs4
|
||||||
|
|
||||||
import personal_data.html_util
|
from ..data import DeduplicateMode, Scraper
|
||||||
import personal_data.parse_util
|
from .. import secrets, parse_util, html_util
|
||||||
from personal_data.data import DeduplicateMode, Scraper
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -37,11 +34,11 @@ class LodestoneAchievementScraper(Scraper):
|
||||||
response = self.session.get(url)
|
response = self.session.get(url)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
NOW = personal_data.parse_util.parse_response_datetime(response)
|
NOW = parse_util.parse_response_datetime(response)
|
||||||
|
|
||||||
# Parse data
|
# Parse data
|
||||||
soup = bs4.BeautifulSoup(response.content, 'lxml')
|
soup = bs4.BeautifulSoup(response.content, 'lxml')
|
||||||
soup = personal_data.html_util.normalize_soup_slightly(
|
soup = html_util.normalize_soup_slightly(
|
||||||
soup,
|
soup,
|
||||||
classes=False,
|
classes=False,
|
||||||
scripts=False,
|
scripts=False,
|
||||||
|
@ -54,7 +51,7 @@ class LodestoneAchievementScraper(Scraper):
|
||||||
time_acquired,
|
time_acquired,
|
||||||
).group(1)
|
).group(1)
|
||||||
time_acquired = int(time_acquired)
|
time_acquired = int(time_acquired)
|
||||||
time_acquired = datetime.datetime.fromtimestamp(time_acquired)
|
time_acquired = datetime.datetime.fromtimestamp(time_acquired,tz=datetime.UTC)
|
||||||
trophy_desc = (
|
trophy_desc = (
|
||||||
entry.select_one('.entry__activity__txt').get_text().strip()
|
entry.select_one('.entry__activity__txt').get_text().strip()
|
||||||
)
|
)
|
||||||
|
|
|
@ -36,10 +36,16 @@ def parse_response_datetime(response) -> datetime.datetime:
|
||||||
).replace(tzinfo=datetime.UTC)
|
).replace(tzinfo=datetime.UTC)
|
||||||
|
|
||||||
|
|
||||||
|
LOCAL_TIMEZONE = datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo
|
||||||
|
|
||||||
|
|
||||||
def parse_time(text: str) -> datetime.datetime:
|
def parse_time(text: str) -> datetime.datetime:
|
||||||
text = text.replace('\n', ' ')
|
text = text.replace('\n', ' ')
|
||||||
text = text.strip()
|
text = text.strip()
|
||||||
return datetime.datetime.strptime(text, '%d %b %Y %I:%M:%S %p')
|
time = datetime.datetime.strptime(text, '%d %b %Y %I:%M:%S %p')
|
||||||
|
time = time.replace(tzinfo=LOCAL_TIMEZONE)
|
||||||
|
assert time.tzinfo is not None, time
|
||||||
|
return time
|
||||||
|
|
||||||
|
|
||||||
def parse_date(text: str) -> datetime.date:
|
def parse_date(text: str) -> datetime.date:
|
||||||
|
|
|
@ -62,6 +62,8 @@ def csv_str_to_value(
|
||||||
def csv_safe_value(v: object) -> str:
|
def csv_safe_value(v: object) -> str:
|
||||||
if isinstance(v, urllib.parse.ParseResult):
|
if isinstance(v, urllib.parse.ParseResult):
|
||||||
return v.geturl()
|
return v.geturl()
|
||||||
|
if isinstance(v, datetime.datetime):
|
||||||
|
assert v.tzinfo is not None, v
|
||||||
return str(v)
|
return str(v)
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user