Improved caching situation
This commit is contained in:
parent
167b2c8f27
commit
6bdd778028
|
@ -10,11 +10,13 @@ from personal_data import secrets
|
|||
from personal_data.data import DeduplicateMode, Scraper
|
||||
|
||||
from .. import parse_util
|
||||
import requests_util
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
URL_PROFILE = 'https://psnprofiles.com/{psn_id}'
|
||||
URL_USER_GAME_TROPHIES = 'https://psnprofiles.com/trophies/{game_id}/{psn_id}'
|
||||
URL_API_ROOT = 'https://psnprofiles.com/'
|
||||
URL_PROFILE = URL_API_ROOT + '{psn_id}'
|
||||
URL_USER_GAME_TROPHIES = URL_API_ROOT + 'trophies/{game_id}/{psn_id}'
|
||||
|
||||
|
||||
def game_psnprofiles_id_from_url(relative_url: str) -> int:
|
||||
|
@ -25,7 +27,6 @@ def game_psnprofiles_id_from_url(relative_url: str) -> int:
|
|||
|
||||
MAX_GAME_ITERATIONS = 10
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class PsnProfilesScraper(Scraper):
|
||||
dataset_name = 'games_played_playstation'
|
||||
|
@ -36,19 +37,27 @@ class PsnProfilesScraper(Scraper):
|
|||
return True
|
||||
|
||||
def scrape(self):
|
||||
games_rows = list(self.scrape_games_overview())
|
||||
self._setup_cache()
|
||||
games_rows = list(self._scrape_games_overview())
|
||||
games_ids = {row['psnprofiles.game_id']: row['game.name'] for row in games_rows}
|
||||
|
||||
yield from games_rows
|
||||
idx = 0
|
||||
for game_id, game_name in games_ids.items():
|
||||
yield from self.scrape_game_trophies(game_id, game_name)
|
||||
yield from self._scrape_game_trophies(game_id, game_name)
|
||||
del game_id
|
||||
idx += 1
|
||||
if idx >= MAX_GAME_ITERATIONS:
|
||||
break
|
||||
|
||||
def scrape_games_overview(self) -> Iterator[dict]:
|
||||
def _setup_cache(self):
|
||||
requests_util.setup_limiter(
|
||||
self.session,
|
||||
URL_API_ROOT,
|
||||
per_minute = 5,
|
||||
)
|
||||
|
||||
def _scrape_games_overview(self) -> Iterator[dict]:
|
||||
# Request to get overview
|
||||
logger.info('Getting Overview')
|
||||
url = URL_PROFILE.format(psn_id=secrets.PLAYSTATION_PSN_ID)
|
||||
|
@ -136,7 +145,7 @@ class PsnProfilesScraper(Scraper):
|
|||
d['me.last_played_time'] = time_played
|
||||
yield d
|
||||
|
||||
def scrape_game_trophies(
|
||||
def _scrape_game_trophies(
|
||||
self,
|
||||
psnprofiles_id: int,
|
||||
game_name: str,
|
||||
|
|
|
@ -12,10 +12,11 @@ import bs4
|
|||
import personal_data.html_util
|
||||
import personal_data.parse_util
|
||||
from personal_data.data import DeduplicateMode, Scraper
|
||||
import requests_util
|
||||
|
||||
URL_API_ROOT = 'https://tavex.dk/'
|
||||
|
||||
def parse_dkk_price(dkk: str) -> Decimal:
|
||||
print(dkk)
|
||||
if dkk.strip() == '-':
|
||||
return None
|
||||
return Decimal(dkk.removesuffix(' DKK').replace(',', '.'))
|
||||
|
@ -33,7 +34,15 @@ class TavexScraperBase(Scraper):
|
|||
def page_url() -> str:
|
||||
pass
|
||||
|
||||
def _setup_cache(self):
|
||||
requests_util.setup_limiter(
|
||||
self.session,
|
||||
URL_API_ROOT,
|
||||
per_minute = 5,
|
||||
)
|
||||
|
||||
def scrape(self):
|
||||
self._setup_cache()
|
||||
response = self.session.get(self.page_url())
|
||||
response.raise_for_status()
|
||||
|
||||
|
@ -77,7 +86,7 @@ class TavexScraperGold(TavexScraperBase):
|
|||
|
||||
@staticmethod
|
||||
def page_url() -> str:
|
||||
return 'https://tavex.dk/guld/1oz-canadisk-maple-leaf-guldmont/'
|
||||
return f'{URL_API_ROOT}/guld/1oz-canadisk-maple-leaf-guldmont/'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
|
@ -87,4 +96,4 @@ class TavexScraperSilver(TavexScraperBase):
|
|||
|
||||
@staticmethod
|
||||
def page_url() -> str:
|
||||
return 'https://tavex.dk/solv/1-oz-american-eagle-solvmont-tidligere-argange/'
|
||||
return f'{URL_API_ROOT}/solv/1-oz-american-eagle-solvmont-tidligere-argange/'
|
||||
|
|
|
@ -43,6 +43,7 @@ if cfscrape:
|
|||
class CachedCfScrape(requests_cache.CacheMixin, cfscrape.CloudflareScraper):
|
||||
pass
|
||||
|
||||
CACHE_EXPIRE_DEFAULT = datetime.timedelta(days=7)
|
||||
|
||||
def get_session(
|
||||
cookiejar: Sequence,
|
||||
|
@ -57,7 +58,7 @@ def get_session(
|
|||
return requests.Session()
|
||||
if cfscrape:
|
||||
session_class = CachedCfScrape
|
||||
session = session_class(OUTPUT_PATH / 'web_cache', cookies=cookiejar, expire_after=datetime.timedelta(days=1))
|
||||
session = session_class(OUTPUT_PATH / 'web_cache', cookies=cookiejar, expire_after=CACHE_EXPIRE_DEFAULT)
|
||||
for cookie in cookiejar:
|
||||
session.cookies.set_cookie(cookie)
|
||||
return session
|
||||
|
|
Loading…
Reference in New Issue
Block a user