100 lines
2.7 KiB
Python
100 lines
2.7 KiB
Python
"""Scrapes Tavex.dk prices.
|
|
|
|
By request of my colleague.
|
|
"""
|
|
|
|
import abc
|
|
import dataclasses
|
|
from decimal import Decimal
|
|
|
|
import bs4
|
|
|
|
import personal_data.html_util
|
|
import personal_data.parse_util
|
|
from personal_data.data import DeduplicateMode, Scraper
|
|
import requests_util
|
|
|
|
URL_API_ROOT = 'https://tavex.dk/'
|
|
|
|
def parse_dkk_price(dkk: str) -> Decimal:
|
|
if dkk.strip() == '-':
|
|
return None
|
|
return Decimal(dkk.removesuffix(' DKK').replace(',', '.'))
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class TavexScraperBase(Scraper):
|
|
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
|
|
|
@staticmethod
|
|
def requires_cfscrape() -> bool:
|
|
return True
|
|
|
|
@abc.abstractmethod
|
|
def page_url() -> str:
|
|
pass
|
|
|
|
def _setup_cache(self):
|
|
requests_util.setup_limiter(
|
|
self.session,
|
|
URL_API_ROOT,
|
|
per_minute = 5,
|
|
)
|
|
|
|
def scrape(self):
|
|
self._setup_cache()
|
|
response = self.session.get(self.page_url())
|
|
response.raise_for_status()
|
|
|
|
NOW = personal_data.parse_util.parse_response_datetime(response)
|
|
|
|
soup = bs4.BeautifulSoup(response.content, 'lxml')
|
|
soup_page = personal_data.html_util.normalize_soup_slightly(
|
|
soup,
|
|
classes=False,
|
|
scripts=True,
|
|
)
|
|
|
|
soup = soup.select_one('.product-poster__box .product-poster__table')
|
|
|
|
table = []
|
|
for soup_row in soup.children:
|
|
if isinstance(soup_row, bs4.NavigableString):
|
|
continue
|
|
table.append(
|
|
[
|
|
soup_cell.get_text().strip()
|
|
for soup_cell in soup_row
|
|
if not isinstance(soup_cell, bs4.NavigableString)
|
|
],
|
|
)
|
|
|
|
yield {
|
|
'time': NOW,
|
|
'buy': parse_dkk_price(table[1][1]),
|
|
'sell': parse_dkk_price(table[1][2]),
|
|
'spread_percentage': Decimal(table[1][3].removesuffix('%'))
|
|
if len(table[1]) > 3
|
|
else None,
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class TavexScraperGold(TavexScraperBase):
|
|
dataset_name = 'prices_tavex/guld-1oz-canadisk-maple-leaf-guldmont'
|
|
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
|
|
|
@staticmethod
|
|
def page_url() -> str:
|
|
return f'{URL_API_ROOT}/guld/1oz-canadisk-maple-leaf-guldmont/'
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class TavexScraperSilver(TavexScraperBase):
|
|
dataset_name = 'prices_tavex/solv-1-oz-american-eagle-solvmont-tidligere-argange'
|
|
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
|
|
|
@staticmethod
|
|
def page_url() -> str:
|
|
return f'{URL_API_ROOT}/solv/1-oz-american-eagle-solvmont-tidligere-argange/'
|