personal-data/personal_data/fetchers/tavex.py

"""Scrapes Tavex.dk prices.

By request of my colleague.
"""

import abc
import dataclasses
from decimal import Decimal

import bs4

import personal_data.html_util
import personal_data.parse_util
from personal_data.data import DeduplicateMode, Scraper
import requests_util

URL_API_ROOT = 'https://tavex.dk/'

def parse_dkk_price(dkk: str) -> Decimal:
    if dkk.strip() == '-':
        return None
    return Decimal(dkk.removesuffix(' DKK').replace(',', '.'))


@dataclasses.dataclass(frozen=True)
class TavexScraperBase(Scraper):
    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS

    @staticmethod
    def requires_cfscrape() -> bool:
        return True

    @abc.abstractmethod
    def page_url() -> str:
        pass

    def _setup_cache(self):
        requests_util.setup_limiter(
            self.session,
            URL_API_ROOT,
            per_minute = 5,
        )

    def scrape(self):
        self._setup_cache()
        response = self.session.get(self.page_url())
        response.raise_for_status()

        NOW = personal_data.parse_util.parse_response_datetime(response)

        soup = bs4.BeautifulSoup(response.content, 'lxml')
        soup_page = personal_data.html_util.normalize_soup_slightly(
            soup,
            classes=False,
            scripts=True,
        )

        soup = soup.select_one('.product-poster__box .product-poster__table')

        table = []
        for soup_row in soup.children:
            if isinstance(soup_row, bs4.NavigableString):
                continue
            table.append(
                [
                    soup_cell.get_text().strip()
                    for soup_cell in soup_row
                    if not isinstance(soup_cell, bs4.NavigableString)
                ],
            )

        yield {
            'time': NOW,
            'buy': parse_dkk_price(table[1][1]),
            'sell': parse_dkk_price(table[1][2]),
            'spread_percentage': Decimal(table[1][3].removesuffix('%'))
            if len(table[1]) > 3
            else None,
        }


@dataclasses.dataclass(frozen=True)
class TavexScraperGold(TavexScraperBase):
    dataset_name = 'prices_tavex/guld-1oz-canadisk-maple-leaf-guldmont'
    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS

    @staticmethod
    def page_url() -> str:
        return f'{URL_API_ROOT}/guld/1oz-canadisk-maple-leaf-guldmont/'


@dataclasses.dataclass(frozen=True)
class TavexScraperSilver(TavexScraperBase):
    dataset_name = 'prices_tavex/solv-1-oz-american-eagle-solvmont-tidligere-argange'
    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS

    @staticmethod
    def page_url() -> str:
        return f'{URL_API_ROOT}/solv/1-oz-american-eagle-solvmont-tidligere-argange/'