36 lines
639 B
Python
36 lines
639 B
Python
import dataclasses
|
|
import requests
|
|
from enum import Enum
|
|
import abc
|
|
|
|
class DeduplicateMode(Enum):
|
|
NONE = 0
|
|
BY_FIRST_COLUMN = 1
|
|
BY_ALL_COLUMNS = 2
|
|
|
|
@dataclasses.dataclass(frozen = True)
|
|
class Scraper(abc.ABC):
|
|
session: requests.Session
|
|
|
|
@staticmethod
|
|
@property
|
|
@abc.abstractmethod
|
|
def dataset_name(self) -> str:
|
|
pass
|
|
|
|
@staticmethod
|
|
@property
|
|
@abc.abstractmethod
|
|
def deduplicate_mode(self) -> DeduplicateMode:
|
|
pass
|
|
|
|
@staticmethod
|
|
@property
|
|
def dataset_format(self) -> str:
|
|
return 'list-of-dicts'
|
|
|
|
@abc.abstractmethod
|
|
def scrape(self):
|
|
pass
|
|
|