1
0
personal-data/personal_data/data.py

35 lines
614 B
Python
Raw Normal View History

2023-12-10 23:27:56 +00:00
import dataclasses
2024-03-03 15:59:03 +00:00
import requests
2024-02-25 19:20:37 +00:00
from enum import Enum
2024-03-03 15:59:03 +00:00
import abc
2024-02-25 19:20:37 +00:00
class DeduplicateMode(Enum):
NONE = 0
BY_FIRST_COLUMN = 1
BY_ALL_COLUMNS = 2
2023-12-10 23:27:56 +00:00
2024-03-03 15:59:03 +00:00
@dataclasses.dataclass(frozen = True)
class Scraper(abc.ABC):
session: requests.Session
@staticmethod
2024-03-03 16:25:34 +00:00
def dataset_name() -> str:
2024-03-03 15:59:03 +00:00
pass
@staticmethod
2024-03-03 16:25:34 +00:00
def deduplicate_mode() -> DeduplicateMode:
2024-03-03 15:59:03 +00:00
pass
@staticmethod
2024-03-03 16:25:34 +00:00
def dataset_format() -> str:
2024-03-03 15:59:03 +00:00
return 'list-of-dicts'
2024-03-03 16:25:34 +00:00
@staticmethod
def requires_cfscrape() -> bool:
return False
2024-03-03 15:59:03 +00:00
@abc.abstractmethod
def scrape(self):
pass
2023-12-10 23:27:56 +00:00