Compare commits
2 Commits
bd3fc56a57
...
088cac75fc
Author | SHA1 | Date | |
---|---|---|---|
088cac75fc | |||
8feb3e2cde |
|
@ -8,6 +8,7 @@ from personal_data.csv_import import determine_possible_keys, load_csv_file, sta
|
||||||
|
|
||||||
logger = getLogger(__name__)
|
logger = getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
|
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
|
||||||
assert len(rows) > 0
|
assert len(rows) > 0
|
||||||
max_title_parts = 2
|
max_title_parts = 2
|
||||||
|
|
|
@ -106,6 +106,7 @@ def import_workout_csv(vault: ObsidianVault, rows: Rows) -> int:
|
||||||
|
|
||||||
MINIMUM_BELIEVABLE_STEP_COUNT = 300
|
MINIMUM_BELIEVABLE_STEP_COUNT = 300
|
||||||
|
|
||||||
|
|
||||||
def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
|
def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
|
||||||
num_updated = 0
|
num_updated = 0
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,9 @@ import frontmatter
|
||||||
import marko
|
import marko
|
||||||
import marko.md_renderer
|
import marko.md_renderer
|
||||||
|
|
||||||
assert hasattr(frontmatter, 'loads'), 'Incorrect frontmatter package installed. Use: pip install python-frontmatter'
|
assert hasattr(frontmatter, 'loads'), (
|
||||||
|
'Incorrect frontmatter package installed. Use: pip install python-frontmatter'
|
||||||
|
)
|
||||||
|
|
||||||
logger = getLogger(__name__)
|
logger = getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -15,18 +15,26 @@ class DeduplicateMode(Enum):
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
@dataclasses.dataclass(frozen=True)
|
||||||
class Scraper(abc.ABC):
|
class Scraper(abc.ABC):
|
||||||
|
"""Base scraper class."""
|
||||||
|
|
||||||
session: requests.Session
|
session: requests.Session
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def dataset_name() -> str:
|
def dataset_name() -> str:
|
||||||
pass
|
"""Indicates the filename of the produced dataset. Must be overwritten
|
||||||
|
by the implementation."""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def deduplicate_mode() -> DeduplicateMode:
|
def deduplicate_mode() -> DeduplicateMode:
|
||||||
pass
|
"""Indicates how the rows should be deduplicated. Must be overwritten
|
||||||
|
by the implementation."""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def deduplicate_ignore_columns() -> list[str]:
|
def deduplicate_ignore_columns() -> list[str]:
|
||||||
|
"""Indicates columns which are not included in the deduplication check.
|
||||||
|
|
||||||
|
SQL comparison: Columns not in this set is part of the primary key.
|
||||||
|
"""
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -35,8 +43,9 @@ class Scraper(abc.ABC):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def requires_cfscrape() -> bool:
|
def requires_cfscrape() -> bool:
|
||||||
|
"""Whether the scraper requires advanced CloudFlare circumvention."""
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def scrape(self) -> Iterator[Mapping[str, object]]:
|
def scrape(self) -> Iterator[Mapping[str, object]]:
|
||||||
pass
|
"""Implementation of the scraper."""
|
||||||
|
|
Loading…
Reference in New Issue
Block a user