Compare commits
No commits in common. "088cac75fce0c03d155db00298fce2899cfd893a" and "bd3fc56a577b6b94a59a8fd8790e1641381cb4c8" have entirely different histories.
088cac75fc
...
bd3fc56a57
|
@ -8,7 +8,6 @@ from personal_data.csv_import import determine_possible_keys, load_csv_file, sta
|
||||||
|
|
||||||
logger = getLogger(__name__)
|
logger = getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
|
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
|
||||||
assert len(rows) > 0
|
assert len(rows) > 0
|
||||||
max_title_parts = 2
|
max_title_parts = 2
|
||||||
|
|
|
@ -106,7 +106,6 @@ def import_workout_csv(vault: ObsidianVault, rows: Rows) -> int:
|
||||||
|
|
||||||
MINIMUM_BELIEVABLE_STEP_COUNT = 300
|
MINIMUM_BELIEVABLE_STEP_COUNT = 300
|
||||||
|
|
||||||
|
|
||||||
def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
|
def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
|
||||||
num_updated = 0
|
num_updated = 0
|
||||||
|
|
||||||
|
|
|
@ -13,9 +13,7 @@ import frontmatter
|
||||||
import marko
|
import marko
|
||||||
import marko.md_renderer
|
import marko.md_renderer
|
||||||
|
|
||||||
assert hasattr(frontmatter, 'loads'), (
|
assert hasattr(frontmatter, 'loads'), 'Incorrect frontmatter package installed. Use: pip install python-frontmatter'
|
||||||
'Incorrect frontmatter package installed. Use: pip install python-frontmatter'
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = getLogger(__name__)
|
logger = getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -15,26 +15,18 @@ class DeduplicateMode(Enum):
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
@dataclasses.dataclass(frozen=True)
|
||||||
class Scraper(abc.ABC):
|
class Scraper(abc.ABC):
|
||||||
"""Base scraper class."""
|
|
||||||
|
|
||||||
session: requests.Session
|
session: requests.Session
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def dataset_name() -> str:
|
def dataset_name() -> str:
|
||||||
"""Indicates the filename of the produced dataset. Must be overwritten
|
pass
|
||||||
by the implementation."""
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def deduplicate_mode() -> DeduplicateMode:
|
def deduplicate_mode() -> DeduplicateMode:
|
||||||
"""Indicates how the rows should be deduplicated. Must be overwritten
|
pass
|
||||||
by the implementation."""
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def deduplicate_ignore_columns() -> list[str]:
|
def deduplicate_ignore_columns() -> list[str]:
|
||||||
"""Indicates columns which are not included in the deduplication check.
|
|
||||||
|
|
||||||
SQL comparison: Columns not in this set is part of the primary key.
|
|
||||||
"""
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -43,9 +35,8 @@ class Scraper(abc.ABC):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def requires_cfscrape() -> bool:
|
def requires_cfscrape() -> bool:
|
||||||
"""Whether the scraper requires advanced CloudFlare circumvention."""
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def scrape(self) -> Iterator[Mapping[str, object]]:
|
def scrape(self) -> Iterator[Mapping[str, object]]:
|
||||||
"""Implementation of the scraper."""
|
pass
|
||||||
|
|
Loading…
Reference in New Issue
Block a user