1
0

Compare commits

..

No commits in common. "119b380f8abac15fe1e92e7c30da8f738e739c2c" and "1c39c8a017e4567eccc23927c37a31e21ade31d5" have entirely different histories.

6 changed files with 34 additions and 36 deletions

View File

@ -6,8 +6,10 @@ from typing import Any
from personal_data.activity import ActivitySample, Label
from personal_data.csv_import import determine_possible_keys, load_csv_file, start_end
print(__name__)
logger = getLogger(__name__)
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
assert len(rows) > 0
max_title_parts = 2

View File

@ -5,7 +5,7 @@ Sub-module for importing time-based data into Obsidian.
import dataclasses
import datetime
from collections.abc import Iterator
from collections.abc import Iterable, Iterator
from logging import getLogger
from pathlib import Path
from typing import Any
@ -104,9 +104,9 @@ def import_workout_csv(vault: ObsidianVault, rows: Rows) -> int:
return num_updated
MINIMUM_BELIEVABLE_STEP_COUNT = 300
def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
MINIMUM_STEPS = 300
num_updated = 0
rows_per_date = {}
@ -121,7 +121,7 @@ def import_step_counts_csv(vault: ObsidianVault, rows: Rows) -> int:
}
for date, steps in steps_per_date.items():
if steps < MINIMUM_BELIEVABLE_STEP_COUNT:
if steps < MINIMUM_STEPS:
continue
was_updated = vault.add_statistic(date, 'Steps', steps)
if was_updated:
@ -155,6 +155,8 @@ def import_stepmania_steps_csv(vault: ObsidianVault, rows: Rows) -> int:
date: sum((row['play.duration'] for row in rows), start=datetime.timedelta())
for date, rows in rows_per_date.items()
}
print(steps_per_date)
print(duration_per_date)
for date in steps_per_date:
was_updated_1 = vault.add_statistic(

View File

@ -13,8 +13,6 @@ import frontmatter
import marko
import marko.md_renderer
assert hasattr(frontmatter, 'loads'), 'Incorrect frontmatter package installed. Use: pip install python-frontmatter'
logger = getLogger(__name__)
StatisticKey = str

View File

@ -1,9 +1,6 @@
import argparse
import logging
logging.basicConfig()
logging.getLogger('personal_data').setLevel('INFO')
import personal_data.main
from personal_data.notification import NotificationType
@ -40,6 +37,9 @@ def parse_arguments():
def main():
logging.basicConfig()
logging.getLogger('personal_data').setLevel('INFO')
args = parse_arguments()
scraper_filter = frozenset(args.fetchers)

View File

@ -12,10 +12,10 @@ from . import data, fetchers, notification, util
logger = logging.getLogger(__name__)
try:
import cloudscraper
import cfscrape
except ImportError:
cloudscraper = None
logger.exception('cloudscraper not installed: Certain fetchers might not work')
cfscrape = None
logger.exception('cfscrape not installed: Certain fetchers might not work')
try:
import browser_cookie3
@ -26,6 +26,11 @@ except ImportError:
OUTPUT_PATH = Path('./output')
logging.basicConfig(
format='%(asctime)s %(levelname)s %(module)s:%(lineno)d - %(message)s',
)
logger.setLevel('INFO')
STANDARD_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
@ -34,9 +39,9 @@ STANDARD_HEADERS = {
}
if cloudscraper:
if cfscrape:
class CachedCfScrape(requests_cache.CacheMixin, cloudscraper.CloudScraper):
class CachedCfScrape(requests_cache.CacheMixin, cfscrape.CloudflareScraper):
pass
@ -49,21 +54,13 @@ def get_session(
with_cfscrape: bool,
ignore_cache: bool,
) -> requests.Session:
session_class = requests_cache.CachedSession
if with_cfscrape:
if cloudscraper:
if with_cfscrape and cfscrape:
session_class = CachedCfScrape
if ignore_cache:
logger.warning('HTTP cache disabled')
return cloudscraper.create_scraper(
interpreter='js2py',
delay=5,
debug=False,
)
else:
logger.error('Expected cloudscraper, but not defined!')
return cfscrape.create_scraper()
else:
session_class = requests_cache.CachedSession
if ignore_cache:
logger.warning('HTTP cache disabled')
return requests.Session()
@ -103,7 +100,6 @@ def get_cookiejar(use_cookiejar: bool):
browser_cookie3.firefox(
'/home/jmaa/.cachy/mbui5xg7.default-release/cookies.sqlite',
)
logger.warning('Cookiejar has %s cookies', len(cookiejar))
if len(cookiejar) > 10:
return cookiejar
logger.warning('No cookiejar is used')
@ -118,23 +114,23 @@ def main(
notification_types: frozenset[notification.NotificationType],
) -> None:
cookiejar = get_cookiejar(use_cookiejar)
logger.warning('Cookiejar has %s cookies', len(cookiejar))
if len(notification_types) == 0:
logger.info('No notifications enabled: Notifications will not be sent!')
for scraper_cls in available_scrapers():
if scraper_cls.__name__ not in scraper_filter:
continue
session = get_session(
cookiejar=cookiejar,
cookiejar,
with_cfscrape=scraper_cls.requires_cfscrape(),
ignore_cache=ignore_cache,
)
scraper = scraper_cls(session)
if scraper_cls.__name__ not in scraper_filter:
continue
logger.info(
'Running %s (%s), appending to "%s"',
'Running %s, appending to "%s"',
scraper_cls.__name__,
type(session).__name__,
scraper.dataset_name,
)
result_rows = []
@ -142,9 +138,8 @@ def main(
for result in scraper.scrape():
result_rows.append(result)
del result
except requests.exceptions.HTTPError as e:
except requests.exceptions.HTTPError:
logger.exception('Failed in running %s', scraper_cls.__name__)
logger.error('User-Agent: %s', e.request.headers['user-agent'])
continue
status = util.extend_csv_file(
OUTPUT_PATH / f'{scraper.dataset_name}.csv',

View File

@ -76,6 +76,7 @@ def find_python_packages() -> list[str]:
for init_file in root_path.rglob('__init__.py'):
packages.add(str(init_file.parent).replace('/', '.'))
print(f'Found following packages: {packages}')
return sorted(packages)
with open(PACKAGE_NAME + '/_version.py') as f: