Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Improved csv sniffing

Browse Source
This commit is contained in:
Jon Michael Aanes 2025-03-16 00:02:08 +01:00
parent 857be3cf2f
commit af42e3ba90
3 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
obsidian_import/__init__.py
View File

@ -261,17 +261,22 @@ PATH_WATCHED = Path('output/show_episodes_watched.csv')
PATH_PLAYED = Path('output/games_played.csv')
PATH_WORKOUT = Path('/home/jmaa/Notes/workout.csv')
PATH_STEP_COUNTS = Path(
'/home/jmaa/personal-archive/misc-data/step_counts_2023-07-26_to_2024-09-21.csv',
'/home/jmaa/Notes/Rawbackupdata/Steps/exportStepCount_2025-03-15_22-58-20',
)
PATH_STEPMANIA = Path('output/stepmania.csv')
IMPORTERS = [
{'path': PATH_WORKOUT, 'import_rows': import_workout_csv},
{'path': PATH_WORKOUT, 'standard_variant': True, 'import_rows': import_workout_csv},
{'path': PATH_STEP_COUNTS, 'import_rows': import_step_counts_csv},
{'path': PATH_STEPMANIA, 'import_rows': import_stepmania_steps_csv},
{
'path': PATH_STEPMANIA,
'standard_variant': True,
'import_rows': import_stepmania_steps_csv,
},
{
'path': PATH_PLAYED,
'standard_variant': True,
'import_rows': lambda vault, rows: import_activity_sample_csv(
vault,
rows,
@ -281,6 +286,7 @@ IMPORTERS = [
},
{
'path': PATH_WATCHED,
'standard_variant': True,
'import_rows': lambda vault, rows: import_activity_sample_csv(
vault,
rows,
@ -301,7 +307,9 @@ def import_data(obsidian_path: Path, dry_run=True):
import_def['path'],
)
continue
rows = load_csv_file(import_def['path'])
rows = load_csv_file(
import_def['path'], sniff=not import_def.get('standard_variant'),
)
logger.info('Loaded CSV with %d lines', len(rows))
num_files_updated = import_def['import_rows'](vault, rows)
logger.info('Updated %d files', num_files_updated)

7
personal_data/csv_import.py
View File

@ -2,6 +2,7 @@ import csv
import dataclasses
import datetime
import decimal
import logging
import typing
import urllib.parse
from collections.abc import Callable
@ -11,6 +12,8 @@ from typing import Any
from frozendict import frozendict
logger = logging.getLogger(__name__)
CSV_DIALECT = 'one_true_dialect'
csv.register_dialect(CSV_DIALECT, lineterminator='\n', skipinitialspace=True)
@ -86,10 +89,12 @@ def load_csv_file(csv_file: Path, sniff=False) -> list[frozendict[str, typing.An
dicts: list[frozendict] = []
with open(csv_file) as csvfile:
if sniff:
dialect = csv.Sniffer().sniff(csvfile.read(1024))
logger.warning('Sniffing CSV variant: %s', csv_file)
dialect = csv.Sniffer().sniff(csvfile.read(1024), delimiters=',;')
csvfile.seek(0)
else:
dialect = CSV_DIALECT
logger.warning('Loading CSV file: %s', csv_file)
reader = csv.DictReader(csvfile, dialect=dialect)
for row in reader:
for k in list(row.keys()):

3
personal_data/fetchers/youtube.py
View File

@ -22,7 +22,8 @@ def scrape(watch_history: bool) -> list[dict[str, str]]:
'yt-dlp',
url,
'--dump-json',
'--cookies-from-browser', 'firefox:/home/jmaa/.cachy/mbui5xg7.default-release',
'--cookies-from-browser',
'firefox:/home/jmaa/.cachy/mbui5xg7.default-release',
]
else:
url = f'https://www.youtube.com/playlist?list={PLAYLIST_ID}'