This commit is contained in:
parent
477fce869d
commit
72be664d82
|
@ -33,9 +33,10 @@ from collections.abc import Iterator
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from personal_data.activity import (
|
from personal_data.activity import (
|
||||||
RealizedActivitySample,
|
|
||||||
ActivitySample,
|
ActivitySample,
|
||||||
|
RealizedActivitySample,
|
||||||
)
|
)
|
||||||
|
|
||||||
from .format import cli, icalendar
|
from .format import cli, icalendar
|
||||||
from .source import csv_file, git_repo
|
from .source import csv_file, git_repo
|
||||||
|
|
||||||
|
@ -85,7 +86,9 @@ def heuristically_realize_samples(
|
||||||
start_at = max(previous_sample_end, end_at - estimated_duration)
|
start_at = max(previous_sample_end, end_at - estimated_duration)
|
||||||
del estimated_duration
|
del estimated_duration
|
||||||
|
|
||||||
yield RealizedActivitySample(labels=sample.labels, end_at=end_at, start_at=start_at)
|
yield RealizedActivitySample(
|
||||||
|
labels=sample.labels, end_at=end_at, start_at=start_at,
|
||||||
|
)
|
||||||
|
|
||||||
previous_sample_end = sample.end_at
|
previous_sample_end = sample.end_at
|
||||||
del sample
|
del sample
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import datetime
|
import datetime
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
|
|
||||||
from personal_data.activity import HIDDEN_LABEL_CATEGORY, RealizedActivitySample, Label
|
from personal_data.activity import HIDDEN_LABEL_CATEGORY, Label, RealizedActivitySample
|
||||||
|
|
||||||
ZERO_DURATION = datetime.timedelta(seconds=0)
|
ZERO_DURATION = datetime.timedelta(seconds=0)
|
||||||
HOUR = datetime.timedelta(hours=1)
|
HOUR = datetime.timedelta(hours=1)
|
||||||
|
|
|
@ -2,7 +2,7 @@ import datetime
|
||||||
|
|
||||||
import icalendar
|
import icalendar
|
||||||
|
|
||||||
from personal_data.activity import HIDDEN_LABEL_CATEGORY, RealizedActivitySample, Label
|
from personal_data.activity import HIDDEN_LABEL_CATEGORY, RealizedActivitySample
|
||||||
|
|
||||||
ZERO_DURATION = datetime.timedelta(seconds=0)
|
ZERO_DURATION = datetime.timedelta(seconds=0)
|
||||||
HOUR = datetime.timedelta(hours=1)
|
HOUR = datetime.timedelta(hours=1)
|
||||||
|
|
|
@ -1,22 +1,17 @@
|
||||||
import datetime
|
|
||||||
import urllib.parse
|
|
||||||
from typing import Any
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from decimal import Decimal
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import dataclasses
|
from typing import Any
|
||||||
|
|
||||||
from personal_data.csv_import import load_csv_file, start_end, determine_possible_keys
|
|
||||||
|
|
||||||
from personal_data.activity import ActivitySample, Label
|
from personal_data.activity import ActivitySample, Label
|
||||||
|
from personal_data.csv_import import determine_possible_keys, load_csv_file, start_end
|
||||||
|
|
||||||
def iterate_samples_from_dicts(rows: list[dict[str,Any]]) -> Iterator[ActivitySample]:
|
|
||||||
|
def iterate_samples_from_dicts(rows: list[dict[str, Any]]) -> Iterator[ActivitySample]:
|
||||||
assert len(rows) > 0
|
assert len(rows) > 0
|
||||||
max_title_parts = 2
|
max_title_parts = 2
|
||||||
|
|
||||||
|
|
||||||
if True:
|
if True:
|
||||||
event_data = rows[len(rows)//2] # Hopefully select a useful representative.
|
event_data = rows[len(rows) // 2] # Hopefully select a useful representative.
|
||||||
possible_keys = determine_possible_keys(event_data)
|
possible_keys = determine_possible_keys(event_data)
|
||||||
del event_data
|
del event_data
|
||||||
|
|
||||||
|
@ -24,13 +19,13 @@ def iterate_samples_from_dicts(rows: list[dict[str,Any]]) -> Iterator[ActivitySa
|
||||||
assert len(possible_keys.image) >= 0
|
assert len(possible_keys.image) >= 0
|
||||||
|
|
||||||
for event_data in rows:
|
for event_data in rows:
|
||||||
'''
|
"""
|
||||||
title = ': '.join(event_data[k] for k in possible_name_keys[:max_title_parts])
|
title = ': '.join(event_data[k] for k in possible_name_keys[:max_title_parts])
|
||||||
description = '\n\n'.join(
|
description = '\n\n'.join(
|
||||||
event_data[k] for k in possible_name_keys[max_title_parts:]
|
event_data[k] for k in possible_name_keys[max_title_parts:]
|
||||||
)
|
)
|
||||||
image = event_data[possible_keys.image[0]] if possible_keys.image else None
|
image = event_data[possible_keys.image[0]] if possible_keys.image else None
|
||||||
'''
|
"""
|
||||||
|
|
||||||
(start_at, end_at) = start_end(event_data, possible_keys)
|
(start_at, end_at) = start_end(event_data, possible_keys)
|
||||||
labels = [Label(k, event_data[k]) for k in possible_keys.misc]
|
labels = [Label(k, event_data[k]) for k in possible_keys.misc]
|
||||||
|
|
|
@ -5,7 +5,7 @@ from pathlib import Path
|
||||||
|
|
||||||
import git
|
import git
|
||||||
|
|
||||||
from personal_data.activity import ActivitySample, Label, HIDDEN_LABEL_CATEGORY
|
from personal_data.activity import HIDDEN_LABEL_CATEGORY, ActivitySample, Label
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ from collections.abc import Sequence
|
||||||
|
|
||||||
HIDDEN_LABEL_CATEGORY = '__'
|
HIDDEN_LABEL_CATEGORY = '__'
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True, order=True)
|
@dataclasses.dataclass(frozen=True, order=True)
|
||||||
class Label:
|
class Label:
|
||||||
category: str
|
category: str
|
||||||
|
@ -14,6 +15,7 @@ class Label:
|
||||||
assert ':' not in self.category
|
assert ':' not in self.category
|
||||||
assert self.label is not None
|
assert self.label is not None
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True, order=True)
|
@dataclasses.dataclass(frozen=True, order=True)
|
||||||
class ActivitySample:
|
class ActivitySample:
|
||||||
labels: Sequence[Label]
|
labels: Sequence[Label]
|
||||||
|
|
|
@ -1,21 +1,13 @@
|
||||||
import datetime
|
|
||||||
import urllib.parse
|
|
||||||
from typing import Any
|
|
||||||
from collections.abc import Iterator
|
|
||||||
from decimal import Decimal
|
|
||||||
from pathlib import Path
|
|
||||||
import dataclasses
|
|
||||||
import _csv
|
|
||||||
import csv
|
import csv
|
||||||
|
import dataclasses
|
||||||
import datetime
|
import datetime
|
||||||
import decimal
|
import decimal
|
||||||
import io
|
|
||||||
import logging
|
|
||||||
import typing
|
import typing
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Callable
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from frozendict import frozendict
|
from frozendict import frozendict
|
||||||
|
|
||||||
|
@ -43,7 +35,7 @@ def csv_str_to_value(
|
||||||
| bool
|
| bool
|
||||||
| None
|
| None
|
||||||
):
|
):
|
||||||
assert not isinstance(s, list) # TODO?
|
assert not isinstance(s, list) # TODO?
|
||||||
|
|
||||||
if s is None:
|
if s is None:
|
||||||
return None
|
return None
|
||||||
|
@ -67,7 +59,6 @@ def csv_str_to_value(
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def load_csv_file(csv_file: Path, sniff=False) -> list[frozendict[str, typing.Any]]:
|
def load_csv_file(csv_file: Path, sniff=False) -> list[frozendict[str, typing.Any]]:
|
||||||
dicts: list[frozendict] = []
|
dicts: list[frozendict] = []
|
||||||
with open(csv_file) as csvfile:
|
with open(csv_file) as csvfile:
|
||||||
|
@ -99,11 +90,10 @@ class PossibleKeys:
|
||||||
image: list[str]
|
image: list[str]
|
||||||
misc: list[str]
|
misc: list[str]
|
||||||
|
|
||||||
|
|
||||||
def determine_possible_keys(event_data: dict[str, Any]) -> PossibleKeys:
|
def determine_possible_keys(event_data: dict[str, Any]) -> PossibleKeys:
|
||||||
# Select data
|
# Select data
|
||||||
time_keys = [
|
time_keys = [k for k, v in event_data.items() if isinstance(v, datetime.date)]
|
||||||
k for k, v in event_data.items() if isinstance(v, datetime.date)
|
|
||||||
]
|
|
||||||
duration_keys = [
|
duration_keys = [
|
||||||
k
|
k
|
||||||
for k, v in event_data.items()
|
for k, v in event_data.items()
|
||||||
|
@ -124,19 +114,26 @@ def determine_possible_keys(event_data: dict[str, Any]) -> PossibleKeys:
|
||||||
misc_keys.remove(k)
|
misc_keys.remove(k)
|
||||||
del k
|
del k
|
||||||
|
|
||||||
time_start_keys = [k for k in time_keys if 'start' in k.lower() ]
|
time_start_keys = [k for k in time_keys if 'start' in k.lower()]
|
||||||
time_end_keys = [k for k in time_keys if 'end' in k.lower() or 'stop' in k.lower() or 'last' in k.lower() ]
|
time_end_keys = [
|
||||||
|
k
|
||||||
|
for k in time_keys
|
||||||
|
if 'end' in k.lower() or 'stop' in k.lower() or 'last' in k.lower()
|
||||||
|
]
|
||||||
|
|
||||||
return PossibleKeys(
|
return PossibleKeys(
|
||||||
time_start = time_start_keys,
|
time_start=time_start_keys,
|
||||||
time_end = time_end_keys,
|
time_end=time_end_keys,
|
||||||
duration = duration_keys,
|
duration=duration_keys,
|
||||||
name = name_keys,
|
name=name_keys,
|
||||||
image = image_keys,
|
image=image_keys,
|
||||||
misc = misc_keys,
|
misc=misc_keys,
|
||||||
)
|
)
|
||||||
|
|
||||||
def start_end(sample: dict[str,Any], keys: PossibleKeys) -> tuple[datetime.datetime | None, datetime.datetime | None]:
|
|
||||||
|
def start_end(
|
||||||
|
sample: dict[str, Any], keys: PossibleKeys,
|
||||||
|
) -> tuple[datetime.datetime | None, datetime.datetime | None]:
|
||||||
if keys.time_start and keys.time_end:
|
if keys.time_start and keys.time_end:
|
||||||
return (sample[keys.time_start[0]], sample[keys.time_end[0]])
|
return (sample[keys.time_start[0]], sample[keys.time_end[0]])
|
||||||
|
|
||||||
|
|
|
@ -57,12 +57,12 @@ def parse_time(text: str) -> datetime.datetime:
|
||||||
time = try_parse(text, '%d %b %Y %I:%M:%S %p')
|
time = try_parse(text, '%d %b %Y %I:%M:%S %p')
|
||||||
time = time or try_parse(text, '%d %b, %Y @ %I:%M%p')
|
time = time or try_parse(text, '%d %b, %Y @ %I:%M%p')
|
||||||
if time is None and (m := try_parse(text, '%d %b @ %I:%M%p')):
|
if time is None and (m := try_parse(text, '%d %b @ %I:%M%p')):
|
||||||
time = m.replace(year = NOW.year)
|
time = m.replace(year=NOW.year)
|
||||||
|
|
||||||
assert time is not None, 'Could not parse format'
|
assert time is not None, 'Could not parse format'
|
||||||
|
|
||||||
if time.tzinfo is None:
|
if time.tzinfo is None:
|
||||||
time = time.replace(tzinfo=LOCAL_TIMEZONE )
|
time = time.replace(tzinfo=LOCAL_TIMEZONE)
|
||||||
|
|
||||||
assert time.tzinfo is not None, time
|
assert time.tzinfo is not None, time
|
||||||
return time
|
return time
|
||||||
|
|
|
@ -1,19 +1,17 @@
|
||||||
import _csv
|
import _csv
|
||||||
import csv
|
import csv
|
||||||
import datetime
|
import datetime
|
||||||
import decimal
|
|
||||||
import io
|
import io
|
||||||
from typing import Any
|
|
||||||
import logging
|
import logging
|
||||||
import typing
|
import typing
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from decimal import Decimal
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from frozendict import frozendict
|
from frozendict import frozendict
|
||||||
|
|
||||||
from . import data, csv_import
|
from . import csv_import, data
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -1,13 +1,21 @@
|
||||||
from personal_data.csv_import import determine_possible_keys
|
|
||||||
import frozendict
|
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
|
import frozendict
|
||||||
|
|
||||||
|
from personal_data.csv_import import determine_possible_keys
|
||||||
|
|
||||||
|
|
||||||
def test_determine_possible_keys():
|
def test_determine_possible_keys():
|
||||||
data = frozendict.frozendict({'game.name': 'Halo', 'me.last_played_time':
|
data = frozendict.frozendict(
|
||||||
datetime.datetime(2021, 6, 13, 19, 12, 21,
|
{
|
||||||
tzinfo=datetime.timezone.utc),
|
'game.name': 'Halo',
|
||||||
'trophy.name': 'Test', 'trophy.desc':
|
'me.last_played_time': datetime.datetime(
|
||||||
'Description'})
|
2021, 6, 13, 19, 12, 21, tzinfo=datetime.timezone.utc,
|
||||||
|
),
|
||||||
|
'trophy.name': 'Test',
|
||||||
|
'trophy.desc': 'Description',
|
||||||
|
},
|
||||||
|
)
|
||||||
keys = determine_possible_keys(data)
|
keys = determine_possible_keys(data)
|
||||||
|
|
||||||
assert keys.time_start == []
|
assert keys.time_start == []
|
||||||
|
|
|
@ -12,11 +12,27 @@ PARSE_MAPPINGS = [
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'2024-07-06 19:30:11+02:00',
|
'2024-07-06 19:30:11+02:00',
|
||||||
datetime.datetime(2024, 7, 6, 19, 30, 11, tzinfo=datetime.timezone(datetime.timedelta(seconds=7200))),
|
datetime.datetime(
|
||||||
|
2024,
|
||||||
|
7,
|
||||||
|
6,
|
||||||
|
19,
|
||||||
|
30,
|
||||||
|
11,
|
||||||
|
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'2023-10-21 11:43:27+02:00',
|
'2023-10-21 11:43:27+02:00',
|
||||||
datetime.datetime(2023, 10, 21, 11, 43, 27, tzinfo=datetime.timezone(datetime.timedelta(seconds=7200))),
|
datetime.datetime(
|
||||||
|
2023,
|
||||||
|
10,
|
||||||
|
21,
|
||||||
|
11,
|
||||||
|
43,
|
||||||
|
27,
|
||||||
|
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'0003791e9f5f3691b8bbbe0d12a7ae9c3f2e89db38',
|
'0003791e9f5f3691b8bbbe0d12a7ae9c3f2e89db38',
|
||||||
|
|
Loading…
Reference in New Issue
Block a user