diff --git a/html_data_format/__main__.py b/html_data_format/__main__.py
index 1e94864..5fbd995 100644
--- a/html_data_format/__main__.py
+++ b/html_data_format/__main__.py
@@ -1,58 +1,48 @@
-import argparse
+import datetime
+import json
import logging
from pathlib import Path
-from typing import Any
+import bottle
-from personal_data.csv_import import load_csv_file
+from personal_data import csv_import
logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ROOT_DIRECTORY = Path('output')
-def parse_arguments():
- parser = argparse.ArgumentParser()
- parser.add_argument(
- 'file',
- type=Path,
- )
- parser.add_argument(
- 'output_file',
- type=Path,
- )
+@bottle.route('//newest')
+def newest_entry(csv_type: str):
+ """
+ Loads a CSV file (default: data.csv, overridable by query param 'file'),
+ finds the newest entry based on the 'time.current' column, and returns it as JSON.
+ """
- return parser.parse_args()
+ path = ROOT_DIRECTORY/f'{csv_type}.csv'
+ bottle.response.content_type = 'application/json'
-def format_data_to_html(data: list[dict[str, Any]], mode: str = 'latest') -> str:
- sample = data[-1]
+ try:
+ data = csv_import.load_csv_file(path)
+ except Exception:
+ logger.exception('Error loading CSV file at %s', path)
+ bottle.response.status = 500
+ return {'error': 'Failed to load CSV'}
- l = []
- l.append('')
- l.append('')
+ if not data:
+ bottle.response.status = 404
+ return {'error': 'CSV file is empty or no data found'}
- for k, v in sample.items():
- l.append(f'{k}: {v}
')
+ TIME_COLUMN = 'time.current'
- l.append('')
- l.append('')
+ if TIME_COLUMN in data[0]:
+ newest = max(data, key=lambda r: r.get(TIME_COLUMN))
+ else:
+ newest = data[-1]
- return ''.join(l)
-
-
-def main():
- logging.basicConfig()
- logging.getLogger('git_time_tracker').setLevel('INFO')
-
- args = parse_arguments()
-
- # Load data
- data = load_csv_file(args.file)
- print(data)
-
- html = format_data_to_html(data)
-
- with open(args.output_file, 'w') as f:
- f.write(html)
+ return {csv_import.csv_safe_value(k):csv_import.csv_safe_value(v) for k,v in newest.items()}
if __name__ == '__main__':
- main()
+ bottle.run(host='localhost', port=8080, debug=True)
diff --git a/html_data_format/webserver.py b/html_data_format/webserver.py
deleted file mode 100644
index 30100ef..0000000
--- a/html_data_format/webserver.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import datetime
-import json
-import logging
-from pathlib import Path
-
-from bottle import request, response, route, run
-
-from personal_data.csv_import import load_csv_file
-
-logger = logging.getLogger('webserver')
-logger.setLevel(logging.INFO)
-
-
-def parse_time(time_str: str) -> datetime.datetime:
- """
- Parse a time string assuming ISO-8601 format.
- Adjust this parser if your CSV timestamps differ.
- """
- return datetime.datetime.fromisoformat(time_str)
-
-
-@route('/newest')
-def newest_entry():
- """
- Loads a CSV file (default: data.csv, overridable by query param 'file'),
- finds the newest entry based on the 'time.current' column, and returns it as JSON.
- """
- file_param = request.query.get('file', 'data.csv')
- csv_path = Path(file_param)
-
- try:
- data = load_csv_file(csv_path)
- except Exception as e:
- logger.error(f'Error loading CSV file at {csv_path}: {e}')
- response.status = 500
- return {'error': f'Failed to load CSV: {str(e)}'}
-
- if not data:
- response.status = 404
- return {'error': 'CSV file is empty or no data found'}
-
- try:
- newest = max(data, key=lambda r: parse_time(r['time.current']))
- except Exception as e:
- logger.error(f'Error processing CSV data: {e}')
- response.status = 500
- return {'error': f'Failed to process CSV data: {str(e)}'}
-
- response.content_type = 'application/json'
- return json.dumps(newest)
-
-
-if __name__ == '__main__':
- run(host='localhost', port=8080, debug=True)
diff --git a/personal_data/fetchers/wanikani_lessons.py b/personal_data/fetchers/wanikani_lessons.py
index 20ceff8..30afdcc 100644
--- a/personal_data/fetchers/wanikani_lessons.py
+++ b/personal_data/fetchers/wanikani_lessons.py
@@ -1,10 +1,10 @@
import dataclasses
import datetime
-import requests
import logging
from collections.abc import Iterator, Mapping
from email.utils import parsedate_to_datetime
+import requests
import requests_util
from personal_data.data import DeduplicateMode, Scraper
@@ -66,11 +66,13 @@ class WaniKaniLessonsFetcher(Scraper):
yield data_item
url = json_resp.get('pages', {}).get('next_url')
+
def date_from_response(response) -> datetime.datetime:
if date_header := response.headers.get('Date'):
return parsedate_to_datetime(date_header)
return datetime.datetime.now(datetime.timezone.utc)
+
@dataclasses.dataclass(frozen=True)
class WaniKaniSummaryFetcher(Scraper):
dataset_name: str = 'wanikani_summary'