From 5c2c843178d9a8da1fc8e0aec50e3f90fb70db20 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Mon, 24 Mar 2025 21:09:56 +0100 Subject: [PATCH] Fixed web-server --- html_data_format/__main__.py | 72 ++++++++++------------ html_data_format/webserver.py | 54 ---------------- personal_data/fetchers/wanikani_lessons.py | 4 +- 3 files changed, 34 insertions(+), 96 deletions(-) delete mode 100644 html_data_format/webserver.py diff --git a/html_data_format/__main__.py b/html_data_format/__main__.py index 1e94864..5fbd995 100644 --- a/html_data_format/__main__.py +++ b/html_data_format/__main__.py @@ -1,58 +1,48 @@ -import argparse +import datetime +import json import logging from pathlib import Path -from typing import Any +import bottle -from personal_data.csv_import import load_csv_file +from personal_data import csv_import logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ROOT_DIRECTORY = Path('output') -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'file', - type=Path, - ) - parser.add_argument( - 'output_file', - type=Path, - ) +@bottle.route('//newest') +def newest_entry(csv_type: str): + """ + Loads a CSV file (default: data.csv, overridable by query param 'file'), + finds the newest entry based on the 'time.current' column, and returns it as JSON. + """ - return parser.parse_args() + path = ROOT_DIRECTORY/f'{csv_type}.csv' + bottle.response.content_type = 'application/json' -def format_data_to_html(data: list[dict[str, Any]], mode: str = 'latest') -> str: - sample = data[-1] + try: + data = csv_import.load_csv_file(path) + except Exception: + logger.exception('Error loading CSV file at %s', path) + bottle.response.status = 500 + return {'error': 'Failed to load CSV'} - l = [] - l.append('') - l.append('') + if not data: + bottle.response.status = 404 + return {'error': 'CSV file is empty or no data found'} - for k, v in sample.items(): - l.append(f'
{k}: {v}
') + TIME_COLUMN = 'time.current' - l.append('') - l.append('') + if TIME_COLUMN in data[0]: + newest = max(data, key=lambda r: r.get(TIME_COLUMN)) + else: + newest = data[-1] - return ''.join(l) - - -def main(): - logging.basicConfig() - logging.getLogger('git_time_tracker').setLevel('INFO') - - args = parse_arguments() - - # Load data - data = load_csv_file(args.file) - print(data) - - html = format_data_to_html(data) - - with open(args.output_file, 'w') as f: - f.write(html) + return {csv_import.csv_safe_value(k):csv_import.csv_safe_value(v) for k,v in newest.items()} if __name__ == '__main__': - main() + bottle.run(host='localhost', port=8080, debug=True) diff --git a/html_data_format/webserver.py b/html_data_format/webserver.py deleted file mode 100644 index 30100ef..0000000 --- a/html_data_format/webserver.py +++ /dev/null @@ -1,54 +0,0 @@ -import datetime -import json -import logging -from pathlib import Path - -from bottle import request, response, route, run - -from personal_data.csv_import import load_csv_file - -logger = logging.getLogger('webserver') -logger.setLevel(logging.INFO) - - -def parse_time(time_str: str) -> datetime.datetime: - """ - Parse a time string assuming ISO-8601 format. - Adjust this parser if your CSV timestamps differ. - """ - return datetime.datetime.fromisoformat(time_str) - - -@route('/newest') -def newest_entry(): - """ - Loads a CSV file (default: data.csv, overridable by query param 'file'), - finds the newest entry based on the 'time.current' column, and returns it as JSON. - """ - file_param = request.query.get('file', 'data.csv') - csv_path = Path(file_param) - - try: - data = load_csv_file(csv_path) - except Exception as e: - logger.error(f'Error loading CSV file at {csv_path}: {e}') - response.status = 500 - return {'error': f'Failed to load CSV: {str(e)}'} - - if not data: - response.status = 404 - return {'error': 'CSV file is empty or no data found'} - - try: - newest = max(data, key=lambda r: parse_time(r['time.current'])) - except Exception as e: - logger.error(f'Error processing CSV data: {e}') - response.status = 500 - return {'error': f'Failed to process CSV data: {str(e)}'} - - response.content_type = 'application/json' - return json.dumps(newest) - - -if __name__ == '__main__': - run(host='localhost', port=8080, debug=True) diff --git a/personal_data/fetchers/wanikani_lessons.py b/personal_data/fetchers/wanikani_lessons.py index 20ceff8..30afdcc 100644 --- a/personal_data/fetchers/wanikani_lessons.py +++ b/personal_data/fetchers/wanikani_lessons.py @@ -1,10 +1,10 @@ import dataclasses import datetime -import requests import logging from collections.abc import Iterator, Mapping from email.utils import parsedate_to_datetime +import requests import requests_util from personal_data.data import DeduplicateMode, Scraper @@ -66,11 +66,13 @@ class WaniKaniLessonsFetcher(Scraper): yield data_item url = json_resp.get('pages', {}).get('next_url') + def date_from_response(response) -> datetime.datetime: if date_header := response.headers.get('Date'): return parsedate_to_datetime(date_header) return datetime.datetime.now(datetime.timezone.utc) + @dataclasses.dataclass(frozen=True) class WaniKaniSummaryFetcher(Scraper): dataset_name: str = 'wanikani_summary'