Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

Fixed web-server

Browse Source
This commit is contained in:
Jon Michael Aanes 2025-03-24 21:09:56 +01:00
parent 0763e07039
commit 5c2c843178
3 changed files with 34 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
html_data_format/__main__.py
View File

@ -1,58 +1,48 @@
import argparse import datetime
import json
import logging import logging
from pathlib import Path from pathlib import Path
from typing import Any import bottle
from personal_data.csv_import import load_csv_file from personal_data import csv_import
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
ROOT_DIRECTORY = Path('output')
def parse_arguments(): @bottle.route('/<csv_type>/newest')
parser = argparse.ArgumentParser() def newest_entry(csv_type: str):
parser.add_argument( """
'file', Loads a CSV file (default: data.csv, overridable by query param 'file'),
type=Path, finds the newest entry based on the 'time.current' column, and returns it as JSON.
) """
parser.add_argument(
'output_file',
type=Path,
)
return parser.parse_args() path = ROOT_DIRECTORY/f'{csv_type}.csv'
bottle.response.content_type = 'application/json'
def format_data_to_html(data: list[dict[str, Any]], mode: str = 'latest') -> str: try:
sample = data[-1] data = csv_import.load_csv_file(path)
except Exception:
logger.exception('Error loading CSV file at %s', path)
bottle.response.status = 500
return {'error': 'Failed to load CSV'}
l = [] if not data:
l.append('<html>') bottle.response.status = 404
l.append('<body>') return {'error': 'CSV file is empty or no data found'}
for k, v in sample.items(): TIME_COLUMN = 'time.current'
l.append(f'<div><b>{k}</b>: {v}</div>')
l.append('</body>') if TIME_COLUMN in data[0]:
l.append('</html>') newest = max(data, key=lambda r: r.get(TIME_COLUMN))
else:
newest = data[-1]
return ''.join(l) return {csv_import.csv_safe_value(k):csv_import.csv_safe_value(v) for k,v in newest.items()}
def main():
logging.basicConfig()
logging.getLogger('git_time_tracker').setLevel('INFO')
args = parse_arguments()
# Load data
data = load_csv_file(args.file)
print(data)
html = format_data_to_html(data)
with open(args.output_file, 'w') as f:
f.write(html)
if __name__ == '__main__': if __name__ == '__main__':
main() bottle.run(host='localhost', port=8080, debug=True)

54
html_data_format/webserver.py
View File

@ -1,54 +0,0 @@
import datetime
import json
import logging
from pathlib import Path
from bottle import request, response, route, run
from personal_data.csv_import import load_csv_file
logger = logging.getLogger('webserver')
logger.setLevel(logging.INFO)
def parse_time(time_str: str) -> datetime.datetime:
"""
Parse a time string assuming ISO-8601 format.
Adjust this parser if your CSV timestamps differ.
"""
return datetime.datetime.fromisoformat(time_str)
@route('/newest')
def newest_entry():
"""
Loads a CSV file (default: data.csv, overridable by query param 'file'),
finds the newest entry based on the 'time.current' column, and returns it as JSON.
"""
file_param = request.query.get('file', 'data.csv')
csv_path = Path(file_param)
try:
data = load_csv_file(csv_path)
except Exception as e:
logger.error(f'Error loading CSV file at {csv_path}: {e}')
response.status = 500
return {'error': f'Failed to load CSV: {str(e)}'}
if not data:
response.status = 404
return {'error': 'CSV file is empty or no data found'}
try:
newest = max(data, key=lambda r: parse_time(r['time.current']))
except Exception as e:
logger.error(f'Error processing CSV data: {e}')
response.status = 500
return {'error': f'Failed to process CSV data: {str(e)}'}
response.content_type = 'application/json'
return json.dumps(newest)
if __name__ == '__main__':
run(host='localhost', port=8080, debug=True)

4
personal_data/fetchers/wanikani_lessons.py
View File

@ -1,10 +1,10 @@
import dataclasses import dataclasses
import datetime import datetime
import requests
import logging import logging
from collections.abc import Iterator, Mapping from collections.abc import Iterator, Mapping
from email.utils import parsedate_to_datetime from email.utils import parsedate_to_datetime
import requests
import requests_util import requests_util
from personal_data.data import DeduplicateMode, Scraper from personal_data.data import DeduplicateMode, Scraper
@ -66,11 +66,13 @@ class WaniKaniLessonsFetcher(Scraper):
yield data_item yield data_item
url = json_resp.get('pages', {}).get('next_url') url = json_resp.get('pages', {}).get('next_url')
def date_from_response(response) -> datetime.datetime: def date_from_response(response) -> datetime.datetime:
if date_header := response.headers.get('Date'): if date_header := response.headers.get('Date'):
return parsedate_to_datetime(date_header) return parsedate_to_datetime(date_header)
return datetime.datetime.now(datetime.timezone.utc) return datetime.datetime.now(datetime.timezone.utc)
@dataclasses.dataclass(frozen=True) @dataclasses.dataclass(frozen=True)
class WaniKaniSummaryFetcher(Scraper): class WaniKaniSummaryFetcher(Scraper):
dataset_name: str = 'wanikani_summary' dataset_name: str = 'wanikani_summary'