import dataclasses import datetime import logging from collections.abc import Iterator, Mapping import requests from .. import secrets from personal_data.data import DeduplicateMode, Scraper logger = logging.getLogger(__name__) @dataclasses.dataclass(frozen=True) class WaniKaniLessonsFetcher(Scraper): dataset_name = 'wanikani_lessons' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS def scrape(self) -> Iterator[Mapping[str, object]]: """ Fetch assignments from the WaniKani API and yield a dict for each assignment with a non-null unlocked_at timestamp. """ url = 'https://api.wanikani.com/v2/assignments' headers = { 'Authorization': f'Bearer {secrets.wanikani_api_key()}', 'Wanikani-Revision': '20170710', } response = self.session.get(url, headers=headers) response.raise_for_status() data = response.json() for assignment in data.get('data', []): data = assignment['data'] print(data) yield data