Compare commits
7 Commits
e787779b58
...
1590586c32
Author | SHA1 | Date | |
---|---|---|---|
1590586c32 | |||
18a8f8dad8 | |||
c5e2a3471d | |||
ed364ee9a7 | |||
f22d224bc9 | |||
4d6bceb418 | |||
7c087d7c61 |
|
@ -308,7 +308,8 @@ def import_data(obsidian_path: Path, dry_run=True):
|
|||
)
|
||||
continue
|
||||
rows = load_csv_file(
|
||||
import_def['path'], sniff=not import_def.get('standard_variant'),
|
||||
import_def['path'],
|
||||
sniff=not import_def.get('standard_variant'),
|
||||
)
|
||||
logger.info('Loaded CSV with %d lines', len(rows))
|
||||
num_files_updated = import_def['import_rows'](vault, rows)
|
||||
|
|
31
personal_data/fetchers/wanikani_lessons.py
Normal file
31
personal_data/fetchers/wanikani_lessons.py
Normal file
|
@ -0,0 +1,31 @@
|
|||
import dataclasses
|
||||
import logging
|
||||
from collections.abc import Iterator, Mapping
|
||||
|
||||
from personal_data.data import DeduplicateMode, Scraper
|
||||
|
||||
from .. import secrets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class WaniKaniLessonsFetcher(Scraper):
|
||||
dataset_name = 'wanikani_lessons'
|
||||
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
||||
|
||||
def scrape(self) -> Iterator[Mapping[str, object]]:
|
||||
"""Fetch assignments from the WaniKani API and yield a dict for each assignment with a non-null unlocked_at timestamp."""
|
||||
url = 'https://api.wanikani.com/v2/assignments'
|
||||
headers = {
|
||||
'Authorization': f'Bearer {secrets.wanikani_api_key()}',
|
||||
'Wanikani-Revision': '20170710',
|
||||
}
|
||||
while url:
|
||||
response = self.session.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
json_resp = response.json()
|
||||
for assignment in json_resp.get('data', []):
|
||||
data_item = assignment['data']
|
||||
yield data_item
|
||||
url = json_resp.get('pages', {}).get('next_url')
|
|
@ -97,7 +97,9 @@ def get_cookiejar(use_cookiejar: bool):
|
|||
cookiejar = browsercookie.firefox()
|
||||
if len(cookiejar) > 10:
|
||||
return cookiejar
|
||||
browsercookie.firefox(['/home/jmaa/.cachy/mbui5xg7.default-release/cookies.sqlite'])
|
||||
browsercookie.firefox(
|
||||
['/home/jmaa/.cachy/mbui5xg7.default-release/cookies.sqlite'],
|
||||
)
|
||||
if len(cookiejar) > 10:
|
||||
return cookiejar
|
||||
logger.warning('No cookiejar is used')
|
||||
|
|
|
@ -71,3 +71,8 @@ JELLYFIN_PASSWORD = secrets.load('JELLYFIN_PASSWORD')
|
|||
WITHINGS_CLIENTID = secrets.load('WITHINGS_CLIENTID')
|
||||
WITHINGS_SECRET = secrets.load('WITHINGS_SECRET')
|
||||
WITHINGS_CALLBACK_URI = secrets.load('WITHINGS_CALLBACK_URI')
|
||||
|
||||
|
||||
# Other
|
||||
def wanikani_api_key():
|
||||
return secrets.load_or_fail('WANIKANI_API_KEY')
|
||||
|
|
Loading…
Reference in New Issue
Block a user