import dataclasses import json import logging import re import urllib.parse from collections.abc import Iterator import bs4 from clients.myanimelist import MyAnimeListClient, MyAnimeListAnime, MyAnimeListSong from clients import init_client from personal_data.data import DeduplicateMode, Scraper from .. import secrets logger = logging.getLogger(__name__) def client(session): return init_client(MyAnimeListClient, session, secrets.secrets, throws=True) @dataclasses.dataclass(frozen=True) class MyAnimeList(Scraper): dataset_name = 'myanimelist_anime' deduplicate_mode = DeduplicateMode.BY_FIRST_COLUMN def scrape(self) -> Iterator[MyAnimeListAnime]: yield from client(self.session).get_my_anime_list() @dataclasses.dataclass(frozen=True) class MyAnimeListSongs(Scraper): dataset_name = 'myanimelist_songs' deduplicate_mode = DeduplicateMode.BY_FIRST_COLUMN def scrape(self) -> Iterator[MyAnimeListSong]: my_client = client(self.session) for anime in my_client.get_my_anime_list(): yield from my_client.get_songs_for_anime(anime)