Dynamic loading of fetchers
This commit is contained in:
parent
feaa6a4a69
commit
8a9f8bf5e8
|
@ -0,0 +1,31 @@
|
||||||
|
import importlib
|
||||||
|
import logging
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def load_backend(name: str) -> object | None:
|
||||||
|
try:
|
||||||
|
return importlib.import_module(__name__ + '.' + name)
|
||||||
|
except Exception:
|
||||||
|
logger.exception('Backend %s failed loading, and have been disabled', name)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_modules(backend_dir: Path) -> Iterator[str]:
|
||||||
|
for f in backend_dir.iterdir():
|
||||||
|
if f.is_file() and f.suffix == '.py':
|
||||||
|
name = f.parts[-1].removesuffix('.py')
|
||||||
|
if name != '__init__':
|
||||||
|
yield name
|
||||||
|
|
||||||
|
|
||||||
|
def load_fetcher_modules() -> None:
|
||||||
|
backend_dir = Path(__name__.replace(r'.', '/'))
|
||||||
|
for module in get_modules(backend_dir):
|
||||||
|
load_backend(module)
|
||||||
|
|
||||||
|
|
||||||
|
load_fetcher_modules()
|
|
@ -1,20 +1,16 @@
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import datetime
|
import datetime
|
||||||
import email.utils
|
|
||||||
import json
|
|
||||||
from frozendict import frozendict
|
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Iterator, Mapping
|
from collections.abc import Iterator, Mapping
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
|
import kucoin.client
|
||||||
from frozendict import frozendict
|
from frozendict import frozendict
|
||||||
|
|
||||||
from personal_data.data import DeduplicateMode, Scraper
|
from personal_data.data import DeduplicateMode, Scraper
|
||||||
|
|
||||||
from .. import secrets
|
from .. import secrets
|
||||||
|
|
||||||
import kucoin.client
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@ -42,6 +38,8 @@ class KucoinAccountBalances(Scraper):
|
||||||
|
|
||||||
for account in client.get_accounts():
|
for account in client.get_accounts():
|
||||||
key = f"balance.{account['currency']}"
|
key = f"balance.{account['currency']}"
|
||||||
data_point[key] = data_point.get(key, Decimal(0)) + Decimal(account['balance'])
|
data_point[key] = data_point.get(key, Decimal(0)) + Decimal(
|
||||||
|
account['balance'],
|
||||||
|
)
|
||||||
|
|
||||||
yield frozendict(data_point)
|
yield frozendict(data_point)
|
||||||
|
|
|
@ -25,11 +25,7 @@ except ImportError:
|
||||||
|
|
||||||
|
|
||||||
import personal_data.data
|
import personal_data.data
|
||||||
import personal_data.fetchers.crunchyroll
|
import personal_data.fetchers
|
||||||
import personal_data.fetchers.ffxiv_lodestone
|
|
||||||
import personal_data.fetchers.partisia_blockchain
|
|
||||||
import personal_data.fetchers.psnprofiles
|
|
||||||
import personal_data.fetchers.kucoin
|
|
||||||
|
|
||||||
from . import mailgun
|
from . import mailgun
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user