Dynamic loading of fetchers

personal_data/fetchers/__init__.py

@@ -0,0 +1,31 @@
+import importlib
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def load_backend(name: str) -> object | None:
+    try:
+        return importlib.import_module(__name__ + '.' + name)
+    except Exception:
+        logger.exception('Backend %s failed loading, and have been disabled', name)
+        return None
+
+
+def get_modules(backend_dir: Path) -> Iterator[str]:
+    for f in backend_dir.iterdir():
+        if f.is_file() and f.suffix == '.py':
+            name = f.parts[-1].removesuffix('.py')
+            if name != '__init__':
+                yield name
+
+
+def load_fetcher_modules() -> None:
+    backend_dir = Path(__name__.replace(r'.', '/'))
+    for module in get_modules(backend_dir):
+        load_backend(module)
+
+
+load_fetcher_modules()

personal_data/fetchers/kucoin.py

@@ -1,20 +1,16 @@
 import dataclasses
 import datetime
-import email.utils
-import json
-from frozendict import frozendict
 import logging
 from collections.abc import Iterator, Mapping
 from decimal import Decimal
 
+import kucoin.client
 from frozendict import frozendict
 
 from personal_data.data import DeduplicateMode, Scraper
 
 from .. import secrets
 
-import kucoin.client
-
 logger = logging.getLogger(__name__)
 
 
@@ -42,6 +38,8 @@ class KucoinAccountBalances(Scraper):
 
         for account in client.get_accounts():
             key = f"balance.{account['currency']}"
-            data_point[key] = data_point.get(key, Decimal(0)) + Decimal(account['balance'])
+            data_point[key] = data_point.get(key, Decimal(0)) + Decimal(
+                account['balance'],
+            )
 
         yield frozendict(data_point)

personal_data/main.py

@@ -25,11 +25,7 @@ except ImportError:
 
 
 import personal_data.data
-import personal_data.fetchers.crunchyroll
-import personal_data.fetchers.ffxiv_lodestone
-import personal_data.fetchers.partisia_blockchain
-import personal_data.fetchers.psnprofiles
-import personal_data.fetchers.kucoin
+import personal_data.fetchers
 
 from . import mailgun