personal-data/personal_data/fetchers/__init__.py

import importlib
import logging
from collections.abc import Iterator
from pathlib import Path

logger = logging.getLogger(__name__)


def load_backend(name: str) -> object | None:
    try:
        return importlib.import_module(__name__ + '.' + name)
    except Exception:
        logger.exception('Backend %s failed loading, and have been disabled', name)
        return None


def get_modules(backend_dir: Path) -> Iterator[str]:
    for f in backend_dir.iterdir():
        if f.is_file() and f.suffix == '.py':
            name = f.parts[-1].removesuffix('.py')
            if name != '__init__':
                yield name

FETCHER_MODULES_LOADED = False


def load_fetcher_modules() -> None:
    global FETCHER_MODULES_LOADED
    if not FETCHER_MODULES_LOADED:
        backend_dir = Path(__name__.replace(r'.', '/'))
        for module in get_modules(backend_dir):
            load_backend(module)
    FETCHER_MODULES_LOADED = True


load_fetcher_modules()