Gitea

This commit was automatically generated by a script: https://gitfub.space/Jmaa/repo-manager

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023-2024 Jon Michael Aanes
+Copyright (c) 2023-2025 Jon Michael Aanes
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -75,7 +75,7 @@ Full list of requirements:
 ```
 MIT License
 
-Copyright (c) 2023-2024 Jon Michael Aanes
+Copyright (c) 2023-2025 Jon Michael Aanes
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

personal_data/fetchers/gitea.py

@@ -0,0 +1,47 @@
+import dataclasses
+import datetime
+import logging
+from collections.abc import Iterator, Mapping
+from decimal import Decimal
+from typing import Any
+
+from personal_data.data import DeduplicateMode, Scraper
+
+from .. import secrets
+
+logger = logging.getLogger(__name__)
+
+def safe_del(d: dict, *keys: str):
+    for key in keys:
+        if key in d:
+            del d[key]
+
+def to_data_point(p: dict[str,Any]) ->Mapping[str, Any]:
+    p['owner'] = p['owner']['login']
+    safe_del(p, 'permissions', 'internal_tracker')
+    return p
+
+@dataclasses.dataclass(frozen=True)
+class Gitea(Scraper):
+    dataset_name = 'gitea_repos'
+    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
+    deduplicate_ignore_columns = []
+
+    @staticmethod
+    def requires_cfscrape() -> bool:
+        return False
+
+    def scrape(self) -> Iterator[Mapping[str, Any]]:
+        response = self.session.get('https://gitfub.space/api/v1/repos/search', params = {
+            #'uid':21,
+            'private': True,
+            'sort':'updated',
+            'order':'desc',
+            'access_token': secrets.gitea_access_token(),
+        })
+        response.raise_for_status()
+
+        data = response.json()
+        logger.info('Got %d results from Gitea', len(data['data']))
+        for p in data['data']:
+            yield to_data_point(p)

personal_data/secrets.py

@@ -23,6 +23,10 @@ def pbc_account_address():
 def steam_username():
     return secrets.load_or_fail('STEAM_USERNAME')
 
+# Gitea
+def gitea_access_token():
+    return secrets.load('GITEA_ACCESS_TOKEN')
+
 
 # Kucoin
 def kucoin_key():

personal_data/util.py

@@ -97,6 +97,7 @@ def dataclass_to_dict(obj) -> dict[str, Any]:
 def normalize_dict(d: dict[str, Any] | frozendict[str, Any]) -> frozendict[str, Any]:
     if not isinstance(d, dict) and not isinstance(d, frozendict):
         d = dataclass_to_dict(d)
+    assert isinstance(d, dict) or isinstance(d, frozendict), 'Not a dict'
     safe_values = [(k, csv_import.csv_str_to_value(csv_import.csv_safe_value(v))) for k, v in d.items() ]
     return frozendict( {k:v for k,v in safe_values if v is not None})