1
0

Compare commits

..

3 Commits

Author SHA1 Message Date
d96f76ae81 Gitea
All checks were successful
Run Python tests (through Pytest) / Test (push) Successful in 35s
Verify Python project can be installed, loaded and have version checked / Test (push) Successful in 30s
2025-02-06 12:32:57 +01:00
3a1a0c0ed7 Better error message 2025-02-06 12:13:46 +01:00
8167b71442 🤖 Repository layout updated to latest version
This commit was automatically generated by a script: https://gitfub.space/Jmaa/repo-manager
2025-02-06 12:13:04 +01:00
5 changed files with 54 additions and 2 deletions

View File

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2023-2024 Jon Michael Aanes
Copyright (c) 2023-2025 Jon Michael Aanes
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -75,7 +75,7 @@ Full list of requirements:
```
MIT License
Copyright (c) 2023-2024 Jon Michael Aanes
Copyright (c) 2023-2025 Jon Michael Aanes
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -0,0 +1,47 @@
import dataclasses
import datetime
import logging
from collections.abc import Iterator, Mapping
from decimal import Decimal
from typing import Any
from personal_data.data import DeduplicateMode, Scraper
from .. import secrets
logger = logging.getLogger(__name__)
def safe_del(d: dict, *keys: str):
for key in keys:
if key in d:
del d[key]
def to_data_point(p: dict[str,Any]) ->Mapping[str, Any]:
p['owner'] = p['owner']['login']
safe_del(p, 'permissions', 'internal_tracker')
return p
@dataclasses.dataclass(frozen=True)
class Gitea(Scraper):
dataset_name = 'gitea_repos'
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
deduplicate_ignore_columns = []
@staticmethod
def requires_cfscrape() -> bool:
return False
def scrape(self) -> Iterator[Mapping[str, Any]]:
response = self.session.get('https://gitfub.space/api/v1/repos/search', params = {
#'uid':21,
'private': True,
'sort':'updated',
'order':'desc',
'access_token': secrets.gitea_access_token(),
})
response.raise_for_status()
data = response.json()
logger.info('Got %d results from Gitea', len(data['data']))
for p in data['data']:
yield to_data_point(p)

View File

@ -23,6 +23,10 @@ def pbc_account_address():
def steam_username():
return secrets.load_or_fail('STEAM_USERNAME')
# Gitea
def gitea_access_token():
return secrets.load('GITEA_ACCESS_TOKEN')
# Kucoin
def kucoin_key():

View File

@ -97,6 +97,7 @@ def dataclass_to_dict(obj) -> dict[str, Any]:
def normalize_dict(d: dict[str, Any] | frozendict[str, Any]) -> frozendict[str, Any]:
if not isinstance(d, dict) and not isinstance(d, frozendict):
d = dataclass_to_dict(d)
assert isinstance(d, dict) or isinstance(d, frozendict), 'Not a dict'
safe_values = [(k, csv_import.csv_str_to_value(csv_import.csv_safe_value(v))) for k, v in d.items() ]
return frozendict( {k:v for k,v in safe_values if v is not None})