1
0

Compare commits

..

No commits in common. "d96f76ae81d4567205cb120585a17b129e2329c1" and "36db389e3c13314278adb89696fc5c5a60d1a679" have entirely different histories.

5 changed files with 2 additions and 54 deletions

View File

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2023-2025 Jon Michael Aanes
Copyright (c) 2023-2024 Jon Michael Aanes
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -75,7 +75,7 @@ Full list of requirements:
```
MIT License
Copyright (c) 2023-2025 Jon Michael Aanes
Copyright (c) 2023-2024 Jon Michael Aanes
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -1,47 +0,0 @@
import dataclasses
import datetime
import logging
from collections.abc import Iterator, Mapping
from decimal import Decimal
from typing import Any
from personal_data.data import DeduplicateMode, Scraper
from .. import secrets
logger = logging.getLogger(__name__)
def safe_del(d: dict, *keys: str):
for key in keys:
if key in d:
del d[key]
def to_data_point(p: dict[str,Any]) ->Mapping[str, Any]:
p['owner'] = p['owner']['login']
safe_del(p, 'permissions', 'internal_tracker')
return p
@dataclasses.dataclass(frozen=True)
class Gitea(Scraper):
dataset_name = 'gitea_repos'
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
deduplicate_ignore_columns = []
@staticmethod
def requires_cfscrape() -> bool:
return False
def scrape(self) -> Iterator[Mapping[str, Any]]:
response = self.session.get('https://gitfub.space/api/v1/repos/search', params = {
#'uid':21,
'private': True,
'sort':'updated',
'order':'desc',
'access_token': secrets.gitea_access_token(),
})
response.raise_for_status()
data = response.json()
logger.info('Got %d results from Gitea', len(data['data']))
for p in data['data']:
yield to_data_point(p)

View File

@ -23,10 +23,6 @@ def pbc_account_address():
def steam_username():
return secrets.load_or_fail('STEAM_USERNAME')
# Gitea
def gitea_access_token():
return secrets.load('GITEA_ACCESS_TOKEN')
# Kucoin
def kucoin_key():

View File

@ -97,7 +97,6 @@ def dataclass_to_dict(obj) -> dict[str, Any]:
def normalize_dict(d: dict[str, Any] | frozendict[str, Any]) -> frozendict[str, Any]:
if not isinstance(d, dict) and not isinstance(d, frozendict):
d = dataclass_to_dict(d)
assert isinstance(d, dict) or isinstance(d, frozendict), 'Not a dict'
safe_values = [(k, csv_import.csv_str_to_value(csv_import.csv_safe_value(v))) for k, v in d.items() ]
return frozendict( {k:v for k,v in safe_values if v is not None})