Compare commits
No commits in common. "d96f76ae81d4567205cb120585a17b129e2329c1" and "36db389e3c13314278adb89696fc5c5a60d1a679" have entirely different histories.
d96f76ae81
...
36db389e3c
2
LICENSE
2
LICENSE
|
@ -1,6 +1,6 @@
|
||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2023-2025 Jon Michael Aanes
|
Copyright (c) 2023-2024 Jon Michael Aanes
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|
|
@ -75,7 +75,7 @@ Full list of requirements:
|
||||||
```
|
```
|
||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2023-2025 Jon Michael Aanes
|
Copyright (c) 2023-2024 Jon Michael Aanes
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|
|
@ -1,47 +0,0 @@
|
||||||
import dataclasses
|
|
||||||
import datetime
|
|
||||||
import logging
|
|
||||||
from collections.abc import Iterator, Mapping
|
|
||||||
from decimal import Decimal
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from personal_data.data import DeduplicateMode, Scraper
|
|
||||||
|
|
||||||
from .. import secrets
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
def safe_del(d: dict, *keys: str):
|
|
||||||
for key in keys:
|
|
||||||
if key in d:
|
|
||||||
del d[key]
|
|
||||||
|
|
||||||
def to_data_point(p: dict[str,Any]) ->Mapping[str, Any]:
|
|
||||||
p['owner'] = p['owner']['login']
|
|
||||||
safe_del(p, 'permissions', 'internal_tracker')
|
|
||||||
return p
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class Gitea(Scraper):
|
|
||||||
dataset_name = 'gitea_repos'
|
|
||||||
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
|
|
||||||
deduplicate_ignore_columns = []
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def requires_cfscrape() -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def scrape(self) -> Iterator[Mapping[str, Any]]:
|
|
||||||
response = self.session.get('https://gitfub.space/api/v1/repos/search', params = {
|
|
||||||
#'uid':21,
|
|
||||||
'private': True,
|
|
||||||
'sort':'updated',
|
|
||||||
'order':'desc',
|
|
||||||
'access_token': secrets.gitea_access_token(),
|
|
||||||
})
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
data = response.json()
|
|
||||||
logger.info('Got %d results from Gitea', len(data['data']))
|
|
||||||
for p in data['data']:
|
|
||||||
yield to_data_point(p)
|
|
|
@ -23,10 +23,6 @@ def pbc_account_address():
|
||||||
def steam_username():
|
def steam_username():
|
||||||
return secrets.load_or_fail('STEAM_USERNAME')
|
return secrets.load_or_fail('STEAM_USERNAME')
|
||||||
|
|
||||||
# Gitea
|
|
||||||
def gitea_access_token():
|
|
||||||
return secrets.load('GITEA_ACCESS_TOKEN')
|
|
||||||
|
|
||||||
|
|
||||||
# Kucoin
|
# Kucoin
|
||||||
def kucoin_key():
|
def kucoin_key():
|
||||||
|
|
|
@ -97,7 +97,6 @@ def dataclass_to_dict(obj) -> dict[str, Any]:
|
||||||
def normalize_dict(d: dict[str, Any] | frozendict[str, Any]) -> frozendict[str, Any]:
|
def normalize_dict(d: dict[str, Any] | frozendict[str, Any]) -> frozendict[str, Any]:
|
||||||
if not isinstance(d, dict) and not isinstance(d, frozendict):
|
if not isinstance(d, dict) and not isinstance(d, frozendict):
|
||||||
d = dataclass_to_dict(d)
|
d = dataclass_to_dict(d)
|
||||||
assert isinstance(d, dict) or isinstance(d, frozendict), 'Not a dict'
|
|
||||||
safe_values = [(k, csv_import.csv_str_to_value(csv_import.csv_safe_value(v))) for k, v in d.items() ]
|
safe_values = [(k, csv_import.csv_str_to_value(csv_import.csv_safe_value(v))) for k, v in d.items() ]
|
||||||
return frozendict( {k:v for k,v in safe_values if v is not None})
|
return frozendict( {k:v for k,v in safe_values if v is not None})
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user