1
0

Compare commits

..

5 Commits

Author SHA1 Message Date
1f989184d2 Use browser_cookie3 instead v2
Some checks failed
Run Python tests (through Pytest) / Test (push) Failing after 32s
Verify Python project can be installed, loaded and have version checked / Test (push) Failing after 25s
2025-05-14 23:55:51 +02:00
0bb93be1be 🤖 Repository layout updated to latest version
This commit was automatically generated by [a script](https://gitfub.space/Jmaa/repo-manager)
2025-05-14 23:54:31 +02:00
2e3d5da34c Use browser_cookie3 instead 2025-05-14 23:51:23 +02:00
35ae8fa2a9 Ruff 2025-05-14 23:50:13 +02:00
51baddc8f2 🤖 Repository layout updated to latest version
This commit was automatically generated by [a script](https://gitfub.space/Jmaa/repo-manager)
2025-05-09 09:50:13 +02:00
11 changed files with 71 additions and 40 deletions

View File

@ -1,3 +1,7 @@
# WARNING!
# THIS IS AN AUTOGENERATED FILE!
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN!
name: Build Python Container
on:
push:

View File

@ -1,3 +1,7 @@
# WARNING!
# THIS IS AN AUTOGENERATED FILE!
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN!
name: Package Python
on:
push:

View File

@ -1,3 +1,7 @@
# WARNING!
# THIS IS AN AUTOGENERATED FILE!
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN!
name: Run Python tests (through Pytest)
on:

View File

@ -1,3 +1,7 @@
# WARNING!
# THIS IS AN AUTOGENERATED FILE!
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN!
name: Verify Python project can be installed, loaded and have version checked
on:

View File

@ -1,3 +1,14 @@
<!-- WARNING! -->
<!-- THIS IS AN AUTOGENERATED FILE! -->
<!-- MANUAL CHANGES CAN AND WILL BE OVERWRITTEN! -->
# Conventions
When contributing code to this project, you MUST follow the requirements
specified here.
## Code Conventions
When contributing code to this project, you MUST follow these principles:
- Code should be easy to read and understand.
@ -10,3 +21,16 @@ When contributing code to this project, you MUST follow these principles:
- Properly handle errors and exceptions to ensure the software's robustness.
- Use exceptions rather than error codes for handling errors.
- Consider security implications of the code. Implement security best practices to protect against vulnerabilities and attacks.
- Documentation should document semantics, not syntax.
- Prefer importing modules, not individual items from modules.
- Do not use f-strings in logging statements.
- Loop variables and walrus-expression-variables should be deleted when
unneeded to keep scope clean, and to avoid accidental use.
## Testing
When contributing test to this project, you MUST follow these principles:
- Do not use any testing libraries other than `pytest`.
- Mocking is the root of all evil. Writing your own stubs is much more
preferable.

View File

@ -1,8 +1,6 @@
<!--- WARNING --->
<!--- THIS IS AN AUTO-GENERATED FILE --->
<!--- MANUAL CHANGES CAN AND WILL BE OVERWRITTEN --->
<!-- WARNING! -->
<!-- THIS IS AN AUTOGENERATED FILE! -->
<!-- MANUAL CHANGES CAN AND WILL BE OVERWRITTEN! -->
# Personal Data Fetcher Systems
@ -63,7 +61,7 @@ Full list of requirements:
- [lxml](https://pypi.org/project/lxml/)
- [requests](https://pypi.org/project/requests/)
- [requests_cache](https://pypi.org/project/requests_cache/)
- [browsercookie](https://pypi.org/project/browsercookie/)
- [browser_cookie3](https://pypi.org/project/browser_cookie3/)
- [cfscrape](https://github.com/landryraccoon/cloudflare-scrape)
- [frozendict](https://pypi.org/project/frozendict/)
- [python-kucoin](https://pypi.org/project/python-kucoin/)
@ -73,13 +71,12 @@ Full list of requirements:
- [fin-depo](https://gitfub.space/Jmaa/fin-depo)
- [secret_loader](https://gitfub.space/Jmaa/secret_loader)
- [requests-util](https://gitfub.space/Jmaa/requests_util)
- [clients](https://gitfub.space/Jmaa/clients)
## Contributing
Feel free to submit pull requests. Please follow the [Code Conventions](CONVENTIONS.md) when doing so.
### Testing
Testing requires the [pytest](https://docs.pytest.org/en/stable/) library.
@ -96,7 +93,6 @@ Test coverage can be run using the [`pytest-cov`](https://pypi.org/project/pytes
pytest --cov=personal_data test
```
## License
```

View File

@ -1,22 +1,21 @@
import dataclasses
import json
import logging
import re
import urllib.parse
from collections.abc import Iterator
import bs4
from clients.myanimelist import MyAnimeListClient, MyAnimeListAnime, MyAnimeListSong
from clients import init_client
from clients.myanimelist import MyAnimeListAnime, MyAnimeListClient, MyAnimeListSong
from personal_data.data import DeduplicateMode, Scraper
from .. import secrets
logger = logging.getLogger(__name__)
def client(session):
return init_client(MyAnimeListClient, session, secrets.secrets, throws=True)
@dataclasses.dataclass(frozen=True)
class MyAnimeList(Scraper):
dataset_name = 'myanimelist_anime'

View File

@ -18,10 +18,10 @@ except ImportError:
logger.exception('cfscrape not installed: Certain fetchers might not work')
try:
import browsercookie
import browser_cookie3
except ImportError:
logger.exception('browsercookie not installed: Certain fetchers might not work')
browsercookie = None
logger.exception('browser_cookie3 not installed: Certain fetchers might not work')
browser_cookie3 = None
OUTPUT_PATH = Path('./output')
@ -94,11 +94,11 @@ def available_scraper_names() -> list[str]:
def get_cookiejar(use_cookiejar: bool):
if use_cookiejar:
logger.warning('Got cookiejar from firefox')
cookiejar = browsercookie.firefox()
cookiejar = browser_cookie3.firefox()
if len(cookiejar) > 10:
return cookiejar
browsercookie.firefox(
['/home/jmaa/.cachy/mbui5xg7.default-release/cookies.sqlite'],
browser_cookie3.firefox(
'/home/jmaa/.cachy/mbui5xg7.default-release/cookies.sqlite'
)
if len(cookiejar) > 10:
return cookiejar

View File

@ -2,6 +2,7 @@ from secret_loader import SecretLoader
secrets = SecretLoader(env_key_prefix='CF_PD')
def load_lazy(key: str):
return lambda: secrets.load_or_fail(key)

View File

@ -2,7 +2,7 @@ beautifulsoup4
lxml
requests
requests_cache
browsercookie
browser_cookie3
cfscrape @ git+https://github.com/landryraccoon/cloudflare-scrape.git
frozendict
python-kucoin

View File

@ -1,8 +1,6 @@
# WARNING
#
# THIS IS AN AUTOGENERATED FILE.
#
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN.
# WARNING!
# THIS IS AN AUTOGENERATED FILE!
# MANUAL CHANGES CAN AND WILL BE OVERWRITTEN!
import re
@ -56,7 +54,6 @@ PACKAGE_DESCRIPTION_SHORT = """
This program collects several small data fetchers, for downloading personal
data spread around the internet.""".strip()
def parse_version_file(text: str) -> str:
match = re.match(r'^__version__\s*=\s*(["\'])([\d\.]+)\1$', text)
if match is None:
@ -64,17 +61,15 @@ def parse_version_file(text: str) -> str:
raise Exception(msg)
return match.group(2)
with open(PACKAGE_NAME + '/_version.py') as f:
version = parse_version_file(f.read())
REQUIREMENTS_MAIN = [
'beautifulsoup4',
'lxml',
'requests',
'requests_cache',
'browsercookie',
'browser_cookie3',
'cfscrape @ git+https://github.com/landryraccoon/cloudflare-scrape.git',
'frozendict',
'python-kucoin',
@ -84,12 +79,12 @@ REQUIREMENTS_MAIN = [
'fin-depo @ git+https://gitfub.space/Jmaa/fin-depo.git',
'secret_loader @ git+https://gitfub.space/Jmaa/secret_loader',
'requests-util @ git+https://gitfub.space/Jmaa/requests_util',
'clients @ git+https://gitfub.space/Jmaa/clients',
]
REQUIREMENTS_TEST = [
'pytest',
]
setup(
name=PACKAGE_NAME,
version=version,