1
0
personal-data/personal_data/main.py

140 lines
3.8 KiB
Python
Raw Normal View History

2024-06-02 16:01:18 +00:00
import inspect
2024-04-16 22:38:57 +00:00
import logging
2024-08-25 18:50:03 +00:00
from collections.abc import Sequence
from pathlib import Path
2024-04-16 22:38:57 +00:00
import requests
import requests_cache
2024-08-25 18:50:03 +00:00
from . import data, notification
2024-08-25 18:08:41 +00:00
from .util import *
2024-05-09 15:58:39 +00:00
logger = logging.getLogger(__name__)
2024-04-16 22:38:57 +00:00
try:
import cfscrape
except ImportError:
cfscrape = None
2024-05-09 15:58:39 +00:00
logger.warning('cfscrape not installed: Certain fetchers might not work')
try:
import browsercookie
except ImportError:
logger.warning('browsercookie not installed: Certain fetchers might not work')
browsercookie = None
2024-04-16 22:38:57 +00:00
2024-07-27 00:14:01 +00:00
OUTPUT_PATH = Path('./output')
2024-04-16 22:38:57 +00:00
logging.basicConfig()
logger.setLevel('INFO')
STANDARD_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
# "Accept": "application/json, text/plain, */*",
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
}
2024-04-17 22:02:13 +00:00
if cfscrape:
2024-04-23 20:58:25 +00:00
2024-04-17 22:02:13 +00:00
class CachedCfScrape(requests_cache.CacheMixin, cfscrape.CloudflareScraper):
pass
2024-04-16 22:38:57 +00:00
2024-08-25 19:18:55 +00:00
CACHE_EXPIRE_DEFAULT = datetime.timedelta(days=7)
2024-04-16 22:38:57 +00:00
2024-06-06 21:50:29 +00:00
def get_session(
cookiejar: Sequence,
*,
with_cfscrape: bool,
ignore_cache: bool,
) -> requests.Session:
2024-04-16 22:38:57 +00:00
assert isinstance(with_cfscrape, bool)
2024-04-17 22:02:13 +00:00
session_class = requests_cache.CachedSession
2024-06-02 22:00:01 +00:00
if ignore_cache:
2024-08-25 19:07:52 +00:00
logger.warning('HTTP cache disabled')
2024-06-02 22:20:46 +00:00
return requests.Session()
2024-04-17 22:02:13 +00:00
if cfscrape:
session_class = CachedCfScrape
2024-08-25 19:18:55 +00:00
session = session_class(OUTPUT_PATH / 'web_cache', cookies=cookiejar, expire_after=CACHE_EXPIRE_DEFAULT)
2024-04-16 22:38:57 +00:00
for cookie in cookiejar:
session.cookies.set_cookie(cookie)
return session
2024-04-23 20:58:25 +00:00
2024-08-25 18:08:41 +00:00
def available_scrapers() -> list[type[data.Scraper]]:
2024-08-25 19:07:52 +00:00
from . import fetchers # noqa
2024-06-02 16:00:55 +00:00
subclasses = []
2024-08-25 18:08:41 +00:00
class_queue = [data.Scraper]
2024-06-02 16:00:55 +00:00
while class_queue:
clazz = class_queue.pop()
if inspect.isabstract(clazz):
class_queue.extend(clazz.__subclasses__())
else:
subclasses.append(clazz)
return subclasses
2024-04-23 20:58:25 +00:00
2024-05-09 14:59:56 +00:00
2024-04-28 21:45:47 +00:00
def available_scraper_names() -> list[str]:
return [scraper_cls.__name__ for scraper_cls in available_scrapers()]
2024-05-09 14:59:56 +00:00
def main(
scraper_filter: frozenset[str],
*,
use_cookiejar: bool,
2024-06-02 22:00:01 +00:00
ignore_cache: bool,
notification_types: frozenset[notification.NotificationType],
2024-05-09 14:59:56 +00:00
) -> None:
2024-04-17 22:13:56 +00:00
if use_cookiejar:
cookiejar = browsercookie.firefox()
logger.info('Got cookiejar from firefox: %s cookies', len(cookiejar))
else:
cookiejar = []
logger.warning('No cookiejar is used')
2024-04-16 22:38:57 +00:00
2024-06-02 21:14:19 +00:00
if len(notification_types) == 0:
2024-06-02 21:16:11 +00:00
logger.info('No notifications enabled: Notifications will not be sent!')
2024-05-15 22:05:30 +00:00
2024-04-28 21:45:47 +00:00
for scraper_cls in available_scrapers():
2024-06-06 21:50:29 +00:00
session = get_session(
cookiejar,
with_cfscrape=scraper_cls.requires_cfscrape(),
ignore_cache=ignore_cache,
)
2024-04-16 22:38:57 +00:00
scraper = scraper_cls(session)
if scraper_cls.__name__ not in scraper_filter:
continue
2024-04-16 22:45:15 +00:00
logger.info(
2024-04-16 22:38:57 +00:00
'Running %s, appending to "%s"',
scraper_cls.__name__,
scraper.dataset_name,
)
2024-04-28 21:45:47 +00:00
result_rows = []
2024-04-16 22:38:57 +00:00
try:
for result in scraper.scrape():
result_rows.append(result)
del result
except requests.exceptions.HTTPError:
logger.exception('Failed in running %s', scraper_cls.__name__)
continue
status = extend_csv_file(
2024-07-27 00:14:01 +00:00
OUTPUT_PATH / f'{scraper.dataset_name}.csv',
2024-04-16 22:38:57 +00:00
result_rows,
deduplicate_mode=scraper.deduplicate_mode,
2024-09-02 09:55:05 +00:00
deduplicate_ignore_columns=scraper.deduplicate_ignore_columns,
2024-04-16 22:38:57 +00:00
)
2024-04-16 22:45:15 +00:00
logger.info('Scraper done: %s', scraper.dataset_name)
2024-04-16 22:38:57 +00:00
2024-06-02 21:14:19 +00:00
if status['extended']:
2024-06-02 21:16:11 +00:00
notification.send_notifications(
2024-06-06 21:50:29 +00:00
session,
scraper_cls.__name__,
status['dicts'][-1],
notification_types,
2024-06-02 21:16:11 +00:00
)
2024-04-16 22:38:57 +00:00
del scraper, session