1
0

Ruff
Some checks failed
Python Ruff Code Quality / ruff (push) Successful in 24s
Run Python tests (through Pytest) / Test (push) Failing after 25s
Verify Python project can be installed, loaded and have version checked / Test (push) Successful in 22s

This commit is contained in:
Jon Michael Aanes 2024-11-09 17:54:15 +01:00
parent 4668372036
commit cf6ef584a0
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
2 changed files with 119 additions and 59 deletions

View File

@ -5,9 +5,9 @@ Python library for parsing and processing URLs of Social Media Sites.
Used by one-page-internet. Used by one-page-internet.
""" """
import dataclasses
import re import re
import urllib.parse import urllib.parse
import dataclasses
import aenum import aenum
import enforce_typing import enforce_typing
@ -205,7 +205,6 @@ WIKIDATA_PROPERTIES: dict[SocialSiteId | int, WikidataInfo] = {
} }
RE_ID = r'@?([^\s/]+)' RE_ID = r'@?([^\s/]+)'
RE_DUAL_ID = r'@?([^\s/]+/[^\s/]+)' RE_DUAL_ID = r'@?([^\s/]+/[^\s/]+)'
RE_ANY_SUBPATH = r'(|\/|\/\S*)$' RE_ANY_SUBPATH = r'(|\/|\/\S*)$'
@ -214,12 +213,14 @@ SPECIAL_REGEX_LITERALS = frozenset({RE_ID, RE_DUAL_ID, RE_ANY_SUBPATH})
REGEX_LITERALS_TO_FORMATTER = {RE_ID: '{id}', RE_DUAL_ID: '{id}', RE_ANY_SUBPATH: ''} REGEX_LITERALS_TO_FORMATTER = {RE_ID: '{id}', RE_DUAL_ID: '{id}', RE_ANY_SUBPATH: ''}
DOES_NOT_NEED_AUTO_SLASH = frozenset({RE_ANY_SUBPATH}) DOES_NOT_NEED_AUTO_SLASH = frozenset({RE_ANY_SUBPATH})
@enforce_typing.enforce_types @enforce_typing.enforce_types
@dataclasses.dataclass(frozen=True) @dataclasses.dataclass(frozen=True)
class SocialPathFormat: class SocialPathFormat:
regex: str regex: str
formatter: str formatter: str
def social_path_format_adv(main_domain: str, *path: str) -> SocialPathFormat: def social_path_format_adv(main_domain: str, *path: str) -> SocialPathFormat:
if main_domain.startswith('www.'): if main_domain.startswith('www.'):
msg = f'Redundant www: {main_domain}' msg = f'Redundant www: {main_domain}'
@ -252,15 +253,20 @@ def social_path_format_adv(main_domain: str, *path: str) -> SocialPathFormat:
formatter=''.join(formatter_builder), formatter=''.join(formatter_builder),
) )
def social_path_format(main_domain: str) -> SocialPathFormat: def social_path_format(main_domain: str) -> SocialPathFormat:
return social_path_format_adv(main_domain, RE_ID) return social_path_format_adv(main_domain, RE_ID)
def re_social_subdomain(main_domain: str) -> SocialPathFormat: def re_social_subdomain(main_domain: str) -> SocialPathFormat:
return SocialPathFormat( return SocialPathFormat(
regex = r'^(?:https?:\/\/)?(?:www\.)?([\w_-]+)\.' + re.escape(main_domain) + r'(\/.*)?$', regex=r'^(?:https?:\/\/)?(?:www\.)?([\w_-]+)\.'
+ re.escape(main_domain)
+ r'(\/.*)?$',
formatter='https://{id}.' + main_domain, formatter='https://{id}.' + main_domain,
) )
def re_social_path(main_domain: str) -> str: def re_social_path(main_domain: str) -> str:
return social_path_format(main_domain).regex return social_path_format(main_domain).regex
@ -286,108 +292,145 @@ PIXIV_USER_ID_URL_2 = (
) )
URL_FORMATS: list[tuple[object, SocialPathFormat]] = [ URL_FORMATS: list[tuple[object, SocialPathFormat]] = [
# Twitter # Twitter
(SocialSiteId.TWITTER, social_path_format_adv('x.com', RE_ID, RE_ANY_SUBPATH)), (SocialSiteId.TWITTER, social_path_format_adv('x.com', RE_ID, RE_ANY_SUBPATH)),
(SocialSiteId.TWITTER, social_path_format_adv('twitter.com', RE_ID, RE_ANY_SUBPATH)), (
SocialSiteId.TWITTER,
social_path_format_adv('twitter.com', RE_ID, RE_ANY_SUBPATH),
),
# Linktr.ee # Linktr.ee
(SocialSiteId.LINKTREE_PAGE, social_path_format('linktr.ee')), (SocialSiteId.LINKTREE_PAGE, social_path_format('linktr.ee')),
# Twitch.tv # Twitch.tv
(SocialSiteId.TWITCH, social_path_format('twitch.tv')), (SocialSiteId.TWITCH, social_path_format('twitch.tv')),
# Wikidata # Wikidata
(SocialSiteId.WIKIDATA, social_path_format_adv('wikidata.org', 'wiki', RE_ID)), (SocialSiteId.WIKIDATA, social_path_format_adv('wikidata.org', 'wiki', RE_ID)),
# Tumblr # Tumblr
(SocialSiteId.TUMBLR, social_path_format('tumblr.com')), (SocialSiteId.TUMBLR, social_path_format('tumblr.com')),
(SocialSiteId.TUMBLR, re_social_subdomain('tumblr.com')), (SocialSiteId.TUMBLR, re_social_subdomain('tumblr.com')),
(SocialSiteId.TUMBLR, social_path_format('tumblr.com/blog')), (SocialSiteId.TUMBLR, social_path_format('tumblr.com/blog')),
(SocialSiteId.TUMBLR, social_path_format('tumblr.com/blog/view')), (SocialSiteId.TUMBLR, social_path_format('tumblr.com/blog/view')),
# Instagram # Instagram
(SocialSiteId.INSTAGRAM_PAGE, social_path_format('instagram.com')), (SocialSiteId.INSTAGRAM_PAGE, social_path_format('instagram.com')),
# Patreon # Patreon
(SocialSiteId.PATREON_PAGE, social_path_format_adv('patreon.com', RE_ID, RE_ANY_SUBPATH)), (
SocialSiteId.PATREON_PAGE,
social_path_format_adv('patreon.com', RE_ID, RE_ANY_SUBPATH),
),
# Artstation # Artstation
(SocialSiteId.ARTSTATION_PAGE, social_path_format_adv('artstation.com', RE_ID, RE_ANY_SUBPATH)), (
SocialSiteId.ARTSTATION_PAGE,
social_path_format_adv('artstation.com', RE_ID, RE_ANY_SUBPATH),
),
(SocialSiteId.ARTSTATION_PAGE, re_social_subdomain('artstation.com')), (SocialSiteId.ARTSTATION_PAGE, re_social_subdomain('artstation.com')),
# Inprnt # Inprnt
(SocialSiteId.INPRNT_PAGE, social_path_format_adv('inprnt.com', 'gallery', RE_ID)), (SocialSiteId.INPRNT_PAGE, social_path_format_adv('inprnt.com', 'gallery', RE_ID)),
# Facebook # Facebook
(SocialSiteId.FACEBOOK_PAGE, social_path_format('facebook.com')), (SocialSiteId.FACEBOOK_PAGE, social_path_format('facebook.com')),
# Substack # Substack
(SocialSiteId.SUBSTACK, re_social_subdomain('substack.com')), (SocialSiteId.SUBSTACK, re_social_subdomain('substack.com')),
# Etsy shop # Etsy shop
(SocialSiteId.ETSY_SHOP, social_path_format_adv('etsy.com', 'shop', RE_ID)), (SocialSiteId.ETSY_SHOP, social_path_format_adv('etsy.com', 'shop', RE_ID)),
# Behance # Behance
(SocialSiteId.BEHANCE_PAGE, social_path_format('behance.net')), (SocialSiteId.BEHANCE_PAGE, social_path_format('behance.net')),
# Tiktok # Tiktok
(SocialSiteId.TIKTOK_USER, social_path_format('tiktok.com')), (SocialSiteId.TIKTOK_USER, social_path_format('tiktok.com')),
# Pixiv # Pixiv
(SocialSiteId.PIXIV_USER_NICKNAME, social_path_format_adv('pixiv.net', 'stacc', RE_ID)), (
SocialSiteId.PIXIV_USER_NICKNAME,
social_path_format_adv('pixiv.net', 'stacc', RE_ID),
),
(SocialSiteId.PIXIV_USER_NICKNAME, re_social_subdomain('fanbox.cc')), (SocialSiteId.PIXIV_USER_NICKNAME, re_social_subdomain('fanbox.cc')),
(SocialSiteId.PIXIV_USER_NICKNAME, social_path_format_adv('sketch.pixiv.net', RE_ID)), (
SocialSiteId.PIXIV_USER_NICKNAME,
social_path_format_adv('sketch.pixiv.net', RE_ID),
),
# Carrd # Carrd
(SocialSiteId.CARRD_PAGE, re_social_subdomain('carrd.co')), (SocialSiteId.CARRD_PAGE, re_social_subdomain('carrd.co')),
# Youtube # Youtube
(SocialSiteId.YOUTUBE_CHANNEL_HANDLE, social_path_format_adv('youtube.com', RE_ID, RE_ANY_SUBPATH)), (
(SocialSiteId.YOUTUBE_CHANNEL_HANDLE, social_path_format_adv('youtube.com', 'c', RE_ID)), SocialSiteId.YOUTUBE_CHANNEL_HANDLE,
(SocialSiteId.YOUTUBE_CHANNEL_ID, social_path_format_adv('youtube.com', 'channel', RE_ID)), social_path_format_adv('youtube.com', RE_ID, RE_ANY_SUBPATH),
),
(
SocialSiteId.YOUTUBE_CHANNEL_HANDLE,
social_path_format_adv('youtube.com', 'c', RE_ID),
),
(
SocialSiteId.YOUTUBE_CHANNEL_ID,
social_path_format_adv('youtube.com', 'channel', RE_ID),
),
# Vimeo # Vimeo
(SocialSiteId.VIMEO_CHANNEL, social_path_format_adv('vimeo.com', RE_ID)), (SocialSiteId.VIMEO_CHANNEL, social_path_format_adv('vimeo.com', RE_ID)),
# Newgrounds # Newgrounds
(SocialSiteId.NEWGROUNDS_PAGE, re_social_subdomain('newgrounds.com')), (SocialSiteId.NEWGROUNDS_PAGE, re_social_subdomain('newgrounds.com')),
# Artsy # Artsy
(SocialSiteId.ARTSY_ARTIST, social_path_format_adv( 'artsy.net', 'artist', RE_ID, RE_ANY_SUBPATH)), (
SocialSiteId.ARTSY_ARTIST,
social_path_format_adv('artsy.net', 'artist', RE_ID, RE_ANY_SUBPATH),
),
# Artnet # Artnet
(SocialSiteId.ARTNET_ARTIST, social_path_format_adv( 'artnet.com', 'artists', RE_ID, RE_ANY_SUBPATH)), (
SocialSiteId.ARTNET_ARTIST,
social_path_format_adv('artnet.com', 'artists', RE_ID, RE_ANY_SUBPATH),
),
# Deviant art # Deviant art
(SocialSiteId.DEVIANT_ART_ACCOUNT, social_path_format_adv('deviantart.com', RE_ID)), (SocialSiteId.DEVIANT_ART_ACCOUNT, social_path_format_adv('deviantart.com', RE_ID)),
(SocialSiteId.DEVIANT_ART_ACCOUNT, re_social_subdomain('deviantart.com')), (SocialSiteId.DEVIANT_ART_ACCOUNT, re_social_subdomain('deviantart.com')),
# Danbooru # Danbooru
(SocialSiteId.DANBOORU_ARTIST, social_path_format_adv('danbooru.donmai.us', 'artists', RE_ID)), (
SocialSiteId.DANBOORU_ARTIST,
social_path_format_adv('danbooru.donmai.us', 'artists', RE_ID),
),
# Bandcamp # Bandcamp
(SocialSiteId.BANDCAMP_PROFILE, re_social_subdomain('bandcamp.com')), (SocialSiteId.BANDCAMP_PROFILE, re_social_subdomain('bandcamp.com')),
# Bluesky # Bluesky
(SocialSiteId.BLUESKY_PROFILE, social_path_format_adv('bsky.app', 'profile', RE_ID)), (
SocialSiteId.BLUESKY_PROFILE,
social_path_format_adv('bsky.app', 'profile', RE_ID),
),
# Medium # Medium
(SocialSiteId.MEDIUM_BLOG, social_path_format_adv('medium.com', RE_ID), ), (
(SocialSiteId.MEDIUM_BLOG, re_social_subdomain('medium.com'), ), SocialSiteId.MEDIUM_BLOG,
social_path_format_adv('medium.com', RE_ID),
),
(
SocialSiteId.MEDIUM_BLOG,
re_social_subdomain('medium.com'),
),
# Ko-fi # Ko-fi
(SocialSiteId.KO_FI, social_path_format_adv('ko-fi.com', RE_ID), ), (
(SocialSiteId.KO_FI, social_path_format_adv('ko-fi.com', RE_ID, 'shop'), ), SocialSiteId.KO_FI,
social_path_format_adv('ko-fi.com', RE_ID),
),
(
SocialSiteId.KO_FI,
social_path_format_adv('ko-fi.com', RE_ID, 'shop'),
),
# Threads # Threads
(SocialSiteId.THREADS_USERNAME, social_path_format_adv('threads.net', RE_ID), ), (
SocialSiteId.THREADS_USERNAME,
social_path_format_adv('threads.net', RE_ID),
),
# Itch.io # Itch.io
(SocialSiteId.ITCH_IO_DEVELOPER, re_social_subdomain('itch.io'), ), (
SocialSiteId.ITCH_IO_DEVELOPER,
re_social_subdomain('itch.io'),
),
# Cohost # Cohost
(SocialSiteId.COHOST_PROFILE, social_path_format_adv('cohost.org', RE_ID), ), (
SocialSiteId.COHOST_PROFILE,
social_path_format_adv('cohost.org', RE_ID),
),
# Soundcloud # Soundcloud
(SocialSiteId.SOUNDCLOUD_ARTIST, social_path_format_adv('soundcloud.com', RE_ID), ), (
SocialSiteId.SOUNDCLOUD_ARTIST,
social_path_format_adv('soundcloud.com', RE_ID),
),
# IGDB # IGDB
(SocialSiteId.IGDB_GAME_ID, social_path_format_adv('igdb.com', 'games', RE_ID), ), (
SocialSiteId.IGDB_GAME_ID,
social_path_format_adv('igdb.com', 'games', RE_ID),
),
# Steam game # Steam game
( (
SocialSiteId.STEAM_APPLICATION_ID, SocialSiteId.STEAM_APPLICATION_ID,
@ -399,16 +442,25 @@ URL_FORMATS: list[tuple[object, SocialPathFormat]] = [
social_path_format_adv('github.com', RE_DUAL_ID, RE_ANY_SUBPATH), social_path_format_adv('github.com', RE_DUAL_ID, RE_ANY_SUBPATH),
), ),
# Plurk # Plurk
(SocialSiteId.PLURK, social_path_format_adv('plurk.com', RE_ID), ), (
SocialSiteId.PLURK,
social_path_format_adv('plurk.com', RE_ID),
),
# Linked in # Linked in
( (
SocialSiteId.LINKEDIN_PERSONAL_PROFILE, SocialSiteId.LINKEDIN_PERSONAL_PROFILE,
social_path_format_adv('linkedin.com', 'in', RE_ID), social_path_format_adv('linkedin.com', 'in', RE_ID),
), ),
# Google Blogger # Google Blogger
(SocialSiteId.GOOGLE_BLOGGER_PAGE, re_social_subdomain('blogspot.com'), ), (
SocialSiteId.GOOGLE_BLOGGER_PAGE,
re_social_subdomain('blogspot.com'),
),
# Cara # Cara
(SocialSiteId.CARA_PROFILE, social_path_format_adv('cara.app', RE_ID, RE_ANY_SUBPATH), ), (
SocialSiteId.CARA_PROFILE,
social_path_format_adv('cara.app', RE_ID, RE_ANY_SUBPATH),
),
] ]
REGEXES: list[tuple[str, object]] = [ REGEXES: list[tuple[str, object]] = [
@ -491,12 +543,17 @@ def to_parse_result(url: str | urllib.parse.ParseResult) -> urllib.parse.ParseRe
msg = f'Expected {urllib.parse.ParseResult} or {str}' msg = f'Expected {urllib.parse.ParseResult} or {str}'
raise TypeError(msg) raise TypeError(msg)
def to_url(social_site_id: SocialSiteId, social_id: str) -> urllib.parse.ParseResult | None:
for (ssi, fmt) in URL_FORMATS: def to_url(
social_site_id: SocialSiteId,
social_id: str,
) -> urllib.parse.ParseResult | None:
for ssi, fmt in URL_FORMATS:
if ssi == social_site_id: if ssi == social_site_id:
return to_parse_result(fmt.formatter.format(id=social_id)) return to_parse_result(fmt.formatter.format(id=social_id))
return None return None
def determine_social_from_url( def determine_social_from_url(
url_not_normalized: str | urllib.parse.ParseResult, url_not_normalized: str | urllib.parse.ParseResult,
) -> SocialLink | None: ) -> SocialLink | None:

View File

@ -212,7 +212,9 @@ NOT_PARSABLE = [
PARSABLE_SOCIAL_IDS_COMBINED, PARSABLE_SOCIAL_IDS_COMBINED,
) )
def test_parse_social_ids( def test_parse_social_ids(
url: str, expected_social_site_id: SocialSiteId, expected_social_id: str, url: str,
expected_social_site_id: SocialSiteId,
expected_social_id: str,
) -> None: ) -> None:
social_link: SocialLink | None = determine_social_from_url(url) social_link: SocialLink | None = determine_social_from_url(url)
assert social_link is not None, url assert social_link is not None, url
@ -243,6 +245,7 @@ def test_determine_social_from_url_internally() -> None:
with pytest.raises(TypeError): with pytest.raises(TypeError):
assert socials_util.determine_social_from_url_internally(None) assert socials_util.determine_social_from_url_internally(None)
def test_normalize_url(): def test_normalize_url():
social_link = determine_social_from_url('http://twitter.com/dril') social_link = determine_social_from_url('http://twitter.com/dril')
assert social_link is not None assert social_link is not None