1
0

Compare commits

..

No commits in common. "64e279e64ed2a83f19f196c2aefc9d295743c9c3" and "6ca8715a7c5c9c79beddd60c14ebd272481cee70" have entirely different histories.

8 changed files with 142 additions and 189 deletions

View File

@ -3,7 +3,7 @@ on: [push]
jobs: jobs:
Package: Package:
uses: jmaa/workflows/.gitea/workflows/python-package.yaml@v6.02 uses: jmaa/workflows/.gitea/workflows/python-package.yaml@main
secrets: secrets:
PIPY_REPO_USER: ${{ secrets.PIPY_REPO_USER }} PIPY_REPO_USER: ${{ secrets.PIPY_REPO_USER }}
PIPY_REPO_PASS: ${{ secrets.PIPY_REPO_PASS }} PIPY_REPO_PASS: ${{ secrets.PIPY_REPO_PASS }}

View File

@ -1,2 +0,0 @@
enforce-typing
aenum

View File

@ -1,42 +1,29 @@
#!/usr/bin/env python #!/usr/bin/env python
from setuptools import setup
import re import re
from setuptools import setup PACKAGE_NAME = 'socials_util'
PACKAGE_NAME = 'socials-util' with open('README.md', 'r') as f:
with open('README.md') as f:
readme = f.read() readme = f.read()
with open(PACKAGE_NAME.replace('-', '_') + '/_version.py') as f: with open(PACKAGE_NAME+'/_version.py', 'r') as f:
text = f.read() match = re.match(r'^__version__\s*=\s*"([\d\.]+)"$', f.read())
match = re.match(r'^__version__\s*=\s*(["\'])([\d\.]+)\1$', text) version = match.group(1)
version = match.group(2) del match
del match, text
with open('requirements.txt') as f:
install_requires = f.read().strip().split('\n')
def get_short_description(readme: str):
readme = re.sub(r'#+[^\n]*\n+', '', readme)
m = re.search(r'^\s*(\w+[\w\s,`]+\.)', readme)
try:
return m.group(1)
except AttributeError as err:
msg = 'Could not determine short description'
raise Exception(msg) from err
setup( setup(
name=PACKAGE_NAME, name=PACKAGE_NAME,
version=version, version=version,
description=get_short_description(readme),
long_description=readme, long_description=readme,
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
author='Jmaa', author='Jmaa',
author_email='jonjmaa@gmail.com', author_email='jonjmaa@gmail.com',
url='https://gitfub.space/Jmaa/' + PACKAGE_NAME, url='https://gitfub.space/Jmaa/socials-util',
packages=[PACKAGE_NAME], packages=[PACKAGE_NAME],
install_requires=install_requires, install_requires=['enforce-typing', 'aenum'],
keywords=[],
classifiers=[],
include_package_data=True,
zip_safe=False
) )

View File

@ -1,6 +1,6 @@
""" '''
Small utility for detecting social websites. Small utility for detecting social websites.
""" '''
from dataclasses import dataclass from dataclasses import dataclass
from enforce_typing import enforce_types from enforce_typing import enforce_types
@ -12,14 +12,13 @@ import urllib.parse
from socials_util._version import __version__ from socials_util._version import __version__
class SocialSiteId(aenum.Enum): class SocialSiteId(aenum.Enum):
""" '''
The great social website enum. The great social website enum.
""" '''
# Reddit-like # Reddit-like
REDDIT = 1 # Should have been named REDDIT_SUBREDDIT REDDIT = 1 # Should have been named REDDIT_SUBREDDIT
REDDIT_USER = 22 REDDIT_USER = 22
REDDIT_SUBREDDIT = REDDIT REDDIT_SUBREDDIT = REDDIT
@ -52,7 +51,7 @@ class SocialSiteId(aenum.Enum):
INPRNT_PAGE = 14 INPRNT_PAGE = 14
FACEBOOK_PAGE = 15 FACEBOOK_PAGE = 15
EMAIL = 16 EMAIL = 16
JSON_LD = 17 # Similar to PAGE_WATCH, but focused on embedded microdata JSON_LD = 17 # Similar to PAGE_WATCH, but focused on embedded microdata
ETSY_SHOP = 19 ETSY_SHOP = 19
KO_FI = 20 KO_FI = 20
BEHANCE_PAGE = 21 BEHANCE_PAGE = 21
@ -90,37 +89,34 @@ class SocialSiteId(aenum.Enum):
def is_aggregator(self): def is_aggregator(self):
return self in AGGERAGOR_SOCIALS return self in AGGERAGOR_SOCIALS
AGGERAGOR_SOCIALS = { AGGERAGOR_SOCIALS = {
SocialSiteId.LINKTREE_PAGE, SocialSiteId.LINKTREE_PAGE,
SocialSiteId.WIKIDATA, SocialSiteId.WIKIDATA,
SocialSiteId.CARRD_PAGE, SocialSiteId.CARRD_PAGE,
SocialSiteId.LINK_COLLECTION_PAGE, SocialSiteId.LINK_COLLECTION_PAGE,
SocialSiteId.DANBOORU_ARTIST, SocialSiteId.DANBOORU_ARTIST,
SocialSiteId.IGDB_GAME_ID, SocialSiteId.IGDB_GAME_ID,
} }
@enforce_types @enforce_types
@dataclass(frozen=True) @dataclass(frozen = True)
class SocialLink(object): class SocialLink(object):
url: urllib.parse.ParseResult url: urllib.parse.ParseResult
social_site_id: SocialSiteId social_site_id: SocialSiteId
social_id: Optional[str] social_id: Optional[str]
@enforce_types @enforce_types
@dataclass(frozen=True) @dataclass(frozen = True)
class WikidataInfo(object): class WikidataInfo(object):
property_id: Optional[int] property_id: Optional[int]
issuer_id: Optional[int] issuer_id: Optional[int]
id_version_of: Optional[SocialSiteId] = None id_version_of: Optional[SocialSiteId] = None
nickname_version_of: Optional[SocialSiteId] = None nickname_version_of: Optional[SocialSiteId] = None
WIKIDATA_PROPERTIES = { WIKIDATA_PROPERTIES = {
SocialSiteId.EMAIL: WikidataInfo(968, None), SocialSiteId.EMAIL: WikidataInfo(968, None),
SocialSiteId.RSS_FEED: WikidataInfo(1079, None), SocialSiteId.RSS_FEED: WikidataInfo(1079, None),
SocialSiteId.FACEBOOK_PAGE: WikidataInfo(2013, None), SocialSiteId.FACEBOOK_PAGE: WikidataInfo(2013, None),
SocialSiteId.INSTAGRAM_PAGE: WikidataInfo(2003, None), SocialSiteId.INSTAGRAM_PAGE: WikidataInfo(2003, None),
SocialSiteId.LINKTREE_PAGE: WikidataInfo(11079, None), SocialSiteId.LINKTREE_PAGE: WikidataInfo(11079, None),
@ -131,25 +127,21 @@ WIKIDATA_PROPERTIES = {
SocialSiteId.TWITCH: WikidataInfo(5797, None), SocialSiteId.TWITCH: WikidataInfo(5797, None),
SocialSiteId.TWITTER: WikidataInfo(2002, None), SocialSiteId.TWITTER: WikidataInfo(2002, None),
SocialSiteId.WIKIDATA: WikidataInfo(None, 2013), SocialSiteId.WIKIDATA: WikidataInfo(None, 2013),
SocialSiteId.TUMBLR: WikidataInfo(3943, None), SocialSiteId.TUMBLR: WikidataInfo(3943, None),
SocialSiteId.TIKTOK_USER: WikidataInfo(7085, None), SocialSiteId.TIKTOK_USER: WikidataInfo(7085, None),
SocialSiteId.PIXIV_USER_ID: WikidataInfo(
5435, 306956, id_version_of=SocialSiteId.PIXIV_USER_NICKNAME SocialSiteId.PIXIV_USER_ID: WikidataInfo(5435, 306956, id_version_of = SocialSiteId.PIXIV_USER_NICKNAME),
), SocialSiteId.PIXIV_USER_NICKNAME: WikidataInfo(None, 306956, nickname_version_of = SocialSiteId.PIXIV_USER_ID),
SocialSiteId.PIXIV_USER_NICKNAME: WikidataInfo(
None, 306956, nickname_version_of=SocialSiteId.PIXIV_USER_ID
),
SocialSiteId.MASTODON_PAGE: WikidataInfo(4033, None), SocialSiteId.MASTODON_PAGE: WikidataInfo(4033, None),
SocialSiteId.PATREON_PAGE: WikidataInfo(4175, 15861362), SocialSiteId.PATREON_PAGE: WikidataInfo(4175, 15861362),
SocialSiteId.ARTSTATION_PAGE: WikidataInfo(None, 65551500), SocialSiteId.ARTSTATION_PAGE: WikidataInfo(None, 65551500),
# SocialSiteId.INPRNT_PAGE: WikidataInfo(None, None), #SocialSiteId.INPRNT_PAGE: WikidataInfo(None, None),
SocialSiteId.CARRD_PAGE: WikidataInfo(None, 106036503), SocialSiteId.CARRD_PAGE: WikidataInfo(None, 106036503),
SocialSiteId.YOUTUBE_CHANNEL_HANDLE: WikidataInfo( SocialSiteId.YOUTUBE_CHANNEL_HANDLE: WikidataInfo(11245, 866, nickname_version_of = SocialSiteId.YOUTUBE_CHANNEL_ID),
11245, 866, nickname_version_of=SocialSiteId.YOUTUBE_CHANNEL_ID SocialSiteId.YOUTUBE_CHANNEL_ID: WikidataInfo(2397, 866, id_version_of = SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
),
SocialSiteId.YOUTUBE_CHANNEL_ID: WikidataInfo(
2397, 866, id_version_of=SocialSiteId.YOUTUBE_CHANNEL_HANDLE
),
SocialSiteId.VIMEO_CHANNEL: WikidataInfo(4015, 156376), SocialSiteId.VIMEO_CHANNEL: WikidataInfo(4015, 156376),
SocialSiteId.NEWGROUNDS_PAGE: WikidataInfo(None, 263655), SocialSiteId.NEWGROUNDS_PAGE: WikidataInfo(None, 263655),
SocialSiteId.ARTSY_ARTIST: WikidataInfo(2042, 4796642), SocialSiteId.ARTSY_ARTIST: WikidataInfo(2042, 4796642),
@ -167,14 +159,16 @@ WIKIDATA_PROPERTIES = {
SocialSiteId.STEAM_APPLICATION_ID: WikidataInfo(1733, None), SocialSiteId.STEAM_APPLICATION_ID: WikidataInfo(1733, None),
SocialSiteId.GITHUB_REPOSITORY: WikidataInfo(None, 364), SocialSiteId.GITHUB_REPOSITORY: WikidataInfo(None, 364),
SocialSiteId.LINKEDIN_PERSONAL_PROFILE: WikidataInfo(6634, None), SocialSiteId.LINKEDIN_PERSONAL_PROFILE: WikidataInfo(6634, None),
SocialSiteId.MEDIUM_BLOG: WikidataInfo(3899, None), SocialSiteId.MEDIUM_BLOG : WikidataInfo(3899, None),
SocialSiteId.SUBSTACK: WikidataInfo(12007, None), SocialSiteId.SUBSTACK : WikidataInfo(12007, None),
SocialSiteId.INPRNT_PAGE: WikidataInfo(None, None), SocialSiteId.INPRNT_PAGE: WikidataInfo(None, None),
SocialSiteId.ETSY_SHOP: WikidataInfo(None, 1353939), SocialSiteId.ETSY_SHOP: WikidataInfo(None, 1353939),
SocialSiteId.KO_FI: WikidataInfo(None, 77949925), SocialSiteId.KO_FI: WikidataInfo(None, 77949925),
SocialSiteId.BEHANCE_PAGE: WikidataInfo(None, 4880667), SocialSiteId.BEHANCE_PAGE: WikidataInfo(None, 4880667),
SocialSiteId.PLURK: WikidataInfo(None, 32111), SocialSiteId.PLURK: WikidataInfo(None, 32111),
SocialSiteId.GOOGLE_BLOGGER_PAGE: WikidataInfo(None, 171186), SocialSiteId.GOOGLE_BLOGGER_PAGE: WikidataInfo(None, 171186),
# Weird internal # Weird internal
SocialSiteId.LINK_COLLECTION_PAGE: WikidataInfo(None, None), SocialSiteId.LINK_COLLECTION_PAGE: WikidataInfo(None, None),
SocialSiteId.PAGE_WATCH: WikidataInfo(None, None), SocialSiteId.PAGE_WATCH: WikidataInfo(None, None),
@ -183,22 +177,18 @@ WIKIDATA_PROPERTIES = {
SocialSiteId.FALKON_PROFILE_BOOKMARKS: WikidataInfo(None, None), SocialSiteId.FALKON_PROFILE_BOOKMARKS: WikidataInfo(None, None),
} }
def re_social_subdomain(main_domain): def re_social_subdomain(main_domain):
# return r'^(?:https?:\/\/)?([\w_-]+)\.'+re.escape(main_domain)+'\/?$' #return r'^(?:https?:\/\/)?([\w_-]+)\.'+re.escape(main_domain)+'\/?$'
return r'^(?:https?:\/\/)?([\w_-]+)\.' + re.escape(main_domain) + r'(\/.*)?$' return r'^(?:https?:\/\/)?([\w_-]+)\.'+re.escape(main_domain)+r'(\/.*)?$'
RE_ID = r'@?([^/]+)' RE_ID = r'@?([^/]+)'
RE_DUAL_ID = r'@?([^/]+/[^/]+)' RE_DUAL_ID = r'@?([^/]+/[^/]+)'
RE_ANY_SUBPATH = r'(|\/|\/.*)$' RE_ANY_SUBPATH = r'(|\/|\/.*)$'
def re_social_path(main_domain): def re_social_path(main_domain):
# return r'^(?:https?:\/\/)?(?:www\.)?'+re.escape(main_domain)+'\/'+RE_ID+'\/?$' #return r'^(?:https?:\/\/)?(?:www\.)?'+re.escape(main_domain)+'\/'+RE_ID+'\/?$'
return re_social_path_adv(main_domain, RE_ID) return re_social_path_adv(main_domain, RE_ID)
def re_social_path_adv(main_domain, *path): def re_social_path_adv(main_domain, *path):
assert not main_domain.startswith('www.'), 'Redundant www.' assert not main_domain.startswith('www.'), 'Redundant www.'
l = [r'^', r'(?:https?:\/\/)?', r'(?:www\.)?', re.escape(main_domain)] l = [r'^', r'(?:https?:\/\/)?', r'(?:www\.)?', re.escape(main_domain)]
@ -212,21 +202,16 @@ def re_social_path_adv(main_domain, *path):
regex = ''.join(l) regex = ''.join(l)
return regex return regex
MAILTO_URL = r'^mailto:(?:[\w._.]+@[\w._.]+)$' MAILTO_URL = r'^mailto:(?:[\w._.]+@[\w._.]+)$'
REDDIT_SUBREDDIT_URL = r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/r\/([\w-]+)\/?$' REDDIT_SUBREDDIT_URL = r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/r\/([\w-]+)\/?$'
REDDIT_USER_URL = ( REDDIT_USER_URL = r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/user\/([\w-]+)(?:|\/submitted)\/?$'
r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/user\/([\w-]+)(?:|\/submitted)\/?$' TWITTER_HANDLE_URL_1 =re_social_path('twitter.com')
) TWITTER_HANDLE_URL_2 =re_social_path('x.com')
TWITTER_HANDLE_URL_1 = re_social_path('twitter.com')
TWITTER_HANDLE_URL_2 = re_social_path('x.com')
LINKTREE_PAGE_URL = re_social_path('linktr.ee') LINKTREE_PAGE_URL = re_social_path('linktr.ee')
TWITCH_STREAM_URL = re_social_path('twitch.tv') TWITCH_STREAM_URL = re_social_path('twitch.tv')
WIKIDATA_ITEM_URL = re_social_path_adv('wikidata.org', 'wiki', RE_ID) WIKIDATA_ITEM_URL = re_social_path_adv('wikidata.org', 'wiki', RE_ID)
SONGKICK_ARTIST_URL = ( SONGKICK_ARTIST_URL = r'^(?:https?:\/\/)?(?:www\.)?songkick\.com\/artists\/(\d+)([\w-]*)\/?$'
r'^(?:https?:\/\/)?(?:www\.)?songkick\.com\/artists\/(\d+)([\w-]*)\/?$'
)
TUMBLR_PAGE_URL = re_social_path('tumblr.com') TUMBLR_PAGE_URL = re_social_path('tumblr.com')
TUMBLR_PAGE_URL_2 = re_social_subdomain('tumblr.com') TUMBLR_PAGE_URL_2 = re_social_subdomain('tumblr.com')
INSTAGRAM_URL = re_social_path('instagram.com') INSTAGRAM_URL = re_social_path('instagram.com')
@ -239,25 +224,19 @@ ETSY_SHOP_URL = re_social_path_adv('etsy.com', 'shop', RE_ID)
BEHANCE_PAGE_URL = re_social_path('behance.net') BEHANCE_PAGE_URL = re_social_path('behance.net')
TIKTOK_USER_URL = re_social_path('tiktok.com') TIKTOK_USER_URL = re_social_path('tiktok.com')
PIXIV_USER_ID_URL = r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/users/(\d+)\/?$' PIXIV_USER_ID_URL = r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/users/(\d+)\/?$'
PIXIV_USER_ID_URL_2 = ( PIXIV_USER_ID_URL_2 = r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/member\.php\/?[?]id=(\d+)$'
r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/member\.php\/?[?]id=(\d+)$'
)
PIXIV_FANBOX_USER_NICKNAME_URL = re_social_subdomain('fanbox.cc') PIXIV_FANBOX_USER_NICKNAME_URL = re_social_subdomain('fanbox.cc')
PIXIV_USER_NICKNAME_URL = re_social_path_adv('pixiv.net', 'stacc', RE_ID) PIXIV_USER_NICKNAME_URL = re_social_path_adv('pixiv.net', 'stacc', RE_ID)
PIXIV_SKETCH_USER_NICKNAME_URL = re_social_path_adv('sketch.pixiv.net', RE_ID) PIXIV_SKETCH_USER_NICKNAME_URL = re_social_path_adv('sketch.pixiv.net', RE_ID)
URL_PARSE_CARRD_PAGE = re_social_subdomain('carrd.co') URL_PARSE_CARRD_PAGE = re_social_subdomain('carrd.co')
URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1 = re_social_path_adv('youtube.com', RE_ID) URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1= re_social_path_adv('youtube.com', RE_ID)
URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2 = re_social_path_adv('youtube.com', 'c', RE_ID) URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2= re_social_path_adv('youtube.com', 'c', RE_ID)
URL_PARSE_YOUTUBE_CHANNEL_ID = re_social_path_adv('youtube.com', 'channel', RE_ID) URL_PARSE_YOUTUBE_CHANNEL_ID= re_social_path_adv('youtube.com', 'channel', RE_ID)
URL_PARSE_VIMEO_CHANNEL = re_social_path_adv('vimeo.com', RE_ID) URL_PARSE_VIMEO_CHANNEL= re_social_path_adv('vimeo.com', RE_ID)
URL_PARSE_NEWGROUNDS_PAGE = re_social_subdomain('newgrounds.com') URL_PARSE_NEWGROUNDS_PAGE = re_social_subdomain('newgrounds.com')
URL_PARSE_ARTSY_ARTIST = re_social_path_adv( URL_PARSE_ARTSY_ARTIST = re_social_path_adv('artsy.net', 'artist', RE_ID, RE_ANY_SUBPATH)
'artsy.net', 'artist', RE_ID, RE_ANY_SUBPATH URL_PARSE_ARTNET_ARTIST = re_social_path_adv('artnet.com', 'artists', RE_ID, RE_ANY_SUBPATH)
)
URL_PARSE_ARTNET_ARTIST = re_social_path_adv(
'artnet.com', 'artists', RE_ID, RE_ANY_SUBPATH
)
URL_PARSE_DEVIANT_ART_ACCOUNT = re_social_path_adv('deviantart.com', RE_ID) URL_PARSE_DEVIANT_ART_ACCOUNT = re_social_path_adv('deviantart.com', RE_ID)
URL_PARSE_DEVIANT_ART_ACCOUNT_2 = re_social_subdomain('deviantart.com') URL_PARSE_DEVIANT_ART_ACCOUNT_2 = re_social_subdomain('deviantart.com')
URL_PARSE_DANBOORU_ARTIST = re_social_path_adv('danbooru.donmai.us', 'artists', RE_ID) URL_PARSE_DANBOORU_ARTIST = re_social_path_adv('danbooru.donmai.us', 'artists', RE_ID)
@ -268,131 +247,160 @@ REGEXES = [
# Reddit # Reddit
(REDDIT_SUBREDDIT_URL, SocialSiteId.REDDIT_SUBREDDIT), (REDDIT_SUBREDDIT_URL, SocialSiteId.REDDIT_SUBREDDIT),
(REDDIT_USER_URL, SocialSiteId.REDDIT_USER), (REDDIT_USER_URL, SocialSiteId.REDDIT_USER),
# Twitter # Twitter
(TWITTER_HANDLE_URL_1, SocialSiteId.TWITTER), (TWITTER_HANDLE_URL_1, SocialSiteId.TWITTER),
(TWITTER_HANDLE_URL_2, SocialSiteId.TWITTER), (TWITTER_HANDLE_URL_2, SocialSiteId.TWITTER),
# Facebook # Facebook
(FACEBOOK_PAGE_URL, SocialSiteId.FACEBOOK_PAGE), (FACEBOOK_PAGE_URL, SocialSiteId.FACEBOOK_PAGE),
# Linktr.ee # Linktr.ee
(LINKTREE_PAGE_URL, SocialSiteId.LINKTREE_PAGE), (LINKTREE_PAGE_URL, SocialSiteId.LINKTREE_PAGE),
# Twitch.tv # Twitch.tv
(TWITCH_STREAM_URL, SocialSiteId.TWITCH), (TWITCH_STREAM_URL, SocialSiteId.TWITCH),
# Wikidata # Wikidata
(WIKIDATA_ITEM_URL, SocialSiteId.WIKIDATA), (WIKIDATA_ITEM_URL, SocialSiteId.WIKIDATA),
# Songkick # Songkick
(SONGKICK_ARTIST_URL, SocialSiteId.SONGKICK_ARTIST), (SONGKICK_ARTIST_URL, SocialSiteId.SONGKICK_ARTIST),
# Tumblr # Tumblr
(TUMBLR_PAGE_URL, SocialSiteId.TUMBLR), (TUMBLR_PAGE_URL, SocialSiteId.TUMBLR),
(TUMBLR_PAGE_URL_2, SocialSiteId.TUMBLR), (TUMBLR_PAGE_URL_2, SocialSiteId.TUMBLR),
# Instagram # Instagram
(INSTAGRAM_URL, SocialSiteId.INSTAGRAM_PAGE), (INSTAGRAM_URL, SocialSiteId.INSTAGRAM_PAGE),
# Tiktok # Tiktok
(TIKTOK_USER_URL, SocialSiteId.TIKTOK_USER), (TIKTOK_USER_URL, SocialSiteId.TIKTOK_USER),
# Pixiv # Pixiv
(PIXIV_USER_ID_URL, SocialSiteId.PIXIV_USER_ID), (PIXIV_USER_ID_URL, SocialSiteId.PIXIV_USER_ID),
(PIXIV_USER_ID_URL_2, SocialSiteId.PIXIV_USER_ID), (PIXIV_USER_ID_URL_2, SocialSiteId.PIXIV_USER_ID),
(PIXIV_FANBOX_USER_NICKNAME_URL, SocialSiteId.PIXIV_USER_NICKNAME), (PIXIV_FANBOX_USER_NICKNAME_URL , SocialSiteId.PIXIV_USER_NICKNAME),
(PIXIV_USER_NICKNAME_URL, SocialSiteId.PIXIV_USER_NICKNAME), (PIXIV_USER_NICKNAME_URL , SocialSiteId.PIXIV_USER_NICKNAME),
(PIXIV_SKETCH_USER_NICKNAME_URL, SocialSiteId.PIXIV_USER_NICKNAME), (PIXIV_SKETCH_USER_NICKNAME_URL, SocialSiteId.PIXIV_USER_NICKNAME),
# Patreon # Patreon
(PATREON_URL, SocialSiteId.PATREON_PAGE), (PATREON_URL, SocialSiteId.PATREON_PAGE),
# Artstation # Artstation
(ARTSTATION_URL, SocialSiteId.ARTSTATION_PAGE), (ARTSTATION_URL, SocialSiteId.ARTSTATION_PAGE),
# Inprnt # Inprnt
(INPRNT_URL, SocialSiteId.INPRNT_PAGE), (INPRNT_URL, SocialSiteId.INPRNT_PAGE),
# Email # Email
(MAILTO_URL, SocialSiteId.EMAIL), (MAILTO_URL, SocialSiteId.EMAIL),
# Substack # Substack
(SUBSTACK_PREFIX_URL, SocialSiteId.SUBSTACK), (SUBSTACK_PREFIX_URL, SocialSiteId.SUBSTACK),
# Medium # Medium
(re_social_path_adv('medium.com', RE_ID), SocialSiteId.MEDIUM_BLOG), (re_social_path_adv('medium.com', RE_ID), SocialSiteId.MEDIUM_BLOG),
(re_social_subdomain('medium.com'), SocialSiteId.MEDIUM_BLOG), (re_social_subdomain('medium.com'), SocialSiteId.MEDIUM_BLOG),
# Etsy shop # Etsy shop
(ETSY_SHOP_URL, SocialSiteId.ETSY_SHOP), (ETSY_SHOP_URL, SocialSiteId.ETSY_SHOP),
# Ko-fi # Ko-fi
(re_social_path_adv('ko-fi.com', RE_ID), SocialSiteId.KO_FI), (re_social_path_adv('ko-fi.com', RE_ID), SocialSiteId.KO_FI),
(re_social_path_adv('ko-fi.com', RE_ID, 'shop'), SocialSiteId.KO_FI), (re_social_path_adv('ko-fi.com', RE_ID, 'shop'), SocialSiteId.KO_FI),
# Behance # Behance
(BEHANCE_PAGE_URL, SocialSiteId.BEHANCE_PAGE), (BEHANCE_PAGE_URL, SocialSiteId.BEHANCE_PAGE),
# Carrd # Carrd
(URL_PARSE_CARRD_PAGE, SocialSiteId.CARRD_PAGE), (URL_PARSE_CARRD_PAGE, SocialSiteId.CARRD_PAGE),
# Youtube # Youtube
(URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1, SocialSiteId.YOUTUBE_CHANNEL_HANDLE), (URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1, SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
(URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2, SocialSiteId.YOUTUBE_CHANNEL_HANDLE), (URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2, SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
(URL_PARSE_YOUTUBE_CHANNEL_ID, SocialSiteId.YOUTUBE_CHANNEL_ID), (URL_PARSE_YOUTUBE_CHANNEL_ID, SocialSiteId.YOUTUBE_CHANNEL_ID),
# Vimeo # Vimeo
(URL_PARSE_VIMEO_CHANNEL, SocialSiteId.VIMEO_CHANNEL), (URL_PARSE_VIMEO_CHANNEL, SocialSiteId.VIMEO_CHANNEL),
# Newgrounds # Newgrounds
(URL_PARSE_NEWGROUNDS_PAGE, SocialSiteId.NEWGROUNDS_PAGE), (URL_PARSE_NEWGROUNDS_PAGE, SocialSiteId.NEWGROUNDS_PAGE),
# Artsy # Artsy
(URL_PARSE_ARTSY_ARTIST, SocialSiteId.ARTSY_ARTIST), (URL_PARSE_ARTSY_ARTIST, SocialSiteId.ARTSY_ARTIST),
(URL_PARSE_ARTNET_ARTIST, SocialSiteId.ARTNET_ARTIST), (URL_PARSE_ARTNET_ARTIST, SocialSiteId.ARTNET_ARTIST),
# Deviant art # Deviant art
(URL_PARSE_DEVIANT_ART_ACCOUNT, SocialSiteId.DEVIANT_ART_ACCOUNT), (URL_PARSE_DEVIANT_ART_ACCOUNT, SocialSiteId.DEVIANT_ART_ACCOUNT),
(URL_PARSE_DEVIANT_ART_ACCOUNT_2, SocialSiteId.DEVIANT_ART_ACCOUNT), (URL_PARSE_DEVIANT_ART_ACCOUNT_2, SocialSiteId.DEVIANT_ART_ACCOUNT),
# Danbooru # Danbooru
(URL_PARSE_DANBOORU_ARTIST, SocialSiteId.DANBOORU_ARTIST), (URL_PARSE_DANBOORU_ARTIST, SocialSiteId.DANBOORU_ARTIST),
# Bandcamp # Bandcamp
(URL_PARSE_BANDCAMP, SocialSiteId.BANDCAMP_PROFILE), (URL_PARSE_BANDCAMP, SocialSiteId.BANDCAMP_PROFILE),
# Bluesky # Bluesky
(URL_PARSE_BLUESKY, SocialSiteId.BLUESKY_PROFILE), (URL_PARSE_BLUESKY, SocialSiteId.BLUESKY_PROFILE),
# Threads # Threads
(re_social_path_adv('threads.net', RE_ID), SocialSiteId.THREADS_USERNAME), (re_social_path_adv('threads.net', RE_ID), SocialSiteId.THREADS_USERNAME),
# Itch.io # Itch.io
(re_social_subdomain('itch.io'), SocialSiteId.ITCH_IO_DEVELOPER), (re_social_subdomain('itch.io'), SocialSiteId.ITCH_IO_DEVELOPER),
# Cohost # Cohost
(re_social_path_adv('cohost.org', RE_ID), SocialSiteId.COHOST_PROFILE), (re_social_path_adv('cohost.org', RE_ID), SocialSiteId.COHOST_PROFILE),
# Soundcloud # Soundcloud
(re_social_path_adv('soundcloud.com', RE_ID), SocialSiteId.SOUNDCLOUD_ARTIST), (re_social_path_adv('soundcloud.com', RE_ID), SocialSiteId.SOUNDCLOUD_ARTIST),
# IGDB # IGDB
(re_social_path_adv('igdb.com', 'games', RE_ID), SocialSiteId.IGDB_GAME_ID), (re_social_path_adv('igdb.com', 'games', RE_ID), SocialSiteId.IGDB_GAME_ID),
# Steam game # Steam game
( (re_social_path_adv('store.steampowered.com', 'app', RE_ID, RE_ANY_SUBPATH), SocialSiteId.STEAM_APPLICATION_ID),
re_social_path_adv('store.steampowered.com', 'app', RE_ID, RE_ANY_SUBPATH),
SocialSiteId.STEAM_APPLICATION_ID,
),
# Github # Github
(re_social_path_adv('github.com', RE_DUAL_ID), SocialSiteId.GITHUB_REPOSITORY), (re_social_path_adv('github.com', RE_DUAL_ID), SocialSiteId.GITHUB_REPOSITORY),
# Plurk # Plurk
(re_social_path_adv('plurk.com', RE_ID), SocialSiteId.PLURK), (re_social_path_adv('plurk.com', RE_ID), SocialSiteId.PLURK),
# Linked in # Linked in
( (re_social_path_adv('linkedin.com', 'in', RE_ID), SocialSiteId.LINKEDIN_PERSONAL_PROFILE),
re_social_path_adv('linkedin.com', 'in', RE_ID),
SocialSiteId.LINKEDIN_PERSONAL_PROFILE,
),
# Google Blogger # Google Blogger
(re_social_subdomain('blogspot.com'), SocialSiteId.GOOGLE_BLOGGER_PAGE), (re_social_subdomain('blogspot.com'), SocialSiteId.GOOGLE_BLOGGER_PAGE),
] ]
WELL_KNOWN_MASTODON_INSTANCES = frozenset( WELL_KNOWN_MASTODON_INSTANCES = frozenset({
{ # Includes all servers with 50 000+ users as of 6 july 2023.
# Includes all servers with 50 000+ users as of 6 july 2023. # based on https://mastodonservers.net/servers/top
# based on https://mastodonservers.net/servers/top 'mastodon.social',
'mastodon.social', #'pawoo.net',
#'pawoo.net', 'baraag.net',
'baraag.net', 'mstdn.jp',
'mstdn.jp', 'mastodon.cloud',
'mastodon.cloud', 'mstdn.social',
'mstdn.social', 'mastodon.online',
'mastodon.online', 'mas.to',
'mas.to', 'mastodon.world',
'mastodon.world', 'mastodon.lol',
'mastodon.lol', 'mastodon.sdf.org',
'mastodon.sdf.org', 'c.im',
'c.im', 'mastodon.uno',
'mastodon.uno', 'mastodonapp.uk',
'mastodonapp.uk', 'fosstodon.org',
'fosstodon.org', 'idlethumbs.social',
'idlethumbs.social', })
}
)
def determine_social_from_url_internally(url: str): def determine_social_from_url_internally(url: str):
assert isinstance(url, str) assert isinstance(url, str)
# Regexes # Regexes
for social_site_url_regex, social_site_id in REGEXES: for (social_site_url_regex, social_site_id) in REGEXES:
if m := re.fullmatch(social_site_url_regex, url, re.I): if m := re.fullmatch(social_site_url_regex, url, re.I):
groups = m.groups() groups = m.groups()
return (social_site_id, groups[0] if len(groups) > 0 else None) return (social_site_id, groups[0] if len(groups) > 0 else None)
@ -410,18 +418,14 @@ def determine_social_from_url_internally(url: str):
return (None, None) return (None, None)
def determine_social_from_url(url): def determine_social_from_url(url):
if isinstance(url, str): if isinstance(url, str):
url = urllib.parse.urlparse(url) url = urllib.parse.urlparse(url)
(social_site_id, social_id) = determine_social_from_url_internally( (social_site_id, social_id) = determine_social_from_url_internally(url._replace(query = '', fragment = '').geturl())
url._replace(query='', fragment='').geturl()
)
if not social_site_id: if not social_site_id:
(social_site_id, social_id) = determine_social_from_url_internally( (social_site_id, social_id) = determine_social_from_url_internally(url._replace(fragment = '').geturl())
url._replace(fragment='').geturl()
)
if not social_site_id: if not social_site_id:
return None return None
return SocialLink(url, social_site_id, social_id) return SocialLink(url, social_site_id, social_id)

View File

@ -1 +1 @@
__version__ = '0.1.1' __version__ = "0.1.1"

View File

@ -1,7 +1,9 @@
import socials_util import socials_util
import pytest import pytest
@pytest.mark.parametrize('social_site_id', list(socials_util.SocialSiteId)) @pytest.mark.parametrize('social_site_id', list(socials_util.SocialSiteId))
def test_consistency(social_site_id): def test_consistency(social_site_id):
assert social_site_id in socials_util.WIKIDATA_PROPERTIES assert social_site_id in socials_util.WIKIDATA_PROPERTIES

View File

@ -1,7 +1,7 @@
from socials_util import * from socials_util import *
import aenum import aenum
def test_extension(): def test_extension():
MY_SECRET_SITE = aenum.extend_enum(SocialSiteId, 'MY_SECRET_SITE', 666) MY_SECRET_SITE = aenum.extend_enum(SocialSiteId, 'MY_SECRET_SITE', 666)
assert MY_SECRET_SITE assert MY_SECRET_SITE
@ -14,3 +14,4 @@ def test_extension():
assert result is not None assert result is not None
assert result.social_site_id == MY_SECRET_SITE assert result.social_site_id == MY_SECRET_SITE
assert result.social_id == '123' assert result.social_id == '123'

View File

@ -1,58 +1,20 @@
from socials_util import *
from socials_util import *
def test_parsing(): def test_parsing():
assert determine_social_from_url('http://www.twitter.com/dril').social_id == 'dril' assert determine_social_from_url('http://www.twitter.com/dril').social_id == 'dril'
assert determine_social_from_url('http://worstdril.tumblr.com/') assert determine_social_from_url('http://worstdril.tumblr.com/')
assert ( assert determine_social_from_url('https://deep-dark-fears.tumblr.com').social_id == 'deep-dark-fears'
determine_social_from_url('https://deep-dark-fears.tumblr.com').social_id assert determine_social_from_url('https://www.etsy.com/shop/aleksiremesart').social_id == 'aleksiremesart'
== 'deep-dark-fears' assert determine_social_from_url('https://ko-fi.com/A627LI1/shop').social_id == 'A627LI1'
) assert determine_social_from_url('https://ko-fi.com/A627LI1/').social_id == 'A627LI1'
assert ( assert determine_social_from_url('https://www.facebook.com/fredagscafeen.dk/').social_id == 'fredagscafeen.dk'
determine_social_from_url('https://www.etsy.com/shop/aleksiremesart').social_id assert determine_social_from_url('https://www.tiktok.com/@depthsofwikipedia?lang=en').social_id == 'depthsofwikipedia'
== 'aleksiremesart' assert determine_social_from_url('https://www.pixiv.net/users/14866303').social_id == '14866303'
) assert determine_social_from_url('https://www.pixiv.net/member.php?id=109710').social_id == '109710'
assert ( assert determine_social_from_url('https://www.deviantart.com/solquiet').social_site_id == SocialSiteId.DEVIANT_ART_ACCOUNT
determine_social_from_url('https://ko-fi.com/A627LI1/shop').social_id assert determine_social_from_url('https://solquiet.deviantart.com/').social_site_id == SocialSiteId.DEVIANT_ART_ACCOUNT
== 'A627LI1' assert determine_social_from_url('https://cohost.org/example').social_site_id == SocialSiteId.COHOST_PROFILE
)
assert (
determine_social_from_url('https://ko-fi.com/A627LI1/').social_id == 'A627LI1'
)
assert (
determine_social_from_url(
'https://www.facebook.com/fredagscafeen.dk/'
).social_id
== 'fredagscafeen.dk'
)
assert (
determine_social_from_url(
'https://www.tiktok.com/@depthsofwikipedia?lang=en'
).social_id
== 'depthsofwikipedia'
)
assert (
determine_social_from_url('https://www.pixiv.net/users/14866303').social_id
== '14866303'
)
assert (
determine_social_from_url(
'https://www.pixiv.net/member.php?id=109710'
).social_id
== '109710'
)
assert (
determine_social_from_url('https://www.deviantart.com/solquiet').social_site_id
== SocialSiteId.DEVIANT_ART_ACCOUNT
)
assert (
determine_social_from_url('https://solquiet.deviantart.com/').social_site_id
== SocialSiteId.DEVIANT_ART_ACCOUNT
)
assert (
determine_social_from_url('https://cohost.org/example').social_site_id
== SocialSiteId.COHOST_PROFILE
)
INSTAGRAMS = [ INSTAGRAMS = [
'https://instagram.com/_richardparry_', 'https://instagram.com/_richardparry_',
@ -64,6 +26,5 @@ def test_parsing():
] ]
for ig in INSTAGRAMS: for ig in INSTAGRAMS:
assert ( assert determine_social_from_url(ig).social_site_id == SocialSiteId.INSTAGRAM_PAGE
determine_social_from_url(ig).social_site_id == SocialSiteId.INSTAGRAM_PAGE
)