socials-util/socials_util/__init__.py

from dataclasses import dataclass
from enforce_typing import enforce_types
from typing import List, Set, Optional, Union
import enum
import datetime
import re
import urllib.parse

class SocialSiteId(enum.Enum):
    REDDIT = 1 # Should have been named REDDIT_SUBREDDIT
    REDDIT_USER = 22
    REDDIT_SUBREDDIT = REDDIT

    TWITTER = 2
    RSS_FEED = 3
    PAGE_WATCH = 4
    LINKTREE_PAGE = 5
    TWITCH = 6
    WIKIDATA = 7
    SONGKICK_ARTIST = 8
    TUMBLR = 9
    MASTODON_PAGE = 10
    INSTAGRAM_PAGE = 11
    PATREON_PAGE = 12
    ARTSTATION_PAGE = 13
    INPRNT_PAGE = 14
    FACEBOOK_PAGE = 15
    EMAIL = 16
    JSON_LD = 17 # Similar to PAGE_WATCH, but focused on embedded microdata
    SUBSTACK = 18
    ETSY_SHOP = 19
    KO_FI = 20
    BEHANCE_PAGE = 21
    TIKTOK_USER = 7085
    PIXIV_USER_ID = 5435
    PIXIV_USER_NICKNAME = 31
    CARRD_PAGE = 24
    YOUTUBE_CHANNEL_HANDLE = 26
    YOUTUBE_CHANNEL_ID = 2397
    VIMEO_CHANNEL = 27
    NEWGROUNDS_PAGE = 28
    ARTSY_ARTIST = 2042
    ARTNET_ARTIST = 3782
    LINK_COLLECTION_PAGE = 29
    DEVIANT_ART_ACCOUNT = 7737
    DANBOORU_ARTIST = 30
    BANDCAMP_PROFILE = 3283
    BLUESKY_PROFILE = 32
    ITCH_IO_DEVELOPER = 8176
    FIREFOX_PROFILE_BOOKMARKS = 33

    def wikidata_property(self, client):
        return client.get(WIKIDATA_PROPERTIES[self])

    def is_aggregator(self):
        return self in AGGERAGOR_SOCIALS

AGGERAGOR_SOCIALS = {
        SocialSiteId.LINKTREE_PAGE,
        SocialSiteId.WIKIDATA,
        SocialSiteId.CARRD_PAGE,
        SocialSiteId.LINK_COLLECTION_PAGE,
        SocialSiteId.DANBOORU_ARTIST,
}

@enforce_types
@dataclass(frozen = True)
class SocialLink(object):
    url: urllib.parse.ParseResult
    social_site_id: SocialSiteId
    social_id: Optional[str]

@enforce_types
@dataclass(frozen = True)
class WikidataInfo(object):
    property_id: Optional[int]
    issuer_id: Optional[int]
    id_version_of: Optional[SocialSiteId] = None
    nickname_version_of: Optional[SocialSiteId] = None

WIKIDATA_PROPERTIES = {
    SocialSiteId.EMAIL: WikidataInfo(968, None),
    SocialSiteId.RSS_FEED: WikidataInfo(1079, None),

    SocialSiteId.FACEBOOK_PAGE: WikidataInfo(2013, None),
    SocialSiteId.INSTAGRAM_PAGE: WikidataInfo(2003, None),
    SocialSiteId.LINKTREE_PAGE: WikidataInfo(11079, None),
    SocialSiteId.REDDIT_SUBREDDIT: WikidataInfo(3984, None),
    SocialSiteId.REDDIT_USER: WikidataInfo(4265, None),
    SocialSiteId.RSS_FEED: WikidataInfo(1019, None),
    SocialSiteId.SONGKICK_ARTIST: WikidataInfo(3478, None),
    SocialSiteId.TWITCH: WikidataInfo(5797, None),
    SocialSiteId.TWITTER: WikidataInfo(2002, None),
    SocialSiteId.WIKIDATA: WikidataInfo(None, 2013),

    SocialSiteId.TUMBLR: WikidataInfo(3943, None),
    SocialSiteId.TIKTOK_USER: WikidataInfo(7085, None),

    SocialSiteId.PIXIV_USER_ID: WikidataInfo(5435, 306956, id_version_of = SocialSiteId.PIXIV_USER_NICKNAME),
    SocialSiteId.PIXIV_USER_NICKNAME: WikidataInfo(None, 306956, nickname_version_of = SocialSiteId.PIXIV_USER_ID),

    SocialSiteId.MASTODON_PAGE: WikidataInfo(4033, None),
    SocialSiteId.PATREON_PAGE: WikidataInfo(4175, 15861362),
    SocialSiteId.ARTSTATION_PAGE: WikidataInfo(None, 65551500),
    #SocialSiteId.INPRNT_PAGE: WikidataInfo(None, None),

    SocialSiteId.CARRD_PAGE: WikidataInfo(None, 106036503),
    SocialSiteId.YOUTUBE_CHANNEL_HANDLE: WikidataInfo(11245, 866, nickname_version_of = SocialSiteId.YOUTUBE_CHANNEL_ID),
    SocialSiteId.YOUTUBE_CHANNEL_ID: WikidataInfo(2397, 866, id_version_of = SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
    SocialSiteId.VIMEO_CHANNEL: WikidataInfo(4015, 156376),
    SocialSiteId.NEWGROUNDS_PAGE: WikidataInfo(None, 263655),
    SocialSiteId.ARTSY_ARTIST: WikidataInfo(2042, 4796642),
    SocialSiteId.ARTNET_ARTIST: WikidataInfo(3782, 266566),
    SocialSiteId.DEVIANT_ART_ACCOUNT: WikidataInfo(7737, None),
    SocialSiteId.DANBOORU_ARTIST: WikidataInfo(None, 64514853),
    SocialSiteId.BANDCAMP_PROFILE: WikidataInfo(3283, 545966),
    SocialSiteId.BLUESKY_PROFILE: WikidataInfo(None, 78194383),
    SocialSiteId.ITCH_IO_DEVELOPER: WikidataInfo(8176, 22905933),
}

def re_social_subdomain(main_domain):
    #return r'^(?:https?:\/\/)?([\w_-]+)\.'+re.escape(main_domain)+'\/?$'
    return r'^(?:https?:\/\/)?([\w_-]+)\.'+re.escape(main_domain)+'(\/.*)?$'

RE_ID = r'@?([^/]+)'
RE_ANY_SUBPATH = r'(|\/|\/.*)$'

def re_social_path(main_domain):
    #return r'^(?:https?:\/\/)?(?:www\.)?'+re.escape(main_domain)+'\/'+RE_ID+'\/?$'
    return re_social_path_adv(main_domain, RE_ID)

def re_social_path_adv(main_domain, *path):
    assert not main_domain.startswith('www.'), 'Redundant www.'
    l = [r'^', '(?:https?:\/\/)?', r'(?:www\.)?', re.escape(main_domain)]

    for p in path:
        if p != RE_ANY_SUBPATH:
            l.append(r'\/')
        l.append(p if p in {RE_ID, RE_ANY_SUBPATH} else re.escape(p))
    if path[-1] != RE_ANY_SUBPATH:
        l.append(r'\/?$')
    regex = ''.join(l)
    return regex

MAILTO_URL = r'^mailto:(?:[\w._.]+@[\w._.]+)$'

REDDIT_SUBREDDIT_URL = r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/r\/([\w-]+)\/?$'
REDDIT_USER_URL = r'^(?:https?:\/\/)?(?:old\.)?reddit\.com\/user\/([\w-]+)(?:|\/submitted)\/?$'
TWITTER_HANDLE_URL_1 =re_social_path('twitter.com')
TWITTER_HANDLE_URL_2 =re_social_path('x.com')
LINKTREE_PAGE_URL = re_social_path('linktr.ee')
TWITCH_STREAM_URL = re_social_path('twitch.tv')
WIKIDATA_ITEM_URL = re_social_path_adv('wikidata.org', 'wiki', RE_ID)
SONGKICK_ARTIST_URL = r'^(?:https?:\/\/)?(?:www\.)?songkick\.com\/artists\/(\d+)([\w-]*)\/?$'
TUMBLR_PAGE_URL = re_social_path('tumblr.com')
TUMBLR_PAGE_URL_2 = re_social_subdomain('tumblr.com')
INSTAGRAM_URL = re_social_path('instagram.com')
PATREON_URL = re_social_path_adv('patreon.com', RE_ID, RE_ANY_SUBPATH)
ARTSTATION_URL = re_social_path('artstation.com')
INPRNT_URL = re_social_path_adv('inprnt.com', 'gallery', RE_ID)
FACEBOOK_PAGE_URL = re_social_path('facebook.com')
SUBSTACK_PREFIX_URL = re_social_subdomain('substack.com')
ETSY_SHOP_URL = re_social_path_adv('etsy.com', 'shop', RE_ID)
BEHANCE_PAGE_URL = re_social_path('behance.net')
TIKTOK_USER_URL = re_social_path('tiktok.com')
PIXIV_USER_ID_URL = r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/users/(\d+)\/?$'
PIXIV_USER_ID_URL_2 = r'^(?:https?:\/\/)?(?:www\.)?pixiv\.net(?:\/en)?\/member\.php\/?[?]id=(\d+)$'
PIXIV_FANBOX_USER_NICKNAME_URL = re_social_subdomain('fanbox.cc')
PIXIV_USER_NICKNAME_URL = re_social_path_adv('pixiv.net', 'stacc', RE_ID)
PIXIV_SKETCH_USER_NICKNAME_URL = re_social_path_adv('sketch.pixiv.net', RE_ID)

URL_PARSE_CARRD_PAGE = re_social_subdomain('carrd.co')
URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1= re_social_path_adv('youtube.com', RE_ID)
URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2= re_social_path_adv('youtube.com', 'c', RE_ID)
URL_PARSE_YOUTUBE_CHANNEL_ID= re_social_path_adv('youtube.com', 'channel', RE_ID)
URL_PARSE_VIMEO_CHANNEL= re_social_path_adv('vimeo.com', RE_ID)
URL_PARSE_NEWGROUNDS_PAGE = re_social_subdomain('newgrounds.com')
URL_PARSE_ARTSY_ARTIST = re_social_path_adv('artsy.net', 'artist', RE_ID, RE_ANY_SUBPATH)
URL_PARSE_ARTNET_ARTIST = re_social_path_adv('artnet.com', 'artists', RE_ID, RE_ANY_SUBPATH)
URL_PARSE_DEVIANT_ART_ACCOUNT = re_social_path_adv('deviantart.com', RE_ID)
URL_PARSE_DEVIANT_ART_ACCOUNT_2 = re_social_subdomain('deviantart.com')
URL_PARSE_DANBOORU_ARTIST = re_social_path_adv('danbooru.donmai.us', 'artists', RE_ID)
URL_PARSE_BANDCAMP = re_social_subdomain('bandcamp.com')
URL_PARSE_BLUESKY = re_social_path_adv('bsky.app', 'profile', RE_ID)
URL_PARSE_ITCH_IO_DEVELOPER = re_social_subdomain('itch.io')

REGEXES = [
    # Reddit
    (REDDIT_SUBREDDIT_URL, SocialSiteId.REDDIT_SUBREDDIT),
    (REDDIT_USER_URL, SocialSiteId.REDDIT_USER),

    # Twitter
    (TWITTER_HANDLE_URL_1, SocialSiteId.TWITTER),
    (TWITTER_HANDLE_URL_2, SocialSiteId.TWITTER),

    # Facebook
    (FACEBOOK_PAGE_URL, SocialSiteId.FACEBOOK_PAGE),

    # Linktr.ee
    (LINKTREE_PAGE_URL, SocialSiteId.LINKTREE_PAGE),

    # Twitch.tv
    (TWITCH_STREAM_URL, SocialSiteId.TWITCH),

    # Wikidata
    (WIKIDATA_ITEM_URL, SocialSiteId.WIKIDATA),

    # Songkick
    (SONGKICK_ARTIST_URL, SocialSiteId.SONGKICK_ARTIST),

    # Tumblr
    (TUMBLR_PAGE_URL, SocialSiteId.TUMBLR),
    (TUMBLR_PAGE_URL_2, SocialSiteId.TUMBLR),

    # Instagram
    (INSTAGRAM_URL, SocialSiteId.INSTAGRAM_PAGE),

    # Tiktok
    (TIKTOK_USER_URL, SocialSiteId.TIKTOK_USER),

    # Pixiv
    (PIXIV_USER_ID_URL, SocialSiteId.PIXIV_USER_ID),
    (PIXIV_USER_ID_URL_2, SocialSiteId.PIXIV_USER_ID),
    (PIXIV_FANBOX_USER_NICKNAME_URL , SocialSiteId.PIXIV_USER_NICKNAME),
    (PIXIV_USER_NICKNAME_URL , SocialSiteId.PIXIV_USER_NICKNAME),
    (PIXIV_SKETCH_USER_NICKNAME_URL, SocialSiteId.PIXIV_USER_NICKNAME),

    # Patreon
    (PATREON_URL, SocialSiteId.PATREON_PAGE),

    # Artstation
    (ARTSTATION_URL, SocialSiteId.ARTSTATION_PAGE),

    # Inprnt
    (INPRNT_URL, SocialSiteId.INPRNT_PAGE),

    # Email
    (MAILTO_URL, SocialSiteId.EMAIL),

    # Substack
    (SUBSTACK_PREFIX_URL, SocialSiteId.SUBSTACK),

    # Etsy shop
    (ETSY_SHOP_URL, SocialSiteId.ETSY_SHOP),

    # Ko-fi
    (re_social_path_adv('ko-fi.com', RE_ID), SocialSiteId.KO_FI),
    (re_social_path_adv('ko-fi.com', RE_ID, 'shop'), SocialSiteId.KO_FI),

    # Behance
    (BEHANCE_PAGE_URL, SocialSiteId.BEHANCE_PAGE),

    # Carrd
    (URL_PARSE_CARRD_PAGE, SocialSiteId.CARRD_PAGE),

    # Youtube
    (URL_PARSE_YOUTUBE_CHANNEL_HANDLE_1, SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
    (URL_PARSE_YOUTUBE_CHANNEL_HANDLE_2, SocialSiteId.YOUTUBE_CHANNEL_HANDLE),
    (URL_PARSE_YOUTUBE_CHANNEL_ID, SocialSiteId.YOUTUBE_CHANNEL_ID),

    # Vimeo
    (URL_PARSE_VIMEO_CHANNEL, SocialSiteId.VIMEO_CHANNEL),

    # Newgrounds
    (URL_PARSE_NEWGROUNDS_PAGE, SocialSiteId.NEWGROUNDS_PAGE),

    # Artsy
    (URL_PARSE_ARTSY_ARTIST, SocialSiteId.ARTSY_ARTIST),
    (URL_PARSE_ARTNET_ARTIST, SocialSiteId.ARTNET_ARTIST),

    # Deviant art
    (URL_PARSE_DEVIANT_ART_ACCOUNT, SocialSiteId.DEVIANT_ART_ACCOUNT),
    (URL_PARSE_DEVIANT_ART_ACCOUNT_2, SocialSiteId.DEVIANT_ART_ACCOUNT),

    # Danbooru
    (URL_PARSE_DANBOORU_ARTIST, SocialSiteId.DANBOORU_ARTIST),

    # Bandcamp
    (URL_PARSE_BANDCAMP, SocialSiteId.BANDCAMP_PROFILE),

    # Bluesky
    (URL_PARSE_BLUESKY, SocialSiteId.BLUESKY_PROFILE),

    # Itch.io
    (URL_PARSE_ITCH_IO_DEVELOPER, SocialSiteId.ITCH_IO_DEVELOPER),
]

WELL_KNOWN_MASTODON_INSTANCES = frozenset({
    # Includes all servers with 50 000+ users as of 6 july 2023.
    # based on https://mastodonservers.net/servers/top
    'mastodon.social',
    #'pawoo.net',
    'baraag.net',
    'mstdn.jp',
    'mastodon.cloud',
    'mstdn.social',
    'mastodon.online',
    'mas.to',
    'mastodon.world',
    'mastodon.lol',
    'mastodon.sdf.org',
    'c.im',
    'mastodon.uno',
    'mastodonapp.uk',
    'fosstodon.org',
})

def determine_social_from_url_internally(url: str):
    assert isinstance(url, str)

    # Regexes
    for (social_site_url_regex, social_site_id) in REGEXES:
        if m := re.fullmatch(social_site_url_regex, url, re.I):
            groups = m.groups()
            return (social_site_id, groups[0] if len(groups) > 0 else None)

    # Mastodon
    for mastodon_hostname in WELL_KNOWN_MASTODON_INSTANCES:
        if url.startswith('https://' + mastodon_hostname):
            return (SocialSiteId.MASTODON_PAGE, None)
    if 'mastodon' in url:
        return (SocialSiteId.MASTODON_PAGE, None)

    # Feed (?)
    elif 'feed' in url or 'xml' in url or 'rss' in url or 'atom' in url:
        return (SocialSiteId.RSS_FEED, None)

    return (None, None)

def determine_social_from_url(url):
    if isinstance(url, str):
        url = urllib.parse.urlparse(url)
    (social_site_id, social_id) = determine_social_from_url_internally(url._replace(query = '', fragment = '').geturl())
    if not social_site_id:
        (social_site_id, social_id) = determine_social_from_url_internally(url._replace(fragment = '').geturl())

    if not social_site_id:
        return None
    return SocialLink(url, social_site_id, social_id)

def run_tests():
    assert determine_social_from_url('http://www.twitter.com/dril').social_id == 'dril'
    assert determine_social_from_url('http://worstdril.tumblr.com/')
    assert determine_social_from_url('https://deep-dark-fears.tumblr.com').social_id == 'deep-dark-fears'
    assert determine_social_from_url('https://www.etsy.com/shop/aleksiremesart').social_id == 'aleksiremesart'
    assert determine_social_from_url('https://ko-fi.com/A627LI1/shop').social_id == 'A627LI1'
    assert determine_social_from_url('https://ko-fi.com/A627LI1/').social_id == 'A627LI1'
    assert determine_social_from_url('https://www.facebook.com/fredagscafeen.dk/').social_id == 'fredagscafeen.dk'
    assert determine_social_from_url('https://www.tiktok.com/@depthsofwikipedia?lang=en').social_id == 'depthsofwikipedia'
    assert determine_social_from_url('https://www.pixiv.net/users/14866303').social_id == '14866303'
    assert determine_social_from_url('https://www.pixiv.net/member.php?id=109710').social_id == '109710'
    assert determine_social_from_url('https://www.deviantart.com/solquiet').social_site_id == SocialSiteId.DEVIANT_ART_ACCOUNT
    assert determine_social_from_url('https://solquiet.deviantart.com/').social_site_id == SocialSiteId.DEVIANT_ART_ACCOUNT

    INSTAGRAMS = [
        'https://instagram.com/_richardparry_',
        'https://instagram.com/j_kmor/',
        'https://instagram.com/cullensartbox/',
        'https://www.instagram.com/timkongart/',
        'https://www.instagram.com/kcn.wu/',
        'https://www.instagram.com/itsbettyjiang',
    ]

    for ig in INSTAGRAMS:
        assert determine_social_from_url(ig).social_site_id == SocialSiteId.INSTAGRAM_PAGE

TEST = True
if TEST:
    run_tests()