personal-data/personal_data/fetchers/defi_partisia_blockchain.py

import dataclasses
import datetime
import email.utils
import json
import logging
from collections.abc import Iterator, Mapping
from decimal import Decimal

import enforce_typing
import requests
from frozendict import frozendict

from personal_data.data import DeduplicateMode, Scraper

from .. import secrets

logger = logging.getLogger(__name__)


# mainnet: https://reader.partisiablockchain.com
# testnet: https://node1.testnet.partisiablockchain.com


HOSTNAME = 'reader.partisiablockchain.com'

URL_ACCOUNT_PLUGIN = 'https://{hostname}/{shard}blockchain/accountPlugin/local'
URL_ACCOUNT_PLUGIN_GLOBAL = 'https://{hostname}/{shard}blockchain/accountPlugin/global'
URL_CONTRACT_STATE = 'https://{hostname}/{shard}blockchain/contracts/{address}?requireContractState=false'


MPC_DECIMALS = 10000


def shard_id_for_address(address: str) -> str:
    if address.endswith('a'):
        return 'shards/Shard0/'  # TODO
    elif address.endswith('2'):
        return 'shards/Shard1/'  # TODO
    else:
        return 'shards/Shard2/'  # TODO


@enforce_typing.enforce_types
@dataclasses.dataclass(frozen=True)
class Balances:
    update_time: datetime.datetime
    balances: Mapping[str, Decimal]


@enforce_typing.enforce_types
@dataclasses.dataclass(frozen=True)
class PbcClient:
    session: requests.Session

    def get_json(
        self,
        url: str,
        data: Mapping[str, str] = frozendict(),
        method='POST',
    ) -> tuple[dict, datetime.datetime]:
        headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/json',
        }

        response = self.session.request(
            method,
            url,
            headers=headers,
            data=json.dumps(data),
        )
        response.raise_for_status()
        date_text = response.headers.get('last-modified') or response.headers.get(
            'date',
        )
        date = email.utils.parsedate_to_datetime(date_text)
        json_data = response.json()
        if json_data is None:
            msg = 'No result data for ' + url
            raise Exception(msg)
        return (json_data, date)

    def determine_coins(self) -> list[dict[str, str]]:
        data: dict = {'path': []}

        url = URL_ACCOUNT_PLUGIN_GLOBAL.format(
            hostname=HOSTNAME,
            shard='',
        )

        json_data, date = self.get_json(url, data=data)
        return json_data['coins']['coins']

    def get_account_balances(self, address: str) -> Balances:
        coins = self.determine_coins()

        url = URL_ACCOUNT_PLUGIN.format(
            hostname=HOSTNAME,
            shard=shard_id_for_address(address),
        )

        data: dict = {
            'path': [
                {'type': 'field', 'name': 'accounts'},
                {'type': 'avl', 'keyType': 'BLOCKCHAIN_ADDRESS', 'key': address},
            ],
        }
        account_data, date = self.get_json(url, data=data)

        balances: dict[str, Decimal] = {}
        balances['MPC'] = Decimal(account_data['mpcTokens']) / MPC_DECIMALS

        for coin_idx, amount_data in enumerate(account_data['accountCoins']):
            coin_data = coins[coin_idx]
            byoc_balance = Decimal(amount_data['balance'])
            denominator = Decimal(coin_data['conversionRate']['denominator'])
            native_balance = byoc_balance / denominator
            balances[coin_data['symbol']] = native_balance
            del coin_idx, coin_data

        return Balances(date, balances)

    def get_contract_state(self, address: str) -> tuple[dict, datetime.datetime]:
        url = URL_CONTRACT_STATE.format(
            hostname=HOSTNAME,
            shard=shard_id_for_address(address),
            address=address,
        )
        data: dict = {'path': []}
        return self.get_json(url, data=data)


@dataclasses.dataclass(frozen=True)
class MpcBalance(Scraper):
    dataset_name = 'defi_mpc_balance'
    deduplicate_mode = DeduplicateMode.ONLY_LATEST
    deduplicate_ignore_columns = ['account.update_time']

    def scrape_balances_for(self, address: str) -> frozendict[str, object]:
        client = PbcClient(self.session)
        balances = client.get_account_balances(address)

        data_point = {
            'account.address': address,
            'account.update_time': balances.update_time,
        }

        for token, amount in balances.balances.items():
            data_point['balance.' + token] = amount
            del token, amount

        return frozendict(data_point)

    def scrape(self) -> Iterator[Mapping[str, object]]:
        yield self.scrape_balances_for(secrets.PBC_ACCOUNT_ADDRESS)


PBC_FOUNDATION_CONTRACT_ADDRESSES = [
    ('012635f1c0a9bffd59853c6496e1c26ebda0e2b4da', 'Foundation Sales'),
    ('0135edec2c9fed33f45cf2538dc06ba139c4bb8f62', 'Foundation Team'),
    ('01ad44bb0277a8df16408006c375a6fa015bb22c97', 'Foundation Eco-System'),
]


@dataclasses.dataclass(frozen=True)
class PbcFoundationBalance(Scraper):
    dataset_name = 'pbc_foundation_balances'
    deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
    deduplicate_ignore_columns = [
        'contract.update_time',
        'contract.name',
        'contract.state.balance',
    ]

    def scrape(self) -> Iterator[Mapping[str, object]]:
        client = PbcClient(self.session)
        for address, contract_name in PBC_FOUNDATION_CONTRACT_ADDRESSES:
            contract_state, update_time = client.get_contract_state(address)
            yield {
                'contract.update_time': update_time,
                'contract.name': contract_name,
                'contract.address': address,
                'contract.state.nonce': contract_state['nonce'],
                'contract.state.balance': contract_state['remainingTokens'],
            }