import dataclasses import datetime import email.utils import json import logging from collections.abc import Iterator, Mapping from decimal import Decimal import requests from frozendict import frozendict from personal_data.data import DeduplicateMode, Scraper from .. import secrets logger = logging.getLogger(__name__) # mainnet: https://reader.partisiablockchain.com # testnet: https://node1.testnet.partisiablockchain.com HOSTNAME = 'reader.partisiablockchain.com' URL_ACCOUNT_PLUGIN = 'https://{hostname}/{shard}blockchain/accountPlugin/local' URL_ACCOUNT_PLUGIN_GLOBAL = 'https://{hostname}/{shard}blockchain/accountPlugin/global' URL_CONTRACT_STATE = 'https://{hostname}/{shard}blockchain/contracts/{address}?requireContractState=false' MPC_DECIMALS = 10000 def shard_id_for_address(address: str) -> str: if address.endswith('a'): return 'shards/Shard0/' # TODO elif address.endswith('2'): return 'shards/Shard1/' # TODO else: return 'shards/Shard2/' # TODO @dataclasses.dataclass(frozen=True) class Balances: update_time: datetime.datetime balances: Mapping[str, Decimal] @dataclasses.dataclass(frozen=True) class PbcClient: session: requests.Session def get_json( self, url: str, data: Mapping[str, str] = frozendict(), method='POST', ) -> tuple[dict, datetime.datetime]: headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', } response = self.session.request( method, url, headers=headers, data=json.dumps(data), ) response.raise_for_status() date_text = response.headers.get('last-modified') or response.headers.get( 'date', ) date = email.utils.parsedate_to_datetime(date_text) json_data = response.json() if json_data is None: msg = 'No result data for ' + url raise Exception(msg) return (json_data, date) def determine_coins(self) -> list[dict[str, str]]: data: dict = {'path': []} url = URL_ACCOUNT_PLUGIN_GLOBAL.format( hostname=HOSTNAME, shard='', ) json_data, date = self.get_json(url, data=data) return json_data['coins']['coins'] def get_account_balances(self, address: str) -> Balances: coins = self.determine_coins() url = URL_ACCOUNT_PLUGIN.format( hostname=HOSTNAME, shard=shard_id_for_address(address), ) data: dict = { 'path': [ {'type': 'field', 'name': 'accounts'}, {'type': 'avl', 'keyType': 'BLOCKCHAIN_ADDRESS', 'key': address}, ], } account_data, date = self.get_json(url, data=data) balances: dict[str, Decimal] = {} balances['MPC'] = Decimal(account_data['mpcTokens']) / MPC_DECIMALS for coin_idx, amount_data in enumerate(account_data['accountCoins']): coin_data = coins[coin_idx] byoc_balance = Decimal(amount_data['balance']) denominator = Decimal(coin_data['conversionRate']['denominator']) native_balance = byoc_balance / denominator balances[coin_data['symbol']] = native_balance del coin_idx, coin_data return Balances(date, balances) def get_contract_state(self, address: str) -> tuple[dict, datetime.datetime]: url = URL_CONTRACT_STATE.format( hostname=HOSTNAME, shard=shard_id_for_address(address), address=address, ) data: dict = {'path': []} return self.get_json(url, data=data) @dataclasses.dataclass(frozen=True) class MpcBalance(Scraper): dataset_name = 'defi_mpc_balance' deduplicate_mode = DeduplicateMode.ONLY_LATEST deduplicate_ignore_columns = ['account.update_time'] def scrape_balances_for(self, address: str) -> frozendict[str, object]: client = PbcClient(self.session) balances = client.get_account_balances(address) data_point = { 'account.address': address, 'account.update_time': balances.update_time, } for token, amount in balances.balances.items(): data_point['balance.' + token] = amount del token, amount return frozendict(data_point) def scrape(self) -> Iterator[Mapping[str, object]]: yield self.scrape_balances_for(secrets.PBC_ACCOUNT_ADDRESS) PBC_FOUNDATION_CONTRACT_ADDRESSES = [ ('012635f1c0a9bffd59853c6496e1c26ebda0e2b4da', 'Foundation Sales'), ('0135edec2c9fed33f45cf2538dc06ba139c4bb8f62', 'Foundation Team'), ('01ad44bb0277a8df16408006c375a6fa015bb22c97', 'Foundation Eco-System'), ] @dataclasses.dataclass(frozen=True) class PbcFoundationBalance(Scraper): dataset_name = 'pbc_foundation_balances' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS deduplicate_ignore_columns = [ 'contract.update_time', 'contract.name', 'contract.state.balance', ] def scrape(self) -> Iterator[Mapping[str, object]]: client = PbcClient(self.session) for address, contract_name in PBC_FOUNDATION_CONTRACT_ADDRESSES: contract_state, update_time = client.get_contract_state(address) yield { 'contract.update_time': update_time, 'contract.name': contract_name, 'contract.address': address, 'contract.state.nonce': contract_state['nonce'], 'contract.state.balance': contract_state['remainingTokens'], }