From 614957c465dc9c7a47a0cf5a546c8150548b8045 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Wed, 17 Apr 2024 00:02:51 +0200 Subject: [PATCH] BYOC --- personal_data/fetchers/partisia_blockchain.py | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/personal_data/fetchers/partisia_blockchain.py b/personal_data/fetchers/partisia_blockchain.py index ce1513a..ee993ae 100644 --- a/personal_data/fetchers/partisia_blockchain.py +++ b/personal_data/fetchers/partisia_blockchain.py @@ -4,6 +4,7 @@ import logging import re import json import secrets +from decimal import Decimal import bs4 @@ -28,8 +29,32 @@ class MpcBalance(Scraper): dataset_name = 'defi_mpc_balance' deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS + def get_json(self, url: str, data: dict) -> tuple[dict,str]: + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + response = self.session.post(url, headers = headers, data=json.dumps(data)) + response.raise_for_status() + date = response.headers.get('last-modified') or response.headers.get('date') + return response.json(), date + + def determine_coins(self) -> list[dict]: + data: dict = {'path':[]} + + url = URL_ACCOUNT_PLUGIN_GLOBAL.format( + hostname = HOSTNAME, + shard = '', + #shard = 'shards/Shard0/', + ) + + json_data, date = self.get_json(url, data=data) + return json_data['coins']['coins'] + def scrape(self): address = '0019e9a28c978dd65114cc4e0bcb876770805b0349' # TODO + coins = self.determine_coins() headers = { 'Content-Type': 'application/json', @@ -41,20 +66,22 @@ class MpcBalance(Scraper): shard = 'shards/Shard0/', ) - data = {'path':[{'type':'field','name':'accounts'},{'type':'avl','keyType':'BLOCKCHAIN_ADDRESS','key':address}]} - response = self.session.post(url, headers = headers, data=json.dumps(data)) - response.raise_for_status() - print(response.headers) - - json_data = response.json() - print(json_data ) + data: dict = {'path':[{'type':'field','name':'accounts'},{'type':'avl','keyType':'BLOCKCHAIN_ADDRESS','key':address}]} + account_data, date = self.get_json(url, data=data) data_point = { 'account.address': address, - 'account.update_time': response.headers.get('last-modified') or response.headers.get('date'), + 'account.update_time': date, } - data_point['balance.MPC'] = json_data['mpcTokens'] - print(data_point) + data_point['balance.MPC'] = account_data['mpcTokens'] + + for coin_idx, amount_data in enumerate(account_data['accountCoins']): + coin_data = coins[coin_idx] + byoc_balance = Decimal(amount_data ['balance']) + denominator = Decimal(coin_data['conversionRate']['denominator']) + native_balance = byoc_balance / denominator + data_point['balance.'+coin_data['symbol']] = native_balance + del coin_idx, coin_data yield data_point