Jmaa/personal-data
1
0
Fork 0
Code Actions Packages

BYOC

Browse Source
This commit is contained in:
Jon Michael Aanes 2024-04-17 00:02:51 +02:00
parent 2a6cd4b9f3
commit 614957c465
Signed by: Jmaa
SSH Key Fingerprint: SHA256:Ab0GfHGCblESJx7JRE4fj4bFy/KRpeLhi41y4pF3sNA
1 changed files with 37 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
personal_data/fetchers/partisia_blockchain.py
View File

@ -4,6 +4,7 @@ import logging
import re import re
import json import json
import secrets import secrets
from decimal import Decimal
import bs4 import bs4
@ -28,8 +29,32 @@ class MpcBalance(Scraper):
dataset_name = 'defi_mpc_balance' dataset_name = 'defi_mpc_balance'
deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS deduplicate_mode = DeduplicateMode.BY_ALL_COLUMNS
def get_json(self, url: str, data: dict) -> tuple[dict,str]:
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
response = self.session.post(url, headers = headers, data=json.dumps(data))
response.raise_for_status()
date = response.headers.get('last-modified') or response.headers.get('date')
return response.json(), date
def determine_coins(self) -> list[dict]:
data: dict = {'path':[]}
url = URL_ACCOUNT_PLUGIN_GLOBAL.format(
hostname = HOSTNAME,
shard = '',
#shard = 'shards/Shard0/',
)
json_data, date = self.get_json(url, data=data)
return json_data['coins']['coins']
def scrape(self): def scrape(self):
address = '0019e9a28c978dd65114cc4e0bcb876770805b0349' # TODO address = '0019e9a28c978dd65114cc4e0bcb876770805b0349' # TODO
coins = self.determine_coins()
headers = { headers = {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@ -41,20 +66,22 @@ class MpcBalance(Scraper):
shard = 'shards/Shard0/', shard = 'shards/Shard0/',
) )
data = {'path':[{'type':'field','name':'accounts'},{'type':'avl','keyType':'BLOCKCHAIN_ADDRESS','key':address}]} data: dict = {'path':[{'type':'field','name':'accounts'},{'type':'avl','keyType':'BLOCKCHAIN_ADDRESS','key':address}]}
response = self.session.post(url, headers = headers, data=json.dumps(data)) account_data, date = self.get_json(url, data=data)
response.raise_for_status()
print(response.headers)
json_data = response.json()
print(json_data )
data_point = { data_point = {
'account.address': address, 'account.address': address,
'account.update_time': response.headers.get('last-modified') or response.headers.get('date'), 'account.update_time': date,
} }
data_point['balance.MPC'] = json_data['mpcTokens'] data_point['balance.MPC'] = account_data['mpcTokens']
print(data_point)
for coin_idx, amount_data in enumerate(account_data['accountCoins']):
coin_data = coins[coin_idx]
byoc_balance = Decimal(amount_data ['balance'])
denominator = Decimal(coin_data['conversionRate']['denominator'])
native_balance = byoc_balance / denominator
data_point['balance.'+coin_data['symbol']] = native_balance
del coin_idx, coin_data
yield data_point yield data_point