From bbf0c2b1bde018f8521cff12959fb88a4ccf4664 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Jun 2024 09:21:02 +0000 Subject: [PATCH] L1-218: Implemented finding the first block which contributed to total issuance imbalance (#1757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Implemented an algorithm that finds the first block hash that positively contributed to a total issuance imbalance. A positive contribution to the total issuance imbalance in block B is a situation in which the total issuance imbalance increases in block B. Total issuance imbalance is a difference between aggregated sum of total bolance over all accounts and balances.total_issuance storage value. There might be a difference in some value X in block B, and some value Y in the parent(B), and X > Y. There might be many such blocks in the chain [start_block_hash; end_block_hash], and this method returns the first one. The method uses a bisection algorithm. It computes mid-range block hash by computing ``` mid_block_number = floor((end_block_number - start_block_number) / 2) ``` Then, calculate the total_issuance imbalance in mid_block_number to start and end range total_issuance imbalance, adjusting the interval ends accordingly to the bisection algorithm. ## Type of change Please delete options that are not relevant. - New feature (non-breaking change which adds functionality) --------- Co-authored-by: Grzegorz Gawryał <31205678+ggawryal@users.noreply.github.com> --- .../accounts-invariants/chain_operations.py | 27 +--- .../pallet-balances-maintenance.py | 68 +++++---- scripts/accounts-invariants/total_issuance.py | 130 ++++++++++++++++++ scripts/accounts-invariants/utils.py | 18 +++ 4 files changed, 194 insertions(+), 49 deletions(-) create mode 100644 scripts/accounts-invariants/total_issuance.py create mode 100644 scripts/accounts-invariants/utils.py diff --git a/scripts/accounts-invariants/chain_operations.py b/scripts/accounts-invariants/chain_operations.py index 0e0a44981b..ddfced4299 100644 --- a/scripts/accounts-invariants/chain_operations.py +++ b/scripts/accounts-invariants/chain_operations.py @@ -1,10 +1,7 @@ -#!/bin/python3 - import substrateinterface -import json -import logging from tqdm import tqdm import sys +import logging log = logging.getLogger() @@ -32,23 +29,19 @@ def filter_accounts(chain_connection, page_size=1000, block_hash=block_hash) total_accounts_count = 0 - total_issuance = 0 for (i, (account_id, info)) in tqdm(iterable=enumerate(account_query), desc="Accounts checked", unit="", file=sys.stdout): total_accounts_count += 1 - free = info['data']['free'].value - reserved = info['data']['reserved'].value - total_issuance += free + reserved if check_accounts_predicate(info, chain_major_version, ed): accounts_that_do_meet_predicate.append([account_id.value, info.serialize()]) log.info( f"Total accounts that match given predicate {check_accounts_predicate_name} is {len(accounts_that_do_meet_predicate)}") log.info(f"Total accounts checked: {total_accounts_count}") - return accounts_that_do_meet_predicate, total_issuance + return accounts_that_do_meet_predicate def format_balance(chain_connection, amount): @@ -108,18 +101,4 @@ def get_all_accounts(chain_connection, block_hash=None): chain_major_version=None, check_accounts_predicate=lambda x, y, z: True, check_accounts_predicate_name="\'all accounts\'", - block_hash=block_hash)[0] - - -def save_accounts_to_json_file(json_file_name, accounts): - with open(json_file_name, 'w') as f: - json.dump(accounts, f) - log.info(f"Wrote file '{json_file_name}'") - - -def chunks(list_of_elements, n): - """ - Lazily split 'list_of_elements' into 'n'-sized chunks. - """ - for i in range(0, len(list_of_elements), n): - yield list_of_elements[i:i + n] + block_hash=block_hash) diff --git a/scripts/accounts-invariants/pallet-balances-maintenance.py b/scripts/accounts-invariants/pallet-balances-maintenance.py index b817ba262f..039803c8ad 100755 --- a/scripts/accounts-invariants/pallet-balances-maintenance.py +++ b/scripts/accounts-invariants/pallet-balances-maintenance.py @@ -2,6 +2,8 @@ from chain_operations import * from aleph_chain_version import * +from utils import * +from total_issuance import * import substrateinterface import argparse @@ -21,6 +23,7 @@ def get_args() -> argparse.Namespace: It has following functionality: * checking pallet balances and account reference counters invariants. * calling pallet operations for maintenance actions +* checking total issuance By default, it connects to a AlephZero Testnet and performs sanity checks only ie not changing state of the chain at all. Accounts that do not satisfy those checks are written to accounts-with-failed-invariants.json file. @@ -49,6 +52,15 @@ def get_args() -> argparse.Namespace: type=str, default='', help='Block hash from which this script should query state from. Default: chain tip.') + parser.add_argument('--start-range-block-hash', + type=str, + default='', + help='A block hash which denotes starting interval when searching for total issuance ' + 'imbalance. Must be present when --check-total-issuance is set.') + parser.add_argument('--check-total-issuance', + action='store_true', + help='Specify this switch if script should compare total issuance aggregated over all ' + 'accounts with StorageValue balances.total_issuance') group = parser.add_mutually_exclusive_group() group.add_argument('--fix-consumers-counter-underflow', action='store_true', @@ -268,9 +280,9 @@ def batch_fix_accounts_consumers_underflow(chain_connection, :return: None. Can raise exception in case of SubstrateRequestException thrown """ for (i, account_ids_chunk) in tqdm(iterable=enumerate(chunks(accounts, input_args.fix_consumers_calls_in_batch)), - desc="Accounts checked", - unit="", - file=sys.stdout): + desc="Accounts checked", + unit="", + file=sys.stdout): operations_calls = list(map(lambda account: chain_connection.compose_call( call_module='Operations', call_function='fix_accounts_consumers_underflow', @@ -296,7 +308,6 @@ def batch_fix_accounts_consumers_underflow(chain_connection, def perform_accounts_sanity_checks(chain_connection, ed, chain_major_version, - total_issuance_from_chain, block_hash=None): """ Checks whether all accounts on a chain matches pallet balances invariants @@ -305,26 +316,17 @@ def perform_accounts_sanity_checks(chain_connection, :param chain_major_version: enum ChainMajorVersion :return:None """ - invalid_accounts, total_issuance_from_accounts = \ - filter_accounts(chain_connection=chain_connection, - ed=ed, - chain_major_version=chain_major_version, - check_accounts_predicate=lambda x, y, z: not check_account_invariants(x, y, z), - check_accounts_predicate_name="\'incorrect account invariants\'", - block_hash=block_hash) + invalid_accounts = filter_accounts(chain_connection=chain_connection, + ed=ed, + chain_major_version=chain_major_version, + check_accounts_predicate=lambda x, y, z: not check_account_invariants(x, y, z), + check_accounts_predicate_name="\'incorrect account invariants\'", + block_hash=block_hash) if len(invalid_accounts) > 0: log.warning(f"Found {len(invalid_accounts)} accounts that do not meet balances invariants!") save_accounts_to_json_file("accounts-with-failed-invariants.json", invalid_accounts) else: log.info(f"All accounts on chain {chain_connection.chain} meet balances invariants.") - total_issuance_from_accounts_human = format_balance(chain_connection, total_issuance_from_accounts) - log.info(f"Total issuance computed from accounts: {total_issuance_from_accounts_human}") - if total_issuance_from_accounts != total_issuance_from_chain: - total_issuance_from_chain_human = format_balance(chain_connection, total_issuance_from_chain) - delta_human = format_balance(chain_connection, - total_issuance_from_chain - total_issuance_from_accounts) - log.warning(f"TotalIssuance from chain: {total_issuance_from_chain_human} is different from computed: " - f"{total_issuance_from_accounts_human}, delta: {delta_human}") if __name__ == "__main__": @@ -357,11 +359,6 @@ def perform_accounts_sanity_checks(chain_connection, f"13.2 version. Exiting.") exit(5) - total_issuance_from_chain = chain_ws_connection.query(module='Balances', - storage_function='TotalIssuance', - block_hash=state_block_hash).value - log.info(f"Chain total issuance is {format_balance(chain_ws_connection, total_issuance_from_chain)}") - existential_deposit = chain_ws_connection.get_constant(module_name="Balances", constant_name="ExistentialDeposit", block_hash=state_block_hash).value @@ -393,10 +390,31 @@ def perform_accounts_sanity_checks(chain_connection, input_args=args, sender_keypair=sender_origin_account_keypair, accounts=list(accounts_with_consumers_underflow_set)) + if args.check_total_issuance: + log.info(f"Comparing total issuance aggregated over all accounts with storage value balances.total_issuance") + total_issuance_from_chain, total_issuance_from_accounts = \ + get_total_issuance_imbalance(chain_connection=chain_ws_connection, + block_hash=state_block_hash) + log_total_issuance_imbalance(chain_connection=chain_ws_connection, + total_issuance_from_chain=total_issuance_from_chain, + total_issuance_from_accounts=total_issuance_from_accounts, + block_hash=state_block_hash) + delta = total_issuance_from_chain - total_issuance_from_accounts + if delta != 0: + if not args.start_range_block_hash: + log.error(f"--start-range-block-hash must be set when --check-total-issuance is set " + f"to perform further actions. Exiting.") + sys.exit(2) + log.warning(f"Total issuance retrieved from the chain storage is different than aggregated sum over" + f" all accounts. Finding first block when it happened.") + first_block_hash_imbalance = find_block_hash_with_imbalance(chain_connection=chain_ws_connection, + start_block_hash=args.start_range_block_hash, + end_block_hash=state_block_hash) + log.info(f"The first block where it happened is {first_block_hash_imbalance}") + log.info(f"Performing pallet balances sanity checks.") perform_accounts_sanity_checks(chain_connection=chain_ws_connection, ed=existential_deposit, chain_major_version=aleph_chain_version, - total_issuance_from_chain=total_issuance_from_chain, block_hash=state_block_hash) log.info(f"DONE") diff --git a/scripts/accounts-invariants/total_issuance.py b/scripts/accounts-invariants/total_issuance.py new file mode 100644 index 0000000000..d154f3ba04 --- /dev/null +++ b/scripts/accounts-invariants/total_issuance.py @@ -0,0 +1,130 @@ +import logging +import itertools +import functools + +from chain_operations import format_balance, get_all_accounts + +log = logging.getLogger() + + +def get_total_issuance_imbalance(chain_connection, block_hash): + """ + Compares total issuance computed from all accounts with balances.total_issuance storage + :param chain_connection: WS handler + :param block_hash: total issuance computed from all accounts + :return: delta between those two values + """ + total_issuance_from_chain = get_total_issuance_from_storage(chain_connection=chain_connection, + block_hash=block_hash) + all_accounts_and_infos = get_all_accounts(chain_connection, block_hash) + total_issuance_from_accounts = calculate_total_issuance( + map(lambda account_and_info: account_and_info[1], all_accounts_and_infos)) + return total_issuance_from_chain, total_issuance_from_accounts + + +def log_total_issuance_imbalance(chain_connection, + total_issuance_from_chain, + total_issuance_from_accounts, + block_hash): + """ + Logs imbalance data in a given block hash in a human-readable format. + :param chain_connection: WS handler + :param total_issuance_from_chain: balances.total_issuance storage value + :param total_issuance_from_accounts: total_issuance as sum aggregated over all accounts + :param block_hash: block hash from which above data was retrieved + :return: None + """ + total_issuance_from_accounts_human = format_balance(chain_connection, total_issuance_from_accounts) + total_issuance_from_chain_human = format_balance(chain_connection, total_issuance_from_chain) + delta = total_issuance_from_chain - total_issuance_from_accounts + delta_human = format_balance(chain_connection, delta) + log.info(f"Total issuance imbalance computed from block {block_hash}") + log.info( + f"balances.total_issuance storage value: {total_issuance_from_chain_human}") + log.info( + f"Total issuance computed as aggregated sum over all accounts: {total_issuance_from_accounts_human}") + log.info(f"Delta is: {delta_human}") + + +def calculate_total_issuance(account_infos): + """ + Calculates total issuance as sum over all accounts free + reserved funds + :param account_infos: A list AccountInfo structs + :return: total issuance as number + """ + + def get_account_total_balance(account_info): + free = account_info['data']['free'] + reserved = account_info['data']['reserved'] + return free + reserved + + return \ + functools.reduce(lambda x, account_info: x + get_account_total_balance(account_info), account_infos, 0) + + +def get_total_issuance_from_storage(chain_connection, block_hash): + """ + Retrieves balances.total_issuance StorageValue + :param chain_connection: WS handler + :param block_hash: A block hash to query state from + :return: total issuance as number + """ + total_issuance_from_chain = chain_connection.query(module='Balances', + storage_function='TotalIssuance', + block_hash=block_hash).value + return total_issuance_from_chain + + +def find_block_hash_with_imbalance(chain_connection, start_block_hash, end_block_hash): + """ + Finds a first block hash that positively contributed to a total issuance imbalance. + + Positive contribution to the total issuance imbalance in block B is a situation in which total issuance imbalance + increases in block B. Total issuance imbalance is a difference between aggregated sum of total bolance over all + accounts and balances.total_issuance storage value. It might happen that difference in some value X in block B, + and some value Y in parent(B), and X > Y. There might be many such blocks in chain [start_block_hash; end_block_hash] + and this method returns the first one. + + Method uses bisection algorithm. It computes mid-range block hash by computing + mid_block_number = floor((end_block_number - start_block_number) / 2) + and then calculating total_issuance imbalance in mid_block_number to start and end range total_issuance imbalance, + adjusting interval ends accordingly to bisection algorith. + + :param chain_connection: WS handler + :param start_block_hash: first block hash in range to check + :param end_block_hash: end block hash in range to check + :return: the first block_hash that contributed positively to total issuance imbalance + """ + start_block_number = chain_connection.get_block_number(start_block_hash) + end_block_number = chain_connection.get_block_number(end_block_hash) + + start_total_issuance_imbalance = get_total_issuance_imbalance(chain_connection, start_block_hash) + log_total_issuance_imbalance(chain_connection=chain_connection, + total_issuance_from_chain=start_total_issuance_imbalance[0], + total_issuance_from_accounts=start_total_issuance_imbalance[1], + block_hash=start_block_hash) + delta_start_imbalance = start_total_issuance_imbalance[0] - start_total_issuance_imbalance[1] + + while end_block_number - 1 > start_block_number: + log.info(f"Finding first block that contributed to total issuance imbalance in range " + f"[{start_block_number}; {end_block_number}]") + + mid_range_block_number = start_block_number + (end_block_number - start_block_number) // 2 + mid_range_block_hash = chain_connection.get_block_hash(mid_range_block_number) + log.info(f"Mid-range block hash: {mid_range_block_hash}, number: {mid_range_block_number}") + mid_total_issuance_imbalance = get_total_issuance_imbalance(chain_connection, mid_range_block_hash) + log_total_issuance_imbalance(chain_connection=chain_connection, + total_issuance_from_chain=mid_total_issuance_imbalance[0], + total_issuance_from_accounts=mid_total_issuance_imbalance[1], + block_hash=mid_range_block_hash) + + delta_mid_imbalance = mid_total_issuance_imbalance[0] - mid_total_issuance_imbalance[1] + if delta_mid_imbalance > delta_start_imbalance: + end_block_hash = mid_range_block_hash + end_block_number = chain_connection.get_block_number(end_block_hash) + else: + start_block_hash = mid_range_block_hash + start_block_number = chain_connection.get_block_number(start_block_hash) + delta_start_imbalance = delta_mid_imbalance + + return chain_connection.get_block_hash(end_block_number) diff --git a/scripts/accounts-invariants/utils.py b/scripts/accounts-invariants/utils.py new file mode 100644 index 0000000000..7ebc80940a --- /dev/null +++ b/scripts/accounts-invariants/utils.py @@ -0,0 +1,18 @@ +import json +import logging + +log = logging.getLogger() + + +def save_accounts_to_json_file(json_file_name, accounts): + with open(json_file_name, 'w') as f: + json.dump(accounts, f) + log.info(f"Wrote file '{json_file_name}'") + + +def chunks(list_of_elements, n): + """ + Lazily split 'list_of_elements' into 'n'-sized chunks. + """ + for i in range(0, len(list_of_elements), n): + yield list_of_elements[i:i + n]