Merge pull request #16 from gnolang/dev/moul/new

WIP: manfred-recheck
gnolang · Nov 4, 2022 · 9258d25 · 9258d25
2 parents d964194 + d40047a
commit 9258d25
Show file tree

Hide file tree

Showing 10 changed files with 365 additions and 0 deletions.
diff --git a/manfred-recheck/.gitignore b/manfred-recheck/.gitignore
@@ -0,0 +1,4 @@
+*.csv
+*.json
+db.sqlite
+account_numbers.txt
diff --git a/manfred-recheck/Makefile b/manfred-recheck/Makefile
@@ -0,0 +1,114 @@
+HEIGHT = 10562840
+EXPORT = snapshot/cosmos_$(HEIGHT)_export.json
+FOCUSES = auth.accounts authz.authorization bank.balances bank.supply \
+	capability.owners distribution feegrant.allowances \
+	gov.proposals ibc.channel_genesis ibc.client_genesis ibc.connection_genesis \
+	interchainaccounts liquidity slashing staking.delegations staking.redelegations \
+	staking.unbonding_delegations staking.validators transfer
+JSON_FOCUSES = $(addprefix snapshot/,$(addsuffix .json,$(FOCUSES)))
+SUMMARIES = summaries/account_numbers.txt summaries/uatom_holders.csv \
+	summaries/uatom_sum_and_count.txt summaries/delegations.csv \
+	summaries/delegations_grouped.csv summaries/delegations_sum_and_count.txt \
+	summaries/undelegations.csv summaries/undelegations_grouped.csv \
+	summaries/undelegations_sum_and_count.txt
+EXPORTS = exports/top10000.csv
+
+all: base summaries sums build-sqlite
+
+##
+## Sqlite
+##
+
+build-sqlite: summaries
+	./build-sqlite.sh
+.PHONY: build-sqlite
+
+run-sqlite: db.sqlite
+	sqlite3 $<
+.PHONY: run-sqlite
+
+$(EXPORTS):
+	@make export-sqlite
+export-sqlite: db.sqlite
+	./export-sqlite.sh
+.PHONY: compute-sqlite
+
+dump-sqlite: db.sqlite
+	echo ".headers on\n.mode csv\nSELECT * FROM accounts ORDER BY cummulative_atoms DESC" | sqlite3 $< > dump.csv
+.PHONY: dump-sqlite
+
+db.sqlite: skip.csv build-sqlite.sh
+	@make build-sqlite
+
+
+##
+## Summaries
+##
+
+sums: shasums.txt
+.PHONY: sums
+shasums.txt: $(JSON_FOCUSES) $(SUMMARIES) $(EXPORTS)
+	sha1sum */*.* | tee shasums.txt
+
+summaries: $(SUMMARIES)
+.PHONY: summaries
+
+summaries/undelegations_sum_and_count.txt: summaries/undelegations_grouped.csv
+	cat summaries/undelegations_grouped.csv | gawk -F',' '{ sum += $$2 } END{ printf "%d %d\n", sum, NR }' | tee $@
+
+summaries/undelegations_grouped.csv: summaries/undelegations.csv
+	cat summaries/undelegations.csv | gawk -M -F',' 'NR == 1 {next} {a[$$1] += $$2} {b[$$1] += 1} END {for (i in a) {if (a[i]>=1) {printf "%s %d %d\n", i, a[i], b[i]}}}' | sort -rnk2 | tr " " "," > $@
+	wc -l $@
+
+summaries/undelegations.csv: snapshot/staking.unbonding_delegations.json
+	cat snapshot/staking.unbonding_delegations.json | jq -r '.[] | .delegator_address + "," + (.entries[].balance)' | sort > $@
+	wc -l $@
+
+summaries/delegations_sum_and_count.txt: summaries/delegations_grouped.csv
+	cat summaries/delegations_grouped.csv | gawk -F',' '{ sum += $$2 } END{ printf "%d %d\n", sum, NR }' | tee $@
+
+summaries/delegations_grouped.csv: summaries/delegations.csv
+	cat summaries/delegations.csv  | gawk -M -F',' 'NR == 1 {next} {a[$$1] += $$2} {b[$$1] += 1} END {for (i in a) {if (a[i]>=1) {printf "%s %d %d\n", i, a[i], b[i]}}}' | sort -rnk2 | tr " " "," > $@
+	wc -l $@
+
+summaries/delegations.csv: snapshot/staking.delegations.json
+	cat snapshot/staking.delegations.json  | jq -r '.[] | .delegator_address + "," + .shares' | sort > $@
+	wc -l $@
+
+summaries/uatom_sum_and_count.txt: summaries/uatom_holders.csv
+	cat summaries/uatom_holders.csv | gawk -F',' '{ sum += $$2 } END{ print sum, NR }' | tee $@
+	wc -l $@
+
+summaries/uatom_holders.csv: snapshot/bank.balances.json
+	cat snapshot/bank.balances.json  | jq -r '.[] | select(.coins[].denom == "uatom") | .address + " " + (.coins[] | select(.denom == "uatom") | .amount)' | sort -rnk2 | tr " " "," > $@
+	wc -l $@
+
+summaries/account_numbers.txt: snapshot/auth.accounts.json
+	cat snapshot/auth.accounts.json | jq -r '.[].account_number' | sort -n > $@
+	wc -l $@
+
+##
+## Snapshot extracts
+##
+
+base: $(JSON_FOCUSES)
+.PHONY: base
+
+$(JSON_FOCUSES): $(EXPORT)
+	cat $< | jq .app_state.$(@:snapshot/%.json=%) > $@
+
+##
+## Raw Snapshot
+##
+
+$(EXPORT):
+	@echo "No such file: $(EXPORT)."
+	@echo
+	@echo "You need to either build it (make gen-export) or download it (make dl-export)."
+	@exit 1
+
+gen-export:
+	gaiad export --height=$(HEIGHT) 2>&1 | jq . | tee $(EXPORT)
+
+dl-export:
+	wget -O $(EXPORT) https://test1.gno.land/static/cosmos_10562840_export.json
diff --git a/manfred-recheck/README.md b/manfred-recheck/README.md
@@ -0,0 +1,3 @@
+# new scripts by @moul
+
+double-checking everything here.
diff --git a/manfred-recheck/build-sqlite.sh b/manfred-recheck/build-sqlite.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+
+rm -f db.sqlite
+
+sqlite3 db.sqlite <<EOF
+SELECT "----- CREATE TABLES";
+CREATE TABLE accounts (
+       address STRING PRIMARY KEY,
+       liquid_atoms DOUBLE DEFAULT 0,
+       staked_atoms DOUBLE DEFAULT 0,
+       unbounding_atoms DOUBLE DEFAULT 0,
+       tag STRING DEFAULT "",
+       comment STRING DEFAULT "",
+       cummulative_atoms DOUBLE
+);
+CREATE TABLE uatom_holders (address STRING PRIMARY KEY, quantity DOUBLE);
+CREATE TABLE delegations_grouped (address STRING PRIMARY KEY, quantity DOUBLE, count INTEGER);
+CREATE TABLE undelegations_grouped (address STRING PRIMARY KEY, quantity DOUBLE, count INTEGER);
+CREATE TABLE skips (address STRING PRIMARY KEY, tag STRING, reason STRING);
+.tables
+
+SELECT "----- IMPORT CSV FILES";
+.separator ,
+.import summaries/uatom_holders.csv uatom_holders
+.import summaries/delegations_grouped.csv delegations_grouped
+.import summaries/undelegations_grouped.csv undelegations_grouped
+.import skip.csv skips
+
+SELECT "----- CHECK CONTENT OF IMPORTED CSV TABLES";
+.mode column
+SELECT COUNT(*) FROM uatom_holders;
+SELECT * FROM uatom_holders LIMIT 1;
+SELECT COUNT(*) FROM delegations_grouped;
+SELECT * FROM delegations_grouped LIMIT 1;
+SELECT COUNT(*) FROM undelegations_grouped;
+SELECT * FROM undelegations_grouped LIMIT 1;
+SELECT COUNT(*) FROM undelegations_grouped;
+SELECT * FROM undelegations_grouped LIMIT 1;
+SELECT COUNT(*) FROM skips;
+SELECT * FROM skips LIMIT 1;
+
+SELECT "----- AGGREGATE TEMP TABLES";
+INSERT INTO accounts(address, liquid_atoms) \
+       SELECT address, quantity FROM uatom_holders;
+INSERT INTO accounts(address, staked_atoms) \
+       SELECT address, quantity FROM delegations_grouped WHERE true \
+       ON CONFLICT(address) DO \
+       UPDATE SET staked_atoms = excluded.staked_atoms;
+INSERT INTO accounts(address, unbounding_atoms) \
+       SELECT address, quantity FROM undelegations_grouped WHERE true \
+       ON CONFLICT(address) DO \
+       UPDATE SET staked_atoms = excluded.unbounding_atoms;
+INSERT INTO accounts(address, tag, comment) \
+       SELECT address, tag, reason FROM skips WHERE true \
+       ON CONFLICT(address) DO \
+       UPDATE SET tag = excluded.tag, comment = excluded.comment;
+UPDATE accounts SET liquid_atoms = liquid_atoms / 1000000, staked_atoms = staked_atoms / 1000000, unbounding_atoms = unbounding_atoms / 1000000;
+UPDATE accounts SET cummulative_atoms = (liquid_atoms + staked_atoms + unbounding_atoms);
+
+SELECT "----- CHECK AGGREGATED TABLE";
+.headers on
+SELECT COUNT(*), SUM(liquid_atoms), SUM(staked_atoms), SUM(unbounding_atoms), SUM(cummulative_atoms) FROM accounts;
+ SELECT * FROM accounts ORDER BY cummulative_atoms DESC LIMIT 10;
+.headers off
+
+SELECT "----- CLEANUP";
+DROP TABLE uatom_holders;
+DROP TABLE delegations_grouped;
+DROP TABLE undelegations_grouped;
+.tables
+EOF
diff --git a/manfred-recheck/export-sqlite.sh b/manfred-recheck/export-sqlite.sh
@@ -0,0 +1,127 @@
+#!/bin/sh
+
+mkdir -p exports
+
+sqlite3 db.sqlite > exports/top10000.csv <<EOF
+.headers on
+.mode csv
+SELECT * FROM accounts ORDER BY cummulative_atoms DESC LIMIT 10000;
+EOF
+
+sqlite3 db.sqlite > exports/top1000.csv <<EOF
+.headers on
+.mode csv
+SELECT * FROM accounts ORDER BY cummulative_atoms DESC LIMIT 1000;
+EOF
+
+
+echo "Examples of summaries with various filters."
+sqlite3 db.sqlite <<EOF
+.headers on
+.mode column
+.width 30
+SELECT
+
+       "nofilter" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms FROM accounts
+       ) UNION ALL SELECT
+
+       "only-stakers" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms FROM accounts WHERE staked_atoms > 0
+       ) UNION ALL SELECT
+
+       "only-staking" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(staked_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT staked_atoms FROM accounts WHERE staked_atoms > 0
+       ) UNION ALL SELECT
+
+       "atom>=20,whalecap-10k" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(capped) AS INTEGER) AS TOTAL FROM (
+            SELECT
+     CASE WHEN cummulative_atoms > 10000 THEN 10000 ELSE cummulative_atoms END as capped
+      FROM accounts
+      WHERE capped >= 20
+
+       ) UNION ALL SELECT
+       "atom>=20,whalecap-10k,filtered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(capped) AS INTEGER) AS TOTAL FROM (
+            SELECT
+     CASE WHEN cummulative_atoms > 10000 THEN 10000 ELSE cummulative_atoms END as capped
+      FROM accounts
+      WHERE capped >= 20
+      AND tag = ""
+
+       ) UNION ALL SELECT
+       "atom>=50,whalecap-10k" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(capped) AS INTEGER) AS TOTAL FROM (
+            SELECT
+     CASE WHEN cummulative_atoms > 10000 THEN 10000 ELSE cummulative_atoms END as capped
+      FROM accounts
+      WHERE capped >= 50
+
+       ) UNION ALL SELECT
+       "atom>=10,unfiltered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 10
+
+       ) UNION ALL SELECT
+       "atom>=10,filtered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 10
+      AND tag = ""
+
+       ) UNION ALL SELECT
+       "atom>=20,unfiltered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 20
+
+       ) UNION ALL SELECT
+       "atom>=20,filtered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 20
+      AND tag = ""
+
+       ) UNION ALL SELECT
+       "atom>=50,unfiltered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 50
+
+       ) UNION ALL SELECT
+       "atom>=50,filtered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 50
+      AND tag = ""
+
+       ) UNION ALL SELECT
+       "atom>=100,unfiltered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 100
+
+       ) UNION ALL SELECT
+       "atom>=100,filtered" as TITLE,
+       COUNT(*) as "PEOPLE", CAST(SUM(cummulative_atoms) AS INTEGER) AS TOTAL FROM (
+            SELECT cummulative_atoms
+      FROM accounts
+      WHERE cummulative_atoms >= 100
+      AND tag = ""
+
+       );
+EOF
+
+echo "Done."
diff --git a/manfred-recheck/shasums.txt b/manfred-recheck/shasums.txt
@@ -0,0 +1,43 @@
+a3c0a0b37934ef04670cf6d6819165607f8e6e90  exports/top1000.csv
+2db9d6d12bb0cd335871f283b2dd01ef85b0634f  exports/top10000.csv
+0a868283e4365ed7ebbfa44a4ac0b28194f3c240  snapshot/auth.accounts.json
+0d1942c61673d0de5aa865b34742722e8fccba35  snapshot/auth.params.json
+cbd4847de79d6bb13ce3504f9fcfcebcdc1cf6a2  snapshot/authz.authorization.json
+ab9236e3183aaec6e7884b12d13c285c3fb484d4  snapshot/bank.balances.json
+3dbf49d8f7db000ac1725da7324157304ac3b089  snapshot/bank.denom_metadata.json
+69c0a4d2ab935c44b7d98e3858cc0c40b87ba785  snapshot/bank.params.json
+180f0d5e22991eaa99268abdf6f378f739f6b27a  snapshot/bank.supply.json
+c7941e3faae696c2aab152cbba544126d0c32b57  snapshot/capability.owners.json
+63702b4da3a52581333374c95daef4d9549c7e84  snapshot/cosmos_10562840_export.json
+87ff8ff82f03ad183883b481e9551e8942eff49b  snapshot/crisis.constant_fee.json
+cf911ce485506dc9f88d7708f139951a998e7b08  snapshot/distribution.json
+cd0d4cc32346750408f7d4f5e78ec9a6e5b79a0d  snapshot/evidence.evidence.json
+9aefc8b87303f6d28194fc87adc03893bf1a6829  snapshot/feegrant.allowances.json
+cd0d4cc32346750408f7d4f5e78ec9a6e5b79a0d  snapshot/genutil.gen_txs.json
+50868ff85682c8dc96249662e102d7a2638f9cd0  snapshot/gov.deposit_params.json
+cd0d4cc32346750408f7d4f5e78ec9a6e5b79a0d  snapshot/gov.deposits.json
+f73bcad28bde4f10b5b4a2b6adda0d15c5d6b1e1  snapshot/gov.proposals.json
+cd0d4cc32346750408f7d4f5e78ec9a6e5b79a0d  snapshot/gov.votes.json
+8c770bff4ea6cf85d63533d91818fe967770a074  snapshot/ibc.channel_genesis.json
+3a5d7bbff959141432f5a76a3ee3c571cff64c74  snapshot/ibc.client_genesis.json
+a45be6d354f92aa9d2dc805647e0cbf8b4fc90ae  snapshot/ibc.connection_genesis.json
+8433190b72eb99c9705b724feea7bd510f2b11a8  snapshot/interchainaccounts.json
+2961042811afeeefeaf226a47aba82722bf3cba0  snapshot/liquidity.json
+8701422969f53dab60f1871b53bb28aeda421185  snapshot/mint.json
+d910b02871075d3156ec8675dfc95b7d5d640aa6  snapshot/packetforwardmiddleware.json
+ba1652ca72943ab72f8ba95cec07a248163fa82d  snapshot/slashing.json
+15c5be633406d03b75f2d4b3b43d1ca6e92d0f4e  snapshot/staking.delegations.json
+277bb61d533c63bce951b5e3a64339a1ae1445f9  snapshot/staking.json
+147de35631dbc761ef3a5e67be556641c7b0148d  snapshot/staking.redelegations.json
+42aa929e39cb5eeb5d61410d78605d79a9d7d1d7  snapshot/staking.unbonding_delegations.json
+4906bf2163bd2487fd46e7ff98fa694b6d0742ba  snapshot/staking.validators.json
+ab859d58e19d74ae4c877a4a92ecff6459b344e6  snapshot/transfer.json
+6c2da7ef70cbf48844fa448272fcc7c062f42026  summaries/account_numbers.txt
+528bb155fe633f4571c0e42f158c0a8819b25185  summaries/delegations.csv
+9c7d4f2cd85c02ec7aa1b1768fd02bcff0ab781b  summaries/delegations_grouped.csv
+e426171bd36d944e6180167974640e9bbe2e3197  summaries/delegations_sum_and_count.txt
+906e263a04377209a0f7088b0820c6526f8e09b1  summaries/uatom_holders.csv
+c2e11002890c5353b4f95e6680218cdf462ee1fb  summaries/uatom_sum_and_count.txt
+e19e81120407ef0ea04ba8da940862ede211afc5  summaries/undelegations.csv
+a065008d231f09164cfadf87453dca12460d667f  summaries/undelegations_grouped.csv
+8e797208157c06f08caed031acb4c2705504e681  summaries/undelegations_sum_and_count.txt
diff --git a/manfred-recheck/snapshot/.gitkeep b/manfred-recheck/snapshot/.gitkeep
diff --git a/manfred-recheck/summaries/delegations_sum_and_count.txt b/manfred-recheck/summaries/delegations_sum_and_count.txt
@@ -0,0 +1 @@
+185158999093913 397417
diff --git a/manfred-recheck/summaries/uatom_sum_and_count.txt b/manfred-recheck/summaries/uatom_sum_and_count.txt
@@ -0,0 +1 @@
+296053688207917 597167
diff --git a/manfred-recheck/summaries/undelegations_sum_and_count.txt b/manfred-recheck/summaries/undelegations_sum_and_count.txt
@@ -0,0 +1 @@
+8724933810396 15138