From 6c25314c1daef52c45b16c667db2c39e940b37dc Mon Sep 17 00:00:00 2001 From: Greg Zaitsev Date: Wed, 24 Jul 2024 16:09:49 -0400 Subject: [PATCH 1/2] Add baedeker scripts that work with Polkadot 1.10+ --- .baedeker/.gitignore | 1 + .baedeker/forkless-data.jsonnet | 46 ++ .baedeker/rewrites.jsonnet | 11 + .baedeker/up.sh | 6 + .baedeker/vendor/baedeker-library/.gitignore | 0 .baedeker/vendor/baedeker-library/LICENSE | 21 + .../baedeker-library/inputs/base.libsonnet | 54 +++ .../baedeker-library/mixin/keys.libsonnet | 55 +++ .../baedeker-library/mixin/raw-spec.libsonnet | 233 ++++++++++ .../baedeker-library/mixin/spec.libsonnet | 438 ++++++++++++++++++ .../vendor/baedeker-library/ops/debug.ejs | 30 ++ .../baedeker-library/ops/devtools.libsonnet | 30 ++ .../baedeker-library/ops/nginx-dev.libsonnet | 19 + .../baedeker-library/ops/nginx.libsonnet | 98 ++++ .../baedeker-library/ops/rewrites.libsonnet | 13 + .../outputs/addressbook.libsonnet | 28 ++ .../outputs/compose.libsonnet | 177 +++++++ .../outputs/composediscover.libsonnet | 30 ++ .../baedeker-library/outputs/debug.libsonnet | 7 + .../util/genesisState.libsonnet | 51 ++ .../baedeker-library/util/meta.libsonnet | 51 ++ .../baedeker-library/util/mixin.libsonnet | 41 ++ docs/baedeker.md | 69 +++ scripts/localnet-baedeker.sh | 65 +++ 24 files changed, 1574 insertions(+) create mode 100644 .baedeker/.gitignore create mode 100644 .baedeker/forkless-data.jsonnet create mode 100644 .baedeker/rewrites.jsonnet create mode 100755 .baedeker/up.sh create mode 100644 .baedeker/vendor/baedeker-library/.gitignore create mode 100644 .baedeker/vendor/baedeker-library/LICENSE create mode 100644 .baedeker/vendor/baedeker-library/inputs/base.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/mixin/keys.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/mixin/spec.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/ops/debug.ejs create mode 100644 .baedeker/vendor/baedeker-library/ops/devtools.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/ops/nginx-dev.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/ops/nginx.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/ops/rewrites.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/outputs/addressbook.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/outputs/compose.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/outputs/composediscover.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/outputs/debug.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/util/genesisState.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/util/meta.libsonnet create mode 100644 .baedeker/vendor/baedeker-library/util/mixin.libsonnet create mode 100644 docs/baedeker.md create mode 100755 scripts/localnet-baedeker.sh diff --git a/.baedeker/.gitignore b/.baedeker/.gitignore new file mode 100644 index 000000000..c5483c281 --- /dev/null +++ b/.baedeker/.gitignore @@ -0,0 +1 @@ +/.bdk-env diff --git a/.baedeker/forkless-data.jsonnet b/.baedeker/forkless-data.jsonnet new file mode 100644 index 000000000..2bff7e7c6 --- /dev/null +++ b/.baedeker/forkless-data.jsonnet @@ -0,0 +1,46 @@ +local +m = import 'baedeker-library/mixin/spec.libsonnet', +rm = import 'baedeker-library/mixin/raw-spec.libsonnet', +; + +function(relay_spec, forked_spec, fork_source) + +local relay = { + name: 'subtensor', + bin: 'bin/subtensor', + spec: {Raw:{ + local modifyRaw = bdk.mixer([ + rm.resetNetworking($), + rm.decodeSpec(), + rm.polkaLaunchPara($), + rm.reencodeSpec(), + ]), + raw_spec: modifyRaw({ + name: "Bittensor", + id: "%s_local" % forked_spec, + chainType: "Development", + codeSubstitutes: {}, + properties: { + ss58Format: 42, + tokenDecimals: 9, + tokenSymbol: "TAO" + }, + genesis: { + raw: { + top: cql.chain(fork_source).latest._preloadKeys._raw, + childrenDefault: {}, + }, + }, + }), + }}, + nodes: { + [name]: { + bin: $.bin, + wantedKeys: 'standalone', + }, + for name in ['alice', 'bob', 'charlie'] + }, +}; + +relay + { +} diff --git a/.baedeker/rewrites.jsonnet b/.baedeker/rewrites.jsonnet new file mode 100644 index 000000000..30b9e199e --- /dev/null +++ b/.baedeker/rewrites.jsonnet @@ -0,0 +1,11 @@ +local dotenv = { + [std.splitLimit(line, "=", 2)[0]]: std.splitLimit(line, "=", 2)[1] + for line in std.split(importstr "../.env", "\n") + if line != "" + if std.member(line, "=") +}; + +function(prev, repoDir) +(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({ + 'bin/subtensor':'%s/target/release/subtensor' % repoDir, +})(prev) diff --git a/.baedeker/up.sh b/.baedeker/up.sh new file mode 100755 index 000000000..164cefa11 --- /dev/null +++ b/.baedeker/up.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e +BDK_DIR=$(dirname $(readlink -f "$0")) +RUST_LOG=info baedeker --spec=docker -J$BDK_DIR/vendor/ --generator=docker_compose=$BDK_DIR/.bdk-env --generator=docker_compose_discover=$BDK_DIR/.bdk-env/discover.env --secret=file=$BDK_DIR/.bdk-env/secret --tla-str=relay_spec=rococo-local --input-modules='lib:baedeker-library/ops/nginx.libsonnet' --input-modules='lib:baedeker-library/ops/devtools.libsonnet' --tla-str=repoDir=$(realpath $BDK_DIR/..) $@ $BDK_DIR/rewrites.jsonnet +cd $BDK_DIR/.bdk-env +#docker compose up -d --wait --remove-orphans diff --git a/.baedeker/vendor/baedeker-library/.gitignore b/.baedeker/vendor/baedeker-library/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/.baedeker/vendor/baedeker-library/LICENSE b/.baedeker/vendor/baedeker-library/LICENSE new file mode 100644 index 000000000..827cc0754 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Unique Network + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.baedeker/vendor/baedeker-library/inputs/base.libsonnet b/.baedeker/vendor/baedeker-library/inputs/base.libsonnet new file mode 100644 index 000000000..83da6ce39 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/inputs/base.libsonnet @@ -0,0 +1,54 @@ +local + genesisState = import '../util/genesisState.libsonnet', + {mixinAllChains, ...} = import '../util/mixin.libsonnet', + k = import '../mixin/keys.libsonnet', +; + +function(prev, final) + +// :code +local WELLKNOWN_CODE = '0x3a636f6465'; + +local genesisMixin = { + // TODO: Process from wasm once native runtime free world lands. + specJson: cql.description('' % self.path, bdk.processSpec(self.bin, self.spec)), + genesisWasm: self.specJson.genesis.raw.top[WELLKNOWN_CODE], + genesisWasmData: cql.runtimeWasm(self.genesisWasm), + genesisStateVersion: self.genesisWasmData.version.state_version, + genesisHead: genesisState(self.specJson, self.genesisStateVersion), + + ss58Format: super?.ss58Format ?? 42, + signatureSchema: super?.signatureSchema ?? 'Sr25519', + // FIXME: Try to guess from runtime metadata. + // If null - try to guess the schema. + // I.e use StashOf of pallet_staking, if staking presents in schema, and so on. + validatorIdAssignment: super?.validatorIdAssignment ?? 'none', + + addressSeed(seed):: cql.addressSeed(self.signatureSchema, seed, self.ss58Format), +}; + +local mergedChains = (prev + mixinAllChains(prev, function(chain, path) genesisMixin + { + path: path, + nodes+: { + [nodename]+: local hostname = '%s-node-%s' % [path, nodename]; { + hostname: hostname, + wantedKeys: + if node?.wantedKeys == 'para' then k.paraWantedKeys($) + else if node?.wantedKeys == 'para-ed' then k.paraWantedKeys($, ed = true) + else if node?.wantedKeys == 'para-nimbus' then k.paraWantedKeys($, nimbus = true) + else if node?.wantedKeys == 'relay' then k.relayWantedKeys($) + else if node?.wantedKeys == 'standalone' then k.standaloneWantedKeys($) + else if std.isObject(node?.wantedKeys) then node?.wantedKeys + else if !('wantedKeys' in node) then {} + else error 'Unknown wantedKeys: %s' % node?.wantedKeys, + }, + for [nodename, node] in (chain?.nodes ?? {}) + }, +})); + +mergedChains + mixinAllChains(mergedChains, function(chain, path) { + nodes+: { + [nodename]+: bdk.ensureKeys(node.hostname, node.wantedKeys, chain.ss58Format), + for [nodename, node] in (chain?.nodes ?? {}) + }, +}) diff --git a/.baedeker/vendor/baedeker-library/mixin/keys.libsonnet b/.baedeker/vendor/baedeker-library/mixin/keys.libsonnet new file mode 100644 index 000000000..aeb344aec --- /dev/null +++ b/.baedeker/vendor/baedeker-library/mixin/keys.libsonnet @@ -0,0 +1,55 @@ +local +needController({validatorIdAssignment, ...}) = + if validatorIdAssignment == 'none' || validatorIdAssignment == 'collatorSelection' then false + else if validatorIdAssignment == 'staking' then true + else error "unknown validatorIdAssignment: %s" % validatorIdAssignment, +; + +{ + relayWantedKeys(root): { + [if needController(root) then '_controller']: root.signatureSchema, + _stash: root.signatureSchema, + + gran: 'Ed25519', + babe: 'Sr25519', + imon: 'Sr25519', + para: 'Sr25519', + asgn: 'Sr25519', + audi: 'Sr25519', + // rococo: beefy is required + beef: 'Ecdsa', + + sessionKeys: { + grandpa: 'gran', + babe: 'babe', + im_online: 'imon', + authority_discovery: 'audi', + para_assignment: 'asgn', + para_validator: 'para', + beefy: 'beef', + }, + }, + paraWantedKeys(root, ed = false, nimbus = false): { + [if needController(root) then '_controller']: root.signatureSchema, + _stash: root.signatureSchema, + + // COMPAT: asset-hub on polkadot uses ed25519 instead of sr25519 for session keys. + // https://github.com/paritytech/cumulus/blob/d4bb2215bb28ee05159c4c7df1b3435177b5bf4e/parachains/common/src/lib.rs#L57-L62 + [if nimbus then 'nmbs' else 'aura']: if ed then 'Ed25519' else 'Sr25519', + // COMPAT: moonbeam only supports setting nimbus key in genesis, yet rand key is required. + [if nimbus then 'rand']: {alias: 'nmbs'}, + + sessionKeys: { + aura: 'aura', + }, + }, + standaloneWantedKeys(root): { + aura: 'Sr25519', + gran: 'Ed25519', + + sessionKeys: { + aura: 'aura', + grandpa: 'gran', + }, + }, +} diff --git a/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet b/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet new file mode 100644 index 000000000..231f5719e --- /dev/null +++ b/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet @@ -0,0 +1,233 @@ +local m = import './spec.libsonnet'; +local {encodeGrandpaKeys} = import '../util/grandpaKeys.libsonnet'; +local strToHex(str) = cql.toHex(std.encodeUTF8(str)); +local + account(name) = cql.sr25519Seed(name), + unwrapNewtype(struct) = local names = std.objectFields(struct); + if std.length(names) == 1 then struct[names[0]] + else struct, + WELLKNOWN_CODE = '0x3a636f6465', +; + +{ + resetSystem: function(prev) prev { + _storage+: { + System+: { + // Magic value mandated by spec, nice + // [69, 69, 69, ..., 69, 69, 69] + local hash69 = '0x' + '45' * 32, + BlockHash: { + "0": hash69, + }, + EventCount: 0, + EventTopics: {}, + Events: [], + ExtrinsicCount: 0, + ExtrinsicData: {}, + + // Block header + ParentHash: hash69, + Number: 0, + Digest: [], + }, + Aura+: { + CurrentSlot: std.bigint('0'), + }, + [if 'CollatorSelection' in prev._storage then 'CollatorSelection']+: { + LastAuthoredBlock: {}, + }, + [if 'Session' in prev._storage then 'Session']+: { + CurrentIndex: 0, + }, + // Full reset + [if 'Authorship' in prev._storage then 'Authorship']: {}, + }, + }, + + setSudo(key): { + _storage+: { + Sudo+: { + Key: key, + }, + }, + }, + giveBalance(address): { + _storage+: { + // Not updating total issuance: no big difference. + System+: { + Account+: std.trace('Altering account %s: %s' % [address, super.Account?.[address] ?? ''], { + [address]+: { + nonce: super?.nonce ?? 0, + // Leaks + consumers: super?.consumers ?? 1, + providers: super?.providers ?? 1, + sufficients: super?.sufficients ?? 0, + data+: { + free+: std.bigint(1000000000), + reserved: super?.reserved ?? std.bigint(0), + frozen: super?.reserved ?? std.bigint(0), + flags: super?.flags ?? std.bigint(0), + }, + }, + }), + }, + }, + }, + resetAuraAuthorities: function(prev) prev { + _storage+: { + Aura+: { + Authorities: [], + }, + [if 'AuraExt' in prev._storage then 'AuraExt']+: { + Authorities: [], + }, + }, + }, + addAuraAuthority(key): function(prev) prev { + _storage+: { + Aura+: { + Authorities+: [cql.ss58(key)], + }, + [if 'AuraExt' in prev._storage then 'AuraExt']+: { + Authorities+: [cql.ss58(key)], + }, + }, + }, + resetGrandpaAuthorities: function(prev) prev { + _storage+: { + Grandpa+: { + Authorities: [], + }, + }, + }, + addGrandpaAuthority(key): function(prev) prev { + _storage+: { + Grandpa+: { + Authorities+: [ + [ + cql.ss58(key), + std.bigint(1), + ] + ], + }, + }, + }, + resetSessionKeys: { + _storage+: { + Session+: { + DisabledValidators: [], + KeyOwner: {}, + NextKeys: {}, + QueuedKeys: [], + Validators: [], + }, + }, + }, + addSessionKey([_accountId, _validatorId, _keys]): { + local accountId = cql.ss58(_accountId), + local validatorId = cql.ss58(_validatorId), + local keys = { + [cql.toHex(std.encodeUTF8(key))]: cql.ss58(data), + for [key, data] in _keys + }, + _storage+: { + // FIXME: Should increase consumers/providers for account + Session+: { + KeyOwner+: { + [std.toString([key, data])]: validatorId, + for [key, data] in keys + }, + NextKeys+: { + [validatorId]: unwrapNewtype(keys), + }, + QueuedKeys+: [ + [ + validatorId, + unwrapNewtype(keys), + ], + ], + Validators+: [validatorId], + }, + }, + }, + resetInvulnerables: function(prev) prev { + _storage+: { + [if 'CollatorSelection' in prev._storage then 'CollatorSelection']+: { + Invulnerables: [], + }, + } + }, + addInvulnerable(key): function(prev) prev { + _storage+: { + [if 'CollatorSelection' in prev._storage then 'CollatorSelection']+: { + Invulnerables+: [cql.ss58(key)], + }, + } + }, + + setCodeRaw(code): function(prev) prev { + genesis+: { + raw+: { + top+: { + [WELLKNOWN_CODE]: code, + }, + }, + }, + }, + + // Compatible, as storage remains the same + resetNetworking: m.resetNetworking, + + decodeSpec(): function(prev) local dump = cql.fullDump(prev.genesis.raw.top); prev { + _originalDump:: dump, + _storage::: dump, + genesis+: { + raw+: { + top:: error "reencode storage first" + }, + }, + }, + reencodeSpec(): function(prev) prev { + _originalDump:: error "decode storage first", + _storage:: error "decode storage first", + genesis+: { + raw+: { + top::: prev._originalDump._rebuild(prev._storage), + }, + }, + }, + + polkaLaunchPara(root): [ + $.resetSystem, + $.setSudo(account('//Alice')), + $.giveBalance(account('//Alice')), + $.resetAuraAuthorities, + [ + $.addAuraAuthority(node.keys.aura), + for [?, node] in root.nodes + ], + $.resetGrandpaAuthorities, + [ + $.addGrandpaAuthority(node.keys.gran), + for [?, node] in root.nodes + ], + function(prev) bdk.mixer(if 'Session' in prev._storage then [ + $.resetSessionKeys, + [ + $.addSessionKey([ + node.keys.aura, + node.keys.aura, + { + aura: node.keys.aura, + }, + ]), + for [?, node] in root.nodes + ], + ] else [])(prev), + $.resetInvulnerables, + [ + $.addInvulnerable(node.keys.aura), + for [?, node] in root.nodes + ], + ], +} diff --git a/.baedeker/vendor/baedeker-library/mixin/spec.libsonnet b/.baedeker/vendor/baedeker-library/mixin/spec.libsonnet new file mode 100644 index 000000000..6c0e70134 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/mixin/spec.libsonnet @@ -0,0 +1,438 @@ +{ + setSudo(address): { + _genesis+: { + sudo+: { + key: address, + }, + } + }, + resetBalances: { + _genesis+: { + balances+: { + balances: [], + }, + }, + }, + giveBalance(address, amount): { + _genesis+: { + balances+: { + balances+: [ + [address, amount], + ], + }, + }, + }, + resetSessionKeys: { + _genesis+: { + session+: { + keys: [], + }, + } + }, + addSessionKey(key): { + _genesis+: { + session+: { + keys+: [key], + }, + }, + }, + resetAuraKeys: { + _genesis+: { + aura+: { + authorities: [], + }, + }, + }, + addAuraKey(key): { + _genesis+: { + aura+: { + authorities+: [key], + }, + }, + }, + resetCollatorSelectionInvulnerables: { + _genesis+: { + collatorSelection+: { + invulnerables: [], + }, + } + }, + addCollatorSelectionInvulnerable(key): { + _genesis+: { + collatorSelection+: { + invulnerables+: [key], + }, + }, + }, + resetParachainStakingCandidates: { + _genesis+: { + parachainStaking+: { + candidates: [], + }, + }, + }, + addParachainStakingCandidate(key): { + _genesis+: { + parachainStaking+: { + candidates+: [key], + }, + }, + }, + resetStakingInvulnerables: { + _genesis+: { + staking+: { + invulnerables: [], + }, + }, + }, + addStakingInvulnerable(key): { + _genesis+: { + staking+: { + invulnerables+: [key], + }, + }, + }, + resetStakingStakers: { + _genesis+: { + staking+: { + stakers: [], + }, + }, + }, + addStakingStaker(key): { + _genesis+: { + staking+: { + stakers+: [key], + }, + }, + }, + setStakingValidatorCount(count): { + _genesis+: { + staking+: { + validatorCount: count, + }, + }, + }, + resetAuthorMappingMappings: { + _genesis+: { + authorMapping+: { + mappings: [], + }, + }, + }, + addAuthorMappingMapping(key): { + _genesis+: { + authorMapping+: { + mappings+: [key], + }, + }, + }, + resetParas: { + _genesis+: { + paras+: { + paras: [], + }, + }, + }, + addPara(para_id, head, wasm, parachain = true): { + _genesis+: { + paras+: { + paras+: [[ + para_id, + { + genesis_head: head, + validation_code: wasm, + parachain: parachain, + }, + ]], + }, + }, + }, + + resetHrmps: { + _genesis+: { + hrmp+: { + preopenHrmpChannels: [], + }, + }, + }, + openHrmp(sender, receiver, maxCapacity, maxMessageSize): { + _genesis+: { + hrmp+: { + preopenHrmpChannels+: [ + [sender, receiver, maxCapacity, maxMessageSize], + ], + }, + }, + }, + + resetNetworking(root): { + assert !(super?._networkingWasReset ?? false): 'network should not be reset twice', + + bootNodes: [ + '/dns/%s/tcp/30333/p2p/%s' % [node.hostname, node.nodeIdentity], + for [?, node] in root.nodes + ], + chainType: 'Development', + telemetryEndpoints: [], + codeSubstitutes: {}, + properties: { + ss58Format: 42, + tokenDecimals: 9, + tokenSymbol: "TAO" + }, + + // COMPAT: cumulus template + // In baedeker, relay chain config is passed explicitly, rendering this argument to not being used + [if 'relay_chain' in root then 'relay_chain']: 'not_used', + // COMPAT: some chains use camelCase here + [if 'relayChain' in root then 'relayChain']: 'not_used', + + _networkingWasReset:: true, + }, + + simplifyGenesisName(): function(prev) + local genesisKind = if 'runtimeGenesis' in prev.genesis then 'sane-1.5-runtimeGenesis' else if 'runtimeAndCode' in prev.genesis then 'deprecated-runtimeAndCode' else if 'runtime_genesis_config' in prev.genesis.runtime then 'rococo' else 'sane'; + prev { + _genesisKind: genesisKind, + } + + if genesisKind == 'rococo' then { + _genesis::: prev.genesis.runtime.runtime_genesis_config + {system+: {code: '0x42424242'}}, + _code::: prev.genesis.runtime.runtime_genesis_config.system.code, + genesis+: { + runtime+: { + runtime_genesis_config:: error 'unsimplify genesis name first', + }, + }, + } else if genesisKind == 'sane' then { + _genesis::: prev.genesis.runtime + {system+: {code: '0x42424242'}}, + _code::: prev.genesis.runtime.system.code, + genesis+: { + runtime:: error 'unsimplify genesis name first', + }, + } else if genesisKind == 'deprecated-runtimeAndCode' then { + _genesis::: prev.genesis.runtimeAndCode.runtime + {system+: {code: '0x42424242'}}, + _code::: prev.genesis.runtimeAndCode.code, + genesis+: { + runtimeAndCode::: error 'unsimplify genesis name first', + }, + } else if genesisKind == 'sane-1.5-runtimeGenesis' then { + _runtimeGenesisKind::: if 'config' in prev.genesis.runtimeGenesis then 'config' else 'patch', + _genesis::: prev.genesis.runtimeGenesis[self._runtimeGenesisKind] + {system+: {code: '0x42424242'}}, + _code::: prev.genesis.runtimeGenesis?.code, + genesis+: { + runtimeGenesis:: error 'unsimplify genesis name first', + }, + }, + + unsimplifyGenesisName(): function(prev) + prev { + _runtimeGenesisKind:: error 'simplify genesis name first', + _genesis:: error 'simplify genesis name first', + _code:: error 'simplify genesis name first', + _genesisKind:: error 'genesis was resimplified', + } + + if prev?._genesisKind == 'rococo' then assert prev._genesis.system.code == '0x42424242' : 'use _code for overriding code!'; { + genesis+: { + runtime+: { + runtime_genesis_config::: prev._genesis + { + system+: { + code: prev._code, + }, + }, + }, + }, + } else if prev?._genesisKind == 'sane' then assert prev._genesis.system.code == '0x42424242' : 'use _code for overriding code!'; { + genesis+: { + runtime::: prev._genesis + { + system+: { + code: prev._code, + }, + }, + }, + } else if prev?._genesisKind == 'deprecated-runtimeAndCode' then assert prev._genesis.system.code == '0x42424242' : 'use _code for overriding code!'; { + genesis+: { + runtimeAndCode::: { + code: prev._code, + runtime: prev._genesis + { + system+: { + code:: error 'use _code for overriding code!', + }, + }, + }, + }, + } else if prev?._genesisKind == 'sane-1.5-runtimeGenesis' then assert prev._genesis.system.code == '0x42424242' : 'use _code for overriding code!'; { + genesis+: { + runtimeGenesis::: { + code: prev._code, + [prev._runtimeGenesisKind]: prev._genesis + { + system+: { + code:: error 'use _code for overriding code!', + }, + }, + }, + }, + } else error 'unknown genesis kind: %s' % [prev._genesis], + + // FIXME: Merge polkaLaunchRelay and polkaLaunchPara? + // Due to refactoring, pararelays are somewhat supported. + + polkaLaunchShared(root): local + isEth = root.signatureSchema == 'Ethereum', + // FIXME: support soft derivations in ecdsaSeed, then unhardcode private keys here. + // Soft derivations here are + // Alith: m/44'/60'/0'/0/0 + // Baltathar: m/44'/60'/0'/0/1 + // Root mnemonic for both is standard substrate "bottom drive obey lake curtain smoke basket hold race lonely fit walk", which is implied by *Seed functions + + // Alice/Alith + accountA = if !isEth then root.addressSeed('//Alice') else '0xf24FF3a9CF04c71Dbc94D0b566f7A27B94566cac', + // Bob/Baltathar + accountB = if !isEth then root.addressSeed('//Bob') else '0x3Cd0A705a2DC65e5b1E1205896BaA2be8A07c6e0', + // Charlie/Charleth + accountC = if !isEth then root.addressSeed('//Charlie') else '0x798d4Ba9baf0064Ec19eB4F0a1a45785ae9D6DFc', + // Dave/Dorothy + accountD = if !isEth then root.addressSeed('//Dave') else '0x773539d4Ac0e786233D90A233654ccEE26a613D9', + // Eve/Ethan + accountE = if !isEth then root.addressSeed('//Eve') else '0xFf64d3F6efE2317EE2807d223a0Bdc4c0c49dfDB', + ; [ + function(prev) if 'sudo' in prev._genesis then bdk.mixer([ + $.setSudo(accountA), + ])(prev) else prev, + $.resetBalances, + $.giveBalance(accountA, 2000000000000000000000000000000), + $.giveBalance(accountB, 2000000000000000000000000000000), + $.giveBalance(accountC, 2000000000000000000000000000000), + $.giveBalance(accountD, 2000000000000000000000000000000), + $.giveBalance(accountE, 2000000000000000000000000000000), + // Regardless of validator id assignment, every method (staking/collator-selection/etc) wants stash to have some + // money. + [ + $.giveBalance(node.wallets.stash, 2000000000000000000000000000000), + for [?, node] in root.nodes + ], + // pallet-session manages pallet-aura/pallet-grandpa, if there is no pallet-session: authority should be set directly for aura. + // pallet-aura also should not have keys, if there keys are specified using pallet-aura. + function(prev) bdk.mixer([ + if 'session' in prev._genesis then $.resetSessionKeys, + if 'aura' in prev._genesis then $.resetAuraKeys, + ])(prev), + function(prev) bdk.mixer(if 'session' in prev._genesis then [ + $.addSessionKey([ + // Account id + if root.validatorIdAssignment == 'staking' then node.wallets.controller + else node.wallets.stash, + // Validator id + node.wallets.stash, + local k = node.keys; { + [name]: k[key] + for [name, key] in node.wantedKeys.sessionKeys + }, + ]) + for [?, node] in root.nodes + ] else if 'aura' in prev._genesis then [ + $.addAuraKey(node.keys.aura) + for [?, node] in root.nodes + ] else [])(prev), + ], + + // Alter spec in the same way as polkadot-launch does this, in most cases this should + // be everything needed to start working node + polkaLaunchRelay(root, hrmp = []): $.polkaLaunchShared(root) + [ + function(prev) if 'staking' in prev._genesis then bdk.mixer([ + $.resetStakingInvulnerables, + $.resetStakingStakers, + [ + [ + $.addStakingInvulnerable(node.wallets.stash), + $.addStakingStaker([ + node.wallets.stash, + node.wallets.controller, + 100000000000000, + 'Validator', + ]), + ], + for [?, node] in root.nodes + ], + $.setStakingValidatorCount(std.length(root.nodes)), + ])(prev) else prev, + function(prev) bdk.mixer([ + [ + $.resetParas, + ], + [ + // FIXME: Also bump parachainRegistrar last id if para_id >= 2000? + $.addPara(para.paraId, para.genesisHead, para.genesisWasm), + for [paraname, para] in root.parachains + ], + ])(prev), + function(prev) bdk.mixer([ + [ + $.resetHrmps, + ], + [ + $.openHrmp(ch[0], ch[1], ch[2], ch[3]), + for ch in hrmp + ], + ])(prev), + function(prev) if 'configuration' in prev._genesis then local + prevConfig = prev?._genesis.configuration?.config ?? {}, + ifExists(f, o) = if f in o then f; + prev { + _genesis+: { + configuration+: { + config+: { + hrmp_max_parachain_outbound_channels: 20, + [ifExists('hrmp_max_parathread_outbound_channels', prevConfig)]: 20, + hrmp_max_parachain_inbound_channels: 20, + [ifExists('hrmp_max_parathread_inbound_channels', prevConfig)]: 20, + [ifExists('pvf_checking_enabled', prevConfig)]: true, + max_validators: 300, + [ifExists('max_validators_per_core', prevConfig)]: 20, + [ifExists('scheduling_lookahead', prevConfig)]: 1 + }, + }, + }, + } else prev, + // function(prev) std.trace(prev), + ], + polkaLaunchPara(root): $.polkaLaunchShared(root) + [ + function(prev) if 'collatorSelection' in prev._genesis then bdk.mixer([ + $.resetCollatorSelectionInvulnerables, + [ + $.addCollatorSelectionInvulnerable(node.wallets.stash), + for [?, node] in root.nodes + ], + ])(prev) else prev, + + // COMPAT: moonbeam + function(prev) if 'parachainStaking' in prev._genesis then bdk.mixer([ + $.resetParachainStakingCandidates, + [ + $.addParachainStakingCandidate([node.wallets.stash, 10000000000000000000000000]), + for [?, node] in root.nodes + ], + ])(prev) else prev, + // COMPAT: moonbeam + function(prev) if 'authorMapping' in prev._genesis then bdk.mixer([ + $.resetAuthorMappingMappings, + [ + $.addAuthorMappingMapping([node.keys?.aura ?? node.keys.nmbs, node.wallets.stash]), + for [?, node] in root.nodes + ], + ])(prev) else prev, + ], + + genericRelay(root, hrmp = []): bdk.mixer([ + $.resetNetworking(root), + $.simplifyGenesisName(), + $.polkaLaunchRelay(root, hrmp), + $.unsimplifyGenesisName(), + ]), + genericPara(root): bdk.mixer([ + $.resetNetworking(root), + $.simplifyGenesisName(), + $.polkaLaunchPara(root), + $.unsimplifyGenesisName(), + ]), +} diff --git a/.baedeker/vendor/baedeker-library/ops/debug.ejs b/.baedeker/vendor/baedeker-library/ops/debug.ejs new file mode 100644 index 000000000..288131ff8 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/ops/debug.ejs @@ -0,0 +1,30 @@ + + + + + Baedeker devtools + + + + + +
+ + + diff --git a/.baedeker/vendor/baedeker-library/ops/devtools.libsonnet b/.baedeker/vendor/baedeker-library/ops/devtools.libsonnet new file mode 100644 index 000000000..f7b1fe4eb --- /dev/null +++ b/.baedeker/vendor/baedeker-library/ops/devtools.libsonnet @@ -0,0 +1,30 @@ +local {flattenChains, flattenNodes, ...} = import '../util/mixin.libsonnet'; + +function(prev) prev { + _output+: { + dockerCompose+: { + _nginxLocations+:: [ + 'location /apps/ { proxy_pass http://polkadot-apps/; }', + ], + _nginxDependencies+:: ['polkadot-apps'], + _composeConfig+:: { + services+: { + 'polkadot-apps': { + // TODO: We can provide custom endpoint list to this container using ENV. But changes to this file are needed. + // https://github.com/polkadot-js/apps/blob/0366991f685a80147f46eb69a23285acb15bc6b7/packages/apps-config/src/endpoints/development.ts#L19 + image: 'jacogr/polkadot-js-apps:latest@sha256:b052771165a82833f68b569a74a198b09d8e1d0cce097e804cf60bc06a4faf7b', + }, + }, + }, + // Yep, sorry for this + 'ops/index.html': std.strReplace(importstr './debug.ejs', 'DATA_JSON', std.manifestJson({ + chains: [ + { + path: chain.path, + }, + for chain in flattenChains(prev) + ], + })), + }, + }, +} diff --git a/.baedeker/vendor/baedeker-library/ops/nginx-dev.libsonnet b/.baedeker/vendor/baedeker-library/ops/nginx-dev.libsonnet new file mode 100644 index 000000000..47326c11b --- /dev/null +++ b/.baedeker/vendor/baedeker-library/ops/nginx-dev.libsonnet @@ -0,0 +1,19 @@ + +local nginx = import './nginx.libsonnet'; + +function(prev, nginxExposePort = 9699, nginxExposeHost = '127.0.0.1') nginx(prev) { + _output+: { + dockerCompose+: { + _wellKnownBalancerUrl:: '%s:%d' % [nginxExposeHost, nginxExposePort], + _composeConfig+: { + services+: { + nginx+: { + ports+: [ + '%s:%d:80' % [nginxExposeHost, nginxExposePort] + ], + }, + }, + }, + }, + }, +} diff --git a/.baedeker/vendor/baedeker-library/ops/nginx.libsonnet b/.baedeker/vendor/baedeker-library/ops/nginx.libsonnet new file mode 100644 index 000000000..7d0d4b396 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/ops/nginx.libsonnet @@ -0,0 +1,98 @@ +local {flattenChains, flattenNodes, ...} = import '../util/mixin.libsonnet'; + +function(prev) + +prev { + _output+: { + dockerCompose+: { + local locations = self._nginxLocations, + local dependencies = self._nginxDependencies, + local composeFiles = self, + _nginxDependencies+:: [ + node.hostname + for node in flattenNodes(prev) + ], + _nginxLocations+:: [ + local shared = { + name: chain.path, + }; + std.join('\n', [ + 'location /%(name)s/ { try_files /nonexistent @%(name)s-$http_upgrade; }' % shared, + 'location @%(name)s-websocket {' % shared, + '\tproxy_pass http://%(name)s-websocket;' % shared, + '\tproxy_http_version 1.1;', + '\tproxy_set_header Upgrade "websocket";', + '\tproxy_set_header Connection "upgrade";', + '}', + 'location @%(name)s- {' % shared, + '\tproxy_pass http://%(name)s-http;' % shared, + '}', + ]), + for chain in flattenChains(prev) + ], + local configStr = std.join('\n\n', [ + local shared = { + name: chain.path, + }; + std.join('\n', [ + 'upstream %(name)s-websocket {' % shared, + '\tip_hash;', + std.join('\n', [ + '\tserver %s:9944;' % node.hostname + for [?, node] in (chain?.nodes ?? {}) + ]), + '}', + 'upstream %(name)s-http {' % shared, + '\tip_hash;', + std.join('\n', [ + '\tserver %s:9944;' % node.hostname + for [?, node] in (chain?.nodes ?? {}) + if !(node?.legacyRpc ?? false) + ] + [ + '\tserver %s:9933;' % node.hostname + for [?, node] in (chain?.nodes ?? {}) + if (node?.legacyRpc ?? false) + ]), + '}', + ]), + for chain in flattenChains(prev) + ] + ['server {', 'listen 80;', 'add_header Access-Control-Allow-Origin *;'] + [ + std.join('\n', locations), + ] + ['}']), + 'ops/nginx.conf': configStr, + _composeConfig+:: { + services+: { + nginx: { + image: 'nginx:latest@sha256:48a84a0728cab8ac558f48796f901f6d31d287101bc8b317683678125e0d2d35', + volumes+: [ + { + type: 'bind', + source: 'ops/nginx.conf', + target: '/etc/nginx/conf.d/default.conf', + read_only: true, + }, + // Introduce arbitrary dependency on config hash to force container restart when it changes + { + type: 'bind', + source: 'ops/nginx.conf', + target: '/config/%s%s' % [ + std.md5(configStr), + std.md5(composeFiles?.['ops/index.html'] ?? ''), + ], + read_only: true, + }, + ] + (if 'ops/index.html' in composeFiles then [ + { + type: 'bind', + source: 'ops/index.html', + target: '/etc/nginx/html/index.html', + read_only: true, + }, + ] else []), + depends_on: dependencies, + }, + }, + }, + }, + }, +} diff --git a/.baedeker/vendor/baedeker-library/ops/rewrites.libsonnet b/.baedeker/vendor/baedeker-library/ops/rewrites.libsonnet new file mode 100644 index 000000000..64c784d8f --- /dev/null +++ b/.baedeker/vendor/baedeker-library/ops/rewrites.libsonnet @@ -0,0 +1,13 @@ +local {mixinRolloutNodes, ...} = import '../util/mixin.libsonnet'; + +{ + rewriteNodePaths(paths, for_nodes = true, for_chain = true, percent = 1, leave = null, extra_node_mixin = {}, extra_chain_mixin = {}): + local mkBin(obj, node) = if 'bin' in obj && std.isString(obj.bin) && obj.bin in paths then ({ + bin: paths[obj.bin], + } + if node then extra_node_mixin else extra_chain_mixin) else {}; + function(prev) prev + mixinRolloutNodes(prev, + function(node) if for_nodes then mkBin(node, true) else {}, + function(chain) if for_chain then mkBin(chain, false) else {}, + percent = percent, leave = leave + ) +} diff --git a/.baedeker/vendor/baedeker-library/outputs/addressbook.libsonnet b/.baedeker/vendor/baedeker-library/outputs/addressbook.libsonnet new file mode 100644 index 000000000..55205b9ca --- /dev/null +++ b/.baedeker/vendor/baedeker-library/outputs/addressbook.libsonnet @@ -0,0 +1,28 @@ +local {flattenNodes, ...} = import '../util/mixin.libsonnet'; + +function(prev) { + _output+: { + addressbook: 'Copy the following snippet to browser console on polkadot apps:\n' + std.join('\n', [ + '', + '// Optional: do not execute if you have something important saved in polkadot apps!', + '// localStorage.clear();' + ] + [ + 'localStorage["address:%s"] = JSON.stringify(%s);' % [cql.ss58(wallet), { + address: wallet, + meta: { + name: "%s (%s)" % [node.hostname, walletname] + }, + }], + for node in flattenNodes(prev) + for [walletname, wallet] in ([ + [walletname, wallet] + for [walletname, wallet] in node.wallets + if walletname == 'stash' + ] + [ + [walletname, wallet] + for [walletname, wallet] in node.keys + if walletname == 'aura' || walletname == 'babe' + ]) + ]), + }, +} diff --git a/.baedeker/vendor/baedeker-library/outputs/compose.libsonnet b/.baedeker/vendor/baedeker-library/outputs/compose.libsonnet new file mode 100644 index 000000000..9cf4488d5 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/outputs/compose.libsonnet @@ -0,0 +1,177 @@ +local {flattenChains, flattenNodes, ...} = import '../util/mixin.libsonnet'; + +function(prev, final) + +local v = { + bind(source, target, read_only = true): { + type: 'bind', + source: source, + target: target, + read_only: read_only, + }, + volume(name, target, nocopy = true): { + type: 'volume', + source: name, + target: target, + volume: { + nocopy: nocopy, + }, + }, + tmpfs(target): { + type: 'tmpfs', + target: target, + }, +}; + +local +hostMounts = bdk.dockerMounts(), +hostVolumes = [ + v.tmpfs('/tmp'), +] + [ + v.bind('/%s' % path, '/%s' % path), + for path in hostMounts +]; + +local binToObj(bin, config) = +if std.isString(bin) then { + image: config.emptyImage, + entrypoint: bin, + dockerBased:: false, + volumes: hostVolumes, +} else if 'dockerImage' in bin then { + image: bin.dockerImage, + [if 'docker' in bin then 'entrypoint']: bin.docker, + dockerBased:: true, +} else { + image: config.emptyImage, + entrypoint: bin['local'], + dockerBased:: false, + volumes: hostVolumes, +}; + +local WELLKNOWN_CODE = '0x3a636f6465'; +local metadataFromKeys(keys) = cql.runtimeWasm(keys[WELLKNOWN_CODE]).metadata; + +// TODO: Show diff +local diffRaw(old, new) = local +oldKeys = std.objectFields(old), newKeys = std.objectFields(new), +oldMetadata = metadataFromKeys(old), newMetadata = metadataFromKeys(new), +fancyDump(meta, data) = std.manifestJson(cql.dump(meta, data, {omit_empty: true, include_defaults: false})), +; +'removed data:\n' + +fancyDump(oldMetadata, { + [key]: old[key], + for key in std.setDiff(oldKeys, newKeys) +}) + +'\n\nadded data:\n' + +fancyDump(newMetadata, { + [key]: new[key], + for key in std.setDiff(newKeys, oldKeys) +}) + +'\n\nupdated, old:\n' + +fancyDump(oldMetadata, { + [key]: old[key], + for key in std.setInter(oldKeys, newKeys) + if old[key] != new[key] +}) + +'\n\nupdated, new:\n' + +fancyDump(newMetadata, { + [key]: new[key], + for key in std.setInter(oldKeys, newKeys) + if old[key] != new[key] +}); + +local assertEqualSpecsReconciler(_old, _new) = local old = std.parseJson(_old), new = std.parseJson(_new); +if old.genesis.raw.top != new.genesis.raw.top then error 'reconcilation disabled, and genesis is not equal:\n' + diffRaw(old.genesis.raw.top, new.genesis.raw.top) else _new; + +{ + _output:: { + dockerCompose+: { + _config+:: { + emptyImage: error 'missing empty image', + outputRoot: error 'missing output root', + }, + }, + }, +} + prev + { + _output+: { + dockerCompose+: { + ['specs/%s.json' % chain.path]: std.manifestJsonEx(chain.specJson, ' ', preserve_order = true) + '\n', + for chain in flattenChains(final) + } + { + ['reconcile_specs/%s.json' % chain.path]:: assertEqualSpecsReconciler, + for chain in flattenChains(final) + } + { + local config = self._config, + _composeConfig+:: { + version: '3.4', + services+: { + [node.hostname]: binToObj(node.bin, config) + { + command: [ + '--name=%s' % node.hostname, + '--validator', + '--base-path=%s' % node?.expectedDataPath ?? '/chaindata', + '--chain=/chain-spec.json', + '--keystore-path=/keystore', + '--node-key-file=/node-key', + '--no-mdns', + // Removed in new versions of substrate, will not escape docker host network anyways + // '--no-private-ipv4', + '--detailed-log-output', + '--execution=wasm', + '--unsafe-rpc-external', + '--rpc-cors=all', + ] + (if node?.legacyRpc ?? false then [ + '--rpc-port=9933', + '--ws-port=9944', + '--unsafe-ws-external', + ] else [ + '--rpc-port=9944', + ]) + (node?.extraArgs ?? []) + (if node._parentChain != null /*&& node.parentConnection == "internal"*/ then ([ + '--', + '--base-path=/chaindata-parent', + '--chain=/chain-spec-parent.json', + '--execution=wasm', + ] + (if node?.legacyRpc ?? false then [ + '--rpc-port=9833', + '--ws-port=9844', + ] else [ + '--rpc-port=9844' + ]) + (node?.extraArgsInternalParent ?? [])) else []), + [if 'rpcPort' in node || 'extraPorts' in node then 'ports']: (if 'rpcPort' in node then [ + '%s:9944' % node.rpcPort, + ] else []) + (node?.extraPorts ?? []), + // TODO: nocopy may cause problems if this directory is already used in container, + // but it is also helps with containers, which are run by unprivileged account. + // Should there be init container, which issues correct chown+chmod? + volumes+: [ + v.bind(bdk.toRelative(config.outputRoot, node.localKeystoreDir), '/keystore'), + v.bind(bdk.toRelative(config.outputRoot, node.localNodeFile), '/node-key'), + v.bind('specs/%s.json' % node._chain.path, '/chain-spec.json'), + v.volume('chaindata-%s' % node.hostname, node?.expectedDataPath ?? '/chaindata', nocopy = false), + ] + (if node._parentChain != null /*&& node.parentConnection == "internal"*/ then [ + v.bind('specs/%s.json' % node._parentChain.path, '/chain-spec-parent.json'), + v.volume('chaindata-%s-parent' % node.hostname, '/chaindata-parent', nocopy = false), + ] else []), + } + (node?.extraCompose ?? {}), + for node in flattenNodes(final) + }, + networks: { + chainnet: { + driver: 'bridge', + }, + }, + volumes: { + ['chaindata-%s' % node.hostname]: null, + for node in flattenNodes(final) + } + { + ['chaindata-%s-parent' % node.hostname]: null, + for node in flattenNodes(final) + if node._parentChain != null + // if node.parentConnection == "internal" + }, + }, + 'docker-compose.yml': std.manifestYamlDoc(self._composeConfig, quote_keys = false, preserve_order = true) + '\n', + }, + }, +} diff --git a/.baedeker/vendor/baedeker-library/outputs/composediscover.libsonnet b/.baedeker/vendor/baedeker-library/outputs/composediscover.libsonnet new file mode 100644 index 000000000..a9197a827 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/outputs/composediscover.libsonnet @@ -0,0 +1,30 @@ +local {flattenNodes, flattenChains, ...} = import '../util/mixin.libsonnet'; + +function(prev, final) +prev + { + _output+:: { + dockerCompose+: { + _wellKnownBalancerUrl:: super?._wellKnownBalancerUrl ?? 'BALANCER_URL', + }, + dockerComposeDiscover+: local + balancerUrl = final._output.dockerCompose._wellKnownBalancerUrl, + ; std.join('\n', [ + 'BDK_BALANCER=http://%s/' % balancerUrl, + ] + [ + '%s_ID=%i' % [std.strReplace(std.asciiUpper(chain.path), '-', '_'), chain.paraId] + for chain in flattenChains(prev) + if 'paraId' in chain + ] + [ + '%s_HTTP_URL=http://%s/%s/' % [std.strReplace(std.asciiUpper(chain.path), '-', '_'), balancerUrl, chain.path] + for chain in flattenChains(prev) + ] + [ + '%s_URL=ws://%s/%s/' % [std.strReplace(std.asciiUpper(chain.path), '-', '_'), balancerUrl, chain.path] + for chain in flattenChains(prev) + ] + [ + '%s_STASH=%s' % [std.strReplace(std.asciiUpper(node.hostname), '-', '_'), node.wallets.stash] + for chain in flattenChains(prev) + if 'paraId' in chain + for node in flattenNodes(chain) + ] + ['']), + }, +} diff --git a/.baedeker/vendor/baedeker-library/outputs/debug.libsonnet b/.baedeker/vendor/baedeker-library/outputs/debug.libsonnet new file mode 100644 index 000000000..d96316003 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/outputs/debug.libsonnet @@ -0,0 +1,7 @@ +function(prev) + +prev + { + _output+:: { + debug: prev, + }, +} diff --git a/.baedeker/vendor/baedeker-library/util/genesisState.libsonnet b/.baedeker/vendor/baedeker-library/util/genesisState.libsonnet new file mode 100644 index 000000000..06d244bd9 --- /dev/null +++ b/.baedeker/vendor/baedeker-library/util/genesisState.libsonnet @@ -0,0 +1,51 @@ +// Implementation of export-genesis-state in jsonnet, exports genesis head in format suitable for polkadot. +local t = import './meta.libsonnet'; + +// Basic header definition, only things required for genesis state building are included. +local types = t.metadata({ + // Although hash/block number is generic, all substrate chains use blake2_256 for hash, and u32 for number. + // Currently, there is no way to query such metadata from the chain, and using other types are not feasible, + // as u32 block number is enough for 136 years of block production, assuming 1 block per second. + header: t.s({ + parent_hash: $.hash, + number: $.number, + state_root: $.hash, + extrinsic_root: $.hash, + digest: $.digest, + }), + + digest: t.s({ + logs: $.vecstub, + }), + vecu8: t.v($.u8), + hash: t.a($.u8, 32), + number: t.c($.u32), + + u8: t.p('u8'), + u32: t.p('u32'), + + // It is impossible to initialize stub type, as it is recursive with no way to stop recursion. + vecstub: t.v($.stub), + stub: t.s({ + __doNotTryToInitialize__: $.stub, + // chainql automatically unwraps newtype structs, this field will make stub struct not newtype. + _: $.stub, + }), +}); + +local storageRoot(storage, stateVersion) = + cql.blake2_256Root(storage.top + { + [key]: cql.blake2_256Root(tree, stateVersion), + for [key, tree] in storage.childrenDefault + }, stateVersion); + +function(spec, stateVersion) +assert spec.genesis.raw != {}: 'not a raw spec!'; + +types._encode(0, { + parent_hash: '0x' + '00' * 32, + number: 0, + state_root: storageRoot(spec.genesis.raw, stateVersion), + extrinsic_root: cql.blake2_256Root({}, stateVersion), + digest: [], +}) diff --git a/.baedeker/vendor/baedeker-library/util/meta.libsonnet b/.baedeker/vendor/baedeker-library/util/meta.libsonnet new file mode 100644 index 000000000..52d08084a --- /dev/null +++ b/.baedeker/vendor/baedeker-library/util/meta.libsonnet @@ -0,0 +1,51 @@ +// json-encoded (chainql-flavored) runtime metadata builder + +local def(t, v) = { + type: { + def: { + [t]: v, + }, + }, +}; + +{ + types(o): std.objectValues(o + { + [name]+: {id: id}, + for [id, name] in std.mapWithIndex(function(i, v) [i, v], std.objectFieldsEx(o, false, preserve_order = true)) + }), + metadata(o): cql.dump({ + types: { + types: $.types(o), + }, + pallets: [], + // Required, but shouldn't be used by callers + extrinsic: {ty: 0, version: 0, signed_extensions: []}, + ty: 0, + }, {}), + + // Primitive type + p(n): def('primitive', n), + // Vec + v(t): def('sequence', { + type: t.id, + }), + // struct, with value types specified in f + s(f): def('composite', { + fields: [ + { + name: key, + type: value.id, + }, + for {key, value} in std.objectKeysValues(f, preserve_order = true) + ], + }), + // [t; s] + a(t, s): def('array', { + len: s, + type: t.id, + }), + // Compact + c(t): def('compact', { + type: t.id, + }), +} diff --git a/.baedeker/vendor/baedeker-library/util/mixin.libsonnet b/.baedeker/vendor/baedeker-library/util/mixin.libsonnet new file mode 100644 index 000000000..66d38b5ef --- /dev/null +++ b/.baedeker/vendor/baedeker-library/util/mixin.libsonnet @@ -0,0 +1,41 @@ +{ + mixinAllChains(chain, mixin, path = chain?.name ?? 'relay'): mixin(chain, path = path) + { + parachains+: { + [paraname]+: $.mixinAllChains(para, mixin, path = "%s-%s" % [path, paraname]) + for [paraname, para] in (chain?.parachains ?? {}) + }, + }, + mixinAllNodes(chain, mixin, mixinChain = function(v) {}): $.mixinAllChains(chain, function(chain, path) { + nodes+: { + [nodename]+: mixin(node), + for [nodename, node] in chain?.nodes + }, + } + mixinChain(chain)), + mixinRolloutNodes(chain, mixin, mixinChain = function(v) {}, percent = 1, leave = null): $.mixinAllChains(chain, function(chain, path) { + nodes+: local length = std.length(chain?.nodes ?? {}); { + [nodename]+: if ((i + 1) / length <= percent) && (leave == null || i < length - leave) then mixin(node) + else {} + for [i, {key: nodename, value: node}] in std.mapWithIndex(function(i, v) [i, v], std.objectKeysValues(chain?.nodes)) + }, + } + mixinChain(chain)), + flattenNodes(chain, parent = null): std.join([], [ + [ + node + { + _chain:: chain, + _parentChain:: parent, + }, + for [?, node] in (chain?.nodes ?? {}) + ], + ] + [ + $.flattenNodes(para, chain), + for [?, para] in (chain?.parachains ?? {}) + ]), + flattenChains(chain): std.join([], [ + [ + chain, + ], + ] + [ + $.flattenChains(para), + for [?, para] in (chain?.parachains ?? {}) + ]), +} diff --git a/docs/baedeker.md b/docs/baedeker.md new file mode 100644 index 000000000..f6a5b6850 --- /dev/null +++ b/docs/baedeker.md @@ -0,0 +1,69 @@ +# Running local network with full production state + +## Baedeker tool overview + +The Baedeker tool helps to pull the full state from a Substrate network, modify it as needed, and output it as a raw spec. + +It also creates node secrets (aura/grandpa keys) and writes them into the modified raw spec's aura and grandpa state. + +## Running locally + +### 1. Install baedeker from source + +```shell +git clone https://github.com/UniqueNetwork/baedeker.git && cd baedeker +cargo build --release +sudo cp ./target/release/baedeker /usr/local/bin +sudo chmod +x /usr/local/bin/baedeker +``` + +### 2. Execute baedeker to pull the state from Finney + +Note 1: The .bdk-env/specs folder should be empty, the chain spec will not be overwritten. +Note 2: Endpoint may be replaced to pull the state from another chain, e.g. from testnet + +```shell +.baedeker/up.sh .baedeker/forkless-data.jsonnet --tla-str=forked_spec=subtensor --tla-str=fork_source=wss://entrypoint-finney.opentensor.ai +``` + +### 3. What to expect + +#### Expected output example + +Note: The process of pulling Finney state from a public archive node may take 10-20 minutes. + +``` + INFO baedeker: evaluating config + INFO baedeker: evaluating input config + WARN baedeker::library: resulting spec will not work on the remote machine, impure bdk.dockerMounts() was used! +loading metadata +preloading all keys +loading keys by prefix [] +loaded 1000 keys +loaded 1000 keys +... +loaded 737 keys +loaded keys, last chunk was 737 +preloading 30000 keys +... +preloading 13734 keys +loading keys by prefix [] +rebuilding pallet Aura +rebuilding pallet Balances +rebuilding storage Account +... +rebuilding storage Voting +rebuilding pallet TriumvirateMembers +``` + +#### Baedeker will provide the following output to `.bdk-env` (added to .gitignore) + + - The raw spec to `.bdk-env/specs/` folder + - Node secrets to `.bdk-env/secret/` folder + +### 4. Run the local network + +```shell +./scripts/localnet-baedeker.sh +``` + diff --git a/scripts/localnet-baedeker.sh b/scripts/localnet-baedeker.sh new file mode 100755 index 000000000..834b619bc --- /dev/null +++ b/scripts/localnet-baedeker.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +: "${BUILD_BINARY:=1}" +# : "${FEATURES:=pow-faucet}" + +FULL_PATH=".baedeker/.bdk-env/specs/subtensor.json" + +if [[ $BUILD_BINARY == "1" ]]; then + echo "*** Building substrate binary..." + # cargo build --release --features "$FEATURES" + cargo build --release + echo "*** Binary compiled" +fi + +echo "*** Purging previous state..." +./target/release/node-subtensor purge-chain -y --base-path /tmp/charlie --chain="$FULL_PATH" >/dev/null 2>&1 +./target/release/node-subtensor purge-chain -y --base-path /tmp/bob --chain="$FULL_PATH" >/dev/null 2>&1 +./target/release/node-subtensor purge-chain -y --base-path /tmp/alice --chain="$FULL_PATH" >/dev/null 2>&1 +echo "*** Previous chainstate purged" + +echo "*** Starting localnet nodes..." +alice_start=( + ./target/release/node-subtensor + --base-path /tmp/alice + --chain="$FULL_PATH" + --keystore-path=./.baedeker/.bdk-env/secret/keystore-subtensor-node-alice + --node-key-file=./.baedeker/.bdk-env/secret/node/subtensor-node-alice + --port 30334 + --rpc-port 9946 + --validator + --rpc-cors=all + --rpc-external + --unsafe-rpc-external + --rpc-methods=unsafe + --allow-private-ipv4 + --discover-local +) + +bob_start=( + ./target/release/node-subtensor + --base-path /tmp/bob + --chain="$FULL_PATH" + --keystore-path=./.baedeker/.bdk-env/secret/keystore-subtensor-node-bob + --node-key-file=./.baedeker/.bdk-env/secret/node/subtensor-node-bob + --port 30335 + --rpc-port 9935 + --validator + --allow-private-ipv4 + --discover-local +) + +charlie_start=( + ./target/release/node-subtensor + --base-path /tmp/charlie + --chain="$FULL_PATH" + --keystore-path=./.baedeker/.bdk-env/secret/keystore-subtensor-node-charlie + --node-key-file=./.baedeker/.bdk-env/secret/node/subtensor-node-charlie + --port 30336 + --rpc-port 9936 + --validator + --allow-private-ipv4 + --discover-local +) + +(trap 'kill 0' SIGINT; ("${alice_start[@]}" 2>&1) & ("${bob_start[@]}" 2>&1) & ("${charlie_start[@]}" 2>&1)) From 9a206a70b4dd90c427ee37d70919fcd4bd12dba3 Mon Sep 17 00:00:00 2001 From: Greg Zaitsev Date: Fri, 26 Jul 2024 14:32:32 -0400 Subject: [PATCH 2/2] Fix testnet spec generation (when Alice account already exists) --- .baedeker/up.sh | 3 +++ .baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.baedeker/up.sh b/.baedeker/up.sh index 164cefa11..2fb3573ca 100755 --- a/.baedeker/up.sh +++ b/.baedeker/up.sh @@ -4,3 +4,6 @@ BDK_DIR=$(dirname $(readlink -f "$0")) RUST_LOG=info baedeker --spec=docker -J$BDK_DIR/vendor/ --generator=docker_compose=$BDK_DIR/.bdk-env --generator=docker_compose_discover=$BDK_DIR/.bdk-env/discover.env --secret=file=$BDK_DIR/.bdk-env/secret --tla-str=relay_spec=rococo-local --input-modules='lib:baedeker-library/ops/nginx.libsonnet' --input-modules='lib:baedeker-library/ops/devtools.libsonnet' --tla-str=repoDir=$(realpath $BDK_DIR/..) $@ $BDK_DIR/rewrites.jsonnet cd $BDK_DIR/.bdk-env #docker compose up -d --wait --remove-orphans + +# patch spec with Alice balance + diff --git a/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet b/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet index 231f5719e..4151c9c52 100644 --- a/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet +++ b/.baedeker/vendor/baedeker-library/mixin/raw-spec.libsonnet @@ -63,7 +63,7 @@ local providers: super?.providers ?? 1, sufficients: super?.sufficients ?? 0, data+: { - free+: std.bigint(1000000000), + free: std.bigint(1000000000), reserved: super?.reserved ?? std.bigint(0), frozen: super?.reserved ?? std.bigint(0), flags: super?.flags ?? std.bigint(0),