From 49318af98d1ead72a6c4cfd90d633811a90920a8 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 19 Apr 2024 15:17:27 +0200 Subject: [PATCH 01/26] [WIP] --- Cargo.lock | 37 +++++++------ Cargo.toml | 6 ++- .../src/chain_spec/builder.rs | 4 +- .../src/chain_spec/cli.rs | 0 .../src/chain_spec/commands.rs | 4 +- .../src/chain_spec/keystore.rs | 3 +- .../src/chain_spec/mod.rs | 10 +--- bin/node/Cargo.toml | 53 ++++++++++--------- bin/node/src/cli.rs | 27 +++++----- bin/node/src/lib.rs | 1 - bin/node/src/main.rs | 2 - bin/node/src/service.rs | 13 +++-- primitives/src/lib.rs | 3 ++ 13 files changed, 83 insertions(+), 80 deletions(-) rename bin/{node => chainspec-generator}/src/chain_spec/builder.rs (97%) rename bin/{node => chainspec-generator}/src/chain_spec/cli.rs (100%) rename bin/{node => chainspec-generator}/src/chain_spec/commands.rs (98%) rename bin/{node => chainspec-generator}/src/chain_spec/keystore.rs (98%) rename bin/{node => chainspec-generator}/src/chain_spec/mod.rs (75%) diff --git a/Cargo.lock b/Cargo.lock index a8cab271c0..2175701fb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,24 +278,19 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "futures", - "hex", - "hex-literal", "jsonrpsee", - "libp2p", "log", - "pallet-staking", "pallet-transaction-payment-rpc", "parity-scale-codec", "primitives", "sc-basic-authorship", - "sc-chain-spec", "sc-cli", "sc-client-api", + "sc-client-db", "sc-consensus", "sc-consensus-aura", "sc-consensus-slots", "sc-executor", - "sc-keystore", "sc-network", "sc-network-sync", "sc-rpc", @@ -307,7 +302,6 @@ dependencies = [ "serde", "serde_json", "sp-api", - "sp-application-crypto", "sp-arithmetic", "sp-block-builder", "sp-blockchain", @@ -315,7 +309,6 @@ dependencies = [ "sp-consensus-aura", "sp-core", "sp-io", - "sp-keystore", "sp-runtime", "sp-timestamp", "sp-transaction-pool", @@ -1266,6 +1259,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chainspec-generator" +version = "0.1.0" +dependencies = [ + "aleph-runtime", + "hex", + "libp2p", + "pallet-staking", + "primitives", + "sc-chain-spec", + "sc-cli", + "sc-client-db", + "sc-keystore", + "sc-service", + "serde", + "serde_json", + "sp-application-crypto", + "sp-core", + "sp-runtime", +] + [[package]] name = "chrono" version = "0.4.38" @@ -3244,12 +3258,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - [[package]] name = "hkdf" version = "0.12.4" @@ -10300,7 +10308,6 @@ dependencies = [ "async-trait", "clap", "frame-remote-externalities", - "frame-try-runtime", "hex", "log", "parity-scale-codec", diff --git a/Cargo.toml b/Cargo.toml index 195e8ba3d3..455564cc18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "baby-liminal-extension", "bin/node", "bin/runtime", + "bin/chainspec-generator", "clique", "finality-aleph", "pallets/aleph", @@ -116,7 +117,8 @@ pallet-proxy = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.gi sc-basic-authorship = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-block-builder = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-chain-spec = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } -sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } +sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.4.0" } +sc-client-db = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.4.0" } sc-client-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-consensus = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-consensus-aura = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } @@ -128,7 +130,7 @@ sc-network-common = { git = "https://github.com/Cardinal-Cryptography/polkadot-s sc-network-sync = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-rpc = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-rpc-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } -sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } +sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.4.0" } sc-telemetry = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-transaction-pool = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } sc-transaction-pool-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.4.0" } diff --git a/bin/node/src/chain_spec/builder.rs b/bin/chainspec-generator/src/chain_spec/builder.rs similarity index 97% rename from bin/node/src/chain_spec/builder.rs rename to bin/chainspec-generator/src/chain_spec/builder.rs index 0746c528f5..570b60c818 100644 --- a/bin/node/src/chain_spec/builder.rs +++ b/bin/chainspec-generator/src/chain_spec/builder.rs @@ -9,8 +9,10 @@ use primitives::{ }; use serde_json::{Number, Value}; use sp_runtime::Perbill; +use crate::chain_spec::AlephNodeChainSpec; +use crate::chain_spec::cli::ChainSpecParams; -use crate::chain_spec::{cli::ChainSpecParams, keystore::AccountSessionKeys, AlephNodeChainSpec}; +use crate::chain_spec::keystore::AccountSessionKeys; fn to_account_ids(authorities: &[AccountSessionKeys]) -> impl Iterator + '_ { authorities.iter().map(|auth| auth.account_id.clone()) diff --git a/bin/node/src/chain_spec/cli.rs b/bin/chainspec-generator/src/chain_spec/cli.rs similarity index 100% rename from bin/node/src/chain_spec/cli.rs rename to bin/chainspec-generator/src/chain_spec/cli.rs diff --git a/bin/node/src/chain_spec/commands.rs b/bin/chainspec-generator/src/chain_spec/commands.rs similarity index 98% rename from bin/node/src/chain_spec/commands.rs rename to bin/chainspec-generator/src/chain_spec/commands.rs index 3c8b20f99a..b9ee0e1ebb 100644 --- a/bin/node/src/chain_spec/commands.rs +++ b/bin/chainspec-generator/src/chain_spec/commands.rs @@ -5,14 +5,14 @@ use sc_cli::{ Error, KeystoreParams, }; use sc_service::config::BasePath; - +use primitives::DEFAULT_BACKUP_FOLDER; use crate::chain_spec::{ builder::build_chain_spec_json, cli::ChainSpecParams, keystore::{ create_abft_backup_dir, create_account_session_keys, create_p2p_key, open_keystore, }, - AlephNodeChainSpec, DEFAULT_BACKUP_FOLDER, + AlephNodeChainSpec, }; /// This command generates session keys and libp2p key for all input accounts. diff --git a/bin/node/src/chain_spec/keystore.rs b/bin/chainspec-generator/src/chain_spec/keystore.rs similarity index 98% rename from bin/node/src/chain_spec/keystore.rs rename to bin/chainspec-generator/src/chain_spec/keystore.rs index c17caa254e..dd3e54de4c 100644 --- a/bin/node/src/chain_spec/keystore.rs +++ b/bin/chainspec-generator/src/chain_spec/keystore.rs @@ -6,11 +6,10 @@ use std::{ use libp2p::identity::ed25519 as libp2p_ed25519; use primitives::{AccountId, AuraId, AuthorityId}; use sc_cli::KeystoreParams; -use sc_keystore::LocalKeystore; +use sc_keystore::{Keystore, LocalKeystore}; use sc_service::{config::KeystoreConfig, BasePath}; use serde::{Deserialize, Serialize}; use sp_core::crypto::key_types; -use sp_keystore::Keystore; #[derive(Clone, Deserialize, Serialize)] pub struct AccountSessionKeys { diff --git a/bin/node/src/chain_spec/mod.rs b/bin/chainspec-generator/src/chain_spec/mod.rs similarity index 75% rename from bin/node/src/chain_spec/mod.rs rename to bin/chainspec-generator/src/chain_spec/mod.rs index a49e90b0f5..7a3e301189 100644 --- a/bin/node/src/chain_spec/mod.rs +++ b/bin/chainspec-generator/src/chain_spec/mod.rs @@ -10,9 +10,9 @@ pub const CHAINTYPE_LOCAL: &str = "local"; pub const CHAINTYPE_LIVE: &str = "live"; pub const DEFAULT_CHAIN_ID: &str = "a0dnet1"; -pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; pub const DEFAULT_SUDO_ACCOUNT_ALICE: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; +// move to primitives pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; use primitives::AccountId; @@ -20,14 +20,6 @@ use sc_chain_spec::ChainType; use sc_cli::Error; use sp_application_crypto::Ss58Codec; -pub fn mainnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::mainnet_chainspec()) -} - -pub fn testnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::testnet_chainspec()) -} - fn parse_chaintype(s: &str) -> Result { Ok(match s { CHAINTYPE_DEV => ChainType::Development, diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index a416bb9e25..041c48aaea 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -21,48 +21,38 @@ log = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } futures = { workspace = true } -hex = { workspace = true } -hex-literal = { workspace = true } -libp2p = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } -sc-basic-authorship = { workspace = true } -sc-chain-spec = { workspace = true } sc-cli = { workspace = true } +sc-basic-authorship = { workspace = true } sc-client-api = { workspace = true } sc-consensus = { workspace = true } sc-consensus-aura = { workspace = true } sc-consensus-slots = { workspace = true } sc-executor = { workspace = true } -sc-keystore = { workspace = true } sc-network = { workspace = true } sc-network-sync = { workspace = true } sc-service = { workspace = true } +sc-client-db = { workspace = true } sc-telemetry = { workspace = true } sc-transaction-pool = { workspace = true } sc-transaction-pool-api = { workspace = true } -sp-application-crypto = { workspace = true } sp-arithmetic = { workspace = true } sp-block-builder = { workspace = true } sp-consensus = { workspace = true } sp-consensus-aura = { workspace = true } sp-core = { workspace = true } sp-io = { workspace = true } -sp-keystore = { workspace = true } sp-runtime = { workspace = true } sp-timestamp = { workspace = true } sp-transaction-pool = { workspace = true } -pallet-staking = { workspace = true } try-runtime-cli = { workspace = true, optional = true } frame-benchmarking-cli = { workspace = true, optional = true } frame-benchmarking = { workspace = true, optional = true } -# this is only neeeded for chainspec generation -aleph-runtime = { workspace = true } - aleph-runtime-interfaces = { workspace = true } finality-aleph = { workspace = true } primitives = { workspace = true } @@ -77,28 +67,41 @@ sp-blockchain = { workspace = true } substrate-frame-rpc-system = { workspace = true } pallet-transaction-payment-rpc = { workspace = true } +aleph-runtime = { workspace = true, optional = true } + [build-dependencies] substrate-build-script-utils = { workspace = true } [features] -default = [] -short_session = [ - "aleph-runtime/short_session", - "primitives/short_session", -] -try-runtime = [ - "aleph-runtime/try-runtime", - "try-runtime-cli/try-runtime", -] -enable_treasury_proposals = [ - "aleph-runtime/enable_treasury_proposals" +default = ["std", "rocksdb"] +std = [ + "parity-scale-codec/std", + "sp-api/std", + "sp-block-builder/std", + "sp-consensus-aura/std", + "sp-core/std", + "sp-runtime/std", + "sp-transaction-pool/std", + "primitives/std", + "sp-io/std", + "aleph-runtime-interfaces/std", ] runtime-benchmarks = [ "frame-benchmarking", "frame-benchmarking-cli/runtime-benchmarks", - "aleph-runtime/runtime-benchmarks", + "aleph-runtime?/runtime-benchmarks", + +] +rocksdb = [ + "sc-cli/rocksdb", + "sc-client-db/rocksdb", + "sc-service/rocksdb", ] +try-runtime = [ + "aleph-runtime?/try-runtime", +] + only_legacy = [ "finality-aleph/only_legacy" ] -local-debugging = [] +local-debugging = ["aleph-runtime"] diff --git a/bin/node/src/cli.rs b/bin/node/src/cli.rs index 565cc9b30a..66a4a0ad37 100644 --- a/bin/node/src/cli.rs +++ b/bin/node/src/cli.rs @@ -3,13 +3,10 @@ use sc_cli::{ PurgeChainCmd, RunCmd, SubstrateCli, }; -use crate::{ - aleph_cli::AlephCli, - chain_spec, - chain_spec::{ - commands::BootstrapChainCmd, mainnet_config, testnet_config, ConvertChainspecToRawCmd, - }, -}; +pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; + +use crate::aleph_cli::AlephCli; +use crate::resources::{mainnet_chainspec, testnet_chainspec}; #[derive(Debug, Parser)] #[clap(subcommand_negates_reqs(true), version(env!("SUBSTRATE_CLI_IMPL_VERSION")))] @@ -24,6 +21,14 @@ pub struct Cli { pub run: RunCmd, } +pub fn mainnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(mainnet_chainspec()) +} + +pub fn testnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(testnet_chainspec()) +} + impl SubstrateCli for Cli { fn impl_name() -> String { "Aleph Node".into() @@ -58,7 +63,7 @@ impl SubstrateCli for Cli { "mainnet" => mainnet_config(), "testnet" => testnet_config(), - _ => chain_spec::AlephNodeChainSpec::from_json_file(id.into()), + _ => AlephNodeChainSpec::from_json_file(id.into()), }; Ok(Box::new(chainspec?)) } @@ -70,12 +75,6 @@ pub enum Subcommand { #[command(subcommand)] Key(sc_cli::KeySubcommand), - /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout - BootstrapChain(BootstrapChainCmd), - - /// Takes a chainspec and generates a corresponding raw chainspec - ConvertChainspecToRaw(ConvertChainspecToRawCmd), - /// Validate blocks. CheckBlock(sc_cli::CheckBlockCmd), diff --git a/bin/node/src/lib.rs b/bin/node/src/lib.rs index d109e58e4d..bec742350e 100644 --- a/bin/node/src/lib.rs +++ b/bin/node/src/lib.rs @@ -1,6 +1,5 @@ mod aleph_cli; mod aleph_node_rpc; -mod chain_spec; mod cli; mod executor; mod resources; diff --git a/bin/node/src/main.rs b/bin/node/src/main.rs index 67f6da5e29..238e0c653e 100644 --- a/bin/node/src/main.rs +++ b/bin/node/src/main.rs @@ -18,8 +18,6 @@ fn main() -> sc_cli::Result<()> { let pruning_config_validation_result = PruningConfigValidator::process(&mut cli); match &cli.subcommand { - Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), - Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::CheckBlock(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/bin/node/src/service.rs b/bin/node/src/service.rs index c57948a007..013bbac4b6 100644 --- a/bin/node/src/service.rs +++ b/bin/node/src/service.rs @@ -6,14 +6,14 @@ use std::{ }; use finality_aleph::{ - run_validator_node, AlephBlockImport, AlephConfig, AllBlockMetrics, BlockImporter, - ChannelProvider, Justification, JustificationTranslator, MillisecsPerBlock, Protocol, - ProtocolNaming, RateLimiterConfig, RedirectingBlockImport, SessionPeriod, SubstrateChainStatus, + AlephBlockImport, AlephConfig, AllBlockMetrics, BlockImporter, ChannelProvider, + Justification, JustificationTranslator, MillisecsPerBlock, Protocol, ProtocolNaming, + RateLimiterConfig, RedirectingBlockImport, run_validator_node, SessionPeriod, SubstrateChainStatus, SubstrateNetwork, SyncOracle, TracingBlockImport, ValidatorAddressCache, }; use log::warn; use primitives::{ - fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, MAX_BLOCK_SIZE, + AlephSessionApi, Block, fake_runtime_api::fake_runtime::RuntimeApi, MAX_BLOCK_SIZE, DEFAULT_BACKUP_FOLDER }; use sc_basic_authorship::ProposerFactory; use sc_client_api::{BlockBackend, HeaderBackend}; @@ -21,16 +21,15 @@ use sc_consensus::ImportQueue; use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams}; use sc_consensus_slots::BackoffAuthoringBlocksStrategy; use sc_network::config::FullNetworkConfiguration; -use sc_service::{error::Error as ServiceError, Configuration, TFullClient, TaskManager}; +use sc_service::{Configuration, error::Error as ServiceError, TaskManager, TFullClient}; use sc_telemetry::{Telemetry, TelemetryWorker}; use sp_api::ProvideRuntimeApi; use sp_arithmetic::traits::BaseArithmetic; use sp_consensus::DisableProofRecording; -use sp_consensus_aura::{sr25519::AuthorityPair as AuraPair, Slot}; +use sp_consensus_aura::{Slot, sr25519::AuthorityPair as AuraPair}; use crate::{ aleph_cli::AlephCli, - chain_spec::DEFAULT_BACKUP_FOLDER, executor::aleph_executor, rpc::{create_full as create_full_rpc, FullDeps as RpcFullDeps}, }; diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index d1fff73bb6..d32c742ace 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -159,6 +159,9 @@ pub const LENIENT_THRESHOLD: Perquintill = Perquintill::from_percent(90); /// Number of non-finalized blocks that halts block production pub const DEFAULT_MAX_NON_FINALIZED_BLOCKS: u32 = 20; +/// A relative folder where to store ABFT backups +pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; + /// Hold set of validators that produce blocks and set of validators that participate in finality /// during session. #[derive(Decode, Encode, TypeInfo, Debug, Clone, PartialEq, Eq)] From 45c28037c480a358c1f0c5d38163752af74d8483 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 19 Apr 2024 15:18:47 +0200 Subject: [PATCH 02/26] WIP --- bin/chainspec-generator/Cargo.toml | 40 +++++++++++++++++++++++++++++ bin/chainspec-generator/src/main.rs | 33 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 bin/chainspec-generator/Cargo.toml create mode 100644 bin/chainspec-generator/src/main.rs diff --git a/bin/chainspec-generator/Cargo.toml b/bin/chainspec-generator/Cargo.toml new file mode 100644 index 0000000000..71c8d651fe --- /dev/null +++ b/bin/chainspec-generator/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "chainspec-generator" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +primitives = { workspace = true } +aleph-runtime = { workspace = true } +pallet-staking = { workspace = true } +libp2p = { workspace = true } +hex = { workspace = true } + +sc-cli = { workspace = true, default-features = false } +sc-chain-spec = { workspace = true } +sc-service = { workspace = true, default-features = false } +sc-client-db = { workspace = true, default-features = false } +sc-keystore = { workspace = true } + +sp-application-crypto = { workspace = true } +sp-runtime = { workspace = true } +sp-core = { workspace = true } + +serde_json = { workspace = true } +serde = { workspace = true } + +[features] +default = [] +short_session = [ + "aleph-runtime/short_session", + "primitives/short_session", +] +enable_treasury_proposals = [ + "aleph-runtime/enable_treasury_proposals" +] +try-runtime = [ + "aleph-runtime/try-runtime", +] diff --git a/bin/chainspec-generator/src/main.rs b/bin/chainspec-generator/src/main.rs new file mode 100644 index 0000000000..efcf018325 --- /dev/null +++ b/bin/chainspec-generator/src/main.rs @@ -0,0 +1,33 @@ +mod chain_spec; + +use sc_cli::{ + clap::{self, Parser, Subcommand as ClapSubcommand}}; +use crate::chain_spec::{BootstrapChainCmd, ConvertChainspecToRawCmd}; + +#[derive(Parser)] +struct Cli { + #[command(subcommand)] + pub subcommand: Option, +} + +#[derive(Debug, ClapSubcommand)] +pub enum Subcommand { + /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout + BootstrapChain(BootstrapChainCmd), + + /// Takes a chainspec and generates a corresponding raw chainspec + ConvertChainspecToRaw(ConvertChainspecToRawCmd), +} + +fn main() -> sc_cli::Result<()> { + let mut cli = Cli::parse(); + + match &cli.subcommand { + Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), + Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), + + None => { + Err(format!("Command was required!").into()) + } + } +} From 881df2a34dcaf9b8bd6599a0365922368c9cfe81 Mon Sep 17 00:00:00 2001 From: Marcin Date: Mon, 22 Apr 2024 15:00:49 +0200 Subject: [PATCH 03/26] Features compile --- .../nightly-check-node-features-build.yml | 4 ++-- BUILD.md | 4 ++-- Cargo.lock | 1 + bin/chainspec-generator/src/main.rs | 2 +- bin/node/Cargo.toml | 12 ++++++++---- bin/node/src/executor.rs | 4 ++-- bin/node/src/lib.rs | 2 +- scripts/run_nodes.sh | 18 ++++++++++-------- 8 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index a393e48e27..4cf4c9a50f 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -32,8 +32,8 @@ jobs: - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked - - name: aleph-node with local-debugging - run: cargo check --profile dev -p aleph-node --features local-debugging --locked + - name: aleph-node with aleph-native-runtime + run: cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked slack-notification: name: Slack notification diff --git a/BUILD.md b/BUILD.md index 1ab7fd5ec4..82bde88e8b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -106,9 +106,9 @@ After a successful build the binary can be found in `target/release/aleph-node`. ### Local debugging -If you'd like to use Rust debugger with aleph-node binary, built it with `local-debugging` feature: +If you'd like to use Rust debugger with aleph-node binary, built it with `aleph-native-runtime` feature: ``` -cargo check --profile dev -p aleph-node --features local-debugging --locked +cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked ``` [nix]: https://nixos.org/download.html diff --git a/Cargo.lock b/Cargo.lock index 2175701fb6..f6c91812a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10308,6 +10308,7 @@ dependencies = [ "async-trait", "clap", "frame-remote-externalities", + "frame-try-runtime", "hex", "log", "parity-scale-codec", diff --git a/bin/chainspec-generator/src/main.rs b/bin/chainspec-generator/src/main.rs index efcf018325..9f35cf56cd 100644 --- a/bin/chainspec-generator/src/main.rs +++ b/bin/chainspec-generator/src/main.rs @@ -20,7 +20,7 @@ pub enum Subcommand { } fn main() -> sc_cli::Result<()> { - let mut cli = Cli::parse(); + let cli = Cli::parse(); match &cli.subcommand { Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index 041c48aaea..f50bfcb714 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -48,8 +48,6 @@ sp-runtime = { workspace = true } sp-timestamp = { workspace = true } sp-transaction-pool = { workspace = true } -try-runtime-cli = { workspace = true, optional = true } - frame-benchmarking-cli = { workspace = true, optional = true } frame-benchmarking = { workspace = true, optional = true } @@ -68,6 +66,7 @@ substrate-frame-rpc-system = { workspace = true } pallet-transaction-payment-rpc = { workspace = true } aleph-runtime = { workspace = true, optional = true } +try-runtime-cli = { workspace = true, optional = true } [build-dependencies] substrate-build-script-utils = { workspace = true } @@ -87,10 +86,10 @@ std = [ "aleph-runtime-interfaces/std", ] runtime-benchmarks = [ + "aleph-runtime-native", "frame-benchmarking", "frame-benchmarking-cli/runtime-benchmarks", "aleph-runtime?/runtime-benchmarks", - ] rocksdb = [ "sc-cli/rocksdb", @@ -98,10 +97,15 @@ rocksdb = [ "sc-service/rocksdb", ] try-runtime = [ + "aleph-runtime-native", + "dep:try-runtime-cli", "aleph-runtime?/try-runtime", + "try-runtime-cli?/try-runtime", +] +aleph-runtime-native = [ + "dep:aleph-runtime" ] only_legacy = [ "finality-aleph/only_legacy" ] -local-debugging = ["aleph-runtime"] diff --git a/bin/node/src/executor.rs b/bin/node/src/executor.rs index 749e94c41f..71693a3755 100644 --- a/bin/node/src/executor.rs +++ b/bin/node/src/executor.rs @@ -6,7 +6,7 @@ use sc_service::Configuration; #[cfg(not(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" )))] pub mod aleph_executor { @@ -27,7 +27,7 @@ pub mod aleph_executor { #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub mod aleph_executor { diff --git a/bin/node/src/lib.rs b/bin/node/src/lib.rs index bec742350e..98bd3744e6 100644 --- a/bin/node/src/lib.rs +++ b/bin/node/src/lib.rs @@ -9,7 +9,7 @@ mod service; pub use cli::{Cli, Subcommand}; #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub use executor::aleph_executor::ExecutorDispatch; diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index bb38ce3107..2af5c9d5dc 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -39,7 +39,8 @@ set -euo pipefail # ------------------------ constants -------------------------------------- -export ALEPH_NODE="target/release/aleph-node" +ALEPH_NODE="target/release/aleph-node" +CHAINSPEC_GENERATOR="target/release/chainspec-generator" NODE_P2P_PORT_RANGE_START=30333 NODE_VALIDATOR_PORT_RANGE_START=30343 NODE_RPC_PORT_RANGE_START=9944 @@ -193,8 +194,8 @@ function run_node() { # ------------------------- input checks ---------------------------------- -if [[ "${VALIDATORS}" -lt 4 ]]; then - error "Number of validators should be at least 4!" +if [[ "${VALIDATORS}" -lt 1 ]]; then + error "Number of validators should be at least 1!" fi if [[ "${RPC_NODES}" -lt 1 ]]; then error "Number of RPC nodes should be at least 1!" @@ -227,10 +228,11 @@ if ! killall -9 aleph-node 2> /dev/null; then fi if [[ -z "${DONT_BUILD_ALEPH_NODE}" ]]; then - info "Building testing aleph-node binary (short session)." - cargo build --release -p aleph-node --features "short_session enable_treasury_proposals" -elif [[ ! -x "${ALEPH_NODE}" ]]; then - error "${ALEPH_NODE} does not exist or it's not an executable file!" + info "Building testing aleph-node binary (short session) and chainspec-generator binary." + cargo build --release -p aleph-node + cargo build --release -p chainspec-generator --features "short_session enable_treasury_proposals" +elif [[ ! -x "${ALEPH_NODE}" || ! -x "${CHAINSPEC_GENERATOR}" ]]; then + error "${ALEPH_NODE} or ${CHAINSPEC_GENERATOR} does not exist or it's not an executable file!" fi NUMBER_OF_NODES_TO_BOOTSTRAP=$(( VALIDATORS + RPC_NODES )) @@ -263,7 +265,7 @@ if [[ -z "${DONT_BOOTSTRAP}" ]]; then validator_ids_string="${validator_ids_string//${IFS:0:1}/,}" info "Populating keystore for all accounts with session keys and libp2p key, and generating chainspec" - "${ALEPH_NODE}" bootstrap-chain \ + "${CHAINSPEC_GENERATOR}" bootstrap-chain \ --raw \ --base-path "${BASE_PATH}" \ --account-ids "${all_account_ids_string}" \ From 4477dc14f5153436ec5157df15a8dd606f64c72c Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 10:13:33 +0200 Subject: [PATCH 04/26] CI changes --- .github/actions/run-e2e-test/action.yml | 11 ++- .github/scripts/run_consensus.sh | 5 +- .github/scripts/test_python_general.sh | 20 ++++- .../workflows/_build-chainspec-generator.yml | 63 ++++++++++++++++ ...e-and-runtime.yml => _build-test-node.yml} | 24 +++--- .github/workflows/deploy-to-devnet.yml | 2 +- .../nightly-check-node-features-build.yml | 9 +++ .github/workflows/nightly-e2e-logic-tests.yml | 26 +++++-- .../nightly-normal-session-e2e-tests.yml | 74 +++++++++++++++---- .github/workflows/on-pull-request-commit.yml | 11 ++- Cargo.lock | 2 +- Cargo.toml | 2 +- .../Cargo.toml | 2 +- bin/chain-bootstrapper/Dockerfile | 6 ++ .../src/chain_spec/builder.rs | 0 .../src/chain_spec/cli.rs | 0 .../src/chain_spec/commands.rs | 0 .../src/chain_spec/keystore.rs | 0 .../src/chain_spec/mod.rs | 0 .../src/main.rs | 0 local-tests/chainrunner/chain.py | 8 +- local-tests/run_nodes.py | 8 +- local-tests/test_force_reorg.py | 9 ++- local-tests/test_major_sync.py | 12 ++- local-tests/test_recover_after_abft_update.py | 8 +- scripts/run_nodes.sh | 6 +- 26 files changed, 243 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/_build-chainspec-generator.yml rename .github/workflows/{_build-test-node-and-runtime.yml => _build-test-node.yml} (67%) rename bin/{chainspec-generator => chain-bootstrapper}/Cargo.toml (97%) create mode 100644 bin/chain-bootstrapper/Dockerfile rename bin/{chainspec-generator => chain-bootstrapper}/src/chain_spec/builder.rs (100%) rename bin/{chainspec-generator => chain-bootstrapper}/src/chain_spec/cli.rs (100%) rename bin/{chainspec-generator => chain-bootstrapper}/src/chain_spec/commands.rs (100%) rename bin/{chainspec-generator => chain-bootstrapper}/src/chain_spec/keystore.rs (100%) rename bin/{chainspec-generator => chain-bootstrapper}/src/chain_spec/mod.rs (100%) rename bin/{chainspec-generator => chain-bootstrapper}/src/main.rs (100%) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 2fada9a7ce..645b2e1c99 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -54,7 +54,7 @@ runs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v5 - - name: Download artifact with docker image + - name: Download node docker image uses: actions/download-artifact@v4 with: name: ${{ inputs.image-path }} @@ -63,6 +63,15 @@ runs: shell: bash run: docker load -i aleph-node.tar + - name: Download chain-booststrapper image + uses: actions/download-artifact@v4 + with: + name: chain-booststrapper + + - name: Load chain-booststrapper image + shell: bash + run: docker load -i chain-booststrapper.tar + - name: Run consensus party shell: bash run: | diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 889fdaf42c..821f9849bf 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -20,6 +20,7 @@ MIN_VALIDATOR_COUNT=${MIN_VALIDATOR_COUNT:-4} DOCKER_COMPOSE=${DOCKER_COMPOSE:-docker/docker-compose.yml} OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""} NODE_IMAGE=${NODE_IMAGE:-"aleph-node:latest"} +CHAIN_BOOTSTRAPPER=${CHAIN_BOOTSTRAPPER:-"chain-bootstrapper:latest"} LOGS_OUTPUT_FILE=${LOGS_OUTPUT_FILE:=""} # ------------------------ argument parsing and usage ----------------------- @@ -81,8 +82,8 @@ function generate_chainspec() { local validator_ids_comma_separated="${validators//${IFS:0:1}/,}" echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}" - docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \ - -c "aleph-node bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" + docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER}" \ + -c "bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" } function generate_bootnode_peer_id() { diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index a7a2e0d89d..49cd19d860 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -8,8 +8,10 @@ function usage(){ cat << EOF Usage: $0 - --aleph-node BINARY] - path to aleph-node-binary + --aleph-node BINARY + path to aleph-node binary + --chain-bootstrapper BINARY + path to chain-bootstrapper binary --testcase NAME name of python file in local-tests directory to run EOF @@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; do ALEPH_NODE_BINARY="$2" shift;shift ;; + --chain-bootstrapper) + CHAIN_BOOTSTRAPPER="$2" + shift;shift + ;; --testcase) TESTCASE="$2" shift;shift @@ -39,8 +45,12 @@ done # relative paths should be improved in python scripts CI, as it matters that below line is pushd local-tests/ > /dev/null -if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then - echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}." +if [[ ! -x "${ALEPH_NODE_BINARY}" ]]; then + echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}, or it's not executable." + exit 1 +fi +if [[ ! -x "${CHAIN_BOOTSTRAPPER}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}, or it's not executable." exit 1 fi if [[ -z "${TESTCASE}" ]]; then @@ -56,6 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then fi chmod +x "${ALEPH_NODE_BINARY}" +chmod +x "${CHAIN_BOOTSTRAPPER}" echo "Installing python requirements" pip install -r requirements.txt @@ -65,6 +76,7 @@ pip install -r requirements.txt # first buffered and that you can see the output of your application. export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY +export CHAIN_BOOTSTRAPPER export WORKDIR=$(mktemp -d) eval "./${file_name_to_run}" popd > /dev/null diff --git a/.github/workflows/_build-chainspec-generator.yml b/.github/workflows/_build-chainspec-generator.yml new file mode 100644 index 0000000000..2c69c22939 --- /dev/null +++ b/.github/workflows/_build-chainspec-generator.yml @@ -0,0 +1,63 @@ +--- +# This workflow builds chain-bootstrapper binary, dedicated fer generating chainspecs for +# test and dev chains +name: Build chain-bootstrapper +on: + workflow_call: + inputs: + ref: + description: 'git ref: hash, branch, tag to build chain-bootstrapper binary from' + type: string + required: true + +jobs: + main: + name: Build chain-bootstrapper + runs-on: [self-hosted, Linux, X64, large] + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + steps: + - name: Checkout aleph-node source code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Call action get-ref-properties + id: get-ref-properties + uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 + + - name: Install Rust toolchain + uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 + with: + targets: wasm32-unknown-unknown + + - name: Build + run: | + cargo build --release -p chain-bootstrapper \ + --features "short_session enable_treasury_proposals only_legacy" + + - name: Upload binary to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper + path: target/release/chain-bootstrapper + if-no-files-found: error + retention-days: 7 + + - name: Build docker + id: build-image + run: | + chmod +x target/release/chain-bootstrapper + docker build --tag chain-bootstrapper:latest -f ./bin/chain-bootstrapper/Dockerfile . + docker save -o chain-bootstrapper.tar chain-bootstrapper:latest + + # this is required by action/run-e2e-test.yml + - name: Upload docker image to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper + path: chain-bootstrapper.tar + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/_build-test-node-and-runtime.yml b/.github/workflows/_build-test-node.yml similarity index 67% rename from .github/workflows/_build-test-node-and-runtime.yml rename to .github/workflows/_build-test-node.yml index 0c70cf297e..65f0077f41 100644 --- a/.github/workflows/_build-test-node-and-runtime.yml +++ b/.github/workflows/_build-test-node.yml @@ -1,7 +1,6 @@ --- -# This workflow builds test version of aleph-node and aleph-runtime, ie with some -# features enabled. Those binaries MUST NOT be used in the production. -name: Build test node and runtime +# This workflow builds test version of aleph-node. It MUST NOT be used in the production. +name: Build test node on: workflow_call: inputs: @@ -11,8 +10,8 @@ on: required: true jobs: - build-artifacts: - name: Build test node and runtime + main: + name: Build test node runs-on: [self-hosted, Linux, X64, large] env: RUST_BACKTRACE: full @@ -30,16 +29,13 @@ jobs: - name: Install Rust toolchain uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 - with: - targets: wasm32-unknown-unknown - - name: Build test binary and runtime + - name: Build run: | - cargo build --release -p aleph-node \ - --features "short_session enable_treasury_proposals only_legacy" + cargo build --release -p aleph-node # this is required for some workflows, yet ideally it should not be required - - name: Upload test binary to GH Artifacts + - name: Upload binary to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-node @@ -47,15 +43,15 @@ jobs: if-no-files-found: error retention-days: 7 - - name: Build test docker image + - name: Build docker id: build-image run: | chmod +x target/release/aleph-node docker build --tag aleph-node:latest -f ./docker/Dockerfile . docker save -o aleph-node.tar aleph-node:latest - # this is solely required by action/run-e2e-test.yml, also it should not be required - - name: Upload test docker image + # this is required by action/run-e2e-test.yml + - name: Upload docker to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-docker diff --git a/.github/workflows/deploy-to-devnet.yml b/.github/workflows/deploy-to-devnet.yml index be63cc9459..76d86e8ae1 100644 --- a/.github/workflows/deploy-to-devnet.yml +++ b/.github/workflows/deploy-to-devnet.yml @@ -98,7 +98,7 @@ jobs: EVE=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint \ - '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}aleph-node:${{ env.RELEASE_TAG }}' \ + '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \ bootstrap-chain --raw --base-path /data --chain-id a0dnet1 \ --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q \ --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD \ diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index 4cf4c9a50f..18199c529f 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -29,12 +29,21 @@ jobs: - name: aleph-runtime with try-runtime run: cargo check --profile production -p aleph-runtime --features try-runtime --locked + - name: aleph-runtime with runtime-benchmarks + run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks --locked + - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked + - name: aleph-node with try-runtime + run: cargo check --profile production -p aleph-node --features try-runtime --locked + - name: aleph-node with aleph-native-runtime run: cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked + - name: chain-bootstrapper with try-runtime + run: cargo check --profile production -p chain-bootstrapper --features try-runtime --locked + slack-notification: name: Slack notification runs-on: ubuntu-20.04 diff --git a/.github/workflows/nightly-e2e-logic-tests.yml b/.github/workflows/nightly-e2e-logic-tests.yml index 97a8f831b3..918c11191f 100644 --- a/.github/workflows/nightly-e2e-logic-tests.yml +++ b/.github/workflows/nightly-e2e-logic-tests.yml @@ -19,9 +19,15 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit - build-test-node-and-runtime: + build-test-node: name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml + with: + ref: ${{ github.ref }} + + build-chain-bootstrapper: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml with: ref: ${{ github.ref }} @@ -36,31 +42,41 @@ jobs: run-e2e-tests: name: Run e2e tests - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node + - build-chain-bootstrapper + - build-aleph-e2e-client-image uses: ./.github/workflows/_run-e2e-tests.yml with: # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} run-recover-after-abft-update-test: - needs: [build-test-node-and-runtime] + needs: [build-test-node, build-chain-bootstrapper] name: Run recovery test after abft update runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download test node from GH artifacts uses: actions/download-artifact@v4 with: name: aleph-test-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper + path: target/release/ + - name: Run test timeout-minutes: 15 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_recover_after_abft_update check-nightly-pipeline-completion: diff --git a/.github/workflows/nightly-normal-session-e2e-tests.yml b/.github/workflows/nightly-normal-session-e2e-tests.yml index 64e3030522..4fa284c125 100644 --- a/.github/workflows/nightly-normal-session-e2e-tests.yml +++ b/.github/workflows/nightly-normal-session-e2e-tests.yml @@ -37,8 +37,17 @@ jobs: name: Build synthetic node uses: ./.github/workflows/_build-synthetic-node.yml + build-chain-bootstrapper: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + run-e2e-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Run high out-latency test runs-on: ubuntu-20.04 steps: @@ -57,7 +66,10 @@ jobs: timeout-minutes: 40 run-e2e-sync-test-one_node_catching_up_and_then_becoming_necessary_for_consensus: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - one node catching up and then becoming necessary for consensus runs-on: ubuntu-20.04 @@ -82,7 +94,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-one_node_catching_up: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - one node catching up runs-on: ubuntu-20.04 steps: @@ -106,7 +121,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_and_one_quorum_and_switch_quorum_between_them: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - into two groups and one quorum and switch quorum between them runs-on: ubuntu-20.04 @@ -131,7 +149,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_multiple_groups_of_two: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - into multiple groups of two runs-on: ubuntu-20.04 steps: @@ -155,7 +176,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_equal_size_groups_with_no_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - into two equal size groups with no quorum runs-on: ubuntu-20.04 @@ -180,7 +204,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_one_with_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Sync test using synthetic-network - into two groups one with quorum runs-on: ubuntu-20.04 @@ -205,7 +232,10 @@ jobs: timeout-minutes: 35 run-e2e-finalization-stall: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Finalization-stall test using synthetic-network runs-on: ubuntu-20.04 steps: @@ -229,50 +259,66 @@ jobs: timeout-minutes: 60 run-major-sync-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper] name: Run major sync test runs-on: ubuntu-20.04 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper + path: target/release/ + - name: Run test timeout-minutes: 65 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_major_sync run-force-reorg-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper] name: Run force reorgs test runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper + path: target/release/ + - name: Run test timeout-minutes: 25 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_force_reorg - run-e2e-no-quorum-without-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper name: Run high out-latency for every quorum runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/on-pull-request-commit.yml b/.github/workflows/on-pull-request-commit.yml index cb91dd95f9..36f6ed79ce 100644 --- a/.github/workflows/on-pull-request-commit.yml +++ b/.github/workflows/on-pull-request-commit.yml @@ -44,6 +44,12 @@ jobs: with: ref: ${{ github.ref }} + build-chain-bootstrapper: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + build-aleph-e2e-client-image: name: Build aleph-e2e-client image uses: ./.github/workflows/_build-aleph-e2e-client-image.yml @@ -53,7 +59,10 @@ jobs: run-e2e-finalization-test: name: Run e2e finalization test - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node-and-runtime + - build-aleph-e2e-client-image + - build-chain-bootstrapper runs-on: ubuntu-20.04 steps: - name: Checkout source code diff --git a/Cargo.lock b/Cargo.lock index f6c91812a0..81157fe62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1260,7 +1260,7 @@ dependencies = [ ] [[package]] -name = "chainspec-generator" +name = "chain-bootstrapper" version = "0.1.0" dependencies = [ "aleph-runtime", diff --git a/Cargo.toml b/Cargo.toml index 455564cc18..f51e34f63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ "baby-liminal-extension", "bin/node", "bin/runtime", - "bin/chainspec-generator", + "bin/chain-bootstrapper", "clique", "finality-aleph", "pallets/aleph", diff --git a/bin/chainspec-generator/Cargo.toml b/bin/chain-bootstrapper/Cargo.toml similarity index 97% rename from bin/chainspec-generator/Cargo.toml rename to bin/chain-bootstrapper/Cargo.toml index 71c8d651fe..ee17515c56 100644 --- a/bin/chainspec-generator/Cargo.toml +++ b/bin/chain-bootstrapper/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "chainspec-generator" +name = "chain-bootstrapper" version = "0.1.0" authors.workspace = true edition.workspace = true diff --git a/bin/chain-bootstrapper/Dockerfile b/bin/chain-bootstrapper/Dockerfile new file mode 100644 index 0000000000..195f7b86d2 --- /dev/null +++ b/bin/chain-bootstrapper/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:jammy + +COPY target/release/chain-bootstrapper /usr/local/bin +RUN chmod +x /usr/local/bin/chain-bootstrapper + +CMD ["/usr/local/bin/chain-bootstrapper"] diff --git a/bin/chainspec-generator/src/chain_spec/builder.rs b/bin/chain-bootstrapper/src/chain_spec/builder.rs similarity index 100% rename from bin/chainspec-generator/src/chain_spec/builder.rs rename to bin/chain-bootstrapper/src/chain_spec/builder.rs diff --git a/bin/chainspec-generator/src/chain_spec/cli.rs b/bin/chain-bootstrapper/src/chain_spec/cli.rs similarity index 100% rename from bin/chainspec-generator/src/chain_spec/cli.rs rename to bin/chain-bootstrapper/src/chain_spec/cli.rs diff --git a/bin/chainspec-generator/src/chain_spec/commands.rs b/bin/chain-bootstrapper/src/chain_spec/commands.rs similarity index 100% rename from bin/chainspec-generator/src/chain_spec/commands.rs rename to bin/chain-bootstrapper/src/chain_spec/commands.rs diff --git a/bin/chainspec-generator/src/chain_spec/keystore.rs b/bin/chain-bootstrapper/src/chain_spec/keystore.rs similarity index 100% rename from bin/chainspec-generator/src/chain_spec/keystore.rs rename to bin/chain-bootstrapper/src/chain_spec/keystore.rs diff --git a/bin/chainspec-generator/src/chain_spec/mod.rs b/bin/chain-bootstrapper/src/chain_spec/mod.rs similarity index 100% rename from bin/chainspec-generator/src/chain_spec/mod.rs rename to bin/chain-bootstrapper/src/chain_spec/mod.rs diff --git a/bin/chainspec-generator/src/main.rs b/bin/chain-bootstrapper/src/main.rs similarity index 100% rename from bin/chainspec-generator/src/main.rs rename to bin/chain-bootstrapper/src/main.rs diff --git a/local-tests/chainrunner/chain.py b/local-tests/chainrunner/chain.py index e3b66eebec..548dc6a93a 100644 --- a/local-tests/chainrunner/chain.py +++ b/local-tests/chainrunner/chain.py @@ -43,16 +43,16 @@ def new(self, binary, validators, nonvalidators=None): self.nodes = self.validator_nodes + self.nonvalidator_nodes - def bootstrap(self, binary, validators, nonvalidators=None, raw=True, **kwargs): + def bootstrap(self, aleph_node_binary, chain_bootstrapper_binary, validators, nonvalidators=None, raw=True, **kwargs): """Bootstrap the chain. `validators` and `nonvalidators` should be lists of strings with public keys. Flags `--account-ids`, `--base-path` and `--raw` are added automatically. All other flags are taken from kwargs""" - self.new(binary, validators, nonvalidators) + self.new(aleph_node_binary, validators, nonvalidators) nonvalidators = nonvalidators or [] all_accounts = list(validators) + list(nonvalidators) - binary = check_file(binary) - cmd = [binary, 'bootstrap-chain', + binary = check_file(chain_bootstrapper_binary) + cmd = [chain_bootstrapper_binary, 'bootstrap-chain', '--base-path', self.path, '--account-ids', ','.join(all_accounts), '--authorities-account-ids', ','.join(validators)] diff --git a/local-tests/run_nodes.py b/local-tests/run_nodes.py index e801ed2fc7..fff35abd92 100755 --- a/local-tests/run_nodes.py +++ b/local-tests/run_nodes.py @@ -9,19 +9,21 @@ nodes = 4 workdir = '.' -binary = '../target/release/aleph-node' +aleph_node_binary = '../target/release/aleph-node' +chain_bootstrapper_binary = '../target/release/chain-bootstrapper' port = 30334 rpc_port = 9944 phrases = ['//Alice', '//Bob', '//Charlie', '//Dave', '//Ezekiel', '//Fanny', '//George', '//Hugo'] -keys_dict = generate_keys(binary, phrases) +keys_dict = generate_keys(aleph_node_binary, phrases) keys = list(keys_dict.values()) nodes = min(nodes, len(phrases)) chain = Chain(workdir) print(f'Bootstrapping chain for {nodes} nodes') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys[:nodes], chain_type='local') chain.set_flags('validator', diff --git a/local-tests/test_force_reorg.py b/local-tests/test_force_reorg.py index eb9ee1cb50..90da775088 100755 --- a/local-tests/test_force_reorg.py +++ b/local-tests/test_force_reorg.py @@ -12,14 +12,17 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary (important DON'T use short-session feature): -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) + phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_major_sync.py b/local-tests/test_major_sync.py index 561d665d49..f9d1c29a8a 100755 --- a/local-tests/test_major_sync.py +++ b/local-tests/test_major_sync.py @@ -12,15 +12,19 @@ workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) logging.info(f"Workdir is {workdir}") -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) -logging.info(f"aleph-node binary is {binary}") +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) +logging.info(f"aleph-node binary is {aleph_node_binary}") +logging.info(f"chain_bootstrapper binary is {chain_bootstrapper_binary}") + phrases = [f'//{i}' for i in range(5)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_recover_after_abft_update.py b/local-tests/test_recover_after_abft_update.py index 8f40586160..8b8926204d 100755 --- a/local-tests/test_recover_after_abft_update.py +++ b/local-tests/test_recover_after_abft_update.py @@ -17,14 +17,16 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary. -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index 2af5c9d5dc..e272181fdd 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -40,7 +40,7 @@ set -euo pipefail # ------------------------ constants -------------------------------------- ALEPH_NODE="target/release/aleph-node" -CHAINSPEC_GENERATOR="target/release/chainspec-generator" +CHAINSPEC_GENERATOR="target/release/chain-bootstrapper" NODE_P2P_PORT_RANGE_START=30333 NODE_VALIDATOR_PORT_RANGE_START=30343 NODE_RPC_PORT_RANGE_START=9944 @@ -228,9 +228,9 @@ if ! killall -9 aleph-node 2> /dev/null; then fi if [[ -z "${DONT_BUILD_ALEPH_NODE}" ]]; then - info "Building testing aleph-node binary (short session) and chainspec-generator binary." + info "Building testing aleph-node binary (short session) and chain-bootstrapper binary." cargo build --release -p aleph-node - cargo build --release -p chainspec-generator --features "short_session enable_treasury_proposals" + cargo build --release -p chain-bootstrapper --features "short_session enable_treasury_proposals" elif [[ ! -x "${ALEPH_NODE}" || ! -x "${CHAINSPEC_GENERATOR}" ]]; then error "${ALEPH_NODE} or ${CHAINSPEC_GENERATOR} does not exist or it's not an executable file!" fi From 1296628ec80b029627e042dc01ddc6965a51ff91 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 10:25:12 +0200 Subject: [PATCH 05/26] Fix CI --- .../workflows/_build-and-push-featurenet-node-image.yml | 8 ++++---- .github/workflows/_check-production-node-and-runtime.yml | 5 ++++- .github/workflows/nightly-check-node-features-build.yml | 3 ++- .github/workflows/on-pull-request-commit.yml | 8 ++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_build-and-push-featurenet-node-image.yml b/.github/workflows/_build-and-push-featurenet-node-image.yml index bcabd1e62b..fe9c6fcfc0 100644 --- a/.github/workflows/_build-and-push-featurenet-node-image.yml +++ b/.github/workflows/_build-and-push-featurenet-node-image.yml @@ -17,10 +17,10 @@ on: type: boolean required: true jobs: - build-test-node-and-runtime: + build-test-node: if: ${{ inputs.short-session == true }} name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ inputs.ref }} @@ -33,8 +33,8 @@ jobs: build-and-push-featurnet-image-to-ecr: name: Build and push featurnet image - needs: [build-test-node-and-runtime, build-production-node-and-runtime] - # to prevent this job to be skipped as at least on of dependant jobs is skipped + needs: [build-test-node, build-production-node-and-runtime] + # this job should not be skipped in case one of its parent job is skipped if: ${{ !cancelled() }} runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/_check-production-node-and-runtime.yml b/.github/workflows/_check-production-node-and-runtime.yml index ec01b19f77..508c012eff 100644 --- a/.github/workflows/_check-production-node-and-runtime.yml +++ b/.github/workflows/_check-production-node-and-runtime.yml @@ -25,5 +25,8 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Check production binary and runtime + - name: Check aleph-node run: cargo check --profile production -p aleph-node + + - name: Check aleph-runtime + run: cargo check --profile production -p aleph-runtime diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index 18199c529f..fd13e2402b 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -30,7 +30,8 @@ jobs: run: cargo check --profile production -p aleph-runtime --features try-runtime --locked - name: aleph-runtime with runtime-benchmarks - run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks --locked + run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks \ + --locked - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked diff --git a/.github/workflows/on-pull-request-commit.yml b/.github/workflows/on-pull-request-commit.yml index 36f6ed79ce..b6cca23aa5 100644 --- a/.github/workflows/on-pull-request-commit.yml +++ b/.github/workflows/on-pull-request-commit.yml @@ -38,9 +38,9 @@ jobs: name: Check production node and runtime uses: ./.github/workflows/_check-production-node-and-runtime.yml - build-test-node-and-runtime: - name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + build-test-node: + name: Build test node + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ github.ref }} @@ -60,7 +60,7 @@ jobs: run-e2e-finalization-test: name: Run e2e finalization test needs: - - build-test-node-and-runtime + - build-test-node - build-aleph-e2e-client-image - build-chain-bootstrapper runs-on: ubuntu-20.04 From d9dc8040b51bf286702366979ada837b1f622059 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 10:25:59 +0200 Subject: [PATCH 06/26] Rename new workflow --- ...uild-chainspec-generator.yml => _build-chain-bootstrapper.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{_build-chainspec-generator.yml => _build-chain-bootstrapper.yml} (100%) diff --git a/.github/workflows/_build-chainspec-generator.yml b/.github/workflows/_build-chain-bootstrapper.yml similarity index 100% rename from .github/workflows/_build-chainspec-generator.yml rename to .github/workflows/_build-chain-bootstrapper.yml From 0b4bc569057d4ecf610f6649003952feaaaa642c Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 11:00:24 +0200 Subject: [PATCH 07/26] Clippy --- .github/workflows/_build-chain-bootstrapper.yml | 2 +- .github/workflows/_build-test-node.yml | 2 +- bin/chain-bootstrapper/src/main.rs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_build-chain-bootstrapper.yml b/.github/workflows/_build-chain-bootstrapper.yml index 2c69c22939..56951c2e21 100644 --- a/.github/workflows/_build-chain-bootstrapper.yml +++ b/.github/workflows/_build-chain-bootstrapper.yml @@ -36,7 +36,7 @@ jobs: - name: Build run: | cargo build --release -p chain-bootstrapper \ - --features "short_session enable_treasury_proposals only_legacy" + --features "short_session enable_treasury_proposals" - name: Upload binary to GH Artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_build-test-node.yml b/.github/workflows/_build-test-node.yml index 65f0077f41..876c02615b 100644 --- a/.github/workflows/_build-test-node.yml +++ b/.github/workflows/_build-test-node.yml @@ -32,7 +32,7 @@ jobs: - name: Build run: | - cargo build --release -p aleph-node + cargo build --release -p aleph-node --features only_legacy # this is required for some workflows, yet ideally it should not be required - name: Upload binary to GH Artifacts diff --git a/bin/chain-bootstrapper/src/main.rs b/bin/chain-bootstrapper/src/main.rs index 9f35cf56cd..215d81507e 100644 --- a/bin/chain-bootstrapper/src/main.rs +++ b/bin/chain-bootstrapper/src/main.rs @@ -10,6 +10,7 @@ struct Cli { pub subcommand: Option, } +#[allow(clippy::large_enum_variant)] #[derive(Debug, ClapSubcommand)] pub enum Subcommand { /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout @@ -27,7 +28,7 @@ fn main() -> sc_cli::Result<()> { Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), None => { - Err(format!("Command was required!").into()) + Err("Command was required!".into()) } } } From e722aef2e8cd1e5a9ef9028bd2652526f71899df Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 11:33:11 +0200 Subject: [PATCH 08/26] CI fixes --- .dockerignore | 2 ++ bin/chain-bootstrapper/src/chain_spec/builder.rs | 4 +--- bin/chain-bootstrapper/src/chain_spec/commands.rs | 3 ++- bin/chain-bootstrapper/src/main.rs | 8 +++----- bin/node/src/cli.rs | 6 ++++-- bin/node/src/service.rs | 13 +++++++------ 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.dockerignore b/.dockerignore index 644ad99be7..fc9865afbe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ **/target target !target/release/aleph-node +!target/release/chain-bootstrapper +!target/production/chain-bootstrapper !target/production/aleph-node !bin/cliain/target/release/cliain diff --git a/bin/chain-bootstrapper/src/chain_spec/builder.rs b/bin/chain-bootstrapper/src/chain_spec/builder.rs index 570b60c818..0746c528f5 100644 --- a/bin/chain-bootstrapper/src/chain_spec/builder.rs +++ b/bin/chain-bootstrapper/src/chain_spec/builder.rs @@ -9,10 +9,8 @@ use primitives::{ }; use serde_json::{Number, Value}; use sp_runtime::Perbill; -use crate::chain_spec::AlephNodeChainSpec; -use crate::chain_spec::cli::ChainSpecParams; -use crate::chain_spec::keystore::AccountSessionKeys; +use crate::chain_spec::{cli::ChainSpecParams, keystore::AccountSessionKeys, AlephNodeChainSpec}; fn to_account_ids(authorities: &[AccountSessionKeys]) -> impl Iterator + '_ { authorities.iter().map(|auth| auth.account_id.clone()) diff --git a/bin/chain-bootstrapper/src/chain_spec/commands.rs b/bin/chain-bootstrapper/src/chain_spec/commands.rs index b9ee0e1ebb..e9cf6a1485 100644 --- a/bin/chain-bootstrapper/src/chain_spec/commands.rs +++ b/bin/chain-bootstrapper/src/chain_spec/commands.rs @@ -1,11 +1,12 @@ use std::{io::Write, path::PathBuf}; +use primitives::DEFAULT_BACKUP_FOLDER; use sc_cli::{ clap::{self, Parser}, Error, KeystoreParams, }; use sc_service::config::BasePath; -use primitives::DEFAULT_BACKUP_FOLDER; + use crate::chain_spec::{ builder::build_chain_spec_json, cli::ChainSpecParams, diff --git a/bin/chain-bootstrapper/src/main.rs b/bin/chain-bootstrapper/src/main.rs index 215d81507e..4ba27a10a9 100644 --- a/bin/chain-bootstrapper/src/main.rs +++ b/bin/chain-bootstrapper/src/main.rs @@ -1,7 +1,7 @@ mod chain_spec; -use sc_cli::{ - clap::{self, Parser, Subcommand as ClapSubcommand}}; +use sc_cli::clap::{self, Parser, Subcommand as ClapSubcommand}; + use crate::chain_spec::{BootstrapChainCmd, ConvertChainspecToRawCmd}; #[derive(Parser)] @@ -27,8 +27,6 @@ fn main() -> sc_cli::Result<()> { Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), - None => { - Err("Command was required!".into()) - } + None => Err("Command was required!".into()), } } diff --git a/bin/node/src/cli.rs b/bin/node/src/cli.rs index 66a4a0ad37..e2c7b811e8 100644 --- a/bin/node/src/cli.rs +++ b/bin/node/src/cli.rs @@ -5,8 +5,10 @@ use sc_cli::{ pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; -use crate::aleph_cli::AlephCli; -use crate::resources::{mainnet_chainspec, testnet_chainspec}; +use crate::{ + aleph_cli::AlephCli, + resources::{mainnet_chainspec, testnet_chainspec}, +}; #[derive(Debug, Parser)] #[clap(subcommand_negates_reqs(true), version(env!("SUBSTRATE_CLI_IMPL_VERSION")))] diff --git a/bin/node/src/service.rs b/bin/node/src/service.rs index 013bbac4b6..df2513ccb8 100644 --- a/bin/node/src/service.rs +++ b/bin/node/src/service.rs @@ -6,14 +6,15 @@ use std::{ }; use finality_aleph::{ - AlephBlockImport, AlephConfig, AllBlockMetrics, BlockImporter, ChannelProvider, - Justification, JustificationTranslator, MillisecsPerBlock, Protocol, ProtocolNaming, - RateLimiterConfig, RedirectingBlockImport, run_validator_node, SessionPeriod, SubstrateChainStatus, + run_validator_node, AlephBlockImport, AlephConfig, AllBlockMetrics, BlockImporter, + ChannelProvider, Justification, JustificationTranslator, MillisecsPerBlock, Protocol, + ProtocolNaming, RateLimiterConfig, RedirectingBlockImport, SessionPeriod, SubstrateChainStatus, SubstrateNetwork, SyncOracle, TracingBlockImport, ValidatorAddressCache, }; use log::warn; use primitives::{ - AlephSessionApi, Block, fake_runtime_api::fake_runtime::RuntimeApi, MAX_BLOCK_SIZE, DEFAULT_BACKUP_FOLDER + fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, DEFAULT_BACKUP_FOLDER, + MAX_BLOCK_SIZE, }; use sc_basic_authorship::ProposerFactory; use sc_client_api::{BlockBackend, HeaderBackend}; @@ -21,12 +22,12 @@ use sc_consensus::ImportQueue; use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams}; use sc_consensus_slots::BackoffAuthoringBlocksStrategy; use sc_network::config::FullNetworkConfiguration; -use sc_service::{Configuration, error::Error as ServiceError, TaskManager, TFullClient}; +use sc_service::{error::Error as ServiceError, Configuration, TFullClient, TaskManager}; use sc_telemetry::{Telemetry, TelemetryWorker}; use sp_api::ProvideRuntimeApi; use sp_arithmetic::traits::BaseArithmetic; use sp_consensus::DisableProofRecording; -use sp_consensus_aura::{Slot, sr25519::AuthorityPair as AuraPair}; +use sp_consensus_aura::{sr25519::AuthorityPair as AuraPair, Slot}; use crate::{ aleph_cli::AlephCli, From d974b94fc8694b915ee1d9a84c71d65f60e97423 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 11:44:19 +0200 Subject: [PATCH 09/26] More CI fixes --- .github/actions/run-e2e-test/action.yml | 2 +- .github/workflows/_build-chain-bootstrapper.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 645b2e1c99..8c38bacb47 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -66,7 +66,7 @@ runs: - name: Download chain-booststrapper image uses: actions/download-artifact@v4 with: - name: chain-booststrapper + name: chain-booststrapper-image - name: Load chain-booststrapper image shell: bash diff --git a/.github/workflows/_build-chain-bootstrapper.yml b/.github/workflows/_build-chain-bootstrapper.yml index 56951c2e21..bff12e47b3 100644 --- a/.github/workflows/_build-chain-bootstrapper.yml +++ b/.github/workflows/_build-chain-bootstrapper.yml @@ -57,7 +57,7 @@ jobs: - name: Upload docker image to GH Artifacts uses: actions/upload-artifact@v4 with: - name: chain-bootstrapper + name: chain-bootstrapper-image path: chain-bootstrapper.tar if-no-files-found: error retention-days: 7 From 828508ddf31e539c21d4da6bfac44e6de09703a0 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 12:05:34 +0200 Subject: [PATCH 10/26] Typos in workflows --- .github/actions/run-e2e-test/action.yml | 2 +- .github/workflows/_build-production-node-and-runtime.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 8c38bacb47..55662f397a 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -66,7 +66,7 @@ runs: - name: Download chain-booststrapper image uses: actions/download-artifact@v4 with: - name: chain-booststrapper-image + name: chain-bootstrapper-image - name: Load chain-booststrapper image shell: bash diff --git a/.github/workflows/_build-production-node-and-runtime.yml b/.github/workflows/_build-production-node-and-runtime.yml index e0e8ce7c2e..05159a7a3a 100644 --- a/.github/workflows/_build-production-node-and-runtime.yml +++ b/.github/workflows/_build-production-node-and-runtime.yml @@ -32,7 +32,7 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Build production binary and runtime + - name: Build production aleph-node run: cargo build --profile production -p aleph-node - name: Upload release binary to GH artifacts @@ -43,6 +43,9 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Build production aleph-runtime + run: cargo build --profile production -p aleph-runtime + # required by _check-runtime-determinism.yml workflow - name: Upload release runtime to GH artifacts uses: actions/upload-artifact@v4 From 0e59bc1ba77627ad3bd482681bce90495fb5a836 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 12:22:30 +0200 Subject: [PATCH 11/26] Fix to how chain-bootstrapper node image is called --- .github/scripts/run_consensus.sh | 4 ++-- .github/workflows/deploy-to-devnet.yml | 2 +- bin/chain-bootstrapper/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 821f9849bf..4edef06a38 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -11,7 +11,7 @@ # consensus size. -set -euo pipefail +set -euox pipefail # ------------------------ constants -------------------------------------- @@ -83,7 +83,7 @@ function generate_chainspec() { echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}" docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER}" \ - -c "bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" + -c "chain-bootstrapper bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" } function generate_bootnode_peer_id() { diff --git a/.github/workflows/deploy-to-devnet.yml b/.github/workflows/deploy-to-devnet.yml index 76d86e8ae1..58d841f32c 100644 --- a/.github/workflows/deploy-to-devnet.yml +++ b/.github/workflows/deploy-to-devnet.yml @@ -98,7 +98,7 @@ jobs: EVE=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint \ - '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \ + '/usr/local/bin/chain-bootstrapper' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \ bootstrap-chain --raw --base-path /data --chain-id a0dnet1 \ --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q \ --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD \ diff --git a/bin/chain-bootstrapper/Dockerfile b/bin/chain-bootstrapper/Dockerfile index 195f7b86d2..92147703a6 100644 --- a/bin/chain-bootstrapper/Dockerfile +++ b/bin/chain-bootstrapper/Dockerfile @@ -3,4 +3,4 @@ FROM ubuntu:jammy COPY target/release/chain-bootstrapper /usr/local/bin RUN chmod +x /usr/local/bin/chain-bootstrapper -CMD ["/usr/local/bin/chain-bootstrapper"] +ENTRYPOINT ["chain-bootstrapper"] From 8d766ee9c6953c712bf705ebe519094a0645fed9 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 12:36:12 +0200 Subject: [PATCH 12/26] Incorrect docker tarball name --- .github/actions/run-e2e-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 55662f397a..afa2d6cd17 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -63,14 +63,14 @@ runs: shell: bash run: docker load -i aleph-node.tar - - name: Download chain-booststrapper image + - name: Download chain-bootstrapper image uses: actions/download-artifact@v4 with: name: chain-bootstrapper-image - - name: Load chain-booststrapper image + - name: Load chain-bootstrapper image shell: bash - run: docker load -i chain-booststrapper.tar + run: docker load -i chain-bootstrapper.tar - name: Run consensus party shell: bash From 14cd62e13f1782c2388b7abb73366bd90b16727a Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 13:56:51 +0200 Subject: [PATCH 13/26] Push chain-bootstrapper on merge to main --- .github/scripts/run_consensus.sh | 13 ++++++-- .github/scripts/test_python_general.sh | 10 +++---- .github/workflows/_push-node-image-to-ecr.yml | 30 ++++++++++++------- .../on-main-or-release-branch-commit.yml | 22 +++++++++++++- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 4edef06a38..1838a04dec 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -20,7 +20,7 @@ MIN_VALIDATOR_COUNT=${MIN_VALIDATOR_COUNT:-4} DOCKER_COMPOSE=${DOCKER_COMPOSE:-docker/docker-compose.yml} OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""} NODE_IMAGE=${NODE_IMAGE:-"aleph-node:latest"} -CHAIN_BOOTSTRAPPER=${CHAIN_BOOTSTRAPPER:-"chain-bootstrapper:latest"} +CHAIN_BOOTSTRAPPER_IMAGE=${CHAIN_BOOTSTRAPPER_IMAGE:-"chain-bootstrapper:latest"} LOGS_OUTPUT_FILE=${LOGS_OUTPUT_FILE:=""} # ------------------------ argument parsing and usage ----------------------- @@ -82,7 +82,7 @@ function generate_chainspec() { local validator_ids_comma_separated="${validators//${IFS:0:1}/,}" echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}" - docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER}" \ + docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER_IMAGE}" \ -c "chain-bootstrapper bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" } @@ -163,6 +163,15 @@ else exit 1 fi +if docker inspect ${NODE_IMAGE} > /dev/null; then + echo "aleph-node image tag ${NODE_IMAGE} found locally" +else + echo "${NODE_IMAGE} not found locally." + echo "Build image first with:" + echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ." + exit 1 +fi + mkdir -p docker/data/ echo "Warning: if you run this script on the same machine more then once, and finalization does not work, remove docker/data." global_account_ids=$(generate_account_ids ${NODE_COUNT}) diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index 49cd19d860..6463962c24 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -25,7 +25,7 @@ while [[ $# -gt 0 ]]; do shift;shift ;; --chain-bootstrapper) - CHAIN_BOOTSTRAPPER="$2" + CHAIN_BOOTSTRAPPER_IMAGE="$2" shift;shift ;; --testcase) @@ -49,8 +49,8 @@ if [[ ! -x "${ALEPH_NODE_BINARY}" ]]; then echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}, or it's not executable." exit 1 fi -if [[ ! -x "${CHAIN_BOOTSTRAPPER}" ]]; then - echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}, or it's not executable." +if [[ ! -x "${CHAIN_BOOTSTRAPPER_IMAGE}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER_IMAGE}, or it's not executable." exit 1 fi if [[ -z "${TESTCASE}" ]]; then @@ -66,7 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then fi chmod +x "${ALEPH_NODE_BINARY}" -chmod +x "${CHAIN_BOOTSTRAPPER}" +chmod +x "${CHAIN_BOOTSTRAPPER_IMAGE}" echo "Installing python requirements" pip install -r requirements.txt @@ -76,7 +76,7 @@ pip install -r requirements.txt # first buffered and that you can see the output of your application. export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY -export CHAIN_BOOTSTRAPPER +export CHAIN_BOOTSTRAPPER_IMAGE export WORKDIR=$(mktemp -d) eval "./${file_name_to_run}" popd > /dev/null diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index 8aa6f55aaf..d3b8c377b7 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -1,21 +1,29 @@ --- -# This workflow builds docker aleph-node image based on binary stored in GH artifacts -name: Build and push node image to ECR +# This workflow builds arbitrary docker node image based on binary downloaded in GH artifacts +name: Build and push docker image to ECR on: workflow_call: inputs: - node-binary: - description: 'Name of aleph-node binary stored in GH artifacts' + binary-artifact-name: + description: 'Name of a binary stored in GH artifacts' + required: true + type: string + binary-name: + description: 'Name of a binary to build docker image on top of' required: true type: string docker-image-name: - description: 'Name of node docker to be uploaded to ECR' + description: 'Name of node docker image to be uploaded to ECR' + required: true + type: string + docker-file-path: + description: 'Path to Dockerfile' required: true type: string jobs: main: - name: Push node docker image to the ECR + name: Push ${{ inputs.binary-name }} docker image to the ECR runs-on: ubuntu-20.04 steps: - name: Checkout aleph-node sources @@ -25,17 +33,17 @@ jobs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 - - name: Download ${{ inputs.node-binary }} binary from artifacts + - name: Download ${{ inputs.binary-artifact-name }} from artifacts uses: actions/download-artifact@v4 with: - name: ${{ inputs.node-binary }} + name: ${{ inputs.binary-artifact-name }} path: target/release/ - - name: Build docker node image + - name: Build ${{ inputs.binary-name }} docker image id: build-image run: | - chmod +x target/release/aleph-node - docker build --tag ${{ inputs.docker-image-name }}:latest -f ./docker/Dockerfile . + chmod +x target/release/${{ inputs.binary-name }} + docker build --tag ${{ inputs.docker-image-name }}:latest -f ${{ inputs.docker-file-path }} . - name: Login to Public Amazon ECR id: login-public-ecr diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 0b47ab36ad..3fc0162d7c 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -13,6 +13,24 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit + build-chain-bootstrapper: + needs: [check-vars-and-secrets] + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + + push-chain-bootstrapper-image-to-ecr: + name: Push chain-bootstrapper docker image to ECR + needs: [build-chain-bootstrapper] + uses: ./.github/workflows/_push-node-image-to-ecr.yml + secrets: inherit + with: + binary-artifact-name: chain-bootstrapper-image + docker-image-name: chain-bootstrapper + binary-name: chain-bootstrapper + docker-file-path: ./bin/chain-bootstrapper/Dockerfile + build-and-push-cliain: needs: [check-vars-and-secrets] name: Build and push cliain @@ -63,8 +81,10 @@ jobs: uses: ./.github/workflows/_push-node-image-to-ecr.yml secrets: inherit with: - node-binary: aleph-production-node + binary-artifact-name: aleph-production-node docker-image-name: aleph-node + binary-name: aleph-node + docker-file-path: ./docker/Dockerfile send-slack-notification-release: name: Send Slack notification about workflow status From 1b82ceb88d398ac99be9aeb70fa91a32d53f94ed Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 13:59:24 +0200 Subject: [PATCH 14/26] yaml linter --- .github/workflows/_push-node-image-to-ecr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index d3b8c377b7..3e1865182a 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -43,7 +43,9 @@ jobs: id: build-image run: | chmod +x target/release/${{ inputs.binary-name }} - docker build --tag ${{ inputs.docker-image-name }}:latest -f ${{ inputs.docker-file-path }} . + docker build \ + --tag ${{ inputs.docker-image-name }}:latest \ + -f ${{ inputs.docker-file-path }} . - name: Login to Public Amazon ECR id: login-public-ecr From 02532771411528ad47ecfc134e0a1b7f9268b848 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 14:12:02 +0200 Subject: [PATCH 15/26] Fix nightly-check-node-features-build.yml --- .github/workflows/nightly-check-node-features-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index fd13e2402b..38d7642002 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -30,8 +30,8 @@ jobs: run: cargo check --profile production -p aleph-runtime --features try-runtime --locked - name: aleph-runtime with runtime-benchmarks - run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks \ - --locked + # yamllint disable-line rule:line-length + run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks --locked - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked From 84f5cedb324eebbd6faf35d66e39738d6a014d38 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 14:26:12 +0200 Subject: [PATCH 16/26] Fixes --- .github/scripts/test_python_general.sh | 8 ++++---- .github/workflows/nightly-check-node-features-build.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index 6463962c24..0b32ca2b09 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -45,12 +45,12 @@ done # relative paths should be improved in python scripts CI, as it matters that below line is pushd local-tests/ > /dev/null -if [[ ! -x "${ALEPH_NODE_BINARY}" ]]; then - echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}, or it's not executable." +if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then + echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}" exit 1 fi -if [[ ! -x "${CHAIN_BOOTSTRAPPER_IMAGE}" ]]; then - echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER_IMAGE}, or it's not executable." +if [[ ! -f "${CHAIN_BOOTSTRAPPER_IMAGE}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER_IMAGE}" exit 1 fi if [[ -z "${TESTCASE}" ]]; then diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index 38d7642002..cb3d503c84 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -39,8 +39,8 @@ jobs: - name: aleph-node with try-runtime run: cargo check --profile production -p aleph-node --features try-runtime --locked - - name: aleph-node with aleph-native-runtime - run: cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked + - name: aleph-node with aleph-runtime-native + run: cargo check --profile dev -p aleph-node --features aleph-runtime-native --locked - name: chain-bootstrapper with try-runtime run: cargo check --profile production -p chain-bootstrapper --features try-runtime --locked From a37d43a41620457faca2c03aa1a0b8d806ddfb0d Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 14:44:09 +0200 Subject: [PATCH 17/26] Teesting on main push workflow --- .github/scripts/run_consensus.sh | 10 +- .github/workflows/_push-node-image-to-ecr.yml | 4 +- .../on-main-or-release-branch-commit.yml | 92 ++++++++++--------- 3 files changed, 54 insertions(+), 52 deletions(-) diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 1838a04dec..5e32e31523 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -11,7 +11,7 @@ # consensus size. -set -euox pipefail +set -euo pipefail # ------------------------ constants -------------------------------------- @@ -163,12 +163,12 @@ else exit 1 fi -if docker inspect ${NODE_IMAGE} > /dev/null; then - echo "aleph-node image tag ${NODE_IMAGE} found locally" +if docker inspect ${CHAIN_BOOTSTRAPPER_IMAGE} > /dev/null; then + echo "chain-bootstrapper image tag ${CHAIN_BOOTSTRAPPER_IMAGE} found locally" else - echo "${NODE_IMAGE} not found locally." + echo "${CHAIN_BOOTSTRAPPER_IMAGE} not found locally." echo "Build image first with:" - echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ." + echo "docker build -t ${CHAIN_BOOTSTRAPPER_IMAGE} -f bin/chain-bootstrapper/Dockerfile ." exit 1 fi diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index 3e1865182a..3ea6765d54 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -62,8 +62,8 @@ jobs: run: | docker tag '${{ env.CURRENT_IMAGE }}' \ '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' - docker push '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' + # docker push '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' if [[ '${{ steps.get-ref-properties.outputs.branch }}' -eq 'main' ]]; then docker tag '${{ env.CURRENT_IMAGE }}' '${{ env.ECR_PUSH_IMAGE }}:latest' - docker push '${{ env.ECR_PUSH_IMAGE }}:latest' + # docker push '${{ env.ECR_PUSH_IMAGE }}:latest' fi diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 3fc0162d7c..c3f4ec49c3 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -6,6 +6,8 @@ on: branches: - main - 'release-*' + # testing, remove + - A0-4231-move-chainspec-generation jobs: check-vars-and-secrets: @@ -45,7 +47,7 @@ jobs: uses: ./.github/workflows/_build-aleph-e2e-client-image.yml with: ref: ${{ github.ref }} - push: true + push: false secrets: inherit build-production-node-and-runtime: @@ -59,22 +61,22 @@ jobs: needs: [build-production-node-and-runtime] uses: ./.github/workflows/_check-runtime-determimism.yml - store-production-node-and-runtime-in-mainnet-s3: - name: Store production node and runtime in Mainnet bucket - runs-on: ubuntu-20.04 - needs: [build-production-node-and-runtime] - steps: - - name: Checkout aleph-node source code - uses: actions/checkout@v4 - - - name: Store production node and runtime - uses: ./.github/actions/store-node-and-runtime - with: - profile: production - aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} - aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} - +# store-production-node-and-runtime-in-mainnet-s3: +# name: Store production node and runtime in Mainnet bucket +# runs-on: ubuntu-20.04 +# needs: [build-production-node-and-runtime] +# steps: +# - name: Checkout aleph-node source code +# uses: actions/checkout@v4 +# +# - name: Store production node and runtime +# uses: ./.github/actions/store-node-and-runtime +# with: +# profile: production +# aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} +# aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} +# push-production-node-image-to-ecr: name: Push production node docker image to ECR needs: [build-production-node-and-runtime] @@ -86,31 +88,31 @@ jobs: binary-name: aleph-node docker-file-path: ./docker/Dockerfile - send-slack-notification-release: - name: Send Slack notification about workflow status - runs-on: ubuntu-20.04 - if: always() && startsWith(github.ref_name, 'release-') - needs: [push-production-node-image-to-ecr] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Send Slack message - uses: Cardinal-Cryptography/github-actions/slack-notification@v6 - with: - notify-on: "always" - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} - - send-slack-notification-main: - name: Send Slack notification about workflow status - runs-on: ubuntu-20.04 - if: always() && github.ref_name == 'main' - needs: [push-production-node-image-to-ecr] - steps: - - name: Send Slack message - uses: Cardinal-Cryptography/github-actions/slack-notification@v6 - with: - notify-on: "failure" - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} +# send-slack-notification-release: +# name: Send Slack notification about workflow status +# runs-on: ubuntu-20.04 +# if: always() && startsWith(github.ref_name, 'release-') +# needs: [push-production-node-image-to-ecr] +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# +# - name: Send Slack message +# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 +# with: +# notify-on: "always" +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} +# +# send-slack-notification-main: +# name: Send Slack notification about workflow status +# runs-on: ubuntu-20.04 +# if: always() && github.ref_name == 'main' +# needs: [push-production-node-image-to-ecr] +# steps: +# - name: Send Slack message +# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 +# with: +# notify-on: "failure" +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} From d00959489ca7d3f915632fe763d1fc7a06fb2515 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 23 Apr 2024 15:26:52 +0200 Subject: [PATCH 18/26] Fixes to main branch build --- .github/workflows/on-main-or-release-branch-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index c3f4ec49c3..ee24891594 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/workflows/_push-node-image-to-ecr.yml secrets: inherit with: - binary-artifact-name: chain-bootstrapper-image + binary-artifact-name: chain-bootstrapper docker-image-name: chain-bootstrapper binary-name: chain-bootstrapper docker-file-path: ./bin/chain-bootstrapper/Dockerfile From c2b0139111921076dfab92bf5f3a226eab45ed49 Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 24 Apr 2024 08:21:07 +0200 Subject: [PATCH 19/26] Fixes to python tests --- .github/scripts/test_python_general.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index 0b32ca2b09..b0f1dcfbdd 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -25,7 +25,7 @@ while [[ $# -gt 0 ]]; do shift;shift ;; --chain-bootstrapper) - CHAIN_BOOTSTRAPPER_IMAGE="$2" + CHAIN_BOOTSTRAPPER="$2" shift;shift ;; --testcase) @@ -49,8 +49,8 @@ if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}" exit 1 fi -if [[ ! -f "${CHAIN_BOOTSTRAPPER_IMAGE}" ]]; then - echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER_IMAGE}" +if [[ ! -f "${CHAIN_BOOTSTRAPPER}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}" exit 1 fi if [[ -z "${TESTCASE}" ]]; then @@ -66,7 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then fi chmod +x "${ALEPH_NODE_BINARY}" -chmod +x "${CHAIN_BOOTSTRAPPER_IMAGE}" +chmod +x "${CHAIN_BOOTSTRAPPER}" echo "Installing python requirements" pip install -r requirements.txt @@ -76,7 +76,7 @@ pip install -r requirements.txt # first buffered and that you can see the output of your application. export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY -export CHAIN_BOOTSTRAPPER_IMAGE +export CHAIN_BOOTSTRAPPER export WORKDIR=$(mktemp -d) eval "./${file_name_to_run}" popd > /dev/null From af6d8a48a60d4dc944e14eae8edea73651d3ecdf Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 24 Apr 2024 08:21:27 +0200 Subject: [PATCH 20/26] Revert "Teesting on main push workflow" This reverts commit a37d43a41620457faca2c03aa1a0b8d806ddfb0d. --- .github/scripts/run_consensus.sh | 10 +- .github/workflows/_push-node-image-to-ecr.yml | 4 +- .../on-main-or-release-branch-commit.yml | 92 +++++++++---------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 5e32e31523..1838a04dec 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -11,7 +11,7 @@ # consensus size. -set -euo pipefail +set -euox pipefail # ------------------------ constants -------------------------------------- @@ -163,12 +163,12 @@ else exit 1 fi -if docker inspect ${CHAIN_BOOTSTRAPPER_IMAGE} > /dev/null; then - echo "chain-bootstrapper image tag ${CHAIN_BOOTSTRAPPER_IMAGE} found locally" +if docker inspect ${NODE_IMAGE} > /dev/null; then + echo "aleph-node image tag ${NODE_IMAGE} found locally" else - echo "${CHAIN_BOOTSTRAPPER_IMAGE} not found locally." + echo "${NODE_IMAGE} not found locally." echo "Build image first with:" - echo "docker build -t ${CHAIN_BOOTSTRAPPER_IMAGE} -f bin/chain-bootstrapper/Dockerfile ." + echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ." exit 1 fi diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index 3ea6765d54..3e1865182a 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -62,8 +62,8 @@ jobs: run: | docker tag '${{ env.CURRENT_IMAGE }}' \ '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' - # docker push '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' + docker push '${{ env.ECR_PUSH_IMAGE }}:${{ steps.get-ref-properties.outputs.sha }}' if [[ '${{ steps.get-ref-properties.outputs.branch }}' -eq 'main' ]]; then docker tag '${{ env.CURRENT_IMAGE }}' '${{ env.ECR_PUSH_IMAGE }}:latest' - # docker push '${{ env.ECR_PUSH_IMAGE }}:latest' + docker push '${{ env.ECR_PUSH_IMAGE }}:latest' fi diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index ee24891594..5d5dead9f3 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -6,8 +6,6 @@ on: branches: - main - 'release-*' - # testing, remove - - A0-4231-move-chainspec-generation jobs: check-vars-and-secrets: @@ -47,7 +45,7 @@ jobs: uses: ./.github/workflows/_build-aleph-e2e-client-image.yml with: ref: ${{ github.ref }} - push: false + push: true secrets: inherit build-production-node-and-runtime: @@ -61,22 +59,22 @@ jobs: needs: [build-production-node-and-runtime] uses: ./.github/workflows/_check-runtime-determimism.yml -# store-production-node-and-runtime-in-mainnet-s3: -# name: Store production node and runtime in Mainnet bucket -# runs-on: ubuntu-20.04 -# needs: [build-production-node-and-runtime] -# steps: -# - name: Checkout aleph-node source code -# uses: actions/checkout@v4 -# -# - name: Store production node and runtime -# uses: ./.github/actions/store-node-and-runtime -# with: -# profile: production -# aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} -# aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} -# + store-production-node-and-runtime-in-mainnet-s3: + name: Store production node and runtime in Mainnet bucket + runs-on: ubuntu-20.04 + needs: [build-production-node-and-runtime] + steps: + - name: Checkout aleph-node source code + uses: actions/checkout@v4 + + - name: Store production node and runtime + uses: ./.github/actions/store-node-and-runtime + with: + profile: production + aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} + aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} + push-production-node-image-to-ecr: name: Push production node docker image to ECR needs: [build-production-node-and-runtime] @@ -88,31 +86,31 @@ jobs: binary-name: aleph-node docker-file-path: ./docker/Dockerfile -# send-slack-notification-release: -# name: Send Slack notification about workflow status -# runs-on: ubuntu-20.04 -# if: always() && startsWith(github.ref_name, 'release-') -# needs: [push-production-node-image-to-ecr] -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# -# - name: Send Slack message -# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 -# with: -# notify-on: "always" -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} -# -# send-slack-notification-main: -# name: Send Slack notification about workflow status -# runs-on: ubuntu-20.04 -# if: always() && github.ref_name == 'main' -# needs: [push-production-node-image-to-ecr] -# steps: -# - name: Send Slack message -# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 -# with: -# notify-on: "failure" -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} + send-slack-notification-release: + name: Send Slack notification about workflow status + runs-on: ubuntu-20.04 + if: always() && startsWith(github.ref_name, 'release-') + needs: [push-production-node-image-to-ecr] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Send Slack message + uses: Cardinal-Cryptography/github-actions/slack-notification@v6 + with: + notify-on: "always" + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} + + send-slack-notification-main: + name: Send Slack notification about workflow status + runs-on: ubuntu-20.04 + if: always() && github.ref_name == 'main' + needs: [push-production-node-image-to-ecr] + steps: + - name: Send Slack message + uses: Cardinal-Cryptography/github-actions/slack-notification@v6 + with: + notify-on: "failure" + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} From b195749debe404110f302c582177d843a6116634 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 25 Apr 2024 08:46:28 +0200 Subject: [PATCH 21/26] Fixes to CI --- .github/actions/run-e2e-test/action.yml | 6 ++- .github/scripts/test_python_general.sh | 2 + .../workflows/_build-chain-bootstrapper.yml | 27 ++++++++++--- .github/workflows/nightly-e2e-logic-tests.yml | 9 +++-- .../nightly-normal-session-e2e-tests.yml | 38 ++++++++++++------- .../on-main-or-release-branch-commit.yml | 3 +- .github/workflows/on-pull-request-commit.yml | 1 + 7 files changed, 60 insertions(+), 26 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index afa2d6cd17..7ed32bee15 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -38,6 +38,10 @@ inputs: aleph-e2e-client-image: description: 'aleph-e2e-client image' required: true + chain-bootstrapper-image: + description: 'chain-bootstrapper image' + required: false + default: chain-bootstrapper-image-test timeout-minutes: description: 'The maximum number of minutes to let a test run before it is canceled and considered failed' @@ -66,7 +70,7 @@ runs: - name: Download chain-bootstrapper image uses: actions/download-artifact@v4 with: - name: chain-bootstrapper-image + name: ${{ inputs.chain-bootstrapper-image }} - name: Load chain-bootstrapper image shell: bash diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index b0f1dcfbdd..d62d517f94 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -77,6 +77,8 @@ pip install -r requirements.txt export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY export CHAIN_BOOTSTRAPPER +export RUST_LOG=debug export WORKDIR=$(mktemp -d) +echo "WORKDIR is ${WORKDIR}" eval "./${file_name_to_run}" popd > /dev/null diff --git a/.github/workflows/_build-chain-bootstrapper.yml b/.github/workflows/_build-chain-bootstrapper.yml index bff12e47b3..d3e5293c6c 100644 --- a/.github/workflows/_build-chain-bootstrapper.yml +++ b/.github/workflows/_build-chain-bootstrapper.yml @@ -9,6 +9,10 @@ on: description: 'git ref: hash, branch, tag to build chain-bootstrapper binary from' type: string required: true + production: + description: 'set to true to use production profile' + type: boolean + required: true jobs: main: @@ -17,6 +21,8 @@ jobs: env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache + CARGO_FOLDER: ${{ inputs.production == true && 'production' || 'release' }} + ARTIFACT_NAME_SUFFIX: ${{ inputs.production == true && 'release' || 'test' }} steps: - name: Checkout aleph-node source code uses: actions/checkout@v4 @@ -33,31 +39,40 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Build + - name: Build without production profile + if: ${{ inputs.production != true }} run: | cargo build --release -p chain-bootstrapper \ --features "short_session enable_treasury_proposals" + - name: Build with production profile + if: ${{ inputs.production == true }} + run: | + cargo build --profile production -p chain-bootstrapper + - name: Upload binary to GH Artifacts uses: actions/upload-artifact@v4 with: - name: chain-bootstrapper - path: target/release/chain-bootstrapper + name: chain-bootstrapper-${{ env.ARTIFACT_NAME_SUFFIX }} + path: target/${{ env.CARGO_FOLDER }}/chain-bootstrapper if-no-files-found: error retention-days: 7 - name: Build docker id: build-image run: | - chmod +x target/release/chain-bootstrapper + chmod +x target/${{ env.CARGO_FOLDER }}/chain-bootstrapper + if [[ ${{ inputs.production }} == true ]]; then + mkdir -p target/release + mv target/production/chain-bootstrapper target/release/ + fi docker build --tag chain-bootstrapper:latest -f ./bin/chain-bootstrapper/Dockerfile . docker save -o chain-bootstrapper.tar chain-bootstrapper:latest - # this is required by action/run-e2e-test.yml - name: Upload docker image to GH Artifacts uses: actions/upload-artifact@v4 with: - name: chain-bootstrapper-image + name: chain-bootstrapper-image-${{ env.ARTIFACT_NAME_SUFFIX }} path: chain-bootstrapper.tar if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/nightly-e2e-logic-tests.yml b/.github/workflows/nightly-e2e-logic-tests.yml index 918c11191f..b000358030 100644 --- a/.github/workflows/nightly-e2e-logic-tests.yml +++ b/.github/workflows/nightly-e2e-logic-tests.yml @@ -25,11 +25,12 @@ jobs: with: ref: ${{ github.ref }} - build-chain-bootstrapper: + build-chain-bootstrapper-test: name: Build chain-bootstrapper uses: ./.github/workflows/_build-chain-bootstrapper.yml with: ref: ${{ github.ref }} + production: false build-aleph-e2e-client-image: needs: [check-vars-and-secrets] @@ -44,7 +45,7 @@ jobs: name: Run e2e tests needs: - build-test-node - - build-chain-bootstrapper + - build-chain-bootstrapper-test - build-aleph-e2e-client-image uses: ./.github/workflows/_run-e2e-tests.yml with: @@ -52,7 +53,7 @@ jobs: aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} run-recover-after-abft-update-test: - needs: [build-test-node, build-chain-bootstrapper] + needs: [build-test-node, build-chain-bootstrapper-test] name: Run recovery test after abft update runs-on: [self-hosted, Linux, X64, medium] steps: @@ -68,7 +69,7 @@ jobs: - name: Download chain-bootstrapper from GH artifacts uses: actions/download-artifact@v4 with: - name: chain-bootstrapper + name: chain-bootstrapper-test path: target/release/ - name: Run test diff --git a/.github/workflows/nightly-normal-session-e2e-tests.yml b/.github/workflows/nightly-normal-session-e2e-tests.yml index 4fa284c125..7c602e45a4 100644 --- a/.github/workflows/nightly-normal-session-e2e-tests.yml +++ b/.github/workflows/nightly-normal-session-e2e-tests.yml @@ -37,17 +37,18 @@ jobs: name: Build synthetic node uses: ./.github/workflows/_build-synthetic-node.yml - build-chain-bootstrapper: + build-chain-bootstrapper-production: name: Build chain-bootstrapper uses: ./.github/workflows/_build-chain-bootstrapper.yml with: ref: ${{ github.ref }} + production: true run-e2e-high-out-latency: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Run high out-latency test runs-on: ubuntu-20.04 steps: @@ -60,6 +61,7 @@ jobs: test-case: high_out_latency_for_all image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} @@ -69,7 +71,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up and then becoming necessary for consensus runs-on: ubuntu-20.04 @@ -87,6 +89,7 @@ jobs: test-case: test::sync::one_node_catching_up_and_then_becoming_necessary_for_consensus image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -97,7 +100,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up runs-on: ubuntu-20.04 steps: @@ -114,6 +117,7 @@ jobs: test-case: test::sync::one_node_catching_up image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -124,7 +128,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups and one quorum and switch quorum between them runs-on: ubuntu-20.04 @@ -142,6 +146,7 @@ jobs: test-case: test::sync::into_two_groups_and_one_quorum_and_switch_quorum_between_them image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -152,7 +157,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into multiple groups of two runs-on: ubuntu-20.04 steps: @@ -169,6 +174,7 @@ jobs: test-case: test::sync::into_multiple_groups_of_two image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -179,7 +185,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two equal size groups with no quorum runs-on: ubuntu-20.04 @@ -197,6 +203,7 @@ jobs: test-case: test::sync::into_two_equal_size_groups_with_no_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -207,7 +214,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups one with quorum runs-on: ubuntu-20.04 @@ -225,6 +232,7 @@ jobs: test-case: test::sync::into_two_groups_one_with_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -235,7 +243,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Finalization-stall test using synthetic-network runs-on: ubuntu-20.04 steps: @@ -253,13 +261,14 @@ jobs: test-case: test::sync::large_finalization_stall image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.finalization_stall_with_pruning.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 60 run-major-sync-test: - needs: [build-production-node-and-runtime, build-chain-bootstrapper] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run major sync test runs-on: ubuntu-20.04 steps: @@ -275,7 +284,7 @@ jobs: - name: Download chain-bootstrapper from GH artifacts uses: actions/download-artifact@v4 with: - name: chain-bootstrapper + name: chain-bootstrapper-release path: target/release/ - name: Run test @@ -287,7 +296,7 @@ jobs: --testcase test_major_sync run-force-reorg-test: - needs: [build-production-node-and-runtime, build-chain-bootstrapper] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run force reorgs test runs-on: [self-hosted, Linux, X64, medium] steps: @@ -303,7 +312,7 @@ jobs: - name: Download chain-bootstrapper from GH artifacts uses: actions/download-artifact@v4 with: - name: chain-bootstrapper + name: chain-bootstrapper-release path: target/release/ - name: Run test @@ -318,7 +327,7 @@ jobs: needs: - build-synthetic-node - build-aleph-e2e-client-image - - build-chain-bootstrapper + - build-chain-bootstrapper-production name: Run high out-latency for every quorum runs-on: ubuntu-20.04 steps: @@ -332,6 +341,7 @@ jobs: image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet compose-file: docker/docker-compose.synthetic-network.yml + chain-bootstrapper-image: chain-bootstrapper-image-release # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 35 diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 5d5dead9f3..011aaa661d 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -19,6 +19,7 @@ jobs: uses: ./.github/workflows/_build-chain-bootstrapper.yml with: ref: ${{ github.ref }} + production: true push-chain-bootstrapper-image-to-ecr: name: Push chain-bootstrapper docker image to ECR @@ -26,7 +27,7 @@ jobs: uses: ./.github/workflows/_push-node-image-to-ecr.yml secrets: inherit with: - binary-artifact-name: chain-bootstrapper + binary-artifact-name: chain-bootstrapper-release docker-image-name: chain-bootstrapper binary-name: chain-bootstrapper docker-file-path: ./bin/chain-bootstrapper/Dockerfile diff --git a/.github/workflows/on-pull-request-commit.yml b/.github/workflows/on-pull-request-commit.yml index b6cca23aa5..d4e56ca0c6 100644 --- a/.github/workflows/on-pull-request-commit.yml +++ b/.github/workflows/on-pull-request-commit.yml @@ -49,6 +49,7 @@ jobs: uses: ./.github/workflows/_build-chain-bootstrapper.yml with: ref: ${{ github.ref }} + production: false build-aleph-e2e-client-image: name: Build aleph-e2e-client image From feb07e9dfeb53256166ec0cfa9b3ce1311b8ab05 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 25 Apr 2024 09:13:51 +0200 Subject: [PATCH 22/26] Testing main branch push --- .../on-main-or-release-branch-commit.yml | 167 +++++++++--------- 1 file changed, 84 insertions(+), 83 deletions(-) diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 011aaa661d..1dc3aa494b 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -6,6 +6,7 @@ on: branches: - main - 'release-*' + - A0-4231-move-chainspec-generation jobs: check-vars-and-secrets: @@ -32,86 +33,86 @@ jobs: binary-name: chain-bootstrapper docker-file-path: ./bin/chain-bootstrapper/Dockerfile - build-and-push-cliain: - needs: [check-vars-and-secrets] - name: Build and push cliain - uses: ./.github/workflows/_build-and-push-cliain.yml - secrets: inherit - with: - check-only: false - - build-aleph-e2e-client-image: - needs: [check-vars-and-secrets] - name: Build aleph-e2e-client image - uses: ./.github/workflows/_build-aleph-e2e-client-image.yml - with: - ref: ${{ github.ref }} - push: true - secrets: inherit - - build-production-node-and-runtime: - needs: [check-vars-and-secrets] - name: Build production node and runtime - uses: ./.github/workflows/_build-production-node-and-runtime.yml - with: - ref: ${{ github.ref }} - - check-runtime-determinism: - needs: [build-production-node-and-runtime] - uses: ./.github/workflows/_check-runtime-determimism.yml - - store-production-node-and-runtime-in-mainnet-s3: - name: Store production node and runtime in Mainnet bucket - runs-on: ubuntu-20.04 - needs: [build-production-node-and-runtime] - steps: - - name: Checkout aleph-node source code - uses: actions/checkout@v4 - - - name: Store production node and runtime - uses: ./.github/actions/store-node-and-runtime - with: - profile: production - aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} - aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} - - push-production-node-image-to-ecr: - name: Push production node docker image to ECR - needs: [build-production-node-and-runtime] - uses: ./.github/workflows/_push-node-image-to-ecr.yml - secrets: inherit - with: - binary-artifact-name: aleph-production-node - docker-image-name: aleph-node - binary-name: aleph-node - docker-file-path: ./docker/Dockerfile - - send-slack-notification-release: - name: Send Slack notification about workflow status - runs-on: ubuntu-20.04 - if: always() && startsWith(github.ref_name, 'release-') - needs: [push-production-node-image-to-ecr] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Send Slack message - uses: Cardinal-Cryptography/github-actions/slack-notification@v6 - with: - notify-on: "always" - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} - - send-slack-notification-main: - name: Send Slack notification about workflow status - runs-on: ubuntu-20.04 - if: always() && github.ref_name == 'main' - needs: [push-production-node-image-to-ecr] - steps: - - name: Send Slack message - uses: Cardinal-Cryptography/github-actions/slack-notification@v6 - with: - notify-on: "failure" - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} +# build-and-push-cliain: +# needs: [check-vars-and-secrets] +# name: Build and push cliain +# uses: ./.github/workflows/_build-and-push-cliain.yml +# secrets: inherit +# with: +# check-only: false +# +# build-aleph-e2e-client-image: +# needs: [check-vars-and-secrets] +# name: Build aleph-e2e-client image +# uses: ./.github/workflows/_build-aleph-e2e-client-image.yml +# with: +# ref: ${{ github.ref }} +# push: true +# secrets: inherit +# +# build-production-node-and-runtime: +# needs: [check-vars-and-secrets] +# name: Build production node and runtime +# uses: ./.github/workflows/_build-production-node-and-runtime.yml +# with: +# ref: ${{ github.ref }} +# +# check-runtime-determinism: +# needs: [build-production-node-and-runtime] +# uses: ./.github/workflows/_check-runtime-determimism.yml +# +# store-production-node-and-runtime-in-mainnet-s3: +# name: Store production node and runtime in Mainnet bucket +# runs-on: ubuntu-20.04 +# needs: [build-production-node-and-runtime] +# steps: +# - name: Checkout aleph-node source code +# uses: actions/checkout@v4 +# +# - name: Store production node and runtime +# uses: ./.github/actions/store-node-and-runtime +# with: +# profile: production +# aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} +# aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} +# +# push-production-node-image-to-ecr: +# name: Push production node docker image to ECR +# needs: [build-production-node-and-runtime] +# uses: ./.github/workflows/_push-node-image-to-ecr.yml +# secrets: inherit +# with: +# binary-artifact-name: aleph-production-node +# docker-image-name: aleph-node +# binary-name: aleph-node +# docker-file-path: ./docker/Dockerfile +# +# send-slack-notification-release: +# name: Send Slack notification about workflow status +# runs-on: ubuntu-20.04 +# if: always() && startsWith(github.ref_name, 'release-') +# needs: [push-production-node-image-to-ecr] +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# +# - name: Send Slack message +# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 +# with: +# notify-on: "always" +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} +# +# send-slack-notification-main: +# name: Send Slack notification about workflow status +# runs-on: ubuntu-20.04 +# if: always() && github.ref_name == 'main' +# needs: [push-production-node-image-to-ecr] +# steps: +# - name: Send Slack message +# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 +# with: +# notify-on: "failure" +# env: +# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} From 04a800622aab1de8182c38d4fda6840376b0de67 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 25 Apr 2024 09:19:42 +0200 Subject: [PATCH 23/26] Revert "Testing main branch push" This reverts commit feb07e9dfeb53256166ec0cfa9b3ce1311b8ab05. --- .../on-main-or-release-branch-commit.yml | 167 +++++++++--------- 1 file changed, 83 insertions(+), 84 deletions(-) diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 1dc3aa494b..011aaa661d 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -6,7 +6,6 @@ on: branches: - main - 'release-*' - - A0-4231-move-chainspec-generation jobs: check-vars-and-secrets: @@ -33,86 +32,86 @@ jobs: binary-name: chain-bootstrapper docker-file-path: ./bin/chain-bootstrapper/Dockerfile -# build-and-push-cliain: -# needs: [check-vars-and-secrets] -# name: Build and push cliain -# uses: ./.github/workflows/_build-and-push-cliain.yml -# secrets: inherit -# with: -# check-only: false -# -# build-aleph-e2e-client-image: -# needs: [check-vars-and-secrets] -# name: Build aleph-e2e-client image -# uses: ./.github/workflows/_build-aleph-e2e-client-image.yml -# with: -# ref: ${{ github.ref }} -# push: true -# secrets: inherit -# -# build-production-node-and-runtime: -# needs: [check-vars-and-secrets] -# name: Build production node and runtime -# uses: ./.github/workflows/_build-production-node-and-runtime.yml -# with: -# ref: ${{ github.ref }} -# -# check-runtime-determinism: -# needs: [build-production-node-and-runtime] -# uses: ./.github/workflows/_check-runtime-determimism.yml -# -# store-production-node-and-runtime-in-mainnet-s3: -# name: Store production node and runtime in Mainnet bucket -# runs-on: ubuntu-20.04 -# needs: [build-production-node-and-runtime] -# steps: -# - name: Checkout aleph-node source code -# uses: actions/checkout@v4 -# -# - name: Store production node and runtime -# uses: ./.github/actions/store-node-and-runtime -# with: -# profile: production -# aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} -# aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} -# -# push-production-node-image-to-ecr: -# name: Push production node docker image to ECR -# needs: [build-production-node-and-runtime] -# uses: ./.github/workflows/_push-node-image-to-ecr.yml -# secrets: inherit -# with: -# binary-artifact-name: aleph-production-node -# docker-image-name: aleph-node -# binary-name: aleph-node -# docker-file-path: ./docker/Dockerfile -# -# send-slack-notification-release: -# name: Send Slack notification about workflow status -# runs-on: ubuntu-20.04 -# if: always() && startsWith(github.ref_name, 'release-') -# needs: [push-production-node-image-to-ecr] -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# -# - name: Send Slack message -# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 -# with: -# notify-on: "always" -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} -# -# send-slack-notification-main: -# name: Send Slack notification about workflow status -# runs-on: ubuntu-20.04 -# if: always() && github.ref_name == 'main' -# needs: [push-production-node-image-to-ecr] -# steps: -# - name: Send Slack message -# uses: Cardinal-Cryptography/github-actions/slack-notification@v6 -# with: -# notify-on: "failure" -# env: -# SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} + build-and-push-cliain: + needs: [check-vars-and-secrets] + name: Build and push cliain + uses: ./.github/workflows/_build-and-push-cliain.yml + secrets: inherit + with: + check-only: false + + build-aleph-e2e-client-image: + needs: [check-vars-and-secrets] + name: Build aleph-e2e-client image + uses: ./.github/workflows/_build-aleph-e2e-client-image.yml + with: + ref: ${{ github.ref }} + push: true + secrets: inherit + + build-production-node-and-runtime: + needs: [check-vars-and-secrets] + name: Build production node and runtime + uses: ./.github/workflows/_build-production-node-and-runtime.yml + with: + ref: ${{ github.ref }} + + check-runtime-determinism: + needs: [build-production-node-and-runtime] + uses: ./.github/workflows/_check-runtime-determimism.yml + + store-production-node-and-runtime-in-mainnet-s3: + name: Store production node and runtime in Mainnet bucket + runs-on: ubuntu-20.04 + needs: [build-production-node-and-runtime] + steps: + - name: Checkout aleph-node source code + uses: actions/checkout@v4 + + - name: Store production node and runtime + uses: ./.github/actions/store-node-and-runtime + with: + profile: production + aws-access-key-id: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }} + aws-bucket: ${{ secrets.CI_MAINNET_S3BUCKET_NAME }} + + push-production-node-image-to-ecr: + name: Push production node docker image to ECR + needs: [build-production-node-and-runtime] + uses: ./.github/workflows/_push-node-image-to-ecr.yml + secrets: inherit + with: + binary-artifact-name: aleph-production-node + docker-image-name: aleph-node + binary-name: aleph-node + docker-file-path: ./docker/Dockerfile + + send-slack-notification-release: + name: Send Slack notification about workflow status + runs-on: ubuntu-20.04 + if: always() && startsWith(github.ref_name, 'release-') + needs: [push-production-node-image-to-ecr] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Send Slack message + uses: Cardinal-Cryptography/github-actions/slack-notification@v6 + with: + notify-on: "always" + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} + + send-slack-notification-main: + name: Send Slack notification about workflow status + runs-on: ubuntu-20.04 + if: always() && github.ref_name == 'main' + needs: [push-production-node-image-to-ecr] + steps: + - name: Send Slack message + uses: Cardinal-Cryptography/github-actions/slack-notification@v6 + with: + notify-on: "failure" + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }} From 7e493f947274f34a43b60d002d4f7c5873e2be4a Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 25 Apr 2024 09:49:28 +0200 Subject: [PATCH 24/26] Final touches before review --- .github/scripts/run_consensus.sh | 2 +- bin/chain-bootstrapper/src/chain_spec/mod.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 1838a04dec..eb52abba26 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -11,7 +11,7 @@ # consensus size. -set -euox pipefail +set -euo pipefail # ------------------------ constants -------------------------------------- diff --git a/bin/chain-bootstrapper/src/chain_spec/mod.rs b/bin/chain-bootstrapper/src/chain_spec/mod.rs index 7a3e301189..ed90c5dc2b 100644 --- a/bin/chain-bootstrapper/src/chain_spec/mod.rs +++ b/bin/chain-bootstrapper/src/chain_spec/mod.rs @@ -12,7 +12,6 @@ pub const CHAINTYPE_LIVE: &str = "live"; pub const DEFAULT_CHAIN_ID: &str = "a0dnet1"; pub const DEFAULT_SUDO_ACCOUNT_ALICE: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; -// move to primitives pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; use primitives::AccountId; From 1299bdc118ef1533850cf9c1579b6f926fc09f92 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 26 Apr 2024 12:39:56 +0200 Subject: [PATCH 25/26] Review --- scripts/run_nodes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index e272181fdd..411138a913 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -230,7 +230,9 @@ fi if [[ -z "${DONT_BUILD_ALEPH_NODE}" ]]; then info "Building testing aleph-node binary (short session) and chain-bootstrapper binary." cargo build --release -p aleph-node - cargo build --release -p chain-bootstrapper --features "short_session enable_treasury_proposals" + if [[ -z "${DONT_BOOTSTRAP}" ]]; then + cargo build --release -p chain-bootstrapper --features "short_session enable_treasury_proposals" + fi elif [[ ! -x "${ALEPH_NODE}" || ! -x "${CHAINSPEC_GENERATOR}" ]]; then error "${ALEPH_NODE} or ${CHAINSPEC_GENERATOR} does not exist or it's not an executable file!" fi From 508de4c7d9b9104fc85ca56c69d3d6b7ba4770e5 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 9 May 2024 14:27:01 +0200 Subject: [PATCH 26/26] Review part 2 --- scripts/run_nodes.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index 411138a913..cb98f9bf7b 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -197,6 +197,9 @@ function run_node() { if [[ "${VALIDATORS}" -lt 1 ]]; then error "Number of validators should be at least 1!" fi +if [[ "${VALIDATORS}" -lt 4 ]]; then + warning "AlephBFT is only supported for more than 4 nodes." +fi if [[ "${RPC_NODES}" -lt 1 ]]; then error "Number of RPC nodes should be at least 1!" fi