A0-4231: Extract chain bootstrap to a separate crate

.dockerignore

@@ -1,6 +1,8 @@
 **/target
 target
 !target/release/aleph-node
+!target/release/chain-bootstrapper
+!target/production/chain-bootstrapper
 !target/production/aleph-node
 !bin/cliain/target/release/cliain
 

.github/actions/run-e2e-test/action.yml

@@ -38,6 +38,10 @@ inputs:
   aleph-e2e-client-image:
     description: 'aleph-e2e-client image'
     required: true
+  chain-bootstrapper-image:
+    description: 'chain-bootstrapper image'
+    required: false
+    default: chain-bootstrapper-image-test
   timeout-minutes:
     description: 'The maximum number of minutes to let a test run before it is canceled and
       considered failed'
@@ -54,7 +58,7 @@ runs:
       id: get-ref-properties
       uses: Cardinal-Cryptography/github-actions/get-ref-properties@v5
 
-    - name: Download artifact with docker image
+    - name: Download node docker image
       uses: actions/download-artifact@v4
       with:
         name: ${{ inputs.image-path }}
@@ -63,6 +67,15 @@ runs:
       shell: bash
       run: docker load -i aleph-node.tar
 
+    - name: Download chain-bootstrapper image
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.chain-bootstrapper-image }}
+
+    - name: Load chain-bootstrapper image
+      shell: bash
+      run: docker load -i chain-bootstrapper.tar
+
     - name: Run consensus party
       shell: bash
       run: |

.github/scripts/run_consensus.sh

@@ -20,6 +20,7 @@ MIN_VALIDATOR_COUNT=${MIN_VALIDATOR_COUNT:-4}
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-docker/docker-compose.yml}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
 NODE_IMAGE=${NODE_IMAGE:-"aleph-node:latest"}
+CHAIN_BOOTSTRAPPER_IMAGE=${CHAIN_BOOTSTRAPPER_IMAGE:-"chain-bootstrapper:latest"}
 LOGS_OUTPUT_FILE=${LOGS_OUTPUT_FILE:=""}
 
 # ------------------------ argument parsing and usage -----------------------
@@ -81,8 +82,8 @@ function generate_chainspec() {
   local validator_ids_comma_separated="${validators//${IFS:0:1}/,}"
 
   echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}"
-  docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \
-  -c "aleph-node bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated}  > /data/chainspec.json"
+  docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER_IMAGE}" \
+  -c "chain-bootstrapper bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated}  > /data/chainspec.json"
 }
 
 function generate_bootnode_peer_id() {
@@ -162,6 +163,15 @@ else
   exit 1
 fi
 
+if docker inspect ${NODE_IMAGE} > /dev/null; then
+  echo "aleph-node image tag ${NODE_IMAGE} found locally"
+else
+  echo "${NODE_IMAGE} not found locally."
+  echo "Build image first with:"
+  echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ."
+  exit 1
+fi
+
 mkdir -p docker/data/
 echo "Warning: if you run this script on the same machine more then once, and finalization does not work, remove docker/data."
 global_account_ids=$(generate_account_ids ${NODE_COUNT})

.github/scripts/test_python_general.sh

@@ -8,8 +8,10 @@ function usage(){
   cat << EOF
 Usage:
    $0
-    --aleph-node BINARY]
-      path to aleph-node-binary
+    --aleph-node BINARY
+      path to aleph-node binary
+    --chain-bootstrapper BINARY
+      path to chain-bootstrapper binary
     --testcase NAME
       name of python file in local-tests directory to run
 EOF
@@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; do
       ALEPH_NODE_BINARY="$2"
       shift;shift
       ;;
+    --chain-bootstrapper)
+      CHAIN_BOOTSTRAPPER="$2"
+      shift;shift
+      ;;
     --testcase)
       TESTCASE="$2"
       shift;shift
@@ -40,7 +46,11 @@ done
 pushd local-tests/ > /dev/null
 
 if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then
-  echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}."
+  echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}"
+  exit 1
+fi
+if [[ ! -f "${CHAIN_BOOTSTRAPPER}" ]]; then
+  echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}"
   exit 1
 fi
 if [[ -z "${TESTCASE}" ]]; then
@@ -56,6 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then
 fi
 
 chmod +x "${ALEPH_NODE_BINARY}"
+chmod +x "${CHAIN_BOOTSTRAPPER}"
 echo "Installing python requirements"
 pip install -r requirements.txt
 
@@ -65,6 +76,9 @@ pip install -r requirements.txt
 # first buffered and that you can see the output of your application.
 export PYTHONUNBUFFERED=y
 export ALEPH_NODE_BINARY
+export CHAIN_BOOTSTRAPPER
+export RUST_LOG=debug
 export WORKDIR=$(mktemp -d)
+echo "WORKDIR is ${WORKDIR}"
 eval "./${file_name_to_run}"
 popd > /dev/null

.github/workflows/_build-and-push-featurenet-node-image.yml

@@ -17,10 +17,10 @@ on:
         type: boolean
         required: true
 jobs:
-  build-test-node-and-runtime:
+  build-test-node:
     if: ${{ inputs.short-session == true }}
     name: Build test node and runtime
-    uses: ./.github/workflows/_build-test-node-and-runtime.yml
+    uses: ./.github/workflows/_build-test-node.yml
     with:
       ref: ${{ inputs.ref }}
 
@@ -33,8 +33,8 @@ jobs:
 
   build-and-push-featurnet-image-to-ecr:
     name: Build and push featurnet image
-    needs: [build-test-node-and-runtime, build-production-node-and-runtime]
-    # to prevent this job to be skipped as at least on of dependant jobs is skipped
+    needs: [build-test-node, build-production-node-and-runtime]
+    # this job should not be skipped in case one of its parent job is skipped
     if: ${{ !cancelled() }}
     runs-on: ubuntu-20.04
     steps:

.github/workflows/_build-chain-bootstrapper.yml

@@ -0,0 +1,78 @@
+---
+# This workflow builds chain-bootstrapper binary, dedicated fer generating chainspecs for
+# test and dev chains
+name: Build chain-bootstrapper
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'git ref: hash, branch, tag to build chain-bootstrapper binary from'
+        type: string
+        required: true
+      production:
+        description: 'set to true to use production profile'
+        type: boolean
+        required: true
+
+jobs:
+  main:
+    name: Build chain-bootstrapper
+    runs-on: [self-hosted, Linux, X64, large]
+    env:
+      RUST_BACKTRACE: full
+      RUSTC_WRAPPER: sccache
+      CARGO_FOLDER: ${{ inputs.production == true && 'production' || 'release' }}
+      ARTIFACT_NAME_SUFFIX: ${{ inputs.production == true && 'release' || 'test' }}
+    steps:
+      - name: Checkout aleph-node source code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+
+      - name: Call action get-ref-properties
+        id: get-ref-properties
+        uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6
+
+      - name: Install Rust toolchain
+        uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6
+        with:
+          targets: wasm32-unknown-unknown
+
+      - name: Build without production profile
+        if: ${{ inputs.production != true }}
+        run: |
+          cargo build --release -p chain-bootstrapper \
+            --features "short_session enable_treasury_proposals"
+
+      - name: Build with production profile
+        if: ${{ inputs.production == true }}
+        run: |
+          cargo build --profile production -p chain-bootstrapper
+
+      - name: Upload binary to GH Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: chain-bootstrapper-${{ env.ARTIFACT_NAME_SUFFIX }}
+          path: target/${{ env.CARGO_FOLDER }}/chain-bootstrapper
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: Build docker
+        id: build-image
+        run: |
+          chmod +x  target/${{ env.CARGO_FOLDER }}/chain-bootstrapper
+          if [[ ${{ inputs.production }} == true ]]; then
+            mkdir -p target/release
+            mv target/production/chain-bootstrapper target/release/
+          fi
+          docker build --tag chain-bootstrapper:latest -f ./bin/chain-bootstrapper/Dockerfile .
+          docker save -o chain-bootstrapper.tar chain-bootstrapper:latest
+
+      - name: Upload docker image to GH Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: chain-bootstrapper-image-${{ env.ARTIFACT_NAME_SUFFIX }}
+          path: chain-bootstrapper.tar
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/_build-production-node-and-runtime.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           targets: wasm32-unknown-unknown
 
-      - name: Build production binary and runtime
+      - name: Build production aleph-node
         run: cargo build --profile production -p aleph-node
 
       - name: Upload release binary to GH artifacts
@@ -43,6 +43,9 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+      - name: Build production aleph-runtime
+        run: cargo build --profile production -p aleph-runtime
+
       # required by _check-runtime-determinism.yml workflow
       - name: Upload release runtime to GH artifacts
         uses: actions/upload-artifact@v4

.github/workflows/_build-test-node-and-runtime.yml → .github/workflows/_build-test-node.yml

@@ -1,7 +1,6 @@
 ---
-#  This workflow builds test version of aleph-node and aleph-runtime, ie with some
-#  features enabled. Those binaries MUST NOT be used in the production.
-name: Build test node and runtime
+#  This workflow builds test version of aleph-node. It MUST NOT be used in the production.
+name: Build test node
 on:
   workflow_call:
     inputs:
@@ -11,8 +10,8 @@ on:
         required: true
 
 jobs:
-  build-artifacts:
-    name: Build test node and runtime
+  main:
+    name: Build test node
     runs-on: [self-hosted, Linux, X64, large]
     env:
       RUST_BACKTRACE: full
@@ -30,32 +29,29 @@ jobs:
 
       - name: Install Rust toolchain
         uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6
-        with:
-          targets: wasm32-unknown-unknown
 
-      - name: Build test binary and runtime
+      - name: Build
         run: |
-          cargo build --release -p aleph-node \
-            --features "short_session enable_treasury_proposals only_legacy"
+          cargo build --release -p aleph-node --features only_legacy
 
       # this is required for some workflows, yet ideally it should not be required
-      - name: Upload test binary to GH Artifacts
+      - name: Upload binary to GH Artifacts
         uses: actions/upload-artifact@v4
         with:
           name: aleph-test-node
           path: target/release/aleph-node
           if-no-files-found: error
           retention-days: 7
 
-      - name: Build test docker image
+      - name: Build docker
         id: build-image
         run: |
           chmod +x target/release/aleph-node
           docker build --tag aleph-node:latest -f ./docker/Dockerfile .
           docker save -o aleph-node.tar aleph-node:latest
 
-      # this is solely required by action/run-e2e-test.yml, also it should not be required
-      - name: Upload test docker image
+      # this is required by action/run-e2e-test.yml
+      - name: Upload docker to GH Artifacts
         uses: actions/upload-artifact@v4
         with:
           name: aleph-test-docker

.github/workflows/_check-production-node-and-runtime.yml

@@ -25,5 +25,8 @@ jobs:
         with:
           targets: wasm32-unknown-unknown
 
-      - name: Check production binary and runtime
+      - name: Check aleph-node
         run: cargo check --profile production -p aleph-node
+
+      - name: Check aleph-runtime
+        run: cargo check --profile production -p aleph-runtime

.github/workflows/_push-node-image-to-ecr.yml

@@ -1,21 +1,29 @@
 ---
-#  This workflow builds docker aleph-node image based on binary stored in GH artifacts
-name: Build and push node image to ECR
+#  This workflow builds arbitrary docker node image based on binary downloaded in GH artifacts
+name: Build and push docker image to ECR
 on:
   workflow_call:
     inputs:
-      node-binary:
-        description: 'Name of aleph-node binary stored in GH artifacts'
+      binary-artifact-name:
+        description: 'Name of a binary stored in GH artifacts'
+        required: true
+        type: string
+      binary-name:
+        description: 'Name of a binary to build docker image on top of'
         required: true
         type: string
       docker-image-name:
-        description: 'Name of node docker to be uploaded to ECR'
+        description: 'Name of node docker image to be uploaded to ECR'
+        required: true
+        type: string
+      docker-file-path:
+        description: 'Path to Dockerfile'
         required: true
         type: string
 
 jobs:
   main:
-    name: Push node docker image to the ECR
+    name: Push ${{ inputs.binary-name }} docker image to the ECR
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout aleph-node sources
@@ -25,17 +33,19 @@ jobs:
         id: get-ref-properties
         uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6
 
-      - name: Download ${{ inputs.node-binary }} binary from artifacts
+      - name: Download ${{ inputs.binary-artifact-name }} from artifacts
         uses: actions/download-artifact@v4
         with:
-          name: ${{ inputs.node-binary }}
+          name: ${{ inputs.binary-artifact-name }}
           path: target/release/
 
-      - name: Build docker node image
+      - name: Build ${{ inputs.binary-name }} docker image
         id: build-image
         run: |
-          chmod +x target/release/aleph-node
-          docker build --tag ${{ inputs.docker-image-name }}:latest -f ./docker/Dockerfile .
+          chmod +x target/release/${{ inputs.binary-name }}
+          docker build \
+            --tag ${{ inputs.docker-image-name }}:latest \
+            -f ${{ inputs.docker-file-path }} .
 
       - name: Login to Public Amazon ECR
         id: login-public-ecr

.github/workflows/deploy-to-devnet.yml

@@ -98,7 +98,7 @@ jobs:
           EVE=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw
 
           docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint \
-            '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}aleph-node:${{ env.RELEASE_TAG }}' \
+            '/usr/local/bin/chain-bootstrapper' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \
             bootstrap-chain --raw --base-path /data --chain-id a0dnet1 \
             --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q \
             --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD \