build-and-test.yml

name: Build and Test

on:
  push:
    branches:
      - "develop"
      - "main"
  pull_request:
  schedule:
    - cron: "0 0 * * 1"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    strategy:
      matrix:
        just_variants:
          - async-std
          - tokio
        test_suites:
          - test-ci-1
          - test-ci-2
          - test-ci-3
          - test-ci-4
          - test-ci-5
          - test-ci-rest
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout Repository

      - name: Install Rust
        uses: mkroening/rust-toolchain-toml@main

      - uses: Swatinem/rust-cache@v2
        name: Enable Rust Caching
        with:
          shared-key: "test"
          prefix-key: ${{ matrix.just_variants }}
          cache-on-failure: "true"
          save-if: ${{ github.ref == 'refs/heads/main' && matrix.test_suites == 'test-ci-rest' }}

      - name: Install Just
        run: |
          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
          sudo cp just /usr/bin/just

      - uses: taiki-e/install-action@nextest

      - name: Unit and integration tests for all crates in workspace
        run: |
          just ${{ matrix.just_variants }} ${{ matrix.test_suites }}
        timeout-minutes: 60
        env:
          RUST_BACKTRACE: full

  test-examples:
    strategy:
      matrix:
        just_variants:
          - async-std
          - tokio
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout Repository

      - name: Install Rust
        uses: mkroening/rust-toolchain-toml@main

      - uses: Swatinem/rust-cache@v2
        name: Enable Rust Caching
        with:
          shared-key: "examples"
          prefix-key: ${{ matrix.just_variants }}
          cache-on-failure: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install Just
        run: |
          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
          sudo cp just /usr/bin/just

      - name: Test examples
        run: |
          just ${{ matrix.just_variants }} example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml
        timeout-minutes: 20

  build-release:
    strategy:
      matrix:
        just_variants:
          - async-std
          - tokio
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout Repository

      - name: Install Rust
        uses: mkroening/rust-toolchain-toml@main

      - uses: Swatinem/rust-cache@v2
        name: Enable Rust Caching
        with:
          shared-key: "build-release"
          prefix-key: ${{ matrix.just_variants }}
          cache-on-failure: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install Just
        run: |
          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
          sudo cp just /usr/bin/just

      - name: Build examples in release mode
        run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features

      - name: Upload Binaries
        uses: actions/upload-artifact@v4
        with:
          name: binaries-amd64-${{ matrix.just_variants }}
          path: |
            target/${{ matrix.just_variants }}/release/examples/counter
            target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p
            target/${{ matrix.just_variants }}/release/examples/validator-libp2p
            target/${{ matrix.just_variants }}/release/examples/validator-combined
            target/${{ matrix.just_variants }}/release/examples/validator-push-cdn
            target/${{ matrix.just_variants }}/release/examples/orchestrator
            target/${{ matrix.just_variants }}/release/examples/cdn-broker
            target/${{ matrix.just_variants }}/release/examples/cdn-marshal

  build-arm-release:
    strategy:
      matrix:
        just_variants:
          - async-std
          - tokio
      fail-fast: false
    runs-on: buildjet-4vcpu-ubuntu-2204-arm
    if: ${{ github.ref == 'refs/heads/main' }}
    container: ghcr.io/espressosystems/devops-rust:stable
    steps:
      - uses: actions/checkout@v4
        name: Checkout Repository

      - uses: Swatinem/rust-cache@v2
        name: Enable Rust Caching
        with:
          shared-key: "build-arm-release"
          prefix-key: ${{ matrix.just_variants }}
          cache-on-failure: "true"
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Build examples in release mode
        run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features

      - name: Upload Binaries
        uses: actions/upload-artifact@v4
        with:
          name: binaries-aarch64-${{ matrix.just_variants }}
          path: |
            target/${{ matrix.just_variants }}/release/examples/counter
            target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p
            target/${{ matrix.just_variants }}/release/examples/validator-libp2p
            target/${{ matrix.just_variants }}/release/examples/validator-combined
            target/${{ matrix.just_variants }}/release/examples/validator-push-cdn
            target/${{ matrix.just_variants }}/release/examples/orchestrator
            target/${{ matrix.just_variants }}/release/examples/cdn-broker
            target/${{ matrix.just_variants }}/release/examples/cdn-marshal

  build-dockers:
    strategy:
      matrix:
        just_variants:
          - async-std
          - tokio
      fail-fast: false
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/main' }}
    needs: [build-release, build-arm-release, test]
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Setup Docker BuildKit (buildx)
        uses: docker/setup-buildx-action@v3

      - name: Login to Github Container Repo
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download AMD executables
        uses: actions/download-artifact@v4
        with:
          name: binaries-amd64-${{ matrix.just_variants }}
          path: target/${{ matrix.just_variants }}/amd64/release/examples

      - name: Download ARM executables
        uses: actions/download-artifact@v4
        with:
          name: binaries-aarch64-${{ matrix.just_variants }}
          path: target/${{ matrix.just_variants }}/arm64/release/examples

      - name: Generate validator-libp2p docker metadata
        uses: docker/metadata-action@v5
        id: validator-libp2p
        with:
          images: ghcr.io/espressosystems/hotshot/validator-libp2p
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Generate validator-combined docker metadata
        uses: docker/metadata-action@v5
        id: validator-combined
        with:
          images: ghcr.io/espressosystems/hotshot/validator-combined
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Generate validator-push-cdn docker metadata
        uses: docker/metadata-action@v5
        id: validator-push-cdn
        with:
          images: ghcr.io/espressosystems/hotshot/validator-push-cdn
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Generate orchestrator docker metadata
        uses: docker/metadata-action@v5
        id: orchestrator
        with:
          images: ghcr.io/espressosystems/hotshot/orchestrator
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Generate cdn-broker docker metadata
        uses: docker/metadata-action@v5
        id: cdn-broker
        with:
          images: ghcr.io/espressosystems/hotshot/cdn-broker
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Generate cdn-marshal docker metadata
        uses: docker/metadata-action@v5
        id: cdn-marshal
        with:
          images: ghcr.io/espressosystems/hotshot/cdn-marshal
          flavor: suffix=-${{ matrix.just_variants }}

      - name: Build and push validator-libp2p docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/validator-libp2p.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.validator-libp2p.outputs.tags }}
          labels: ${{ steps.validator-libp2p.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}

      - name: Build and push validator-combined docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/validator-combined.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.validator-combined.outputs.tags }}
          labels: ${{ steps.validator-combined.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}

      - name: Build and push validator-push-cdn docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/validator-cdn.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.validator-push-cdn.outputs.tags }}
          labels: ${{ steps.validator-push-cdn.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}

      - name: Build and push orchestrator docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/orchestrator.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.orchestrator.outputs.tags }}
          labels: ${{ steps.orchestrator.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}

      - name: Build and push cdn-broker docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/cdn-broker.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.cdn-broker.outputs.tags }}
          labels: ${{ steps.cdn-broker.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}

      - name: Build and push cdn-marshal docker
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./docker/cdn-marshal.Dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.cdn-marshal.outputs.tags }}
          labels: ${{ steps.cdn-marshal.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            ASYNC_EXECUTOR=${{ matrix.just_variants }}