feat: run performance simulation on merge queue (#83)

.github/workflows/performance.yml

@@ -0,0 +1,109 @@
+name: Run Performance tests
+
+on:
+  merge_group:
+    branches: ['main']
+  schedule:
+    - cron: '0 0 * * *' # Run every midnight UTC
+  workflow_dispatch: # manually triggered
+
+env:
+  CARGO_TERM_COLOR: always
+  SCCACHE_GHA_ENABLED: 'true'
+  RUSTC_WRAPPER: 'sccache'
+
+jobs:
+  generate-matrix:
+    name: Generate Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      networks: ${{ steps.generate-matrix.outputs.networks }}
+      build_tag: ${{ steps.generate-matrix.outputs.build_tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Generate network matrix
+        id: generate-matrix
+        run: |
+          NETWORKS=$(ls networks | jq -R -s -c '. | gsub(".yaml"; "") | split("\n")[:-1]')
+          echo "Networks:"
+          echo ${NETWORKS}
+          echo "networks=${NETWORKS}" >> $GITHUB_OUTPUT
+          # Choose unique name for this build
+          BUILD_TAG="$(echo ${{ github.sha }} | head -c 8)-${{ github.run_id }}"
+          echo "Build tag:"
+          echo ${BUILD_TAG}
+          echo "build_tag=${BUILD_TAG}" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+
+  build-driver:
+    name: Build 'hermetic-driver' binary
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # The prefix cache key, this can be changed to start a new cache manually.
+          # default: "v0-rust"
+          prefix-key: v0
+          # Cache only the cargo registry
+          cache-targets: false
+      - uses: mozilla-actions/[email protected]
+      - name: Build Tester
+        run: make BUILD_PROFILE=release driver
+      - uses: actions/upload-artifact@master
+        with:
+          name: hermetic-driver
+          path: ./target/release/hermetic-driver
+          retention-days: 1
+
+  run-performance-tests:
+    name: Performance Tests
+    runs-on: ubuntu-latest
+    needs:
+      - generate-matrix
+      - build-driver
+    strategy:
+      fail-fast: false
+      matrix:
+        networks: ${{ fromJSON(needs.generate-matrix.outputs.networks) }}
+        # TODO: add simulations matrix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup GKE auth
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.GKE_SA_KEY }}
+      - name: Get GKE credentials
+        uses: 'google-github-actions/get-gke-credentials@v1'
+        with:
+          cluster_name: ${{ secrets.GKE_CLUSTER }}
+          location: ${{ secrets.GKE_ZONE }}
+      - uses: actions/download-artifact@master
+        with:
+          name: hermetic-driver
+          path: ./bin
+      - name: Test ${{ matrix.networks }}
+        run: |
+          export BUILD_TAG=${{ needs.generate-matrix.outputs.build_tag }}
+          export TEST_NETWORK=./networks/${{ matrix.networks }}.yaml
+
+          export TEST_SIMULATION=./networks/basic-simulation.yaml
+          chmod +x ./bin/hermetic-driver
+          make HERMETIC_CMD=./bin/hermetic-driver performance-tests
+
+  collect-results:
+    name: Performance Test Results
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [run-performance-tests]
+    steps:
+      - run: exit 1
+        # see https://stackoverflow.com/a/67532120/4907315
+        if: >-
+          ${{
+                contains(needs.*.result, 'failure')
+            || contains(needs.*.result, 'cancelled')
+          }}

Makefile

@@ -10,6 +10,8 @@ BUILD_PROFILE ?= release
 BUILD_TAG ?= dev-run
 # Path to network to test against.
 TEST_NETWORK ?= ./networks/basic-go-rust.yaml
+# Path to performance simulation.
+TEST_SIMULATION ?= ./networks/basic-simulation.yaml
 # Name for the test suite image, without any tag
 TEST_SUITE_IMAGE_NAME ?= public.ecr.aws/r5b3e0r5/3box/ceramic-tests-suite
 # Full path of the test suite image with a tag
@@ -104,6 +106,15 @@ hermetic-tests:
 		--test-image "${TEST_SUITE_IMAGE}" \
 		--test-selector "${TEST_SELECTOR}"
 
+.PHONY: performance-tests
+performance-tests:
+	${HERMETIC_CMD} test \
+		--network "${TEST_NETWORK}" \
+		--flavor perf \
+		--suffix "${HERMETIC_SUFFIX}" \
+		--network-ttl ${HERMETIC_TTL} \
+		--simulation ${TEST_SIMULATION}
+
 .PHONY: durable-tests
 durable-tests:
 	BUILD_TAG="${BUILD_TAG}" IMAGE_NAME="${TEST_SUITE_IMAGE_NAME}" ./durable/durable-driver.sh "${DURABLE_ENV}" "${TEST_SELECTOR}"

hermetic/src/cli.rs

@@ -1,6 +1,10 @@
 use std::{fmt::Display, path::PathBuf};
 
+use anyhow::anyhow;
 use clap::{builder::PossibleValue, Args, Parser, Subcommand, ValueEnum};
+
+use self::tester::{Flavor, TestConfig};
+
 pub mod tester;
 
 #[derive(Parser, Debug)]
@@ -22,6 +26,10 @@ pub struct TestOpts {
     #[arg(short, long)]
     network: PathBuf,
 
+    /// Path to simulation yaml file
+    #[arg(long)]
+    simulation: Option<PathBuf>,
+
     /// Name and label of the specific test image to use.
     /// Defaults to latest published image.
     /// Setting this value implies an image pull policy of IfNotPresent
@@ -30,7 +38,7 @@ pub struct TestOpts {
 
     /// Type of tests to run.
     #[arg(short, long)]
-    flavor: Flavor,
+    flavor: FlavorOpts,
 
     /// Optional suffix to apply to network name.
     /// Used to create unique networks.
@@ -55,34 +63,78 @@ pub struct TestOpts {
 }
 
 #[derive(Debug, Clone)]
-pub enum Flavor {
+pub enum FlavorOpts {
     /// Property based tests
     Property,
     /// Smoke tests
     Smoke,
+    /// Performance tests
+    Performance,
 }
 
-impl Flavor {
+impl FlavorOpts {
     fn name(&self) -> &'static str {
         match self {
-            Flavor::Property => "prop",
-            Flavor::Smoke => "smoke",
+            FlavorOpts::Property => "prop",
+            FlavorOpts::Smoke => "smoke",
+            FlavorOpts::Performance => "perf",
         }
     }
 }
 
-impl ValueEnum for Flavor {
+impl ValueEnum for FlavorOpts {
     fn value_variants<'a>() -> &'a [Self] {
-        &[Flavor::Property, Flavor::Smoke]
+        &[
+            FlavorOpts::Property,
+            FlavorOpts::Smoke,
+            FlavorOpts::Performance,
+        ]
     }
 
     fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
         Some(PossibleValue::new(self.name()))
     }
 }
 
-impl Display for Flavor {
+impl Display for FlavorOpts {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.name())
     }
 }
+
+impl TryFrom<TestOpts> for TestConfig {
+    type Error = anyhow::Error;
+
+    fn try_from(opts: TestOpts) -> Result<Self, Self::Error> {
+        let TestOpts {
+            network,
+            simulation,
+            test_image,
+            flavor,
+            suffix,
+            network_ttl,
+            network_timeout,
+            job_timeout,
+            test_selector,
+        } = opts;
+
+        let flavor = match flavor {
+            FlavorOpts::Property => Flavor::Property,
+            FlavorOpts::Smoke => Flavor::Smoke,
+            FlavorOpts::Performance => Flavor::Performance(
+                simulation.ok_or(anyhow!("Simulation file required for performance tests"))?,
+            ),
+        };
+
+        Ok(TestConfig {
+            network,
+            test_image,
+            flavor,
+            suffix,
+            network_ttl,
+            network_timeout,
+            job_timeout,
+            test_selector,
+        })
+    }
+}