feat: add workflows for property-based testing

.github/workflows/test.yml

@@ -42,14 +42,19 @@ jobs:
         with:
           cluster_name: ${{ secrets.GKE_CLUSTER }}
           location: ${{ secrets.GKE_ZONE }}
+      -
+        # Add a "_sha_runId" suffix to differentiate test runs
+        name: Setup test env
+        run: |
+          TEST_SUFFIX="-$(echo ${{ github.sha }} | head -c 12)-${{ github.run_id }}"
+          echo "TEST_SUFFIX=$TEST_SUFFIX" >> $GITHUB_ENV
       -
         name: Test ${{ matrix.networks }}
         working-directory: k8s/tests/scripts
-        run: |
-          # Add a "_sha_runId" suffix to differentiate test runs
-          TEST_SUFFIX="_$(echo ${{ github.sha }} | head -c 12)_${{ github.run_id }}"
-          ./run-tests.sh ${{ matrix.networks }}
+        run: ./run-tests.sh ${{ matrix.networks }}
       -
+        # Always cleanup resources at the end, whether the tests pass or fail.
         name: Delete ${{ matrix.networks }}
+        if: always()
         working-directory: k8s/tests/scripts
         run: ./delete-tests.sh ${{ matrix.networks }}

Makefile

@@ -37,7 +37,3 @@ check-clippy:
 publish-docker:
 	./ci-scripts/publish.sh
 
-.PHONY:
-publish-docker:
-	./ci-scripts/publish.sh
-

ci-scripts/build-test-binaries.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 BUILD_MODE=$1
-if [ -z $BUILD_MODE ]
+if [ -z "$BUILD_MODE" ]
 then
     echo "Must pass build mode arg"
     exit 1
@@ -12,11 +12,11 @@ mkdir test-binaries
 for b in $(RUSTFLAGS='-D warnings' cargo test \
     --no-run \
     --locked \
-    --profile $BUILD_MODE \
+    --profile "$BUILD_MODE" \
     -q \
     --message-format=json \
     | jq -r 'select(.executable != null) | .executable')
 do
-    echo $b
-    cp $b test-binaries/
+    echo "$b"
+    cp "$b" test-binaries/
 done

entrypoint.sh

@@ -6,8 +6,6 @@ set -e
 for tb in /test-binaries/*
 do
     echo "Running tests in: $tb"
-    $tb
+    # Always exit successfully so that the test job isn't restarted
+    $tb || exit 0
 done
-
-# Always exit successfully so that the test job isn't restarted
-exit 0

k8s/tests/scripts/run-tests.sh

@@ -9,9 +9,17 @@ yq "
 
 # Create the network.
 kubectl create configmap check-network --from-file=check-network.sh -n "keramik-$1$TEST_SUFFIX"
-yq ".metadata.name = \"keramik-$1$TEST_SUFFIX\"" ../../networks/"$1".yaml | kubectl apply -f -
+yq ".metadata.name = \"$1$TEST_SUFFIX\"" ../../networks/"$1".yaml | kubectl apply -f -
 
 # Create the test job
 kubectl apply -f ../manifests/tests.yaml -n "keramik-$1$TEST_SUFFIX"
 
-# Wait for tests to complete
+# Wait for tests to complete then collect the results
+kubectl wait --for=condition=complete job/ceramic-tests -n "keramik-$1$TEST_SUFFIX" --timeout=20m
+TEST_RESULTS=$(kubectl logs job.batch/ceramic-tests -c ceramic-tests -n "keramik-$1$TEST_SUFFIX")
+echo "$TEST_RESULTS"
+# Return a failure from here instead of from the tests. The test job is setup to restart on failure, which is needed for
+# it to be setup correctly with the right Ceramic/ComposeDB URLs.
+if [[ "$TEST_RESULTS" =~ "FAILED" ]]; then
+   exit 1
+fi

tests/api.rs

@@ -12,7 +12,7 @@ use tokio::sync::OnceCell;
 use tracing_test::traced_test;
 use url::Url;
 
-use rust_ceramic_migration_tests::ceramic::{composedb_client, ComposeDbClient};
+use ceramic_tests::ceramic::{composedb_client, ComposeDbClient};
 
 const CERAMIC_URLS: &str = "CERAMIC_URLS";
 const COMPOSEDB_URLS: &str = "COMPOSEDB_URLS";