feat: port forward and connect to durable GKE networks (#69)

.github/workflows/run-durable.yml

@@ -23,7 +23,7 @@ on:
         type: string
         description: Path regex passed to Jest to select which tests to run
         required: true
-        default: .
+        default: fast
       # The "job_id" input is needed for the CD manager to be able to track a workflow run as part of a CD manager job.
       # The GitHub API does not return the workflow run ID for a run created via the API. In order to track a workflow,
       # we're forced to inject the CD manager job ID via a tagged job step. This allows the CD manager to lookup
@@ -62,6 +62,18 @@ jobs:
           password: ${{ env.AWS_SECRET_ACCESS_KEY }}
         env:
           AWS_REGION: us-east-1
+      -
+        # TODO: Use a separate cluster for durable nodes, and eventually separate clusters for each network.
+        name: Setup GKE auth
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.GKE_SA_KEY }}
+      -
+        name: Get GKE credentials
+        uses: 'google-github-actions/get-gke-credentials@v1'
+        with:
+          cluster_name: ${{ secrets.GKE_CLUSTER }}
+          location: ${{ secrets.GKE_ZONE }}
       -
         name: Test ${{ matrix.network }}
         if: ${{ github.event.inputs.environment == matrix.network }}
@@ -72,6 +84,14 @@ jobs:
           if [[ -z "$BUILD_TAG" ]]; then
             BUILD_TAG=latest
           fi
+          # Expose the Keramik ComposeDB port to the host so that the tests can connect to it
+          source ./port-forward.sh "keramik-${{ matrix.network }}"
+          # If we found any Keramik ComposeDB endpoints, we'll also have Ceramic API endpoints. Add them to the config.
+          if [[ -n "$COMPOSEDB_URLS" ]];
+          then
+            sed -i "s|COMPOSEDB_URLS.*|&,$COMPOSEDB_URLS|" suite/env/.env."${{ matrix.network }}"
+            sed -i "s|CERAMIC_URLS.*|&,$CERAMIC_URLS|" suite/env/.env."${{ matrix.network }}"
+          fi
           make DURABLE_ENV=${{ matrix.network }} durable-tests
 
   collect-results:

README.md

@@ -1,22 +1,27 @@
-# Ceramic End To End Tests
+# Ceramic End-To-End Tests
 
-This repo contains an end to end test suite for the [Ceramic Network](https://github.com/ceramicnetwork/js-ceramic) and [ComposeDB](https://github.com/ceramicnetwork/js-composedb).
+This repo contains an end-to-end test suite for the [Ceramic Network](https://github.com/ceramicnetwork/js-ceramic) and
+[ComposeDB](https://github.com/ceramicnetwork/js-composedb).
 
 ## Design
 
 This crate provides three entities:
 
-* A test suite of end to end tests.
+* A test suite of end-to-end tests.
 * A hermetic test driver, which runs the test suite against isolated infrastructure.
 * A durable test driver, which runs the test suite against live public network infrastructure.
 
-The test suite is compiled into a docker image and expects to be configured with the endpoints of the infrastructure to test against.
+The test suite is compiled into a docker image and expects to be configured with the endpoints of the infrastructure to
+test against.
 The test drivers run the test image providing the correct configuration for their intended infrastructure.
 
-Hermetic infrastructure refers to infrastructure this is isolated from the rest of the world. In other words it controls all of its dependencies.
-Testing against hermetic infrastructure allows for easier debugging as all dependencies are controlled and can be isolated for potential bugs.
+Hermetic infrastructure refers to infrastructure this is isolated from the rest of the world. In other words it controls
+all of its dependencies.
+Testing against hermetic infrastructure allows for easier debugging as all dependencies are controlled and can be
+isolated for potential bugs.
 
-Durable infrastructure refers to infrastructure running connected to the rest of the world. For example other anonymous Ceramic nodes may connect to this infrastructure and it relies on a public Ethereum blockchain.
+Durable infrastructure refers to infrastructure running connected to the rest of the world. For example other anonymous
+Ceramic nodes may connect to this infrastructure, and it relies on a public Ethereum blockchain.
 Testing against durable infrastructure allows for better coverage of real world edge cases.
 
 ## Flavors
@@ -41,9 +46,8 @@ Smoke tests live outside of this repo.
 ## Prerequisites
 
 These test assume you have already set up Keramik.  If you are not also making changes to Keramik itself,
-you will likely want to configure Keramik to always use the 'latest' imageTag.  See comments in k8s/operator/kustomization.yaml
-for instructions on how to do that.
-
+you will likely want to configure Keramik to always use the 'latest' imageTag.  See comments in
+k8s/operator/kustomization.yaml for instructions on how to do that.
 
 ## Environment Variables
 
@@ -55,14 +59,17 @@ The test suite is configured via environment variables.
 | CERAMIC_URLS   | Comma separated list of URLs to Ceramic API endpoints   |
 
 Note, these names use the new API boundaries being currently constructed.
-So confusingly the `js-ceramic` process exposes the `ComposeDB API` while the `rust-ceramic` process exposes the `Ceramic API`.
+So confusingly the `js-ceramic` process exposes the `ComposeDB API` while the `rust-ceramic` process exposes the
+`Ceramic API`.
 
-Eventually we expect the `js-ceramic` repo/process to be renamed to `js-composedb` and the `js-composedb` repo being renamed to `js-compsedb-client` or other similar names.
+Eventually we expect the `js-ceramic` repo/process to be renamed to `js-composedb` and the `js-composedb` repo being
+renamed to `js-compsedb-client` or other similar names.
 
 
 ## Running tests locally
 
-Tests are run in CI via the build docker image. However locally its useful to be able to run the tests without doing an image build.
+Tests are run in CI via the build docker image. However, locally it's useful to be able to run the tests without doing
+an image build.
 There are some helpful scripts in this repo to make testing locally possible:
 ```bash
     # Create the network
@@ -78,7 +85,7 @@ There are some helpful scripts in this repo to make testing locally possible:
     pnpm run test --testPathPattern fast
 ```
 >NOTE: The port-forward script will leave `kubectl` process running in the background to forward the ports.
-The script kills those processes and creates new ones. However if you need you can kill them directly.
+The script kills those processes and creates new ones. However, if you need you can kill them directly.
 The script should be run anytime a pod in the network restarts.
 
 You can also run these tests against durable infrastructure (e.g. QA):
@@ -95,7 +102,8 @@ Use this script:
 
 The script will create a `topology.svg` (view it in a browser) of how each of the nodes are connected to one another.
 
->NOTE: The topology.mjs script expects the same environment variables to exist in order to communicate with the nodes to determine their topology.
+>NOTE: The topology.mjs script expects the same environment variables to exist in order to communicate with the nodes to
+> determine their topology.
 Source the `port-forward.sh` script in order to expose the environment locally.
 
 ## Using make
@@ -106,7 +114,8 @@ The make file exposes variables that can be set in the environment to control ho
 
 ### Example workflow
 
-Using this workflow you can build the test suite image locally, load the image into kind and then invoke the driver to run the tests in the network.
+Using this workflow you can build the test suite image locally, load the image into kind and then invoke the driver to
+run the tests in the network.
 
     # Choose a meaningful but local name for the image
     export TEST_SUITE_IMAGE_NAME=3box/ceramic-tests-suite
@@ -117,17 +126,17 @@ Using this workflow you can build the test suite image locally, load the image i
     # Build and run the hermetic test driver using the test suite image
     make hermetic-tests
 
-Using make is not required, feel free to peak inside the Makefile in order to call the test driver directly for whatever workflow you may need.
-
+Using make is not required, feel free to peak inside the Makefile in order to call the test driver directly for whatever
+workflow you may need.
 
 ### Debugging failures
 
-If you want to get the logs from any process started as part of running tests (the ceramic node, the CAS, the test output itself), you can use:
+If you want to get the logs from any process started as part of running tests (the ceramic node, the CAS, the test
+output itself), you can use:
 `kubectl logs -n <namespace> <pod name>`
 
 You can see a list of running kubernetes pods and their namespace and pod name by running `k9s`.
 
-
 ### Cleaning up
 
 To tear down all the pods created by running the tests, you can delete the network that they were set up in.

durable/durable-driver.sh

@@ -26,11 +26,14 @@ AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-.}
 AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-.}
 AWS_REGION=${AWS_REGION-us-east-1}
 
-docker run \
+# TODO: We are using the host network to make port forwarding work. This can be removed once we have public URLs for
+# Keramik endpoints.
+docker run --network="host" \
     -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
     -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
     -e AWS_REGION="${AWS_REGION}" \
     -e DB_ENDPOINT="${DB_ENDPOINT}" \
-    -e DOTENV_CONFIG_PATH="./env/.env.${DURABLE_ENV}" \
+    -v $(pwd)/suite/env/.env."${DURABLE_ENV}":/config/.env \
+    -e DOTENV_CONFIG_PATH="/config/.env" \
     -e TEST_SELECTOR="${TEST_SELECTOR}" \
     "${IMAGE_NAME}":"$tag"

port-forward.sh

@@ -8,8 +8,13 @@ set -m
 pids=$(jobs -p)
 if [ -n "$pids" ]
 then
-    kill $pids
-    wait $pids
+    kill "$pids"
+    wait "$pids"
+fi
+
+if [ -n "$1" ]
+then
+  namespace_flag="-n $1"
 fi
 
 composedb=7007
@@ -20,10 +25,10 @@ step=1
 COMPOSEDB_URLS=''
 CERAMIC_URLS=''
 
-for pod in $(kubectl get pods -l app=ceramic -o json | jq -r '.items[].metadata.name')
+for pod in $(kubectl $namespace_flag get pods -l app=ceramic -o json | jq -r '.items[].metadata.name')
 do
-    composedb_local=$((composedb + $offset))
-    ceramic_local=$((ceramic + $offset))
+    composedb_local=$((composedb + offset))
+    ceramic_local=$((ceramic + offset))
 
     if [ $offset != 1 ]
     then
@@ -35,9 +40,9 @@ do
     COMPOSEDB_URLS="${COMPOSEDB_URLS}http://localhost:$composedb_local"
     CERAMIC_URLS="${CERAMIC_URLS}http://localhost:$ceramic_local"
 
-    kubectl port-forward $pod $composedb_local:$composedb $ceramic_local:$ceramic >/dev/null  &
+    kubectl port-forward $namespace_flag "$pod" $composedb_local:$composedb $ceramic_local:$ceramic >/dev/null  &
 
-    offset=$((offset + $step))
+    offset=$((offset + step))
 done
 
 export COMPOSEDB_URLS=$COMPOSEDB_URLS

suite/Dockerfile

@@ -8,15 +8,11 @@ RUN apk add --no-cache curl jq
 
 COPY package.json pnpm-*.yaml jest.config.json tsconfig.json entrypoint.sh update.sh ./
 
-# install all npm dependencies including dev dependencies
+# Install all npm dependencies including dev dependencies
 RUN pnpm install
 
-# copy all files not excluded by .dockerignore of current directory to docker container
+# Copy all files not excluded by .dockerignore of current directory to docker container
 COPY src/ ./src
-COPY env/ ./env
-
-# Set the default .env path for Jest to load
-ENV DOTENV_CONFIG_PATH "./env/.env.dev"
 
 # Select all tests by default
 ENV TEST_SELECTOR "."