node: Add Transfer Verifier mechanism

.golangci.yml

@@ -28,3 +28,5 @@ issues:
       text: "^func.*supervisor.*(waitSettle|waitSettleError).*$"
       linters:
         - unused
+    - path: pkg/transfer-verifier/transfer-verifier-sui_test.go
+      text: "G101: Potential hardcoded credentials"

Tiltfile

@@ -463,6 +463,7 @@ k8s_resource(
     trigger_mode = trigger_mode,
 )
 
+
 if solana or pythnet:
     # solana client cli (used for devnet setup)
 
@@ -604,6 +605,11 @@ if evm2:
     )
 
 
+# Note that ci_tests requires other resources in order to build properly:
+# - eth-devnet  -- required by: accountant_tests, ntt_accountant_tests, tx-verifier
+# - eth-devnet2 -- required by: accountant_tests, ntt_accountant_tests
+# - wormchain   -- required by: accountant_tests, ntt_accountant_tests
+# - solana      -- required by: spydk-ci-tests
 if ci_tests:
     docker_build(
         ref = "sdk-test-image",
@@ -635,6 +641,16 @@ if ci_tests:
             sync("./testing", "/app/testing"),
         ],
     )
+    docker_build(
+        ref = "tx-verifier-monitor", 
+        context = "./devnet/tx-verifier-monitor/",
+        dockerfile = "./devnet/tx-verifier-monitor/Dockerfile"
+    )
+    docker_build(
+        ref = "tx-verifier-test", 
+        context = "./devnet/tx-verifier-monitor/",
+        dockerfile = "./devnet/tx-verifier-monitor/Dockerfile.cast"
+    )
 
     k8s_yaml_with_ns(
         encode_yaml_stream(
@@ -644,6 +660,11 @@ if ci_tests:
                     "BOOTSTRAP_PEERS", str(ccqBootstrapPeers)),
                     "MAX_WORKERS", max_workers))
     )
+
+    # transfer-verifier -- daemon and log monitoring
+    k8s_yaml_with_ns("devnet/tx-verifier.yaml")
+
+    k8s_yaml_with_ns("devnet/tx-verifier-test.yaml")
 
     # separate resources to parallelize docker builds
     k8s_resource(
@@ -676,6 +697,20 @@ if ci_tests:
         trigger_mode = trigger_mode,
         resource_deps = [], # testing/querysdk.sh handles waiting for query-server, not having deps gets the build earlier
     )
+    # launches tx-verifier binary and sets up monitoring script
+    k8s_resource(
+        "tx-verifier-with-monitor",
+        resource_deps = ["eth-devnet"],
+        labels = ["evm", "tx-verifier"],
+        trigger_mode = trigger_mode,
+    )
+    # triggers the integration tests that will be detected by the monitor
+    k8s_resource(
+        "tx-verifier-test",
+        resource_deps = ["eth-devnet", "tx-verifier-with-monitor"],
+        labels = ["evm", "tx-verifier"],
+        trigger_mode = trigger_mode,
+    )
 
 if terra_classic:
     docker_build(

devnet/tx-verifier-monitor.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: monitor
+spec:
+  selector:
+    matchLabels:
+      app: monitor
+  template:
+    metadata:
+      labels:
+        app: monitor
+    spec:
+      containers:
+      - name: monitor
+        image: monitor
+        volumeMounts:
+        - name: log-volume
+          mountPath: /logs
+        env:
+        - name: ERROR_PATTERN
+          # This error string comes from the transfer-verifier binary in node/
+          value: "invalid receipt: no deposits and no transfers"
+        - name: ERROR_LOG_PATH
+          value: "/logs/error.log"
+      volumes:
+      - name: log-volume
+        emptyDir: {}

devnet/tx-verifier-monitor/Dockerfile

@@ -0,0 +1,10 @@
+# There's nothing special about this version, it is simply the `latest` as of
+# the creation date of this file.
+FROM alpine:3.20.3@sha256:1e42bbe2508154c9126d48c2b8a75420c3544343bf86fd041fb7527e017a4b4a
+
+RUN apk add --no-cache inotify-tools
+
+COPY monitor.sh /monitor.sh
+RUN chmod +x /monitor.sh
+
+CMD ["/monitor.sh"]

devnet/tx-verifier-monitor/Dockerfile.cast

@@ -0,0 +1,13 @@
+# These versions are pinned to match the Dockerfile in the `ethereum/`
+# directory. Otherwise, there is nothing special about them and they can be
+# updated alongside the other Dockerfile.
+FROM ghcr.io/foundry-rs/foundry:nightly-55bf41564f605cae3ca4c95ac5d468b1f14447f9@sha256:8c15d322da81a6deaf827222e173f3f81c653136a3518d5eeb41250a0f2e17ea as foundry
+# node is required to install Foundry
+FROM node:19.6.1-slim@sha256:a1ba21bf0c92931d02a8416f0a54daad66cb36a85d2b73af9d73b044f5f57cfc
+
+COPY --from=foundry /usr/local/bin/cast /bin/cast
+
+COPY transfer-verifier-test.sh /transfer-verifier-test.sh
+RUN chmod +x /transfer-verifier-test.sh
+
+CMD ["/transfer-verifier-test.sh"]

devnet/tx-verifier-monitor/monitor.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+log_file="${ERROR_LOG_PATH:-/logs/error.log}"
+error_pattern="${ERROR_PATTERN:-ERROR}"
+status_file="/logs/status"
+
+# Wait for log file to exist and be non-empty
+while [ ! -s "${log_file}" ]; do
+    echo "Waiting for ${log_file} to be created and contain data..."
+    sleep 5
+done
+
+# Initialize status
+echo "RUNNING" > "$status_file"
+echo "Monitoring file '${log_file}' for error pattern: '${error_pattern}'"
+
+# Watch for changes in the log file. If we find the error pattern that means we have
+# succeeded. (Transfer verifier should correctly detect errors.
+inotifywait -m -e modify "${log_file}" | while read -r directory events filename; do
+    if grep -q "$error_pattern" "$log_file"; then
+        echo "SUCCESS" > "$status_file"
+        echo "Found error pattern. Exiting."
+        exit 0
+    fi
+done

devnet/tx-verifier-monitor/transfer-verifier-test.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RPC="${RPC_URL:-ws://eth-devnet:8545}"
+
+# mainnet values
+# export CORE_CONTRACT="0x98f3c9e6E3fAce36bAAd05FE09d375Ef1464288B"
+# export TOKEN_BRIDGE_CONTRACT="0x3ee18B2214AFF97000D974cf647E7C347E8fa585"
+
+# TODO these could be CLI params from the sh/devnet script
+CORE_BRIDGE_CONTRACT=0xC89Ce4735882C9F0f0FE26686c53074E09B0D550
+TOKEN_BRIDGE_CONTRACT=0x0290FB167208Af455bB137780163b7B7a9a10C16
+
+MNEMONIC=0x4f3edf983ac636a65a842ce7c78d9aa706d3b113bce9c46f30d7d21715b23b1d
+
+ERC20_ADDR="0x47bdB2D7d6528C760b6f228b3B8F9F650169a10f" # Test token A
+
+VALUE="1000" # Wei value sent as msg.value
+TRANSFER_AMOUNT="10"
+
+ANVIL_USER0="0x90F8bf6A479f320ead074411a4B0e7944Ea8c9C1" # Account0 reported by anvil when run using $MNEMONIC
+ANVIL_USER1="0xFFcf8FDEE72ac11b5c542428B35EEF5769C409f0" 
+ETH_WHALE="${ANVIL_USER0}"
+FROM="${ETH_WHALE}"
+RECIPIENT="0x00000000000000000000000090F8bf6A479f320ead074411a4B0e7944Ea8c9C1" # Anvil user0 normalized to Wormhole size. Doesn't matter what the value is
+NONCE="234" # arbitrary
+
+# Build the payload for token transfers. Declared on multiple lines to
+# be more legible. Data pulled from an arbitrary LogMessagePublished event
+# on etherscan. Metadata and fees commented out, leaving only the payload
+PAYLOAD="0x"
+declare -a SLOTS=(
+   # "0000000000000000000000000000000000000000000000000000000000055baf"
+   # "0000000000000000000000000000000000000000000000000000000000000000"
+   # "0000000000000000000000000000000000000000000000000000000000000080"
+   # "0000000000000000000000000000000000000000000000000000000000000001"
+   # "00000000000000000000000000000000000000000000000000000000000000ae"
+   "030000000000000000000000000000000000000000000000000000000005f5e1"
+   "000000000000000000000000002260fac5e5542a773aa44fbcfedf7c193bc2c5"
+   "9900020000000000000000000000000000000000000000000000000000000000"
+   "000816001000000000000000000000000044eca3f6295d6d559ca1d99a5ef5a8"
+   "f72b4160f10001010200c91f01004554480044eca3f6295d6d559ca1d99a5ef5"
+   "a8f72b4160f10000000000000000000000000000000000000000000000000000"
+)
+for i in "${SLOTS[@]}"
+do
+   PAYLOAD="$PAYLOAD$i"
+done
+
+echo "DEBUG:"
+echo "- RPC=${RPC}"
+echo "- CORE_BRIDGE_CONTRACT=${CORE_BRIDGE_CONTRACT}"
+echo "- TOKEN_BRIDGE_CONTRACT=${TOKEN_BRIDGE_CONTRACT}"
+echo "- MNEMONIC=${MNEMONIC}"
+echo "- FROM=${FROM}"
+echo "- VALUE=${VALUE}" 
+echo "- RECIPIENT=${RECIPIENT}" 
+echo 
+
+# Fund the token bridge from User0
+echo "Start impersonating User0"
+cast rpc \
+   anvil_impersonateAccount "${ANVIL_USER0}" \
+   --rpc-url "${RPC}"
+echo "Funding token bridge using user0's balance"
+cast send --unlocked \
+   --rpc-url "${RPC}" \
+   --from $ANVIL_USER0 \
+   --value 100000000000000 \
+   ${TOKEN_BRIDGE_CONTRACT}
+echo ""
+echo "End impersonating User0"
+cast rpc \
+   anvil_stopImpersonatingAccount "${ANVIL_USER0}" \
+   --rpc-url "${RPC}"
+
+BALANCE_CORE=$(cast balance --rpc-url "${RPC}" $CORE_BRIDGE_CONTRACT)
+BALANCE_TOKEN=$(cast balance --rpc-url "${RPC}" $TOKEN_BRIDGE_CONTRACT)
+BALANCE_USER0=$(cast balance --rpc-url "${RPC}" $ANVIL_USER0)
+echo "BALANCES:"
+echo "- CORE_BRIDGE_CONTRACT=${BALANCE_CORE}"
+echo "- TOKEN_BRIDGE_CONTRACT=${BALANCE_TOKEN}"
+echo "- ANVIL_USER0=${BALANCE_USER0}"
+echo 
+
+# === Malicious call to transferTokensWithPayload()
+# This is the exploit scenario: the token bridge has called publishMessage() without a ERC20 Transfer or Deposit
+# being present in the same receipt.
+# This is done by impersonating the token bridge contract and sending a message directly to the core bridge.
+# Ensure that anvil is using `--auto-impersonate` or else that account impersonation is enabled in your local environment.
+# --private-key "$MNEMONIC" \
+# --max-fee 500000 \
+echo "Start impersonate token bridge" 
+cast rpc \
+   --rpc-url "${RPC}" \
+   anvil_impersonateAccount "${TOKEN_BRIDGE_CONTRACT}"
+echo "Calling publishMessage as ${TOKEN_BRIDGE_CONTRACT}" 
+cast send --unlocked \
+   --rpc-url "${RPC}" \
+   --json \
+   --gas-limit 10000000 \
+   --priority-gas-price 1 \
+   --from "${TOKEN_BRIDGE_CONTRACT}" \
+   --value "0" \
+   "${CORE_BRIDGE_CONTRACT}" \
+   "publishMessage(uint32,bytes,uint8)" \
+   0 "${PAYLOAD}" 1
+echo ""
+cast rpc \
+   --rpc-url "${RPC}" \
+   anvil_stopImpersonatingAccount "${TOKEN_BRIDGE_CONTRACT}"
+echo "End impersonate token bridge" 
+
+# TODO add the 'multicall' scenario encoded in the forge script
+
+echo "Done Transfer Verifier integration test."
+echo "Exiting."

devnet/tx-verifier-test.yaml

@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tx-verifier-test
+spec:
+  # Number of successful pod completions needed
+  completions: 1
+  # Number of pods to run in parallel
+  parallelism: 1
+  # Time limit after which the job is terminated (optional)
+  # activeDeadlineSeconds: 100
+  # Number of retries before marking as failed
+  backoffLimit: 4
+  template:
+    metadata:
+      labels:
+        app: tx-verifier-test
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: tx-verifier-test
+          image: tx-verifier-test
+          command:
+            - /bin/bash
+            - -c
+            - "/transfer-verifier-test.sh"
+          env:
+            - name: RPC_URL
+              value: "ws://eth-devnet:8545"
+      volumes:
+      - name: log-volume
+        emptyDir: {}

devnet/tx-verifier.yaml

@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tx-verifier-with-monitor
+spec:
+  selector:
+    matchLabels:
+      app: tx-verifier-with-monitor
+  template:
+    metadata:
+      labels:
+        app: tx-verifier-with-monitor
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: tx-verifier
+          image: guardiand-image
+          volumeMounts:
+           - name: log-volume
+             mountPath: /logs
+          command:
+            ["/bin/sh", "-c"]
+          # See `ethereum/.env.test` and related shell scripts for how these values are configured in localnet testing.
+          args:
+            - |
+              exec /guardiand \
+                           transfer-verifier \
+                           evm \
+                           --rpcUrl ws://eth-devnet:8545 \
+                           --coreContract 0xC89Ce4735882C9F0f0FE26686c53074E09B0D550 \
+                           --tokenContract 0x0290FB167208Af455bB137780163b7B7a9a10C16 \
+                           --wrappedNativeContract 0xDDb64fE46a91D46ee29420539FC25FD07c5FEa3E \
+                           --logLevel=info \
+                           2> /logs/error.log \
+        - name: tx-verifier-monitor
+          image: tx-verifier-monitor
+          volumeMounts:
+          - name: log-volume
+            mountPath: /logs
+          env:
+            - name: ERROR_PATTERN
+              # This error string comes from the transfer-verifier binary in node/
+              value: "invalid receipt: no deposits and no transfers"
+            - name: ERROR_LOG_PATH
+              value: "/logs/error.log"
+      volumes:
+      - name: log-volume
+        emptyDir: {}