Upgrade dependencies and update referential integrity checks (#17)

* Upgrade dependencies and update referential integrity queries

* Update GitHub workflow to run tests

.github/workflows/tests.yml

@@ -3,9 +3,9 @@ on:
   workflow_call:
 
 env:
-  STACK_ORCHESTRATOR_REF: "b3cb26e93b7e387d96417c81f880a3b8699b67db"
-  GO_ETHEREUM_REF: "v1.10.19-statediff-4.0.2-alpha" # Use the tag, we are going to download the bin not build it.
-  IPLD_ETH_DB_REF: "4e948c58ce20c20ab633289f986d2ed2a1fe02ec"
+  STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702"
+  GO_ETHEREUM_REF: "v1.10.19-statediff-4.1.0-alpha" # Use the tag, we are going to download the bin not build it.
+  IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9"
 
 jobs:
   integrationtest:
@@ -17,33 +17,41 @@ jobs:
     steps:
       - name: Create GOPATH
         run: mkdir -p /tmp/go
+
       - uses: actions/setup-go@v3
         with:
           go-version: ">=1.18.0"
           check-latest: true
+
       - uses: actions/checkout@v2
         with:
           path: "./ipld-eth-db-validator"
+
       - uses: actions/checkout@v2
         with:
           ref: ${{ env.STACK_ORCHESTRATOR_REF }}
           path: "./stack-orchestrator/"
           repository: vulcanize/stack-orchestrator
+
       - uses: actions/checkout@v2
         with:
           ref: ${{ env.IPLD_ETH_DB_REF }}
           repository: vulcanize/ipld-eth-db
           path: "./ipld-eth-db/"
+
       - name: Create config file
         run: |
           echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
           echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
+          echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
           echo db_write=$DB_WRITE >> ./config.sh
           cat ./config.sh
-      - name: Download Geth geth
+
+      - name: Download Geth
         run: |
           cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
           wget https://github.com/vulcanize/go-ethereum/releases/download/${{env.GO_ETHEREUM_REF}}/geth-linux-amd64
+
       - name: Run docker compose
         run: |
           docker-compose  \
@@ -52,49 +60,32 @@ jobs:
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
           --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
+
       - name: Run integration test.
         run: |
           cd $GITHUB_WORKSPACE/ipld-eth-db-validator
           ./scripts/run_integration_test.sh
 
   unittest:
     name: Run unit tests
-    env:
-      GOPATH: /tmp/go
-      DB_WRITE: true
     runs-on: ubuntu-latest
     steps:
       - name: Create GOPATH
         run: mkdir -p /tmp/go
+
       - uses: actions/setup-go@v3
         with:
           go-version: ">=1.18.0"
           check-latest: true
-      - uses: actions/checkout@v2
-        with:
-          path: "./ipld-eth-db-validator"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
-          path: "./stack-orchestrator/"
-          repository: vulcanize/stack-orchestrator
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ env.IPLD_ETH_DB_REF }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
-          echo db_write=$DB_WRITE >> ./config.sh
-          cat ./config.sh
-      - name: Run docker compose
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Spin up database
         run: |
-          docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
-          --env-file "$GITHUB_WORKSPACE/config.sh" \
-          up -d --build
-      - name: Run unit test.
+          docker-compose up -d
+
+      - name: Run unit tests
         run: |
-          cd $GITHUB_WORKSPACE/ipld-eth-db-validator
+          sleep 30
           PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

.gitignore

@@ -0,0 +1,2 @@
+ipld-eth-db-validator
+.vscode

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+    image: vulcanize/ipld-eth-db:v4.2.0-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "vulcanize_testing"

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
 	github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha
-	github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha
+	github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha
 )
 
 require (
@@ -275,4 +275,4 @@ require (
 	lukechampine.com/blake3 v1.1.6 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha
+replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha

go.sum

@@ -1562,12 +1562,12 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha h1:sDDK4eOdW3JEds+0eP/OGgPr7T7SS4jtje5W4VjhCgM=
 github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha/go.mod h1:/pHfZd1IWsSTpCtGq6nnzUZBAkLV+zMrRh6Z3Hr3NFc=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha h1:xD4fA2khoAnhBEk84JwrIEGvQCndVXpQGv5n7a9cgwc=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha h1:8ge2ban6t/e53XDwe6s28jcCevT7Ggo51lNJ0Eo1PgA=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha h1:xak1uYmFWqJ2Hz3pM+0jDcqdlwYwRWeSkQV6B8IxD/0=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha/go.mod h1:pHbLbW4Hk1IFpxrY9yi50IuoPPzmSY7lwOqpFAa369k=
-github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha h1:yNpTBjP71ZaUFz2Q8XFUbupvi1yGc4rtniBCTWLtmLk=
-github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha/go.mod h1:rYP+9ejznXwuN8eKeoFGdzi7wjzm2MOaWcI1XObPdBo=
+github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha h1:UYPRvG9fKgUTLQOMgTMxxPPHgST5IpLRI4FT9goAHXY=
+github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha/go.mod h1:HbXa8Zz4g0TIS8vEq6XdK9J1Wc2x4Oa1+YaHwzmLMCQ=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
 github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA=
 github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=

pkg/validator/ref_integrity_queries.go

@@ -48,6 +48,7 @@ const (
 						FROM eth.receipt_cids
 						LEFT JOIN eth.transaction_cids ON (
 							receipt_cids.tx_id = transaction_cids.tx_hash
+							AND receipt_cids.header_id = transaction_cids.header_id
 							AND receipt_cids.block_number = transaction_cids.block_number
 						)
 						WHERE
@@ -110,6 +111,7 @@ const (
 						FROM eth.log_cids
 						LEFT JOIN eth.receipt_cids ON (
 							log_cids.rct_id = receipt_cids.tx_id
+							AND log_cids.header_id = receipt_cids.header_id
 							AND log_cids.block_number = receipt_cids.block_number
 						)
 						WHERE

test/README.md

@@ -4,20 +4,27 @@
 
 - For running integration tests:
 
-  - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories.
+  - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator), [go-ethereum](https://github.com/vulcanize/go-ethereum) and [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db) repositories.
 
-  - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) in go-ethereum repo.
+  - Checkout [v4 release](https://github.com/vulcanize/ipld-eth-db/releases/tag/v4.2.0-alpha) in ipld-eth-db repo.
+
+    ```bash
+    # In ipld-eth-db repo.
+    git checkout v4.2.0-alpha
+    ```
+
+  - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.19-statediff-4.1.0-alpha) in go-ethereum repo.
 
     ```bash
     # In go-ethereum repo.
-    git checkout v1.10.18-statediff-4.0.2-alpha
+    git checkout v1.10.19-statediff-4.1.0-alpha
     ```
 
   - Checkout working commit in stack-orchestrator repo.
 
     ```bash
     # In stack-orchestrator repo.
-    git checkout 382aca8e42bc5e33f301f77cdd2e09cc80602fc3
+    git checkout f2fd766f5400fcb9eb47b50675d2e3b1f2753702
     ```
 
 ## Run
@@ -48,12 +55,16 @@
       ```bash
       #!/bin/bash
 
+      # Path to ipld-eth-server repo.
+      vulcanize_ipld_eth_db=~/ipld-eth-db/
+
       # Path to go-ethereum repo.
       vulcanize_go_ethereum=~/go-ethereum
 
       # Path to contract folder.
       vulcanize_test_contract=~/ipld-eth-db-validator/test/contract
 
+      genesis_file_path='start-up-files/go-ethereum/genesis.json'
       db_write=true
       ```
 
@@ -65,7 +76,7 @@
 
       ./wrapper.sh \
       -e docker \
-      -d ../docker/latest/docker-compose-db-sharding.yml \
+      -d ../docker/local/docker-compose-db-sharding.yml \
       -d ../docker/local/docker-compose-go-ethereum.yml \
       -d ../docker/local/docker-compose-contract.yml \
       -v remove \