tmp: testing gateway conformance

.env-gwc

@@ -1,10 +1,12 @@
 # ENV vars recommended for running gateway-conformance tests
-USE_LIBP2P=false
+USE_LIBP2P=true
 USE_BITSWAP=true
-PORT=8090 # helia-http-gateway should be running here
-TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here
-DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here
-DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*'
+PORT="8080" # helia-http-gateway should be running here
+KUBO_PORT="8081" # Kubo should be running here
+TRUSTLESS_GATEWAYS="http://127.0.0.1:8081" # Kubo should be running here
+DELEGATED_ROUTING_V1_HOST="http://127.0.0.1:8081" # Kubo should be running here
+# DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*'
+DEBUG='helia*,helia*:trace'
 USE_TRUSTLESS_GATEWAYS=true
 USE_DELEGATED_ROUTING=true
 
@@ -13,4 +15,15 @@ USE_DELEGATED_ROUTING=true
 # FILE_BLOCKSTORE_PATH=./data/blockstore
 
 # Uncomment the below to see request & response headers in the logs
-# ECHO_HEADERS=true
+GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0
+
+# things pass when we skip all these.
+GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$"
+
+# re-enable TestPathing
+GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$"
+GWC_GATEWAY_URL="http://helia-http-gateway.localhost"
+# GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost"
+# GWC_GATEWAY_URL="http://127.0.0.1:8080"
+GWC_GATEWAY_URL="http://host.docker.internal:8080"
+GWC_SUBDOMAIN_URL="http://host.docker.internal:8080"

.github/workflows/gateway-conformance.yml

@@ -101,7 +101,8 @@ jobs:
           #
           # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific)
           # for now disabling these test cases
-          args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'
+          args: -skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'
+          # args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'
 
       # 7. Upload the results
       - name: Upload MD summary

package.json

@@ -63,10 +63,10 @@
     "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"",
     "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"",
     "test:gwc-kubo": "node ./scripts/kubo-init.js",
-    "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc",
-    "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"",
-    "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url='http://helia-http-gateway.localhost' --subdomain-url='http://helia-http-gateway.localhost' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m",
-    "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"",
+    "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace",
+    "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"",
+    "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json $GWC_SPECS -- -test.skip=\"$GWC_SKIP\" -timeout 30m",
+    "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"",
     "healthcheck": "node dist/src/healthcheck.js",
     "debug:until-death": "./debugging/until-death.sh",
     "debug:test-gateways": "./debugging/test-gateways.sh",

scripts/kubo-init.js

@@ -15,6 +15,7 @@ debug.enable('kubo-init*')
 
 const kuboFilePath = './scripts/tmp/kubo-path.txt'
 const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures`
+const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE ?? 'ghcr.io/ipfs/gateway-conformance:v0.5.0'
 
 async function main () {
   await $`mkdir -p ${dirname(kuboFilePath)}`
@@ -54,7 +55,7 @@ function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) {
 async function attemptKuboInit (tmpDir) {
   const execaOptions = getExecaOptions({ tmpDir })
   try {
-    await $(execaOptions)`npx -y kubo init`
+    await $(execaOptions)`npx -y kubo init --profile test`
     log('Kubo initialized at %s', tmpDir)
   } catch (e) {
     if (!e.stderr.includes('already exists!')) {
@@ -86,8 +87,14 @@ async function writeKuboMetaData () {
 async function configureKubo (tmpDir) {
   const execaOptions = getExecaOptions({ tmpDir })
   try {
-    await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080`
+    await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}`
+    await $(execaOptions)`npx -y kubo config --json Bootstrap ${JSON.stringify([])}`
+    await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true`
+    await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false`
+    await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true`
     await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true`
+    await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin ${JSON.stringify(['*'])}`
+    await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods ${JSON.stringify(['GET', 'POST', 'PUT', 'OPTIONS'])}`
     log('Kubo configured')
   } catch (e) {
     error('Failed to configure Kubo', e)
@@ -97,7 +104,7 @@ async function configureKubo (tmpDir) {
 async function downloadFixtures () {
   log('Downloading fixtures')
   try {
-    await $`docker run -v ${process.cwd()}:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false`
+    await $`docker run -v ${process.cwd()}:/workspace -w /workspace ${GWC_DOCKER_IMAGE} extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false`
   } catch (e) {
     error('Error downloading fixtures, assuming current or previous success', e)
   }

src/helia-server.ts

@@ -58,17 +58,6 @@ export class HeliaServer {
 
     this.log('heliaServer Started!')
     this.routes = [
-      {
-        // without this non-wildcard postfixed path, the '/*' route will match first.
-        path: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid
-        type: 'GET',
-        handler: async (request, reply): Promise<void> => this.handleEntry({ request, reply })
-      },
-      {
-        path: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath
-        type: 'GET',
-        handler: async (request, reply): Promise<void> => this.handleEntry({ request, reply })
-      },
       {
         path: '/api/v0/version',
         type: 'POST',
@@ -106,11 +95,12 @@ export class HeliaServer {
         path: '/*',
         type: 'GET',
         handler: async (request, reply): Promise<void> => {
-          try {
-            await this.fetch({ request, reply })
-          } catch {
-            await reply.code(200).send('try /ipfs/<cid> or /ipns/<name>')
-          }
+          // try {
+          await this.fetch({ request, reply })
+          // } catch (err) {
+          //   this.log.error('error fetching:', err)
+          //   await reply.code(500).send('try /ipfs/<cid> or /ipns/<name>')
+          // }
         }
       },
       {
@@ -130,25 +120,22 @@ export class HeliaServer {
     ]
   }
 
-  /**
-   * Redirects to the subdomain gateway.
-   */
-  private async handleEntry ({ request, reply }: RouteHandler): Promise<void> {
+  #redirectUrl ({ request, reply }: RouteHandler): string | null {
     const { params } = request
     this.log('fetch request %s', request.url)
     const { ns: namespace, '*': relativePath, address } = params as EntryParams
 
     this.log('handling entry: ', { address, namespace, relativePath })
     if (!USE_SUBDOMAINS) {
       this.log('subdomains are disabled, fetching without subdomain')
-      return this.fetch({ request, reply })
+      return null
     } else {
       this.log('subdomains are enabled, redirecting to subdomain')
     }
 
     const { peerId, cid } = getIpnsAddressDetails(address)
     if (peerId != null) {
-      return this.fetch({ request, reply })
+      return null
     }
     const cidv1Address = cid?.toString()
 
@@ -169,35 +156,17 @@ export class HeliaServer {
       // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`)
     }
     let encodedDnsLink = address
-    if (!isInlinedDnsLink(address)) {
+    if (address != null && !isInlinedDnsLink(address)) {
       encodedDnsLink = dnsLinkLabelEncoder(address)
     }
 
-    const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}`
-    this.log('redirecting to final URL:', finalUrl)
-    await reply
-      .headers({
-        Location: finalUrl
-      })
-      .code(301)
-      .send()
+    return `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}`
   }
 
   #getFullUrlFromFastifyRequest (request: FastifyRequest): string {
-    let query = ''
-    if (request.query != null) {
-      this.log('request.query:', request.query)
-      const pairs: string[] = []
-      Object.keys(request.query).forEach((key: string) => {
-        const value = (request.query as Record<string, string>)[key]
-        pairs.push(`${key}=${value}`)
-      })
-      if (pairs.length > 0) {
-        query += '?' + pairs.join('&')
-      }
-    }
+    // FYI: request.url includes the query string
 
-    return `${request.protocol}://${request.hostname}${request.url}${query}`
+    return `${request.protocol}://${request.hostname}${request.url}`
   }
 
   #convertVerifiedFetchResponseToFastifyReply = async (verifiedFetchResponse: Response, reply: FastifyReply): Promise<void> => {
@@ -223,6 +192,12 @@ export class HeliaServer {
     for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) {
       headers[headerName] = headerValue
     }
+
+    const redirectUrl = this.#redirectUrl({ request: reply.request, reply })
+    if (redirectUrl != null) {
+      headers.Location = redirectUrl
+    }
+
     // Fastify really does not like streams despite what the documentation and github issues say.
     const reader = verifiedFetchResponse.body.getReader()
     reply.raw.writeHead(verifiedFetchResponse.status, headers)
@@ -285,7 +260,18 @@ export class HeliaServer {
     const signal = this.#getRequestAwareSignal(request, url)
 
     await this.isReady
-    const resp = await this.heliaFetch(url, { signal, redirect: 'manual' })
+    // pass headers from the original request (IncomingHttpHeaders) to HeadersInit
+    const headers: Record<string, string> = {}
+    for (const [headerName, headerValue] of Object.entries(request.headers)) {
+      if (headerValue != null) {
+        if (typeof headerValue === 'string') {
+          headers[headerName] = headerValue
+        } else {
+          headers[headerName] = headerValue.join(',')
+        }
+      }
+    }
+    const resp = await this.heliaFetch(url, { signal, redirect: 'manual', headers })
     await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply)
   }