From eab694ddf186b5fbd3066f486b03b6399b42c0d2 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:50:06 -0700 Subject: [PATCH 1/9] tmp: testing --- .env-gwc | 1 + package.json | 2 +- scripts/kubo-init.js | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.env-gwc b/.env-gwc index f9a2c6b..758e1fe 100644 --- a/.env-gwc +++ b/.env-gwc @@ -14,3 +14,4 @@ USE_DELEGATED_ROUTING=true # Uncomment the below to see request & response headers in the logs # ECHO_HEADERS=true +GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 diff --git a/package.json b/package.json index 9cbd65e..fec0055 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "test:gwc-kubo": "node ./scripts/kubo-init.js", "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc", "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url='http://helia-http-gateway.localhost' --subdomain-url='http://helia-http-gateway.localhost' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url='http://localhost:8090' --subdomain-url='http://localhost:8090' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index 8912fa8..06f52a0 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -15,6 +15,7 @@ debug.enable('kubo-init*') const kuboFilePath = './scripts/tmp/kubo-path.txt' const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` +const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE || 'ghcr.io/ipfs/gateway-conformance:v0.5.0' async function main () { await $`mkdir -p ${dirname(kuboFilePath)}` @@ -97,7 +98,7 @@ async function configureKubo (tmpDir) { async function downloadFixtures () { log('Downloading fixtures') try { - await $`docker run -v ${process.cwd()}:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` + await $`docker run -v ${process.cwd()}:/workspace -w /workspace ${GWC_DOCKER_IMAGE} extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` } catch (e) { error('Error downloading fixtures, assuming current or previous success', e) } From 2b1eb7270f5df65ad49f97d032a99a70e9a10322 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:54:34 -0700 Subject: [PATCH 2/9] chore: run some tests in CI --- .github/workflows/gateway-conformance.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 615dfe9..4c2ba98 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -101,7 +101,8 @@ jobs: # # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + args: -skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + # args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' # 7. Upload the results - name: Upload MD summary From f5f566d8e87b43c03d1d5c98894831436f9522c1 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:58:59 -0700 Subject: [PATCH 3/9] fix: pass headers from fastify to verifiedFetch --- src/heliaServer.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/heliaServer.ts b/src/heliaServer.ts index 333e7af..0867fd6 100644 --- a/src/heliaServer.ts +++ b/src/heliaServer.ts @@ -237,7 +237,18 @@ export class HeliaServer { } }) await this.isReady - const resp = await this.heliaFetch(url, { signal: opController.signal, redirect: 'manual' }) + // pass headers from the original request (IncomingHttpHeaders) to HeadersInit + const headers: Record = {} + for (const [headerName, headerValue] of Object.entries(request.headers)) { + if (headerValue != null) { + if (typeof headerValue === 'string') { + headers[headerName] = headerValue + } else { + headers[headerName] = headerValue.join(',') + } + } + } + const resp = await this.heliaFetch(url, { signal: opController.signal, redirect: 'manual', headers }) await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply) } From bbb2a9fb72b5feb588f60e0155b939c4638b9686 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:10:00 -0700 Subject: [PATCH 4/9] chore: allow env var to change specs tested --- .env-gwc | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env-gwc b/.env-gwc index 758e1fe..99f9bd8 100644 --- a/.env-gwc +++ b/.env-gwc @@ -15,3 +15,5 @@ USE_DELEGATED_ROUTING=true # Uncomment the below to see request & response headers in the logs # ECHO_HEADERS=true GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 +#GWC_SPECS="subdomain-ipns-gateway,subdomain-ipfs-gateway" +GWC_SPECS="subdomain-ipfs-gateway" diff --git a/package.json b/package.json index fec0055..0db8c4f 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "test:gwc-kubo": "node ./scripts/kubo-init.js", "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc", "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url='http://localhost:8090' --subdomain-url='http://localhost:8090' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url='http://localhost:8090' --subdomain-url='http://localhost:8090' --verbose --json gwc-report.json --specs ${GWC_SPECS} -- -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", From 2b7fd1735cfc35887f9f09ad95058b6bdc904a3d Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:39:57 -0700 Subject: [PATCH 5/9] fix: actually getting responses from http-gateway now --- .env-gwc | 19 +++++++++++++------ package.json | 8 ++++---- scripts/kubo-init.js | 10 ++++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/.env-gwc b/.env-gwc index 99f9bd8..cf7552e 100644 --- a/.env-gwc +++ b/.env-gwc @@ -1,10 +1,12 @@ # ENV vars recommended for running gateway-conformance tests -USE_LIBP2P=false +USE_LIBP2P=true USE_BITSWAP=true -PORT=8090 # helia-http-gateway should be running here -TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here -DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here -DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +PORT="8080" # helia-http-gateway should be running here +KUBO_PORT="8081" # Kubo should be running here +TRUSTLESS_GATEWAYS="http://127.0.0.1:8081" # Kubo should be running here +DELEGATED_ROUTING_V1_HOST="http://127.0.0.1:8081" # Kubo should be running here +# DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +DEBUG='helia*,helia*:trace' USE_TRUSTLESS_GATEWAYS=true USE_DELEGATED_ROUTING=true @@ -16,4 +18,9 @@ USE_DELEGATED_ROUTING=true # ECHO_HEADERS=true GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 #GWC_SPECS="subdomain-ipns-gateway,subdomain-ipfs-gateway" -GWC_SPECS="subdomain-ipfs-gateway" +# GWC_SPECS="--specs subdomain-ipfs-gateway" +GWC_SKIP="-skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'" +# GWC_GATEWAY_URL="http://helia-http-gateway.localhost" +# GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" +GWC_GATEWAY_URL="http://host.docker.internal:8080" +GWC_SUBDOMAIN_URL="http://host.docker.internal:8080" diff --git a/package.json b/package.json index 277c475..0b240dc 100644 --- a/package.json +++ b/package.json @@ -63,10 +63,10 @@ "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", - "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc", - "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url='http://localhost:8090' --subdomain-url='http://localhost:8090' --verbose --json gwc-report.json --specs ${GWC_SPECS} -- -timeout 30m", - "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", + "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace", + "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json ${GWC_SPECS} -- ${GWC_SKIP} -timeout 30m", + "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", "debug:test-gateways": "./debugging/test-gateways.sh", diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index 06f52a0..f88e07c 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -15,7 +15,7 @@ debug.enable('kubo-init*') const kuboFilePath = './scripts/tmp/kubo-path.txt' const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` -const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE || 'ghcr.io/ipfs/gateway-conformance:v0.5.0' +const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE ?? 'ghcr.io/ipfs/gateway-conformance:v0.5.0' async function main () { await $`mkdir -p ${dirname(kuboFilePath)}` @@ -87,8 +87,14 @@ async function writeKuboMetaData () { async function configureKubo (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080` + await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}` + await $(execaOptions)`npx -y kubo config --json Bootstrap '[]'` + await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true` + await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false` + await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true` await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin '["*"]'` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods '["GET", "POST", "PUT", "OPTIONS"]'` log('Kubo configured') } catch (e) { error('Failed to configure Kubo', e) From 6fa4ff78205e3fd5ef16810fe1b641eaaa5bb7ae Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:51:31 -0700 Subject: [PATCH 6/9] fix: gateway conformance expects 301s to include content --- .env-gwc | 2 +- src/helia-server.ts | 35 ++++++++++------------------------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/.env-gwc b/.env-gwc index cf7552e..ea7996d 100644 --- a/.env-gwc +++ b/.env-gwc @@ -19,7 +19,7 @@ USE_DELEGATED_ROUTING=true GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 #GWC_SPECS="subdomain-ipns-gateway,subdomain-ipfs-gateway" # GWC_SPECS="--specs subdomain-ipfs-gateway" -GWC_SKIP="-skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'" +# GWC_SKIP="-skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'" # GWC_GATEWAY_URL="http://helia-http-gateway.localhost" # GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" GWC_GATEWAY_URL="http://host.docker.internal:8080" diff --git a/src/helia-server.ts b/src/helia-server.ts index 75161d2..deb045c 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -58,17 +58,6 @@ export class HeliaServer { this.log('heliaServer Started!') this.routes = [ - { - // without this non-wildcard postfixed path, the '/*' route will match first. - path: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, - { - path: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, { path: '/api/v0/version', type: 'POST', @@ -130,10 +119,7 @@ export class HeliaServer { ] } - /** - * Redirects to the subdomain gateway. - */ - private async handleEntry ({ request, reply }: RouteHandler): Promise { + #redirectUrl ({ request, reply }: RouteHandler): string | null { const { params } = request this.log('fetch request %s', request.url) const { ns: namespace, '*': relativePath, address } = params as EntryParams @@ -141,14 +127,14 @@ export class HeliaServer { this.log('handling entry: ', { address, namespace, relativePath }) if (!USE_SUBDOMAINS) { this.log('subdomains are disabled, fetching without subdomain') - return this.fetch({ request, reply }) + return null } else { this.log('subdomains are enabled, redirecting to subdomain') } const { peerId, cid } = getIpnsAddressDetails(address) if (peerId != null) { - return this.fetch({ request, reply }) + return null } const cidv1Address = cid?.toString() @@ -173,14 +159,7 @@ export class HeliaServer { encodedDnsLink = dnsLinkLabelEncoder(address) } - const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` - this.log('redirecting to final URL:', finalUrl) - await reply - .headers({ - Location: finalUrl - }) - .code(301) - .send() + return `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` } #getFullUrlFromFastifyRequest (request: FastifyRequest): string { @@ -223,6 +202,12 @@ export class HeliaServer { for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { headers[headerName] = headerValue } + + const redirectUrl = this.#redirectUrl({ request: reply.request, reply }) + if (redirectUrl != null) { + headers.Location = redirectUrl + } + // Fastify really does not like streams despite what the documentation and github issues say. const reader = verifiedFetchResponse.body.getReader() reply.raw.writeHead(verifiedFetchResponse.status, headers) From dbb42a8211716385608c9a802069705980a5835a Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 17:01:08 -0700 Subject: [PATCH 7/9] fix: fastify request.url includes query string --- src/helia-server.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/helia-server.ts b/src/helia-server.ts index deb045c..c7b8651 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -163,20 +163,9 @@ export class HeliaServer { } #getFullUrlFromFastifyRequest (request: FastifyRequest): string { - let query = '' - if (request.query != null) { - this.log('request.query:', request.query) - const pairs: string[] = [] - Object.keys(request.query).forEach((key: string) => { - const value = (request.query as Record)[key] - pairs.push(`${key}=${value}`) - }) - if (pairs.length > 0) { - query += '?' + pairs.join('&') - } - } + // FYI: request.url includes the query string - return `${request.protocol}://${request.hostname}${request.url}${query}` + return `${request.protocol}://${request.hostname}${request.url}` } #convertVerifiedFetchResponseToFastifyReply = async (verifiedFetchResponse: Response, reply: FastifyReply): Promise => { From ed79f713542a70c58797b64cb0a35b0bdf8af0d8 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 17:21:01 -0700 Subject: [PATCH 8/9] fix: issue calling isInlinedDnsLink when empty --- src/helia-server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/helia-server.ts b/src/helia-server.ts index c7b8651..1f8c519 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -97,7 +97,8 @@ export class HeliaServer { handler: async (request, reply): Promise => { try { await this.fetch({ request, reply }) - } catch { + } catch (err) { + this.log.error('error fetching:', err) await reply.code(200).send('try /ipfs/ or /ipns/') } } @@ -155,7 +156,7 @@ export class HeliaServer { // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`) } let encodedDnsLink = address - if (!isInlinedDnsLink(address)) { + if (address != null && !isInlinedDnsLink(address)) { encodedDnsLink = dnsLinkLabelEncoder(address) } From 0764f0b228ba4c6fbddda506a318f011c79ce95b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 21:22:13 -0700 Subject: [PATCH 9/9] chore: making progress on TestPathing conformance --- .env-gwc | 13 ++++++++----- package.json | 2 +- scripts/kubo-init.js | 8 ++++---- src/helia-server.ts | 12 ++++++------ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.env-gwc b/.env-gwc index ea7996d..4418eed 100644 --- a/.env-gwc +++ b/.env-gwc @@ -15,12 +15,15 @@ USE_DELEGATED_ROUTING=true # FILE_BLOCKSTORE_PATH=./data/blockstore # Uncomment the below to see request & response headers in the logs -# ECHO_HEADERS=true GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 -#GWC_SPECS="subdomain-ipns-gateway,subdomain-ipfs-gateway" -# GWC_SPECS="--specs subdomain-ipfs-gateway" -# GWC_SKIP="-skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'" -# GWC_GATEWAY_URL="http://helia-http-gateway.localhost" + +# things pass when we skip all these. +GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" + +# re-enable TestPathing +GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" +GWC_GATEWAY_URL="http://helia-http-gateway.localhost" # GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" +# GWC_GATEWAY_URL="http://127.0.0.1:8080" GWC_GATEWAY_URL="http://host.docker.internal:8080" GWC_SUBDOMAIN_URL="http://host.docker.internal:8080" diff --git a/package.json b/package.json index 0b240dc..a7fa5a0 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "test:gwc-kubo": "node ./scripts/kubo-init.js", "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace", "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json ${GWC_SPECS} -- ${GWC_SKIP} -timeout 30m", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json $GWC_SPECS -- -test.skip=\"$GWC_SKIP\" -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index f88e07c..2e7d461 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -55,7 +55,7 @@ function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) { async function attemptKuboInit (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo init` + await $(execaOptions)`npx -y kubo init --profile test` log('Kubo initialized at %s', tmpDir) } catch (e) { if (!e.stderr.includes('already exists!')) { @@ -88,13 +88,13 @@ async function configureKubo (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}` - await $(execaOptions)`npx -y kubo config --json Bootstrap '[]'` + await $(execaOptions)`npx -y kubo config --json Bootstrap ${JSON.stringify([])}` await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true` await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false` await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true` await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` - await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin '["*"]'` - await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods '["GET", "POST", "PUT", "OPTIONS"]'` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin ${JSON.stringify(['*'])}` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods ${JSON.stringify(['GET', 'POST', 'PUT', 'OPTIONS'])}` log('Kubo configured') } catch (e) { error('Failed to configure Kubo', e) diff --git a/src/helia-server.ts b/src/helia-server.ts index 1f8c519..8cf5e67 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -95,12 +95,12 @@ export class HeliaServer { path: '/*', type: 'GET', handler: async (request, reply): Promise => { - try { - await this.fetch({ request, reply }) - } catch (err) { - this.log.error('error fetching:', err) - await reply.code(200).send('try /ipfs/ or /ipns/') - } + // try { + await this.fetch({ request, reply }) + // } catch (err) { + // this.log.error('error fetching:', err) + // await reply.code(500).send('try /ipfs/ or /ipns/') + // } } }, {