From ede443f040c6910683b03773d8c9ae44b347bbdf Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:01:55 -0700 Subject: [PATCH 1/3] fix: catch uncaught exceptions & gc handles request aborts --- src/get-custom-helia.ts | 4 ++-- src/helia-server.ts | 34 ++++++++++++++++++++++++---------- src/index.ts | 25 ++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/get-custom-helia.ts b/src/get-custom-helia.ts index 29b083b..c3558b1 100644 --- a/src/get-custom-helia.ts +++ b/src/get-custom-helia.ts @@ -24,12 +24,12 @@ export async function getCustomHelia (): Promise { } let blockstore: HeliaInit['blockstore'] | undefined - if (FILE_BLOCKSTORE_PATH != null) { + if (FILE_BLOCKSTORE_PATH != null && FILE_BLOCKSTORE_PATH !== '') { blockstore = new LevelBlockstore(FILE_BLOCKSTORE_PATH) } let datastore: HeliaInit['datastore'] | undefined - if (FILE_DATASTORE_PATH != null) { + if (FILE_DATASTORE_PATH != null && FILE_DATASTORE_PATH !== '') { datastore = new LevelDatastore(FILE_DATASTORE_PATH) } diff --git a/src/helia-server.ts b/src/helia-server.ts index f9809f2..87b5ef8 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -242,13 +242,7 @@ export class HeliaServer { } } - /** - * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia. - */ - async fetch ({ request, reply }: RouteHandler): Promise { - const url = this.#getFullUrlFromFastifyRequest(request) - this.log('fetching url "%s" with @helia/verified-fetch', url) - + #getRequestAwareSignal (request: FastifyRequest, url = this.#getFullUrlFromFastifyRequest(request), timeout?: number): AbortSignal { const opController = new AbortController() setMaxListeners(Infinity, opController.signal) const cleanupFn = (): void => { @@ -272,8 +266,26 @@ export class HeliaServer { */ request.raw.on('close', cleanupFn) + if (timeout != null) { + setTimeout(() => { + this.log.trace('request timed out for url "%s"', url) + opController.abort() + }, timeout) + } + return opController.signal + } + + /** + * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia. + */ + async fetch ({ request, reply }: RouteHandler): Promise { + const url = this.#getFullUrlFromFastifyRequest(request) + this.log('fetching url "%s" with @helia/verified-fetch', url) + + const signal = this.#getRequestAwareSignal(request, url) + await this.isReady - const resp = await this.heliaFetch(url, { signal: opController.signal, redirect: 'manual' }) + const resp = await this.heliaFetch(url, { signal, redirect: 'manual' }) await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply) } @@ -319,10 +331,12 @@ export class HeliaServer { /** * GC the node */ - async gc ({ reply }: RouteHandler): Promise { + async gc ({ reply, request }: RouteHandler): Promise { await this.isReady this.log('running `gc` on Helia node') - await this.heliaNode?.gc({ signal: AbortSignal.timeout(20000) }) + const signal = this.#getRequestAwareSignal(request, undefined, 20000) + await this.heliaNode?.gc({ signal }) + await this.heliaNode?.gc() await reply.code(200).send('OK') } diff --git a/src/index.ts b/src/index.ts index 0b37afc..861706e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -249,7 +249,7 @@ const stopWebServer = async (): Promise => { } let shutdownRequested = false -async function closeGracefully (signal: number): Promise { +async function closeGracefully (signal: number | string): Promise { log(`Received signal to terminate: ${signal}`) if (shutdownRequested) { log('closeGracefully: shutdown already requested, exiting callback.') @@ -268,3 +268,26 @@ async function closeGracefully (signal: number): Promise { // eslint-disable-next-line @typescript-eslint/no-misused-promises process.once(signal, closeGracefully) }) + +/** + * Unless ALLOW_UNHANDLED_ERROR_RECOVERY is set to false, we will attempt to recover from named unhandled errors in the recoverableErrors array. + */ +const allowUnhandledErrorRecovery = process.env.ALLOW_UNHANDLED_ERROR_RECOVERY !== 'false' +const recoverableErrors = ['ERR_STREAM_PREMATURE_CLOSE'] +process.on('uncaughtException', (error: any) => { + log.error('Uncaught Exception:', error) + if (allowUnhandledErrorRecovery && recoverableErrors.includes(error.code)) { + log.trace('Ignoring known error') + return + } + void closeGracefully('SIGTERM') +}) + +process.on('unhandledRejection', (error: any) => { + log.error('Unhandled rejection:', error) + if (allowUnhandledErrorRecovery && recoverableErrors.includes(error.code)) { + log.trace('Ignoring known error') + return + } + void closeGracefully('SIGTERM') +}) From 75d0d018731e28e6cff1d25e410ed57b5714b109 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:21:33 -0700 Subject: [PATCH 2/3] feat: allow configurable recovery errors --- src/constants.ts | 25 +++++++++++++++++++++++++ src/index.ts | 26 ++++++++------------------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 0b2a69d..c6f14f8 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -55,3 +55,28 @@ export const USE_DELEGATED_ROUTING = process.env.USE_DELEGATED_ROUTING !== 'fals * If not set, we will default delegated routing to `https://delegated-ipfs.dev` */ export const DELEGATED_ROUTING_V1_HOST = process.env.DELEGATED_ROUTING_V1_HOST ?? 'https://delegated-ipfs.dev' + +/** + * You can set `RECOVERABLE_ERRORS` to a comma delimited list of errors to recover from. + * If you want to recover from all errors, set `RECOVERABLE_ERRORS` to 'all'. + * If you want to recover from no errors, set `RECOVERABLE_ERRORS` to ''. + */ +export const RECOVERABLE_ERRORS = (() => { + if (process.env.RECOVERABLE_ERRORS === 'all') { + return 'all' + } + if (process.env.RECOVERABLE_ERRORS === '') { + return '' + } + return process.env.RECOVERABLE_ERRORS?.split(',') ?? 'all' +})() + +export const ALLOW_UNHANDLED_ERROR_RECOVERY = (() => { + if (RECOVERABLE_ERRORS === 'all') { + return true + } + if (RECOVERABLE_ERRORS === '') { + return false + } + return RECOVERABLE_ERRORS.length > 0 +})() diff --git a/src/index.ts b/src/index.ts index 861706e..0ff1409 100644 --- a/src/index.ts +++ b/src/index.ts @@ -68,6 +68,7 @@ * | `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` | * | `USE_DELEGATED_ROUTING` | Whether to use the delegated routing v1 API | `true` | * | `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1 | `https://delegated-ipfs.dev` | + * | `RECOVERABLE_ERRORS` | A comma delimited list of errors to recover from. These errors are checked in `uncaughtException` and `unhandledRejection` callbacks | `all` | * *