From 2c6fef072e802bfc71dcbe81bbd44a988b170f24 Mon Sep 17 00:00:00 2001 From: g1nt0ki <99907941+g1nt0ki@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:51:36 +0200 Subject: [PATCH] run dune queries only between 1-3 a.m --- helpers/dune.ts | 88 +++++++++++++++++++++++++----------------- helpers/duneRequest.ts | 6 +-- helpers/env.ts | 1 + 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/helpers/dune.ts b/helpers/dune.ts index f70cacd19a..bbd4480370 100644 --- a/helpers/dune.ts +++ b/helpers/dune.ts @@ -3,12 +3,15 @@ import { getEnv } from "./env"; const plimit = require('p-limit'); const limit = plimit(1); -const API_KEYS =getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"] +const isRestrictedMode = getEnv('DUNE_RESTRICTED_MODE') === 'true' +const API_KEYS = getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"] let API_KEY_INDEX = 0; const NOW_TIMESTAMP = Math.trunc((Date.now()) / 1000) const getLatestData = async (queryId: string) => { + checkCanRunDuneQuery() + const url = `https://api.dune.com/api/v1/query/${queryId}/results` try { const latest_result = (await limit(() => httpGet(url, { @@ -34,7 +37,9 @@ async function randomDelay() { return new Promise((resolve) => setTimeout(resolve, delay * 1000)) } -const inquiryStatus = async (execution_id: string, queryId:string) => { +const inquiryStatus = async (execution_id: string, queryId: string) => { + checkCanRunDuneQuery() + let _status = undefined; do { try { @@ -55,45 +60,49 @@ const inquiryStatus = async (execution_id: string, queryId:string) => { } const submitQuery = async (queryId: string, query_parameters = {}) => { - let query: undefined | any = undefined + checkCanRunDuneQuery() + + let query: undefined | any = undefined + try { + query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, { + headers: { + "x-dune-api-key": API_KEYS[API_KEY_INDEX], + 'Content-Type': 'application/json' + } + })) + if (query?.execution_id) { + return query?.execution_id + } else { + throw new Error("error query data: " + query) + } + } catch (e: any) { + throw e; + } +} + + +export const queryDune = async (queryId: string, query_parameters: any = {}) => { + checkCanRunDuneQuery() + + if (Object.keys(query_parameters).length === 0) { + const latest_result = await getLatestData(queryId) + if (latest_result !== undefined) return latest_result + } + const execution_id = await submitQuery(queryId, query_parameters) + const _status = await inquiryStatus(execution_id, queryId) + if (_status === 'QUERY_STATE_COMPLETED') { + const API_KEY = API_KEYS[API_KEY_INDEX] try { - query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, { + const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=5&offset=0`, { headers: { - "x-dune-api-key": API_KEYS[API_KEY_INDEX], - 'Content-Type': 'application/json' + "x-dune-api-key": API_KEY } })) - if (query?.execution_id) { - return query?.execution_id - } else { - throw new Error("error query data: " + query) - } + return queryStatus.result.rows } catch (e: any) { throw e; } -} - - -export const queryDune = async (queryId: string, query_parameters:any = {}) => { - if (Object.keys(query_parameters).length === 0) { - const latest_result = await getLatestData(queryId) - if (latest_result !== undefined) return latest_result - } - const execution_id = await submitQuery(queryId, query_parameters) - const _status = await inquiryStatus(execution_id, queryId) - if (_status === 'QUERY_STATE_COMPLETED') { - const API_KEY = API_KEYS[API_KEY_INDEX] - try { - const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=5&offset=0`, { - headers: { - "x-dune-api-key": API_KEY - } - })) - return queryStatus.result.rows - } catch (e: any) { - throw e; - } - } + } } const tableName = { @@ -102,9 +111,18 @@ const tableName = { base: "base", } as any -export const queryDuneSql = (options:any, query:string) => { +export const queryDuneSql = (options: any, query: string) => { + checkCanRunDuneQuery() + return queryDune("3996608", { fullQuery: query.replace("CHAIN", tableName[options.chain] ?? options.chain).replace("TIME_RANGE", `block_time >= from_unixtime(${options.startTimestamp}) AND block_time <= from_unixtime(${options.endTimestamp})`) }) } + +export function checkCanRunDuneQuery() { + if (!isRestrictedMode) return; + const currentHour = new Date().getHours(); + if (currentHour >= 1 && currentHour <= 3) return; // 1am - 3am - any time other than this, throw error + throw new Error("In restricted mode, can run dune queries only between 1am - 3am UTC"); +} \ No newline at end of file diff --git a/helpers/duneRequest.ts b/helpers/duneRequest.ts index 79d409f3a7..63025e3334 100644 --- a/helpers/duneRequest.ts +++ b/helpers/duneRequest.ts @@ -1,6 +1,7 @@ import retry from "async-retry"; import fetchURL from "../utils/fetchURL"; import { getEnv } from "./env"; +import { checkCanRunDuneQuery } from "./dune"; const API_KEYS = getEnv('DUNE_API_KEYS')?.split(",") ?? []; type IRequest = { @@ -9,9 +10,8 @@ type IRequest = { const requests: IRequest = {} export async function fetchURLWithRetry(url: string) { - /* const error = new Error("Dune: queryId is required") - delete error.stack - throw error */ + checkCanRunDuneQuery() + if (!requests[url]) requests[url] = _fetchURLWithRetry(url) return requests[url] diff --git a/helpers/env.ts b/helpers/env.ts index f98149f203..ee87b9f94d 100644 --- a/helpers/env.ts +++ b/helpers/env.ts @@ -18,6 +18,7 @@ export const ENV_KEYS = new Set([ 'INDEXA_DB', 'FLIPSIDE_API_KEY', 'DUNE_API_KEYS', + 'DUNE_RESTRICTED_MODE', 'ALLIUM_API_KEY', 'BIT_QUERY_API_KEY', 'SMARDEX_SUBGRAPH_API_KEY',