run dune queries only between 1-3 a.m

DefiLlama · Oct 4, 2024 · 2c6fef0 · 2c6fef0
1 parent 72d1598
commit 2c6fef0
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 38 deletions.
diff --git a/helpers/dune.ts b/helpers/dune.ts
@@ -3,12 +3,15 @@ import { getEnv } from "./env";
 const plimit = require('p-limit');
 const limit = plimit(1);
 
-const API_KEYS =getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"]
+const isRestrictedMode = getEnv('DUNE_RESTRICTED_MODE') === 'true'
+const API_KEYS = getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"]
 let API_KEY_INDEX = 0;
 
 const NOW_TIMESTAMP = Math.trunc((Date.now()) / 1000)
 
 const getLatestData = async (queryId: string) => {
+  checkCanRunDuneQuery()
+
   const url = `https://api.dune.com/api/v1/query/${queryId}/results`
   try {
     const latest_result = (await limit(() => httpGet(url, {
@@ -34,7 +37,9 @@ async function randomDelay() {
   return new Promise((resolve) => setTimeout(resolve, delay * 1000))
 }
 
-const inquiryStatus = async (execution_id: string, queryId:string) => {
+const inquiryStatus = async (execution_id: string, queryId: string) => {
+  checkCanRunDuneQuery()
+
   let _status = undefined;
   do {
     try {
@@ -55,45 +60,49 @@ const inquiryStatus = async (execution_id: string, queryId:string) => {
 }
 
 const submitQuery = async (queryId: string, query_parameters = {}) => {
-    let query: undefined | any = undefined
+  checkCanRunDuneQuery()
+
+  let query: undefined | any = undefined
+  try {
+    query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, {
+      headers: {
+        "x-dune-api-key": API_KEYS[API_KEY_INDEX],
+        'Content-Type': 'application/json'
+      }
+    }))
+    if (query?.execution_id) {
+      return query?.execution_id
+    } else {
+      throw new Error("error query data: " + query)
+    }
+  } catch (e: any) {
+    throw e;
+  }
+}
+
+
+export const queryDune = async (queryId: string, query_parameters: any = {}) => {
+  checkCanRunDuneQuery()
+
+  if (Object.keys(query_parameters).length === 0) {
+    const latest_result = await getLatestData(queryId)
+    if (latest_result !== undefined) return latest_result
+  }
+  const execution_id = await submitQuery(queryId, query_parameters)
+  const _status = await inquiryStatus(execution_id, queryId)
+  if (_status === 'QUERY_STATE_COMPLETED') {
+    const API_KEY = API_KEYS[API_KEY_INDEX]
     try {
-      query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, {
+      const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=5&offset=0`, {
         headers: {
-          "x-dune-api-key": API_KEYS[API_KEY_INDEX],
-          'Content-Type': 'application/json'
+          "x-dune-api-key": API_KEY
         }
       }))
-      if (query?.execution_id) {
-        return query?.execution_id
-      } else {
-        throw new Error("error query data: " + query)
-      }
+      return queryStatus.result.rows
     } catch (e: any) {
       throw e;
     }
-}
-
-
-export const queryDune = async (queryId: string, query_parameters:any = {}) => {
-    if (Object.keys(query_parameters).length === 0) {
-      const latest_result = await getLatestData(queryId)
-      if (latest_result !== undefined) return latest_result
-    }
-    const execution_id = await submitQuery(queryId, query_parameters)
-    const _status = await inquiryStatus(execution_id, queryId)
-    if (_status === 'QUERY_STATE_COMPLETED') {
-      const API_KEY = API_KEYS[API_KEY_INDEX]
-      try {
-        const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=5&offset=0`, {
-          headers: {
-            "x-dune-api-key": API_KEY
-          }
-        }))
-        return queryStatus.result.rows
-      } catch (e: any) {
-        throw e;
-      }
-    }
+  }
 }
 
 const tableName = {
@@ -102,9 +111,18 @@ const tableName = {
   base: "base",
 } as any
 
-export const queryDuneSql = (options:any, query:string) => {
+export const queryDuneSql = (options: any, query: string) => {
+  checkCanRunDuneQuery()
+
   return queryDune("3996608", {
     fullQuery: query.replace("CHAIN", tableName[options.chain] ?? options.chain).replace("TIME_RANGE", `block_time >= from_unixtime(${options.startTimestamp})
   AND block_time <= from_unixtime(${options.endTimestamp})`)
   })
 }
+
+export function checkCanRunDuneQuery() {
+  if (!isRestrictedMode) return;
+  const currentHour = new Date().getHours();
+  if (currentHour >= 1 && currentHour <= 3) return; // 1am - 3am - any time other than this, throw error
+  throw new Error("In restricted mode, can run dune queries only between 1am - 3am UTC");
+}
diff --git a/helpers/duneRequest.ts b/helpers/duneRequest.ts
@@ -1,6 +1,7 @@
 import retry from "async-retry";
 import fetchURL from "../utils/fetchURL";
 import { getEnv } from "./env";
+import { checkCanRunDuneQuery } from "./dune";
 
 const API_KEYS = getEnv('DUNE_API_KEYS')?.split(",") ?? [];
 type IRequest = {
@@ -9,9 +10,8 @@ type IRequest = {
 const requests: IRequest = {}
 
 export async function fetchURLWithRetry(url: string) {
-  /* const error = new Error("Dune: queryId is required")
-  delete error.stack
-  throw error */
+  checkCanRunDuneQuery()
+
   if (!requests[url])
     requests[url] = _fetchURLWithRetry(url)
   return requests[url]

diff --git a/helpers/env.ts b/helpers/env.ts
@@ -18,6 +18,7 @@ export const ENV_KEYS = new Set([
   'INDEXA_DB',
   'FLIPSIDE_API_KEY',
   'DUNE_API_KEYS',
+  'DUNE_RESTRICTED_MODE',
   'ALLIUM_API_KEY',
   'BIT_QUERY_API_KEY',
   'SMARDEX_SUBGRAPH_API_KEY',