TableAPI: Create & Insert (#39)

Add support for the new table endpoints in the SDK:

- https://docs.dune.com/api-reference/tables/endpoint/create
- https://docs.dune.com/api-reference/tables/endpoint/insert

Additionally `uploadCsv` was moved from client with the new endpoints into Table API to consolidate all table endpoints in one file.

Skipped tests (that work) were added.

src/api/client.ts

@@ -12,16 +12,17 @@ import {
   QueryParameter,
   GetStatusResponse,
   ExecutionResponseCSV,
-  SuccessResponse,
+  ExecutionParams,
+  RunQueryArgs,
+  RunSqlArgs,
 } from "../types";
 import { sleep } from "../utils";
 import log from "loglevel";
 import { logPrefix } from "../utils";
 import { ExecutionAPI } from "./execution";
 import { POLL_FREQUENCY_SECONDS } from "../constants";
-import { ExecutionParams, UploadCSVArgs } from "../types/requestPayload";
 import { QueryAPI } from "./query";
-import { RunQueryArgs, RunSqlArgs } from "../types/client";
+import { TableAPI } from "./table";
 
 /// Various states of query execution that are "terminal".
 const TERMINAL_STATES = [
@@ -40,10 +41,13 @@ export class DuneClient {
   exec: ExecutionAPI;
   /// Query Management Interface.
   query: QueryAPI;
+  /// Table Management Interface
+  table: TableAPI;
 
   constructor(apiKey: string) {
     this.exec = new ExecutionAPI(apiKey);
     this.query = new QueryAPI(apiKey);
+    this.table = new TableAPI(apiKey);
   }
 
   /**
@@ -190,23 +194,6 @@ export class DuneClient {
     return results;
   }
 
-  /**
-   * Allows for anyone to upload a CSV as a table in Dune.
-   * The size limit per upload is currently 200MB.
-   * Storage is limited by plan, 1MB on free, 15GB on plus, and 50GB on premium.
-   *
-   * @param args UploadCSVParams relevant fields related to dataset upload.
-   * @returns boolean representing if upload was successful.
-   */
-  async uploadCsv(args: UploadCSVArgs): Promise<boolean> {
-    const response = await this.exec.post<SuccessResponse>("table/upload/csv", args);
-    try {
-      return Boolean(response.success);
-    } catch (err) {
-      throw new DuneError(`UploadCsvResponse ${JSON.stringify(response)}`);
-    }
-  }
-
   private async _runInner(
     queryID: number,
     params?: ExecutionParams,

src/api/execution.ts

@@ -8,15 +8,13 @@ import {
   concatResultCSV,
   SuccessResponse,
   LatestResultsResponse,
+  ExecutionParams,
+  ExecutionPerformance,
+  GetResultParams,
 } from "../types";
 import log from "loglevel";
 import { ageInHours, logPrefix, withDefaults } from "../utils";
 import { Router } from "./router";
-import {
-  ExecutionParams,
-  ExecutionPerformance,
-  GetResultParams,
-} from "../types/requestPayload";
 import {
   DEFAULT_GET_PARAMS,
   DUNE_CSV_NEXT_OFFSET_HEADER,

src/api/index.ts

@@ -2,3 +2,4 @@ export * from "./client";
 export * from "./execution";
 export * from "./query";
 export * from "./router";
+export * from "./table";

src/api/query.ts

@@ -1,7 +1,11 @@
-// Assuming the existence of these imports based on your Python code
 import { Router } from "./router";
-import { DuneQuery, CreateQueryResponse, DuneError } from "../types";
-import { CreateQueryParams, UpdateQueryParams } from "../types/requestPayload";
+import {
+  DuneQuery,
+  CreateQueryResponse,
+  DuneError,
+  CreateQueryParams,
+  UpdateQueryParams,
+} from "../types";
 import log from "loglevel";
 
 interface EditQueryResponse {

src/api/router.ts

@@ -1,13 +1,14 @@
-import { DuneError } from "../types";
-import fetch from "cross-fetch";
-import log from "loglevel";
-import { logPrefix } from "../utils";
 import {
+  ContentType,
+  DuneError,
   RequestPayload,
   payloadJSON,
   payloadSearchParams,
-} from "../types/requestPayload";
+} from "../types";
 import { version } from "../../package.json";
+import fetch from "cross-fetch";
+import log from "loglevel";
+import { logPrefix } from "../utils";
 
 const BASE_URL = "https://api.dune.com/api";
 
@@ -37,8 +38,18 @@ export class Router {
    * @param params payload sent with request (should be aligned with what the interface supports)
    * @returns a flexible data type representing whatever is expected to be returned from the request.
    */
-  async post<T>(route: string, params?: RequestPayload): Promise<T> {
-    return this._request<T>(RequestMethod.POST, this.url(route), params);
+  async post<T>(
+    route: string,
+    params?: RequestPayload,
+    content_type: ContentType = ContentType.Json,
+  ): Promise<T> {
+    return this._request<T>(
+      RequestMethod.POST,
+      this.url(route),
+      params,
+      false,
+      content_type,
+    );
   }
 
   protected async _handleResponse<T>(responsePromise: Promise<Response>): Promise<T> {
@@ -82,18 +93,25 @@ export class Router {
     url: string,
     payload?: RequestPayload,
     raw: boolean = false,
+    content_type: ContentType = ContentType.Json,
   ): Promise<T> {
-    const payloadData = payloadJSON(payload);
-    log.debug(logPrefix, `${method} received input url=${url}, payload=${payloadData}`);
+    let body;
+    if (Buffer.isBuffer(payload)) {
+      body = payload;
+    } else {
+      body = payloadJSON(payload);
+    }
+    log.debug(logPrefix, `${method} received input url=${url}, payload=${body}`);
     const requestData: RequestInit = {
       method,
       headers: {
         "x-dune-api-key": this.apiKey,
         "User-Agent": `client-sdk@${version} (https://www.npmjs.com/package/@duneanalytics/client-sdk)`,
+        "Content-Type": content_type,
       },
       // conditionally add the body property
       ...(method !== RequestMethod.GET && {
-        body: payloadData,
+        body,
       }),
     };
     let queryParams = "";

src/api/table.ts

@@ -0,0 +1,68 @@
+import { Router } from "./router";
+import {
+  CreateTableResult,
+  DuneError,
+  SuccessResponse,
+  UploadCSVArgs,
+  CreateTableArgs,
+  InsertTableArgs,
+  InsertTableResult,
+} from "../types";
+import { withDefaults } from "../utils";
+
+/**
+ * Table Management Interface (includes uploadCSV)
+ * https://docs.dune.com/api-reference/tables/
+ */
+export class TableAPI extends Router {
+  /**
+   * Allows for anyone to upload a CSV as a table in Dune.
+   * The size limit per upload is currently 200MB.
+   * Storage is limited by plan, 1MB on free, 15GB on plus, and 50GB on premium.
+   *
+   * @param args UploadCSVParams relevant fields related to dataset upload.
+   * @returns boolean representing if upload was successful.
+   */
+  async uploadCsv(args: UploadCSVArgs): Promise<boolean> {
+    const response = await this.post<SuccessResponse>("table/upload/csv", args);
+    try {
+      return Boolean(response.success);
+    } catch (err) {
+      throw new DuneError(`UploadCsvResponse ${JSON.stringify(response)}`);
+    }
+  }
+
+  /**
+   * https://docs.dune.com/api-reference/tables/endpoint/create
+   * The create table endpoint allows you to create an empty table
+   * with a specific schema in Dune.
+   *
+   * The only limitations are:
+   * - If a table already exists with the same name, the request will fail.
+   * - Column names in the table can’t start with a special character or a digit.
+   * @param args
+   */
+  async create(args: CreateTableArgs): Promise<CreateTableResult> {
+    return this.post<CreateTableResult>(
+      "table/create",
+      withDefaults<CreateTableArgs>(args, { description: "", is_private: false }),
+    );
+  }
+
+  /**
+   * https://docs.dune.com/api-reference/tables/endpoint/insert
+   * The insert table endpoint allows you to insert data into an existing table in Dune.
+   * The only limitations are:
+   * - The file has to be in json or csv format
+   * - The file has to have the same schema as the table
+   * @param args
+   * @returns
+   */
+  async insert(args: InsertTableArgs): Promise<InsertTableResult> {
+    return this.post<InsertTableResult>(
+      `table/${args.namespace}/${args.table_name}/insert`,
+      args.data,
+      args.content_type,
+    );
+  }
+}

src/types/index.ts

@@ -1,4 +1,6 @@
+export * from "./client";
 export * from "./error";
 export * from "./query";
 export * from "./queryParameter";
+export * from "./requestPayload";
 export * from "./response";

src/types/requestPayload.ts

@@ -25,7 +25,10 @@ export type RequestPayload =
   | ExecuteQueryParams
   | UpdateQueryParams
   | CreateQueryParams
-  | UploadCSVArgs;
+  | UploadCSVArgs
+  | CreateTableArgs
+  | InsertTableArgs
+  | Buffer;
 
 /// Utility method used by router to parse request payloads.
 export function payloadJSON(payload?: RequestPayload): string {
@@ -109,3 +112,40 @@ export interface CreateQueryParams extends BaseParams {
   /// Whether the query should be created as private.
   is_private?: boolean;
 }
+
+export enum ColumnType {
+  Varchar = "varchar",
+  Integer = "integer",
+  Double = "double",
+  Boolean = "boolean",
+  Timestamp = "timestamp",
+}
+
+export interface SchemaRecord {
+  name: string;
+  type: ColumnType;
+}
+
+export interface CreateTableArgs {
+  namespace: string;
+  table_name: string;
+  schema: SchemaRecord[];
+  description?: string;
+  is_private?: boolean;
+}
+
+/**
+ * All supported API content types
+ */
+export enum ContentType {
+  Json = "application/json",
+  Csv = "text/csv",
+  NDJson = "application/x-ndjson",
+}
+
+export interface InsertTableArgs {
+  namespace: string;
+  table_name: string;
+  data: Buffer;
+  content_type: ContentType;
+}

src/types/response.ts

@@ -188,3 +188,14 @@ function concatResultMetadata(
     ...remainingValues,
   };
 }
+
+export interface CreateTableResult {
+  example_query: string;
+  full_name: string;
+  namespace: string;
+  table_name: string;
+}
+
+export interface InsertTableResult {
+  rows_written: number;
+}

tests/e2e/client.spec.ts

@@ -4,7 +4,7 @@ import log from "loglevel";
 import { BASIC_KEY, PLUS_KEY } from "./util";
 import * as fs from "fs/promises";
 
-log.setLevel(log.levels.DEBUG, true);
+log.setLevel("silent", true);
 
 describe("DuneClient Extensions", () => {
   let client: DuneClient;
@@ -113,21 +113,4 @@ describe("DuneClient Extensions", () => {
     const query = await premiumClient.query.readQuery(queryID);
     expect(query.is_archived).to.be.equal(true);
   });
-
-  it("uploadCSV", async () => {
-    const premiumClient = new DuneClient(PLUS_KEY);
-    const public_success = await premiumClient.uploadCsv({
-      table_name: "ts_client_test",
-      description: "testing csv upload from node",
-      data: "column1,column2\nvalue1,value2\nvalue3,value4",
-    });
-    expect(public_success).to.be.equal(true);
-
-    const private_success = await premiumClient.uploadCsv({
-      table_name: "ts_client_test_private",
-      data: "column1,column2\nvalue1,value2\nvalue3,value4",
-      is_private: true,
-    });
-    expect(private_success).to.be.equal(true);
-  });
 });