From abef0d0f8c31c07ccf461a4038b6e770ddb25360 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Mon, 25 Mar 2024 12:27:26 +0100 Subject: [PATCH] TableAPI: Create & Insert (#39) Add support for the new table endpoints in the SDK: - https://docs.dune.com/api-reference/tables/endpoint/create - https://docs.dune.com/api-reference/tables/endpoint/insert Additionally `uploadCsv` was moved from client with the new endpoints into Table API to consolidate all table endpoints in one file. Skipped tests (that work) were added. --- src/api/client.ts | 27 +++------ src/api/execution.ts | 8 +-- src/api/index.ts | 1 + src/api/query.ts | 10 +++- src/api/router.ts | 38 ++++++++---- src/api/table.ts | 68 +++++++++++++++++++++ src/types/index.ts | 2 + src/types/requestPayload.ts | 42 ++++++++++++- src/types/response.ts | 11 ++++ tests/e2e/client.spec.ts | 19 +----- tests/e2e/table.spec.ts | 78 +++++++++++++++++++++++++ tests/fixtures/sample_table_insert.csv | 2 + tests/fixtures/sample_table_insert.json | 1 + 13 files changed, 250 insertions(+), 57 deletions(-) create mode 100644 src/api/table.ts create mode 100644 tests/e2e/table.spec.ts create mode 100644 tests/fixtures/sample_table_insert.csv create mode 100644 tests/fixtures/sample_table_insert.json diff --git a/src/api/client.ts b/src/api/client.ts index 252aafc..1f23f2c 100644 --- a/src/api/client.ts +++ b/src/api/client.ts @@ -12,16 +12,17 @@ import { QueryParameter, GetStatusResponse, ExecutionResponseCSV, - SuccessResponse, + ExecutionParams, + RunQueryArgs, + RunSqlArgs, } from "../types"; import { sleep } from "../utils"; import log from "loglevel"; import { logPrefix } from "../utils"; import { ExecutionAPI } from "./execution"; import { POLL_FREQUENCY_SECONDS } from "../constants"; -import { ExecutionParams, UploadCSVArgs } from "../types/requestPayload"; import { QueryAPI } from "./query"; -import { RunQueryArgs, RunSqlArgs } from "../types/client"; +import { TableAPI } from "./table"; /// Various states of query execution that are "terminal". const TERMINAL_STATES = [ @@ -40,10 +41,13 @@ export class DuneClient { exec: ExecutionAPI; /// Query Management Interface. query: QueryAPI; + /// Table Management Interface + table: TableAPI; constructor(apiKey: string) { this.exec = new ExecutionAPI(apiKey); this.query = new QueryAPI(apiKey); + this.table = new TableAPI(apiKey); } /** @@ -190,23 +194,6 @@ export class DuneClient { return results; } - /** - * Allows for anyone to upload a CSV as a table in Dune. - * The size limit per upload is currently 200MB. - * Storage is limited by plan, 1MB on free, 15GB on plus, and 50GB on premium. - * - * @param args UploadCSVParams relevant fields related to dataset upload. - * @returns boolean representing if upload was successful. - */ - async uploadCsv(args: UploadCSVArgs): Promise { - const response = await this.exec.post("table/upload/csv", args); - try { - return Boolean(response.success); - } catch (err) { - throw new DuneError(`UploadCsvResponse ${JSON.stringify(response)}`); - } - } - private async _runInner( queryID: number, params?: ExecutionParams, diff --git a/src/api/execution.ts b/src/api/execution.ts index ccb2371..040533d 100644 --- a/src/api/execution.ts +++ b/src/api/execution.ts @@ -8,15 +8,13 @@ import { concatResultCSV, SuccessResponse, LatestResultsResponse, + ExecutionParams, + ExecutionPerformance, + GetResultParams, } from "../types"; import log from "loglevel"; import { ageInHours, logPrefix, withDefaults } from "../utils"; import { Router } from "./router"; -import { - ExecutionParams, - ExecutionPerformance, - GetResultParams, -} from "../types/requestPayload"; import { DEFAULT_GET_PARAMS, DUNE_CSV_NEXT_OFFSET_HEADER, diff --git a/src/api/index.ts b/src/api/index.ts index c171d23..f025600 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -2,3 +2,4 @@ export * from "./client"; export * from "./execution"; export * from "./query"; export * from "./router"; +export * from "./table"; diff --git a/src/api/query.ts b/src/api/query.ts index df952bf..afc7bb7 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -1,7 +1,11 @@ -// Assuming the existence of these imports based on your Python code import { Router } from "./router"; -import { DuneQuery, CreateQueryResponse, DuneError } from "../types"; -import { CreateQueryParams, UpdateQueryParams } from "../types/requestPayload"; +import { + DuneQuery, + CreateQueryResponse, + DuneError, + CreateQueryParams, + UpdateQueryParams, +} from "../types"; import log from "loglevel"; interface EditQueryResponse { diff --git a/src/api/router.ts b/src/api/router.ts index 03023ce..b9d7813 100644 --- a/src/api/router.ts +++ b/src/api/router.ts @@ -1,13 +1,14 @@ -import { DuneError } from "../types"; -import fetch from "cross-fetch"; -import log from "loglevel"; -import { logPrefix } from "../utils"; import { + ContentType, + DuneError, RequestPayload, payloadJSON, payloadSearchParams, -} from "../types/requestPayload"; +} from "../types"; import { version } from "../../package.json"; +import fetch from "cross-fetch"; +import log from "loglevel"; +import { logPrefix } from "../utils"; const BASE_URL = "https://api.dune.com/api"; @@ -37,8 +38,18 @@ export class Router { * @param params payload sent with request (should be aligned with what the interface supports) * @returns a flexible data type representing whatever is expected to be returned from the request. */ - async post(route: string, params?: RequestPayload): Promise { - return this._request(RequestMethod.POST, this.url(route), params); + async post( + route: string, + params?: RequestPayload, + content_type: ContentType = ContentType.Json, + ): Promise { + return this._request( + RequestMethod.POST, + this.url(route), + params, + false, + content_type, + ); } protected async _handleResponse(responsePromise: Promise): Promise { @@ -82,18 +93,25 @@ export class Router { url: string, payload?: RequestPayload, raw: boolean = false, + content_type: ContentType = ContentType.Json, ): Promise { - const payloadData = payloadJSON(payload); - log.debug(logPrefix, `${method} received input url=${url}, payload=${payloadData}`); + let body; + if (Buffer.isBuffer(payload)) { + body = payload; + } else { + body = payloadJSON(payload); + } + log.debug(logPrefix, `${method} received input url=${url}, payload=${body}`); const requestData: RequestInit = { method, headers: { "x-dune-api-key": this.apiKey, "User-Agent": `client-sdk@${version} (https://www.npmjs.com/package/@duneanalytics/client-sdk)`, + "Content-Type": content_type, }, // conditionally add the body property ...(method !== RequestMethod.GET && { - body: payloadData, + body, }), }; let queryParams = ""; diff --git a/src/api/table.ts b/src/api/table.ts new file mode 100644 index 0000000..5d38339 --- /dev/null +++ b/src/api/table.ts @@ -0,0 +1,68 @@ +import { Router } from "./router"; +import { + CreateTableResult, + DuneError, + SuccessResponse, + UploadCSVArgs, + CreateTableArgs, + InsertTableArgs, + InsertTableResult, +} from "../types"; +import { withDefaults } from "../utils"; + +/** + * Table Management Interface (includes uploadCSV) + * https://docs.dune.com/api-reference/tables/ + */ +export class TableAPI extends Router { + /** + * Allows for anyone to upload a CSV as a table in Dune. + * The size limit per upload is currently 200MB. + * Storage is limited by plan, 1MB on free, 15GB on plus, and 50GB on premium. + * + * @param args UploadCSVParams relevant fields related to dataset upload. + * @returns boolean representing if upload was successful. + */ + async uploadCsv(args: UploadCSVArgs): Promise { + const response = await this.post("table/upload/csv", args); + try { + return Boolean(response.success); + } catch (err) { + throw new DuneError(`UploadCsvResponse ${JSON.stringify(response)}`); + } + } + + /** + * https://docs.dune.com/api-reference/tables/endpoint/create + * The create table endpoint allows you to create an empty table + * with a specific schema in Dune. + * + * The only limitations are: + * - If a table already exists with the same name, the request will fail. + * - Column names in the table can’t start with a special character or a digit. + * @param args + */ + async create(args: CreateTableArgs): Promise { + return this.post( + "table/create", + withDefaults(args, { description: "", is_private: false }), + ); + } + + /** + * https://docs.dune.com/api-reference/tables/endpoint/insert + * The insert table endpoint allows you to insert data into an existing table in Dune. + * The only limitations are: + * - The file has to be in json or csv format + * - The file has to have the same schema as the table + * @param args + * @returns + */ + async insert(args: InsertTableArgs): Promise { + return this.post( + `table/${args.namespace}/${args.table_name}/insert`, + args.data, + args.content_type, + ); + } +} diff --git a/src/types/index.ts b/src/types/index.ts index 06c387f..325d79b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,4 +1,6 @@ +export * from "./client"; export * from "./error"; export * from "./query"; export * from "./queryParameter"; +export * from "./requestPayload"; export * from "./response"; diff --git a/src/types/requestPayload.ts b/src/types/requestPayload.ts index 4470b5b..5df596b 100644 --- a/src/types/requestPayload.ts +++ b/src/types/requestPayload.ts @@ -25,7 +25,10 @@ export type RequestPayload = | ExecuteQueryParams | UpdateQueryParams | CreateQueryParams - | UploadCSVArgs; + | UploadCSVArgs + | CreateTableArgs + | InsertTableArgs + | Buffer; /// Utility method used by router to parse request payloads. export function payloadJSON(payload?: RequestPayload): string { @@ -109,3 +112,40 @@ export interface CreateQueryParams extends BaseParams { /// Whether the query should be created as private. is_private?: boolean; } + +export enum ColumnType { + Varchar = "varchar", + Integer = "integer", + Double = "double", + Boolean = "boolean", + Timestamp = "timestamp", +} + +export interface SchemaRecord { + name: string; + type: ColumnType; +} + +export interface CreateTableArgs { + namespace: string; + table_name: string; + schema: SchemaRecord[]; + description?: string; + is_private?: boolean; +} + +/** + * All supported API content types + */ +export enum ContentType { + Json = "application/json", + Csv = "text/csv", + NDJson = "application/x-ndjson", +} + +export interface InsertTableArgs { + namespace: string; + table_name: string; + data: Buffer; + content_type: ContentType; +} diff --git a/src/types/response.ts b/src/types/response.ts index a6cfadf..b75e9cb 100644 --- a/src/types/response.ts +++ b/src/types/response.ts @@ -188,3 +188,14 @@ function concatResultMetadata( ...remainingValues, }; } + +export interface CreateTableResult { + example_query: string; + full_name: string; + namespace: string; + table_name: string; +} + +export interface InsertTableResult { + rows_written: number; +} diff --git a/tests/e2e/client.spec.ts b/tests/e2e/client.spec.ts index 54e852a..1ede01d 100644 --- a/tests/e2e/client.spec.ts +++ b/tests/e2e/client.spec.ts @@ -4,7 +4,7 @@ import log from "loglevel"; import { BASIC_KEY, PLUS_KEY } from "./util"; import * as fs from "fs/promises"; -log.setLevel(log.levels.DEBUG, true); +log.setLevel("silent", true); describe("DuneClient Extensions", () => { let client: DuneClient; @@ -113,21 +113,4 @@ describe("DuneClient Extensions", () => { const query = await premiumClient.query.readQuery(queryID); expect(query.is_archived).to.be.equal(true); }); - - it("uploadCSV", async () => { - const premiumClient = new DuneClient(PLUS_KEY); - const public_success = await premiumClient.uploadCsv({ - table_name: "ts_client_test", - description: "testing csv upload from node", - data: "column1,column2\nvalue1,value2\nvalue3,value4", - }); - expect(public_success).to.be.equal(true); - - const private_success = await premiumClient.uploadCsv({ - table_name: "ts_client_test_private", - data: "column1,column2\nvalue1,value2\nvalue3,value4", - is_private: true, - }); - expect(private_success).to.be.equal(true); - }); }); diff --git a/tests/e2e/table.spec.ts b/tests/e2e/table.spec.ts new file mode 100644 index 0000000..9dd0b79 --- /dev/null +++ b/tests/e2e/table.spec.ts @@ -0,0 +1,78 @@ +import { expect } from "chai"; +import log from "loglevel"; +import { PLUS_KEY } from "./util"; +import * as fs from "fs/promises"; +import { TableAPI } from "../../src/api"; +import { ColumnType, ContentType } from "../../src"; + +log.setLevel("silent", true); + +describe("Table API", () => { + let tableClient: TableAPI; + + beforeEach(() => { + tableClient = new TableAPI(PLUS_KEY); + }); + + it("uploads CSV", async () => { + const public_success = await tableClient.uploadCsv({ + table_name: "ts_client_test", + description: "testing csv upload from node", + data: "column1,column2\nvalue1,value2\nvalue3,value4", + }); + expect(public_success).to.be.equal(true); + + const private_success = await tableClient.uploadCsv({ + table_name: "ts_client_test_private", + data: "column1,column2\nvalue1,value2\nvalue3,value4", + is_private: true, + }); + expect(private_success).to.be.equal(true); + }); + + // Skipped because needs valid user name. + it.skip("creates table", async () => { + const namespace = "your_username"; + const table_name = "dataset_e2e_test"; + const createResult = await tableClient.create({ + namespace, + table_name, + description: "e2e test table", + schema: [ + { name: "date", type: ColumnType.Timestamp }, + { name: "dgs10", type: ColumnType.Double }, + ], + is_private: false, + }); + + expect(createResult).to.be.deep.equal({ + namespace: namespace, + table_name: table_name, + full_name: `dune.${namespace}.${table_name}`, + example_query: `select * from dune.${namespace}.${table_name} limit 10`, + }); + }); + + it.skip("inserts JSON to Table", async () => { + const data: Buffer = await fs.readFile("./tests/fixtures/sample_table_insert.json"); + const insertResult = await tableClient.insert({ + namespace: "your_username", + table_name: "dataset_e2e_test", + data, + content_type: ContentType.NDJson, + }); + + expect(insertResult).to.be.deep.equal({ rows_written: 1 }); + }); + + it.skip("inserts CSV to Table", async () => { + const data = await fs.readFile("./tests/fixtures/sample_table_insert.csv"); + const insertResult = await tableClient.insert({ + namespace: "your_username", + table_name: "dataset_e2e_test", + data, + content_type: ContentType.Csv, + }); + expect(insertResult).to.be.deep.equal({ rows_written: 1 }); + }); +}); diff --git a/tests/fixtures/sample_table_insert.csv b/tests/fixtures/sample_table_insert.csv new file mode 100644 index 0000000..70b2f9e --- /dev/null +++ b/tests/fixtures/sample_table_insert.csv @@ -0,0 +1,2 @@ +date,dgs10 +2020-12-01T23:33:00,10 \ No newline at end of file diff --git a/tests/fixtures/sample_table_insert.json b/tests/fixtures/sample_table_insert.json new file mode 100644 index 0000000..f2b93f0 --- /dev/null +++ b/tests/fixtures/sample_table_insert.json @@ -0,0 +1 @@ +{"date":"2020-12-01T23:33:00","dgs10":10} \ No newline at end of file