duckdb aliases, platforms config

observablehq · Nov 2, 2024 · ad768f2 · ad768f2
1 parent 27bc864
commit ad768f2
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 36 deletions.
diff --git a/src/config.ts b/src/config.ts
@@ -8,7 +8,7 @@ import {pathToFileURL} from "node:url";
 import he from "he";
 import type MarkdownIt from "markdown-it";
 import wrapAnsi from "wrap-ansi";
-import {DUCKDB_CORE_EXTENSIONS, DUCKDB_PLATFORMS} from "./duckdb.js";
+import {DUCKDB_CORE_ALIASES, DUCKDB_CORE_EXTENSIONS} from "./duckdb.js";
 import {visitFiles} from "./files.js";
 import {formatIsoDate, formatLocaleDate} from "./format.js";
 import type {FrontMatter} from "./frontMatter.js";
@@ -78,7 +78,7 @@ export interface SearchConfigSpec {
 }
 
 export interface DuckDBConfig {
-  platforms: Record<string, true>;
+  platforms: {[name: string]: true};
   extensions: {[name: string]: DuckDBExtensionConfig};
 }
 
@@ -522,21 +522,22 @@ export function stringOrNull(spec: unknown): string | null {
   return spec == null || spec === false ? null : String(spec);
 }
 
-// TODO configure platforms?
 function normalizeDuckDB(spec: unknown): DuckDBConfig {
+  const {mvp = true, eh = true} = spec?.["platforms"] ?? {};
   const extensions: {[name: string]: DuckDBExtensionConfig} = {};
   let extspec: Record<string, unknown> = spec?.["extensions"] ?? {};
   if (Array.isArray(extspec)) extspec = Object.fromEntries(extspec.map((name) => [name, {}]));
   if (extspec.json === undefined) extspec = {...extspec, json: false};
   if (extspec.parquet === undefined) extspec = {...extspec, parquet: false};
-  for (const name in extspec) {
+  for (let name in extspec) {
     if (!/^\w+$/.test(name)) throw new Error(`invalid extension: ${name}`);
     const vspec = extspec[name];
     if (vspec == null) continue;
+    name = DUCKDB_CORE_ALIASES[name] ?? name;
     const {
-      source = DUCKDB_CORE_EXTENSIONS.some(([n]) => n === name) ? "core" : "community",
+      source = name in DUCKDB_CORE_EXTENSIONS ? "core" : "community",
       install = true,
-      load = !DUCKDB_CORE_EXTENSIONS.find(([n]) => n === name)?.[1]
+      load = !DUCKDB_CORE_EXTENSIONS[name]
     } = typeof vspec === "boolean"
       ? {load: vspec}
       : typeof vspec === "string"
@@ -548,7 +549,15 @@ function normalizeDuckDB(spec: unknown): DuckDBConfig {
       load: Boolean(load)
     };
   }
-  return {platforms: DUCKDB_PLATFORMS, extensions};
+  return {
+    platforms: Object.fromEntries(
+      [
+        ["mvp", mvp],
+        ["eh", eh]
+      ].filter(([, enabled]) => enabled)
+    ),
+    extensions
+  };
 }
 
 function normalizeDuckDBSource(source: string): string {

diff --git a/src/duckdb.ts b/src/duckdb.ts
@@ -8,40 +8,50 @@ const downloadRequests = new Map<string, Promise<string>>();
 
 export const DUCKDB_WASM_VERSION = "1.29.0";
 export const DUCKDB_VERSION = "1.1.1";
-export const DUCKDB_PLATFORMS: DuckDBConfig["platforms"] = {eh: true, mvp: true};
 
 // https://duckdb.org/docs/extensions/core_extensions.html
-export const DUCKDB_CORE_EXTENSIONS: [name: string, autoload: boolean][] = [
-  ["arrow", false],
-  ["autocomplete", true],
-  ["aws", true],
-  ["azure", true],
-  ["delta", true],
-  ["excel", true],
-  ["fts", true],
-  ["httpfs", true],
-  ["iceberg", false],
-  ["icu", true],
-  ["inet", true],
-  ["jemalloc", false],
-  ["json", true],
-  ["mysql", false],
-  ["parquet", true],
-  ["postgres", true],
-  ["spatial", false],
-  ["sqlite", true],
-  ["substrait", false],
-  ["tpcds", true],
-  ["tpch", true],
-  ["vss", false]
-];
+export const DUCKDB_CORE_ALIASES: Record<string, keyof typeof DUCKDB_CORE_EXTENSIONS> = {
+  sqlite: "sqlite_scanner",
+  sqlite3: "sqlite_scanner",
+  postgres_scanner: "postgres",
+  http: "httpfs",
+  https: "httpfs",
+  s3: "httpfs"
+} as const;
+
+// https://duckdb.org/docs/extensions/core_extensions.html
+// https://duckdb.org/docs/api/wasm/extensions.html#list-of-officially-available-extensions
+export const DUCKDB_CORE_EXTENSIONS = {
+  arrow: false,
+  autocomplete: true,
+  aws: true,
+  azure: true,
+  delta: true,
+  excel: true,
+  fts: true,
+  httpfs: true,
+  iceberg: false,
+  icu: true,
+  inet: true,
+  jemalloc: false,
+  json: true,
+  mysql: false,
+  parquet: true,
+  postgres: true,
+  spatial: false,
+  sqlite_scanner: true,
+  substrait: false,
+  tpcds: true,
+  tpch: true,
+  vss: false
+} as const;
 
 export async function getDuckDBManifest(
   {platforms, extensions}: DuckDBConfig,
   {root, aliases}: {root: string; aliases?: Map<string, string>}
 ) {
   return {
-    platforms,
+    platforms: {mvp: "mvp" in platforms, eh: "eh" in platforms},
     extensions: Object.fromEntries(
       await Promise.all(
         Object.entries(extensions).map(([name, {install, load, source}]) =>

diff --git a/test/config-test.ts b/test/config-test.ts
@@ -1,11 +1,15 @@
 import assert from "node:assert";
 import {resolve} from "node:path";
 import MarkdownIt from "markdown-it";
+import type {DuckDBConfig} from "../src/config.js";
 import {normalizeConfig as config, mergeToc, readConfig, setCurrentDate} from "../src/config.js";
 import {LoaderResolver} from "../src/loader.js";
 
-const DUCKDB_DEFAULTS = {
-  bundles: ["eh", "mvp"],
+const DUCKDB_DEFAULTS: DuckDBConfig = {
+  platforms: {
+    eh: true,
+    mvp: true
+  },
   extensions: {
     json: {
       source: "https://extensions.duckdb.org/",

diff --git a/test/libraries-test.ts b/test/libraries-test.ts
@@ -53,7 +53,7 @@ describe("getImplicitStylesheets(imports)", () => {
 describe("getImplicitDownloads(imports)", () => {
   it("supports known imports", () => {
     assert.deepStrictEqual(
-      getImplicitDownloads(["npm:@observablehq/duckdb"]),
+      getImplicitDownloads(["npm:@observablehq/duckdb"], {extensions: {}, platforms: {mvp: true, eh: true}}),
       new Set([
         "npm:@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm",
         "npm:@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js",