note loaders (#1776)

Add source of pages, exported files, and modules, to the build manifest.

---------

Co-authored-by: Mike Bostock <[email protected]>

src/build.ts

@@ -4,7 +4,7 @@ import {copyFile, readFile, rm, stat, writeFile} from "node:fs/promises";
 import {basename, dirname, extname, join} from "node:path/posix";
 import type {Config} from "./config.js";
 import {getDuckDBManifest} from "./duckdb.js";
-import {CliError} from "./error.js";
+import {CliError, enoent} from "./error.js";
 import {getClientPath, prepareOutput} from "./files.js";
 import {findModule, getModuleHash, readJavaScript} from "./javascript/module.js";
 import {transpileModule} from "./javascript/transpile.js";
@@ -54,7 +54,7 @@ export async function build(
   {config}: BuildOptions,
   effects: BuildEffects = new FileBuildEffects(config.output, join(config.root, ".observablehq", "cache"))
 ): Promise<void> {
-  const {root, loaders, duckdb} = config;
+  const {root, loaders, title, duckdb} = config;
   Telemetry.record({event: "build", step: "start"});
 
   // Prepare for build (such as by emptying the existing output root).
@@ -75,6 +75,25 @@ export async function build(
   let assetCount = 0;
   let pageCount = 0;
   const pagePaths = new Set<string>();
+
+  const buildManifest: BuildManifest = {
+    ...(title && {title}),
+    config: {root},
+    pages: [],
+    modules: [],
+    files: []
+  };
+
+  // file is the serving path relative to the base (e.g., /foo)
+  // path is the source file relative to the source root (e.g., /foo.md)
+  const addToManifest = (type: string, file: string, {title, path}: {title?: string | null; path: string}) => {
+    buildManifest[type].push({
+      path: config.normalizePath(file),
+      source: join("/", path), // TODO have route return path with leading slash?
+      ...(title != null && {title})
+    });
+  };
+
   for await (const path of config.paths()) {
     effects.output.write(`${faint("load")} ${path} `);
     const start = performance.now();
@@ -91,6 +110,7 @@ export async function build(
         effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`);
         outputs.set(path, {type: "module", resolvers});
         ++assetCount;
+        addToManifest("modules", path, module);
         continue;
       }
     }
@@ -99,6 +119,7 @@ export async function build(
       effects.output.write(`${faint("copy")} ${join(root, path)} ${faint("→")} `);
       const sourcePath = join(root, await file.load({useStale: true}, effects));
       await effects.copyFile(sourcePath, path);
+      addToManifest("files", path, file);
       ++assetCount;
       continue;
     }
@@ -209,7 +230,10 @@ export async function build(
   // Copy over referenced files, accumulating hashed aliases.
   for (const file of files) {
     effects.output.write(`${faint("copy")} ${join(root, file)} ${faint("→")} `);
-    const sourcePath = join(root, await loaders.loadFile(join("/", file), {useStale: true}, effects));
+    const path = join("/", file);
+    const loader = loaders.find(path);
+    if (!loader) throw enoent(path);
+    const sourcePath = join(root, await loader.load({useStale: true}, effects));
     const contents = await readFile(sourcePath);
     const hash = createHash("sha256").update(contents).digest("hex").slice(0, 8);
     const alias = applyHash(join("/_file", file), hash);
@@ -338,15 +362,13 @@ export async function build(
   }
 
   // Render pages!
-  const buildManifest: BuildManifest = {pages: []};
-  if (config.title) buildManifest.title = config.title;
   for (const [path, output] of outputs) {
     effects.output.write(`${faint("render")} ${path} ${faint("→")} `);
     if (output.type === "page") {
       const {page, resolvers} = output;
       const html = await renderPage(page, {...config, path, resolvers});
       await effects.writeFile(`${path}.html`, html);
-      buildManifest.pages.push({path: config.normalizePath(path), title: page.title});
+      addToManifest("pages", path, page);
     } else {
       const {resolvers} = output;
       const source = await renderModule(root, path, resolvers);
@@ -507,5 +529,8 @@ export class FileBuildEffects implements BuildEffects {
 
 export interface BuildManifest {
   title?: string;
-  pages: {path: string; title: string | null}[];
+  config: {root: string};
+  pages: {path: string; title?: string | null; source?: string}[];
+  modules: {path: string; source?: string}[];
+  files: {path: string; source?: string}[];
 }

src/loader.ts

@@ -73,25 +73,15 @@ export class LoaderResolver {
     );
   }
 
-  /**
-   * Loads the file at the specified path, returning a promise to the path to
-   * the (possibly generated) file relative to the source root.
-   */
-  async loadFile(path: string, options?: LoadOptions, effects?: LoadEffects): Promise<string> {
-    const loader = this.find(path);
-    if (!loader) throw enoent(path);
-    return await loader.load(options, effects);
-  }
-
   /**
    * Loads the page at the specified path, returning a promise to the parsed
    * page object.
    */
   async loadPage(path: string, options: LoadOptions & ParseOptions, effects?: LoadEffects): Promise<MarkdownPage> {
     const loader = this.findPage(path);
     if (!loader) throw enoent(path);
-    const source = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
-    return parseMarkdown(source, {params: loader.params, ...options});
+    const input = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
+    return parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
   }
 
   /**

src/markdown.ts

@@ -40,6 +40,7 @@ export interface MarkdownPage {
   data: FrontMatter;
   style: string | null;
   code: MarkdownCode[];
+  path: string;
   params?: Params;
 }
 
@@ -216,6 +217,7 @@ export interface ParseOptions {
   head?: Config["head"];
   header?: Config["header"];
   footer?: Config["footer"];
+  source?: string;
   params?: Params;
 }
 
@@ -242,7 +244,7 @@ export function createMarkdownIt({
 }
 
 export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
-  const {md, path, params} = options;
+  const {md, path, source = path, params} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
   const context: ParseContext = {code, startLine: 0, currentLine: 0, path, params};
@@ -258,6 +260,7 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
     title,
     style: getStyle(data, options),
     code,
+    path: source,
     params
   };
 }

src/observableApiClient.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import type {BuildManifest} from "./build.js";
 import type {ClackEffects} from "./clack.js";
 import {CliError, HttpError, isApiError} from "./error.js";
 import {formatByteSize} from "./format.js";
@@ -196,7 +197,7 @@ export class ObservableApiClient {
     });
   }
 
-  async postDeployUploaded(deployId: string, buildManifest: PostDeployUploadedRequest | null): Promise<DeployInfo> {
+  async postDeployUploaded(deployId: string, buildManifest: BuildManifest | null): Promise<DeployInfo> {
     return await this._fetch<DeployInfo>(new URL(`/cli/deploy/${deployId}/uploaded`, this._apiOrigin), {
       method: "POST",
       headers: {"content-type": "application/json"},
@@ -317,10 +318,3 @@ export interface PostDeployManifestResponse {
     detail: string | null;
   }[];
 }
-
-export interface PostDeployUploadedRequest {
-  pages: {
-    path: string;
-    title: string | null;
-  }[];
-}

src/preview.ts

@@ -177,7 +177,10 @@ export class PreviewServer {
         }
         throw enoent(path);
       } else if (pathname.startsWith("/_file/")) {
-        send(req, await loaders.loadFile(pathname.slice("/_file".length)), {root}).pipe(res);
+        const path = pathname.slice("/_file".length);
+        const loader = loaders.find(path);
+        if (!loader) throw enoent(path);
+        send(req, await loader.load(), {root}).pipe(res);
       } else {
         if ((pathname = normalize(pathname)).startsWith("..")) throw new Error("Invalid path: " + pathname);
 

test/build-test.ts

@@ -124,28 +124,67 @@ describe("build", () => {
       join(inputDir, "weather.md"),
       "# It's going to be ${weather}!" +
         "\n\n" +
-        "```js\nconst weather = await FileAttachment('weather.txt').text(); display(weather);\n```"
+        "```js\nconst weather = await FileAttachment('weather.txt').text(); display(weather);\n```" +
+        "\n\n" +
+        "```js\nconst generated = await FileAttachment('generated.txt').text(); display(generated);\n```" +
+        "\n\n" +
+        "```js\nconst internal = await FileAttachment('internal.txt').text(); display(internal);\n```" +
+        "\n\n" +
+        "```js\nconst thing = await FileAttachment('parameterized-thing.txt').text(); display(thing);\n```" +
+        "\n\n" +
+        "```js\nimport * from '/module-internal.js';\n```"
     );
     await mkdir(join(inputDir, "cities"));
     await writeFile(join(inputDir, "cities", "index.md"), "# Cities");
     await writeFile(join(inputDir, "cities", "portland.md"), "# Portland");
-    // A non-page file that should not be included
+    // exported files
     await writeFile(join(inputDir, "weather.txt"), "sunny");
+    await writeFile(join(inputDir, "generated.txt.ts"), "process.stdout.write('hello');");
+    await writeFile(join(inputDir, "parameterized-[page].txt.ts"), "process.stdout.write('hello');");
+    // /module-exported.js, /module-internal.js
+    await writeFile(join(inputDir, "module-[type].js"), "console.log(observable.params.type);");
+    // not exported
+    await writeFile(join(inputDir, "internal.txt.ts"), "process.stdout.write('hello');");
 
     const outputDir = await mkdtemp(tmpPrefix + "output-");
     const cacheDir = await mkdtemp(tmpPrefix + "output-");
 
-    const config = normalizeConfig({root: inputDir, output: outputDir}, inputDir);
+    const config = normalizeConfig(
+      {
+        root: inputDir,
+        output: outputDir,
+        dynamicPaths: [
+          "/module-exported.js",
+          "/weather.txt",
+          "/generated.txt",
+          "/parameterized-thing.txt",
+          "/parameterized-[page].txt"
+        ]
+      },
+      inputDir
+    );
     const effects = new LoggingBuildEffects(outputDir, cacheDir);
     await build({config}, effects);
     effects.buildManifest!.pages.sort((a, b) => ascending(a.path, b.path));
-    assert.deepEqual(effects.buildManifest, {
+    const {
+      config: {root},
+      ...manifest
+    } = effects.buildManifest!;
+    assert.equal(typeof root, "string");
+    assert.deepEqual(manifest, {
       pages: [
-        {path: "/", title: "Hello, world!"},
-        {path: "/cities/", title: "Cities"},
-        {path: "/cities/portland", title: "Portland"},
-        {path: "/weather", title: "It's going to be !"}
-      ]
+        {path: "/", title: "Hello, world!", source: "/index.md"},
+        {path: "/cities/", title: "Cities", source: "/cities/index.md"},
+        {path: "/cities/portland", title: "Portland", source: "/cities/portland.md"},
+        {path: "/weather", title: "It's going to be !", source: "/weather.md"}
+      ],
+      files: [
+        {path: "/weather.txt", source: "/weather.txt"},
+        {path: "/generated.txt", source: "/generated.txt.ts"},
+        {path: "/parameterized-thing.txt", source: "/parameterized-[page].txt.ts"},
+        {path: "/parameterized-[page].txt", source: "/parameterized-[page].txt.ts"}
+      ],
+      modules: [{path: "/module-exported.js", source: "/module-[type].js"}]
     });
 
     await Promise.all([inputDir, cacheDir, outputDir].map((dir) => rm(dir, {recursive: true}))).catch(() => {});

test/deploy-test.ts

@@ -8,7 +8,7 @@ import type {DeployEffects, DeployOptions} from "../src/deploy.js";
 import {deploy, promptDeployTarget} from "../src/deploy.js";
 import {CliError, isHttpError} from "../src/error.js";
 import {visitFiles} from "../src/files.js";
-import type {ObservableApiClientOptions, PostDeployUploadedRequest} from "../src/observableApiClient.js";
+import type {ObservableApiClientOptions} from "../src/observableApiClient.js";
 import type {GetCurrentUserResponse} from "../src/observableApiClient.js";
 import {ObservableApiClient} from "../src/observableApiClient.js";
 import type {DeployConfig} from "../src/observableApiConfig.js";
@@ -724,7 +724,7 @@ describe("deploy", () => {
 
   it("includes a build manifest if one was generated", async () => {
     const deployId = "deploy456";
-    let buildManifestPages: PostDeployUploadedRequest["pages"] | null = null;
+    let buildManifestPages: BuildManifest["pages"] | null = null;
     getCurrentObservableApi()
       .handleGetCurrentUser()
       .handleGetProject(DEPLOY_CONFIG)
@@ -744,7 +744,12 @@ describe("deploy", () => {
       deployConfig: DEPLOY_CONFIG,
       fixedInputStatTime: new Date("2024-03-09"),
       fixedOutputStatTime: new Date("2024-03-10"),
-      buildManifest: {pages: [{path: "/", title: "Build test case"}]}
+      buildManifest: {
+        config: {root: "src"},
+        pages: [{path: "/", title: "Build test case"}],
+        modules: [],
+        files: []
+      }
     });
     effects.clack.inputs = ["fix some bugs"]; // "what changed?"
     await deploy(TEST_OPTIONS, effects);

test/output/block-expression.md.json

@@ -36,5 +36,6 @@
       },
       "mode": "inline"
     }
-  ]
+  ],
+  "path": "block-expression.md"
 }

test/output/comment.md.json

@@ -2,5 +2,6 @@
   "data": {},
   "title": null,
   "style": null,
-  "code": []
+  "code": [],
+  "path": "comment.md"
 }

test/output/dollar-expression.md.json

@@ -36,5 +36,6 @@
       },
       "mode": "inline"
     }
-  ]
+  ],
+  "path": "dollar-expression.md"
 }

test/output/dot-graphviz.md.json

@@ -52,5 +52,6 @@
       },
       "mode": "block"
     }
-  ]
+  ],
+  "path": "dot-graphviz.md"
 }

test/output/double-quote-expression.md.json

@@ -23,5 +23,6 @@
       },
       "mode": "inline"
     }
-  ]
+  ],
+  "path": "double-quote-expression.md"
 }

test/output/embedded-expression.md.json

@@ -36,5 +36,6 @@
       },
       "mode": "inline"
     }
-  ]
+  ],
+  "path": "embedded-expression.md"
 }

test/output/escaped-expression.md.json

@@ -163,5 +163,6 @@
       },
       "mode": "inline"
     }
-  ]
+  ],
+  "path": "escaped-expression.md"
 }

test/output/fenced-code-options.md.json

@@ -275,5 +275,6 @@
       },
       "mode": "block"
     }
-  ]
+  ],
+  "path": "fenced-code-options.md"
 }

test/output/fenced-code.md.json

@@ -89,5 +89,6 @@
       },
       "mode": "block"
     }
-  ]
+  ],
+  "path": "fenced-code.md"
 }

test/output/fetch-parent-dir.md.json

@@ -245,5 +245,6 @@
       },
       "mode": "block"
     }
-  ]
+  ],
+  "path": "fetch-parent-dir.md"
 }