Added some read-only endpoints for getting and listing files. (#25)

This provides an HTTP-based alternative to accessing the R2 bucket via the S3
API. It supports downloading individual files, getting the HEAD, and listing
the contents of the bucket. The aim is to allow web apps to just get/list files
via the usual fetch() calls instead of being forced to bundle the AWS S3 SDK.

src/index.js

@@ -12,6 +12,7 @@ import * as gh from "./utils/github.js";
 import * as auth from "./utils/permissions.js";
 import * as http from "./utils/http.js";
 import * as s3 from "./utils/s3.js";
+import * as read from "./read.js";
 
 const router = Router();
 
@@ -83,6 +84,14 @@ router.post("/refresh/latest/:project/:asset", version.refreshLatestVersionHandl
 
 router.post("/refresh/usage/:project", quota.refreshQuotaUsageHandler);
 
+/*** Download ***/
+
+router.head("/file/:key", read.headFileHandler);
+
+router.get("/file/:key", read.downloadFileHandler);
+
+router.get("/list", read.listFilesHandler);
+
 /*** Setting up the listener ***/
 
 router.get("/", () => {

src/read.js

@@ -0,0 +1,44 @@
+import * as http from "./utils/http.js";
+import * as s3 from "./utils/s3.js";
+
+function createHeaders(payload) {
+    const headers = new Headers();
+    payload.writeHttpMetadata(headers);
+    headers.set('etag', payload.httpEtag);
+    headers.set('Last-Modified', payload.uploaded.toUTCString());
+    headers.set("Content-Length", payload.size);
+    return headers;
+}
+
+export async function headFileHandler(request, env, nonblockers) {
+    const payload = await env.BOUND_BUCKET.head(decodeURIComponent(request.params.key));
+    if (payload === null) {
+        throw new http.HttpError("object not found", 404);
+    }
+    const headers = createHeaders(payload);
+    return new Response(null, { headers });
+}
+
+export async function downloadFileHandler(request, env, nonblockers) {
+    const payload = await env.BOUND_BUCKET.get(decodeURIComponent(request.params.key));
+    if (payload === null) {
+        throw new http.HttpError("object not found", 404);
+    }
+    const headers = createHeaders(payload);
+    return new Response(payload.body, { headers });
+}
+
+export async function listFilesHandler(request, env, nonblockers) {
+    const params = request.query;
+    let prefix = null;
+    if ("prefix" in params) {
+        prefix = decodeURIComponent(params.prefix);
+    }
+    let recursive = false;
+    if ("recursive" in params) {
+        recursive = params.recursive == "true";
+    }
+    let collected = [];
+    await s3.listApply(prefix, x => collected.push(x), env, { trimPrefix: false, stripTrailingSlash: false, local: !recursive });
+    return new http.jsonResponse(collected, 200);
+}

src/utils/s3.js

@@ -76,7 +76,7 @@ export async function quickFetchJson(path, env, { mustWork = true } = {}) {
     }
 }
 
-export async function listApply(prefix, op, env, { namesOnly = true, trimPrefix = true, local = false, list_limit = 1000 } = {}) {
+export async function listApply(prefix, op, env, { namesOnly = true, trimPrefix = true, stripTrailingSlash = true, local = false, list_limit = 1000 } = {}) {
     let list_options = { limit: list_limit };
     if (prefix != null) {
         list_options.prefix = prefix;
@@ -93,9 +93,9 @@ export async function listApply(prefix, op, env, { namesOnly = true, trimPrefix
 
         if (local) {
             if (trimPrefix) {
-                listing.delimitedPrefixes.forEach(p => op(p.slice(prefix.length, p.length - 1))); // remove the prefix and the slash.
+                listing.delimitedPrefixes.forEach(p => op(p.slice(prefix.length, stripTrailingSlash ? p.length - 1 : p.length)));
             } else {
-                listing.delimitedPrefixes.forEach(p => op(p.slice(0, p.length - 1))); // remove the trailing slash.
+                listing.delimitedPrefixes.forEach(p => op(p.slice(0, stripTrailingSlash ? p.length - 1 : p.length)));
             }
         } 
 

swagger.json

@@ -562,6 +562,75 @@
                 },
                 "tags": [ "Refresh" ]
             }
+        },
+
+        "/file/{key}": {
+            "get": {
+                "summary": "Download the file with the specified key in the bucket. This is a REST-based replacement for `get-object` from the AWS S3 API.",
+                "parameters": [
+                    { "$ref": "#/components/parameters/key" }
+                ],
+                "responses": {
+                    "200": { "description": "Contents of the requested file." },
+                    "404": { "$ref": "#/components/responses/404" }
+                },
+                "tags": [ "Read-only" ]
+            },
+
+            "head": {
+                "summary": "Obtain metadata about the specified key in the bucket.",
+                "parameters": [
+                    { "$ref": "#/components/parameters/key" }
+                ],
+                "responses": {
+                    "200": { "description": "Metadata for the requested file." },
+                    "404": { "$ref": "#/components/responses/404" }
+                },
+                "tags": [ "Read-only" ]
+            }
+        },
+
+        "/list": {
+            "get": {
+                "summary": "List the contents of the bucket, possibly with a prefix. This is a REST-based replacement for `list-objects-v2` from the AWS S3 API.",
+                "parameters": [
+                    {
+                        "in": "query",
+                        "name": "prefix",
+                        "schema": {
+                            "type": "string"
+                        },
+                        "description": "URL-encoded prefix of the keys to be listed. To represent a \"directory\", the decoded prefix should end with a forward slash.",
+                        "required": false 
+                    },
+                    {
+                        "in": "query",
+                        "name": "recursive",
+                        "schema": {
+                            "type": "boolean",
+                            "default": false
+                        },
+                        "description": "Whether to list keys \"recursively\", treating slash-separated components of the key as subdirectories. If true, all keys (or all keys starting with `prefix`, if supplied) are listed. If false, keys are only listed if there are no more slashes after `prefix`; otherwise, the slash-delimited common prefix is shown (i.e., the \"subdirectory\" path).", 
+                        "required": false 
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "All keys (or common prefixes) in the bucket that start with `prefix`.",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                "tags": [ "Read-only" ]
+            }
         }
 
     },
@@ -594,6 +663,16 @@
                 },
                 "description": "Name of the version of the asset inside a project. This should not contain '/' or start with '..'.",
                 "required": true
+            },
+
+            "key": {
+                "in": "path",
+                "name": "key",
+                "schema": {
+                    "type": "string"
+                },
+                "description": "URL-encoded key of the file.",
+                "required": true
             }
         },
 
@@ -613,6 +692,14 @@
                         "schema":  { "$ref": "#/components/schemas/error" }
                     }
                 }
+            },
+            "404": { 
+                "description": "Entity does not exist.",
+                "content": {
+                    "application/json": {
+                        "schema":  { "$ref": "#/components/schemas/error" }
+                    }
+                }
             }
         },
 

tests/read.test.js

@@ -0,0 +1,106 @@
+import * as read from "../src/read.js";
+import * as setup from "./setup.js";
+
+beforeAll(async () => {
+    const env = getMiniflareBindings();
+    await setup.simpleMockProject(env);
+})
+
+test("headFileHandler works correctly", async () => {
+    const env = getMiniflareBindings();
+    const req = new Request("http://localhost", { method: "HEAD" });
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/whee.txt") };
+        const res = await read.headFileHandler(req, env, []);
+        expect(res.body).toBeNull();
+        const hh = res.headers;
+        expect(hh.get("Last-Modified").length).toBeGreaterThan(0);
+        expect(hh.get("etag").length).toBeGreaterThan(0);
+        expect(Number(hh.get("Content-Length"))).toBeGreaterThan(0);
+    }
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/..summary") };
+        const res = await read.headFileHandler(req, env, []);
+        expect(res.body).toBeNull();
+        const hh = res.headers;
+        expect(hh.get("content-type")).toBe("application/json");
+        expect(Number(hh.get("Content-Length"))).toBeGreaterThan(0);
+    }
+})
+
+test("downloadFileHandler works correctly", async () => {
+    const env = getMiniflareBindings();
+    const req = new Request("http://localhost", { method: "GET" });
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/whee.txt") };
+        const res = await read.downloadFileHandler(req, env, []);
+        const body = await res.text();
+        expect(body.startsWith("Aaron")).toBe(true);
+
+        const hh = res.headers;
+        expect(hh.get("Last-Modified").length).toBeGreaterThan(0);
+        expect(hh.get("etag").length).toBeGreaterThan(0);
+        expect(Number(hh.get("Content-Length"))).toBeGreaterThan(0);
+    }
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/..summary") };
+        const res = await read.downloadFileHandler(req, env, []);
+        const body = await res.json();
+        expect("upload_user_id" in body).toBe(true);
+
+        const hh = res.headers;
+        expect(hh.get("content-type")).toBe("application/json");
+    }
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/foo/bar.txt") };
+        const res = await read.downloadFileHandler(req, env, []);
+        const body = await res.text();
+        expect(body.startsWith("1\n")).toBe(true);
+    }
+
+    {
+        req.params = { key: encodeURIComponent("test/blob/v1/absent.txt") };
+        await expect(read.downloadFileHandler(req, env, [])).rejects.toThrow("not found");
+    }
+})
+
+test("listFilesHandler works correctly", async () => {
+    const env = getMiniflareBindings();
+    const req = new Request("http://localhost", { method: "GET" });
+
+    {
+        req.query = {};
+        const res = await read.listFilesHandler(req, env, []);
+        const body = await res.json();
+        expect(body.indexOf("test/") >= 0).toBe(true);
+    }
+
+    {
+        req.query = { prefix: encodeURIComponent("test/blob") };
+        const res = await read.listFilesHandler(req, env, []);
+        const body = await res.json();
+        expect(body.indexOf("test/blob/") >= 0).toBe(true);
+    }
+
+    {
+        req.query = { prefix: encodeURIComponent("test/blob/") };
+        const res = await read.listFilesHandler(req, env, []);
+        const body = await res.json();
+        expect(body.indexOf("test/blob/..latest") >= 0).toBe(true);
+        expect(body.indexOf("test/blob/v1/") >= 0).toBe(true);
+    }
+
+    {
+        req.query = { prefix: encodeURIComponent("test/blob"), recursive: "true" };
+        const res = await read.listFilesHandler(req, env, []);
+        const body = await res.json();
+        expect(body.indexOf("test/blob/v1/..summary") >= 0).toBe(true);
+        expect(body.indexOf("test/blob/v1/foo/bar.txt") >= 0).toBe(true);
+        expect(body.indexOf("test/..permissions") == -1).toBe(true);
+    }
+})

tests/utils/s3.test.js

@@ -86,6 +86,14 @@ test("listApply works as expected", async () => {
         survivors.sort();
         expect(survivors).toEqual(["alpha/alex.txt", "alpha/bravo", "alpha/bravo2"]);
     }
+
+    // Don't trim the trailing slash.
+    {
+        let survivors = [];
+        await s3.listApply("alpha/", p => survivors.push(p), env, { local: true, stripTrailingSlash: false });
+        survivors.sort();
+        expect(survivors).toEqual(["alex.txt", "bravo/", "bravo2/"]);
+    }
 })
 
 test("quick recursive delete works as expected", async () => {