Commit to share code with internal team members

package.json

@@ -13,8 +13,8 @@
   },
   "dependencies": {
     "@composedb/client": "^0.7.1",
-    "cartonne": "^3.0.1",
     "@composedb/devtools-node": "^0.7.1",
+    "@didtools/codecs": "^3.0.0",
     "@google-cloud/local-auth": "2.1.0",
     "@notionhq/client": "^2.2.15",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -49,9 +49,11 @@
     "alchemy-sdk": "^3.3.1",
     "axios": "^1.7.2",
     "bullmq": "^5.7.15",
+    "cartonne": "^3.0.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "cmdk": "^1.0.0",
+    "codeco": "^1.2.3",
     "date-fns": "^3.6.0",
     "did-session": "^2.1.1",
     "dids": "^5.0.2",
@@ -70,6 +72,7 @@
     "next-auth": "^4.24.6",
     "next-themes": "^0.3.0",
     "node-libcurl": "^4.0.0",
+    "object-sizeof": "^2.6.4",
     "pg": "8.6.0",
     "react": "18.2.0",
     "react-code-blocks": "^0.1.6",
@@ -85,6 +88,7 @@
     "tailwindcss-animate": "^1.0.7",
     "tsx": "^4.9.3",
     "twitter-api-sdk": "^1.2.1",
+    "varint": "^6.0.0",
     "viem": "^1.12.2",
     "wagmi": "^1.4.2",
     "zod": "^3.22.4"
@@ -100,6 +104,7 @@
     "@typescript-eslint/eslint-plugin": "^7.1.1",
     "@typescript-eslint/parser": "^7.1.1",
     "@web3modal/wagmi": "^3.0.1",
+    "@types/varint": "^6.0.3",
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.1.3",
     "postcss": "^8.4.34",

src/components/services/constants.ts

@@ -0,0 +1,17 @@
+export const STREAMID_CODEC = 206
+
+export const STREAM_TYPES = {
+  tile: 0,
+  'caip10-link': 1,
+  model: 2,
+  MID: 3,
+  UNLOADABLE: 4,
+} as const satisfies Record<string, number>
+
+type StreamTypes = typeof STREAM_TYPES
+
+export type StreamTypeName = keyof StreamTypes
+
+export type StreamTypeCode = StreamTypes[StreamTypeName]
+
+export type StreamType = StreamTypeCode | StreamTypeName

src/components/services/encoding.ts

@@ -0,0 +1,89 @@
+import { type CAR, CARFactory, CarBlock } from "cartonne";
+import { type Codec } from "codeco";
+import { SignedEvent } from "./events";
+import { DagJWS } from "@didtools/codecs";
+import * as dagJson from "@ipld/dag-json";
+import * as dagJose from "dag-jose";
+import { bases } from "multiformats/basics";
+import { CID } from "multiformats/cid";
+import { toString as bytesToString, fromString } from "uint8arrays";
+import { sha256 } from "multihashes-sync/sha2";
+
+export const MAX_BLOCK_SIZE = 256000; // 256 KB
+
+export function base64urlToJSON<T = Record<string, unknown>>(value: string): T {
+  return JSON.parse(bytesToString(fromString(value, "base64url")));
+}
+
+/**
+ * Restricts block size to MAX_BLOCK_SIZE.
+ *
+ * @param block - Uint8Array of IPLD block
+ * @param cid - Commit CID
+ */
+export function restrictBlockSize(block: Uint8Array, cid: CID): void {
+  const size = block.byteLength;
+  if (size > MAX_BLOCK_SIZE) {
+    throw new Error(
+      `${cid} commit size ${size} exceeds the maximum block size of ${MAX_BLOCK_SIZE}`,
+    );
+  }
+}
+
+const carFactory = new CARFactory();
+carFactory.codecs.add(dagJose);
+carFactory.codecs.add(dagJson);
+carFactory.hashers.add(sha256);
+
+export type Base = keyof typeof bases;
+
+export const DEFAULT_BASE: Base = "base64url";
+
+export function signedEventToCAR(event: SignedEvent): CAR {
+  const { jws, linkedBlock, cacaoBlock } = event;
+  const car = carFactory.build();
+
+  // if cacao is present, put it into ipfs dag
+  if (cacaoBlock != null) {
+    const header = base64urlToJSON<{ cap: string }>(
+      jws.signatures[0]!.protected,
+    );
+    const capCID = CID.parse(header.cap.replace("ipfs://", ""));
+    car.blocks.put(new CarBlock(capCID, cacaoBlock));
+    restrictBlockSize(cacaoBlock, capCID);
+  }
+
+  const payloadCID = jws.link;
+  if (payloadCID != null) {
+    // Encode payload
+    car.blocks.put(new CarBlock(payloadCID, linkedBlock));
+    restrictBlockSize(linkedBlock, payloadCID);
+  }
+
+  // Encode JWS itself
+  const cid = car.put(jws, {
+    codec: "dag-jose",
+    hasher: "sha2-256",
+    isRoot: true,
+  });
+  // biome-ignore lint/style/noNonNullAssertion: added to CAR file right before
+  const cidBlock = car.blocks.get(cid)!.payload;
+  restrictBlockSize(cidBlock, cid);
+
+  return car;
+}
+
+export function encodeEventToCAR(codec: Codec<unknown>, event: unknown): CAR {
+  const car = carFactory.build();
+  const cid = car.put(codec.encode(event), { isRoot: true });
+  // biome-ignore lint/style/noNonNullAssertion: added to CAR file right before
+  const cidBlock = car.blocks.get(cid)!.payload;
+  restrictBlockSize(cidBlock, cid);
+  return car;
+}
+
+export function eventToCAR(codec: Codec<unknown>, event: unknown): CAR {
+  return SignedEvent.is(event)
+    ? signedEventToCAR(event)
+    : encodeEventToCAR(codec, event);
+}