Admin settings config

packages/db/config/config.ts

@@ -0,0 +1,203 @@
+import { z } from "zod";
+
+import { ConfigType, ConfigValue, InferenceProviderEnum } from "./configValue";
+
+export const SectionSymbol = Symbol("section");
+
+export interface SectionInformation {
+  name: string;
+}
+
+export interface ConfigSubSection {
+  [SectionSymbol]: SectionInformation;
+  [configValue: string]: ConfigValue<ConfigType>;
+}
+
+export enum ConfigSectionName {
+  GENERAL_CONFIG = "generalConfig",
+  CRAWLER_CONFIG = "crawlerConfig",
+  INFERENCE_CONFIG = "inferenceConfig",
+}
+
+export enum ConfigKeys {
+  DISABLE_SIGNUPS = "DISABLE_SIGNUPS",
+  MAX_ASSET_SIZE_MB = "MAX_ASSET_SIZE_MB",
+  DISABLE_NEW_RELEASE_CHECK = "DISABLE_NEW_RELEASE_CHECK",
+  CRAWLER_DOWNLOAD_BANNER_IMAGE = "CRAWLER_DOWNLOAD_BANNER_IMAGE",
+  CRAWLER_STORE_SCREENSHOT = "CRAWLER_STORE_SCREENSHOT",
+  CRAWLER_FULL_PAGE_SCREENSHOT = "CRAWLER_FULL_PAGE_SCREENSHOT",
+  CRAWLER_FULL_PAGE_ARCHIVE = "CRAWLER_FULL_PAGE_ARCHIVE",
+  CRAWLER_JOB_TIMEOUT_SEC = "CRAWLER_JOB_TIMEOUT_SEC",
+  CRAWLER_NAVIGATE_TIMEOUT_SEC = "CRAWLER_NAVIGATE_TIMEOUT_SEC",
+  INFERENCE_PROVIDER = "INFERENCE_PROVIDER",
+  OPENAI_API_KEY = "OPENAI_API_KEY",
+  OPENAI_BASE_URL = "OPENAI_BASE_URL",
+  OLLAMA_BASE_URL = "OLLAMA_BASE_URL",
+  INFERENCE_TEXT_MODEL = "INFERENCE_TEXT_MODEL",
+  INFERENCE_IMAGE_MODEL = "INFERENCE_IMAGE_MODEL",
+  INFERENCE_LANG = "INFERENCE_LANG",
+}
+
+export type ServerConfig = Record<ConfigSectionName, ConfigSubSection>;
+
+export const serverConfig: ServerConfig = {
+  [ConfigSectionName.GENERAL_CONFIG]: {
+    [SectionSymbol]: {
+      name: "General Config",
+    },
+    [ConfigKeys.DISABLE_SIGNUPS]: new ConfigValue({
+      key: ConfigKeys.DISABLE_SIGNUPS,
+      name: "Disable Signups",
+      type: ConfigType.BOOLEAN,
+      defaultValue: false,
+      validator: z.boolean(),
+    }),
+    [ConfigKeys.MAX_ASSET_SIZE_MB]: new ConfigValue({
+      key: ConfigKeys.MAX_ASSET_SIZE_MB,
+      name: "Maximum Asset size(MB)",
+      type: ConfigType.NUMBER,
+      defaultValue: 4,
+      validator: z.coerce.number().positive(),
+    }),
+    [ConfigKeys.DISABLE_NEW_RELEASE_CHECK]: new ConfigValue({
+      key: ConfigKeys.DISABLE_NEW_RELEASE_CHECK,
+      name: "Disable new release check",
+      type: ConfigType.BOOLEAN,
+      defaultValue: false,
+      validator: z.boolean(),
+    }),
+  },
+  [ConfigSectionName.CRAWLER_CONFIG]: {
+    [SectionSymbol]: {
+      name: "Crawler Config",
+    },
+    [ConfigKeys.CRAWLER_DOWNLOAD_BANNER_IMAGE]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_DOWNLOAD_BANNER_IMAGE,
+      name: "Download Banner Image",
+      type: ConfigType.BOOLEAN,
+      defaultValue: true,
+      validator: z.boolean(),
+    }),
+    [ConfigKeys.CRAWLER_STORE_SCREENSHOT]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_STORE_SCREENSHOT,
+      name: "Disable screenshots",
+      type: ConfigType.BOOLEAN,
+      defaultValue: true,
+      validator: z.boolean(),
+    }),
+    [ConfigKeys.CRAWLER_FULL_PAGE_SCREENSHOT]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_FULL_PAGE_SCREENSHOT,
+      name: "Store full page screenshots",
+      type: ConfigType.BOOLEAN,
+      defaultValue: false,
+      validator: z.boolean(),
+      dependsOn: [ConfigKeys.CRAWLER_STORE_SCREENSHOT],
+      renderIf: (value) => {
+        return !value;
+      },
+    }),
+    [ConfigKeys.CRAWLER_FULL_PAGE_ARCHIVE]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_FULL_PAGE_ARCHIVE,
+      name: "Store full page archive",
+      type: ConfigType.BOOLEAN,
+      defaultValue: false,
+      validator: z.boolean(),
+    }),
+    [ConfigKeys.CRAWLER_JOB_TIMEOUT_SEC]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_JOB_TIMEOUT_SEC,
+      name: "Job Timeout (sec)",
+      type: ConfigType.NUMBER,
+      defaultValue: 60,
+      validator: z.coerce.number().positive(),
+    }),
+    [ConfigKeys.CRAWLER_NAVIGATE_TIMEOUT_SEC]: new ConfigValue({
+      key: ConfigKeys.CRAWLER_NAVIGATE_TIMEOUT_SEC,
+      name: "Navigate Timeout (sec)",
+      type: ConfigType.NUMBER,
+      defaultValue: 30,
+      validator: z.coerce.number().positive(),
+    }),
+  },
+  [ConfigSectionName.INFERENCE_CONFIG]: {
+    [SectionSymbol]: {
+      name: "Inference Config",
+    },
+    [ConfigKeys.INFERENCE_PROVIDER]: new ConfigValue({
+      key: ConfigKeys.INFERENCE_PROVIDER,
+      name: "Inference Provider",
+      type: ConfigType.INFERENCE_PROVIDER_ENUM,
+      defaultValue: InferenceProviderEnum.DISABLED,
+      validator: [
+        z.literal(InferenceProviderEnum.DISABLED),
+        z.literal(InferenceProviderEnum.OPEN_AI),
+        z.literal(InferenceProviderEnum.OLLAMA),
+      ],
+    }),
+    [ConfigKeys.OPENAI_API_KEY]: new ConfigValue({
+      key: ConfigKeys.OPENAI_API_KEY,
+      name: "OpenAPI Key",
+      type: ConfigType.PASSWORD,
+      defaultValue: "",
+      validator: z.string(),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value === InferenceProviderEnum.OPEN_AI;
+      },
+    }),
+    [ConfigKeys.OPENAI_BASE_URL]: new ConfigValue({
+      key: ConfigKeys.OPENAI_BASE_URL,
+      name: "OpenAPI base URL",
+      type: ConfigType.URL,
+      defaultValue: "",
+      validator: z.string().url(),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value === InferenceProviderEnum.OPEN_AI;
+      },
+    }),
+    [ConfigKeys.OLLAMA_BASE_URL]: new ConfigValue({
+      key: ConfigKeys.OLLAMA_BASE_URL,
+      name: "Ollama base URL",
+      type: ConfigType.URL,
+      defaultValue: "",
+      validator: z.string().url(),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value === InferenceProviderEnum.OLLAMA;
+      },
+    }),
+    [ConfigKeys.INFERENCE_TEXT_MODEL]: new ConfigValue({
+      key: ConfigKeys.INFERENCE_TEXT_MODEL,
+      name: "Inference text model",
+      type: ConfigType.STRING,
+      defaultValue: "gpt-3.5-turbo-0125",
+      validator: z.string().min(1),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value !== InferenceProviderEnum.DISABLED;
+      },
+    }),
+    [ConfigKeys.INFERENCE_IMAGE_MODEL]: new ConfigValue({
+      key: ConfigKeys.INFERENCE_IMAGE_MODEL,
+      name: "Inference image model",
+      type: ConfigType.STRING,
+      defaultValue: "gpt-4o-2024-05-13",
+      validator: z.string().min(1),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value !== InferenceProviderEnum.DISABLED;
+      },
+    }),
+    [ConfigKeys.INFERENCE_LANG]: new ConfigValue({
+      key: ConfigKeys.INFERENCE_LANG,
+      name: "Inference language",
+      type: ConfigType.STRING,
+      defaultValue: "english",
+      validator: z.string().min(1),
+      dependsOn: [ConfigKeys.INFERENCE_PROVIDER],
+      renderIf: (value) => {
+        return value !== InferenceProviderEnum.DISABLED;
+      },
+    }),
+  },
+};

packages/db/config/configDatabaseUtils.ts

@@ -0,0 +1,102 @@
+import { eq } from "drizzle-orm";
+
+import { db } from "../index";
+import { config } from "../schema";
+import { ConfigType, ConfigTypeMap, ConfigValue } from "./configValue";
+
+/**
+ * @returns the value only from the database without taking the value from the environment into consideration. Undefined if it does not exist
+ */
+export async function getConfigValueFromDB<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+): Promise<ConfigTypeMap[T]> {
+  const rows = await db
+    .select()
+    .from(config)
+    .where(eq(config.key, configValue.key));
+  if (rows.length === 0) {
+    return configValue.defaultValue;
+  }
+  return parseValue(configValue, rows[0].value);
+}
+
+/**
+ * @returns the value from the environment variable. Undefined if it does not exist.
+ */
+export function getConfigValueFromEnv<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+): ConfigTypeMap[T] | undefined {
+  const environmentValue = process.env[configValue.key];
+  if (environmentValue === undefined) {
+    return void 0;
+  }
+  return parseValue(configValue, environmentValue);
+}
+
+/**
+ * @returns the value of the config, considering the database, the environment variable and the default value. Will always return a value
+ */
+export async function getConfigValue<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+): Promise<ConfigTypeMap[T]> {
+  const dbValue = await getConfigValueFromDB(configValue);
+  if (dbValue) {
+    return dbValue;
+  }
+  const envValue = getConfigValueFromEnv(configValue);
+  if (envValue) {
+    return envValue;
+  }
+  return configValue.defaultValue;
+}
+
+export async function storeConfigValue<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+  valueToStore: ConfigTypeMap[T],
+): Promise<void> {
+  const value = valueToStore.toString();
+  await db
+    .insert(config)
+    .values({ key: configValue.key, value })
+    .onConflictDoUpdate({ target: config.key, set: { value } })
+    .execute();
+}
+
+export async function deleteConfigValue<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+): Promise<void> {
+  await db.delete(config).where(eq(config.key, configValue.key));
+}
+
+/**
+ * Parses a value based on the configValue configuration
+ * @param configValue the configValue that is being parsed
+ * @param value the value to parse
+ */
+function parseValue<T extends ConfigType>(
+  configValue: ConfigValue<T>,
+  value: string,
+): ConfigTypeMap[T] {
+  if (Array.isArray(configValue.validator)) {
+    for (const validator of configValue.validator) {
+      const parsed = validator.safeParse(value);
+      if (parsed.success) {
+        return parsed.data as ConfigTypeMap[T];
+      }
+    }
+    throw new Error(
+      `"${configValue.key}" contains an invalid value: "${value}"`,
+    );
+  }
+  // Zod parsing for booleans is considering everything truthy as true, so it needs special handling
+  if (configValue.type === ConfigType.BOOLEAN) {
+    return (value === "true") as ConfigTypeMap[T];
+  }
+  const parsed = configValue.validator.safeParse(value);
+  if (!parsed.success) {
+    throw new Error(
+      `"${configValue.key}" contains an invalid value: "${value}"`,
+    );
+  }
+  return parsed.data as ConfigTypeMap[T];
+}