diff --git a/packages/db/config/config.ts b/packages/db/config/config.ts new file mode 100644 index 00000000..dcde336d --- /dev/null +++ b/packages/db/config/config.ts @@ -0,0 +1,203 @@ +import { z } from "zod"; + +import { ConfigType, ConfigValue, InferenceProviderEnum } from "./configValue"; + +export const SectionSymbol = Symbol("section"); + +export interface SectionInformation { + name: string; +} + +export interface ConfigSubSection { + [SectionSymbol]: SectionInformation; + [configValue: string]: ConfigValue; +} + +export enum ConfigSectionName { + GENERAL_CONFIG = "generalConfig", + CRAWLER_CONFIG = "crawlerConfig", + INFERENCE_CONFIG = "inferenceConfig", +} + +export enum ConfigKeys { + DISABLE_SIGNUPS = "DISABLE_SIGNUPS", + MAX_ASSET_SIZE_MB = "MAX_ASSET_SIZE_MB", + DISABLE_NEW_RELEASE_CHECK = "DISABLE_NEW_RELEASE_CHECK", + CRAWLER_DOWNLOAD_BANNER_IMAGE = "CRAWLER_DOWNLOAD_BANNER_IMAGE", + CRAWLER_STORE_SCREENSHOT = "CRAWLER_STORE_SCREENSHOT", + CRAWLER_FULL_PAGE_SCREENSHOT = "CRAWLER_FULL_PAGE_SCREENSHOT", + CRAWLER_FULL_PAGE_ARCHIVE = "CRAWLER_FULL_PAGE_ARCHIVE", + CRAWLER_JOB_TIMEOUT_SEC = "CRAWLER_JOB_TIMEOUT_SEC", + CRAWLER_NAVIGATE_TIMEOUT_SEC = "CRAWLER_NAVIGATE_TIMEOUT_SEC", + INFERENCE_PROVIDER = "INFERENCE_PROVIDER", + OPENAI_API_KEY = "OPENAI_API_KEY", + OPENAI_BASE_URL = "OPENAI_BASE_URL", + OLLAMA_BASE_URL = "OLLAMA_BASE_URL", + INFERENCE_TEXT_MODEL = "INFERENCE_TEXT_MODEL", + INFERENCE_IMAGE_MODEL = "INFERENCE_IMAGE_MODEL", + INFERENCE_LANG = "INFERENCE_LANG", +} + +export type ServerConfig = Record; + +export const serverConfig: ServerConfig = { + [ConfigSectionName.GENERAL_CONFIG]: { + [SectionSymbol]: { + name: "General Config", + }, + [ConfigKeys.DISABLE_SIGNUPS]: new ConfigValue({ + key: ConfigKeys.DISABLE_SIGNUPS, + name: "Disable Signups", + type: ConfigType.BOOLEAN, + defaultValue: false, + validator: z.boolean(), + }), + [ConfigKeys.MAX_ASSET_SIZE_MB]: new ConfigValue({ + key: ConfigKeys.MAX_ASSET_SIZE_MB, + name: "Maximum Asset size(MB)", + type: ConfigType.NUMBER, + defaultValue: 4, + validator: z.coerce.number().positive(), + }), + [ConfigKeys.DISABLE_NEW_RELEASE_CHECK]: new ConfigValue({ + key: ConfigKeys.DISABLE_NEW_RELEASE_CHECK, + name: "Disable new release check", + type: ConfigType.BOOLEAN, + defaultValue: false, + validator: z.boolean(), + }), + }, + [ConfigSectionName.CRAWLER_CONFIG]: { + [SectionSymbol]: { + name: "Crawler Config", + }, + [ConfigKeys.CRAWLER_DOWNLOAD_BANNER_IMAGE]: new ConfigValue({ + key: ConfigKeys.CRAWLER_DOWNLOAD_BANNER_IMAGE, + name: "Download Banner Image", + type: ConfigType.BOOLEAN, + defaultValue: true, + validator: z.boolean(), + }), + [ConfigKeys.CRAWLER_STORE_SCREENSHOT]: new ConfigValue({ + key: ConfigKeys.CRAWLER_STORE_SCREENSHOT, + name: "Disable screenshots", + type: ConfigType.BOOLEAN, + defaultValue: true, + validator: z.boolean(), + }), + [ConfigKeys.CRAWLER_FULL_PAGE_SCREENSHOT]: new ConfigValue({ + key: ConfigKeys.CRAWLER_FULL_PAGE_SCREENSHOT, + name: "Store full page screenshots", + type: ConfigType.BOOLEAN, + defaultValue: false, + validator: z.boolean(), + dependsOn: [ConfigKeys.CRAWLER_STORE_SCREENSHOT], + renderIf: (value) => { + return !value; + }, + }), + [ConfigKeys.CRAWLER_FULL_PAGE_ARCHIVE]: new ConfigValue({ + key: ConfigKeys.CRAWLER_FULL_PAGE_ARCHIVE, + name: "Store full page archive", + type: ConfigType.BOOLEAN, + defaultValue: false, + validator: z.boolean(), + }), + [ConfigKeys.CRAWLER_JOB_TIMEOUT_SEC]: new ConfigValue({ + key: ConfigKeys.CRAWLER_JOB_TIMEOUT_SEC, + name: "Job Timeout (sec)", + type: ConfigType.NUMBER, + defaultValue: 60, + validator: z.coerce.number().positive(), + }), + [ConfigKeys.CRAWLER_NAVIGATE_TIMEOUT_SEC]: new ConfigValue({ + key: ConfigKeys.CRAWLER_NAVIGATE_TIMEOUT_SEC, + name: "Navigate Timeout (sec)", + type: ConfigType.NUMBER, + defaultValue: 30, + validator: z.coerce.number().positive(), + }), + }, + [ConfigSectionName.INFERENCE_CONFIG]: { + [SectionSymbol]: { + name: "Inference Config", + }, + [ConfigKeys.INFERENCE_PROVIDER]: new ConfigValue({ + key: ConfigKeys.INFERENCE_PROVIDER, + name: "Inference Provider", + type: ConfigType.INFERENCE_PROVIDER_ENUM, + defaultValue: InferenceProviderEnum.DISABLED, + validator: [ + z.literal(InferenceProviderEnum.DISABLED), + z.literal(InferenceProviderEnum.OPEN_AI), + z.literal(InferenceProviderEnum.OLLAMA), + ], + }), + [ConfigKeys.OPENAI_API_KEY]: new ConfigValue({ + key: ConfigKeys.OPENAI_API_KEY, + name: "OpenAPI Key", + type: ConfigType.PASSWORD, + defaultValue: "", + validator: z.string(), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value === InferenceProviderEnum.OPEN_AI; + }, + }), + [ConfigKeys.OPENAI_BASE_URL]: new ConfigValue({ + key: ConfigKeys.OPENAI_BASE_URL, + name: "OpenAPI base URL", + type: ConfigType.URL, + defaultValue: "", + validator: z.string().url(), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value === InferenceProviderEnum.OPEN_AI; + }, + }), + [ConfigKeys.OLLAMA_BASE_URL]: new ConfigValue({ + key: ConfigKeys.OLLAMA_BASE_URL, + name: "Ollama base URL", + type: ConfigType.URL, + defaultValue: "", + validator: z.string().url(), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value === InferenceProviderEnum.OLLAMA; + }, + }), + [ConfigKeys.INFERENCE_TEXT_MODEL]: new ConfigValue({ + key: ConfigKeys.INFERENCE_TEXT_MODEL, + name: "Inference text model", + type: ConfigType.STRING, + defaultValue: "gpt-3.5-turbo-0125", + validator: z.string().min(1), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value !== InferenceProviderEnum.DISABLED; + }, + }), + [ConfigKeys.INFERENCE_IMAGE_MODEL]: new ConfigValue({ + key: ConfigKeys.INFERENCE_IMAGE_MODEL, + name: "Inference image model", + type: ConfigType.STRING, + defaultValue: "gpt-4o-2024-05-13", + validator: z.string().min(1), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value !== InferenceProviderEnum.DISABLED; + }, + }), + [ConfigKeys.INFERENCE_LANG]: new ConfigValue({ + key: ConfigKeys.INFERENCE_LANG, + name: "Inference language", + type: ConfigType.STRING, + defaultValue: "english", + validator: z.string().min(1), + dependsOn: [ConfigKeys.INFERENCE_PROVIDER], + renderIf: (value) => { + return value !== InferenceProviderEnum.DISABLED; + }, + }), + }, +}; diff --git a/packages/db/config/configDatabaseUtils.ts b/packages/db/config/configDatabaseUtils.ts new file mode 100644 index 00000000..2b95e912 --- /dev/null +++ b/packages/db/config/configDatabaseUtils.ts @@ -0,0 +1,102 @@ +import { eq } from "drizzle-orm"; + +import { db } from "../index"; +import { config } from "../schema"; +import { ConfigType, ConfigTypeMap, ConfigValue } from "./configValue"; + +/** + * @returns the value only from the database without taking the value from the environment into consideration. Undefined if it does not exist + */ +export async function getConfigValueFromDB( + configValue: ConfigValue, +): Promise { + const rows = await db + .select() + .from(config) + .where(eq(config.key, configValue.key)); + if (rows.length === 0) { + return configValue.defaultValue; + } + return parseValue(configValue, rows[0].value); +} + +/** + * @returns the value from the environment variable. Undefined if it does not exist. + */ +export function getConfigValueFromEnv( + configValue: ConfigValue, +): ConfigTypeMap[T] | undefined { + const environmentValue = process.env[configValue.key]; + if (environmentValue === undefined) { + return void 0; + } + return parseValue(configValue, environmentValue); +} + +/** + * @returns the value of the config, considering the database, the environment variable and the default value. Will always return a value + */ +export async function getConfigValue( + configValue: ConfigValue, +): Promise { + const dbValue = await getConfigValueFromDB(configValue); + if (dbValue) { + return dbValue; + } + const envValue = getConfigValueFromEnv(configValue); + if (envValue) { + return envValue; + } + return configValue.defaultValue; +} + +export async function storeConfigValue( + configValue: ConfigValue, + valueToStore: ConfigTypeMap[T], +): Promise { + const value = valueToStore.toString(); + await db + .insert(config) + .values({ key: configValue.key, value }) + .onConflictDoUpdate({ target: config.key, set: { value } }) + .execute(); +} + +export async function deleteConfigValue( + configValue: ConfigValue, +): Promise { + await db.delete(config).where(eq(config.key, configValue.key)); +} + +/** + * Parses a value based on the configValue configuration + * @param configValue the configValue that is being parsed + * @param value the value to parse + */ +function parseValue( + configValue: ConfigValue, + value: string, +): ConfigTypeMap[T] { + if (Array.isArray(configValue.validator)) { + for (const validator of configValue.validator) { + const parsed = validator.safeParse(value); + if (parsed.success) { + return parsed.data as ConfigTypeMap[T]; + } + } + throw new Error( + `"${configValue.key}" contains an invalid value: "${value}"`, + ); + } + // Zod parsing for booleans is considering everything truthy as true, so it needs special handling + if (configValue.type === ConfigType.BOOLEAN) { + return (value === "true") as ConfigTypeMap[T]; + } + const parsed = configValue.validator.safeParse(value); + if (!parsed.success) { + throw new Error( + `"${configValue.key}" contains an invalid value: "${value}"`, + ); + } + return parsed.data as ConfigTypeMap[T]; +} diff --git a/packages/db/config/configSchemaUtils.ts b/packages/db/config/configSchemaUtils.ts new file mode 100644 index 00000000..a5d0b441 --- /dev/null +++ b/packages/db/config/configSchemaUtils.ts @@ -0,0 +1,136 @@ +import { z, ZodLiteral, ZodTypeAny } from "zod"; + +import { ConfigKeys, ConfigSectionName, serverConfig } from "./config"; + +/* + * A general description of what is happening here: + * The serverConfig contains the definition of all the config values possible and how they depend on each other: + * * fullscreen screenshots only make sense if screenshot taking is turned on. + * * when inference is turned off, there is no need to provide Ollama/OpenAPI information + * + * To properly validate this with Zod, we need to generate different schemas for different scenarios. + * This file generates those schemas, by figuring out the dependencies between config flags, then generating the permutations of the values a config can take on + * and then using this information to check which fields of the config would be rendered if a config flag has a certain value. + * Those permutations are gathered up and the zod schemas are returned as a union for the UI to run the validations against. + * + * Sample: + * * There are flags "a" and "b", which are both boolean, but "b" only makes sense if "a" is "true". + * * First we figure out that b depends on a + * * We figure out that a can take value "true" and "false" + * * The permutations are simply "true" and "false" + * * We then check which fields would be rendered if: + * * "a" is "true" --> "a" and "b" --> schema 1 + * * "a" is "false" --> "a" --> schema 2 + * * These 2 schemas are returned with z.union([schema1, schema2]) and then the validations can be performed + */ + +type AllowedMultiplications = ZodLiteral[] | ZodLiteral[]; +type Multiplicators = Record; +type Permutations = Record< + ConfigKeys, + ZodLiteral | ZodLiteral +>; + +/** + * Turns zod validators into their separate possible values they can take on + * @param validator the validator from a ConfigValue to transform + */ +function mapValidatorToValues( + validator: ZodTypeAny | ZodLiteral[], +): AllowedMultiplications { + if (Array.isArray(validator)) { + return validator; + } + return [z.literal(true), z.literal(false)]; +} + +/** + * @param input an object where the key is the config that can take multiple values and the value is the list of possible values for this key. + * @returns an array with all the different permutations of the keys and values provided + */ +function calculatePermutations(input: Partial): Permutations[] { + const keys: ConfigKeys[] = Object.keys(input) as ConfigKeys[]; + if (keys.length === 0) { + return [{} as Permutations]; + } + const [firstKey, ...restKeys] = keys; + const restInput = restKeys.reduce( + (obj, key) => ({ ...obj, [key]: input[key] }), + {} as Multiplicators, + ); + const restPermutations = calculatePermutations(restInput); + return input[firstKey]!.flatMap((value) => + restPermutations.map((permutation) => ({ + [firstKey]: value, + ...permutation, + })), + ); +} + +/** + * @param configSectionName the name of the configSection to check for Dependencies + * @returns an object containing all the config values another config value is depending on and the values the config can take on + */ +function calculateDependencies( + configSectionName: ConfigSectionName, +): Partial { + const configSection = serverConfig[configSectionName]; + const dependencies: Partial = {}; + for (const configValue of Object.values(configSection)) { + if (configValue.dependsOn.length) { + for (const dependentConfigKey of configValue.dependsOn) { + dependencies[dependentConfigKey] = mapValidatorToValues( + configSection[dependentConfigKey].validator, + ); + } + } + } + return dependencies; +} + +/** + * @param permutation The permutation to convert + * @returns a new record where the zod validator is replaced with the actual value it can take on + */ +function convertPermutation( + permutation: Partial, +): Record { + const result: Record = {}; + for (const key in permutation) { + result[key] = permutation[key as ConfigKeys]!.value; + } + return result; +} + +/** + * @param configSectionName the name of the section to create the schema for + * @returns the config schema for the passed in config section + */ +export function getConfigSchema( + configSectionName: ConfigSectionName, +): z.ZodType { + const dependencies = calculateDependencies(configSectionName); + const permutations = calculatePermutations(dependencies); + + const configSubSection = serverConfig[configSectionName]; + const results: Record[] = []; + for (const permutation of permutations) { + const convertedPermutation = convertPermutation(permutation); + const result: Record = {}; + for (const configValue of Object.values(configSubSection)) { + if (configValue.shouldRender(convertedPermutation)) { + if (Object.keys(permutation).includes(configValue.key)) { + result[configValue.key] = permutation[configValue.key]; + } else { + result[configValue.key] = configValue.validator as ZodTypeAny; + } + } + } + results.push(result); + } + if (results.length === 1) { + return z.object(results[0]); + } + // @ts-expect-error -- z.union is not type-able with dynamic values right now, so it does not work + return z.union(results.map((result) => z.object(result))); +} diff --git a/packages/db/config/configValue.ts b/packages/db/config/configValue.ts new file mode 100644 index 00000000..7f25a453 --- /dev/null +++ b/packages/db/config/configValue.ts @@ -0,0 +1,75 @@ +import { ZodLiteral, ZodTypeAny } from "zod"; + +import { ConfigKeys } from "./config"; + +export enum ConfigType { + BOOLEAN, + STRING, + PASSWORD, + URL, + NUMBER, + INFERENCE_PROVIDER_ENUM, +} + +export enum InferenceProviderEnum { + DISABLED = "disabled", + OPEN_AI = "openai", + OLLAMA = "ollama", +} + +export interface ConfigTypeMap { + [ConfigType.BOOLEAN]: boolean; + [ConfigType.STRING]: string; + [ConfigType.PASSWORD]: string; + [ConfigType.URL]: string; + [ConfigType.NUMBER]: number; + [ConfigType.INFERENCE_PROVIDER_ENUM]: string; +} + +export type ConfigTypes = boolean | number | string; + +export interface ConfigProperties { + key: ConfigKeys; + name: string; + type: T; + defaultValue: ConfigTypeMap[T]; + validator: ZodTypeAny | ZodLiteral[]; + dependsOn?: ConfigKeys[]; + renderIf?: (value: ConfigTypes) => boolean; +} + +export class ConfigValue { + key: ConfigKeys; + name: string; + type: T; + defaultValue: ConfigTypeMap[T]; + validator: ZodTypeAny | ZodLiteral[]; + value: ConfigTypeMap[T] | undefined; + dependsOn: ConfigKeys[]; + renderIf?: (value: ConfigTypes) => boolean; + + constructor(props: ConfigProperties) { + this.key = props.key; + this.name = props.name; + this.type = props.type; + this.defaultValue = props.defaultValue; + this.validator = props.validator; + this.dependsOn = props.dependsOn ?? []; + this.renderIf = props.renderIf; + } + + shouldRender(currentValues: Record): boolean { + if (this.renderIf) { + for (const key of Object.keys(currentValues)) { + const configKey = key as ConfigKeys; + if ( + this.dependsOn.includes(configKey) && + !this.renderIf(currentValues[configKey]) + ) { + return false; + } + } + } + return true; + } +} diff --git a/packages/shared/types/tags.ts b/packages/shared/types/tags.ts index c9fe2a93..a3c0a339 100644 --- a/packages/shared/types/tags.ts +++ b/packages/shared/types/tags.ts @@ -13,6 +13,6 @@ export const zGetTagResponseSchema = z.object({ id: z.string(), name: z.string(), count: z.number(), - countAttachedBy: z.record(zAttachedByEnumSchema, z.number()), + countAttachedBy: z.record(z.string(), z.number()), }); export type ZGetTagResponse = z.infer;