Skip to content

Commit

Permalink
Add ability to add models from the deployed site (#407)
Browse files Browse the repository at this point in the history
  • Loading branch information
joeyballentine authored May 9, 2024
1 parent f2d3332 commit d3b7860
Show file tree
Hide file tree
Showing 8 changed files with 273 additions and 82 deletions.
68 changes: 2 additions & 66 deletions scripts/validate-db.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,7 @@
import fs from 'fs/promises';
import path from 'path';
import { MODEL_PROPS, validateType } from '../src/lib/model-props';
import { ArchId, ModelId, TagId, UserId } from '../src/lib/schema';
import { canonicalizeModelId } from '../src/lib/schema-util';
import { fileApi } from '../src/lib/server/file-data';
import { typedEntries } from '../src/lib/util';

interface Report {
message: string;
fix?: () => Promise<void>;
}
import { Report, validateModel } from '../src/lib/validate-model';

const getAllFiles = async (dir: string): Promise<string[]> => {
const names = await fs.readdir(dir);
Expand Down Expand Up @@ -37,63 +29,7 @@ const getReports = async (): Promise<Report[]> => {

const errors: Report[] = [];
for (const [modelId, model] of modelData) {
const report = (message: string, fix?: () => Promise<void>) =>
errors.push({ message: `Model ${modelId}: ${message}`, fix });

if (modelId.startsWith(`${model.scale}x`)) {
const expected = canonicalizeModelId(modelId);
if (expected !== modelId) {
report(`Model ID should be ${expected}`, () => fileApi.models.changeId(modelId, expected));
}
} else {
report(`Model ID must start with scale`, () =>
fileApi.models.changeId(modelId, `${model.scale}x-${modelId.replace(/^\d+x-?/, '')}` as ModelId)
);
}

if (model.thumbnail || model.images.some((image) => image.thumbnail)) {
report(`Thumbnails are automatically generated and should not appear in the database`, async () => {
const model = await fileApi.models.get(modelId);
delete model.thumbnail;
for (const image of model.images) {
delete image.thumbnail;
}
await fileApi.models.update([[modelId, model]]);
});
}

for (const [key, prop] of typedEntries(MODEL_PROPS)) {
const value = model[key];

if (value === null || value === undefined) {
if (!prop.optional) report(`Missing required property '${key}'`);
continue;
}

const error = validateType(value, prop, `'${key}'`, {
isValidModelId: (id) => modelData.has(id as ModelId),
isValidUserId: (id) => userData.has(id as UserId),
isValidTagId: (id) => tagData.has(id as TagId),
isValidArchitectureId: (id) => architectureData.has(id as ArchId),
});
if (error) {
const { message, fix } = error;
report(
message,
fix &&
(async () => {
const model = await fileApi.models.get(modelId);
const newValue = fix();
if (newValue === undefined) {
delete model[key];
} else {
model[key] = newValue as never;
}
await fileApi.models.update([[modelId, model]]);
})
);
}
}
errors.push(...validateModel(model, modelId, modelData, architectureData, tagData, userData, fileApi));
}

const jsonFiles = (await getAllFiles('data/')).filter((file) => file.endsWith('.json'));
Expand Down
20 changes: 20 additions & 0 deletions src/lib/data-session.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
export const createMapFromSessionStorage = <Id, Value>(key: string, map = new Map<Id, Value>()): Map<Id, Value> => {
const cachedItem = sessionStorage.getItem(key);
if (cachedItem) {
const cachedMap = new Map<Id, Value>(JSON.parse(cachedItem || '[]') as []);
for (const id of cachedMap.keys()) {
// Only add the value if it doesn't already exist
// AKA only add user-made data, update everything else from api
if (!map.has(id)) {
const value = cachedMap.get(id);
if (value) {
map.set(id, value);
}
}
}
}
window.addEventListener('beforeunload', () => {
sessionStorage.setItem(key, JSON.stringify([...map]));
});
return map;
};
9 changes: 6 additions & 3 deletions src/lib/hooks/use-web-api.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import React, { createContext, useCallback, useContext, useEffect, useState } from 'react';
import { DBApi } from '../data-api';
import { IS_DEPLOYED } from '../site-data';
import { noop } from '../util';
import { getWebApi } from '../web-api';

Expand All @@ -26,10 +27,12 @@ export function WebApiProvider({ children }: React.PropsWithChildren<unknown>) {

export type UseWebApi = { webApi: DBApi; editMode: true } | { webApi: undefined; editMode: false };

export function useWebApi(): UseWebApi {
export function useWebApi(override = false): UseWebApi {
const { webApi, enabled } = useContext(WebApiContext);

return webApi && enabled ? { webApi, editMode: true } : { webApi: undefined, editMode: false };
return webApi && enabled && (!IS_DEPLOYED || override)
? { webApi, editMode: true }
: { webApi: undefined, editMode: false };
}

export interface UseEditModeToggle {
Expand All @@ -41,7 +44,7 @@ export interface UseEditModeToggle {
export function useEditModeToggle(): UseEditModeToggle {
const { webApi, enabled, toggleEnabled } = useContext(WebApiContext);

const editModeAvailable = webApi !== undefined;
const editModeAvailable = webApi !== undefined && !IS_DEPLOYED;

return {
editModeAvailable,
Expand Down
80 changes: 80 additions & 0 deletions src/lib/validate-model.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { DBApi } from './data-api';
import { MODEL_PROPS, validateType } from './model-props';
import { Arch, ArchId, Model, ModelId, Tag, TagId, User, UserId } from './schema';
import { canonicalizeModelId } from './schema-util';
import { typedEntries } from './util';

export interface Report {
message: string;
fix?: () => Promise<void>;
}

export const validateModel = (
model: Model,
modelId: ModelId,
modelData: ReadonlyMap<ModelId, Model>,
architectureData: ReadonlyMap<ArchId, Arch>,
tagData: ReadonlyMap<TagId, Tag>,
userData: ReadonlyMap<UserId, User>,
api: DBApi
): Report[] => {
const errors: Report[] = [];
const report = (message: string, fix?: () => Promise<void>) =>
errors.push({ message: `Model ${modelId}: ${message}`, fix });

if (modelId.startsWith(`${model.scale}x`)) {
const expected = canonicalizeModelId(modelId);
if (expected !== modelId) {
report(`Model ID should be ${expected}`, () => api.models.changeId(modelId, expected));
}
} else {
report(`Model ID must start with scale`, () =>
api.models.changeId(modelId, `${model.scale}x-${modelId.replace(/^\d+x-?/, '')}` as ModelId)
);
}

if (model.thumbnail || model.images.some((image) => image.thumbnail)) {
report(`Thumbnails are automatically generated and should not appear in the database`, async () => {
const model = await api.models.get(modelId);
delete model.thumbnail;
for (const image of model.images) {
delete image.thumbnail;
}
await api.models.update([[modelId, model]]);
});
}

for (const [key, prop] of typedEntries(MODEL_PROPS)) {
const value = model[key];

if (value === null || value === undefined) {
if (!prop.optional) report(`Missing required property '${key}'`);
continue;
}

const error = validateType(value, prop, `'${key}'`, {
isValidModelId: (id) => modelData.has(id as ModelId),
isValidUserId: (id) => userData.has(id as UserId),
isValidTagId: (id) => tagData.has(id as TagId),
isValidArchitectureId: (id) => architectureData.has(id as ArchId),
});
if (error) {
const { message, fix } = error;
report(
message,
fix &&
(async () => {
const model = await api.models.get(modelId);
const newValue = fix();
if (newValue === undefined) {
delete model[key];
} else {
model[key] = newValue as never;
}
await api.models.update([[modelId, model]]);
})
);
}
}
return errors;
};
52 changes: 45 additions & 7 deletions src/lib/web-api.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { CollectionApi, DBApi, notifyOnWrite } from './data-api';
import { CollectionApi, DBApi, MapCollection, notifyOnWrite } from './data-api';
import { JsonApiCollection, JsonApiRequestHandler, JsonRequest, JsonResponse, Method } from './data-json-api';
import { IS_DEPLOYED } from './site-data';
import { createMapFromSessionStorage } from './data-session';
import { Arch, ArchId, Model, ModelId, Tag, TagCategory, TagCategoryId, TagId, User, UserId } from './schema';
import { IS_DEPLOYED, SITE_URL } from './site-data';
import { delay, lazy, noop } from './util';

const updateListeners = new Set<() => void>();
Expand Down Expand Up @@ -69,19 +71,55 @@ function createWebCollection<Id, Value>(path: string): CollectionApi<Id, Value>
},
});
}
export const getWebApi = lazy(async (): Promise<DBApi | undefined> => {
if (IS_DEPLOYED) {
// we only have API access locally
return Promise.resolve(undefined);

async function createMapCollection<Id, Value>(path: string): Promise<CollectionApi<Id, Value>> {
const url = new URL(path, SITE_URL).href;
const res = await fetch(url);
if (res.status !== 200) {
throw new Error(res.statusText);
}
const map = new Map<Id, Value>();
const data = (await res.json()) as Record<string, Value>[];
for (const [id, value] of Object.entries(data)) {
map.set(id as Id, value as Value);
}
return notifyOnWrite(new MapCollection(createMapFromSessionStorage(path, map)), {
after: () => {
mutationCounter++;
notifyListeners();
},
});
}

const getDbAPI = async (): Promise<DBApi> => {
if (IS_DEPLOYED) {
const [models, users, tags, tagCategories, architectures] = await Promise.all([
createMapCollection<ModelId, Model>('/api/v1/models.json'),
createMapCollection<UserId, User>('/api/v1/users.json'),
createMapCollection<TagId, Tag>('/api/v1/tags.json'),
createMapCollection<TagCategoryId, TagCategory>('/api/v1/tagCategories.json'),
createMapCollection<ArchId, Arch>('/api/v1/architectures.json'),
]);

const webApi: DBApi = {
return {
models,
users,
tags,
tagCategories,
architectures,
};
}
return {
models: createWebCollection('/api/models'),
users: createWebCollection('/api/users'),
tags: createWebCollection('/api/tags'),
tagCategories: createWebCollection('/api/tag-categories'),
architectures: createWebCollection('/api/architectures'),
};
};

export const getWebApi = lazy(async (): Promise<DBApi | undefined> => {
const webApi = await getDbAPI();

// we do an empty update to test the waters
return webApi.tags.update([]).then(
Expand Down
12 changes: 10 additions & 2 deletions src/pages/add-model.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { hashSha256 } from '../lib/model-files';
import { ParseResult, parseDiscordMessage } from '../lib/parse-discord-message';
import { Arch, ArchId, Model, ModelId, Tag, TagId } from '../lib/schema';
import { canonicalizeModelId } from '../lib/schema-util';
import { IS_DEPLOYED } from '../lib/site-data';
import type { ResponseJson } from './api/pth-metadata';

function getCommonPretrained(modelData: ReadonlyMap<ModelId, Model>): ModelId[] {
Expand Down Expand Up @@ -115,7 +116,7 @@ function PageContent() {
const { archData } = useArchitectures();
const { tagData } = useTags();
const router = useRouter();
const { webApi, editMode } = useWebApi();
const { webApi, editMode } = useWebApi(IS_DEPLOYED);

const [processing, setProcessing] = useState(false);

Expand All @@ -124,7 +125,7 @@ function PageContent() {
const [partialId, setPartialId] = useState<string>();
const [scale, setScale] = useState(1);
const [scaleDefinedBy, setScaleDefinedBy] = useState<string>();
const fullId = canonicalizeModelId(`${scale}x-${partialId ?? name}`);
let fullId = canonicalizeModelId(`${scale}x-${partialId ?? name}`);
const partialIdFromFull = fullId.slice(`${scale}x-`.length);

const [hasMainPth, setHasMainPth] = useState(false);
Expand Down Expand Up @@ -255,6 +256,13 @@ function PageContent() {
model.tags = guessTags(model, tagData);

setProcessing(true);

if (IS_DEPLOYED) {
sessionStorage.setItem('dummy-modelId', fullId);
sessionStorage.setItem('dummy-model', JSON.stringify(model));
fullId = 'OMDB_ADDMODEL_DUMMY' as ModelId;
}

await webApi.models.update([[fullId, model]]);

// fetch before navigating to ensure the model page is available
Expand Down
32 changes: 32 additions & 0 deletions src/pages/models/OMDB_ADDMODEL_DUMMY.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { GetStaticProps } from 'next';
import { useEffect, useState } from 'react';
import { Model, ModelId } from '../../lib/schema';
import ModelsPage from './[id]';

const modelId = 'OMDB_ADDMODEL_DUMMY' as ModelId;

export default function Page() {
const [model, setModel] = useState<Model | null>(null);

useEffect(() => {
const model = JSON.parse(sessionStorage.getItem('dummy-model') ?? '{}') as Model;
setModel(model);
}, []);

if (!model) return null;

return (
<div>
<ModelsPage
editModeOverride
modelData={{ [modelId]: model }}
modelId={modelId}
similar={[]}
/>
</div>
);
}

export const getStaticProps: GetStaticProps = () => {
return { props: {} };
};
Loading

0 comments on commit d3b7860

Please sign in to comment.