(core) updates from grist-core

README.md

@@ -275,7 +275,7 @@ Grist can be configured in many ways. Here are the main environment variables it
 | GRIST_ENABLE_REQUEST_FUNCTION      | enables the REQUEST function. This function performs HTTP requests in a similar way to `requests.request`. This function presents a significant security risk, since it can let users call internal endpoints when Grist is available publicly. This function can also cause performance issues. Unset by default.                                            |
 | GRIST_HIDE_UI_ELEMENTS             | comma-separated list of UI features to disable. Allowed names of parts: `helpCenter,billing,templates,createSite,multiSite,multiAccounts,sendToDrive,tutorials,supportGrist`. If a part also exists in GRIST_UI_FEATURES, it will still be disabled.                                                                                                          |
 | GRIST_HOST                         | hostname to use when listening on a port.                                                                                                                                                                                                                                                                                                                     |
-| GRIST_HTTPS_PROXY                  | if set, use this proxy for webhook payload delivery.                                                                                                                                                                                                                                                                                                          |
+| GRIST_HTTPS_PROXY                  | if set, use this proxy for webhook payload delivery or fetching custom widgets repository from url.                                                                                                                                                                                                                                                                                                          |
 | GRIST_ID_PREFIX                    | for subdomains of form o-*, expect or produce o-${GRIST_ID_PREFIX}*.                                                                                                                                                                                                                                                                                          |
 | GRIST_IGNORE_SESSION               | if set, Grist will not use a session for authentication.                                                                                                                                                                                                                                                                                                      |
 | GRIST_INCLUDE_CUSTOM_SCRIPT_URL    | if set, will load the referenced URL in a `<script>` tag on all app pages.                                                                                                                                                                                                                                                                                    |

app/gen-server/lib/DocWorkerMap.ts

@@ -580,8 +580,10 @@ let dummyDocWorkerMap: DummyDocWorkerMap|null = null;
 
 export function getDocWorkerMap(): IDocWorkerMap {
   if (process.env.REDIS_URL) {
+    log.info("Creating Redis-based DocWorker");
     return new DocWorkerMap();
   } else {
+    log.info("Creating local/dummy DocWorker");
     dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap();
     return dummyDocWorkerMap;
   }

app/server/lib/Assistance.ts

@@ -158,8 +158,7 @@ export class OpenAIAssistant implements Assistant {
   private _maxTokens = process.env.ASSISTANT_MAX_TOKENS ?
       parseInt(process.env.ASSISTANT_MAX_TOKENS, 10) : undefined;
 
-  public constructor() {
-    const apiKey = process.env.ASSISTANT_API_KEY || process.env.OPENAI_API_KEY;
+  public constructor(apiKey: string | undefined) {
     const endpoint = process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT;
     if (!apiKey && !endpoint) {
       throw new Error('Please set either OPENAI_API_KEY or ASSISTANT_CHAT_COMPLETION_ENDPOINT');
@@ -485,11 +484,13 @@ class EchoAssistant implements Assistant {
  * Instantiate an assistant, based on environment variables.
  */
 export function getAssistant() {
-  if (process.env.OPENAI_API_KEY === 'test') {
+  const apiKey = process.env.ASSISTANT_API_KEY || process.env.OPENAI_API_KEY;
+
+  if (apiKey === 'test') {
     return new EchoAssistant();
   }
-  if (process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT) {
-    return new OpenAIAssistant();
+  if (apiKey || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT) {
+    return new OpenAIAssistant(apiKey);
   }
   throw new Error('Please set OPENAI_API_KEY or ASSISTANT_CHAT_COMPLETION_ENDPOINT');
 }

app/server/lib/DocManager.ts

@@ -638,6 +638,7 @@ export class DocManager extends EventEmitter {
         throw new Error('Grist docs must be uploaded individually');
       }
       const first = uploadInfo.files[0].origName;
+      log.debug(`DocManager._doImportDoc: Received doc with name ${first}`);
       const ext = extname(first);
       const basename = path.basename(first, ext).trim() || "Untitled upload";
       let id: string;
@@ -662,6 +663,7 @@ export class DocManager extends EventEmitter {
       }
       await options.register?.(id, basename);
       if (ext === '.grist') {
+        log.debug(`DocManager._doImportDoc: Importing .grist doc`);
         // If the import is a grist file, copy it to the docs directory.
         // TODO: We should be skeptical of the upload file to close a possible
         // security vulnerability. See https://phab.getgrist.com/T457.

app/server/lib/HostedMetadataManager.ts

@@ -1,6 +1,9 @@
-import {DocumentMetadata, HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {DocumentMetadata} from 'app/gen-server/lib/homedb/HomeDBManager';
 import log from 'app/server/lib/log';
 
+// Callback that persists the updated metadata to storage for each document.
+export type SaveDocsMetadataFunc = (metadata: { [docId: string]: DocumentMetadata }) => Promise<any>;
+
 /**
  * HostedMetadataManager handles pushing document metadata changes to the Home database when
  * a doc is updated. Currently updates doc updatedAt time and usage.
@@ -29,7 +32,7 @@ export class HostedMetadataManager {
    * Create an instance of HostedMetadataManager.
    * The minPushDelay is the default delay in seconds between metadata pushes to the database.
    */
-  constructor(private _dbManager: HomeDBManager, minPushDelay: number = 60) {
+  constructor(private _saveDocsMetadata: SaveDocsMetadataFunc, minPushDelay: number = 60) {
     this._minPushDelayMs = minPushDelay * 1000;
   }
 
@@ -68,7 +71,7 @@ export class HostedMetadataManager {
   }
 
   public setDocsMetadata(docUpdateMap: {[docId: string]: DocumentMetadata}): Promise<any> {
-    return this._dbManager.setDocsMetadata(docUpdateMap);
+    return this._saveDocsMetadata(docUpdateMap);
   }
 
   /**

app/server/lib/HostedStorageManager.ts

@@ -8,7 +8,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
 import {buildUrlId, parseUrlId} from 'app/common/gristUrls';
 import {KeyedOps} from 'app/common/KeyedOps';
 import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
-import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
 import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
@@ -19,14 +18,15 @@ import {
   ExternalStorageCreator, ExternalStorageSettings,
   Unchanged
 } from 'app/server/lib/ExternalStorage';
-import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
+import {HostedMetadataManager, SaveDocsMetadataFunc} from 'app/server/lib/HostedMetadataManager';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {LogMethods} from "app/server/lib/LogMethods";
 import {fromCallback} from 'app/server/lib/serverUtils';
 import * as fse from 'fs-extra';
 import * as path from 'path';
 import uuidv4 from "uuid/v4";
-import { OpenMode, SQLiteDB } from './SQLiteDB';
+import {OpenMode, SQLiteDB} from './SQLiteDB';
+import {Features} from "app/common/Features";
 
 // Check for a valid document id.
 const docIdRegex = /^[-=_\w~%]+$/;
@@ -52,6 +52,13 @@ function checkValidDocId(docId: string): void {
   }
 }
 
+export interface HostedStorageCallbacks {
+  // Saves the given metadata for the specified documents.
+  setDocsMetadata: SaveDocsMetadataFunc,
+  // Retrieves account features enabled for the given document.
+  getDocFeatures: (docId: string) => Promise<Features | undefined>
+}
+
 export interface HostedStorageOptions {
   secondsBeforePush: number;
   secondsBeforeFirstRetry: number;
@@ -133,7 +140,7 @@ export class HostedStorageManager implements IDocStorageManager {
     private _docWorkerId: string,
     private _disableS3: boolean,
     private _docWorkerMap: IDocWorkerMap,
-    dbManager: HomeDBManager,
+    callbacks: HostedStorageCallbacks,
     createExternalStorage: ExternalStorageCreator,
     options: HostedStorageOptions = defaultOptions
   ) {
@@ -144,7 +151,7 @@ export class HostedStorageManager implements IDocStorageManager {
     if (!externalStoreDoc) { this._disableS3 = true; }
     const secondsBeforePush = options.secondsBeforePush;
     if (options.pushDocUpdateTimes) {
-      this._metadataManager = new HostedMetadataManager(dbManager);
+      this._metadataManager = new HostedMetadataManager(callbacks.setDocsMetadata);
     }
     this._uploads = new KeyedOps(key => this._pushToS3(key), {
       delayBeforeOperationMs: secondsBeforePush * 1000,
@@ -178,7 +185,7 @@ export class HostedStorageManager implements IDocStorageManager {
           return path.join(dir, 'meta.json');
         },
         async docId => {
-          const features = await dbManager.getDocFeatures(docId);
+          const features = await callbacks.getDocFeatures(docId);
           return features?.snapshotWindow;
         },
       );
@@ -639,7 +646,7 @@ export class HostedStorageManager implements IDocStorageManager {
 
       const existsLocally = await fse.pathExists(this.getPath(docName));
       if (existsLocally) {
-        if (!docStatus.docMD5 || docStatus.docMD5 === DELETED_TOKEN) {
+        if (!docStatus.docMD5 || docStatus.docMD5 === DELETED_TOKEN || docStatus.docMD5 === 'unknown') {
           // New doc appears to already exist, but may not exist in S3.
           // Let's check.
           const head = await this._ext.head(docName);

app/server/lib/WidgetRepository.ts

@@ -9,6 +9,7 @@ import {GristServer} from 'app/server/lib/GristServer';
 import LRUCache from 'lru-cache';
 import * as url from 'url';
 import { AsyncCreate } from 'app/common/AsyncCreate';
+import { proxyAgent } from 'app/server/lib/ProxyAgent';
 
 // Static url for UrlWidgetRepository
 const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
@@ -109,7 +110,7 @@ export class UrlWidgetRepository implements IWidgetRepository {
       return [];
     }
     try {
-      const response = await fetch(this._staticUrl);
+      const response = await fetch(this._staticUrl, { agent: proxyAgent(new URL(this._staticUrl)) });
       if (!response.ok) {
         if (response.status === 404) {
           throw new ApiError('WidgetRepository: Remote widget list not found', 404);

app/server/lib/sendAppPage.ts

@@ -89,7 +89,9 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi
     supportedLngs: readLoadedLngs(req?.i18n),
     namespaces: readLoadedNamespaces(req?.i18n),
     featureComments: isAffirmative(process.env.COMMENTS),
-    featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT),
+    featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY ||
+      process.env.ASSISTANT_API_KEY  ||
+      process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT),
     assistantService: process.env.OPENAI_API_KEY ? 'OpenAI' : undefined,
     permittedCustomWidgets: getPermittedCustomWidgets(server),
     supportEmail: SUPPORT_EMAIL,