From 2dc07f1659daa0db448b05c34efcf3bbaa7f6cd4 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 9 Oct 2023 10:42:03 +0200 Subject: [PATCH 1/7] feat add technical id for tableId --- app/server/lib/ActiveDoc.ts | 18 ++++++++++++++++++ app/server/lib/DocApi.ts | 5 +++-- buildtools/prepare_python.sh | 1 + 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 0413f2eae2..93ead73435 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2784,6 +2784,10 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table const [, , tableRefs, tableData] = metaTables._grist_Tables; const tableRowIndex = tableData.tableId.indexOf(tableId); if (tableRowIndex === -1) { + // Test if the tableId argument is directly the ref + if(parseInt(tableId) && tableRefs.indexOf(parseInt(tableId)) >= 0){ + return parseInt(tableId); + } throw new ApiError(`Table not found "${tableId}"`, 404); } return tableRefs[tableRowIndex]; @@ -2799,11 +2803,25 @@ export function colIdToRef(metaTables: {[p: string]: TableDataAction}, tableId: columnData.colId[i] === colId && columnData.parentId[i] === tableRef )); if (colRowIndex === -1) { + // Test if the colId argument is directly the ref + if(parseInt(colId) && colRefs.indexOf(parseInt(colId)) >= 0){ + return parseInt(colId); + } throw new ApiError(`Column not found "${colId}"`, 404); } return colRefs[colRowIndex]; } +// Helper that check if tableRef is used instead of tableId and return real tableId +export function getRealTableId(metaTables: { [p: string]: TableDataAction }, tableId: string): string { + const [, , tableRefs, tableData] = metaTables._grist_Tables; + if(parseInt(tableId) && tableRefs.indexOf(parseInt(tableId)) >= 0){ + const tableRowIndex = tableRefs.indexOf(parseInt(tableId)); + return tableData.tableId[tableRowIndex]!.toString(); + } + return tableId; +} + export function sanitizeApplyUAOptions(options?: ApplyUAOptions): ApplyUAOptions { return pick(options||{}, ['desc', 'otherId', 'linkId', 'parseStrings']); } diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index bcf93c7a36..aff6e39f13 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -32,7 +32,7 @@ import { TableOperationsImpl, TableOperationsPlatform } from 'app/plugin/TableOperationsImpl'; -import {ActiveDoc, colIdToRef as colIdToReference, tableIdToRef} from "app/server/lib/ActiveDoc"; +import {ActiveDoc, colIdToRef as colIdToReference, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc"; import {appSettings} from "app/server/lib/AppSettings"; import {sendForCompletion} from 'app/server/lib/Assistance'; import { @@ -201,7 +201,8 @@ export class DocWorkerApi { if (!Object.keys(filters).every(col => Array.isArray(filters[col]))) { throw new ApiError("Invalid query: filter values must be arrays", 400); } - const tableId = optTableId || req.params.tableId; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, optTableId || req.params.tableId); const session = docSessionFromRequest(req); const {tableData} = await handleSandboxError(tableId, [], activeDoc.fetchQuery( session, {tableId, filters}, !immediate)); diff --git a/buildtools/prepare_python.sh b/buildtools/prepare_python.sh index 716f979c23..34c4ec4d3b 100755 --- a/buildtools/prepare_python.sh +++ b/buildtools/prepare_python.sh @@ -28,3 +28,4 @@ else # Make sure python3 isn't around. rm -rf sandbox_venv3 fi +echo "toto" From 60fc3800ce0ed8173d4ad2e4ff143040262386b9 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 9 Oct 2023 11:33:02 +0200 Subject: [PATCH 2/7] delete echo --- buildtools/prepare_python.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/buildtools/prepare_python.sh b/buildtools/prepare_python.sh index 34c4ec4d3b..716f979c23 100755 --- a/buildtools/prepare_python.sh +++ b/buildtools/prepare_python.sh @@ -28,4 +28,3 @@ else # Make sure python3 isn't around. rm -rf sandbox_venv3 fi -echo "toto" From 89f4066f9a67c700df40b94370562cf0d89bc00c Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 9 Oct 2023 15:30:08 +0200 Subject: [PATCH 3/7] add technical tableId compatibility in api calls for post and patch --- app/server/lib/ActiveDoc.ts | 4 ---- app/server/lib/DocApi.ts | 43 ++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 93ead73435..cc1d7003e0 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2784,10 +2784,6 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table const [, , tableRefs, tableData] = metaTables._grist_Tables; const tableRowIndex = tableData.tableId.indexOf(tableId); if (tableRowIndex === -1) { - // Test if the tableId argument is directly the ref - if(parseInt(tableId) && tableRefs.indexOf(parseInt(tableId)) >= 0){ - return parseInt(tableId); - } throw new ApiError(`Table not found "${tableId}"`, 404); } return tableRefs[tableRowIndex]; diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index aff6e39f13..7fd397e1e7 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -338,7 +338,8 @@ export class DocWorkerApi { const trigger = webhookId ? activeDoc.triggers.getWebhookTriggerRecord(webhookId) : undefined; let currentTableId = trigger ? tablesTable.getValue(trigger.tableRef, 'tableId')! : undefined; const {url, eventTypes, isReadyColumn, name} = webhook; - const tableId = req.params.tableId || webhook.tableId; + const tableId = getRealTableId(metaTables, req.params.tableId || webhook.tableId); + const fields: Partial = {}; if (url && !isUrlAllowed(url)) { @@ -388,7 +389,8 @@ export class DocWorkerApi { // Get the columns of the specified table in recordish format this._app.get('/api/docs/:docId/tables/:tableId/columns', canView, withDoc(async (activeDoc, req, res) => { - const tableId = req.params.tableId; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); const includeHidden = isAffirmative(req.query.hidden); const columns = await handleSandboxError('', [], activeDoc.getTableCols(docSessionFromRequest(req), tableId, includeHidden)); @@ -499,7 +501,9 @@ export class DocWorkerApi { withDoc(async (activeDoc, req, res) => { const colValues = req.body as BulkColValues; const count = colValues[Object.keys(colValues)[0]].length; - const op = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const op = getTableOperations(req, activeDoc, tableId); const ids = await op.addRecords(count, colValues); res.json(ids); }) @@ -528,7 +532,9 @@ export class DocWorkerApi { } } validateCore(RecordsPost, req, body); - const ops = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const ops = getTableOperations(req, activeDoc, tableId); const records = await ops.create(body.records); res.json({records}); }) @@ -559,7 +565,8 @@ export class DocWorkerApi { this._app.post('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPost), withDoc(async (activeDoc, req, res) => { const body = req.body as Types.ColumnsPost; - const {tableId} = req.params; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); const actions = body.columns.map(({fields, id: colId}) => // AddVisibleColumn adds the column to all widgets of the table. // This isn't necessarily what the user wants, but it seems like a good default. @@ -591,7 +598,9 @@ export class DocWorkerApi { this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => { const rowIds = req.body; - const op = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const op = getTableOperations(req, activeDoc, tableId); await op.destroy(rowIds); res.json(null); })); @@ -660,7 +669,9 @@ export class DocWorkerApi { const rowIds = columnValues.id; // sandbox expects no id column delete columnValues.id; - const ops = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const ops = getTableOperations(req, activeDoc, tableId); await ops.updateRecords(columnValues, rowIds); res.json(null); }) @@ -670,7 +681,9 @@ export class DocWorkerApi { this._app.patch('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPatch), withDoc(async (activeDoc, req, res) => { const body = req.body as Types.RecordsPatch; - const ops = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const ops = getTableOperations(req, activeDoc, tableId); await ops.update(body.records); res.json(null); }) @@ -681,7 +694,8 @@ export class DocWorkerApi { withDoc(async (activeDoc, req, res) => { const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables"); const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column"); - const {tableId} = req.params; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); const tableRef = tablesTable.findMatchingRowId({tableId}); if (!tableRef) { throw new ApiError(`Table not found "${tableId}"`, 404); @@ -721,7 +735,9 @@ export class DocWorkerApi { // Add or update records given in records format this._app.put('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPut), withDoc(async (activeDoc, req, res) => { - const ops = getTableOperations(req, activeDoc); + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); + const ops = getTableOperations(req, activeDoc, tableId); const body = req.body as Types.RecordsPut; const options = { add: !isAffirmative(req.query.noadd), @@ -741,7 +757,8 @@ export class DocWorkerApi { withDoc(async (activeDoc, req, res) => { const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables"); const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column"); - const {tableId} = req.params; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); const tableRef = tablesTable.findMatchingRowId({tableId}); if (!tableRef) { throw new ApiError(`Table not found "${tableId}"`, 404); @@ -786,7 +803,9 @@ export class DocWorkerApi { this._app.delete('/api/docs/:docId/tables/:tableId/columns/:colId', canEdit, withDoc(async (activeDoc, req, res) => { - const {tableId, colId} = req.params; + const {colId} = req.params; + const metaTables = await getMetaTables(activeDoc, req); + const tableId = getRealTableId(metaTables, req.params.tableId); const actions = [ [ 'RemoveColumn', tableId, colId ] ]; await handleSandboxError(tableId, [colId], activeDoc.applyUserActions(docSessionFromRequest(req), actions) From 62bdd4e7ad108e26e80861639e0f041eadbfcee6 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 9 Oct 2023 16:14:19 +0200 Subject: [PATCH 4/7] feat: create function to get real col id --- app/server/lib/ActiveDoc.ts | 20 +++++++++++++++----- app/server/lib/DocApi.ts | 4 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index cc1d7003e0..e2ffeac881 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2791,7 +2791,6 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table // Helper that converts a Grist column colId to a ref given the corresponding table. export function colIdToRef(metaTables: {[p: string]: TableDataAction}, tableId: string, colId: string) { - const tableRef = tableIdToRef(metaTables, tableId); const [, , colRefs, columnData] = metaTables._grist_Tables_column; @@ -2799,10 +2798,6 @@ export function colIdToRef(metaTables: {[p: string]: TableDataAction}, tableId: columnData.colId[i] === colId && columnData.parentId[i] === tableRef )); if (colRowIndex === -1) { - // Test if the colId argument is directly the ref - if(parseInt(colId) && colRefs.indexOf(parseInt(colId)) >= 0){ - return parseInt(colId); - } throw new ApiError(`Column not found "${colId}"`, 404); } return colRefs[colRowIndex]; @@ -2818,6 +2813,21 @@ export function getRealTableId(metaTables: { [p: string]: TableDataAction }, tab return tableId; } +// Helper that check if colRef is used instead of colId and return real colId +export function getRealColId(metaTables: { [p: string]: TableDataAction }, tableId: string, colId: string): string { + const tableRef = tableIdToRef(metaTables, tableId); + const [, , colRefs, columnData] = metaTables._grist_Tables_column; + if(parseInt(colId)){ + const colRowIndex = colRefs.findIndex((_, i) => ( + colRefs[i] === parseInt(colId) && columnData.parentId[i] === tableRef + )); + if(colRowIndex >= 0) { + return columnData.colId[colRowIndex]!.toString(); + } + } + return colId; +} + export function sanitizeApplyUAOptions(options?: ApplyUAOptions): ApplyUAOptions { return pick(options||{}, ['desc', 'otherId', 'linkId', 'parseStrings']); } diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 7fd397e1e7..1ae14714fb 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -32,7 +32,7 @@ import { TableOperationsImpl, TableOperationsPlatform } from 'app/plugin/TableOperationsImpl'; -import {ActiveDoc, colIdToRef as colIdToReference, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc"; +import {ActiveDoc, colIdToRef as colIdToReference, getRealColId, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc"; import {appSettings} from "app/server/lib/AppSettings"; import {sendForCompletion} from 'app/server/lib/Assistance'; import { @@ -803,9 +803,9 @@ export class DocWorkerApi { this._app.delete('/api/docs/:docId/tables/:tableId/columns/:colId', canEdit, withDoc(async (activeDoc, req, res) => { - const {colId} = req.params; const metaTables = await getMetaTables(activeDoc, req); const tableId = getRealTableId(metaTables, req.params.tableId); + const colId = getRealColId(metaTables, tableId, req.params.colId); const actions = [ [ 'RemoveColumn', tableId, colId ] ]; await handleSandboxError(tableId, [colId], activeDoc.applyUserActions(docSessionFromRequest(req), actions) From 29461071cfd17d8b086549697948ad0b19d58405 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 9 Oct 2023 16:15:19 +0200 Subject: [PATCH 5/7] remove unappropiate code --- app/server/lib/ActiveDoc.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index cc1d7003e0..dd0ffd0b7e 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2799,10 +2799,6 @@ export function colIdToRef(metaTables: {[p: string]: TableDataAction}, tableId: columnData.colId[i] === colId && columnData.parentId[i] === tableRef )); if (colRowIndex === -1) { - // Test if the colId argument is directly the ref - if(parseInt(colId) && colRefs.indexOf(parseInt(colId)) >= 0){ - return parseInt(colId); - } throw new ApiError(`Column not found "${colId}"`, 404); } return colRefs[colRowIndex]; From 647bb5774584e417b7637156a85fbaa965393b4b Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 10 Oct 2023 11:13:20 +0200 Subject: [PATCH 6/7] remove line to reload test in github --- app/server/lib/ActiveDoc.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index dd0ffd0b7e..2f60e8fd0a 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2791,7 +2791,6 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table // Helper that converts a Grist column colId to a ref given the corresponding table. export function colIdToRef(metaTables: {[p: string]: TableDataAction}, tableId: string, colId: string) { - const tableRef = tableIdToRef(metaTables, tableId); const [, , colRefs, columnData] = metaTables._grist_Tables_column; From 1d7c807c980f16958d66e2bb05ab0387cb40fac4 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Thu, 12 Oct 2023 09:51:47 +0200 Subject: [PATCH 7/7] add print to understand where to change the code --- app/server/lib/ActiveDoc.ts | 5 +++++ app/server/lib/DocApi.ts | 11 +++++++++-- sandbox/grist/engine.py | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index e2ffeac881..4905f93668 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -2509,6 +2509,9 @@ export class ActiveDoc extends EventEmitter { private async _fetchQueryFromDB(query: ServerQuery, onDemand: boolean): Promise { // Expand query to compute formulas (or include placeholders for them). const expandedQuery = expandQuery(query, this.docData!, onDemand); + console.log("--------------fetchQueryFromDB----------------------"); + console.log(expandedQuery); + console.log(this.docData!); const marshalled = await this.docStorage.fetchQuery(expandedQuery); const table = this.docStorage.decodeMarshalledData(marshalled, query.tableId); @@ -2523,6 +2526,8 @@ export class ActiveDoc extends EventEmitter { } private async _fetchQueryFromDataEngine(query: ServerQuery): Promise { + console.log("--------------_fetchQueryFromDataEngine----------------------"); + console.log(await this._pyCall('fetch_table', query.tableId, true, query.filters)); return this._pyCall('fetch_table', query.tableId, true, query.filters); } diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 1ae14714fb..37f5f07d73 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -208,6 +208,9 @@ export class DocWorkerApi { session, {tableId, filters}, !immediate)); // For metaTables we don't need to specify columns, search will infer it from the sort expression. const isMetaTable = tableId.startsWith('_grist'); + console.log("////////////////////////////// in getTableData"); + console.log("---- tableData"); + console.log(tableData); const columns = isMetaTable ? null : await handleSandboxError('', [], activeDoc.getTableCols(session, tableId, true)); const params = getQueryParameters(req); @@ -217,9 +220,13 @@ export class DocWorkerApi { } async function getTableRecords( - activeDoc: ActiveDoc, req: RequestWithLogin, opts?: { optTableId?: string; includeHidden?: boolean } + activeDoc: ActiveDoc, + req: RequestWithLogin, + opts?: { optTableId?: string; includeHidden?: boolean, useColRef?: boolean } ): Promise { const columnData = await getTableData(activeDoc, req, opts?.optTableId); + console.log("------------------- colummnDATA in getTableRecords"); + console.log(columnData); const fieldNames = Object.keys(columnData).filter((k) => { if (k === "id") { return false; @@ -257,7 +264,7 @@ export class DocWorkerApi { this._app.get('/api/docs/:docId/tables/:tableId/records', canView, withDoc(async (activeDoc, req, res) => { const records = await getTableRecords(activeDoc, req, - { includeHidden: isAffirmative(req.query.hidden) } + { includeHidden: isAffirmative(req.query.hidden), useColRef: isAffirmative(req.query.use_col_ref) } ); res.json({records}); }) diff --git a/sandbox/grist/engine.py b/sandbox/grist/engine.py index 1e2dae2ea6..6e6b1fdf5f 100644 --- a/sandbox/grist/engine.py +++ b/sandbox/grist/engine.py @@ -383,6 +383,7 @@ def fetch_table(self, table_id, formulas=True, private=False, query=None): """ Returns TableData object representing all data in this table. """ + print("Je suis dans le fichier de PYTHON YOUHOUUUUUUU", file=sys.stderr) table = self.tables[table_id] column_values = {}