Merge branch 'master' into s3-blob-storage-wip

lib/model/query/datasets.js

@@ -179,11 +179,18 @@ createPublishedDataset.audit.withResult = true;
 const _createPublishedProperty = (property) => sql`
 INSERT INTO ds_properties ("name", "datasetId", "publishedAt")
 VALUES (${property.name}, ${property.datasetId}, clock_timestamp())
+ON CONFLICT ("name", "datasetId")
+DO UPDATE SET "publishedAt" = clock_timestamp()
+WHERE ds_properties."publishedAt" IS NULL
 RETURNING *`;
 
 // eslint-disable-next-line no-unused-vars
-const createPublishedProperty = (property, dataset) => ({ one }) =>
-  one(_createPublishedProperty(property));
+const createPublishedProperty = (property, dataset) => async ({ all }) => {
+  const result = await all(_createPublishedProperty(property));
+  if (result.length === 0)
+    throw Problem.user.uniquenessViolation({ table: 'ds_properties', fields: [ 'name', 'datasetId' ], values: [ property.name, dataset.id ] });
+  return result[0];
+};
 
 createPublishedProperty.audit = (property, dataset) => (log) =>
   log('dataset.update', dataset, { properties: [property.name] });

lib/model/query/entities.js

@@ -99,6 +99,7 @@ createNew.audit = (newEntity, dataset, partial, subDef) => (log) => {
       entityDefId: newEntity.aux.currentVersion.id, // Added in v2023.3 and backfilled
       entity: { uuid: newEntity.uuid, dataset: dataset.name }
     });
+  return Promise.resolve();
 };
 createNew.audit.withResult = true;
 
@@ -528,9 +529,12 @@ const _deleteHeldSubmissionByEventId = (eventId) => ({ run }) => run(sql`
 ////////////////////////////////////////////////////////////////////////////////
 // FORCE PROCESSING SUBMISSIONS FROM BACKLOG
 
-const DAY_RANGE = config.has('default.taskSchedule.forceProcess')
-  ? config.get('default.taskSchedule.forceProcess')
-  : 7; // Default is 7 days
+// Submissions that have been held in the backlog for longer than 5 days
+// will be force-processed, including out-of-order updates and updates
+// applied as entity creates.
+const DAY_RANGE = config.has('default.taskSchedule.entitySubmissionBacklog')
+  ? config.get('default.taskSchedule.entitySubmissionBacklog')
+  : 5; // Default is 5 days
 
 const _getHeldSubmissionsAsEvents = (force) => ({ all }) => all(sql`
   SELECT audits.* FROM entity_submission_backlog

lib/model/query/forms.js

@@ -435,7 +435,7 @@ with redacted_audits as (
   ), deleted_forms as (
     delete from forms
     where ${_trashedFilter(force, id, projectId, xmlFormId)}
-    returning *
+    returning 1
   )
 select count(*) from deleted_forms`)
     .then((count) => Blobs.purgeUnattached()

test/integration/api/datasets.js

@@ -409,6 +409,34 @@ describe('datasets and entities', () => {
           });
       }));
 
+      it('should reject if creating a dataset property that already exists', testService(async (service) => {
+        const asAlice = await service.login('alice');
+
+        await asAlice.post('/v1/projects/1/datasets')
+          .send({
+            name: 'trees'
+          })
+          .expect(200);
+
+        // Create a property
+        await asAlice.post('/v1/projects/1/datasets/trees/properties')
+          .send({
+            name: 'height'
+          })
+          .expect(200);
+
+        // Second time should fail
+        await asAlice.post('/v1/projects/1/datasets/trees/properties')
+          .send({
+            name: 'height'
+          })
+          .expect(409)
+          .then(({ body }) => {
+            body.code.should.equal(409.3);
+            body.message.should.startWith('A resource already exists with name,datasetId value(s) of height');
+          });
+      }));
+
       it('should log an event for creating a new dataset property', testService(async (service) => {
         const asAlice = await service.login('alice');
 
@@ -434,6 +462,44 @@ describe('datasets and entities', () => {
             logs[0].details.properties.should.eql([ 'circumference' ]);
           });
       }));
+
+      it('should allow property name from a deleted draft form be used', testService(async (service) => {
+        const asAlice = await service.login('alice');
+
+        // Create a draft dataset
+        await asAlice.post('/v1/projects/1/forms')
+          .send(testData.forms.simpleEntity)
+          .set('Content-Type', 'application/xml')
+          .expect(200);
+
+        // Delete the draft form
+        await asAlice.delete('/v1/projects/1/forms/simpleEntity')
+          .expect(200);
+
+        // Create the dataset with the same name as the dataset in the deleted form
+        await asAlice.post('/v1/projects/1/datasets')
+          .send({ name: 'people' })
+          .expect(200);
+
+        // Create a property with a name that wasn't originally in the dataset in the form
+        await asAlice.post('/v1/projects/1/datasets/people/properties')
+          .send({ name: 'nickname' })
+          .expect(200);
+
+        // Create a property with a name that was in the dataset in the form
+        await asAlice.post('/v1/projects/1/datasets/people/properties')
+          .send({ name: 'age' })
+          .expect(200);
+
+        // Re-creating an existing property should result in an error
+        await asAlice.post('/v1/projects/1/datasets/people/properties')
+          .send({ name: 'age' })
+          .expect(409)
+          .then(({ body }) => {
+            body.code.should.equal(409.3);
+            body.message.should.startWith('A resource already exists with name,datasetId value(s) of age');
+          });
+      }));
     });
   });
 

test/integration/api/offline-entities.js

@@ -1105,11 +1105,23 @@ describe('Offline Entities', () => {
 
         await exhaust(container);
 
+        // One submission should be in the backlog
         let backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`);
         backlogCount.should.equal(1);
 
+        // Update the timestamp on this to in the past, but less than the default hold duration
+        await container.run(sql`UPDATE entity_submission_backlog SET "loggedAt" = "loggedAt" - interval '4 days'`);
+
+        // The submission should not have been processed
+        let count = await container.Entities.processBacklog();
+        count.should.equal(0);
+
+        // The submission should still be held in the backlog
+        backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`);
+        backlogCount.should.equal(1);
+
         // Update the timestamp on this backlog
-        await container.run(sql`UPDATE entity_submission_backlog SET "loggedAt" = "loggedAt" - interval '8 days'`);
+        await container.run(sql`UPDATE entity_submission_backlog SET "loggedAt" = "loggedAt" - interval '6 days'`);
 
         // Send the next submission, which will also be held in the backlog.
         // This submission immediately follows the previous one, but force-processing
@@ -1132,7 +1144,7 @@ describe('Offline Entities', () => {
 
         // Process submissions that have been in the backlog for a long time
         // (only 1 of 2 should be processed)
-        const count = await container.Entities.processBacklog();
+        count = await container.Entities.processBacklog();
         count.should.equal(1);
 
         await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc')

test/integration/task/purge.js

@@ -98,7 +98,7 @@ describe('task: purge deleted forms', () => {
               count.should.equal(0);
             })))));
 
-    it('should not purge all forms if no form ID supplied', testTask(({ Forms }) =>
+    it('should purge all forms if no form ID supplied', testTask(({ Forms }) =>
       Forms.getByProjectAndXmlFormId(1, 'simple')
         .then((form) => Forms.del(form.get()))
         .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat')
@@ -120,7 +120,7 @@ describe('task: purge deleted forms', () => {
   });
 
   describe('with xmlFormId', () => {
-    it('should thow error if xmlFormId specified without projectId', testTask(async ({ Forms }) => {
+    it('should throw error if xmlFormId specified without projectId', testTask(async ({ Forms }) => {
       const form = await Forms.getByProjectAndXmlFormId(1, 'simple');
       await Forms.del(form.get());
       await assert.throws(() => { purgeForms(true, null, null, 'simple'); }, (err) => {
@@ -179,7 +179,7 @@ describe('task: purge deleted forms', () => {
       count.should.equal(2);
     }));
 
-    it('should purged named form only from specified project', testService(async (service, container) => {
+    it('should purge named form only from specified project', testService(async (service, container) => {
       const asAlice = await service.login('alice');
 
       // delete simple form in project 1 (but don't purge it)