Include entity create 2022.1 forms in 2024.1 upgrade

lib/data/schema.js

@@ -641,9 +641,11 @@ const _updateEntityVersion = (xml, oldVersion, newVersion) => new Promise((pass,
 // If there are any problems with updating the XML, this will just
 // return the unaltered XML which will then be a clue for the worker
 // to not change anything about the Form.
-const updateEntityForm = (xml, oldVersion, newVersion, suffix) =>
+// 2022.1 -> 2024.1 forms only have version changed and suffix added.
+// 2023.1 -> 2024.1 forms (for updating) also have branchId and trunkVersion added.
+const updateEntityForm = (xml, oldVersion, newVersion, suffix, addOfflineParams) =>
   _updateEntityVersion(xml, oldVersion, newVersion)
-    .then(_addBranchIdAndTrunkVersion)
+    .then(x => (addOfflineParams ? _addBranchIdAndTrunkVersion(x) : x))
     .then(x => addVersionSuffix(x, suffix))
     .catch(() => xml);
 

lib/model/migrations/20241029-01-schedule-entity-form-upgrade-create-forms.js

@@ -0,0 +1,34 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+// The previous migration only added this event for forms with an
+// update action, which should have been entity spec version 2023.1.0.
+// We also need to flag 2022.1.0 forms with the create action.
+// To avoid flagging forms that do both create + update twice, this
+// migration captures the complement set of forms.
+// Basically, every existing form should be flagged, but I didn't want
+// to change an old migration.
+
+const up = (db) => db.raw(`
+  INSERT INTO audits ("action", "acteeId", "loggedAt", "details")
+  SELECT 'upgrade.process.form.entities_version', forms."acteeId", clock_timestamp(),
+    '{"upgrade": "As part of upgrading Central to v2024.3, this form is being updated to the latest entities-version spec."}'
+  FROM forms
+  JOIN form_defs fd ON forms."id" = fd."formId"
+  JOIN dataset_form_defs dfd ON fd."id" = dfd."formDefId"
+  JOIN projects ON projects."id" = forms."projectId"
+  WHERE NOT dfd."actions" @> '["update"]'
+  AND forms."deletedAt" IS NULL
+  AND projects."deletedAt" IS NULL
+  GROUP BY forms."acteeId";
+`);
+
+const down = () => {};
+
+module.exports = { up, down };

lib/worker/form.js

@@ -44,10 +44,21 @@ const updateDraftSet = pushDraftToEnketo;
 const updatePublish = pushFormToEnketo;
 
 const _upgradeEntityVersion = async (form) => {
-  const xml = await updateEntityForm(form.xml, '2023.1.0', '2024.1.0', '[upgrade]');
-  // If the XML doesnt change (not the version in question, or a parsing error), don't return the new partial Form
+  // We need to upgrade both 2022.1 and 2023.1 forms to 2024, and we are not sure which it is
+  // without parsing the form.
+  // Try one upgrade and then the other.
+
+  // Attempt the 2023.1 upgrade first:
+  let xml = await updateEntityForm(form.xml, '2023.1.0', '2024.1.0', '[upgrade]', true);
+
+  // If the XML doesnt change (not the version in question, or a parsing error), try the 2022.1 upgrade:
+  if (xml === form.xml)
+    xml = await updateEntityForm(form.xml, '2022.1.0', '2024.1.0', '[upgrade]', false);
+
+  // If the XML still has not changed, don't return a partial.
   if (xml === form.xml)
     return null;
+
   const partial = await Form.fromXml(xml);
   return partial.withAux('xls', { xlsBlobId: form.def.xlsBlobId });
 };

test/integration/other/form-entities-version.js

@@ -29,6 +29,29 @@ const upgradedUpdateEntity = `<?xml version="1.0"?>
   </h:head>
 </h:html>`;
 
+const upgradedSimpleEntity = `<?xml version="1.0"?>
+<h:html xmlns="http://www.w3.org/2002/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:jr="http://openrosa.org/javarosa" xmlns:entities="http://www.opendatakit.org/xforms">
+  <h:head>
+    <model entities:entities-version="2024.1.0">
+      <instance>
+        <data id="simpleEntity" orx:version="1.0[upgrade]">
+          <name/>
+          <age/>
+          <hometown/>
+          <meta>
+            <entity dataset="people" id="" create="">
+              <label/>
+            </entity>
+          </meta>
+        </data>
+      </instance>
+      <bind nodeset="/data/name" type="string" entities:saveto="first_name"/>
+      <bind nodeset="/data/age" type="int" entities:saveto="age"/>
+      <bind nodeset="/data/hometown" type="string"/>
+    </model>
+  </h:head>
+</h:html>`;
+
 describe('Update / migrate entities-version within form', () => {
   describe('upgrading a 2023.1.0 update form', () => {
     it('should upgrade a form with only a published version', testService(async (service, container) => {
@@ -469,6 +492,49 @@ describe('Update / migrate entities-version within form', () => {
     }));
   });
 
+  describe('upgrading a 2022.1.0 create form', () => {
+    it('should upgrade a form with a draft version and a published version', testService(async (service, container) => {
+      const { Forms, Audits } = container;
+      const asAlice = await service.login('alice');
+
+      // Upload a form and publish it
+      await asAlice.post('/v1/projects/1/forms?publish=true')
+        .send(testData.forms.simpleEntity)
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      // Convert the published form to a draft
+      await asAlice.post('/v1/projects/1/forms/simpleEntity/draft');
+
+      const { acteeId } = await Forms.getByProjectAndXmlFormId(1, 'simpleEntity').then(o => o.get());
+      await Audits.log(null, 'upgrade.process.form.entities_version', { acteeId });
+
+      // Run form upgrade
+      await exhaust(container);
+
+      // The version on the draft does change even though it is updated in place
+      await asAlice.get('/v1/projects/1/forms/simpleEntity/draft')
+        .then(({ body }) => {
+          body.version.should.equal('1.0[upgrade]');
+        });
+
+      await asAlice.get('/v1/projects/1/forms/simpleEntity/versions')
+        .then(({ body }) => {
+          body.length.should.equal(2);
+          body[0].version.should.equal('1.0[upgrade]');
+          body[1].version.should.equal('1.0');
+        });
+
+      // The published form XML is updated
+      await asAlice.get('/v1/projects/1/forms/simpleEntity.xml')
+        .then(({ text }) => text.should.equal(upgradedSimpleEntity));
+
+      // The draft XML is updated
+      await asAlice.get('/v1/projects/1/forms/simpleEntity/draft.xml')
+        .then(({ text }) => text.should.equal(upgradedSimpleEntity));
+    }));
+  });
+
   describe('audit logging and errors', () => {
     it('should log events about the upgrade for a published form', testService(async (service, container) => {
       const { Forms, Audits } = container;

test/unit/data/schema.js

@@ -2089,7 +2089,7 @@ describe('form schema', () => {
 
   describe('updateEntityForm', () => {
     it('should change version 2023->2024, add trunkVersion, and add branchId', (async () => {
-      const result = await updateEntityForm(testData.forms.updateEntity, '2023.1.0', '2024.1.0', '[upgrade]');
+      const result = await updateEntityForm(testData.forms.updateEntity, '2023.1.0', '2024.1.0', '[upgrade]', true);
       // entities-version has been updated
       // version has suffix
       // trunkVersion and branchId are present
@@ -2116,14 +2116,53 @@ describe('form schema', () => {
 </h:html>`);
     }));
 
+    it('should change version 2022->2024', (async () => {
+      const result = await updateEntityForm(testData.forms.simpleEntity, '2022.1.0', '2024.1.0', '[upgrade]', false);
+      // entities-version has been updated
+      // version has suffix
+      // trunkVersion and branchId are NOT added
+      result.should.equal(`<?xml version="1.0"?>
+<h:html xmlns="http://www.w3.org/2002/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:jr="http://openrosa.org/javarosa" xmlns:entities="http://www.opendatakit.org/xforms">
+  <h:head>
+    <model entities:entities-version="2024.1.0">
+      <instance>
+        <data id="simpleEntity" orx:version="1.0[upgrade]">
+          <name/>
+          <age/>
+          <hometown/>
+          <meta>
+            <entity dataset="people" id="" create="">
+              <label/>
+            </entity>
+          </meta>
+        </data>
+      </instance>
+      <bind nodeset="/data/name" type="string" entities:saveto="first_name"/>
+      <bind nodeset="/data/age" type="int" entities:saveto="age"/>
+      <bind nodeset="/data/hometown" type="string"/>
+    </model>
+  </h:head>
+</h:html>`);
+    }));
+
+    // updateEntityForm takes the old version to replace as an argument
+    // these tests show it will not change a 2022.1 (create-only) form when 2023.1 is provided
     it('should not alter a version 2022.1.0 form when the old version to replace is 2023.1.0', (async () => {
-      const result = await updateEntityForm(testData.forms.simpleEntity, '2023.1.0', '2024.1.0', '[upgrade]');
+      const result = await updateEntityForm(testData.forms.simpleEntity, '2023.1.0', '2024.1.0', '[upgrade]', true);
       result.should.equal(testData.forms.simpleEntity);
     }));
+
     it('should not alter a version 2024.1.0 form when the old version to replace is 2023.1.0', (async () => {
-      const result = await updateEntityForm(testData.forms.offlineEntity, '2023.1.0', '2024.1.0', '[upgrade]');
+      const result = await updateEntityForm(testData.forms.offlineEntity, '2023.1.0', '2024.1.0', '[upgrade]', true);
       result.should.equal(testData.forms.offlineEntity);
     }));
+
+    // these tests show it will not change a 2023.1 (update) form when 2022.1 is provided
+    it('should not alter a version 2023.1.0 form when the old version to replace is 2022.1.0', (async () => {
+      const result = await updateEntityForm(testData.forms.updateEntity, '2022.1.0', '2024.1.0', '[upgrade]', true);
+      result.should.equal(testData.forms.updateEntity);
+    }));
+
   });
 });