Merge branch 'master' into feature/ACS-8766-Adjust-live-ingester-logg…

…ing-to-be-more-descriptive-in-case-of-error # Conflicts: # live-ingester/src/main/java/org/alfresco/hxi_connector/live_ingester/adapters/config/jackson/DeleteNodeEventSerializer.java
Alfresco · Sep 23, 2024 · 5b79ae4 · 5b79ae4
2 parents 11cf350 + 614be2b
commit 5b79ae4
Show file tree

Hide file tree

Showing 26 changed files with 177 additions and 117 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ env:
   # As the release is triggered by a commit message with "[release]" keyword on a release branch,
   # setting these variables to new values can be done in the same commit and will indicate the release and the dev versions in it.
   DEVELOPMENT_VERSION: "1.0.0-SNAPSHOT" # The version that will be set in pom files after the release (next dev version)
-  RELEASE_VERSION: "1.0.0-A15" # The version of the release (tag).
+  RELEASE_VERSION: "1.0.0-A16" # The version of the release (tag).
   GITHUB_ACTIONS_DEPLOY_TIMEOUT: 60
   MAVEN_CLI_OPTS: "-B -e -fae -V -DinstallAtEnd=true -DfailIfNoTests=false -U -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Pdistribution "
   MAVEN_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
@@ -35,15 +35,15 @@ jobs:
       java_version: ${{ env.JAVA_VERSION }}
       java_version_supported_by_repo: ${{ env.JAVA_VERSION_SUPPORTED_BY_REPO }}
     steps:
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - id: changed-files
-        uses: Alfresco/alfresco-build-tools/.github/actions/github-list-changes@v6.1.0
+        uses: Alfresco/alfresco-build-tools/.github/actions/github-list-changes@v7.0.0
         with:
           write-list-to-env: true
-      - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.0.0
 
   veracode_sca:
     name: "Veracode - Source Clear Scan (SCA)"
@@ -55,13 +55,12 @@ jobs:
       !contains(github.event.head_commit.message, '[skip build]')
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/veracode@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/veracode@v7.0.0
         continue-on-error: true
         with:
           srcclr-api-token: ${{ secrets.SRCCLR_API_TOKEN }}
-          srcclr-install-options: -Dmaven.test.skip -P !test -pl "!common-test,!e2e-test"
 
   veracode_sast:
     name: "Pipeline SAST Scan"
@@ -74,8 +73,8 @@ jobs:
       !contains(github.event.head_commit.message, '[skip build]')
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/github-download-file@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/github-download-file@v7.0.0
         with:
           token: ${{ secrets.BOT_GITHUB_TOKEN }}
           repository: "Alfresco/veracode-baseline-archive"
@@ -109,8 +108,8 @@ jobs:
       !contains(github.event.head_commit.message, '[skip tests]')
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - uses: Alfresco/[email protected]
 
   test_application:
@@ -129,14 +128,14 @@ jobs:
                       "live-ingester", "prediction-applier", "hxinsight-extension" ]
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Test application"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean verify -pl '${{ matrix.subproject }}' -am
 
@@ -155,14 +154,14 @@ jobs:
         repoVersion: [ 7.3.2, 7.4.2, 23.3.0 ]
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Build application with distribution profile"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean install -DskipTests -Dalfresco-platform.version=${{ matrix.repoVersion }}
 
@@ -182,14 +181,14 @@ jobs:
         repoVersion: [ 7.3.2, 7.4.2, 23.3.0 ]
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Build application"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean install -DskipTests -Dalfresco-platform.version=${{ matrix.repoVersion }}
       - name: "Build docker images"
@@ -215,14 +214,14 @@ jobs:
         repoVersion: [ 7.3.2, 7.4.2 ]
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Build application"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean install -DskipTests -Dalfresco-platform.version=${{ matrix.repoVersion }} -Dalfresco-platform.java.version=${{ env.JAVA_VERSION_SUPPORTED_BY_REPO }}
       - name: "Build docker images"
@@ -247,13 +246,13 @@ jobs:
       - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Build application"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean install -DskipTests -Dalfresco-platform.java.version=${{ env.JAVA_VERSION_SUPPORTED_BY_REPO }}
       - name: "Push docker images"
@@ -277,9 +276,9 @@ jobs:
       !contains(github.event.head_commit.message, '[skip build]')
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/get-build-info@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/get-build-info@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Publish SNAPSHOT artifacts"
         timeout-minutes: ${{ fromJSON(env.GITHUB_ACTIONS_DEPLOY_TIMEOUT) }}
         run: mvn ${{ env.MAVEN_CLI_OPTS }} deploy -DskipTests -Dalfresco-platform.java.version=${{ env.JAVA_VERSION_SUPPORTED_BY_REPO }}
@@ -302,9 +301,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/configure-git-author@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/configure-git-author@v7.0.0
         with:
           username: ${{ secrets.BOT_GITHUB_USERNAME }}
           email: ${{ secrets.BOT_GITHUB_EMAIL }}
@@ -349,13 +348,13 @@ jobs:
       - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
       - name: "Login to Quay.io"
-        uses: docker/login-action@v3.3.0
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v6.1.0
-      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.0.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v7.0.0
       - name: "Build application"
         run: mvn ${{ env.MAVEN_CLI_OPTS }} clean install -DskipTests -Dproject.revision.key=${{ github.sha }} -Dalfresco-platform.java.version=${{ env.JAVA_VERSION_SUPPORTED_BY_REPO }}
       - name: "Deploy release version to quay.io"

diff --git a/.github/workflows/manual_release.yml b/.github/workflows/manual_release.yml
@@ -22,7 +22,7 @@ jobs:
       contents: write
     steps:
       - uses: actions/checkout@v4
-      - uses: Alfresco/alfresco-build-tools/.github/actions/configure-git-author@v6.1.0
+      - uses: Alfresco/alfresco-build-tools/.github/actions/configure-git-author@v7.0.0
         with:
           username: ${{ secrets.BOT_GITHUB_USERNAME }}
           email: ${{ secrets.BOT_GITHUB_EMAIL }}

diff --git a/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/BulkIngesterE2eTest.java b/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/BulkIngesterE2eTest.java
@@ -104,7 +104,7 @@ void shouldIncludeBasicProperties()
             assertEquals(nodeId, event.get("objectId").asText());
             assertEquals(sourceId, event.get("sourceId").asText());
             assertEquals(eventType, event.get("eventType").asText());
-            assertTrue(event.has("timestamp"));
+            assertTrue(event.has("sourceTimestamp"));
             assertTrue(event.has("properties"));
         }, 15, 200);
     }

diff --git a/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/CreateNodeE2eTest.java b/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/CreateNodeE2eTest.java
@@ -161,7 +161,7 @@ final void testCreateNodeContainingImageFile() throws IOException
 
             WireMock.verify(exactly(1), postRequestedFor(urlEqualTo("/presigned-urls")));
             WireMock.verify(moreThanOrExactly(2), postRequestedFor(urlEqualTo("/ingestion-events"))
-                    .withRequestBody(containing(createdNode.id()).and(containing("timestamp")))
+                    .withRequestBody(containing(createdNode.id()).and(containing("sourceTimestamp")))
                     .withHeader(USER_AGENT, matching(getAppInfoRegex())));
         }, INITIAL_DELAY_MS);
     }
@@ -189,7 +189,7 @@ final void testCreateNodeContainingTextFile() throws IOException
 
             WireMock.verify(exactly(1), postRequestedFor(urlEqualTo("/presigned-urls")));
             WireMock.verify(moreThanOrExactly(2), postRequestedFor(urlEqualTo("/ingestion-events"))
-                    .withRequestBody(containing(createdNode.id()).and(containing("timestamp")))
+                    .withRequestBody(containing(createdNode.id()).and(containing("sourceTimestamp")))
                     .withHeader(USER_AGENT, matching(getAppInfoRegex())));
         }, INITIAL_DELAY_MS);
     }

diff --git a/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/DeleteNodeE2eTest.java b/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/DeleteNodeE2eTest.java
@@ -116,7 +116,7 @@ void testReceiveDeleteEvent() throws IOException
         RetryUtils.retryWithBackoff(() -> verify(exactly(1), postRequestedFor(urlEqualTo("/ingestion-events"))
                 .withRequestBody(containing("\"objectId\":\"%s\"".formatted(createdNode.id())))
                 .withRequestBody(containing("\"sourceId\":\"alfresco-dummy-source-id-0a63de491876\""))
-                .withRequestBody(containing("\"timestamp\""))
+                .withRequestBody(containing("\"sourceTimestamp\""))
                 .withRequestBody(containing("\"eventType\":\"delete\""))
                 .withHeader(USER_AGENT, matching(getAppInfoRegex()))), DELAY_MS);
     }

diff --git a/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/UpdateNodeE2eTest.java b/e2e-test/src/test/java/org/alfresco/hxi_connector/e2e_test/UpdateNodeE2eTest.java
@@ -236,7 +236,7 @@ final void testSendTimestampToHxi()
         // then
         RetryUtils.retryWithBackoff(() -> verify(exactly(1), postRequestedFor(urlEqualTo("/ingestion-events"))
                 .withRequestBody(containing(updatedNode.id()))
-                .withRequestBody(containing("timestamp"))
+                .withRequestBody(containing("sourceTimestamp"))
                 .withHeader(USER_AGENT, matching(getAppInfoRegex()))));
     }
 

diff --git a/...ctor/live_ingester/domain/usecase/e2e/bulk_ingester/BulkIngesterEventIntegrationTest.java b/...ctor/live_ingester/domain/usecase/e2e/bulk_ingester/BulkIngesterEventIntegrationTest.java
@@ -69,14 +69,17 @@ void shouldIngestOnlyMetadataIfThereIsNoContent()
                     "objectId" : "5018ff83-ec45-4a11-95c4-681761752aa7",
                     "sourceId" : "alfresco-dummy-source-id-0a63de491876",
                     "eventType" : "create",
-                    "timestamp": 1707153500,
+                    "sourceTimestamp": 1707153500,
                     "properties" : {
                       "type": {"value": "cm:category"},
                       "createdAt": {"value": 1707153552},
                       "createdBy": {"value": "System"},
                       "modifiedBy": {"value": "admin"},
                       "aspectsNames": {"value": ["cm:auditable"]},
-                      "cm:name": {"value": "Mexican Spanish"}
+                      "cm:name": {
+                        "value": "Mexican Spanish",
+                        "annotation" : "name"
+                      }
                     }
                   }
                 ]""";
@@ -122,14 +125,17 @@ void shouldIngestMetadataAndContent()
                     "objectId" : "37be157c-741c-4e51-b781-20d36e4e335a",
                     "sourceId" : "alfresco-dummy-source-id-0a63de491876",
                     "eventType" : "create",
-                    "timestamp": 1308061016,
+                    "sourceTimestamp": 1308061016,
                     "properties" : {
                       "type": {"value": "cm:content"},
                       "createdBy": {"value": "admin"},
                       "modifiedBy": {"value": "hr_user"},
                       "aspectsNames": {"value": ["cm:indexControl", "cm:auditable"]},
                       "createdAt": {"value": 1308061016},
-                      "cm:name": {"value": "dashboard.xml"},
+                      "cm:name": {
+                        "value": "dashboard.xml",
+                        "annotation" : "name"
+                      },
                       "cm:isContentIndexed": {"value": true},
                       "cm:isIndexed": {"value": false},
                       "cm:content": {

diff --git a/...ain/usecase/e2e/bulk_ingester/BulkIngesterEventMatchingContentMappingIntegrationTest.java b/...ain/usecase/e2e/bulk_ingester/BulkIngesterEventMatchingContentMappingIntegrationTest.java
@@ -83,14 +83,17 @@ void givenMappingForImage_whenContentWithMatchingTypeIngested_thenProcessWithTra
                     "objectId" : "37be157c-741c-4e51-b781-20d36e4e335a",
                     "sourceId" : "alfresco-dummy-source-id-0a63de491876",
                     "eventType" : "create",
-                    "timestamp": %s,
+                    "sourceTimestamp": %s,
                     "properties" : {
                       "type": {"value": "cm:content"},
                       "createdBy": {"value": "admin"},
                       "modifiedBy": {"value": "hr_user"},
                       "aspectsNames": {"value": ["cm:indexControl", "cm:auditable"]},
                       "createdAt": {"value": 1308061016},
-                      "cm:name": {"value": "dashboard.xml"},
+                      "cm:name": {
+                        "value": "dashboard.xml",
+                        "annotation" : "name"
+                      },
                       "cm:isContentIndexed": {"value": true},
                       "cm:isIndexed": {"value": false},
                       "cm:content": {
@@ -168,14 +171,17 @@ void givenMappingForNonImage_whenContentWithMatchingTypeIngested_thenProcessWith
                     "objectId" : "37be157c-741c-4e51-b781-20d36e4e335a",
                     "sourceId" : "alfresco-dummy-source-id-0a63de491876",
                     "eventType" : "create",
-                    "timestamp": %s,
+                    "sourceTimestamp": %s,
                     "properties" : {
                       "type": {"value": "cm:content"},
                       "createdBy": {"value": "admin"},
                       "modifiedBy": {"value": "hr_user"},
                       "aspectsNames": {"value": ["cm:indexControl", "cm:auditable"]},
                       "createdAt": {"value": 1308061016},
-                      "cm:name": {"value": "dashboard.xml"},
+                      "cm:name": {
+                        "value": "dashboard.xml",
+                        "annotation" : "name"
+                      },
                       "cm:isContentIndexed": {"value": true},
                       "cm:isIndexed": {"value": false},
                       "cm:content": {

diff --git a/.../usecase/e2e/bulk_ingester/BulkIngesterEventNonMatchingContentMappingIntegrationTest.java b/.../usecase/e2e/bulk_ingester/BulkIngesterEventNonMatchingContentMappingIntegrationTest.java
@@ -77,14 +77,17 @@ void givenExactAndWildcardMimeTypeMappingForContentConfigured_whenContentWithNot
                     "objectId" : "37be157c-741c-4e51-b781-20d36e4e335a",
                     "sourceId" : "alfresco-dummy-source-id-0a63de491876",
                     "eventType" : "create",
-                    "timestamp" : %s,
+                    "sourceTimestamp" : %s,
                     "properties" : {
                       "type": {"value": "cm:content"},
                       "createdBy": {"value": "admin"},
                       "modifiedBy": {"value": "hr_user"},
                       "aspectsNames": {"value": ["cm:indexControl", "cm:auditable"]},
                       "createdAt": {"value": 1308061016},
-                      "cm:name": {"value": "dashboard.xml"},
+                      "cm:name": {
+                        "value": "dashboard.xml",
+                        "annotation" : "name"
+                      },
                       "cm:isContentIndexed": {"value": true},
                       "cm:isIndexed": {"value": false},
                       "cm:content": {