Enable default splunk and confluent dependencies (#1785)

* Enable default splunk and confluent dependencies Signed-off-by: Jeffrey Kinard <[email protected]> * fix site-report plugin Signed-off-by: Jeffrey Kinard <[email protected]> --------- Signed-off-by: Jeffrey Kinard <[email protected]>
GoogleCloudPlatform · Aug 13, 2024 · 87ae0eb · 87ae0eb
1 parent ffdb13c
commit 87ae0eb
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 35 deletions.
diff --git a/.github/actions/publish-site-report/action.yml b/.github/actions/publish-site-report/action.yml
@@ -30,7 +30,7 @@ runs:
   steps:
     - name: Generate Site Report
       shell: bash
-      run: mvn -B surefire-report:report-only -f pom.xml -Daggregate=true
+      run: mvn -B surefire-report:report-only -f pom.xml -Daggregate=true -Denforcer.skip=true
     - name: Publish Site Report
       uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v3.1.2
       with:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -98,7 +98,7 @@ jobs:
       if: always()
     - name: Create artifacts and push
       run: |
-        mvn verify -PtemplatesRelease,splunkDeps,missing-artifact-repos \
+        mvn verify -PtemplatesRelease \
         -DprojectId="dataflow-templates" \
         -DbucketName="dataflow-templates-staging" \
         -DlibrariesBucketName="dataflow-templates-libraries" \

diff --git a/cicd/internal/workflows/maven-workflows.go b/cicd/internal/workflows/maven-workflows.go
@@ -106,15 +106,15 @@ func (*mvnFlags) FailAtTheEnd() string {
 }
 
 func (*mvnFlags) RunIntegrationTests() string {
-	return "-PtemplatesIntegrationTests,splunkDeps,missing-artifact-repos"
+	return "-PtemplatesIntegrationTests"
 }
 
 func (*mvnFlags) RunIntegrationSmokeTests() string {
-	return "-PtemplatesIntegrationSmokeTests,splunkDeps,missing-artifact-repos"
+	return "-PtemplatesIntegrationSmokeTests"
 }
 
 func (*mvnFlags) RunLoadTests() string {
-	return "-PtemplatesLoadTests,splunkDeps,missing-artifact-repos"
+	return "-PtemplatesLoadTests"
 }
 
 // The number of modules Maven is going to build in parallel in a multi-module project.

diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java
@@ -393,7 +393,7 @@ private String[] buildMavenStageCommand(
       "-pl",
       moduleBuild,
       "-am",
-      "-PtemplatesStage,pluginOutputDir,splunkDeps,missing-artifact-repos",
+      "-PtemplatesStage,pluginOutputDir",
       "-DpluginRunId=" + RandomStringUtils.randomAlphanumeric(16),
       // Skip shading for now due to flakiness / slowness in the process.
       "-DskipShade=" + skipShade,

diff --git a/...src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateDataflowJobResourceManager.java b/...src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateDataflowJobResourceManager.java
@@ -218,7 +218,7 @@ String[] buildMavenStageCommand(
       "-pl",
       moduleBuild,
       "-am",
-      "-PtemplatesStage,pluginOutputDir,splunkDeps,missing-artifact-repos",
+      "-PtemplatesStage,pluginOutputDir",
       // Skip shading for now due to flakiness / slowness in the process.
       "-DskipShade=" + true,
       "-DskipTests",

diff --git a/...test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateDataflowJobResourceManagerTest.java b/...test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateDataflowJobResourceManagerTest.java
@@ -49,7 +49,7 @@ public void testBuildMavenStageCommand() {
     assertThat(String.join(" ", Arrays.copyOfRange(actual, 0, 5)))
         .isEqualTo("mvn compile package -q -f");
     String expected =
-        "-pl metadata,v2/common,v2/spanner-change-streams-to-sharded-file-sink -am -PtemplatesStage,pluginOutputDir,splunkDeps,missing-artifact-repos -DskipShade=true -DskipTests -Dmaven.test.skip -Dcheckstyle.skip -Dmdep.analyze.skip -Dspotless.check.skip -Denforcer.skip -DprojectId=testProject -Dregion=us-central1 -DbucketName=TestBucketName -DgcpTempLocation=TestBucketName -DstagePrefix=TestClassName -DtemplateName=Spanner_Change_Streams_to_Sharded_File_Sink -DunifiedWorker=true -e";
+        "-pl metadata,v2/common,v2/spanner-change-streams-to-sharded-file-sink -am -PtemplatesStage,pluginOutputDir -DskipShade=true -DskipTests -Dmaven.test.skip -Dcheckstyle.skip -Dmdep.analyze.skip -Dspotless.check.skip -Denforcer.skip -DprojectId=testProject -Dregion=us-central1 -DbucketName=TestBucketName -DgcpTempLocation=TestBucketName -DstagePrefix=TestClassName -DtemplateName=Spanner_Change_Streams_to_Sharded_File_Sink -DunifiedWorker=true -e";
     assertThat(String.join(" ", Arrays.copyOfRange(actual, 6, 25))).isEqualTo(expected);
     assertThat(actual[5]).endsWith("pom.xml");
     assertThat(actual[25]).isNotEmpty();

diff --git a/pom.xml b/pom.xml
@@ -391,7 +391,19 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-report-plugin</artifactId>
+        <version>2.6</version>
+      </plugin>
     </plugins>
+    <extensions>
+      <extension>
+        <groupId>com.google.cloud.artifactregistry</groupId>
+        <artifactId>artifactregistry-maven-wagon</artifactId>
+        <version>2.2.0</version>
+      </extension>
+    </extensions>
   </build>
 
   <profiles>
@@ -600,10 +612,32 @@
         </repository>
       </repositories>
     </profile>
+    <profile>
+      <id>internalMaven</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <repositories>
+        <repository>
+          <id>central</id>
+          <name>Maven Central remote repository</name>
+          <url>artifactregistry://us-maven.pkg.dev/artifact-foundry-prod/maven-3p</url>
+          <layout>default</layout>
+          <releases>
+            <enabled>true</enabled>
+          </releases>
+          <snapshots>
+            <enabled>true</enabled>
+          </snapshots>
+        </repository>
+      </repositories>
+    </profile>
     <profile>
       <id>splunkDeps</id>
       <activation>
-        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>!skipSplunkDeps</name>
+        </property>
       </activation>
       <repositories>
         <repository>
@@ -623,6 +657,21 @@
         </pluginRepository>
       </pluginRepositories>
     </profile>
+    <profile>
+      <id>confluentDeps</id>
+      <activation>
+        <property>
+          <name>!skipConfluentDeps</name>
+        </property>
+      </activation>
+      <repositories>
+        <repository>
+          <!-- Required for io.confluent:kafka-avro-serializer artifact -->
+          <id>confluent</id>
+          <url>https://packages.confluent.io/maven/</url>
+        </repository>
+      </repositories>
+    </profile>
     <profile>
       <!-- mvn clean deploy -Prelease -pl plugins/templates-maven-plugin -pl v2/common -am -->
       <id>release</id>

diff --git a/v1/pom.xml b/v1/pom.xml
@@ -948,19 +948,6 @@
   </build>
 
   <profiles>
-    <profile>
-      <id>missing-artifact-repos</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <repositories>
-        <repository>
-          <!-- Required for io.confluent:kafka-avro-serializer artifact -->
-          <id>confluent</id>
-          <url>https://packages.confluent.io/maven/</url>
-        </repository>
-      </repositories>
-    </profile>
     <profile>
       <id>shade</id>
       <activation>

diff --git a/v2/pom.xml b/v2/pom.xml
@@ -467,19 +467,6 @@
     </build>
 
     <profiles>
-        <profile>
-            <id>missing-artifact-repos</id>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
-            <repositories>
-                <repository>
-                    <!-- Required for io.confluent:kafka-avro-serializer artifact -->
-                    <id>confluent</id>
-                    <url>https://packages.confluent.io/maven/</url>
-                </repository>
-            </repositories>
-        </profile>
         <profile>
             <id>shade</id>
             <activation>