refactor YAML_Template to yaml module

.github/workflows/java-pr.yml

@@ -181,6 +181,34 @@ jobs:
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
         if: always()
+  java_yaml_integration_tests_templates:
+      name: Dataflow Templates YAML Integration Tests
+      timeout-minutes: 240
+      # Run on any runner that matches all the specified runs-on values.
+      runs-on: [self-hosted, it]
+      steps:
+        - name: Checkout Code
+          uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        - name: Setup Environment
+          id: setup-env
+          uses: ./.github/actions/setup-env
+        - name: Run Integration Tests
+          run: |
+            ./cicd/run-it-yaml-tests \
+            --modules-to-build="YAML" \
+            --it-region="us-central1" \
+            --it-project="cloud-teleport-testing" \
+            --it-artifact-bucket="cloud-teleport-testing-it-gitactions"
+        - name: Upload Integration Tests Report
+          uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+          if: always() # always run even if the previous step fails
+          with:
+            name: surefire-test-results
+            path: '**/surefire-reports/TEST-*.xml'
+            retention-days: 1
+        - name: Cleanup Java Environment
+          uses: ./.github/actions/cleanup-java-env
+          if: always()
   java_load_tests_templates:
     if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
     name: Dataflow Templates Load Tests

cicd/cmd/run-it-yaml-tests/main.go

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/flags"
+	"github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/workflows"
+)
+
+func main() {
+	flags.RegisterCommonFlags()
+	flags.RegisterItFlags()
+	flag.Parse()
+
+	// Run mvn install before running integration tests
+	mvnFlags := workflows.NewMavenFlags()
+	err := workflows.MvnCleanInstall().Run(
+		mvnFlags.IncludeDependencies(),
+		mvnFlags.IncludeDependents(),
+		mvnFlags.SkipDependencyAnalysis(),
+		mvnFlags.SkipCheckstyle(),
+		mvnFlags.SkipJib(),
+		mvnFlags.SkipTests(),
+		mvnFlags.SkipJacoco(),
+		mvnFlags.SkipShade(),
+		mvnFlags.ThreadCount(8),
+		mvnFlags.InternalMaven())
+	if err != nil {
+		log.Fatalf("%v\n", err)
+	}
+
+	// Run integration tests
+	mvnFlags = workflows.NewMavenFlags()
+	err = workflows.MvnVerify().Run(
+		mvnFlags.IncludeDependencies(),
+		mvnFlags.IncludeDependents(),
+		mvnFlags.SkipDependencyAnalysis(),
+		mvnFlags.SkipCheckstyle(),
+		mvnFlags.SkipJib(),
+		mvnFlags.SkipShade(),
+		mvnFlags.RunYamlIntegrationTests(),
+		mvnFlags.ThreadCount(4),
+		mvnFlags.IntegrationTestParallelism(3),
+		mvnFlags.StaticBigtableInstance("teleport"),
+		mvnFlags.StaticSpannerInstance("teleport"),
+		mvnFlags.InternalMaven(),
+		flags.Region(),
+		flags.Project(),
+		flags.ArtifactBucket(),
+		flags.StageBucket(),
+		flags.HostIp(),
+		flags.PrivateConnectivity(),
+		flags.SpannerHost(),
+		flags.FailureMode(),
+		flags.RetryFailures(),
+		flags.StaticOracleHost(),
+		flags.StaticOracleSysPassword(),
+		flags.CloudProxyHost(),
+		flags.CloudProxyMySqlPort(),
+		flags.CloudProxyPostgresPort(),
+		flags.CloudProxyPassword(),
+		flags.UnifiedWorkerHarnessContainerImage(),
+		flags.CloudProxyPassword())
+	if err != nil {
+		log.Fatalf("%v\n", err)
+	}
+	log.Println("Build Successful!")
+}

cicd/internal/flags/common-flags.go

@@ -26,6 +26,7 @@ const (
 	DEFAULT = "DEFAULT" // Modules other than those excluded
 	KAFKA   = "KAFKA"
 	SPANNER = "SPANNER"
+	YAML    = "YAML"
 )
 
 // Avoid making these vars public.
@@ -48,6 +49,7 @@ var (
 			"v2/spanner-to-sourcedb/",
 			"v2/spanner-custom-shard",
 			"plugins/templates-maven-plugin"},
+		YAML: {"yaml/"},
 	}
 )
 

cicd/internal/workflows/maven-workflows.go

@@ -50,6 +50,7 @@ type MavenFlags interface {
 	SkipIntegrationTests() string
 	FailAtTheEnd() string
 	RunIntegrationTests() string
+	RunYamlIntegrationTests() string
 	RunIntegrationSmokeTests() string
 	RunLoadTests() string
 	RunLoadTestObserver() string
@@ -111,6 +112,10 @@ func (*mvnFlags) RunIntegrationTests() string {
 	return "-PtemplatesIntegrationTests"
 }
 
+func (*mvnFlags) RunYamlIntegrationTests() string {
+	return "-PtemplatesYamlIntegrationTests"
+}
+
 func (*mvnFlags) RunIntegrationSmokeTests() string {
 	return "-PtemplatesIntegrationSmokeTests"
 }

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/JDBCBaseIT.java

@@ -20,7 +20,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.function.Function;
 import org.apache.beam.it.common.utils.IORedirectUtil;
+import org.apache.beam.it.gcp.storage.GcsResourceManager;
 import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,10 +38,10 @@
   private static final String JAR_SUFFIX = ".jar";
 
   // The JDBC Driver fully-qualified class names
-  protected static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
-  protected static final String POSTGRES_DRIVER = "org.postgresql.Driver";
-  protected static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver";
-  protected static final String MSSQL_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+  public static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
+  public static final String POSTGRES_DRIVER = "org.postgresql.Driver";
+  public static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver";
+  public static final String MSSQL_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
 
   // The relative path to the JDBC drivers under Maven's `.m2/repository` directory
   private static final String MYSQL_LOCAL_PATH = "mysql/mysql-connector-java";
@@ -63,6 +65,10 @@
   @Before
   public void setUpJDBC() throws IOException {
 
+    uploadArtifacts(gcsClient);
+  }
+
+  public static void uploadArtifacts(GcsResourceManager gcsClient) throws IOException {
     String basePath = getMvnBaseRepoPath();
 
     String mySqlDriverGCSRelativePath = GCS_PREFIX + mySqlDriverLocalJar();
@@ -77,19 +83,35 @@
   }
 
   protected String mySqlDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + mySqlDriverLocalJar());
+    return mySqlDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String mySqlDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + mySqlDriverLocalJar());
   }
 
   protected String postgresDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + postgresDriverLocalJar());
+    return postgresDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String postgresDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + postgresDriverLocalJar());
   }
 
   protected String oracleDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + oracleDriverLocalJar());
+    return oracleDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String oracleDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + oracleDriverLocalJar());
   }
 
   protected String msSqlDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + msSqlDriverLocalJar());
+    return msSqlDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String msSqlDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + msSqlDriverLocalJar());
   }
 
   private static String mySqlDriverLocalJar() {
@@ -125,7 +147,7 @@
     return String.join("/", basePath, MSSQL_LOCAL_PATH, MSSQL_VERSION, msSqlDriverLocalJar());
   }
 
-  private String getMvnBaseRepoPath() {
+  private static String getMvnBaseRepoPath() {
     // Try to get specified maven repo path from args
     if (System.getProperty("mavenRepository") != null) {
       String basePath = System.getProperty("mavenRepository");

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java

@@ -137,16 +137,20 @@
 
   @Before
   public void setUpBase() throws ExecutionException {
+    setUp(null);
+  }
 
+  protected void setUp(TemplateIntegrationTest annotation) throws ExecutionException {
     testId = PipelineUtils.createJobName("test", 10);
 
-    TemplateIntegrationTest annotation = null;
     MultiTemplateIntegrationTest multiAnnotation =
         getClass().getAnnotation(MultiTemplateIntegrationTest.class);
     usingDirectRunner = System.getProperty("directRunnerTest") != null;
     try {
       Method testMethod = getClass().getMethod(testName);
-      annotation = testMethod.getAnnotation(TemplateIntegrationTest.class);
+      if (annotation == null) {
+        annotation = testMethod.getAnnotation(TemplateIntegrationTest.class);
+      }
       Category category = testMethod.getAnnotation(Category.class);
       if (category != null) {
         usingDirectRunner =
@@ -167,7 +171,7 @@
     }
     if (annotation != null && multiAnnotation != null) {
       LOG.warn(
-          "{} specifies both @TemplateIntegrationTest or @MultiTemplateIntegrationTest, please use"
+          "{} specifies both @TemplateIntegrationTest and @MultiTemplateIntegrationTest, please use"
               + " only of either.",
           getClass());
       return;
@@ -387,7 +391,7 @@
       "mvn",
       "compile",
       "package",
-      "-q",
+      //      "-q",
       "-f",
       pomPath,
       "-pl",

metadata/src/main/java/com/google/cloud/teleport/metadata/YAMLTemplateIntegrationTest.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.cloud.teleport.metadata;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/** Annotation that marks the test of a Dataflow Template. */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface YAMLTemplateIntegrationTest {}