temporarily increase timeout

Signed-off-by: Jeffrey Kinard <[email protected]>
GoogleCloudPlatform · Nov 1, 2024 · deb8f2f · deb8f2f
1 parent 0bcc4b6
commit deb8f2f
Show file tree

Hide file tree

Showing 11 changed files with 337 additions and 212 deletions.
diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml
@@ -154,7 +154,7 @@ jobs:
   java_integration_tests_templates:
     name: Dataflow Templates Integration Tests
     needs: [java_integration_smoke_tests_templates]
-    timeout-minutes: 240
+    timeout-minutes: 180
     # Run on any runner that matches all the specified runs-on values.
     runs-on: [self-hosted, it]
     steps:
@@ -181,6 +181,29 @@ jobs:
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
         if: always()
+  java_stage_templates_test:
+    name: Dataflow Templates Stage Templates Test
+    needs: [java_integration_smoke_tests_templates]
+    timeout-minutes: 180
+    runs-on: [self-hosted, it]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Stage Templates
+        run: |
+          mvn clean verify -PtemplatesStage \
+            -DprojectId="cloud-teleport-testing" \
+            -DbucketName="cloud-teleport-testing-it-gitactions" \
+            -DstagePrefix="test-images" \
+            -DgenerateSBOM="true" \
+            -Dmaven.test.skip -T8 -e \
+            --settings=.mvn/settings.xml
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
   java_load_tests_templates:
     if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
     name: Dataflow Templates Load Tests

diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java
@@ -387,6 +387,7 @@ private String[] buildMavenStageCommand(
       "mvn",
       "compile",
       "package",
+      "-q",
       "-f",
       pomPath,
       "-pl",

diff --git a/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/TemplatePluginUtils.java b/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/TemplatePluginUtils.java
@@ -30,7 +30,8 @@ public final class TemplatePluginUtils {
    * @param inputStream The InputStream to redirect.
    * @param log The logger to redirect the InputStream to.
    */
-  public static void redirectLinesLog(InputStream inputStream, Logger log) {
+  public static void redirectLinesLog(
+      InputStream inputStream, Logger log, StringBuilder cloudBuildLogs) {
     new Thread(
             () -> {
               try (InputStreamReader isr =
@@ -40,6 +41,9 @@ public static void redirectLinesLog(InputStream inputStream, Logger log) {
                 String line;
                 while ((line = bis.readLine()) != null) {
                   log.info(line);
+                  if (cloudBuildLogs != null && line.contains("Logs are available at")) {
+                    cloudBuildLogs.append(line);
+                  }
                 }
               } catch (Exception e) {
                 log.error("Error redirecting stream", e);

diff --git a/...ven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesReleaseMojo.java b/...ven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesReleaseMojo.java
@@ -116,7 +116,7 @@ public class TemplatesReleaseMojo extends TemplatesBaseMojo {
   @Parameter(defaultValue = "${unifiedWorker}", readonly = true, required = false)
   protected boolean unifiedWorker;
 
-  @Parameter(defaultValue = "false", property = "generateSBOM", readonly = true, required = false)
+  @Parameter(defaultValue = "true", property = "generateSBOM", readonly = true, required = false)
   protected boolean generateSBOM;
 
   public void execute() throws MojoExecutionException {

diff --git a/...s-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesRunMojo.java b/...s-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesRunMojo.java
@@ -133,7 +133,7 @@ public class TemplatesRunMojo extends TemplatesBaseMojo {
   @Parameter(defaultValue = "${parameters}", readonly = true, required = true)
   protected String parameters;
 
-  @Parameter(defaultValue = "true", property = "generateSBOM", readonly = true, required = false)
+  @Parameter(defaultValue = "false", property = "generateSBOM", readonly = true, required = false)
   protected boolean generateSBOM;
 
   public void execute() throws MojoExecutionException {

diff --git a/...maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java b/...maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java
@@ -155,7 +155,6 @@ public class TemplatesStageMojo extends TemplatesBaseMojo {
   @Parameter(defaultValue = "${unifiedWorker}", readonly = true, required = false)
   protected boolean unifiedWorker;
 
-<<<<<<< HEAD
   @Parameter(defaultValue = "${saSecretName}", readonly = true, required = false)
   protected String saSecretName;
 
@@ -283,8 +282,8 @@ public String stageTemplate(
       if (generateSBOM) {
         String imagePath = imageSpec.getImage();
         File buildDir = new File(outputClassesDirectory.getAbsolutePath());
-        Failsafe.with(sbomRetryPolicy()).run(() -> generateSystemSBOM(imagePath));
         performVulnerabilityScanAndGenerateUserSBOM(imagePath, projectId, buildDir);
+        Failsafe.with(sbomRetryPolicy()).run(() -> generateSystemSBOM(imagePath));
       }
       return stagedTemplate;
     }
@@ -595,7 +594,9 @@ private void stageFlexJavaTemplate(
               destRequirements.toPath(),
               StandardCopyOption.REPLACE_EXISTING);
         }
+
         // Generate Dockerfile
+        LOG.info("Generating dockerfile " + dockerfilePath);
         Set<String> directoriesToCopy = Set.of(containerName);
         DockerfileGenerator.Builder dockerfileBuilder =
             DockerfileGenerator.builder(
@@ -646,29 +647,6 @@ private void stageFlexJavaTemplate(
       if (definition.getTemplateAnnotation().stageImageOnly()) {
         return;
       }
-
-      flexTemplateBuildCmd =
-          new String[] {
-            "gcloud",
-            "dataflow",
-            "flex-template",
-            "build",
-            templatePath,
-            "--image",
-            imagePath,
-            "--project",
-            projectId,
-            "--sdk-language",
-            definition.getTemplateAnnotation().type().name(),
-            "--metadata-file",
-            outputClassesDirectory.getAbsolutePath() + "/" + metadataFile,
-            "--additional-user-labels",
-            "goog-dataflow-provided-template-name="
-                + currentTemplateName.toLowerCase()
-                + ",goog-dataflow-provided-template-version="
-                + TemplateDefinitionsParser.parseVersion(stagePrefix)
-                + ",goog-dataflow-provided-template-type=flex"
-          };
     }
 
     // Skip GCS spec file creation
@@ -735,6 +713,7 @@ private void stageFlexYamlTemplate(
       }
 
       // Generate Dockerfile
+      LOG.info("Generating dockerfile " + dockerfilePath);
       DockerfileGenerator.Builder dockerfileBuilder =
           DockerfileGenerator.builder(
                   definition.getTemplateAnnotation().type(),
@@ -757,6 +736,7 @@ private void stageFlexYamlTemplate(
       dockerfileBuilder.build().generate();
     }
 
+    LOG.info("Staging YAML image using Dockerfile");
     stageYamlUsingDockerfile(imagePath, containerName);
 
     // Skip GCS spec file creation
@@ -809,7 +789,6 @@ private void stageFlexPythonTemplate(
       throws IOException, InterruptedException, TemplateException {
     String dockerfileContainer = outputClassesDirectory.getPath() + "/" + containerName;
     String dockerfilePath = dockerfileContainer + "/Dockerfile";
-    LOG.info("Generating dockerfile " + dockerfilePath);
     File dockerfile = new File(dockerfilePath);
     if (!dockerfile.exists()) {
       List<String> filesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy());
@@ -832,6 +811,7 @@ private void stageFlexPythonTemplate(
       }
 
       // Generate Dockerfile
+      LOG.info("Generating dockerfile " + dockerfilePath);
       DockerfileGenerator.Builder dockerfileBuilder =
           DockerfileGenerator.builder(
                   definition.getTemplateAnnotation().type(),
@@ -851,6 +831,8 @@ private void stageFlexPythonTemplate(
 
       dockerfileBuilder.build().generate();
     }
+
+    LOG.info("Staging PYTHON image using Dockerfile");
     stagePythonUsingDockerfile(imagePath, containerName);
 
     // Skip GCS spec file creation
@@ -935,6 +917,8 @@ private void stageYamlUsingDockerfile(String imagePath, String yamlTemplateName)
                   : ""));
     }
 
+    LOG.info("Submitting Cloud Build job with config: " + cloudbuildFile.getAbsolutePath());
+    StringBuilder cloudBuildLogs = new StringBuilder();
     Process stageProcess =
         runCommand(
             new String[] {
@@ -950,12 +934,15 @@ private void stageYamlUsingDockerfile(String imagePath, String yamlTemplateName)
               "--project",
               projectId
             },
-            directory);
+            directory,
+            cloudBuildLogs);
 
     // Ideally this should raise an exception, but in GitHub Actions this returns NZE even for
     // successful runs.
     if (stageProcess.waitFor() != 0) {
-      LOG.warn("Possible error building container image using gcloud. Check logs for details.");
+      LOG.warn(
+          "Possible error building container image using gcloud. Check logs for details. {}",
+          cloudBuildLogs);
     }
   }
 
@@ -1000,6 +987,8 @@ private void stagePythonUsingDockerfile(String imagePath, String containerName)
                   : ""));
     }
 
+    LOG.info("Submitting Cloud Build job with config: " + cloudbuildFile.getAbsolutePath());
+    StringBuilder cloudBuildLogs = new StringBuilder();
     Process stageProcess =
         runCommand(
             new String[] {
@@ -1015,12 +1004,15 @@ private void stagePythonUsingDockerfile(String imagePath, String containerName)
               "--project",
               projectId
             },
-            directory);
+            directory,
+            cloudBuildLogs);
 
     // Ideally this should raise an exception, but in GitHub Actions this returns NZE even for
     // successful runs.
     if (stageProcess.waitFor() != 0) {
-      LOG.warn("Possible error building container image using gcloud. Check logs for details.");
+      LOG.warn(
+          "Possible error building container image using gcloud. Check logs for details. {}",
+          cloudBuildLogs);
     }
   }
 
@@ -1088,6 +1080,8 @@ private void stageFlexTemplateUsingCloudBuild(File tarFile, String imagePath)
               + "  requestedVerifyOption: VERIFIED");
     }
 
+    LOG.info("Submitting Cloud Build job with config: " + cloudbuildFile.getAbsolutePath());
+    StringBuilder cloudBuildLogs = new StringBuilder();
     Process stageProcess =
         runCommand(
             new String[] {
@@ -1099,12 +1093,15 @@ private void stageFlexTemplateUsingCloudBuild(File tarFile, String imagePath)
               "--project",
               projectId
             },
-            directory);
+            directory,
+            cloudBuildLogs);
 
     // Ideally this should raise an exception, but in GitHub Actions this returns NZE even for
     // successful runs.
     if (stageProcess.waitFor() != 0) {
-      LOG.warn("Possible error building container image using gcloud. Check logs for details.");
+      LOG.warn(
+          "Possible error building container image using gcloud. Check logs for details. {}",
+          cloudBuildLogs);
     }
   }
 
@@ -1152,7 +1149,9 @@ private void stageXlangUsingDockerfile(String imagePath, String containerName)
                       + "  requestedVerifyOption: VERIFIED"
                   : ""));
     }
-    LOG.info("Submitting cloudbuild job with config: " + cloudbuildFile.getAbsolutePath());
+
+    LOG.info("Submitting Cloud Build job with config: " + cloudbuildFile.getAbsolutePath());
+    StringBuilder cloudBuildLogs = new StringBuilder();
     Process stageProcess =
         runCommand(
             new String[] {
@@ -1168,12 +1167,15 @@ private void stageXlangUsingDockerfile(String imagePath, String containerName)
               "--project",
               projectId
             },
-            directory);
+            directory,
+            cloudBuildLogs);
 
     // Ideally this should raise an exception, but in GitHub Actions this returns NZE even for
     // successful runs.
     if (stageProcess.waitFor() != 0) {
-      LOG.warn("Possible error building container image using gcloud. Check logs for details.");
+      LOG.warn(
+          "Possible error building container image using gcloud. Check logs for details. {}",
+          cloudBuildLogs);
     }
   }
 
@@ -1250,6 +1252,8 @@ private static void performVulnerabilityScanAndGenerateUserSBOM(
               + ":latest");
     }
 
+    LOG.info("Submitting Cloud Build job with config: " + cloudbuildFile.getAbsolutePath());
+    StringBuilder cloudBuildLogs = new StringBuilder();
     Process stageProcess =
         runCommand(
             new String[] {
@@ -1261,22 +1265,24 @@ private static void performVulnerabilityScanAndGenerateUserSBOM(
               "--project",
               projectId
             },
-            buildDir);
+            buildDir,
+            cloudBuildLogs);
 
     if (stageProcess.waitFor() != 0) {
-      throw new IllegalStateException("Error scanning container. Check logs for details.");
+      throw new IllegalStateException(
+          "Error scanning container. Check logs for details. " + cloudBuildLogs);
     }
   }
 
   private static void generateSystemSBOM(String imagePath)
       throws InterruptedException, IOException {
     LOG.info("Generating system SBOM for {}...", imagePath);
-    Process stageProcess =
+    Process sbomCommand =
         runCommand(
             new String[] {"gcloud", "artifacts", "sbom", "export", "--uri", imagePath + ":latest"},
             null);
 
-    if (stageProcess.waitFor() != 0) {
+    if (sbomCommand.waitFor() != 0) {
       throw new IllegalStateException("Error generating SBOM. Check logs for details.");
     }
   }
@@ -1292,12 +1298,17 @@ private static <T> RetryPolicy<T> sbomRetryPolicy() {
         .build();
   }
 
-  private static Process runCommand(String[] gcloudBuildsCmd, File directory) throws IOException {
+  private static Process runCommand(
+      String[] gcloudBuildsCmd, File directory, StringBuilder cloudBuildLogs) throws IOException {
     LOG.info("Running: {}", String.join(" ", gcloudBuildsCmd));
 
     Process process = Runtime.getRuntime().exec(gcloudBuildsCmd, null, directory);
-    TemplatePluginUtils.redirectLinesLog(process.getInputStream(), LOG);
-    TemplatePluginUtils.redirectLinesLog(process.getErrorStream(), LOG);
+    TemplatePluginUtils.redirectLinesLog(process.getInputStream(), LOG, cloudBuildLogs);
+    TemplatePluginUtils.redirectLinesLog(process.getErrorStream(), LOG, cloudBuildLogs);
     return process;
   }
+
+  private static Process runCommand(String[] gcloudBuildsCmd, File directory) throws IOException {
+    return runCommand(gcloudBuildsCmd, directory, null);
+  }
 }
diff --git a/python/generate_all_dependencies.sh b/python/generate_all_dependencies.sh
@@ -23,7 +23,7 @@ sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_yaml_requiremen
 cp $SCRIPTPATH/__build__/default_python_requirements.txt $SCRIPTPATH/../python/src/main/python/word-count-python/requirements.txt
 
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/yaml-template/requirements.txt
-cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/yaml-template/job-builder-server/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/job-builder-server/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-elasticsearch/src/main/resources/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-googlecloud/src/main/resources/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-splunk/src/main/resources/requirements.txt

diff --git a/python/src/main/java/com/google/cloud/teleport/templates/python/JobBuilderServer.java b/python/src/main/java/com/google/cloud/teleport/templates/python/JobBuilderServer.java
@@ -28,7 +28,7 @@
             + "This image is based on the Yaml Template image, but should not be run as a template itself, or released."
             + "as part of the Google-provided Dataflow Templates suite."
             + "For more information on Beam YAML, see https://cloud.google.com/dataflow/docs/guides/job-builder",
-    flexContainerName = "yaml-template/job-builder-server",
+    flexContainerName = "job-builder-server",
     entryPoint = {"python", "server.py"},
     filesToCopy = {"server.py", "requirements.txt"},
     stageImageOnly = true)