From 7569f33ebdab46e389869b5d534e2cb1e79690f6 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Mon, 21 Oct 2024 22:14:30 +0530 Subject: [PATCH] added retry mechanism --- .github/workflows/e2e.yml | 9 +++++ pom.xml | 13 +++++++ .../runners/multitablerunner/TestRunner.java | 3 +- .../runners/sinkrunner/TestRunner.java | 3 +- .../runners/sourcerunner/TestRunner.java | 3 +- .../bigqueryexecute/runner/TestRunner.java | 3 +- .../plugin/bigtable/runners/TestRunner.java | 3 +- .../runners/common/RetryTestRunner.java | 36 +++++++++++++++++++ .../plugin/datastore/runner/TestRunner.java | 3 +- .../gcs/runners/sinkrunner/TestRunner.java | 3 +- .../gcs/runners/sourcerunner/TestRunner.java | 3 +- .../plugin/gcscopy/runners/TestRunner.java | 3 +- .../plugin/gcscreate/runners/TestRunner.java | 3 +- .../plugin/gcsdelete/runners/TestRunner.java | 3 +- .../plugin/gcsmove/runners/TestRunner.java | 3 +- .../pubsub/runners/sinkrunner/TestRunner.java | 3 +- .../runners/sinkrunner/TestRunner.java | 3 +- .../runners/sourcerunner/TestRunner.java | 3 +- 18 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 src/e2e-test/java/io/cdap/plugin/common/runners/common/RetryTestRunner.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 505cc8624c..baf5abd334 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -74,6 +74,15 @@ jobs: - name: Run all e2e tests if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true' run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/${{ matrix.tests }}/**/TestRunner.java + # Step to check if there were failures and run retry runner if needed + - name: Check for Failed Tests and Run Retry Runner + if: always() # Always run this step to check for failures + run: if [ -f ./plugin/target/failed_scenarios.txt ]; then + echo "Found failed scenarios. Running retry tests." + python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/${{ matrix.tests }}/**/RetryTestRunner.java + else + echo "No failed scenarios found." + fi - name: Upload debug files uses: actions/upload-artifact@v3 if: always() diff --git a/pom.xml b/pom.xml index f3b98ceb3d..e5be211b36 100644 --- a/pom.xml +++ b/pom.xml @@ -1202,6 +1202,7 @@ src/e2e-test/java TestRunner.java + RetryTestRunner.java @@ -1254,6 +1255,18 @@ integration-test + + retry-failed-tests + integration-test + + integration-test + + + + ${RETRY_RUNNER} + + + diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java index 6dd78f79ae..6a41219df6 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/bigquerymultitable-sink", "json:target/cucumber-reports/cucumber-bigquerymultitable-sink.json", - "junit:target/cucumber-reports/cucumber-bigquerymultitable-sink.xml"} + "junit:target/cucumber-reports/cucumber-bigquerymultitable-sink.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java index b6085ccb1e..63d44af9a5 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java @@ -32,7 +32,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink", "json:target/cucumber-reports/cucumber-bigquery-sink.json", - "junit:target/cucumber-reports/cucumber-bigquery-sink.xml"} + "junit:target/cucumber-reports/cucumber-bigquery-sink.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sourcerunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sourcerunner/TestRunner.java index 82893e8733..a49510f011 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sourcerunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/runners/sourcerunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/bigquery-source", "json:target/cucumber-reports/cucumber-bigquery-source.json", - "junit:target/cucumber-reports/cucumber-bigquery-source.xml"} + "junit:target/cucumber-reports/cucumber-bigquery-source.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/bigqueryexecute/runner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigqueryexecute/runner/TestRunner.java index 50a03625d0..71e41ff9fb 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigqueryexecute/runner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/bigqueryexecute/runner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/bqExecute", "json:target/cucumber-reports/cucumber-bqExecute.json", - "junit:target/cucumber-reports/cucumber-bqExecute.xml"} + "junit:target/cucumber-reports/cucumber-bqExecute.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java index d135cc7e54..c2483ce93f 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/bigtable", "json:target/cucumber-reports/cucumber-bigtable.json", - "junit:target/cucumber-reports/cucumber-bigtable.xml"} + "junit:target/cucumber-reports/cucumber-bigtable.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/common/runners/common/RetryTestRunner.java b/src/e2e-test/java/io/cdap/plugin/common/runners/common/RetryTestRunner.java new file mode 100644 index 0000000000..01facc57cc --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/common/runners/common/RetryTestRunner.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.common.runners.common; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Retry Test Runner to execute failed scenarios test cases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"@target/failed_scenarios.txt"}, // This reruns only failed scenarios, + glue = {"io.cdap.plugin.gcs.stepsdesign", "io.cdap.plugin.bigquery.stepsdesign", "stepsdesign", + "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.pubsub.stepsdesign", + "io.cdap.plugin.gcsmove.stepsdesign", "io.cdap.plugin.spanner.stepsdesign", + "io.cdap.plugin.gcsdelete.stepsdesign"}, + monochrome = true, + plugin = {"pretty", "json:target/cucumber-reports/failed-scenarios.json"} +) +public class RetryTestRunner { +} diff --git a/src/e2e-test/java/io/cdap/plugin/datastore/runner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/datastore/runner/TestRunner.java index 1057d0c1d7..49db28e320 100644 --- a/src/e2e-test/java/io/cdap/plugin/datastore/runner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/datastore/runner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/datastore", "json:target/cucumber-reports/cucumber-datastore.json", - "junit:target/cucumber-reports/cucumber-datastore.xml"} + "junit:target/cucumber-reports/cucumber-datastore.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcs/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcs/runners/sinkrunner/TestRunner.java index 9526475ac0..3b89c0fde0 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcs/runners/sinkrunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcs/runners/sinkrunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcs-sink", "json:target/cucumber-reports/cucumber-gcs-sink.json", - "junit:target/cucumber-reports/cucumber-gcs-sink.xml"} + "junit:target/cucumber-reports/cucumber-gcs-sink.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcs/runners/sourcerunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcs/runners/sourcerunner/TestRunner.java index b998732aba..06066b3840 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcs/runners/sourcerunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcs/runners/sourcerunner/TestRunner.java @@ -34,7 +34,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcs-source", "json:target/cucumber-reports/cucumber-gcs-source.json", - "junit:target/cucumber-reports/cucumber-gcs-source.xml"} + "junit:target/cucumber-reports/cucumber-gcs-source.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java index 91ae566df9..ab990aa276 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcscopy-action", "json:target/cucumber-reports/cucumber-gcscopy-action.json", - "junit:target/cucumber-reports/cucumber-gcscopy-action.xml"} + "junit:target/cucumber-reports/cucumber-gcscopy-action.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcscreate/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcscreate/runners/TestRunner.java index 0d126a139e..062987700c 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcscreate/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcscreate/runners/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcscreate", "json:target/cucumber-reports/cucumber-gcscreate.json", - "junit:target/cucumber-reports/cucumber-gcscreate.xml"} + "junit:target/cucumber-reports/cucumber-gcscreate.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcsdelete/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcsdelete/runners/TestRunner.java index 4a450e0576..5f65b35bc7 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcsdelete/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcsdelete/runners/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcsdelete", "json:target/cucumber-reports/cucumber-gcsdelete.json", - "junit:target/cucumber-reports/cucumber-gcsdelete.xml"} + "junit:target/cucumber-reports/cucumber-gcsdelete.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java index 0cfc443c58..40255ad139 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcsmove-action", "json:target/cucumber-reports/cucumber-gcsmove-action.json", - "junit:target/cucumber-reports/cucumber-gcsmove-action.xml"} + "junit:target/cucumber-reports/cucumber-gcsmove-action.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/pubsub/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/pubsub/runners/sinkrunner/TestRunner.java index c49b12f04c..592a68aecb 100644 --- a/src/e2e-test/java/io/cdap/plugin/pubsub/runners/sinkrunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/pubsub/runners/sinkrunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/pubsub-sink", "json:target/cucumber-reports/cucumber-pubsub-sink.json", - "junit:target/cucumber-reports/cucumber-pubsub-sink.xml"} + "junit:target/cucumber-reports/cucumber-pubsub-sink.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { diff --git a/src/e2e-test/java/io/cdap/plugin/spanner/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/spanner/runners/sinkrunner/TestRunner.java index f87e35cedd..36b6f9575a 100644 --- a/src/e2e-test/java/io/cdap/plugin/spanner/runners/sinkrunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/spanner/runners/sinkrunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/spanner-sink", "json:target/cucumber-reports/cucumber-spanner-sink.json", - "junit:target/cucumber-reports/cucumber-spanner-sink.xml"} + "junit:target/cucumber-reports/cucumber-spanner-sink.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { } diff --git a/src/e2e-test/java/io/cdap/plugin/spanner/runners/sourcerunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/spanner/runners/sourcerunner/TestRunner.java index e58738bbbe..769d44708a 100644 --- a/src/e2e-test/java/io/cdap/plugin/spanner/runners/sourcerunner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/spanner/runners/sourcerunner/TestRunner.java @@ -31,7 +31,8 @@ monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/spanner-source", "json:target/cucumber-reports/cucumber-spanner-source.json", - "junit:target/cucumber-reports/cucumber-spanner-source.xml"} + "junit:target/cucumber-reports/cucumber-spanner-source.xml", + "rerun:target/failed_scenarios.txt"} // Store failed scenarios here}} ) public class TestRunner { }