From f6bba81ebd36f22c0fa82865953f35c4d54b0500 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 22 May 2023 13:51:56 +0200 Subject: [PATCH 1/4] cicd: New check-junit-test workflow #TASK-4445 --- .github/workflows/check-junit-test.yml | 41 +++++++++++++++++++ .github/workflows/scripts/get_profiles.sh | 30 ++++++++++++++ .github/workflows/test-analysis.yml | 8 +++- .github/workflows/test-long-test-analysis.yml | 14 ------- 4 files changed, 77 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/check-junit-test.yml create mode 100755 .github/workflows/scripts/get_profiles.sh delete mode 100644 .github/workflows/test-long-test-analysis.yml diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml new file mode 100644 index 00000000000..183fee32ea0 --- /dev/null +++ b/.github/workflows/check-junit-test.yml @@ -0,0 +1,41 @@ +name: LONG TEST ANALYSIS + +on: + workflow_dispatch: + inputs: + short_tests: + type: boolean + required: false + default: true + medium_tests: + type: boolean + required: false + default: false + long_tests: + type: boolean + required: false + default: false + mvn_opts: + type: string + required: false + default: "" + +jobs: + test: + name: Test JUnit + runs-on: ubuntu-22.04 + steps: + - name: Building string profiles to run + id: get_profiles + run: | + if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then + chmod +x ./.github/workflows/scripts/get_profiles.sh + echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT + fi + - name: Building string profiles to run + uses: ./.github/workflows/test-analysis.yml + secrets: inherit + with: + test_profile: ${{ steps.get_profiles.outputs.profiles }} + report_context: development + mvn_opts: ${{ inputs.mvn_opts }} diff --git a/.github/workflows/scripts/get_profiles.sh b/.github/workflows/scripts/get_profiles.sh new file mode 100755 index 00000000000..fc205968984 --- /dev/null +++ b/.github/workflows/scripts/get_profiles.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ $# -eq 0 ]; then + echo "The arguments must be 3" + exit 1 +fi +PROFILE="" + +if [ $1 == "true" ]; then + PROFILE="${PROFILE}runShortTests," +fi +if [ $2 == "true" ]; then + PROFILE="${PROFILE}runMediumTests," +fi +if [ $3 == "true" ]; then + PROFILE="${PROFILE}runLongTests" +fi + +if [[ "${PROFILE}" == *"," ]]; then + PROFILE="${PROFILE%?}" +fi + +if [ -z "${PROFILE}" ]; then + echo "There must be at least one active profile" + exit 1 +fi + +echo "${PROFILE}" +exit 0 + diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 8c72b649a33..37f1215a802 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -8,6 +8,10 @@ on: report_context: type: string required: true + mvn_opts: + type: string + required: false + default: "" secrets: SONAR_TOKEN: required: true @@ -61,7 +65,7 @@ jobs: mongodb-version: 6.0 mongodb-replica-set: rs-test - name: Run Junit tests - run: mvn -B verify surefire-report:report -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + run: mvn -B verify surefire-report:report -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' ${{ inputs.mvn_opts }} - name: Upload result dir uses: actions/upload-artifact@v3 with: @@ -73,7 +77,7 @@ jobs: needs: test strategy: matrix: - module: ["opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps"] + module: [ "opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps" ] steps: - name: Download result dir uses: actions/download-artifact@v3 diff --git a/.github/workflows/test-long-test-analysis.yml b/.github/workflows/test-long-test-analysis.yml deleted file mode 100644 index fe516b75a98..00000000000 --- a/.github/workflows/test-long-test-analysis.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: LONG TEST ANALYSIS - -on: - push: - branches: - - TASK-4149 - -jobs: - test: - uses: ./.github/workflows/test-analysis.yml - secrets: inherit - with: - test_profile: runLongTests,runMediumTests,runShortTests - report_context: development From bc7f00e5f0692e947f42e1f9e8f08e124e62967c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 22 May 2023 13:53:38 +0200 Subject: [PATCH 2/4] cicd: Rename check-junit-test #TASK-4445 --- .github/workflows/check-junit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml index 183fee32ea0..58f71978a27 100644 --- a/.github/workflows/check-junit-test.yml +++ b/.github/workflows/check-junit-test.yml @@ -1,4 +1,4 @@ -name: LONG TEST ANALYSIS +name: Check junits on: workflow_dispatch: From 571f76c479ae3e3a56aa79aa6584e40cb53d0f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 24 May 2023 16:12:01 +0100 Subject: [PATCH 3/4] storage: Fix HadoopVariantStatisticsManagerTest::testJulieTool #TASK-4445 --- .../opencga/storage/core/metadata/models/ProjectMetadata.java | 4 ---- .../storage/core/variant/annotation/DummyTestAnnotator.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java index c27111d1c03..e64deedff64 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java @@ -85,10 +85,6 @@ public VariantAnnotationMetadata() { sourceVersion = new ArrayList<>(); } - public VariantAnnotationMetadata(int id, String name, Date creationDate, VariantAnnotatorProgram annotator, - List sourceVersion, DataRelease dataRelease) { - - } public VariantAnnotationMetadata(int id, String name, Date creationDate, VariantAnnotatorProgram annotator, List sourceVersion, DataRelease dataRelease, List privateSources) { this.id = id; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java index 71f5135e08c..33e2860868d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java @@ -59,7 +59,7 @@ public List annotate(List variants) throws VariantAn public ProjectMetadata.VariantAnnotationMetadata getVariantAnnotationMetadata() throws VariantAnnotatorException { return new ProjectMetadata.VariantAnnotationMetadata(-1, null, null, new ProjectMetadata.VariantAnnotatorProgram("MyAnnotator", key, null), - Collections.singletonList(new ObjectMap("data", "genes")), null); + Collections.singletonList(new ObjectMap("data", "genes")), null, null); } } From 37ba31205a843de2f3cd5a81e6d558a751d49056 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 25 May 2023 17:13:24 +0200 Subject: [PATCH 4/4] cicd:Synch with the latest version of workflow from develop --- .github/workflows/check-junit-test.yml | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml index 58f71978a27..f953642c9eb 100644 --- a/.github/workflows/check-junit-test.yml +++ b/.github/workflows/check-junit-test.yml @@ -21,21 +21,36 @@ on: default: "" jobs: - test: + get_profiles: name: Test JUnit runs-on: ubuntu-22.04 + outputs: + profiles: ${{ steps.getter.outputs.profiles }} steps: + - uses: actions/checkout@v3 + with: + fetch-depth: '10' - name: Building string profiles to run - id: get_profiles + id: getter run: | if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then chmod +x ./.github/workflows/scripts/get_profiles.sh echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT fi - - name: Building string profiles to run - uses: ./.github/workflows/test-analysis.yml - secrets: inherit - with: - test_profile: ${{ steps.get_profiles.outputs.profiles }} - report_context: development - mvn_opts: ${{ inputs.mvn_opts }} + + echo_log: + needs: get_profiles + runs-on: ubuntu-22.04 + steps: + - name: Echo profiles + id: log + run: echo "Executing testing profiles -> ${{ needs.get_profiles.outputs.profiles }}" + + test: + needs: [ get_profiles, echo_log ] + uses: ./.github/workflows/test-analysis.yml + with: + test_profile: ${{ needs.get_profiles.outputs.profiles }} + report_context: development + mvn_opts: ${{ inputs.mvn_opts }} + secrets: inherit \ No newline at end of file