Run unit tests during Docker build procedure (#277)

Test containers during build
comorment · Oct 10, 2024 · 3cbf616 · 3cbf616
1 parent 35cfed4
commit 3cbf616
Show file tree

Hide file tree

Showing 20 changed files with 505 additions and 145 deletions.
diff --git a/.github/workflows/docker_build_gwas.yml b/.github/workflows/docker_build_gwas.yml
@@ -11,10 +11,21 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  LATEST_TAG: ghcr.io/comorment/gwas:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+          sparse-checkout: |
+            test-requirements.txt
+            tests
       - 
         name: Docker meta
         id: meta
@@ -41,9 +52,24 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.LATEST_TAG }}
+          file:
+            ./dockerfiles/gwas/Dockerfile
+      - 
+        name: Test
+        run: |
+          git lfs pull -I tests/extras/*.tar.gz
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_gwas.py
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:docker"
           push: False

diff --git a/.github/workflows/docker_build_hello.yml b/.github/workflows/docker_build_hello.yml
@@ -11,10 +11,16 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  TEST_TAG: ghcr.io/comorment/hello:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v3
       - 
         name: Docker meta
         id: meta
@@ -41,9 +47,24 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          file:
+            ./dockerfiles/hello/Dockerfile
+      - 
+        name: Test
+        run: |
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_hello.py
+          docker run --rm -v ${{ github.workspace }}:/home -w /home ${{ env.TEST_TAG }} py.test -v tests/test_hello.py
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:docker"
           push: False

diff --git a/.github/workflows/docker_build_push_gwas.yml b/.github/workflows/docker_build_push_gwas.yml
@@ -6,10 +6,16 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  LATEST_TAG: ghcr.io/comorment/gwas:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v4
       - 
         name: Docker meta
         id: meta
@@ -36,6 +42,21 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.LATEST_TAG }}
+          file:
+            ./dockerfiles/gwas/Dockerfile
+      - 
+        name: Test
+        run: |
+          git lfs pull -I tests/extras/*.tar.gz
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_gwas.py
       -
         name: Build and push
         uses: docker/build-push-action@v5

diff --git a/.github/workflows/docker_build_push_hello.yml b/.github/workflows/docker_build_push_hello.yml
@@ -6,10 +6,16 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  TEST_TAG: ghcr.io/comorment/hello:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v3
       - 
         name: Docker meta
         id: meta
@@ -36,9 +42,24 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          file:
+            ./dockerfiles/hello/Dockerfile
+      - 
+        name: Test
+        run: |
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_hello.py
+          docker run --rm -v ${{ github.workspace }}:/home -w /home ${{ env.TEST_TAG }} py.test -v tests/test_hello.py
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:docker"
           push: ${{ github.event_name != 'pull_request' }}

diff --git a/.github/workflows/docker_build_push_python3.yml b/.github/workflows/docker_build_push_python3.yml
@@ -6,10 +6,16 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  TEST_TAG: ghcr.io/comorment/python3:test
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v3
       - 
         name: Docker meta
         id: meta
@@ -36,9 +42,21 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          file:
+            ./dockerfiles/python3/Dockerfile
+      - name: Test
+        run: |
+          docker run --rm -v ${{ github.workspace }}:/home -w /home ${{ env.TEST_TAG }} py.test -v tests/test_python3.py
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:docker"
           push: ${{ github.event_name != 'pull_request' }}

diff --git a/.github/workflows/docker_build_push_r.yml b/.github/workflows/docker_build_push_r.yml
@@ -6,10 +6,21 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  LATEST_TAG: ghcr.io/comorment/r:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+          sparse-checkout: |
+            test-requirements.txt
+            tests
       - 
         name: Docker meta
         id: meta
@@ -36,6 +47,25 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        env:
+          github_pat: ${{ secrets.COMORMENT_GITHUB_PAT }}
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          secrets: |
+            "github_pat=${{ env.github_pat }}"
+          tags: ${{ env.LATEST_TAG }}
+          file:
+            ./dockerfiles/r/Dockerfile
+      - 
+        name: Test
+        run: |
+          git lfs pull -I tests/extras/*.tar.gz
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_r.py
       -
         name: Build and push
         uses: docker/build-push-action@v6
@@ -85,4 +115,4 @@ jobs:
           docker run --rm --privileged -v $(pwd):/work kaczmarj/apptainer build r.sif docker://"$first_tag"
           oras push "$push_tags" --artifact-type application/vnd.acme.rocket.config r.sif
           rm r.sif
-        shell: sh
+        shell: sh
diff --git a/.github/workflows/docker_build_python3.yml b/.github/workflows/docker_build_python3.yml
@@ -11,10 +11,16 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  TEST_TAG: ghcr.io/comorment/python3:test
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v3
       - 
         name: Docker meta
         id: meta
@@ -40,10 +46,22 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}      
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - 
+        name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          file:
+            ./dockerfiles/python3/Dockerfile
+      - name: Test
+        run: |
+          docker run --rm -v ${{ github.workspace }}:/home -w /home ${{ env.TEST_TAG }} py.test -v tests/test_python3.py
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:docker"
           push: False

diff --git a/.github/workflows/docker_build_r.yml b/.github/workflows/docker_build_r.yml
@@ -11,10 +11,21 @@ on:
     tags:
       - "v*.*.*"
 
+env:
+  LATEST_TAG: ghcr.io/comorment/r:latest
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
+      - 
+        name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+          sparse-checkout: |
+            test-requirements.txt
+            tests
       - 
         name: Docker meta
         id: meta
@@ -41,6 +52,25 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}      
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        env:
+          github_pat: ${{ secrets.COMORMENT_GITHUB_PAT }}
+        with:
+          context: "{{defaultContext}}:docker"
+          load: true
+          secrets: |
+            "github_pat=${{ env.github_pat }}"
+          tags: ${{ env.LATEST_TAG }}
+          file:
+            ./dockerfiles/r/Dockerfile
+      - 
+        name: Test
+        run: |
+          git lfs pull -I tests/extras/*.tar.gz
+          pip3 install -r test-requirements.txt
+          py.test -v tests/test_r.py
       -
         name: Build and push
         uses: docker/build-push-action@v6

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,7 @@ If MD5 sum is not listed for a certain release then it means that the container
 
 ### Added
 
+* Add unit test runs as part of the GitHub Actions workflow for building Docker containers
 * Add Python packages `imbalanced-learn, lightgbm, openpyxl` + PRSice_linux binary to `python3.sif` container
 * Add Conda environment file for project dependencies
 * Add Python packages `scikit-survival, pandas-plink, numba, xmltodict, pyliftover, configparser, intervaltree` to `python3.sif` container
@@ -36,6 +37,8 @@ If MD5 sum is not listed for a certain release then it means that the container
 
 ### Updated
 
+* update testing scripts to support both Docker and Singularity containers
+* Update REGENIE binary to version 3.6 in `gwas.sif` container
 * Update LDAK binary to version 6 in gwas.sif (from 5.2)
 * Rebuilt `gwas.sif` container with md5sum checksum:
 

diff --git a/docker/scripts/R/github.R b/docker/scripts/R/github.R
@@ -14,15 +14,15 @@ packages <- list(
     'jamesliley/cFDR-common-controls' = '9b923fea283e2373ee8effa2909620a1930004bd',
     # gwasvcf deps:
     'MRCIEU/genetics.binaRies' = 'b0324f180476d80c43bba2ab026b72c5be426a92',
-    'MRCIEU/gwasglue2' = '5d4e35bad211299e95e25151972382a1d7b84092',
+    'MRCIEU/gwasglue2' = 'c93b3a1fca7d2eae5d40bd62117091b1ad57f0fa',
     'MRCIEU/gwasvcf' = '477b365da8522e9a47f3bce51993d5f36df49ceb',
-    # gwasglue deps:
+    # gwasglue/TwoSampleMR deps:
     'rondolab/MR-PRESSO' = '3e3c92d7eda6dce0d1d66077373ec0f7ff4f7e87',
-    'MRCIEU/ieugwasr' = '127d1ea5a6ead6fa6c037dcc44ec77d39f952f21',
-    'MRCIEU/TwoSampleMR' = 'c174107cfd9ba47cf2f780849a263f37ac472a0e',
-    'WSpiller/RadialMR' = 'abe00170076284cb79ae711c96d3832da3879267',
-    'gqi/MRMix' = '56afdb2bc96760842405396f5d3f02e60e305039',
-    'stephenslab/susieR' = 'c0314a6113dfb54ea3817103ee01c027849f0eb7',
+    'MRCIEU/ieugwasr' = '8aa24f74d6a6cc5beca77ff204feb2089ae90ffc',
+    'gqi/MRMix'='56afdb2bc96760842405396f5d3f02e60e305039',
+    'WSpiller/RadialMR' = '0ed91f83aebf265a09482561c128c830e58ed697',
+    'MRCIEU/TwoSampleMR' = '578c68fa754c57d764553812bf85d69ecf43b011',
+    'stephenslab/susieR' = 'ced6a9c83a45f792d4d2ef2a9ae0846e164bf92c',
     'MRCIEU/gwasglue' = 'c2d5660eed389e1a9b3e04406b88731d642243f1',
     'noahlorinczcomi/MRBEE' = '96971e346099b89585a6eff4a6f22bbcf25d6ca8',
     'norment/normentR' = 'dfa1fbae9587db6c3613b0405df4f9cfa98ee0e1',