Helm Unit Tests (#29)

* Add some bare bones unittests

* WS

* Rename to unit_test

* reformat cronjob test

* Test seem to be working

* ws

* lowercase

* ws

* Fix ingress keys and test elasticsearch password presence

* use string

* try dep update

* use ct

* lint passing

* add build

* add bitnami

* remove some unnecessary

* Remove a few more lines

* Update values.yaml

* Need to reference seqr-secrets

* Simplify cronjob tests

* Remove comment

* Rename step

* Fix bug

* Try hail search install

* some bugfixes

* Comment out for now

* just remove for now

* ws

* Remove unnecessary

* Try to run seqr

* Use localhost

* Try to be more explicit

* revert some stuff

* connect to postgres on local

* Try to skip db setup

* Add some comments

.github/workflows/release_chart.yaml

@@ -1,7 +1,10 @@
 name: Release Charts
 
 on:
-  push:
+  workflow_run:
+    workflows: ["unit tests"]
+    types:
+      - completed
     branches:
       - main
 

.github/workflows/unit_tests.yaml

@@ -0,0 +1,64 @@
+name: unit tests
+
+# Run the test suite on pushes (incl. merges) to main
+# Run the test suite when a PR is opened, pushed to, or reopened
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: "super-secure-password"
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up chart-testing
+        uses: helm/[email protected]
+      - name: Add bitnami
+        run: helm repo add bitnami https://charts.bitnami.com/bitnami
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Chart Lint
+        run: ct lint --all
+      - name: Create kind cluster
+        uses: helm/[email protected]
+      - name: Run Chart Templating Unit Tests
+        run: python3 -m unittest unit_test/**/*.py
+      - name: Test Seqr Chart Installation
+        run: |
+          # This hack us skip some expensive setup within the seqr startup
+          export PGPASSWORD='super-secure-password'; psql -h localhost -U postgres -c 'CREATE DATABASE seqrdb';
+
+          # Add secrets
+          kubectl create secret generic postgres-secrets --from-literal=password='super-secure-password'
+          kubectl create secret generic seqr-secrets --from-literal=django_key='securely-generated-key'
+
+          # The extra variable is to allow kubernetes to ping our postrgres running locally
+          ct install --charts charts/seqr --namespace default --helm-extra-set-args "--set=environment.POSTGRES_SERVICE_HOSTNAME=172.17.0.1"

charts/hail-search/Chart.yaml

@@ -4,6 +4,9 @@ description: A Helm chart for deploying the hail backend of Seqr, an open source
 sources:
   - https://github.com/broadinstitute/seqr
   - https://github.com/broadinstitute/seqr-helm
+maintainers:
+  - name: seqr
+    email: [email protected]
 
 # A chart can be either an 'application' or a 'library' chart.
 #

charts/hail-search/values.yaml

@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 4
+replicaCount: 1
 
 image:
   repository: gcr.io/seqr-project/seqr-hail-search
@@ -26,7 +26,7 @@ service:
   type: ClusterIP
   port: 5000
 
-resources: 
+resources:
   {}
 
 # default environment variables
@@ -52,7 +52,7 @@ initContainers: |-
   {{ end }}
   {{- range $path, $queries := $.Values.cachedReferenceDatasetQueries -}}
   {{- range $query := $queries -}}
-  - name: sync-cached-reference-dataset-queries-{{ $path | replace "/" "-" | replace "_" "-" | lower}}
+  - name: sync-crdq-{{ $path | replace "/" "-" | replace "_" "-" | lower}}-{{ $query | replace "_" "-" | lower}}
     image: gcr.io/google.com/cloudsdktool/google-cloud-cli:latest
     command: ['gsutil', '-m', 'cp', '-r', '{{ $.Values.sourceCachedReferenceDatasetQueriesDir }}/{{ $path }}/cached_reference_dataset_queries/{{ $query }}.ht', '{{ $.Values.environment.DATASETS_DIR }}/{{ $path }}']
     volumeMounts:
@@ -81,7 +81,7 @@ affinity: |-
                 - seqr
           topologyKey: "kubernetes.io/hostname"
 
-nodeSelector: 
+nodeSelector:
   {}
 
 sourceCachedReferenceDatasetQueriesDir: 'gs://seqr-reference-data/v03'

charts/seqr/Chart.yaml

@@ -5,6 +5,9 @@ home: https://seqr.broadinstitute.org
 sources:
   - https://github.com/broadinstitute/seqr
   - https://github.com/broadinstitute/seqr-helm
+maintainers:
+  - name: seqr
+    email: [email protected]
 # A chart can be either an 'application' or a 'library' chart.
 #
 # Application charts are a collection of templates that can be packaged into versioned archives

charts/seqr/values.yaml

@@ -103,13 +103,13 @@ environment:
   ELASTICSEARCH_SERVICE_PORT: "9200"
   # -- The URL protocol that seqr should use to connect to elasticsearch
   ELASTICSEARCH_PROTOCOL: "http"
-  #ELASTICSEARCH_CA_PATH: "/elasticsearch-certs/ca.crt"
+  # ELASTICSEARCH_CA_PATH: "/elasticsearch-certs/ca.crt"
   # KIBANA_SERVICE_HOSTNAME: "kibana-kb-http"
   # KIBANA_SERVICE_PORT: "5601"
   # -- The hostname of the redis cache that seqr should use
   REDIS_SERVICE_HOSTNAME: "seqr-redis-master"
   # -- The port that the seqr server should listen on
-  SEQR_SERVICE_PORT: "8000" # TODO: this should probably always match the service.port setting?
+  SEQR_SERVICE_PORT: "8000"  # TODO: this should probably always match the service.port setting?
   # -- If storing static media files in a local filesystem, the path to that filesystem
   STATIC_MEDIA_DIR: null
 
@@ -170,7 +170,7 @@ jobAfterHook: ""
 jobBeforeHook: ""
 cronJobs: {}
 # Examples
-#cronJobs:
+# cronJobs:
 #  - name: sleep
 #    schedule: "5-59/5 * * * *"
 #    command: "/bin/sleep 2"

unit_test/hail_search/test_hail_search_chart.py

@@ -0,0 +1,27 @@
+import os
+import subprocess
+import unittest
+
+WORK_DIR = os.path.dirname(__file__)
+DEFAULT_ARGS = [
+    'helm',
+    'install',
+    'test-hail-search', 
+    'charts/hail-search', 
+    '--dry-run',
+    '--debug', 
+    '-f',
+    os.path.join(WORK_DIR, 'values.yaml')
+]
+
+class TestHailSearchChart(unittest.TestCase):
+
+    def test_values(self):
+        p = subprocess.run(DEFAULT_ARGS, capture_output=True, text=True) # NB: text=True here to avoid opening the output in binary mode
+        p.check_returncode()
+        self.assertIn('serviceAccountName: test-hail-search', p.stdout)
+        self.assertIn('name: sync-datasets-grch38-snv-indel', p.stdout)
+
+
+if __name__ == '__main__':
+    unittest.main()

unit_test/hail_search/values.yaml

@@ -0,0 +1,5 @@
+global:
+  hail_search:
+    sourceDatasetsDir: 's3://datasets/v01'
+    datasetVersions:
+      GRCh38/SNV_INDEL: 2023-11-02T23-31-23.149902+00-00

unit_test/seqr/incorrectly-formatted-cronjob.yaml

@@ -0,0 +1,4 @@
+cronJobs:
+  - name: test/cron/1
+    schedule: "1 0 * * 0"
+    command: "python /seqr/manage.py test_cron_1"

unit_test/seqr/no-deployment-sidecars.yaml

@@ -0,0 +1 @@
+deployment_sidecars: {}

unit_test/seqr/no-service-account.yaml

@@ -0,0 +1,2 @@
+serviceAccount:
+  create: false

unit_test/seqr/test_seqr_chart.py

@@ -0,0 +1,51 @@
+import os
+import subprocess
+import unittest
+
+WORK_DIR = os.path.dirname(__file__)
+DEFAULT_ARGS = [
+    'helm',
+    'install',
+    'test-seqr', 
+    'charts/seqr', 
+    '--dry-run',
+    '--debug', 
+    '-f',
+    os.path.join(WORK_DIR, 'values.yaml')
+]
+
+class TestSeqrChart(unittest.TestCase):
+
+    def test_open_source_values(self):
+        p = subprocess.run(DEFAULT_ARGS[:-2], capture_output=True, text=True) # NB: text=True here to avoid opening the output in binary mode
+        p.check_returncode()
+        self.assertEqual(p.stdout.count('kind: CronJob'), 0)
+
+    def test_values(self):
+        p = subprocess.run(DEFAULT_ARGS, capture_output=True, text=True)
+        p.check_returncode()
+        self.assertIn('kind: Deployment\nmetadata:\n  name: seqr', p.stdout)
+        self.assertIn('echo starting CronJob test-cron-1;', p.stdout)
+        self.assertIn('iam.gke.io/gcp-service-account: [email protected]', p.stdout)
+        self.assertIn('cloud-sql-proxy', p.stdout)
+        self.assertIn('serviceAccountName: test-seqr', p.stdout)
+        self.assertIn('SEQR_ES_PASSWORD', p.stdout)
+        self.assertEqual(p.stdout.count('kind: CronJob'), 2)
+
+    def test_no_deployment_sidecars(self):
+        p = subprocess.run([*DEFAULT_ARGS, '-f', os.path.join(WORK_DIR, 'no-deployment-sidecars.yaml')], capture_output=True, text=True)
+        p.check_returncode()
+        self.assertNotIn('cloud-sql-proxy', p.stdout)
+
+    def test_no_service_account(self):
+        p = subprocess.run([*DEFAULT_ARGS, '-f', os.path.join(WORK_DIR, 'no-service-account.yaml')], capture_output=True, text=True)
+        p.check_returncode()
+        self.assertNotIn('serviceAccountName: test-seqr\n', p.stdout)
+
+    def test_incorrectly_formatted_cronjob(self):
+        p = subprocess.run([*DEFAULT_ARGS, '-f', os.path.join(WORK_DIR, 'incorrectly-formatted-cronjob.yaml')], capture_output=True, text=True)
+        self.assertRaises(subprocess.CalledProcessError, p.check_returncode)
+        self.assertIn('invalid resource name "test-seqr-test/cron/1-cronjob": [may not contain \'/\']\nhelm.go', p.stderr)
+
+if __name__ == '__main__':
+    unittest.main()

unit_test/seqr/values.yaml

@@ -0,0 +1,112 @@
+replicaCount: 4
+podAnnotations:
+  linkerd.io/inject: enabled
+
+nodeSelector:
+  cloud.google.com/gke-nodepool: "test-pool"
+
+deployment_type: "test"
+cluster_hostname: &cluster_hostname "test-seqr.myinstitute.org"
+enable_elasticsearch_auth: true
+
+gunicorn_worker_threads: 20 # number of webserver threads
+
+resources:
+  requests:
+    memory: "0.5Gi"
+    cpu: "0.05"
+  limits:
+    memory: "35Gi"
+    cpu: "8"
+
+environment:
+  DEPLOYMENT_TYPE: "test"
+  BASE_URL: "https://test-seqr.myinstitute.org/"
+
+redis:
+  enabled: yes
+  architecture: standalone
+  auth:
+    enabled: no
+  master:
+    podAnnotations:
+      linkerd.io/inject: enabled
+      config.linkerd.io/opaque-ports: "6379"
+    resources:
+      limits:
+        memory: "10Gi"
+        cpu: "1"
+      requests:
+        memory: "0.25Gi"
+        cpu: "0.01"
+
+deployment_sidecars: |-
+  - name: cloud-sql-proxy
+    image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:1.0.0
+    command:
+      - "/cloud-sql-proxy"
+    startupProbe:
+      httpGet:
+        path: /startup
+        port: 9090
+      periodSeconds: 1
+      timeoutSeconds: 5
+      failureThreshold: 20
+    livenessProbe:
+      httpGet:
+        path: /liveness
+        port: 9090
+      initialDelaySeconds: 0
+      periodSeconds: 60
+      timeoutSeconds: 30
+      failureThreshold: 5
+
+
+jobAfterHook: "curl -s -m 60 -o /dev/null -X POST localhost:9091/quitquitquit; curl -s -m 60 -o /dev/null -X POST localhost:4191/shutdown; echo 'Pinged shutdown routes'"
+jobBeforeHook: "/wait_for_routes localhost:9091/readiness localhost:4191/ready"
+cronJobs:
+  - name: test-cron-1
+    schedule: "1 0 * * 0"
+    command: "python /seqr/manage.py test_1 key=$KEY"
+  - name: test-cron-2
+    schedule: "2 0 * * 0"
+    command: "python /seqr/manage.py test_2"
+
+volumes:
+  - name: matchbox-secrets-volume
+    secret:
+      secretName: matchbox-secrets
+volumeMounts:
+  - name: matchbox-secrets-volume
+    mountPath: /mme
+
+ingress:
+  enabled: true
+  nameOverride: ingress-nginx
+  className: "nginx"
+  annotations:
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header l5d-dst-override $service_name.$namespace.svc.cluster.local:$service_port;
+    nginx.ingress.kubernetes.io/modsecurity-snippet: |
+      SecRuleEngine On
+  hosts:
+    - host: *cluster_hostname
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls:
+    - secretName: nginx-secrets-cert-manager
+      hosts:
+        - *cluster_hostname
+
+additional_secrets:
+  - name: OMIM_KEY
+    valueFrom:
+      secretKeyRef:
+        name: seqr-secrets
+        key: omim_key
+
+serviceAccount:
+  create: true
+  annotations:
+    iam.gke.io/gcp-service-account: [email protected]