Merge stable into master for release 1.5.0. (#425)

* Update bucket-monitor helm chart with unified STORAGE_BUCKET env var. (#414) * Update copyright to 2021 (#415) * Upgrade default GKE version to 1.16 (#412) * Update autoscaler and janitor to latest release for better compatibility with 1.16. (#419) * Add new consumer and zip consumer for "multiplex" jobs. (#420) * Quote all annotations in chart templates. (#421) * Refactor multiplex consumers to mesmer consumers (#422) * Update referenced version to 1.5.0 (#423) * Bump tracking model to TensorFlow 2 based version. (#426) * Bump frontend chart to 0.3.0 and redis-consumer chart to 0.5.0 (#427)
vanvalenlab · Apr 6, 2021 · 2770f0e · 2770f0e
1 parent 13f5edb
commit 2770f0e
Show file tree

Hide file tree

Showing 31 changed files with 383 additions and 123 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -12,6 +12,7 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
+
 1. Run '...'
 2. Enter '....'
 3. See error
@@ -22,9 +23,19 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**Desktop (please complete the following information):**
- - OS: [e.g. macOS, Ubuntu]
- - Version [e.g. 22]
+**Ennvironment:**
+
+- OS: [e.g. macOS, Ubuntu]
+- Kiosk version: [e.g. 1.5.0]
+- `kubectl version`:
+- `helm version`:
+- `helmfile version`:
 
 **Additional context**
 Add any other context about the problem here.
+
+If the bug is about the behavior of a pod, please include:
+
+- currently running pods [e.g. `kubectl get po`]
+- details of the failed pod [e.g. `kubectl describe po [pod_name]`]
+- pod output before failure [e.g. `kubectl logs [pod_name]`].
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -73,7 +73,7 @@ jobs:
           ${{ env.IMAGE }}
         docker exec kiosk make test/integration/${{ matrix.CLOUD_PROVIDER }}/deploy
         docker kill kiosk && docker rm kiosk
-  
+
   elk-integration-tests:
 
     runs-on: ubuntu-latest

diff --git a/Dockerfile b/Dockerfile
@@ -39,7 +39,7 @@ ENV GPU_NODE_MIN_SIZE="0"
 ENV GPU_PER_NODE="1"
 
 # gcloud config
-ENV KUBERNETES_VERSION="1.15"
+ENV KUBERNETES_VERSION="1.16"
 ENV CLOUDSDK_CORE_PROJECT=""
 ENV CLOUDSDK_CONTAINER_CLUSTER=""
 ENV CLOUDSDK_BUCKET=""

diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ Tracked Image
 Start a terminal shell and install the DeepCell Kiosk wrapper script:
 
 ```bash
-docker run -e DOCKER_TAG=1.4.0 vanvalenlab/kiosk-console:1.4.0 | sudo bash
+docker run -e DOCKER_TAG=1.5.0 vanvalenlab/kiosk-console:1.5.0 | sudo bash
 ```
 
 To start the kiosk, just run `kiosk-console` from the terminal shell.
@@ -106,5 +106,5 @@ This kiosk was developed with [Cloud Posse, LLC](https://cloudposse.com). They c
 
 ## Copyright
 
-Copyright © 2018-2020 [The Van Valen Lab](http://www.vanvalen.caltech.edu/) at the California Institute of Technology (Caltech), with support from the Shurl and Kay Curci Foundation, the Paul Allen Family Foundation, Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
+Copyright © 2018-2021 [The Van Valen Lab](http://www.vanvalen.caltech.edu/) at the California Institute of Technology (Caltech), with support from the Shurl and Kay Curci Foundation, the Paul Allen Family Foundation, Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
 All rights reserved.
diff --git a/conf/charts/bucket-monitor/Chart.yaml b/conf/charts/bucket-monitor/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: bucket-monitor
-version: 0.1.0
+version: 0.2.0
 #kubeVersion: ^1.10.0
 description: This project monitors a cloud bucket (Google or AWS) and waits for files to be uploaded whose names match certain specifictions. These files will have entries written to the cluster's Redis database for them.
 keywords:
@@ -13,6 +13,6 @@ maintainers:
     email: [email protected]
     url: vanvalen.caltech.edu
 engine: gotpl
-appVersion: 0.1.0
+appVersion: 0.4.0
 deprecated: false
 tillerVersion: ^2.9.1
diff --git a/conf/charts/bucket-monitor/values.yaml b/conf/charts/bucket-monitor/values.yaml
@@ -12,11 +12,10 @@ image:
 resources: {}
 
 env:
-  DEBUG: "true"
+  LOG_LEVEL: "DEBUG"
   INTERVAL: "21600"
   AGE_THRESHOLD: "604800"
   PREFIX: "uploads/,output/"
-  CLOUD_PROVIDER: "overwrite_this"
-  BUCKET: "overwrite_this"
+  STORAGE_BUCKET: "s3://default-bucket"
   REDIS_HOST: "redis-master"
   REDIS_PORT: "6379"
diff --git a/conf/charts/frontend/Chart.yaml b/conf/charts/frontend/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: frontend
-version: 0.2.1
+version: 0.3.0
 #kubeVersion: ^1.9.8
 description: This project provides the frontend interface for the Tensorflow-serving backend of Deepcell.
 keywords:
@@ -13,6 +13,6 @@ maintainers:
     email: [email protected]
     url: vanvalen.caltech.edu
 engine: gotpl
-appVersion: 0.6.0
+appVersion: 0.7.0
 deprecated: false
 tillerVersion: ^2.9.1
diff --git a/conf/charts/frontend/templates/ingress.yaml b/conf/charts/frontend/templates/ingress.yaml
@@ -11,9 +11,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
   annotations:
-{{- if .Values.service.annotations }}
-  annotations:
-{{ toYaml .Values.service.annotations | indent 4 }}
+{{- range $name, $value := .Values.ingress.annotations }}
+{{- if not ( empty $value) }}
+    {{ $name }}: {{ $value | quote }}
+{{- end }}
 {{- end }}
 spec:
   rules:

diff --git a/conf/charts/frontend/templates/service.yaml b/conf/charts/frontend/templates/service.yaml
@@ -7,9 +7,11 @@ metadata:
     chart: {{ template "chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
-{{- if .Values.service.annotations }}
   annotations:
-{{ toYaml .Values.service.annotations | indent 4 }}
+{{- range $name, $value := .Values.service.annotations }}
+{{- if not ( empty $value) }}
+    {{ $name }}: {{ $value | quote }}
+{{- end }}
 {{- end }}
 spec:
   type: {{ .Values.service.type }}

diff --git a/conf/charts/frontend/values.yaml b/conf/charts/frontend/values.yaml
@@ -6,7 +6,7 @@ replicas: 1
 
 image:
   repository: vanvalenlab/kiosk-frontend
-  tag: 0.6.0
+  tag: 0.7.0
   pullPolicy: IfNotPresent
 
 ingress:

diff --git a/conf/charts/redis-consumer/Chart.yaml b/conf/charts/redis-consumer/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: redis-consumer
-version: 0.2.1
+version: 0.5.0
 #kubeVersion: ^1.9.8
 description: This project consumes and processes redis event, placing the final result back to redis.
 keywords:
@@ -13,6 +13,6 @@ maintainers:
     email: [email protected]
     url: vanvalen.caltech.edu
 engine: gotpl
-appVersion: 0.8.3
+appVersion: 0.11.0
 deprecated: false
 tillerVersion: ^2.9.1
diff --git a/conf/charts/redis-consumer/templates/service.yaml b/conf/charts/redis-consumer/templates/service.yaml
@@ -4,9 +4,11 @@ metadata:
   name: {{ template "fullname" . }}
   labels:
     app: {{ template "fullname" . }}
-{{- if .Values.service.annotations }}
   annotations:
-{{ toYaml .Values.service.annotations | indent 4 }}
+{{- range $name, $value := .Values.service.annotations }}
+{{- if not ( empty $value) }}
+    {{ $name }}: {{ $value | quote }}
+{{- end }}
 {{- end }}
 spec:
   type: {{ .Values.service.type }}

diff --git a/conf/charts/redis-consumer/values.yaml b/conf/charts/redis-consumer/values.yaml
@@ -6,7 +6,7 @@ replicas: 1
 
 image:
   repository: vanvalenlab/kiosk-redis-consumer
-  tag: 0.8.3
+  tag: 0.11.0
   pullPolicy: IfNotPresent
 
 resources: {}
@@ -34,9 +34,9 @@ hpa:
   metrics: {}
 
 env:
-  DEBUG: "True"
+  LOG_LEVEL: "DEBUG"
   INTERVAL: 10
-  CONSUMER_TYPE: "image"
+  CONSUMER_TYPE: "segmentation"
   QUEUE: "segmentation"
   SEGMENTATION_QUEUE: "segmentation"
   EMPTY_QUEUE_TIMEOUT: 5
@@ -51,29 +51,25 @@ env:
   REDIS_PORT: "6379"
   REDIS_TIMEOUT: 3
 
-  TF_HOST: "overwrite_this"
-  TF_PORT: "sepcify_me"
+  TF_HOST: "tf-serving"
+  TF_PORT: 8500
   TF_MAX_BATCH_SIZE: 1
   TF_MIN_MODEL_SIZE: 128
 
-  CLOUD_PROVIDER: "aws"
   AWS_REGION: "us-east-1"
   GKE_COMPUTE_ZONE: "us-west1-b"
 
-  NUCLEAR_MODEL: "NuclearSegmentation:0"
-  NUCLEAR_POSTPROCESS: "deep_watershed"
+  NUCLEAR_MODEL: "NuclearSegmentation:3"
 
   PHASE_MODEL: "PhaseCytoSegmentation:0"
-  PHASE_POSTPROCESS: "deep_watershed"
 
-  CYTOPLASM_MODEL: "FluoCytoSegmentation:0"
-  CYTOPLASM_POSTPROCESS: "deep_watershed"
+  CYTOPLASM_MODEL: "FluoCytoSegmentation:1"
 
   LABEL_DETECT_ENABLED: "false"
-  LABEL_DETECT_MODEL: "LabelDetection:0"
+  LABEL_DETECT_MODEL: "LabelDetection:1"
 
   SCALE_DETECT_ENABLED: "false"
-  SCALE_DETECT_MODEL: "ScaleDetection:0"
+  SCALE_DETECT_MODEL: "ScaleDetection:1"
 
   DRIFT_CORRECT_ENABLED: "false"
   NORMALIZE_TRACKING: "false"
@@ -85,5 +81,4 @@ env:
 secrets:
   AWS_ACCESS_KEY_ID: ""
   AWS_SECRET_ACCESS_KEY: ""
-  AWS_S3_BUCKET: "s3://example-bucket"
-  GKE_BUCKET: "gs://example-bucket"
+  STORAGE_BUCKET: "gs://example-bucket"
diff --git a/conf/charts/tf-serving/Chart.yaml b/conf/charts/tf-serving/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: tf-serving
-version: 0.4.0
+version: 0.4.1
 #kubeVersion: ^1.9.8
 description: This helm chart provides the Tensorflow-serving backend functionality of the Deepcell application.
 keywords:

diff --git a/conf/charts/tf-serving/templates/deployment.yaml b/conf/charts/tf-serving/templates/deployment.yaml
@@ -16,7 +16,11 @@ spec:
   template:
     metadata:
       annotations:
-{{ toYaml .Values.annotations | indent 8 }}
+{{- range $name, $value := .Values.annotations }}
+{{- if not ( empty $value) }}
+        {{ $name }}: {{ $value | quote }}
+{{- end }}
+{{- end }}
       labels:
         app: {{ template "name" . }}
         release: {{ .Release.Name }}

diff --git a/conf/charts/tf-serving/templates/service.yaml b/conf/charts/tf-serving/templates/service.yaml
@@ -7,9 +7,11 @@ metadata:
     release: {{ .Release.Name | quote }}
     chart: {{ template "chart" . }}
     app: {{ template "name" . }}
-{{- if .Values.service.annotations }}
   annotations:
-{{ toYaml .Values.service.annotations | indent 4 }}
+{{- range $name, $value := .Values.service.annotations }}
+{{- if not ( empty $value) }}
+    {{ $name }}: {{ $value | quote }}
+{{- end }}
 {{- end }}
 spec:
   type: {{ .Values.service.type }}

diff --git a/conf/helmfile.d/0000.ingress.yaml b/conf/helmfile.d/0000.ingress.yaml
@@ -39,9 +39,8 @@ releases:
           requests:
             cpu: 50m
             memory: 128Mi
-        stats:
-          ### Optional: NGINX_INGRESS_CONTROLLER_STATS_ENABLED;
-          enabled: {{ env "NGINX_INGRESS_CONTROLLER_STATS_ENABLED" | default "true" }}
+        config:
+          proxy-body-size: 1g
         metrics:
           ### Optional: NGINX_INGRESS_CONTROLLER_METRICS_ENABLED; enable Prometheus metrics (controller.stats.enabled must be true as well)
           enabled: {{ env "NGINX_INGRESS_CONTROLLER_METRICS_ENABLED" | default "true" }}

diff --git a/conf/helmfile.d/0110.prometheus-redis-exporter.yaml b/conf/helmfile.d/0110.prometheus-redis-exporter.yaml
@@ -73,6 +73,7 @@ releases:
 
             queues[#queues+1] = "segmentation"
             queues[#queues+1] = "tracking"
+            queues[#queues+1] = "mesmer"
 
             for _,queue in ipairs(queues) do
               local zip_queue = queue .. "-zip"

diff --git a/conf/helmfile.d/0200.bucket-monitor.yaml b/conf/helmfile.d/0200.bucket-monitor.yaml
@@ -17,7 +17,7 @@ releases:
     vendor: vanvalenlab
     default: true
   chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/bucket-monitor'
-  version: 0.1.0
+  version: 0.2.0
   wait: true
   timeout: 300
   atomic: true
@@ -27,7 +27,7 @@ releases:
 
       image:
         repository: vanvalenlab/kiosk-bucket-monitor
-        tag: 0.3.0
+        tag: 0.4.0
 
       resources:
         requests:
@@ -38,15 +38,13 @@ releases:
         #   memory: 64Mi
 
       env:
-        DEBUG: "true"
         AGE_THRESHOLD: "259200"
         PREFIX: "uploads/,output/"
         INTERVAL: "21600"
         REDIS_HOST: "redis"
         REDIS_PORT: "26379"
-        CLOUD_PROVIDER: {{ env "CLOUD_PROVIDER" | default "broken_default" }}
 {{ if eq (env "CLOUD_PROVIDER" | default "aws") "aws" }}
-        BUCKET: {{ env "BUCKET" | default "broken_default" }}
+        STORAGE_BUCKET: 's3://{{ env "BUCKET" | default "broken_default" }}'
 {{ else }}
-        BUCKET: {{ env "CLOUDSDK_BUCKET" | default "broken_default" }}
+        STORAGE_BUCKET: 'gs://{{ env "CLOUDSDK_BUCKET" | default "broken_default" }}'
 {{ end }}
diff --git a/conf/helmfile.d/0210.autoscaler.yaml b/conf/helmfile.d/0210.autoscaler.yaml
@@ -27,7 +27,7 @@ releases:
 
       image:
         repository: vanvalenlab/kiosk-autoscaler
-        tag: 0.4.1
+        tag: 0.5.0
 
       resources:
         requests:
@@ -38,7 +38,7 @@ releases:
         #   memory: 64Mi
 
       env:
-        QUEUES: "segmentation,tracking"
+        QUEUES: "mesmer,segmentation,tracking"
         QUEUE_DELIMITER: ","
         RESOURCE_NAME: "tf-serving"
         RESOURCE_TYPE: "deployment"

diff --git a/conf/helmfile.d/0220.redis-janitor.yaml b/conf/helmfile.d/0220.redis-janitor.yaml
@@ -27,7 +27,7 @@ releases:
 
       image:
         repository: vanvalenlab/kiosk-redis-janitor
-        tag: 0.3.2
+        tag: 0.4.0
 
       resources:
         requests:
@@ -38,7 +38,7 @@ releases:
         #   memory: 64Mi
 
       env:
-        QUEUES: "segmentation,tracking"
+        QUEUES: "mesmer,segmentation,tracking"
         DEBUG: "true"
         INTERVAL: "5"
         REDIS_HOST: "redis"