Merge pull request #318 from n1analytics/release-v1.9.3

Release v1.9.3 into master

Jenkinsfile.groovy

@@ -281,8 +281,7 @@ node('helm && kubectl') {
                 helm upgrade --install --wait --namespace ${NAMESPACE} ${DEPLOYMENT} . \
                     -f values.yaml -f minimal-values.yaml -f test-versions.yaml \
                     --set api.app.debug=true \
-                    --set api.ingress.enabled=false \
-                    --set api.certManager.enabled=false                    
+                    --set api.ingress.enabled=false
                 """
               // give the cluster a chance to provision volumes etc, assign an IP to the service, then create a new job to test it
               sleep(time: 120, unit: "SECONDS")

backend/entityservice/VERSION

@@ -1 +1 @@
-v1.9.2
+v1.9.3

backend/entityservice/async_worker.py

@@ -8,18 +8,17 @@
 
 
 celery = Celery('tasks',
-                broker=Config.BROKER_URL,
+                broker=Config.CELERY_BROKER_URL,
                 backend=Config.CELERY_RESULT_BACKEND
                 )
 
-celery.conf.CELERY_TASK_SERIALIZER = 'json'
-celery.conf.CELERY_ACCEPT_CONTENT = ['json']
-celery.conf.CELERY_RESULT_SERIALIZER = 'json'
-celery.conf.CELERY_ANNOTATIONS = Config.CELERY_ANNOTATIONS
-celery.conf.CELERYD_PREFETCH_MULTIPLIER = Config.CELERYD_PREFETCH_MULTIPLIER
-celery.conf.CELERYD_MAX_TASKS_PER_CHILD = Config.CELERYD_MAX_TASKS_PER_CHILD
-celery.conf.CELERY_ACKS_LATE = Config.CELERY_ACKS_LATE
-celery.conf.CELERY_ROUTES = Config.CELERY_ROUTES
+celery.conf.task_annotations = Config.CELERY_ANNOTATIONS
+celery.conf.task_acks_late = Config.CELERY_ACKS_LATE
+celery.conf.task_routes = Config.CELERY_ROUTES
+celery.conf.broker_transport_options = Config.CELERY_BROKER_TRANSPORT_OPTIONS
+celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS
+celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER
+celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD
 
 
 structlog.configure(

backend/entityservice/cache.py

@@ -1,4 +1,5 @@
 import redis
+from redis.sentinel import Sentinel
 
 import pickle
 
@@ -15,8 +16,22 @@
 redis_pass = config.REDIS_PASSWORD
 
 
-def connect_to_redis():
-    r = redis.StrictRedis(host=redis_host, password=redis_pass, port=6379, db=0)
+def connect_to_redis(read_only=False):
+    """
+    Get a connection to the master redis service.
+
+    """
+    logger.debug("Connecting to redis")
+    if config.REDIS_USE_SENTINEL:
+        sentinel = Sentinel([(redis_host, 26379)], password=redis_pass, socket_timeout=5)
+        if read_only:
+            logger.debug("Looking up read only redis slave using sentinel protocol")
+            r = sentinel.slave_for('mymaster')
+        else:
+            logger.debug("Looking up redis master using sentinel protocol")
+            r = sentinel.master_for('mymaster')
+    else:
+        r = redis.StrictRedis(host=redis_host, password=redis_pass)
     return r
 
 
@@ -36,7 +51,7 @@ def get_deserialized_filter(dp_id):
     """
     logger.debug("Getting filters")
     key = 'clk-pkl-{}'.format(dp_id)
-    r = connect_to_redis()
+    r = connect_to_redis(read_only=True)
 
     # Check if this dp_id is already saved in redis?
     if r.exists(key):
@@ -73,7 +88,7 @@ def update_progress(comparisons, run_id):
 
 
 def get_progress(run_id):
-    r = connect_to_redis()
+    r = connect_to_redis(read_only=True)
     key = 'progress-{}'.format(run_id)
     res = r.get(key)
     # redis returns bytes, and None if not present
@@ -90,7 +105,7 @@ def clear_progress(run_id):
 
 
 def get_status():
-    r = connect_to_redis()
+    r = connect_to_redis(read_only=True)
     key = 'entityservice-status'
     if r.exists(key):
         logger.debug("returning status from cache")

backend/entityservice/settings.py

@@ -14,8 +14,8 @@ class Config(object):
 
     DEBUG = os.getenv("DEBUG", "false") == "true"
 
-    CONNEXION_STRICT_VALIDATION = os.getenv("CONNEXION_STRICT_VALIDATION", "true") == "true"
-    CONNEXION_RESPONSE_VALIDATION = os.getenv("CONNEXION_RESPONSE_VALIDATION", "true") == "true"
+    CONNEXION_STRICT_VALIDATION = os.getenv("CONNEXION_STRICT_VALIDATION", "true").lower() == "true"
+    CONNEXION_RESPONSE_VALIDATION = os.getenv("CONNEXION_RESPONSE_VALIDATION", "true").lower() == "true"
 
     LOGFILE = os.getenv("LOGFILE")
     fileFormat = logging.Formatter('%(asctime)-15s %(name)-12s %(levelname)-8s: %(message)s')
@@ -24,6 +24,7 @@ class Config(object):
 
     REDIS_SERVER = os.getenv('REDIS_SERVER', 'redis')
     REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
+    REDIS_USE_SENTINEL = os.getenv('REDIS_USE_SENTINEL', 'false').lower() == "true"
 
     MINIO_SERVER = os.getenv('MINIO_SERVER', 'minio:9000')
     MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', '')
@@ -35,13 +36,20 @@ class Config(object):
     DATABASE_USER = os.getenv('DATABASE_USER', 'postgres')
     DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
 
-    BROKER_URL = os.getenv('CELERY_BROKER_URL',
-                           'redis://:{}@{}:6379/0'.format(REDIS_PASSWORD, REDIS_SERVER))
+    CELERY_BROKER_URL = os.getenv(
+        'CELERY_BROKER_URL',
+        ('sentinel://:{}@{}:26379/0' if REDIS_USE_SENTINEL else 'redis://:{}@{}:6379/0').format(REDIS_PASSWORD, REDIS_SERVER)
+    )
+
+    CELERY_BROKER_TRANSPORT_OPTIONS = {'master_name': "mymaster"} if REDIS_USE_SENTINEL else {}
+
+    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+    CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = {'master_name': "mymaster"} if REDIS_USE_SENTINEL else {}
 
-    CELERY_RESULT_BACKEND = BROKER_URL
     CELERY_ANNOTATIONS = {
         'async_worker.calculate_mapping': {'rate_limit': '1/s'}
     }
+
     CELERY_ROUTES = {
         'async_worker.calculate_mapping': {'queue': 'celery'},
         'async_worker.compute_similarity': {'queue': 'compute'},

backend/requirements.txt

@@ -1,26 +1,24 @@
+anonlink==0.10.0
+bitmath==1.3.1.2
+celery==4.2.1
+clkhash==0.12.0
+colorama==0.4.0 # required for structlog
+connexion==1.4
+Flask-Opentracing==0.2.0
 flask==0.12.2
-Werkzeug==0.12.2
-requests==2.18.4
-gunicorn==19.7.1
-psycopg2==2.7.5
-celery==4.1.1
 flower==0.9.2
-redis==2.10.5
-pympler==0.4.3
-setproctitle
-minio==4.0.4
+gunicorn==19.7.1
 ijson==2.3
-bitmath==1.3.1.2
-anonlink==0.10.0
-clkhash==0.12.0
-pytest==3.5.1
 iso8601==0.1.12
-connexion==1.4
+jaeger-client==3.12.0
 marshmallow==3.0.0b10
-structlog
-colorama
-opentracing<2
-jaeger-client
-Flask-Opentracing
-opentracing_instrumentation
-more_itertools
+minio==4.0.4
+more_itertools==4.3.0
+opentracing==1.3.0
+opentracing_instrumentation==2.4.3
+psycopg2==2.7.5
+pytest==3.5.1
+redis==3.0.1
+requests==2.18.4
+setproctitle==1.1.10 # used by celery to change process name
+structlog==18.2.0

deployment/README.md

@@ -48,9 +48,15 @@ we will have to add one. The chart has been tested with nginx and treafik.
 
 Helm can be used to easily deploy the system to a kubernetes cluster.
 
-Pull the dependencies:
+Add the helm repository:
+
+    helm repo add n1charts https://n1analytics.github.io/charts
+    helm repo update
+
+Install the `entity-service`:
+
+    helm install n1charts/entity-service
 
-    helm dependency update
 
 ## Configuring the deployment
 

deployment/entity-service/Chart.yaml

@@ -1,6 +1,6 @@
 name: entity-service
-appVersion: 1.9.1
-version: 1.8
+appVersion: 1.9.3
+version: 1.9.3
 description: Privacy preserving record linkage service
 sources:
   - https://github.com/n1analytics/entity-service

deployment/entity-service/minimal-values.yaml

@@ -50,6 +50,7 @@ postgresql:
       cpu: 50m
 
   persistence:
+    enabled: false
     size: 1Gi
 
 
@@ -59,6 +60,8 @@ minio:
     size: 1Gi
 
 redis-ha:
+  persistentVolume:
+    enabled: false
   resources:
     server:
       requests:

deployment/entity-service/requirements.yaml

@@ -1,6 +1,6 @@
 dependencies:
   - name: redis-ha
-    version: 2.2.3
+    version: 3.0.3
     repository: https://kubernetes-charts.storage.googleapis.com
     condition: provision.redis
   - name: minio

deployment/entity-service/templates/api-deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ .Release.Name }}-api
@@ -8,6 +8,11 @@ metadata:
     tier: frontend
 spec:
   replicas: {{ .Values.api.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "es.fullname" . }}
+      component: "{{ .Values.api.name }}"
+      tier: frontend
   {{- if .Values.api.strategy }}
   strategy:
 {{ toYaml .Values.api.strategy | indent 4 }}

deployment/entity-service/templates/configmap.yaml

@@ -6,9 +6,11 @@ metadata:
     {{- include "es.release_labels" . | indent 4 }}
 data:
   {{ if .Values.provision.redis }}
-  REDIS_SERVER: {{ .Release.Name }}-{{ index .Values "redis-ha" "nameOverride" }}-master-svc
+  REDIS_SERVER: {{ .Release.Name }}-{{ index .Values "redis-ha" "nameOverride" }}
+  REDIS_USE_SENTINEL: "true"
   {{ else }}
   REDIS_SERVER: {{ .Values.redis.server }}
+  REDIS_USE_SENTINEL: {{ .Values.redis.use_sentinel }}
   {{ end }}
   DATABASE_SERVER: "{{ .Release.Name }}-{{ .Values.postgresql.nameOverride }}"
   DEBUG: {{ .Values.workers.debug | quote }}

deployment/entity-service/templates/highmemory-worker-deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
@@ -8,6 +8,11 @@ metadata:
   name: {{ .Release.Name }}-highmemory-worker
 spec:
   replicas: {{ .Values.workers.highmemory.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "es.fullname" . }}
+      component: "{{ .Values.workers.name }}-highmemory"
+      tier: backend
   {{- if .Values.workers.strategy }}
   strategy:
 {{ toYaml .Values.workers.strategy | indent 4 }}

deployment/entity-service/templates/monitor-deployment.yaml

@@ -1,12 +1,17 @@
 {{- if .Values.workers.monitor.enabled }}
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ template "name" . }}-monitor
+  name: {{ .Release.Name }}-monitor
   labels:
     {{- include "es.release_labels" . | indent 4 }}
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "es.fullname" . }}
+      tier: aux
+      run: celery-monitor
   template:
     metadata:
       labels:

deployment/entity-service/templates/worker-deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
@@ -8,6 +8,11 @@ metadata:
   name: {{ .Release.Name }}-worker
 spec:
   replicas: {{ .Values.workers.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "es.fullname" . }}
+      component: "{{ .Values.workers.name }}"
+      tier: backend
   {{- if .Values.workers.strategy }}
   strategy:
 {{ toYaml .Values.workers.strategy | indent 4 }}