✨(eucalyptus/3/wb) add missing support for redis sentinel

To connect celery to redis-sentinel we use the plugin celery-redis-sentinel
in our eucalyptus fork. Also to use session in redis, the setting allowing
to change the backend to use was missing and setting to configure redis
were also missing. We added them, it is now possible to use redis to
store sessions.

releases/eucalyptus/3/wb/CHANGELOG.md

@@ -15,6 +15,7 @@ release.
 
 - First experimental release of OpenEdx `eucalyptus.3` (wb flavor).
 - Set replicaSet and read_preference in mongodb connection
+- Add missing support for redis sentinel
 
 [unreleased]: https://github.com/openfun/openedx-docker/compare/eucalyptus.3-1.0.0-wb...HEAD
 [eucalyptus.3-1.0.0-wb]: https://github.com/openfun/openedx-docker/releases/tag/eucalyptus.3-1.0.0-wb

releases/eucalyptus/3/wb/config/cms/docker_run_production.py

@@ -14,6 +14,7 @@
 import os
 import platform
 
+from celery_redis_sentinel import register
 from lms.envs.fun.utils import Configuration
 from openedx.core.lib.logsettings import get_logger_config
 from path import Path as path
@@ -189,6 +190,32 @@
     "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool
 )
 
+# Configuration to use session with redis
+# To use redis, change SESSION_ENGINE with value redis_sessions.session
+SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis")
+SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int)
+SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int)
+SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None)
+SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session")
+SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int)
+SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool)
+
+SESSION_REDIS = config(
+    "SESSION_REDIS",
+    default={
+        "host": SESSION_REDIS_HOST,
+        "port": SESSION_REDIS_PORT,
+        "db": SESSION_REDIS_DB,  # db 0 is used for Celery Broker
+        "password": SESSION_REDIS_PASSWORD,
+        "prefix": SESSION_REDIS_PREFIX,
+        "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT,
+        "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT,
+    },
+    formatter=json.loads,
+)
+SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads)
+SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None)
+
 # social sharing settings
 SOCIAL_SHARING_SETTINGS = config(
     "SOCIAL_SHARING_SETTINGS", default=SOCIAL_SHARING_SETTINGS, formatter=json.loads
@@ -519,13 +546,18 @@
 DATADOG["api_key"] = config("DATADOG_API", default=None)
 
 # Celery Broker
+# For redis sentinel you the transport redis-sentinel
 CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis")
 CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="")
 CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="")
 CELERY_BROKER_HOST = config("CELERY_BROKER_HOST", default="redis")
 CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int)
 CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int)
 
+if CELERY_BROKER_TRANSPORT == "redis-sentinel":
+    # register redis sentinel schema in celery
+    register()
+
 BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format(
     transport=CELERY_BROKER_TRANSPORT,
     user=CELERY_BROKER_USER,
@@ -534,6 +566,9 @@
     port=CELERY_BROKER_PORT,
     vhost=CELERY_BROKER_VHOST,
 )
+# To use redis-sentinel, refer to the documenation here 
+# https://celery-redis-sentinel.readthedocs.io/en/latest/
+BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads)
 
 # Event tracking
 TRACKING_BACKENDS.update(config("TRACKING_BACKENDS", default={}, formatter=json.loads))

releases/eucalyptus/3/wb/config/lms/docker_run_production.py

@@ -23,6 +23,7 @@
 import platform
 import warnings
 
+from celery_redis_sentinel import register
 from openedx.core.lib.logsettings import get_logger_config
 from path import Path as path
 from xmodule.modulestore.modulestore_settings import (
@@ -195,6 +196,32 @@
     "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool
 )
 
+# Configuration to use session with redis
+# To use redis, change SESSION_ENGINE with value redis_sessions.session
+SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis")
+SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int)
+SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int)
+SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None)
+SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session")
+SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int)
+SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool)
+
+SESSION_REDIS = config(
+    "SESSION_REDIS",
+    default={
+        "host": SESSION_REDIS_HOST,
+        "port": SESSION_REDIS_PORT,
+        "db": SESSION_REDIS_DB,  # db 0 is used for Celery Broker
+        "password": SESSION_REDIS_PASSWORD,
+        "prefix": SESSION_REDIS_PREFIX,
+        "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT,
+        "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT,
+    },
+    formatter=json.loads,
+)
+SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads)
+SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None)
+
 AWS_SES_REGION_NAME = config("AWS_SES_REGION_NAME", default="us-east-1")
 AWS_SES_REGION_ENDPOINT = config(
     "AWS_SES_REGION_ENDPOINT", default="email.us-east-1.amazonaws.com"
@@ -782,13 +809,18 @@
 EDX_API_KEY = config("EDX_API_KEY", default=None)
 
 # Celery Broker
+# For redis sentinel you the transport redis-sentinel
 CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis")
 CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="")
 CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="")
 CELERY_BROKER_HOST = config("CELERY_BROKER_HOST", default="redis")
 CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int)
 CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int)
 
+if CELERY_BROKER_TRANSPORT == "redis-sentinel":
+    # register redis sentinel schema in celery
+    register()
+
 BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format(
     transport=CELERY_BROKER_TRANSPORT,
     user=CELERY_BROKER_USER,
@@ -797,6 +829,9 @@
     port=CELERY_BROKER_PORT,
     vhost=CELERY_BROKER_VHOST,
 )
+# To use redis-sentinel, refer to the documenation here 
+# https://celery-redis-sentinel.readthedocs.io/en/latest/
+BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads)
 
 # upload limits
 STUDENT_FILEUPLOAD_MAX_SIZE = config(

releases/eucalyptus/3/wb/requirements.txt

@@ -13,3 +13,4 @@ xblock-utils2==0.3.0
 # ==== third-party apps ====
 raven==6.9.0
 django-redis-sessions==0.6.1
+celery-redis-sentinel==0.3.0