From f98ebd217cbac44d90765a543fe53ac2bea4340e Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Wed, 13 Nov 2019 20:50:02 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8(project)=20add=20eucalyptus/3/wb?= =?UTF-8?q?=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install and configure fun-apps in the edx-platform eucalyptus.3 release. --- .circleci/config.yml | 7 + releases/eucalyptus/3/wb/CHANGELOG.md | 20 + releases/eucalyptus/3/wb/Dockerfile | 237 ++++ releases/eucalyptus/3/wb/activate | 5 + .../eucalyptus/3/wb/config/cms/__init__.py | 0 .../3/wb/config/cms/docker_build.py | 8 + .../wb/config/cms/docker_build_development.py | 1 + .../wb/config/cms/docker_build_production.py | 1 + .../eucalyptus/3/wb/config/cms/docker_run.py | 3 + .../3/wb/config/cms/docker_run_development.py | 25 + .../3/wb/config/cms/docker_run_preprod.py | 14 + .../3/wb/config/cms/docker_run_production.py | 668 +++++++++ .../3/wb/config/cms/docker_run_staging.py | 14 + releases/eucalyptus/3/wb/config/cms/fun.py | 53 + .../eucalyptus/3/wb/config/lms/__init__.py | 0 .../3/wb/config/lms/docker_build.py | 8 + .../wb/config/lms/docker_build_development.py | 1 + .../wb/config/lms/docker_build_production.py | 1 + .../eucalyptus/3/wb/config/lms/docker_run.py | 3 + .../3/wb/config/lms/docker_run_development.py | 24 + .../3/wb/config/lms/docker_run_preprod.py | 14 + .../3/wb/config/lms/docker_run_production.py | 1233 +++++++++++++++++ .../3/wb/config/lms/docker_run_staging.py | 14 + releases/eucalyptus/3/wb/config/lms/fun.py | 77 + releases/eucalyptus/3/wb/config/lms/utils.py | 110 ++ releases/eucalyptus/3/wb/entrypoint.sh | 8 + releases/eucalyptus/3/wb/requirements.txt | 15 + 27 files changed, 2564 insertions(+) create mode 100644 releases/eucalyptus/3/wb/CHANGELOG.md create mode 100644 releases/eucalyptus/3/wb/Dockerfile create mode 100644 releases/eucalyptus/3/wb/activate create mode 100644 releases/eucalyptus/3/wb/config/cms/__init__.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_build.py create mode 120000 releases/eucalyptus/3/wb/config/cms/docker_build_development.py create mode 120000 releases/eucalyptus/3/wb/config/cms/docker_build_production.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_run.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_run_development.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_run_preprod.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_run_production.py create mode 100644 releases/eucalyptus/3/wb/config/cms/docker_run_staging.py create mode 100644 releases/eucalyptus/3/wb/config/cms/fun.py create mode 100644 releases/eucalyptus/3/wb/config/lms/__init__.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_build.py create mode 120000 releases/eucalyptus/3/wb/config/lms/docker_build_development.py create mode 120000 releases/eucalyptus/3/wb/config/lms/docker_build_production.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_run.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_run_development.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_run_preprod.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_run_production.py create mode 100644 releases/eucalyptus/3/wb/config/lms/docker_run_staging.py create mode 100644 releases/eucalyptus/3/wb/config/lms/fun.py create mode 100644 releases/eucalyptus/3/wb/config/lms/utils.py create mode 100755 releases/eucalyptus/3/wb/entrypoint.sh create mode 100644 releases/eucalyptus/3/wb/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 6182c087..48893a1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,6 +137,9 @@ jobs: eucalyptus/3/bare: <<: [*defaults, *build_steps] + eucalyptus/3/wb: + <<: [*defaults, *build_steps] + hawthorn/1/bare: <<: [*defaults, *build_steps] @@ -231,6 +234,10 @@ workflows: filters: tags: ignore: /.*/ + - eucalyptus/3/wb: + filters: + tags: + ignore: /.*/ - hawthorn/1/oee: filters: tags: diff --git a/releases/eucalyptus/3/wb/CHANGELOG.md b/releases/eucalyptus/3/wb/CHANGELOG.md new file mode 100644 index 00000000..08f406b6 --- /dev/null +++ b/releases/eucalyptus/3/wb/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic +Versioning](https://semver.org/spec/v2.0.0.html) for each flavored OpenEdx +release. + +## [Unreleased] + +## [eucalyptus.3-1.0.0-wb] - 2019-11-14 + +### Added + +- First experimental release of OpenEdx `eucalyptus.3` (wb flavor). +- Set replicaSet and read_preference in mongodb connection + +[unreleased]: https://github.com/openfun/openedx-docker/compare/eucalyptus.3-1.0.0-wb...HEAD +[eucalyptus.3-1.0.0-wb]: https://github.com/openfun/openedx-docker/releases/tag/eucalyptus.3-1.0.0-wb diff --git a/releases/eucalyptus/3/wb/Dockerfile b/releases/eucalyptus/3/wb/Dockerfile new file mode 100644 index 00000000..6c75fbbd --- /dev/null +++ b/releases/eucalyptus/3/wb/Dockerfile @@ -0,0 +1,237 @@ +# EDX-PLATFORM multi-stage docker build + +# Change release to build, by providing the EDX_RELEASE_REF build argument to +# your build command: +# +# $ docker build \ +# --build-arg EDX_RELEASE_REF="open-release/eucalyptus.3" \ +# -t edxapp:eucalyptus.3 \ +# . +ARG DOCKER_UID=1000 +ARG DOCKER_GID=1000 +ARG EDX_RELEASE_REF=fun/whitebrand +ARG EDX_ARCHIVE_URL=https://github.com/openfun/edx-platform/archive/fun/whitebrand.tar.gz + +# === BASE === +FROM ubuntu:16.04 as base + +# Configure locales & timezone +RUN apt-get update && \ + apt-get install -y \ + gettext \ + locales \ + tzdata && \ + rm -rf /var/lib/apt/lists/* +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + + +# === DOWNLOAD === +FROM base as downloads + +WORKDIR /downloads + +# Install curl +RUN apt-get update && \ + apt-get install -y curl + +# Download pip installer +RUN curl -sLo get-pip.py https://bootstrap.pypa.io/get-pip.py + +# Download edxapp release +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF +ARG EDX_ARCHIVE_URL +RUN curl -sLo edxapp.tgz $EDX_ARCHIVE_URL && \ + tar xzf edxapp.tgz + + +# === EDXAPP === +FROM base as edxapp + +# Install base system dependencies +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y python && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /edx/app/edxapp/edx-platform + +# Get default EDX_RELEASE_REF value (defined on top) +ARG EDX_RELEASE_REF +COPY --from=downloads /downloads/edx-platform-* . + +COPY ./requirements.txt /edx/app/edxapp/edx-platform/requirements/edx/fun.txt + +# We copy default configuration files to "/config" and we point to them via +# symlinks. That allows to easily override default configurations by mounting a +# docker volume. +COPY ./config /config +RUN ln -sf /config/lms /edx/app/edxapp/edx-platform/lms/envs/fun && \ + ln -sf /config/cms /edx/app/edxapp/edx-platform/cms/envs/fun + +# Add node_modules/.bin to the PATH so that paver-related commands can execute +# node scripts +ENV PATH="/edx/app/edxapp/edx-platform/node_modules/.bin:${PATH}" + +# === BUILDER === +FROM edxapp as builder + +WORKDIR /builder + +# Install builder system dependencies +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + build-essential \ + gettext \ + git \ + graphviz-dev \ + libgeos-dev \ + libjpeg8-dev \ + libmysqlclient-dev \ + libpng12-dev \ + libxml2-dev \ + libxmlsec1-dev \ + nodejs \ + nodejs-legacy \ + npm \ + python-dev && \ + rm -rf /var/lib/apt/lists/* + +# Install the latest pip release +COPY --from=downloads /downloads/get-pip.py ./get-pip.py +RUN python get-pip.py + +WORKDIR /edx/app/edxapp/edx-platform + +# Install python dependencies +# +# Note that we force some pinned release installations before installing github +# dependencies to prevent secondary dependencies installation to fail while +# trying to install a python 2.7 incompatible release +RUN pip install -r requirements/edx/pre.txt +RUN pip install \ + astroid==1.6.0 \ + django==1.8.15 \ + pip==9.0.3 +RUN pip install --src /usr/local/src -r requirements/edx/github.txt +RUN pip install -r requirements/edx/base.txt +RUN pip install -r requirements/edx/paver.txt +RUN pip install -r requirements/edx/post.txt +RUN pip install -r requirements/edx/local.txt +# Redis is an extra requirement of Celery, we need to install it explicitly so +# that celery workers are effective +RUN pip install redis==3.3.7 +# Installing FUN requirements needs a recent pip release (we are using +# setup.cfg declarative packages) +RUN pip install -r requirements/edx/fun.txt + +# Install Javascript requirements +RUN npm install + +# Force the reinstallation of edx-ui-toolkit's dependencies inside its +# node_modules because someone is poking files from there when updating assets. +RUN cd node_modules/edx-ui-toolkit && \ + npm install + +# Update assets skipping collectstatic (it should be done during deployment) +RUN NO_PREREQ_INSTALL=1 \ + paver update_assets --settings=fun.docker_build_production --skip-collect + + +# === DEVELOPMENT === +FROM builder as development + +ARG DOCKER_UID +ARG DOCKER_GID +ARG EDX_RELEASE_REF + +# Install system dependencies +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + libsqlite3-dev \ + mongodb && \ + rm -rf /var/lib/apt/lists/* + +RUN groupadd --gid ${DOCKER_GID} edx || \ + echo "Group with ID ${DOCKER_GID} already exists." && \ + useradd \ + --create-home \ + --home-dir /home/edx \ + --uid ${DOCKER_UID} \ + --gid ${DOCKER_GID} \ + edx + +# To prevent permission issues related to the non-priviledged user running in +# development, we will install development dependencies in a python virtual +# environment belonging to that user +RUN pip install virtualenv + +# Create the virtualenv directory where we will install python development +# dependencies +RUN mkdir -p /edx/app/edxapp/venv && \ + chown -R ${DOCKER_UID}:${DOCKER_GID} /edx/app/edxapp/venv + +# Change edxapp directory owner to allow the development image docker user to +# perform installations from edxapp sources (yeah, I know...) +RUN chown -R ${DOCKER_UID}:${DOCKER_GID} /edx/app/edxapp + +# Copy the entrypoint that will activate the virtualenv +COPY ./entrypoint.sh /usr/local/bin/entrypoint.sh + +# Switch to an un-privileged user matching the host user to prevent permission +# issues with volumes (host folders) +USER ${DOCKER_UID}:${DOCKER_GID} + +# Create the virtualenv with a non-priviledged user +RUN virtualenv -p python2.7 --system-site-packages /edx/app/edxapp/venv + +# Install development dependencies in a virtualenv +RUN bash -c "source /edx/app/edxapp/venv/bin/activate && \ + pip install --no-cache-dir -r requirements/edx/local.txt && \ + pip install --no-cache-dir -r requirements/edx/development.txt" + +ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] + + +# === PRODUCTION === +FROM edxapp as production + +# Install runner system dependencies +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + libgeos-dev \ + libjpeg8 \ + libmysqlclient20 \ + libpng12-0 \ + libxml2 \ + libxmlsec1-dev \ + lynx \ + nodejs \ + nodejs-legacy \ + tzdata && \ + rm -rf /var/lib/apt/lists/* + +# Copy installed dependencies +COPY --from=builder /usr/local /usr/local + +# Copy modified sources (sic!) +COPY --from=builder /edx/app/edxapp/edx-platform /edx/app/edxapp/edx-platform + +# Now that dependencies are installed and configuration has been set, the above +# statements will run with a un-privileged user. +USER 10000 + +# To start the CMS, inject the SERVICE_VARIANT=cms environment variable +# (defaults to "lms") +ENV SERVICE_VARIANT=lms + +# Use Gunicorn in production as web server +CMD DJANGO_SETTINGS_MODULE=${SERVICE_VARIANT}.envs.fun.docker_run \ + gunicorn --name=${SERVICE_VARIANT} --bind=0.0.0.0:8000 --max-requests=1000 ${SERVICE_VARIANT}.wsgi:application diff --git a/releases/eucalyptus/3/wb/activate b/releases/eucalyptus/3/wb/activate new file mode 100644 index 00000000..a42e31b5 --- /dev/null +++ b/releases/eucalyptus/3/wb/activate @@ -0,0 +1,5 @@ +export EDX_RELEASE="eucalyptus.3" +export FLAVOR="wb" +export EDX_RELEASE_REF="fun/whitebrand" +export EDX_ARCHIVE_URL="https://github.com/openfun/edx-platform/archive/${EDX_RELEASE_REF}.tar.gz" +export EDX_DEMO_RELEASE_REF="open-release/eucalyptus.3" diff --git a/releases/eucalyptus/3/wb/config/cms/__init__.py b/releases/eucalyptus/3/wb/config/cms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/releases/eucalyptus/3/wb/config/cms/docker_build.py b/releases/eucalyptus/3/wb/config/cms/docker_build.py new file mode 100644 index 00000000..c0d6be28 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_build.py @@ -0,0 +1,8 @@ +from ..common import * + +# This is a minimal settings file allowing us to run "update_assets" +# in the Dockerfile + +DATABASES = {"default": {}} + +XQUEUE_INTERFACE = {"url": None, "django_auth": None} diff --git a/releases/eucalyptus/3/wb/config/cms/docker_build_development.py b/releases/eucalyptus/3/wb/config/cms/docker_build_development.py new file mode 120000 index 00000000..d8fcf3a2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_build_development.py @@ -0,0 +1 @@ +docker_build.py \ No newline at end of file diff --git a/releases/eucalyptus/3/wb/config/cms/docker_build_production.py b/releases/eucalyptus/3/wb/config/cms/docker_build_production.py new file mode 120000 index 00000000..d8fcf3a2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_build_production.py @@ -0,0 +1 @@ +docker_build.py \ No newline at end of file diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run.py b/releases/eucalyptus/3/wb/config/cms/docker_run.py new file mode 100644 index 00000000..a1e4f25e --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_run.py @@ -0,0 +1,3 @@ +# This file is meant to import the environment related settings file + +from docker_run_production import * diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_development.py b/releases/eucalyptus/3/wb/config/cms/docker_run_development.py new file mode 100644 index 00000000..e290fe84 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_development.py @@ -0,0 +1,25 @@ +# This file includes overrides to build the `development` environment for the CMS, starting from +# the settings of the `production` environment + +from docker_run_production import * +from lms.envs.fun.utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +if "sentry" in LOGGING.get("handlers"): + LOGGING["handlers"]["sentry"]["environment"] = "development" + +DEBUG = True +REQUIRE_DEBUG = True + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) + + +PIPELINE_ENABLED = False +STATICFILES_STORAGE = "openedx.core.storage.DevelopmentStorage" + +ALLOWED_HOSTS = ["*"] +FEATURES["AUTOMATIC_AUTH_FOR_TESTING"] = True diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_preprod.py b/releases/eucalyptus/3/wb/config/cms/docker_run_preprod.py new file mode 100644 index 00000000..db742252 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_preprod.py @@ -0,0 +1,14 @@ +# This file includes overrides to build the `preprod` environment for the LMS +# starting from the settings of the `production` environment + +from docker_run_production import * +from lms.envs.fun.utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +LOGGING["handlers"]["sentry"]["environment"] = "preprod" + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_production.py b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py new file mode 100644 index 00000000..58faf166 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py @@ -0,0 +1,668 @@ +""" +This is the default template for our main set of AWS servers. +""" + +# We intentionally define lots of variables that aren't used, and +# pylint: disable=wildcard-import, unused-wildcard-import + +# Pylint gets confused by path.py instances, which report themselves as class +# objects. As a result, pylint applies the wrong regex in validating names, +# and throws spurious errors. Therefore, we disable invalid-name checking. +# pylint: disable=invalid-name + +import json +import os +import platform + +from lms.envs.fun.utils import Configuration +from openedx.core.lib.logsettings import get_logger_config +from path import Path as path +from xmodule.modulestore.modulestore_settings import ( + convert_module_store_setting_if_needed, + update_module_store_settings, +) + +from .fun import * + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +# edX has now started using "settings.ENV_TOKENS" and "settings.AUTH_TOKENS" everywhere in the +# project, not just in the settings. Let's make sure our settings still work in this case +ENV_TOKENS = config +AUTH_TOKENS = config + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = config("SERVICE_VARIANT", default=None) + +# CONFIG_ROOT specifies the directory where the JSON configuration +# files are expected to be found. If not specified, use the project +# directory. +CONFIG_ROOT = path(config("CONFIG_ROOT", default=ENV_ROOT)) + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + +############### ALWAYS THE SAME ################################ + +RELEASE = config("RELEASE", default=None) +DEBUG = False + +EMAIL_BACKEND = "django_ses.SESBackend" +SESSION_ENGINE = "django.contrib.sessions.backends.cache" + +# IMPORTANT: With this enabled, the server must always be behind a proxy that +# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise, +# a user can fool our server into thinking it was an https connection. +# See +# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header +# for other warnings. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +###################################### CELERY ################################ + +CELERY_ALWAYS_EAGER = config("CELERY_ALWAYS_EAGER", default=False, formatter=bool) + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = "djcelery.backends.cache:CacheBackend" + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 60.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +CELERY_DEFAULT_EXCHANGE = "edx.cms.core" + +# Celery queues +DEFAULT_PRIORITY_QUEUE = config( + "DEFAULT_PRIORITY_QUEUE", default="edx.cms.core.default" +) +HIGH_PRIORITY_QUEUE = config("HIGH_PRIORITY_QUEUE", default="edx.cms.core.high") +LOW_PRIORITY_QUEUE = config("LOW_PRIORITY_QUEUE", default="edx.cms.core.low") + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = config( + "CELERY_QUEUES", + default={ + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + }, + formatter=json.loads, +) + +# Setup alternate queues, to allow access to cross-process workers +ALTERNATE_QUEUE_ENVS = config("ALTERNATE_WORKER_QUEUES", default="").split() +ALTERNATE_QUEUES = [ + DEFAULT_PRIORITY_QUEUE.replace("cms.", alternate + ".") + for alternate in ALTERNATE_QUEUE_ENVS +] +CELERY_QUEUES.update( + { + alternate: {} + for alternate in ALTERNATE_QUEUES + if alternate not in CELERY_QUEUES.keys() + } +) + +CELERY_ROUTES = "cms.celery.Router" + +############# NON-SECURE ENV CONFIG ############################## +# Things like server locations, ports, etc. + +# DEFAULT_COURSE_ABOUT_IMAGE_URL specifies the default image to show for +# courses that don't provide one +DEFAULT_COURSE_ABOUT_IMAGE_URL = config( + "DEFAULT_COURSE_ABOUT_IMAGE_URL", default=DEFAULT_COURSE_ABOUT_IMAGE_URL +) + +# GITHUB_REPO_ROOT is the base directory +# for course data +GITHUB_REPO_ROOT = config("GITHUB_REPO_ROOT", default=GITHUB_REPO_ROOT) + +STATIC_URL = "/static/studio/" +STATIC_ROOT_BASE = path("/edx/app/edxapp/staticfiles") +STATIC_ROOT = path("/edx/app/edxapp/staticfiles/studio") + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend" +) +EMAIL_FILE_PATH = config("EMAIL_FILE_PATH", default=None) + +EMAIL_HOST = config("EMAIL_HOST", default="localhost") +EMAIL_PORT = config("EMAIL_PORT", default=25, formatter=int) +EMAIL_USE_TLS = config("EMAIL_USE_TLS", default=False, formatter=bool) + +LMS_BASE = config("LMS_BASE", default="localhost:8072") +CMS_BASE = config("CMS_BASE", default="localhost:8082") + +SITE_NAME = config("SITE_NAME", default=CMS_BASE) + +ALLOWED_HOSTS = config( + "ALLOWED_HOSTS", default=[CMS_BASE.split(":")[0]], formatter=json.loads +) + +LOG_DIR = config("LOG_DIR", default="/edx/var/logs/edx") + +MEMCACHED_HOST = config("MEMCACHED_HOST", default="memcached") +MEMCACHED_PORT = config("MEMCACHED_PORT", default=11211, formatter=int) + +CACHES = config( + "CACHES", + default={ + "loc_cache": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "edx_location_mem_cache", + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": "{}:{}".format(MEMCACHED_HOST, MEMCACHED_PORT), + }, + }, + formatter=json.loads, +) + +SESSION_COOKIE_DOMAIN = config("SESSION_COOKIE_DOMAIN", default=None) +SESSION_COOKIE_HTTPONLY = config( + "SESSION_COOKIE_HTTPONLY", default=True, formatter=bool +) +SESSION_ENGINE = config( + "SESSION_ENGINE", default="django.contrib.sessions.backends.cache" +) +SESSION_COOKIE_SECURE = config( + "SESSION_COOKIE_SECURE", default=SESSION_COOKIE_SECURE, formatter=bool +) +SESSION_SAVE_EVERY_REQUEST = config( + "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool +) + +# social sharing settings +SOCIAL_SHARING_SETTINGS = config( + "SOCIAL_SHARING_SETTINGS", default=SOCIAL_SHARING_SETTINGS, formatter=json.loads +) + +REGISTRATION_EMAIL_PATTERNS_ALLOWED = config( + "REGISTRATION_EMAIL_PATTERNS_ALLOWED", default=None, formatter=json.loads +) + +# allow for environments to specify what cookie name our login subsystem should use +# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can +# happen with some browsers (e.g. Firefox) +if config("SESSION_COOKIE_NAME", default=None): + # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this + # being a str() + SESSION_COOKIE_NAME = str(config("SESSION_COOKIE_NAME")) + +# Set the names of cookies shared with the marketing site +# These have the same cookie domain as the session, which in production +# usually includes subdomains. +EDXMKTG_LOGGED_IN_COOKIE_NAME = config( + "EDXMKTG_LOGGED_IN_COOKIE_NAME", default=EDXMKTG_LOGGED_IN_COOKIE_NAME +) +EDXMKTG_USER_INFO_COOKIE_NAME = config( + "EDXMKTG_USER_INFO_COOKIE_NAME", default=EDXMKTG_USER_INFO_COOKIE_NAME +) + +# Determines whether the CSRF token can be transported on +# unencrypted channels. It is set to False here for backward compatibility, +# but it is highly recommended that this is True for enviroments accessed +# by end users. +CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=False) + + +# Email overrides +DEFAULT_FROM_EMAIL = config("DEFAULT_FROM_EMAIL", default=DEFAULT_FROM_EMAIL) +DEFAULT_FEEDBACK_EMAIL = config( + "DEFAULT_FEEDBACK_EMAIL", default=DEFAULT_FEEDBACK_EMAIL +) +ADMINS = config("ADMINS", default=ADMINS, formatter=json.loads) +SERVER_EMAIL = config("SERVER_EMAIL", default=SERVER_EMAIL) +MKTG_URLS = config("MKTG_URLS", default=MKTG_URLS, formatter=json.loads) +TECH_SUPPORT_EMAIL = config("TECH_SUPPORT_EMAIL", default=TECH_SUPPORT_EMAIL) + +for name, value in config("CODE_JAIL", default={}, formatter=json.loads).items(): + oldvalue = CODE_JAIL.get(name) + if isinstance(oldvalue, dict): + for subname, subvalue in value.items(): + oldvalue[subname] = subvalue + else: + CODE_JAIL[name] = value + +COURSES_WITH_UNSAFE_CODE = config( + "COURSES_WITH_UNSAFE_CODE", default=[], formatter=json.loads +) + +LOCALE_PATHS = config("LOCALE_PATHS", default=LOCALE_PATHS, formatter=json.loads) + +ASSET_IGNORE_REGEX = config("ASSET_IGNORE_REGEX", default=ASSET_IGNORE_REGEX) + +# Theme overrides +THEME_NAME = config("THEME_NAME", default=None) + +# following setting is for backward compatibility +if config("COMPREHENSIVE_THEME_DIR", default=None): + COMPREHENSIVE_THEME_DIR = config("COMPREHENSIVE_THEME_DIR") + +COMPREHENSIVE_THEME_DIRS = ( + config( + "COMPREHENSIVE_THEME_DIRS", + default=COMPREHENSIVE_THEME_DIRS, + formatter=json.loads, + ) + or [] +) +DEFAULT_SITE_THEME = config("DEFAULT_SITE_THEME", default=DEFAULT_SITE_THEME) +ENABLE_COMPREHENSIVE_THEMING = config( + "ENABLE_COMPREHENSIVE_THEMING", default=ENABLE_COMPREHENSIVE_THEMING +) + +# Timezone overrides +TIME_ZONE = config("TIME_ZONE", default=TIME_ZONE) + +# Push to LMS overrides +GIT_REPO_EXPORT_DIR = config( + "GIT_REPO_EXPORT_DIR", default="/edx/var/edxapp/export_course_repos" +) + +# Translation overrides +LANGUAGES = config("LANGUAGES", default=LANGUAGES, formatter=json.loads) +LANGUAGE_CODE = config("LANGUAGE_CODE", default=LANGUAGE_CODE) + +USE_I18N = config("USE_I18N", default=USE_I18N, formatter=bool) +ALL_LANGUAGES = config("ALL_LANGUAGES", default=ALL_LANGUAGES, formatter=json.loads) + +# Override feature by feature by whatever is being redefined in the settings.yaml file +CONFIG_FEATURES = config("FEATURES", default={}, formatter=json.loads) +FEATURES.update(CONFIG_FEATURES) + + +# Additional installed apps +for app in config("ADDL_INSTALLED_APPS", default=[], formatter=json.loads): + INSTALLED_APPS += (app,) + +WIKI_ENABLED = config("WIKI_ENABLED", default=WIKI_ENABLED, formatter=bool) + +# Configure Logging + +# Default format for syslog logging +standard_format = "%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s" +syslog_format = ( + "[variant:cms][%(name)s][env:sandbox] %(levelname)s " + "[{hostname} %(process)d] [%(filename)s:%(lineno)d] - %(message)s" +).format(hostname=platform.node().split(".")[0]) + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "local": { + "formatter": "syslog_format", + "class": "logging.StreamHandler", + "level": "INFO", + }, + "tracking": { + "formatter": "raw", + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "console": { + "formatter": "standard", + "class": "logging.StreamHandler", + "level": "INFO", + }, + }, + "formatters": { + "raw": {"format": "%(message)s"}, + "syslog_format": {"format": syslog_format}, + "standard": {"format": standard_format}, + }, + "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, + "loggers": { + "": {"level": "INFO", "propagate": False, "handlers": ["console", "local"]}, + "tracking": {"level": "DEBUG", "propagate": False, "handlers": ["tracking"]}, + }, +} + +SENTRY_DSN = config("SENTRY_DSN", default=None) +if SENTRY_DSN: + LOGGING["loggers"][""]["handlers"].append("sentry") + LOGGING["handlers"]["sentry"] = { + "class": "raven.handlers.logging.SentryHandler", + "dsn": SENTRY_DSN, + "level": "ERROR", + "environment": "production", + "release": RELEASE, + } + +# FIXME: the PLATFORM_NAME and PLATFORM_DESCRIPTION settings should be set to lazy translatable +# strings but edX tries to serialize them with a default json serializer which breaks. We should +# submit a PR to fix it in edx-platform +PLATFORM_NAME = config("PLATFORM_NAME", default="Your Platform Name Here") +PLATFORM_DESCRIPTION = config( + "PLATFORM_DESCRIPTION", default="Your Platform Description Here" +) +STUDIO_NAME = config("STUDIO_NAME", default=STUDIO_NAME) +STUDIO_SHORT_NAME = config("STUDIO_SHORT_NAME", default=STUDIO_SHORT_NAME) + +# Event Tracking +TRACKING_IGNORE_URL_PATTERNS = config( + "TRACKING_IGNORE_URL_PATTERNS", + default=TRACKING_IGNORE_URL_PATTERNS, + formatter=json.loads, +) + +# Django CAS external authentication settings +CAS_EXTRA_LOGIN_PARAMS = config( + "CAS_EXTRA_LOGIN_PARAMS", default=None, formatter=json.loads +) +if FEATURES.get("AUTH_USE_CAS"): + CAS_SERVER_URL = config("CAS_SERVER_URL", default=None) + AUTHENTICATION_BACKENDS = [ + "django.contrib.auth.backends.ModelBackend", + "django_cas.backends.CASBackend", + ] + INSTALLED_APPS += ("django_cas",) + MIDDLEWARE_CLASSES.append("django_cas.middleware.CASMiddleware") + CAS_ATTRIBUTE_CALLBACK = config( + "CAS_ATTRIBUTE_CALLBACK", default=None, formatter=json.loads + ) + if CAS_ATTRIBUTE_CALLBACK: + import importlib + + CAS_USER_DETAILS_RESOLVER = getattr( + importlib.import_module(CAS_ATTRIBUTE_CALLBACK["module"]), + CAS_ATTRIBUTE_CALLBACK["function"], + ) + +################ SECURE AUTH ITEMS ############################### + +############### XBlock filesystem field config ########## +DJFS = config( + "DJFS", + default={ + "directory_root": "/edx/var/edxapp/django-pyfs/static/django-pyfs", + "type": "osfs", + "url_root": "/static/django-pyfs", + }, + formatter=json.loads, +) + +EMAIL_HOST_USER = config("EMAIL_HOST_USER", default=EMAIL_HOST_USER) +EMAIL_HOST_PASSWORD = config("EMAIL_HOST_PASSWORD", default=EMAIL_HOST_PASSWORD) + +AWS_SES_REGION_NAME = config("AWS_SES_REGION_NAME", default="us-east-1") +AWS_SES_REGION_ENDPOINT = config( + "AWS_SES_REGION_ENDPOINT", default="email.us-east-1.amazonaws.com" +) + +# Note that this is the Studio key for Segment. There is a separate key for the LMS. +CMS_SEGMENT_KEY = config("SEGMENT_KEY", default=None) + +SECRET_KEY = config("SECRET_KEY", default="ThisIsAnExampleKeyForDevPurposeOnly") + +DEFAULT_FILE_STORAGE = config( + "DEFAULT_FILE_STORAGE", default="django.core.files.storage.FileSystemStorage" +) + +# Databases + +DATABASE_ENGINE = config("DATABASE_ENGINE", default="django.db.backends.mysql") +DATABASE_HOST = config("DATABASE_HOST", default="mysql") +DATABASE_PORT = config("DATABASE_PORT", default=3306, formatter=int) +DATABASE_NAME = config("DATABASE_NAME", default="edxapp") +DATABASE_USER = config("DATABASE_USER", default="edxapp_user") +DATABASE_PASSWORD = config("DATABASE_PASSWORD", default="password") + +DATABASES = config( + "DATABASES", + default={ + "default": { + "ENGINE": DATABASE_ENGINE, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + } + }, + formatter=json.loads, +) + +# The normal database user does not have enough permissions to run migrations. +# Migrations are run with separate credentials, given as DB_MIGRATION_* +# environment variables +for name, database in DATABASES.items(): + if name != "read_replica": + database.update( + { + "ENGINE": config("DB_MIGRATION_ENGINE", default=database["ENGINE"]), + "USER": config("DB_MIGRATION_USER", default=database["USER"]), + "PASSWORD": config("DB_MIGRATION_PASS", default=database["PASSWORD"]), + "NAME": config("DB_MIGRATION_NAME", default=database["NAME"]), + "HOST": config("DB_MIGRATION_HOST", default=database["HOST"]), + "PORT": config("DB_MIGRATION_PORT", default=database["PORT"]), + } + ) + +MODULESTORE = convert_module_store_setting_if_needed( + config("MODULESTORE", default=MODULESTORE, formatter=json.loads) +) + +MODULESTORE_FIELD_OVERRIDE_PROVIDERS = config( + "MODULESTORE_FIELD_OVERRIDE_PROVIDERS", + default=MODULESTORE_FIELD_OVERRIDE_PROVIDERS, + formatter=json.loads, +) + +XBLOCK_FIELD_DATA_WRAPPERS = config( + "XBLOCK_FIELD_DATA_WRAPPERS", + default=XBLOCK_FIELD_DATA_WRAPPERS, + formatter=json.loads, +) + +MONGODB_PASSWORD = config("MONGODB_PASSWORD", default="") +MONGODB_HOST = config("MONGODB_HOST", default="mongodb") +MONGODB_PORT = config("MONGODB_PORT", default=27017, formatter=int) +MONGODB_NAME = config("MONGODB_NAME", default="edxapp") +MONGODB_USER = config("MONGODB_USER", default=None) +MONGODB_SSL = config("MONGODB_SSL", default=False, formatter=bool) +MONGODB_REPLICASET = config("MONGODB_REPLICASET", default=None) +# Accepted read_preference value can be found here https://github.com/mongodb/mongo-python-driver/blob/2.9.1/pymongo/read_preferences.py#L54 +MONGODB_READ_PREFERENCE = config("MONGODB_READ_PREFERENCE", default="PRIMARY") + +DOC_STORE_CONFIG = config( + "DOC_STORE_CONFIG", + default={ + "collection": "modulestore", + "host": MONGODB_HOST, + "port": MONGODB_PORT, + "db": MONGODB_NAME, + "user": MONGODB_USER, + "password": MONGODB_PASSWORD, + "ssl": MONGODB_SSL, + "replicaSet": MONGODB_REPLICASET, + "read_preference": MONGODB_READ_PREFERENCE, + }, + formatter=json.loads, +) + +update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) + +MONGODB_LOG = config("MONGODB_LOG", default={}, formatter=json.loads) + +CONTENTSTORE = config( + "CONTENTSTORE", + default={ + "DOC_STORE_CONFIG": DOC_STORE_CONFIG, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + }, + formatter=json.loads, +) + +# Datadog for events! +DATADOG = config("DATADOG", default={}, formatter=json.loads) + +# TODO: deprecated (compatibility with previous settings) +DATADOG["api_key"] = config("DATADOG_API", default=None) + +# Celery Broker +CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") +CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") +CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") +CELERY_BROKER_HOST = config("CELERY_BROKER_HOST", default="redis") +CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) +CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) + +BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( + transport=CELERY_BROKER_TRANSPORT, + user=CELERY_BROKER_USER, + password=CELERY_BROKER_PASSWORD, + host=CELERY_BROKER_HOST, + port=CELERY_BROKER_PORT, + vhost=CELERY_BROKER_VHOST, +) + +# Event tracking +TRACKING_BACKENDS.update(config("TRACKING_BACKENDS", default={}, formatter=json.loads)) +EVENT_TRACKING_BACKENDS["tracking_logs"]["OPTIONS"]["backends"].update( + config("EVENT_TRACKING_BACKENDS", default={}, formatter=json.loads) +) +EVENT_TRACKING_BACKENDS["segmentio"]["OPTIONS"]["processors"][0]["OPTIONS"][ + "whitelist" +].extend( + config("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", default=[], formatter=json.loads) +) + +VIRTUAL_UNIVERSITIES = config("VIRTUAL_UNIVERSITIES", default=[], formatter=json.loads) + +##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### +MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = config( + "MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", default=5, formatter=int +) +MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = config( + "MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", default=15 * 60, formatter=int +) + +#### PASSWORD POLICY SETTINGS ##### +PASSWORD_MIN_LENGTH = config("PASSWORD_MIN_LENGTH", default=12, formatter=int) +PASSWORD_MAX_LENGTH = config("PASSWORD_MAX_LENGTH", default=None, formatter=int) + +PASSWORD_COMPLEXITY = config( + "PASSWORD_COMPLEXITY", + default={"UPPER": 1, "LOWER": 1, "DIGITS": 1}, + formatter=json.loads, +) +PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = config( + "PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD", + default=PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD, + formatter=int, +) +PASSWORD_DICTIONARY = config("PASSWORD_DICTIONARY", default=[], formatter=json.loads) + +### INACTIVITY SETTINGS #### +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = config( + "SESSION_INACTIVITY_TIMEOUT_IN_SECONDS", default=None, formatter=int +) + +##### X-Frame-Options response header settings ##### +X_FRAME_OPTIONS = config("X_FRAME_OPTIONS", default=X_FRAME_OPTIONS) + +##### ADVANCED_SECURITY_CONFIG ##### +ADVANCED_SECURITY_CONFIG = config( + "ADVANCED_SECURITY_CONFIG", default={}, formatter=json.loads +) + +################ ADVANCED COMPONENT/PROBLEM TYPES ############### + +ADVANCED_PROBLEM_TYPES = config( + "ADVANCED_PROBLEM_TYPES", default=ADVANCED_PROBLEM_TYPES, formatter=json.loads +) +DEPRECATED_ADVANCED_COMPONENT_TYPES = config( + "DEPRECATED_ADVANCED_COMPONENT_TYPES", + default=DEPRECATED_ADVANCED_COMPONENT_TYPES, + formatter=json.loads, +) + +################ VIDEO UPLOAD PIPELINE ############### + +VIDEO_UPLOAD_PIPELINE = config( + "VIDEO_UPLOAD_PIPELINE", default=VIDEO_UPLOAD_PIPELINE, formatter=json.loads +) + +################ PUSH NOTIFICATIONS ############### + +PARSE_KEYS = config("PARSE_KEYS", default={}, formatter=json.loads) + + +# Video Caching. Pairing country codes with CDN URLs. +# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='} +VIDEO_CDN_URL = config("VIDEO_CDN_URL", default={}, formatter=json.loads) + +if FEATURES["ENABLE_COURSEWARE_INDEX"] or FEATURES["ENABLE_LIBRARY_INDEX"]: + # Use ElasticSearch for the search engine + SEARCH_ENGINE = "search.elastic.ElasticSearchEngine" + +ELASTIC_SEARCH_CONFIG = config("ELASTIC_SEARCH_CONFIG", default=[{}]) + +XBLOCK_SETTINGS = config("XBLOCK_SETTINGS", default={}, formatter=json.loads) +XBLOCK_SETTINGS.setdefault("VideoDescriptor", {})["licensing_enabled"] = FEATURES.get( + "LICENSING", False +) +XBLOCK_SETTINGS.setdefault("VideoModule", {})["YOUTUBE_API_KEY"] = FEATURES.get( + "YOUTUBE_API_KEY", YOUTUBE_API_KEY +) + +################# PROCTORING CONFIGURATION ################## + +PROCTORING_BACKEND_PROVIDER = config( + "PROCTORING_BACKEND_PROVIDER", default=PROCTORING_BACKEND_PROVIDER +) +PROCTORING_SETTINGS = config( + "PROCTORING_SETTINGS", default=PROCTORING_SETTINGS, fomatter=json.loads +) + +################# MICROSITE #################### +MICROSITE_CONFIGURATION = config( + "MICROSITE_CONFIGURATION", default={}, formatter=json.loads +) +MICROSITE_ROOT_DIR = path(config("MICROSITE_ROOT_DIR", default="")) +# this setting specify which backend to be used when pulling microsite specific configuration +MICROSITE_BACKEND = config("MICROSITE_BACKEND", default=MICROSITE_BACKEND) +# this setting specify which backend to be used when loading microsite specific templates +MICROSITE_TEMPLATE_BACKEND = config( + "MICROSITE_TEMPLATE_BACKEND", default=MICROSITE_TEMPLATE_BACKEND +) +# TTL for microsite database template cache +MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = config( + "MICROSITE_DATABASE_TEMPLATE_CACHE_TTL", + default=MICROSITE_DATABASE_TEMPLATE_CACHE_TTL, + formatter=int, +) + +############################ OAUTH2 Provider ################################### + +# OpenID Connect issuer ID. Normally the URL of the authentication endpoint. +OAUTH_OIDC_ISSUER = config("OAUTH_OIDC_ISSUER", default=None) + +######################## CUSTOM COURSES for EDX CONNECTOR ###################### +if FEATURES.get("CUSTOM_COURSES_EDX"): + INSTALLED_APPS += ("openedx.core.djangoapps.ccxcon",) + +# Partner support link for CMS footer +PARTNER_SUPPORT_EMAIL = config("PARTNER_SUPPORT_EMAIL", default=PARTNER_SUPPORT_EMAIL) + +# Affiliate cookie tracking +AFFILIATE_COOKIE_NAME = config("AFFILIATE_COOKIE_NAME", default=AFFILIATE_COOKIE_NAME) diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_staging.py b/releases/eucalyptus/3/wb/config/cms/docker_run_staging.py new file mode 100644 index 00000000..bdf5ffcb --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_staging.py @@ -0,0 +1,14 @@ +# This file includes overrides to build the `staging` environment for the LMS starting from the +# settings of the `production` environment + +from docker_run_production import * +from lms.envs.fun.utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +LOGGING["handlers"]["sentry"]["environment"] = "staging" + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) diff --git a/releases/eucalyptus/3/wb/config/cms/fun.py b/releases/eucalyptus/3/wb/config/cms/fun.py new file mode 100644 index 00000000..b0ca7866 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/cms/fun.py @@ -0,0 +1,53 @@ +import json + +from lms.envs.fun.utils import Configuration + +from ..common import * + + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +# Fun-apps configuration +INSTALLED_APPS += ( + "fun", + "videoproviders", + "teachers", + "courses", + "haystack", + "universities", + "easy_thumbnails", + "ckeditor", + "selftest", + "raven.contrib.django.raven_compat", +) + +ROOT_URLCONF = "fun.cms.urls" + +# This constant as nothing to do with github. +# Path is used to store tar.gz courses before import process +GITHUB_REPO_ROOT = DATA_DIR + +# ### THIRD-PARTY SETTINGS ### + +# Haystack configuration (default is minimal working configuration) +HAYSTACK_CONNECTIONS = config( + "HAYSTACK_CONNECTIONS", + default={ + "default": {"ENGINE": "courses.search_indexes.ConfigurableElasticSearchEngine"} + }, + formatter=json.loads, +) + +CKEDITOR_UPLOAD_PATH = "./" + +# ### FUN-APPS SETTINGS ### +# -- Base -- +FUN_BASE_ROOT = path(os.path.dirname(imp.find_module("funsite")[1])) + +# Add 'theme/cms/templates' directory to MAKO template finder to override some +# CMS templates +MAKO_TEMPLATES["main"] = [FUN_BASE_ROOT / "fun/templates/cms"] + MAKO_TEMPLATES["main"] + +# Generic LTI configuration +LTI_XBLOCK_CONFIGURATIONS = [{"display_name": "LTI consumer"}] diff --git a/releases/eucalyptus/3/wb/config/lms/__init__.py b/releases/eucalyptus/3/wb/config/lms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/releases/eucalyptus/3/wb/config/lms/docker_build.py b/releases/eucalyptus/3/wb/config/lms/docker_build.py new file mode 100644 index 00000000..c0d6be28 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_build.py @@ -0,0 +1,8 @@ +from ..common import * + +# This is a minimal settings file allowing us to run "update_assets" +# in the Dockerfile + +DATABASES = {"default": {}} + +XQUEUE_INTERFACE = {"url": None, "django_auth": None} diff --git a/releases/eucalyptus/3/wb/config/lms/docker_build_development.py b/releases/eucalyptus/3/wb/config/lms/docker_build_development.py new file mode 120000 index 00000000..d8fcf3a2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_build_development.py @@ -0,0 +1 @@ +docker_build.py \ No newline at end of file diff --git a/releases/eucalyptus/3/wb/config/lms/docker_build_production.py b/releases/eucalyptus/3/wb/config/lms/docker_build_production.py new file mode 120000 index 00000000..d8fcf3a2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_build_production.py @@ -0,0 +1 @@ +docker_build.py \ No newline at end of file diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run.py b/releases/eucalyptus/3/wb/config/lms/docker_run.py new file mode 100644 index 00000000..a1e4f25e --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_run.py @@ -0,0 +1,3 @@ +# This file is meant to import the environment related settings file + +from docker_run_production import * diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_development.py b/releases/eucalyptus/3/wb/config/lms/docker_run_development.py new file mode 100644 index 00000000..ef0996b4 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_development.py @@ -0,0 +1,24 @@ +# This file includes overrides to build the `development` environment for the LMS starting from the +# settings of the `production` environment + +from docker_run_production import * +from .utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +if "sentry" in LOGGING.get("handlers"): + LOGGING["handlers"]["sentry"]["environment"] = "development" + +DEBUG = True +REQUIRE_DEBUG = True + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) + +PIPELINE_ENABLED = False +STATICFILES_STORAGE = "openedx.core.storage.DevelopmentStorage" + +ALLOWED_HOSTS = ["*"] +FEATURES["AUTOMATIC_AUTH_FOR_TESTING"] = True diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_preprod.py b/releases/eucalyptus/3/wb/config/lms/docker_run_preprod.py new file mode 100644 index 00000000..ed5983f2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_preprod.py @@ -0,0 +1,14 @@ +# This file includes overrides to build the `preprod` environment for the LMS +# starting from the settings of the `production` environment + +from docker_run_production import * +from .utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +LOGGING["handlers"]["sentry"]["environment"] = "preprod" + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_production.py b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py new file mode 100644 index 00000000..0b6829ed --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py @@ -0,0 +1,1233 @@ +""" +This is the default template for our main set of servers. This does NOT +cover the content machines, which use content.py + +Common traits: +* Use memcached, and cache-backed sessions +* Use a MySQL 5.1 database +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=wildcard-import, unused-wildcard-import + +# Pylint gets confused by path.py instances, which report themselves as class +# objects. As a result, pylint applies the wrong regex in validating names, +# and throws spurious errors. Therefore, we disable invalid-name checking. +# pylint: disable=invalid-name + +import datetime +import dateutil +import json +import os +import platform +import warnings + +from openedx.core.lib.logsettings import get_logger_config +from path import Path as path +from xmodule.modulestore.modulestore_settings import ( + convert_module_store_setting_if_needed, + update_module_store_settings, +) + +from .utils import Configuration +from .fun import * + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +# edX has now started using "settings.ENV_TOKENS" and "settings.AUTH_TOKENS" everywhere in the +# project, not just in the settings. Let's make sure our settings still work in this case +ENV_TOKENS = config +AUTH_TOKENS = config + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = config("SERVICE_VARIANT", default=None) + +# CONFIG_ROOT specifies the directory where the JSON configuration +# files are expected to be found. If not specified, use the project +# directory. +CONFIG_ROOT = path(config("CONFIG_ROOT", default=ENV_ROOT)) + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + +################################ ALWAYS THE SAME ############################## + +RELEASE = config("RELEASE", default=None) +DEBUG = False +DEFAULT_TEMPLATE_ENGINE["OPTIONS"]["debug"] = False + +SESSION_ENGINE = config( + "SESSION_ENGINE", default="django.contrib.sessions.backends.cache" +) + +# IMPORTANT: With this enabled, the server must always be behind a proxy that +# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise, +# a user can fool our server into thinking it was an https connection. +# See +# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header +# for other warnings. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +###################################### CELERY ################################ + +CELERY_ALWAYS_EAGER = config("CELERY_ALWAYS_EAGER", default=False, formatter=bool) + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = "djcelery.backends.cache:CacheBackend" + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 60.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +# Celery queues + +DEFAULT_PRIORITY_QUEUE = config( + "DEFAULT_PRIORITY_QUEUE", default="edx.lms.core.default" +) +HIGH_PRIORITY_QUEUE = config("HIGH_PRIORITY_QUEUE", default="edx.lms.core.high") +LOW_PRIORITY_QUEUE = config("LOW_PRIORITY_QUEUE", default="edx.lms.core.low") +HIGH_MEM_QUEUE = config("HIGH_MEM_QUEUE", default="edx.lms.core.high_mem") + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = config( + "CELERY_QUEUES", + default={ + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, + }, + formatter=json.loads, +) + +# Setup alternate queues, to allow access to cross-process workers +ALTERNATE_QUEUE_ENVS = config("ALTERNATE_WORKER_QUEUES", default="").split() +ALTERNATE_QUEUES = [ + DEFAULT_PRIORITY_QUEUE.replace("lms.", alternate + ".") + for alternate in ALTERNATE_QUEUE_ENVS +] +CELERY_QUEUES.update( + { + alternate: {} + for alternate in ALTERNATE_QUEUES + if alternate not in CELERY_QUEUES.keys() + } +) +CELERY_ROUTES = "lms.celery.Router" + +# Force accepted content to "json" only. If we also accept pickle-serialized +# messages, the worker will crash when it's running with a privileged user (even +# if it's not the root user but a user belonging to the root group, which is our +# case with OpenShift). +CELERY_ACCEPT_CONTENT = ["json"] + +CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict + +########################## NON-SECURE ENV CONFIG ############################## +# Things like server locations, ports, etc. + +STATIC_ROOT_BASE = path("/edx/app/edxapp/staticfiles") +STATIC_ROOT = STATIC_ROOT_BASE +STATIC_URL = "/static/" + +MEDIA_ROOT = path("/edx/var/edxapp/media/") +MEDIA_URL = "/media/" + +# DEFAULT_COURSE_ABOUT_IMAGE_URL specifies the default image to show for courses that don't provide one +DEFAULT_COURSE_ABOUT_IMAGE_URL = config( + "DEFAULT_COURSE_ABOUT_IMAGE_URL", default=DEFAULT_COURSE_ABOUT_IMAGE_URL +) + + +PLATFORM_NAME = config("PLATFORM_NAME", default=PLATFORM_NAME) +# For displaying on the receipt. At Stanford PLATFORM_NAME != MERCHANT_NAME, but PLATFORM_NAME is a fine default +PLATFORM_TWITTER_ACCOUNT = config( + "PLATFORM_TWITTER_ACCOUNT", default=PLATFORM_TWITTER_ACCOUNT +) +PLATFORM_FACEBOOK_ACCOUNT = config( + "PLATFORM_FACEBOOK_ACCOUNT", default=PLATFORM_FACEBOOK_ACCOUNT +) + +SOCIAL_SHARING_SETTINGS = config( + "SOCIAL_SHARING_SETTINGS", default=SOCIAL_SHARING_SETTINGS, formatter=json.loads +) + +# Social media links for the page footer +SOCIAL_MEDIA_FOOTER_URLS = config( + "SOCIAL_MEDIA_FOOTER_URLS", default=SOCIAL_MEDIA_FOOTER_URLS, formatter=json.loads +) + +CC_MERCHANT_NAME = config("CC_MERCHANT_NAME", default=PLATFORM_NAME) +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend" +) +EMAIL_FILE_PATH = config("EMAIL_FILE_PATH", default=None) +EMAIL_HOST = config("EMAIL_HOST", default="localhost") +EMAIL_PORT = config("EMAIL_PORT", default=25) # django default is 25 +EMAIL_USE_TLS = config("EMAIL_USE_TLS", default=False) # django default is False +SITE_NAME = config("SITE_NAME", default=None) +HTTPS = config("HTTPS", default=HTTPS) +SESSION_ENGINE = config("SESSION_ENGINE", default=SESSION_ENGINE) +SESSION_COOKIE_DOMAIN = config("SESSION_COOKIE_DOMAIN", default=None) +SESSION_COOKIE_HTTPONLY = config( + "SESSION_COOKIE_HTTPONLY", default=True, formatter=bool +) +SESSION_COOKIE_SECURE = config( + "SESSION_COOKIE_SECURE", default=SESSION_COOKIE_SECURE, formatter=bool +) +SESSION_SAVE_EVERY_REQUEST = config( + "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool +) + +AWS_SES_REGION_NAME = config("AWS_SES_REGION_NAME", default="us-east-1") +AWS_SES_REGION_ENDPOINT = config( + "AWS_SES_REGION_ENDPOINT", default="email.us-east-1.amazonaws.com" +) + +REGISTRATION_EXTRA_FIELDS = config( + "REGISTRATION_EXTRA_FIELDS", default=REGISTRATION_EXTRA_FIELDS, formatter=json.loads +) +REGISTRATION_EXTENSION_FORM = config( + "REGISTRATION_EXTENSION_FORM", default=REGISTRATION_EXTENSION_FORM +) +REGISTRATION_EMAIL_PATTERNS_ALLOWED = config( + "REGISTRATION_EMAIL_PATTERNS_ALLOWED", default=None +) + + +# Set the names of cookies shared with the marketing site +# These have the same cookie domain as the session, which in production +# usually includes subdomains. +EDXMKTG_LOGGED_IN_COOKIE_NAME = config( + "EDXMKTG_LOGGED_IN_COOKIE_NAME", default=EDXMKTG_LOGGED_IN_COOKIE_NAME +) +EDXMKTG_USER_INFO_COOKIE_NAME = config( + "EDXMKTG_USER_INFO_COOKIE_NAME", default=EDXMKTG_USER_INFO_COOKIE_NAME +) + +# Override feature by feature by whatever is being redefined in the settings.yaml file +CONFIG_FEATURES = config("FEATURES", default={}, formatter=json.loads) +FEATURES.update(CONFIG_FEATURES) + +# Backward compatibility for deprecated feature names +if "ENABLE_S3_GRADE_DOWNLOADS" in FEATURES: + warnings.warn( + "'ENABLE_S3_GRADE_DOWNLOADS' is deprecated. Please use 'ENABLE_GRADE_DOWNLOADS' instead", + DeprecationWarning, + ) + FEATURES["ENABLE_GRADE_DOWNLOADS"] = FEATURES["ENABLE_S3_GRADE_DOWNLOADS"] + +LMS_BASE = config("LMS_BASE", default="localhost:8072") +CMS_BASE = config("CMS_BASE", default="localhost:8082") + +SITE_NAME = config("SITE_NAME", default=LMS_BASE) + +ALLOWED_HOSTS = config( + "ALLOWED_HOSTS", default=[LMS_BASE.split(":")[0]], formatter=json.loads +) +if FEATURES.get("PREVIEW_LMS_BASE"): + ALLOWED_HOSTS.append(FEATURES["PREVIEW_LMS_BASE"]) + +# allow for environments to specify what cookie name our login subsystem should use +# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can +# happen with some browsers (e.g. Firefox) +if config("SESSION_COOKIE_NAME", default=None): + # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this + # being a str() + SESSION_COOKIE_NAME = str(config("SESSION_COOKIE_NAME")) + +MEMCACHED_HOST = config("MEMCACHED_HOST", default="memcached") +MEMCACHED_PORT = config("MEMCACHED_PORT", default=11211, formatter=int) + +CACHES = config( + "CACHES", + default={ + "loc_cache": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "edx_location_mem_cache", + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": "{}:{}".format(MEMCACHED_HOST, MEMCACHED_PORT), + }, + }, + formatter=json.loads, +) + +# Email overrides +DEFAULT_FROM_EMAIL = config("DEFAULT_FROM_EMAIL", default=DEFAULT_FROM_EMAIL) +DEFAULT_FEEDBACK_EMAIL = config( + "DEFAULT_FEEDBACK_EMAIL", default=DEFAULT_FEEDBACK_EMAIL +) +ADMINS = config("ADMINS", default=ADMINS, formatter=json.loads) +SERVER_EMAIL = config("SERVER_EMAIL", default=SERVER_EMAIL) +TECH_SUPPORT_EMAIL = config("TECH_SUPPORT_EMAIL", default=TECH_SUPPORT_EMAIL) +CONTACT_EMAIL = config("CONTACT_EMAIL", default=CONTACT_EMAIL) +BUGS_EMAIL = config("BUGS_EMAIL", default=BUGS_EMAIL) +PAYMENT_SUPPORT_EMAIL = config("PAYMENT_SUPPORT_EMAIL", default=PAYMENT_SUPPORT_EMAIL) +FINANCE_EMAIL = config("FINANCE_EMAIL", default=FINANCE_EMAIL) +UNIVERSITY_EMAIL = config("UNIVERSITY_EMAIL", default=UNIVERSITY_EMAIL) +PRESS_EMAIL = config("PRESS_EMAIL", default=PRESS_EMAIL) + +CONTACT_MAILING_ADDRESS = config( + "CONTACT_MAILING_ADDRESS", default=CONTACT_MAILING_ADDRESS +) + +# Currency +PAID_COURSE_REGISTRATION_CURRENCY = config( + "PAID_COURSE_REGISTRATION_CURRENCY", default=PAID_COURSE_REGISTRATION_CURRENCY +) + +# Payment Report Settings +PAYMENT_REPORT_GENERATOR_GROUP = config( + "PAYMENT_REPORT_GENERATOR_GROUP", default=PAYMENT_REPORT_GENERATOR_GROUP +) + +# Bulk Email overrides +BULK_EMAIL_DEFAULT_FROM_EMAIL = config( + "BULK_EMAIL_DEFAULT_FROM_EMAIL", default=BULK_EMAIL_DEFAULT_FROM_EMAIL +) +BULK_EMAIL_EMAILS_PER_TASK = config( + "BULK_EMAIL_EMAILS_PER_TASK", default=BULK_EMAIL_EMAILS_PER_TASK, formatter=int +) +BULK_EMAIL_DEFAULT_RETRY_DELAY = config( + "BULK_EMAIL_DEFAULT_RETRY_DELAY", + default=BULK_EMAIL_DEFAULT_RETRY_DELAY, + formatter=int, +) +BULK_EMAIL_MAX_RETRIES = config( + "BULK_EMAIL_MAX_RETRIES", default=BULK_EMAIL_MAX_RETRIES, formatter=int +) +BULK_EMAIL_INFINITE_RETRY_CAP = config( + "BULK_EMAIL_INFINITE_RETRY_CAP", + default=BULK_EMAIL_INFINITE_RETRY_CAP, + formatter=int, +) +BULK_EMAIL_LOG_SENT_EMAILS = config( + "BULK_EMAIL_LOG_SENT_EMAILS", default=BULK_EMAIL_LOG_SENT_EMAILS, formatter=bool +) +BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS = config( + "BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS", + default=BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS, + formatter=int, +) +# We want Bulk Email running on the high-priority queue, so we define the +# routing key that points to it. At the moment, the name is the same. +# We have to reset the value here, since we have changed the value of the queue name. +BULK_EMAIL_ROUTING_KEY = config("BULK_EMAIL_ROUTING_KEY", default=HIGH_PRIORITY_QUEUE) + +# We can run smaller jobs on the low priority queue. See note above for why +# we have to reset the value here. +BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = LOW_PRIORITY_QUEUE + +# Theme overrides +THEME_NAME = config("THEME_NAME", default=None) + +# following setting is for backward compatibility +if config("COMPREHENSIVE_THEME_DIR", default=None): + COMPREHENSIVE_THEME_DIR = config("COMPREHENSIVE_THEME_DIR") + +COMPREHENSIVE_THEME_DIRS = ( + config( + "COMPREHENSIVE_THEME_DIRS", + default=COMPREHENSIVE_THEME_DIRS, + formatter=json.loads, + ) + or [] +) +DEFAULT_SITE_THEME = config("DEFAULT_SITE_THEME", default=DEFAULT_SITE_THEME) +ENABLE_COMPREHENSIVE_THEMING = config( + "ENABLE_COMPREHENSIVE_THEMING", default=ENABLE_COMPREHENSIVE_THEMING +) + +# Marketing link overrides +MKTG_URL_LINK_MAP.update(config("MKTG_URL_LINK_MAP", default={}, formatter=json.loads)) + +SUPPORT_SITE_LINK = config("SUPPORT_SITE_LINK", default=SUPPORT_SITE_LINK) + +# Mobile store URL overrides +MOBILE_STORE_URLS = config("MOBILE_STORE_URLS", default=MOBILE_STORE_URLS) + +# Timezone overrides +TIME_ZONE = config("TIME_ZONE", default=TIME_ZONE) + +# Translation overrides +LANGUAGES = config("LANGUAGES", default=LANGUAGES, formatter=json.loads) +LANGUAGE_DICT = dict(LANGUAGES) +LANGUAGE_CODE = config("LANGUAGE_CODE", default=LANGUAGE_CODE) +USE_I18N = config("USE_I18N", default=USE_I18N) + +# Additional installed apps +for app in config("ADDL_INSTALLED_APPS", default=[], formatter=json.loads): + INSTALLED_APPS += (app,) + +WIKI_ENABLED = config("WIKI_ENABLED", default=WIKI_ENABLED, formatter=bool) +local_loglevel = config("LOCAL_LOGLEVEL", default="INFO") + +# Configure Logging + +LOG_DIR = config("LOG_DIR", default="/edx/var/logs/edx") +DATA_DIR = config("DATA_DIR", default="/edx/var/edxapp") + +# Default format for syslog logging +standard_format = "%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s" +syslog_format = ( + "[variant:lms][%(name)s][env:sandbox] %(levelname)s " + "[{hostname} %(process)d] [%(filename)s:%(lineno)d] - %(message)s" +).format(hostname=platform.node().split(".")[0]) + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "local": { + "formatter": "syslog_format", + "class": "logging.StreamHandler", + "level": "INFO", + }, + "tracking": { + "formatter": "raw", + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "console": { + "formatter": "standard", + "class": "logging.StreamHandler", + "level": "INFO", + }, + }, + "formatters": { + "raw": {"format": "%(message)s"}, + "syslog_format": {"format": syslog_format}, + "standard": {"format": standard_format}, + }, + "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, + "loggers": { + "": {"level": "INFO", "propagate": False, "handlers": ["console", "local"]}, + "tracking": {"level": "DEBUG", "propagate": False, "handlers": ["tracking"]}, + }, +} + +SENTRY_DSN = config("SENTRY_DSN", default=None) +if SENTRY_DSN: + LOGGING["loggers"][""]["handlers"].append("sentry") + LOGGING["handlers"]["sentry"] = { + "class": "raven.handlers.logging.SentryHandler", + "dsn": SENTRY_DSN, + "level": "ERROR", + "environment": "production", + "release": RELEASE, + } + + +COURSE_LISTINGS = config("COURSE_LISTINGS", default={}, formatter=json.loads) +VIRTUAL_UNIVERSITIES = config("VIRTUAL_UNIVERSITIES", default=[]) +META_UNIVERSITIES = config("META_UNIVERSITIES", default={}, formatter=json.loads) +COMMENTS_SERVICE_URL = config("COMMENTS_SERVICE_URL", default="") +COMMENTS_SERVICE_KEY = config("COMMENTS_SERVICE_KEY", default="") +CERT_QUEUE = config("CERT_QUEUE", default="test-pull") +ZENDESK_URL = config("ZENDESK_URL", default=None) +FEEDBACK_SUBMISSION_EMAIL = config("FEEDBACK_SUBMISSION_EMAIL", default=None) +MKTG_URLS = config("MKTG_URLS", default=MKTG_URLS, formatter=json.loads) + +# Badgr API +BADGR_API_TOKEN = config("BADGR_API_TOKEN", default=BADGR_API_TOKEN) +BADGR_BASE_URL = config("BADGR_BASE_URL", default=BADGR_BASE_URL) +BADGR_ISSUER_SLUG = config("BADGR_ISSUER_SLUG", default=BADGR_ISSUER_SLUG) +BADGR_TIMEOUT = config("BADGR_TIMEOUT", default=BADGR_TIMEOUT) + +# git repo loading environment +GIT_REPO_DIR = config("GIT_REPO_DIR", default="/edx/var/edxapp/course_repos") +GIT_IMPORT_STATIC = config("GIT_IMPORT_STATIC", default=True) + +for name, value in config("CODE_JAIL", default={}, formatter=json.loads).items(): + oldvalue = CODE_JAIL.get(name) + if isinstance(oldvalue, dict): + for subname, subvalue in value.items(): + oldvalue[subname] = subvalue + else: + CODE_JAIL[name] = value + +COURSES_WITH_UNSAFE_CODE = config( + "COURSES_WITH_UNSAFE_CODE", default=[], formatter=json.loads +) + +LOCALE_PATHS = config("LOCALE_PATHS", default=LOCALE_PATHS, formatter=json.loads) + +ASSET_IGNORE_REGEX = config("ASSET_IGNORE_REGEX", default=ASSET_IGNORE_REGEX) + +# Event Tracking +TRACKING_IGNORE_URL_PATTERNS = config( + "TRACKING_IGNORE_URL_PATTERNS", + default=TRACKING_IGNORE_URL_PATTERNS, + formatter=json.loads, +) + +# SSL external authentication settings +SSL_AUTH_EMAIL_DOMAIN = config("SSL_AUTH_EMAIL_DOMAIN", default="MIT.EDU") +SSL_AUTH_DN_FORMAT_STRING = config("SSL_AUTH_DN_FORMAT_STRING", default=None) + +# Django CAS external authentication settings +CAS_EXTRA_LOGIN_PARAMS = config( + "CAS_EXTRA_LOGIN_PARAMS", default=None, formatter=json.loads +) +if FEATURES.get("AUTH_USE_CAS"): + CAS_SERVER_URL = config("CAS_SERVER_URL", default=None) + AUTHENTICATION_BACKENDS = [ + "django.contrib.auth.backends.ModelBackend", + "django_cas.backends.CASBackend", + ] + INSTALLED_APPS += ("django_cas",) + MIDDLEWARE_CLASSES.append("django_cas.middleware.CASMiddleware") + CAS_ATTRIBUTE_CALLBACK = config( + "CAS_ATTRIBUTE_CALLBACK", default=None, formatter=json.loads + ) + if CAS_ATTRIBUTE_CALLBACK: + import importlib + + CAS_USER_DETAILS_RESOLVER = getattr( + importlib.import_module(CAS_ATTRIBUTE_CALLBACK["module"]), + CAS_ATTRIBUTE_CALLBACK["function"], + ) + +# Video Caching. Pairing country codes with CDN URLs. +# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='} +VIDEO_CDN_URL = config("VIDEO_CDN_URL", default={}, formatter=json.loads) + +# Branded footer +FOOTER_OPENEDX_URL = config("FOOTER_OPENEDX_URL", default=FOOTER_OPENEDX_URL) +FOOTER_OPENEDX_LOGO_IMAGE = config( + "FOOTER_OPENEDX_LOGO_IMAGE", default=FOOTER_OPENEDX_LOGO_IMAGE +) +FOOTER_ORGANIZATION_IMAGE = config( + "FOOTER_ORGANIZATION_IMAGE", default=FOOTER_ORGANIZATION_IMAGE +) +FOOTER_CACHE_TIMEOUT = config( + "FOOTER_CACHE_TIMEOUT", default=FOOTER_CACHE_TIMEOUT, formatter=int +) +FOOTER_BROWSER_CACHE_MAX_AGE = config( + "FOOTER_BROWSER_CACHE_MAX_AGE", default=FOOTER_BROWSER_CACHE_MAX_AGE, formatter=int +) + +# Credit notifications settings +NOTIFICATION_EMAIL_CSS = config( + "NOTIFICATION_EMAIL_CSS", default=NOTIFICATION_EMAIL_CSS +) +NOTIFICATION_EMAIL_EDX_LOGO = config( + "NOTIFICATION_EMAIL_EDX_LOGO", default=NOTIFICATION_EMAIL_EDX_LOGO +) + +# Determines whether the CSRF token can be transported on +# unencrypted channels. It is set to False here for backward compatibility, +# but it is highly recommended that this is True for enviroments accessed +# by end users. +CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=False) + +############# CORS headers for cross-domain requests ################# + +if FEATURES.get("ENABLE_CORS_HEADERS") or FEATURES.get( + "ENABLE_CROSS_DOMAIN_CSRF_COOKIE" +): + CORS_ALLOW_CREDENTIALS = True + CORS_ORIGIN_WHITELIST = config( + "CORS_ORIGIN_WHITELIST", default=(), formatter=json.loads + ) + CORS_ORIGIN_ALLOW_ALL = config( + "CORS_ORIGIN_ALLOW_ALL", default=False, formatter=bool + ) + CORS_ALLOW_INSECURE = config("CORS_ALLOW_INSECURE", default=False, formatter=bool) + + # If setting a cross-domain cookie, it's really important to choose + # a name for the cookie that is DIFFERENT than the cookies used + # by each subdomain. For example, suppose the applications + # at these subdomains are configured to use the following cookie names: + # + # 1) foo.example.com --> "csrftoken" + # 2) baz.example.com --> "csrftoken" + # 3) bar.example.com --> "csrftoken" + # + # For the cross-domain version of the CSRF cookie, you need to choose + # a name DIFFERENT than "csrftoken"; otherwise, the new token configured + # for ".example.com" could conflict with the other cookies, + # non-deterministically causing 403 responses. + # + # Because of the way Django stores cookies, the cookie name MUST + # be a `str`, not unicode. Otherwise there will `TypeError`s will be raised + # when Django tries to call the unicode `translate()` method with the wrong + # number of parameters. + CROSS_DOMAIN_CSRF_COOKIE_NAME = str(config("CROSS_DOMAIN_CSRF_COOKIE_NAME")) + + # When setting the domain for the "cross-domain" version of the CSRF + # cookie, you should choose something like: ".example.com" + # (note the leading dot), where both the referer and the host + # are subdomains of "example.com". + # + # Browser security rules require that + # the cookie domain matches the domain of the server; otherwise + # the cookie won't get set. And once the cookie gets set, the client + # needs to be on a domain that matches the cookie domain, otherwise + # the client won't be able to read the cookie. + CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = config("CROSS_DOMAIN_CSRF_COOKIE_DOMAIN") + + +# Field overrides. To use the IDDE feature, add +# 'courseware.student_field_overrides.IndividualStudentOverrideProvider'. +FIELD_OVERRIDE_PROVIDERS = tuple( + config("FIELD_OVERRIDE_PROVIDERS", default=[], formatter=json.loads) +) + +############################## SECURE AUTH ITEMS ############### +# Secret things: passwords, access keys, etc. + +############### XBlock filesystem field config ########## +DJFS = config( + "DJFS", + default={ + "directory_root": "/edx/var/edxapp/django-pyfs/static/django-pyfs", + "type": "osfs", + "url_root": "/static/django-pyfs", + }, + formatter=json.loads, +) + +############### Module Store Items ########## +HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = config( + "HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS", default={}, formatter=json.loads +) +# PREVIEW DOMAIN must be present in HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS for +# the preview to show draft changes +if "PREVIEW_LMS_BASE" in FEATURES and FEATURES["PREVIEW_LMS_BASE"] != "": + PREVIEW_DOMAIN = FEATURES["PREVIEW_LMS_BASE"].split(":")[0] + # update dictionary with preview domain regex + HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS.update({PREVIEW_DOMAIN: "draft-preferred"}) + +MODULESTORE_FIELD_OVERRIDE_PROVIDERS = config( + "MODULESTORE_FIELD_OVERRIDE_PROVIDERS", + default=MODULESTORE_FIELD_OVERRIDE_PROVIDERS, + formatter=json.loads, +) + +XBLOCK_FIELD_DATA_WRAPPERS = config( + "XBLOCK_FIELD_DATA_WRAPPERS", + default=XBLOCK_FIELD_DATA_WRAPPERS, + formatter=json.loads, +) + +# PREVIEW DOMAIN must be present in HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS for the preview to show draft changes +if "PREVIEW_LMS_BASE" in FEATURES and FEATURES["PREVIEW_LMS_BASE"] != "": + PREVIEW_DOMAIN = FEATURES["PREVIEW_LMS_BASE"].split(":")[0] + # update dictionary with preview domain regex + HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS.update({PREVIEW_DOMAIN: "draft-preferred"}) + +############### Mixed Related(Secure/Not-Secure) Items ########## +LMS_SEGMENT_KEY = config("SEGMENT_KEY", default=None) + +CC_PROCESSOR_NAME = config("CC_PROCESSOR_NAME", default=CC_PROCESSOR_NAME) +CC_PROCESSOR = config("CC_PROCESSOR", default=CC_PROCESSOR) + +SECRET_KEY = config("SECRET_KEY", default="ThisisAnExampleKeyForDevPurposeOnly") + +DEFAULT_FILE_STORAGE = config( + "DEFAULT_FILE_STORAGE", default="django.core.files.storage.FileSystemStorage" +) + +# Specific setting for the File Upload Service to store media in a bucket. +FILE_UPLOAD_STORAGE_BUCKET_NAME = config( + "FILE_UPLOAD_STORAGE_BUCKET_NAME", default=FILE_UPLOAD_STORAGE_BUCKET_NAME +) +FILE_UPLOAD_STORAGE_PREFIX = config( + "FILE_UPLOAD_STORAGE_PREFIX", default=FILE_UPLOAD_STORAGE_PREFIX +) + +# If there is a database called 'read_replica', you can use the use_read_replica_if_available +# function in util/query.py, which is useful for very large database reads + +DATABASE_ENGINE = config("DATABASE_ENGINE", default="django.db.backends.mysql") +DATABASE_HOST = config("DATABASE_HOST", default="mysql") +DATABASE_PORT = config("DATABASE_PORT", default=3306, formatter=int) +DATABASE_NAME = config("DATABASE_NAME", default="edxapp") +DATABASE_USER = config("DATABASE_USER", default="edxapp_user") +DATABASE_PASSWORD = config("DATABASE_PASSWORD", default="password") + +DATABASES = config( + "DATABASES", + default={ + "default": { + "ENGINE": DATABASE_ENGINE, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + } + }, + formatter=json.loads, +) + +# The normal database user does not have enough permissions to run migrations. +# Migrations are run with separate credentials, given as DB_MIGRATION_* +# environment variables +for name, database in DATABASES.items(): + if name != "read_replica": + database.update( + { + "ENGINE": config("DB_MIGRATION_ENGINE", default=database["ENGINE"]), + "USER": config("DB_MIGRATION_USER", default=database["USER"]), + "PASSWORD": config("DB_MIGRATION_PASS", default=database["PASSWORD"]), + "NAME": config("DB_MIGRATION_NAME", default=database["NAME"]), + "HOST": config("DB_MIGRATION_HOST", default=database["HOST"]), + "PORT": config("DB_MIGRATION_PORT", default=database["PORT"]), + } + ) + +XQUEUE_INTERFACE = config( + "XQUEUE_INTERFACE", + default={"url": None, "basic_auth": None, "django_auth": None}, + formatter=json.loads, +) + +# Configure the MODULESTORE +MODULESTORE = convert_module_store_setting_if_needed( + config("MODULESTORE", default=MODULESTORE, formatter=json.loads) +) + +MONGODB_PASSWORD = config("MONGODB_PASSWORD", default="") +MONGODB_HOST = config("MONGODB_HOST", default="mongodb") +MONGODB_PORT = config("MONGODB_PORT", default=27017, formatter=int) +MONGODB_NAME = config("MONGODB_NAME", default="edxapp") +MONGODB_USER = config("MONGODB_USER", default=None) +MONGODB_SSL = config("MONGODB_SSL", default=False, formatter=bool) +MONGODB_REPLICASET = config("MONGODB_REPLICASET", default=None) +# Accepted read_preference value can be found here https://github.com/mongodb/mongo-python-driver/blob/2.9.1/pymongo/read_preferences.py#L54 +MONGODB_READ_PREFERENCE = config("MONGODB_READ_PREFERENCE", default="PRIMARY") + +DOC_STORE_CONFIG = config( + "DOC_STORE_CONFIG", + default={ + "collection": "modulestore", + "host": MONGODB_HOST, + "port": MONGODB_PORT, + "db": MONGODB_NAME, + "user": MONGODB_USER, + "password": MONGODB_PASSWORD, + "ssl": MONGODB_SSL, + "replicaSet": MONGODB_REPLICASET, + "read_preference": MONGODB_READ_PREFERENCE, + }, + formatter=json.loads, +) + +update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) + +MONGODB_LOG = config("MONGODB_LOG", default={}, formatter=json.loads) + +CONTENTSTORE = config( + "CONTENTSTORE", + default={ + "DOC_STORE_CONFIG": DOC_STORE_CONFIG, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + }, + formatter=json.loads, +) + +EMAIL_HOST_USER = config("EMAIL_HOST_USER", default="") # django default is '' +EMAIL_HOST_PASSWORD = config("EMAIL_HOST_PASSWORD", default="") # django default is '' + +# Datadog for events! +DATADOG = config("DATADOG", default={}, formatter=json.loads) + +# TODO: deprecated (compatibility with previous settings) +DATADOG_API = config("DATADOG_API", default=None) + +# Analytics dashboard server +ANALYTICS_SERVER_URL = config("ANALYTICS_SERVER_URL", default=None) +ANALYTICS_API_KEY = config("ANALYTICS_API_KEY", default="") + +# Analytics data source +ANALYTICS_DATA_URL = config("ANALYTICS_DATA_URL", default=ANALYTICS_DATA_URL) +ANALYTICS_DATA_TOKEN = config("ANALYTICS_DATA_TOKEN", default=ANALYTICS_DATA_TOKEN) + +# Analytics Dashboard +ANALYTICS_DASHBOARD_URL = config( + "ANALYTICS_DASHBOARD_URL", default=ANALYTICS_DASHBOARD_URL +) +ANALYTICS_DASHBOARD_NAME = config( + "ANALYTICS_DASHBOARD_NAME", default=PLATFORM_NAME + " Insights" +) + +# Mailchimp New User List +MAILCHIMP_NEW_USER_LIST_ID = config("MAILCHIMP_NEW_USER_LIST_ID", default=None) + +# Zendesk +ZENDESK_USER = config("ZENDESK_USER", default=None) +ZENDESK_API_KEY = config("ZENDESK_API_KEY", default=None) + +# API Key for inbound requests from Notifier service +EDX_API_KEY = config("EDX_API_KEY", default=None) + +# Celery Broker +CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") +CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") +CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") +CELERY_BROKER_HOST = config("CELERY_BROKER_HOST", default="redis") +CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) +CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) + +BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( + transport=CELERY_BROKER_TRANSPORT, + user=CELERY_BROKER_USER, + password=CELERY_BROKER_PASSWORD, + host=CELERY_BROKER_HOST, + port=CELERY_BROKER_PORT, + vhost=CELERY_BROKER_VHOST, +) + +# upload limits +STUDENT_FILEUPLOAD_MAX_SIZE = config( + "STUDENT_FILEUPLOAD_MAX_SIZE", default=STUDENT_FILEUPLOAD_MAX_SIZE, formatter=int +) + +# Event tracking +TRACKING_BACKENDS.update(config("TRACKING_BACKENDS", default={}, formatter=json.loads)) +EVENT_TRACKING_BACKENDS["tracking_logs"]["OPTIONS"]["backends"].update( + config("EVENT_TRACKING_BACKENDS", default={}, formatter=json.loads) +) +EVENT_TRACKING_BACKENDS["segmentio"]["OPTIONS"]["processors"][0]["OPTIONS"][ + "whitelist" +].extend( + config("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", default=[], formatter=json.loads) +) +TRACKING_SEGMENTIO_WEBHOOK_SECRET = config( + "TRACKING_SEGMENTIO_WEBHOOK_SECRET", default=TRACKING_SEGMENTIO_WEBHOOK_SECRET +) +TRACKING_SEGMENTIO_ALLOWED_TYPES = config( + "TRACKING_SEGMENTIO_ALLOWED_TYPES", + default=TRACKING_SEGMENTIO_ALLOWED_TYPES, + formatter=json.loads, +) +TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES = config( + "TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES", + default=TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES, + formatter=json.loads, +) +TRACKING_SEGMENTIO_SOURCE_MAP = config( + "TRACKING_SEGMENTIO_SOURCE_MAP", + default=TRACKING_SEGMENTIO_SOURCE_MAP, + formatter=json.loads, +) + +# Student identity verification settings +VERIFY_STUDENT = config("VERIFY_STUDENT", default=VERIFY_STUDENT, formatter=json.loads) + +# Grades download +GRADES_DOWNLOAD_ROUTING_KEY = config( + "GRADES_DOWNLOAD_ROUTING_KEY", default=HIGH_MEM_QUEUE +) + +GRADES_DOWNLOAD = config( + "GRADES_DOWNLOAD", default=GRADES_DOWNLOAD, formatter=json.loads +) + +GRADES_DOWNLOAD = config("GRADES_DOWNLOAD", default=GRADES_DOWNLOAD) + +# financial reports +FINANCIAL_REPORTS = config( + "FINANCIAL_REPORTS", default=FINANCIAL_REPORTS, formatter=json.loads +) + +##### ORA2 ###### +# Prefix for uploads of example-based assessment AI classifiers +# This can be used to separate uploads for different environments +# within the same S3 bucket. +ORA2_FILE_PREFIX = config("ORA2_FILE_PREFIX", default=ORA2_FILE_PREFIX) + +##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### +MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = config( + "MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", default=5, formatter=int +) +MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = config( + "MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", default=15 * 60, formatter=int +) + +#### PASSWORD POLICY SETTINGS ##### +PASSWORD_MIN_LENGTH = config("PASSWORD_MIN_LENGTH", default=12, formatter=int) +PASSWORD_MAX_LENGTH = config("PASSWORD_MAX_LENGTH", default=None, formatter=int) + +PASSWORD_COMPLEXITY = config( + "PASSWORD_COMPLEXITY", + default={"UPPER": 1, "LOWER": 1, "DIGITS": 1}, + formatter=json.loads, +) +PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = config( + "PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD", + default=PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD, + formatter=int, +) +PASSWORD_DICTIONARY = config("PASSWORD_DICTIONARY", default=[], formatter=json.loads) + +### INACTIVITY SETTINGS #### +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = config( + "SESSION_INACTIVITY_TIMEOUT_IN_SECONDS", default=None, formatter=int +) + +##### LMS DEADLINE DISPLAY TIME_ZONE ####### +TIME_ZONE_DISPLAYED_FOR_DEADLINES = config( + "TIME_ZONE_DISPLAYED_FOR_DEADLINES", default=TIME_ZONE_DISPLAYED_FOR_DEADLINES +) + +##### X-Frame-Options response header settings ##### +X_FRAME_OPTIONS = config("X_FRAME_OPTIONS", default=X_FRAME_OPTIONS) + +##### Third-party auth options ################################################ +if FEATURES.get("ENABLE_THIRD_PARTY_AUTH"): + AUTHENTICATION_BACKENDS = config( + "THIRD_PARTY_AUTH_BACKENDS", + [ + "social.backends.google.GoogleOAuth2", + "social.backends.linkedin.LinkedinOAuth2", + "social.backends.facebook.FacebookOAuth2", + "social.backends.azuread.AzureADOAuth2", + "third_party_auth.saml.SAMLAuthBackend", + "third_party_auth.lti.LTIAuthBackend", + ], + ) + list(AUTHENTICATION_BACKENDS) + + # The reduced session expiry time during the third party login pipeline. (Value in seconds) + SOCIAL_AUTH_PIPELINE_TIMEOUT = config("SOCIAL_AUTH_PIPELINE_TIMEOUT", default=600) + + # Most provider configuration is done via ConfigurationModels but for a few sensitive values + # we allow configuration via AUTH_TOKENS instead (optionally). + # The SAML private/public key values do not need the delimiter lines (such as + # "-----BEGIN PRIVATE KEY-----", default="-----END PRIVATE KEY-----" etc.) but they may be included + # if you want (though it's easier to format the key values as JSON without the delimiters). + SOCIAL_AUTH_SAML_SP_PRIVATE_KEY = AUTH_TOKENS.get( + "SOCIAL_AUTH_SAML_SP_PRIVATE_KEY", default="" + ) + SOCIAL_AUTH_SAML_SP_PUBLIC_CERT = AUTH_TOKENS.get( + "SOCIAL_AUTH_SAML_SP_PUBLIC_CERT", default="" + ) + SOCIAL_AUTH_OAUTH_SECRETS = config( + "SOCIAL_AUTH_OAUTH_SECRETS", default={}, formatter=json.loads + ) + SOCIAL_AUTH_LTI_CONSUMER_SECRETS = config( + "SOCIAL_AUTH_LTI_CONSUMER_SECRETS", default={}, formatter=json.loads + ) + + # third_party_auth config moved to ConfigurationModels. This is for data migration only: + THIRD_PARTY_AUTH_OLD_CONFIG = config("THIRD_PARTY_AUTH", default=None) + + if ( + config("THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS", default=24, formatter=int) + is not None + ): + CELERYBEAT_SCHEDULE["refresh-saml-metadata"] = { + "task": "third_party_auth.fetch_saml_metadata", + "schedule": datetime.timedelta( + hours=config( + "THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS", + default=24, + formatter=int, + ) + ), + } + + # The following can be used to integrate a custom login form with third_party_auth. + # It should be a dict where the key is a word passed via ?auth_entry=, and the value is a + # dict with an arbitrary 'secret_key' and a 'url'. + THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS = config( + "THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS", default={}, formatter=json.loads + ) + +##### OAUTH2 Provider ############## +if FEATURES.get("ENABLE_OAUTH2_PROVIDER"): + OAUTH_OIDC_ISSUER = config("OAUTH_OIDC_ISSUER", default=None) + OAUTH_ENFORCE_SECURE = config("OAUTH_ENFORCE_SECURE", default=True, formatter=bool) + OAUTH_ENFORCE_CLIENT_SECURE = config( + "OAUTH_ENFORCE_CLIENT_SECURE", default=True, formatter=bool + ) + +# Defaults for the following are defined in lms.envs.common +OAUTH_EXPIRE_DELTA = datetime.timedelta( + days=config( + "OAUTH_EXPIRE_CONFIDENTIAL_CLIENT_DAYS", + default=OAUTH_EXPIRE_CONFIDENTIAL_CLIENT_DAYS, + formatter=int, + ) +) +OAUTH_EXPIRE_DELTA_PUBLIC = datetime.timedelta( + days=config( + "OAUTH_EXPIRE_PUBLIC_CLIENT_DAYS", + default=OAUTH_EXPIRE_PUBLIC_CLIENT_DAYS, + formatter=int, + ) +) +OAUTH_ID_TOKEN_EXPIRATION = config( + "OAUTH_ID_TOKEN_EXPIRATION", default=OAUTH_ID_TOKEN_EXPIRATION, formatter=int +) + + +##### ADVANCED_SECURITY_CONFIG ##### +ADVANCED_SECURITY_CONFIG = config( + "ADVANCED_SECURITY_CONFIG", default={}, formatter=json.loads +) + +##### GOOGLE ANALYTICS IDS ##### +GOOGLE_ANALYTICS_ACCOUNT = config("GOOGLE_ANALYTICS_ACCOUNT", default=None) +GOOGLE_ANALYTICS_LINKEDIN = config("GOOGLE_ANALYTICS_LINKEDIN", default=None) + +##### OPTIMIZELY PROJECT ID ##### +OPTIMIZELY_PROJECT_ID = config("OPTIMIZELY_PROJECT_ID", default=OPTIMIZELY_PROJECT_ID) + +#### Course Registration Code length #### +REGISTRATION_CODE_LENGTH = config("REGISTRATION_CODE_LENGTH", default=8, formatter=int) + +# REGISTRATION CODES DISPLAY INFORMATION +INVOICE_CORP_ADDRESS = config("INVOICE_CORP_ADDRESS", default=INVOICE_CORP_ADDRESS) +INVOICE_PAYMENT_INSTRUCTIONS = config( + "INVOICE_PAYMENT_INSTRUCTIONS", default=INVOICE_PAYMENT_INSTRUCTIONS +) + +# Which access.py permission names to check; +# We default this to the legacy permission 'see_exists'. +COURSE_CATALOG_VISIBILITY_PERMISSION = config( + "COURSE_CATALOG_VISIBILITY_PERMISSION", default=COURSE_CATALOG_VISIBILITY_PERMISSION +) +COURSE_ABOUT_VISIBILITY_PERMISSION = config( + "COURSE_ABOUT_VISIBILITY_PERMISSION", default=COURSE_ABOUT_VISIBILITY_PERMISSION +) + +# Enrollment API Cache Timeout +ENROLLMENT_COURSE_DETAILS_CACHE_TIMEOUT = config( + "ENROLLMENT_COURSE_DETAILS_CACHE_TIMEOUT", default=60, formatter=int +) + +# PDF RECEIPT/INVOICE OVERRIDES +PDF_RECEIPT_TAX_ID = config("PDF_RECEIPT_TAX_ID", default=PDF_RECEIPT_TAX_ID) +PDF_RECEIPT_FOOTER_TEXT = config( + "PDF_RECEIPT_FOOTER_TEXT", default=PDF_RECEIPT_FOOTER_TEXT +) +PDF_RECEIPT_DISCLAIMER_TEXT = config( + "PDF_RECEIPT_DISCLAIMER_TEXT", default=PDF_RECEIPT_DISCLAIMER_TEXT +) +PDF_RECEIPT_BILLING_ADDRESS = config( + "PDF_RECEIPT_BILLING_ADDRESS", default=PDF_RECEIPT_BILLING_ADDRESS +) +PDF_RECEIPT_TERMS_AND_CONDITIONS = config( + "PDF_RECEIPT_TERMS_AND_CONDITIONS", default=PDF_RECEIPT_TERMS_AND_CONDITIONS +) +PDF_RECEIPT_TAX_ID_LABEL = config( + "PDF_RECEIPT_TAX_ID_LABEL", default=PDF_RECEIPT_TAX_ID_LABEL +) +PDF_RECEIPT_LOGO_PATH = config("PDF_RECEIPT_LOGO_PATH", default=PDF_RECEIPT_LOGO_PATH) +PDF_RECEIPT_COBRAND_LOGO_PATH = config( + "PDF_RECEIPT_COBRAND_LOGO_PATH", default=PDF_RECEIPT_COBRAND_LOGO_PATH +) +PDF_RECEIPT_LOGO_HEIGHT_MM = config( + "PDF_RECEIPT_LOGO_HEIGHT_MM", default=PDF_RECEIPT_LOGO_HEIGHT_MM, formatter=int +) +PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM = config( + "PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM", + default=PDF_RECEIPT_COBRAND_LOGO_HEIGHT_MM, + formatter=int, +) + +if ( + FEATURES.get("ENABLE_COURSEWARE_SEARCH") + or FEATURES.get("ENABLE_DASHBOARD_SEARCH") + or FEATURES.get("ENABLE_COURSE_DISCOVERY") + or FEATURES.get("ENABLE_TEAMS") +): + # Use ElasticSearch as the search engine herein + SEARCH_ENGINE = "search.elastic.ElasticSearchEngine" + +ELASTIC_SEARCH_CONFIG = config( + "ELASTIC_SEARCH_CONFIG", default=[{}], formatter=json.loads +) + +# Facebook app +FACEBOOK_API_VERSION = config("FACEBOOK_API_VERSION", default=None) +FACEBOOK_APP_SECRET = config("FACEBOOK_APP_SECRET", default=None) +FACEBOOK_APP_ID = config("FACEBOOK_APP_ID", default=None) + +XBLOCK_SETTINGS = config("XBLOCK_SETTINGS", default={}, formatter=json.loads) +XBLOCK_SETTINGS.setdefault("VideoDescriptor", {})["licensing_enabled"] = FEATURES.get( + "LICENSING", False +) +XBLOCK_SETTINGS.setdefault("VideoModule", {})["YOUTUBE_API_KEY"] = config( + "YOUTUBE_API_KEY", default=YOUTUBE_API_KEY +) + +##### CDN EXPERIMENT/MONITORING FLAGS ##### +CDN_VIDEO_URLS = config("CDN_VIDEO_URLS", default=CDN_VIDEO_URLS) +ONLOAD_BEACON_SAMPLE_RATE = config( + "ONLOAD_BEACON_SAMPLE_RATE", default=ONLOAD_BEACON_SAMPLE_RATE +) + +##### ECOMMERCE API CONFIGURATION SETTINGS ##### +ECOMMERCE_PUBLIC_URL_ROOT = config( + "ECOMMERCE_PUBLIC_URL_ROOT", default=ECOMMERCE_PUBLIC_URL_ROOT +) +ECOMMERCE_API_URL = config("ECOMMERCE_API_URL", default=ECOMMERCE_API_URL) +ECOMMERCE_API_TIMEOUT = config( + "ECOMMERCE_API_TIMEOUT", default=ECOMMERCE_API_TIMEOUT, formatter=int +) + +ECOMMERCE_SERVICE_WORKER_USERNAME = config( + "ECOMMERCE_SERVICE_WORKER_USERNAME", default=ECOMMERCE_SERVICE_WORKER_USERNAME +) +ECOMMERCE_API_TIMEOUT = config("ECOMMERCE_API_TIMEOUT", default=ECOMMERCE_API_TIMEOUT) + +COURSE_CATALOG_API_URL = config( + "COURSE_CATALOG_API_URL", default=COURSE_CATALOG_API_URL +) + +##### Custom Courses for EdX ##### +if FEATURES.get("CUSTOM_COURSES_EDX"): + INSTALLED_APPS += ("lms.djangoapps.ccx", "openedx.core.djangoapps.ccxcon") + MODULESTORE_FIELD_OVERRIDE_PROVIDERS += ( + "lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider", + ) +CCX_MAX_STUDENTS_ALLOWED = config( + "CCX_MAX_STUDENTS_ALLOWED", default=CCX_MAX_STUDENTS_ALLOWED +) + +##### Individual Due Date Extensions ##### +if FEATURES.get("INDIVIDUAL_DUE_DATES"): + FIELD_OVERRIDE_PROVIDERS += ( + "courseware.student_field_overrides.IndividualStudentOverrideProvider", + ) + +##### Self-Paced Course Due Dates ##### +XBLOCK_FIELD_DATA_WRAPPERS += ( + "lms.djangoapps.courseware.field_overrides:OverrideModulestoreFieldData.wrap", +) + +MODULESTORE_FIELD_OVERRIDE_PROVIDERS += ( + "courseware.self_paced_overrides.SelfPacedDateOverrideProvider", +) + +# PROFILE IMAGE CONFIG +PROFILE_IMAGE_BACKEND = config("PROFILE_IMAGE_BACKEND", default=PROFILE_IMAGE_BACKEND) +PROFILE_IMAGE_SECRET_KEY = AUTH_TOKENS.get( + "PROFILE_IMAGE_SECRET_KEY", default=PROFILE_IMAGE_SECRET_KEY +) +PROFILE_IMAGE_MAX_BYTES = config( + "PROFILE_IMAGE_MAX_BYTES", default=PROFILE_IMAGE_MAX_BYTES, formatter=int +) +PROFILE_IMAGE_MIN_BYTES = config( + "PROFILE_IMAGE_MIN_BYTES", default=PROFILE_IMAGE_MIN_BYTES, formatter=int +) +PROFILE_IMAGE_DEFAULT_FILENAME = "images/profiles/default" + +# EdxNotes config + +EDXNOTES_PUBLIC_API = config("EDXNOTES_PUBLIC_API", default=EDXNOTES_PUBLIC_API) +EDXNOTES_INTERNAL_API = config("EDXNOTES_INTERNAL_API", default=EDXNOTES_INTERNAL_API) + +EDXNOTES_CONNECT_TIMEOUT = config( + "EDXNOTES_CONNECT_TIMEOUT", default=EDXNOTES_CONNECT_TIMEOUT +) +EDXNOTES_READ_TIMEOUT = config("EDXNOTES_READ_TIMEOUT", default=EDXNOTES_READ_TIMEOUT) + +##### Credit Provider Integration ##### + +CREDIT_PROVIDER_SECRET_KEYS = config( + "CREDIT_PROVIDER_SECRET_KEYS", default={}, formatter=json.loads +) + +##################### LTI Provider ##################### +if FEATURES.get("ENABLE_LTI_PROVIDER"): + INSTALLED_APPS += ("lti_provider",) + AUTHENTICATION_BACKENDS += ("lti_provider.users.LtiBackend",) + +LTI_USER_EMAIL_DOMAIN = config("LTI_USER_EMAIL_DOMAIN", default="lti.example.com") + +# For more info on this, see the notes in common.py +LTI_AGGREGATE_SCORE_PASSBACK_DELAY = config( + "LTI_AGGREGATE_SCORE_PASSBACK_DELAY", default=LTI_AGGREGATE_SCORE_PASSBACK_DELAY +) + +##################### Credit Provider help link #################### +CREDIT_HELP_LINK_URL = config("CREDIT_HELP_LINK_URL", default=CREDIT_HELP_LINK_URL) + +#### JWT configuration #### +JWT_AUTH.update(config("JWT_AUTH", default={})) +PUBLIC_RSA_KEY = config("PUBLIC_RSA_KEY", default=PUBLIC_RSA_KEY) +PRIVATE_RSA_KEY = config("PRIVATE_RSA_KEY", default=PRIVATE_RSA_KEY) + +################# PROCTORING CONFIGURATION ################## + +PROCTORING_BACKEND_PROVIDER = config( + "PROCTORING_BACKEND_PROVIDER", default=PROCTORING_BACKEND_PROVIDER +) +PROCTORING_SETTINGS = config( + "PROCTORING_SETTINGS", default=PROCTORING_SETTINGS, formatter=json.loads +) + +################# MICROSITE #################### +MICROSITE_CONFIGURATION = config( + "MICROSITE_CONFIGURATION", default={}, formatter=json.loads +) +MICROSITE_ROOT_DIR = path(config("MICROSITE_ROOT_DIR", default="")) +# this setting specify which backend to be used when pulling microsite specific configuration +MICROSITE_BACKEND = config("MICROSITE_BACKEND", default=MICROSITE_BACKEND) +# this setting specify which backend to be used when loading microsite specific templates +MICROSITE_TEMPLATE_BACKEND = config( + "MICROSITE_TEMPLATE_BACKEND", default=MICROSITE_TEMPLATE_BACKEND +) +# TTL for microsite database template cache +MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = config( + "MICROSITE_DATABASE_TEMPLATE_CACHE_TTL", + default=MICROSITE_DATABASE_TEMPLATE_CACHE_TTL, + formatter=int, +) + +# Course Content Bookmarks Settings +MAX_BOOKMARKS_PER_COURSE = config( + "MAX_BOOKMARKS_PER_COURSE", default=MAX_BOOKMARKS_PER_COURSE, formatter=int +) + +# Offset for pk of courseware.StudentModuleHistoryExtended +STUDENTMODULEHISTORYEXTENDED_OFFSET = config( + "STUDENTMODULEHISTORYEXTENDED_OFFSET", + default=STUDENTMODULEHISTORYEXTENDED_OFFSET, + formatter=int, +) + +# Cutoff date for granting audit certificates +if config("AUDIT_CERT_CUTOFF_DATE", default=None): + AUDIT_CERT_CUTOFF_DATE = dateutil.parser.parse(config("AUDIT_CERT_CUTOFF_DATE")) + +################################ Settings for Credentials Service ################################ + +CREDENTIALS_GENERATION_ROUTING_KEY = HIGH_PRIORITY_QUEUE + +# The extended StudentModule history table +if FEATURES.get("ENABLE_CSMH_EXTENDED"): + INSTALLED_APPS += ("coursewarehistoryextended",) + +API_ACCESS_MANAGER_EMAIL = config("API_ACCESS_MANAGER_EMAIL", default=None) +API_ACCESS_FROM_EMAIL = config("API_ACCESS_FROM_EMAIL", default=None) + +# Mobile App Version Upgrade config +APP_UPGRADE_CACHE_TIMEOUT = config( + "APP_UPGRADE_CACHE_TIMEOUT", default=APP_UPGRADE_CACHE_TIMEOUT, formatter=int +) + +AFFILIATE_COOKIE_NAME = config("AFFILIATE_COOKIE_NAME", default=AFFILIATE_COOKIE_NAME) diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_staging.py b/releases/eucalyptus/3/wb/config/lms/docker_run_staging.py new file mode 100644 index 00000000..7c878eb2 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_staging.py @@ -0,0 +1,14 @@ +# This file includes overrides to build the `staging` environment for the LMS starting from the +# settings of the `production` environment + +from docker_run_production import * +from .utils import Configuration + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +LOGGING["handlers"]["sentry"]["environment"] = "staging" + +EMAIL_BACKEND = config( + "EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend" +) diff --git a/releases/eucalyptus/3/wb/config/lms/fun.py b/releases/eucalyptus/3/wb/config/lms/fun.py new file mode 100644 index 00000000..ffcaa0bc --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/fun.py @@ -0,0 +1,77 @@ +import imp +import json +import os.path + +from ..common import * +from .utils import Configuration + + +# Load custom configuration parameters from yaml files +config = Configuration(os.path.dirname(__file__)) + +# Fun-apps configuration +INSTALLED_APPS += ( + "rest_framework.authtoken", + "backoffice", + "fun", + "fun_api", + "fun_instructor", + "course_dashboard", + "courses", + "courses_api", + "course_pages", + "universities", + "videoproviders", + "bootstrapform", + "raven.contrib.django.raven_compat", + "pure_pagination", + "selftest", + "teachers", + "edx_gea", +) + +ROOT_URLCONF = "fun.lms.urls_wb" + +# ### THIRD-PARTY SETTINGS ### + +# Haystack configuration (default is minimal working configuration) +HAYSTACK_CONNECTIONS = config( + "HAYSTACK_CONNECTIONS", + default={ + "default": {"ENGINE": "courses.search_indexes.ConfigurableElasticSearchEngine"} + }, + formatter=json.loads, +) + +# ### FUN-APPS SETTINGS ### +# -- Base -- +FUN_BASE_ROOT = path(os.path.dirname(imp.find_module("funsite")[1])) +SHARED_ROOT = DATA_DIR / "shared" + +# Add FUN applications templates directories to MAKO template finder before edX's ones +MAKO_TEMPLATES["main"] = [ + # overrides template in edx-platform/lms/templates + FUN_BASE_ROOT + / "course_dashboard/templates" +] + MAKO_TEMPLATES["main"] + +FUN_SMALL_LOGO_RELATIVE_PATH = "funsite/images/logos/fun61.png" +FUN_BIG_LOGO_RELATIVE_PATH = "funsite/images/logos/fun195.png" + +# -- Certificates +CERTIFICATE_BASE_URL = "/attestations/" +CERTIFICATES_DIRECTORY = "/edx/var/edxapp/attestations/" +FUN_LOGO_PATH = FUN_BASE_ROOT / "funsite/static" / FUN_BIG_LOGO_RELATIVE_PATH +STUDENT_NAME_FOR_TEST_CERTIFICATE = "Test User" + +# Used by pure-pagination app, +# https://github.com/jamespacileo/django-pure-pagination for information about +# the constants : +# https://camo.githubusercontent.com/51defa6771f5db2826a1869eca7bed82d9fb3120/687474703a2f2f692e696d6775722e636f6d2f4c437172742e676966 +PAGINATION_SETTINGS = { + # same formatting as in github issues, seems to be sane. + "PAGE_RANGE_DISPLAYED": 4, + "MARGIN_PAGES_DISPLAYED": 2, +} + +NUMBER_DAYS_TOO_LATE = 31 diff --git a/releases/eucalyptus/3/wb/config/lms/utils.py b/releases/eucalyptus/3/wb/config/lms/utils.py new file mode 100644 index 00000000..9359baa9 --- /dev/null +++ b/releases/eucalyptus/3/wb/config/lms/utils.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +import yaml +import os + +from django.core.exceptions import ImproperlyConfigured + + +class Configuration(dict): + """ + Try getting a setting from the settings.yml or secrets.yml files placed in + the directory passed when initializing the configuration instance. + """ + + def __init__(self, dir=None, *args, **kwargs): + """ + Initialize with the path to the directory in which the configuration is + to be found. + """ + super(Configuration, self).__init__(*args, **kwargs) + + if dir is None: + self.settings = {} + + else: + # Load the content of a `settings.yml` file placed in the current + # directory if any. This file is where customizable settings are stored + # for a given environment. + try: + with open(os.path.join(dir, "settings.yml")) as f: + settings = yaml.load(f.read()) or {} + except IOError: + settings = {} + + # Load the content of a `secrets.yml` file placed in the current + # directory if any. This file is where sensitive credentials are stored + # for a given environment. + try: + with open(os.path.join(dir, "secrets.yml")) as f: + credentials = yaml.load(f.read()) or {} + except IOError: + credentials = {} + + settings.update(credentials) + self.settings = settings + + def __call__(self, var_name, formatter=str, *args, **kwargs): + """ + The config returns in order of priority: + + - the value set in the secrets.yml file, + - the value set in the settings.yml file, + - the value set as environment variable + - the value passed as default. + + If the value is passed as a string, a type is forced via the function passed in + the "formatter" kwarg. + + Raise an "ImproperlyConfigured" error if the name is not found, except + if the `default` key is given in kwargs (using kwargs allows to pass a + default to None, which is different from not passing any default): + + $ config = Configuration('path/to/config/directory') + $ config('foo') # raise ImproperlyConfigured error if `foo` is not defined + $ config('foo', default='bar') # return 'bar' if `foo` is not defined + $ config('foo', default=None) # return `None` if `foo` is not defined + """ + try: + value = self.settings[var_name] + except KeyError: + try: + value = formatter(os.environ[var_name]) + except KeyError: + if "default" in kwargs: + value = kwargs["default"] + else: + raise ImproperlyConfigured( + 'Please set the "{:s}" variable in a settings.yml file, a secrets.yml ' + "file or an environment variable.".format(var_name) + ) + # If a formatter is specified, force the value but only if it was passed as a string + if isinstance(value, basestring): + value = formatter(value) + + return value + + def get(self, name, *args, **kwargs): + """ + edX is loading the content of 2 json files to settings.ENV_TOKEN and settings.AUTH_TOKEN + They have started calling these attributes anywhere in the code base, so we must make + sure that the following call works (and the same for AUTH_TOKEN): + + settings.ENV_TOKEN.get('ANY_SETTING_NAME') + + That's what this method will do after we add this to our settings: + ``` + config = Configuration('path/to/my/settings/directory.yml') + ENV_TOKEN = config + AUTH_TOKEN = config + ``` + """ + try: + default = args[0] + except IndexError: + # As a first approach, all defaults that are not provided by Open edX are set to None. + # If this creates a problem, we can either: + # - make sure we provide a value for this setting in our yaml files, + # - make a PR to Open edX to provide a better default for this setting. + default = None + return self(name, default=default) diff --git a/releases/eucalyptus/3/wb/entrypoint.sh b/releases/eucalyptus/3/wb/entrypoint.sh new file mode 100755 index 00000000..07331910 --- /dev/null +++ b/releases/eucalyptus/3/wb/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# +# Development entrypoint +# + +# Activate user's virtualenv +source /edx/app/edxapp/venv/bin/activate +exec "$@" diff --git a/releases/eucalyptus/3/wb/requirements.txt b/releases/eucalyptus/3/wb/requirements.txt new file mode 100644 index 00000000..802ff33a --- /dev/null +++ b/releases/eucalyptus/3/wb/requirements.txt @@ -0,0 +1,15 @@ +# FUN dependencies +--extra-index-url https://pypi.fury.io/openfun/ + +# ==== core ==== +configurable-lti-consumer-xblock==1.3.0 +fun-apps==2.0.1+wb +edx-gea==0.2.0 +libcast-xblock==0.5.0 +password-container-xblock==0.3.0 +xblock-proctor-exam==0.9.0b0 +xblock-utils2==0.3.0 + +# ==== third-party apps ==== +raven==6.9.0 +django-redis-sessions==0.6.1 From bc598f8af230f171285ede61cb56f53f88ae3d94 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 2 Dec 2019 16:59:55 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8(eucalyptus/3/wb)=20add=20missing?= =?UTF-8?q?=20support=20for=20redis=20sentinel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To connect celery to redis-sentinel we use the plugin celery-redis-sentinel. Also to use session in redis, the setting allowing to change the backend to use was missing and setting to configure redis were also missing. We added them, it is now possible to use redis to store sessions. --- releases/eucalyptus/3/wb/CHANGELOG.md | 1 + .../3/wb/config/cms/docker_run_production.py | 35 +++++++++++++++++++ .../3/wb/config/lms/docker_run_production.py | 35 +++++++++++++++++++ releases/eucalyptus/3/wb/requirements.txt | 1 + 4 files changed, 72 insertions(+) diff --git a/releases/eucalyptus/3/wb/CHANGELOG.md b/releases/eucalyptus/3/wb/CHANGELOG.md index 08f406b6..2220814e 100644 --- a/releases/eucalyptus/3/wb/CHANGELOG.md +++ b/releases/eucalyptus/3/wb/CHANGELOG.md @@ -15,6 +15,7 @@ release. - First experimental release of OpenEdx `eucalyptus.3` (wb flavor). - Set replicaSet and read_preference in mongodb connection +- Add missing support for redis sentinel [unreleased]: https://github.com/openfun/openedx-docker/compare/eucalyptus.3-1.0.0-wb...HEAD [eucalyptus.3-1.0.0-wb]: https://github.com/openfun/openedx-docker/releases/tag/eucalyptus.3-1.0.0-wb diff --git a/releases/eucalyptus/3/wb/config/cms/docker_run_production.py b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py index 58faf166..ea3e5d27 100644 --- a/releases/eucalyptus/3/wb/config/cms/docker_run_production.py +++ b/releases/eucalyptus/3/wb/config/cms/docker_run_production.py @@ -14,6 +14,7 @@ import os import platform +from celery_redis_sentinel import register from lms.envs.fun.utils import Configuration from openedx.core.lib.logsettings import get_logger_config from path import Path as path @@ -189,6 +190,32 @@ "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool ) +# Configuration to use session with redis +# To use redis, change SESSION_ENGINE with value redis_sessions.session +SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis") +SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int) +SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int) +SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None) +SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session") +SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int) +SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool) + +SESSION_REDIS = config( + "SESSION_REDIS", + default={ + "host": SESSION_REDIS_HOST, + "port": SESSION_REDIS_PORT, + "db": SESSION_REDIS_DB, # db 0 is used for Celery Broker + "password": SESSION_REDIS_PASSWORD, + "prefix": SESSION_REDIS_PREFIX, + "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT, + "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT, + }, + formatter=json.loads, +) +SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads) +SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None) + # social sharing settings SOCIAL_SHARING_SETTINGS = config( "SOCIAL_SHARING_SETTINGS", default=SOCIAL_SHARING_SETTINGS, formatter=json.loads @@ -519,6 +546,7 @@ DATADOG["api_key"] = config("DATADOG_API", default=None) # Celery Broker +# For redis sentinel you the transport redis-sentinel CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") @@ -526,6 +554,10 @@ CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) +if CELERY_BROKER_TRANSPORT == "redis-sentinel": + # register redis sentinel schema in celery + register() + BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( transport=CELERY_BROKER_TRANSPORT, user=CELERY_BROKER_USER, @@ -534,6 +566,9 @@ port=CELERY_BROKER_PORT, vhost=CELERY_BROKER_VHOST, ) +# To use redis-sentinel, refer to the documenation here +# https://celery-redis-sentinel.readthedocs.io/en/latest/ +BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads) # Event tracking TRACKING_BACKENDS.update(config("TRACKING_BACKENDS", default={}, formatter=json.loads)) diff --git a/releases/eucalyptus/3/wb/config/lms/docker_run_production.py b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py index 0b6829ed..43f68d2d 100644 --- a/releases/eucalyptus/3/wb/config/lms/docker_run_production.py +++ b/releases/eucalyptus/3/wb/config/lms/docker_run_production.py @@ -23,6 +23,7 @@ import platform import warnings +from celery_redis_sentinel import register from openedx.core.lib.logsettings import get_logger_config from path import Path as path from xmodule.modulestore.modulestore_settings import ( @@ -195,6 +196,32 @@ "SESSION_SAVE_EVERY_REQUEST", default=SESSION_SAVE_EVERY_REQUEST, formatter=bool ) +# Configuration to use session with redis +# To use redis, change SESSION_ENGINE with value redis_sessions.session +SESSION_REDIS_HOST = config("SESSION_REDIS_HOST", default="redis") +SESSION_REDIS_PORT = config("SESSION_REDIS_HOST", default=6379, formatter=int) +SESSION_REDIS_DB = config("SESSION_REDIS_DB", default=1, formatter=int) +SESSION_REDIS_PASSWORD = config("SESSION_REDIS_PASSWORD", default=None) +SESSION_REDIS_PREFIX = config("SESSION_REDIS_PREFIX", default="session") +SESSION_REDIS_SOCKET_TIMEOUT = config("SESSION_REDIS_SOCKET_TIMEOUT", default=1, formatter=int) +SESSION_REDIS_RETRY_ON_TIMEOUT = config("SESSION_REDIS_RETRY_ON_TIMEOUT", default=False, formatter=bool) + +SESSION_REDIS = config( + "SESSION_REDIS", + default={ + "host": SESSION_REDIS_HOST, + "port": SESSION_REDIS_PORT, + "db": SESSION_REDIS_DB, # db 0 is used for Celery Broker + "password": SESSION_REDIS_PASSWORD, + "prefix": SESSION_REDIS_PREFIX, + "socket_timeout": SESSION_REDIS_SOCKET_TIMEOUT, + "retry_on_timeout": SESSION_REDIS_RETRY_ON_TIMEOUT, + }, + formatter=json.loads, +) +SESSION_REDIS_SENTINEL_LIST = config("SESSION_REDIS_SENTINEL_LIST", default=None, formatter=json.loads) +SESSION_REDIS_SENTINEL_MASTER_ALIAS = config("SESSION_REDIS_SENTINEL_MASTER_ALIAS", default=None) + AWS_SES_REGION_NAME = config("AWS_SES_REGION_NAME", default="us-east-1") AWS_SES_REGION_ENDPOINT = config( "AWS_SES_REGION_ENDPOINT", default="email.us-east-1.amazonaws.com" @@ -782,6 +809,7 @@ EDX_API_KEY = config("EDX_API_KEY", default=None) # Celery Broker +# For redis sentinel you the transport redis-sentinel CELERY_BROKER_TRANSPORT = config("CELERY_BROKER_TRANSPORT", default="redis") CELERY_BROKER_USER = config("CELERY_BROKER_USER", default="") CELERY_BROKER_PASSWORD = config("CELERY_BROKER_PASSWORD", default="") @@ -789,6 +817,10 @@ CELERY_BROKER_PORT = config("CELERY_BROKER_PORT", default=6379, formatter=int) CELERY_BROKER_VHOST = config("CELERY_BROKER_VHOST", default=0, formatter=int) +if CELERY_BROKER_TRANSPORT == "redis-sentinel": + # register redis sentinel schema in celery + register() + BROKER_URL = "{transport}://{user}:{password}@{host}:{port}/{vhost}".format( transport=CELERY_BROKER_TRANSPORT, user=CELERY_BROKER_USER, @@ -797,6 +829,9 @@ port=CELERY_BROKER_PORT, vhost=CELERY_BROKER_VHOST, ) +# To use redis-sentinel, refer to the documenation here +# https://celery-redis-sentinel.readthedocs.io/en/latest/ +BROKER_TRANSPORT_OPTIONS = config("BROKER_TRANSPORT_OPTIONS", default={}, formatter=json.loads) # upload limits STUDENT_FILEUPLOAD_MAX_SIZE = config( diff --git a/releases/eucalyptus/3/wb/requirements.txt b/releases/eucalyptus/3/wb/requirements.txt index 802ff33a..a390c80f 100644 --- a/releases/eucalyptus/3/wb/requirements.txt +++ b/releases/eucalyptus/3/wb/requirements.txt @@ -13,3 +13,4 @@ xblock-utils2==0.3.0 # ==== third-party apps ==== raven==6.9.0 django-redis-sessions==0.6.1 +celery-redis-sentinel==0.3.0