Merge branch 'containerize' into kmp_C4-627_workaround_AcceptValidHea…

…der_best_match_with_containerize

Dockerfile

@@ -0,0 +1,151 @@
+# Fourfront (Production) Dockerfile
+# Based off of the cgap-portal Dockerfile
+# Note that images are pinned via sha256 as opposed to tag
+# so that we don't pick up new images unintentionally
+
+# Debian Buster with Python 3.6.15
+# Note image is updated from cgap-portal
+FROM python:3.6.15-slim-buster
+
+MAINTAINER William Ronchetti "[email protected]"
+
+# Build Arguments
+ARG INI_BASE
+ENV INI_BASE=${INI_BASE:-"fourfront_any_alpha.ini"}
+
+# Configure (global) Env
+ENV NGINX_USER=nginx
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+ENV PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100 \
+  POETRY_VERSION=1.1.12 \
+  NODE_VERSION=12.22.9
+
+# Install nginx, base system
+COPY deploy/docker/production/install_nginx.sh /
+RUN bash /install_nginx.sh && \
+    apt-get update && \
+    apt-get install -y curl vim emacs postgresql-client net-tools ca-certificates \
+    gcc zlib1g-dev libpq-dev git make
+
+# Configure Fourfront User (nginx)
+WORKDIR /home/nginx/.nvm
+
+# Install Node
+ENV NVM_DIR=/home/nginx/.nvm
+RUN apt install -y curl
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+ENV PATH="/home/nginx/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
+RUN node --version
+RUN npm --version
+
+WORKDIR /home/nginx
+
+# Configure venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python -m venv /opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Upgrade pip, install in layer
+RUN pip install --upgrade pip && \
+    pip install poetry==$POETRY_VERSION
+
+# Adjust permissions
+RUN chown -R nginx:nginx /opt/venv && \
+    mkdir -p /home/nginx/fourfront
+
+WORKDIR /home/nginx/fourfront
+
+# Do the back-end dependency install
+COPY pyproject.toml .
+COPY poetry.lock .
+RUN poetry install --no-root
+
+# Do the front-end dependency install
+COPY package.json .
+COPY package-lock.json .
+RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python
+
+# Build front-end
+# XXX: this fails but might be able to be made to succeed? - Will Oct 7 2021
+# COPY *.js .
+# COPY src/encoded/static .
+# RUN npm run build && \
+  #    npm run build-scss
+
+# Copy over the rest of the code
+COPY . .
+
+# Build front-end, remove node_modules when done
+RUN npm run build && \
+    npm run build-scss && \
+    rm -rf node_modules/
+
+# Build remaining back-end
+RUN poetry install && \
+    python setup_eb.py develop && \
+    make fix-dist-info
+
+# Misc
+RUN make aws-ip-ranges && \
+    cat /dev/urandom | head -c 256 | base64 > session-secret.b64
+
+# Copy config files in (down here for quick debugging)
+# Remove default configuration from Nginx
+RUN rm /etc/nginx/nginx.conf && \
+    rm /etc/nginx/conf.d/default.conf
+COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf
+
+# nginx filesystem setup
+RUN chown -R nginx:nginx /var/cache/nginx && \
+    chown -R nginx:nginx /var/log/nginx && \
+    chown -R nginx:nginx /etc/nginx/conf.d && \
+    touch /var/run/nginx.pid && \
+    chown -R nginx:nginx /var/run/nginx.pid && \
+    rm -f /var/log/nginx/* && \
+    touch /var/log/nginx/access.log && \
+    chown -R nginx:nginx /var/log/nginx/access.log && \
+    touch /var/log/nginx/error.log && \
+    chown -R nginx:nginx /var/log/nginx/error.log && \
+    mkdir -p /data/nginx/cache && \
+    chown -R nginx:nginx /data/nginx/cache
+
+# Pull all required files
+# Note that *.ini must match the env name in secrets manager!
+# Note that deploy/docker/production/entrypoint.sh resolves which entrypoint to run
+# based on env variable "application_type".
+COPY deploy/docker/local/docker_development.ini development.ini
+COPY deploy/docker/local/entrypoint.bash entrypoint_local.bash
+RUN chown nginx:nginx development.ini
+RUN chmod +x entrypoint_local.bash
+
+# Production setup
+RUN touch production.ini
+RUN chown nginx:nginx production.ini
+COPY deploy/docker/production/$INI_BASE deploy/ini_files/.
+COPY deploy/docker/production/entrypoint.bash .
+COPY deploy/docker/production/entrypoint_portal.bash .
+COPY deploy/docker/production/entrypoint_deployment.bash .
+COPY deploy/docker/production/entrypoint_indexer.bash .
+# Note that fourfront does not have an ingester
+# COPY deploy/docker/production/entrypoint_ingester.sh .
+COPY deploy/docker/production/assume_identity.py .
+RUN chmod +x entrypoint.bash
+RUN chmod +x entrypoint_deployment.bash
+RUN chmod +x entrypoint_deployment.bash
+RUN chmod +x entrypoint_indexer.bash
+RUN chmod +x assume_identity.py
+EXPOSE 8000
+
+# Container does not run as root
+USER nginx
+
+ENTRYPOINT ["/home/nginx/fourfront/entrypoint.bash"]

Makefile

@@ -145,6 +145,40 @@ remote-test-unit:  # Note this does the 'indexing' tests
 update:  # updates dependencies
 	poetry update
 
+build-docker-local:
+	docker-compose build
+
+build-docker-local-clean:
+	docker-compose build --no-cache
+
+deploy-docker-local:
+	docker-compose up -V
+
+deploy-docker-local-daemon:
+	docker-compose up -d -V
+
+ENV_NAME ?= fourfront-mastertest
+AWS_ACCOUNT ?= 643366669028
+
+ecr-login:
+	@echo "Making ecr-login AWS_ACCOUNT=${AWS_ACCOUNT} ..."
+	aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com
+
+build-docker-mastertest:
+	scripts/build-docker-test --login --ecosystem fourfront-mastertest
+
+build-docker-production:
+	@echo "Making build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
+	docker build -t ${ENV_NAME}:latest .
+	make tag-and-push-docker-production ENV_NAME=${ENV_NAME} AWS_ACCOUNT=${AWS_ACCOUNT}
+
+tag-and-push-docker-production:
+	@echo "Making tag-and-push-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
+	docker tag ${ENV_NAME}:latest ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
+	date
+	docker push ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
+	date
+
 help:
 	@make info
 

buildspec.yml

@@ -0,0 +1,19 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - touch deploy/docker/local/docker_development.ini  # cheap substitute for prepare-docker to make ignored file
+      - echo Logging in to Amazon ECR...
+      - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+  build:
+    commands:
+      - echo Build started on `date`
+      - echo Building the Docker image...
+      - docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .
+      - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
+  post_build:
+    commands:
+      - echo Build completed on `date`
+      - echo Pushing the Docker image...
+      - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG

deploy/docker/elasticsearch/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.14
+
+MAINTAINER William Ronchetti "[email protected]"
+
+ENV ELASTICSEARCH_VERSION="6.8.14"
+ENV ELASTICSEARCH_SERVICE_PORT=9200
+EXPOSE $ELASTICSEARCH_SERVICE_PORT

deploy/docker/local/docker_development.ini

@@ -0,0 +1,96 @@
+###
+# Docker App Configuration for local deployment
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
+# NOTE: Still needs to be customized for (automated use by) multiple users
+###
+
+[app:app]
+use = config:base.ini#app
+session.secret = %(here)s/session-secret.b64
+env.name = ffdkr
+sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres
+elasticsearch.server = search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
+elasticsearch.aws_auth = true
+#elasticsearch.server = es:9200
+blob_bucket = encoded-4dn-blobs
+load_test_only = true
+create_tables = true
+testing = true
+postgresql.statement_timeout = 20
+mpindexer = true
+indexer = true
+pyramid.reload_templates = true
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.default_locale_name = en
+# this line determines which load function is used in load_data
+# most deployments use: "load_test_data = encoded.loadxl:load_test_data"
+load_test_data = encoded.loadxl:load_test_data
+encoded_version = 100.200.300
+snovault_version = 200.300.400
+utils_version = 300.400.500
+eb_app_version = app-v-development-simulation
+
+[pipeline:debug]
+pipeline =
+    egg:PasteDeploy#prefix
+    egg:repoze.debug#pdbpm
+    app
+set pyramid.includes =
+    pyramid_translogger
+
+[composite:main]
+use = egg:rutter#urlmap
+/ = debug
+/_indexer = indexer
+
+[composite:indexer]
+use = config:base.ini#indexer
+
+###
+# wsgi server configuration
+###
+
+[server:main]
+use = egg:waitress#main
+host = 0.0.0.0
+port = 6543
+threads = 1
+
+###
+# logging configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, wsgi, encoded
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_wsgi]
+level = DEBUG
+handlers =
+qualname = wsgi
+
+[logger_encoded]
+level = DEBUG
+handlers =
+qualname = encoded
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

deploy/docker/local/entrypoint.bash

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [  -z ${TEST+x} ]; then
+
+    if [ ! -z ${LOAD+x} ]; then
+
+        # Clear db/es since this is the local entry point
+        poetry run clear-db-es-contents development.ini --app-name app --env "$FOURFRONT_ENV_NAME"
+
+        # Create mapping
+        poetry run create-mapping-on-deploy development.ini --app-name app
+
+        # Load Data (based on development.ini, for now just master-inserts)
+        poetry run load-data development.ini --app-name app --prod
+
+    fi
+
+    # Start nginx proxy
+    service nginx start
+
+    # Start application
+    make deploy2
+
+else
+
+    echo "Not starting serving application"
+    echo "Enter the container with docker exec"
+    sleep 100000000
+
+fi

deploy/docker/postgres/Dockerfile

@@ -0,0 +1,15 @@
+# TODO: upgrade to latest version we can tolerate
+FROM postgres:12.3
+
+MAINTAINER William Ronchetti "[email protected]"
+
+# Install some system level dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates htop vim emacs curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy over our custom conf, enable inbound connections
+COPY postgresql.conf /etc/postgresql/postgresql.conf
+RUN echo "host    all             all              0.0.0.0/0                       trust" >> /etc/postgresql/pg_hba.conf
+
+ENV PGDATA=/var/lib/postgresql/data/pgdata