From e96490af13ea766dcf31d4bbb4cb98892410f601 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Tue, 21 Nov 2023 09:40:36 -0500 Subject: [PATCH 1/5] Fix requirements; Add dev documentation --- .docker-compose.vital-strategies-theme.yaml | 2 +- Makefile | 47 +++++--- README.md | 118 ++++++++++++++++++++ ckan/Dockerfile | 3 + docker-compose/ckan-secrets.dat | 2 +- jenkins/Dockerfile | 10 +- 6 files changed, 160 insertions(+), 22 deletions(-) diff --git a/.docker-compose.vital-strategies-theme.yaml b/.docker-compose.vital-strategies-theme.yaml index 332dae6..9fd04c9 100644 --- a/.docker-compose.vital-strategies-theme.yaml +++ b/.docker-compose.vital-strategies-theme.yaml @@ -26,7 +26,7 @@ services: EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/" PRE_INSTALL: "sed -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt" POST_INSTALL: | - install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.0.1 &&\ + install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\ install_standard_ckan_extension_github -r ckan/ckanext-geoview && \ install_standard_ckan_extension_github -r okfn/ckanext-sentry && \ install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \ diff --git a/Makefile b/Makefile index c3f97aa..22adcb5 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,54 @@ -.PHONY: start stop build pull shell down remove logs user sysadmin secret cron +.PHONY: start stop build pull shell down remove remove-images logs logs-less exec user sysadmin secret cron clean-rebuild + +COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml start: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml up -d --build nginx && make cron + docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron stop: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml stop + docker-compose $(COMPOSE_FILES) stop build: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml build + docker-compose $(COMPOSE_FILES) build pull: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml pull + docker-compose $(COMPOSE_FILES) pull shell: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml exec $S $C + docker-compose $(COMPOSE_FILES) exec $S $C down: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml down + docker-compose $(COMPOSE_FILES) down remove: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml down -v + docker-compose $(COMPOSE_FILES) down -v + +remove-images: + docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f logs: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml logs -f $S + docker-compose $(COMPOSE_FILES) logs -f $S + +logs-less: + docker-compose $(COMPOSE_FILES) logs $S | less + +exec: + docker-compose $(COMPOSE_FILES) exec $S $C + user: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml \ - exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini + docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini + sysadmin: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml \ - exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini + docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini + secret: python create_secrets.py + cron: - docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml exec --user=root ckan service cron start + docker-compose $(COMPOSE_FILES) exec --user=root ckan service cron start + +clean-rebuild: + docker-compose $(COMPOSE_FILES) down -v + docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f + docker-compose $(COMPOSE_FILES) build --no-cache + docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron diff --git a/README.md b/README.md index 4b5256a..bc2eeb0 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,90 @@ Install Docker for [Windows](https://store.docker.com/editions/community/docker- ## Generate or update files with secrets + Run and follow all steps: ``` ./create_secrets.py ``` +If you're setting up a local environment for development and testing, you can leave all of the secret values as is. Just press enter when prompted for a value. + +## Running locally for development and testing + +If you want to run this environment locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps and started the environment, you can access CKAN at http://127.0.0.1:5000. + +**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. + +### Use traefik.dev.toml + +By default, `traefik` will attempt to generate a certificate and use https. This will cause issues with the local development environment. To fix this, you can use the `traefik.dev.toml` by updating the `proxy` service in `docker-compose.yaml` to use the `traefik.dev.toml` file. The dev version doesn't generate a certificate and uses http instead of https. + +``` + proxy: + image: traefik:1.7.2-alpine + restart: always + volumes: + - ./traefik/traefik.dev.toml:/traefik.toml # <-- Replace ./traefik/traefik.toml with ./traefik/traefik.dev.toml as shown here + - ./traefik/acme.json:/acme.json + networks: + - ckan-multi +``` + +### Expose port 5000 for CKAN + +In your project specific `docker-compose` file, you must expose port 5000 for CKAN. Otherwise, CKAN will not be accessible from the host machine. For example, if you want to run `.docker-compose.vital-strategies-theme.yaml` locally, you would add the ports section as shown below: + +``` + ckan: + depends_on: + - datapusher + links: + - datapusher + image: viderum/ckan-cloud-docker:ckan-latest-vital-strategies-theme + build: + context: ckan + args: + CKAN_BRANCH: ckan-2.7.3 + EXTRA_PACKAGES: cron + EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/" + PRE_INSTALL: "sed -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt" + POST_INSTALL: | + install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\ + install_standard_ckan_extension_github -r ckan/ckanext-geoview && \ + install_standard_ckan_extension_github -r okfn/ckanext-sentry && \ + install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \ + install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b fix-null-content-type && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l es -f && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l fr -f && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l km -f && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l pt_BR -f && \ + cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_CN -f + environment: + - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme- + ports: # <-- Add this section to expose port 5000 + - 5000:5000 +``` + +### Remove unused plugins from CKAN + +Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-production.ini`) to avoid issues (unless you want to properly configure them): + +``` +ckan.plugins = image_view + text_view + recline_view + datastore + datapusher + resource_proxy + geojson_view + querytool + stats + sentry # <-- Remove this line + s3filestore # <-- Remove this line + googleanalytics # <-- Remove this line +``` + ## Running a CKAN instance using the docker-compose environment (optional) Clear any existing compose environment to ensure a fresh start @@ -77,6 +156,45 @@ To start the jobs server for uploading to the datastore DB: docker-compose up -d jobs ``` +### Optionally, use make commands + +The following commands use the `vital-strategies` project template as an example. Replace `vital-strategies` with the name of your project template. **Note**: Using the commands below still requires adding `nginx` to your hosts file as shown above. + +Build the images: + +``` +make build O=vital-strategies +``` + +Start the environment (this will also build the images if they haven't been built yet): + +``` +make start O=vital-strategies +``` + +Stop the environment: + +``` +make stop O=vital-strategies +``` + +Remove the containers and volumes: + +``` +make remote O=vital-strategies +``` + +Remove the associated images: + +``` +make remove-images O=vital-strategies +``` + +Completely remove and then rebuild the environment (this will remove containers, volumes, and images): + +``` +make clean-rebuild O=vital-strategies +``` ## Making modifications to the docker images / configuration diff --git a/ckan/Dockerfile b/ckan/Dockerfile index e67de4f..0dc2bd8 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -69,6 +69,9 @@ RUN CKAN_BRANCH="${CKAN_BRANCH:-ckan-2.8.1}" && CKAN_REPO="${CKAN_REPO:-ckan/cka ARG PRE_INSTALL RUN eval "${PRE_INSTALL}" +RUN ckan-pip install psycopg2-binary==2.8.4 +RUN sed -i '/psycopg2/d' $CKAN_VENV/src/ckan/requirements.txt + RUN touch $CKAN_VENV/src/ckan/requirement-setuptools.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt RUN touch $CKAN_VENV/src/ckan/requirements.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt diff --git a/docker-compose/ckan-secrets.dat b/docker-compose/ckan-secrets.dat index 9a376c5..9fce1bf 100644 --- a/docker-compose/ckan-secrets.dat +++ b/docker-compose/ckan-secrets.dat @@ -14,7 +14,7 @@ ckan required SMTP_SERVER mail.example.com Enter SMTP server address ckan required SMTP_USER info Enter SMTP server username ckan optional SMTP_PASSWORD empty Enter SMTP server password ckan optional SMTP_MAIL_FROM empty Enter SMTP mail from -ckan optional CKAN_SITE_URL http://ckan.localhost:5000 Enter Website URL (including https://) +ckan optional CKAN_SITE_URL http://127.0.0.1:5000 Enter Website URL (including https:// or http://) ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key (if any) ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key (if any) ckan optional SENTRY_DSN https://@sentry.io/ Enter Sentry DSN URL with token and ID diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile index d73036a..01203f0 100644 --- a/jenkins/Dockerfile +++ b/jenkins/Dockerfile @@ -3,9 +3,9 @@ FROM jenkins/jenkins:lts ARG PIP_INDEX_URL ENV PIP_INDEX_URL=$PIP_INDEX_URL -RUN /usr/local/bin/install-plugins.sh \ - build-timeout envfile copyartifact extensible-choice-parameter fail-the-build file-operations \ - filesystem-list-parameter fstrigger generic-webhook-trigger git-parameter github-branch-source \ +RUN jenkins-plugin-cli --plugins \ + build-timeout envfile copyartifact extensible-choice-parameter file-operations \ + fstrigger generic-webhook-trigger git-parameter github-branch-source \ global-variable-string-parameter http_request jobgenerator join managed-scripts matrix-combinations-parameter \ persistent-parameter workflow-aggregator pipeline-github-lib python ssh-slaves timestamper urltrigger \ ws-cleanup @@ -20,8 +20,6 @@ RUN chmod +x /usr/local/bin/docker-compose && echo "jenkins ALL=NOPASSWD: ALL" > echo "export CKAN_CLOUD_DOCKER_JENKINS=1" > /etc/profile.d/ckan_cloud_docker_jenkins &&\ chmod +x /etc/profile.d/ckan_cloud_docker_jenkins RUN apt update && apt install -y python3-pip -RUN python3 -m pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} pyyaml +RUN python3 -m pip install --break-system-packages --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} pyyaml USER jenkins - -RUN /usr/local/bin/install-plugins.sh rebuild From 900fcf8af39ee735139a54f80dbf1fa4c257a944 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Tue, 21 Nov 2023 11:26:35 -0500 Subject: [PATCH 2/5] Update README; Add traefik dev file --- README.md | 4 ++-- traefik/traefik.dev.toml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 traefik/traefik.dev.toml diff --git a/README.md b/README.md index bc2eeb0..a64e58f 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,9 @@ If you're setting up a local environment for development and testing, you can le ## Running locally for development and testing -If you want to run this environment locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps and started the environment, you can access CKAN at http://127.0.0.1:5000. +If you want to run this locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps below and started the environment, you can access CKAN at http://127.0.0.1:5000. -**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. +**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. Once you handle the overrides below, you can proceed with the "Running a CKAN instance using the docker-compose environment" section. ### Use traefik.dev.toml diff --git a/traefik/traefik.dev.toml b/traefik/traefik.dev.toml new file mode 100644 index 0000000..5ff6d94 --- /dev/null +++ b/traefik/traefik.dev.toml @@ -0,0 +1,32 @@ +debug = false +defaultEntryPoints = ["http"] + +[entryPoints] + [entryPoints.http] + address = ":80" + + [entryPoints.api] + address = ":8081" + + [api] + entryPoint = "api" + + [ping] + entryPoint = "http" + + [accessLog] + + [file] + watch = true + + [backends] + [backends.ckan] + [backends.ckan.servers.server1] + url = "http://nginx:8080" + + [frontends] + [frontends.ckan] + backend="ckan" + passHostHeader = true + [frontends.ckan.routes.route1] + rule = "Host:localhost" From 6fea7b6c87cf7a3dd37c846d4f8f9ee15ebb8e8d Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Tue, 21 Nov 2023 11:27:57 -0500 Subject: [PATCH 3/5] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a64e58f..4d4d2b6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ If you're setting up a local environment for development and testing, you can le If you want to run this locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps below and started the environment, you can access CKAN at http://127.0.0.1:5000. -**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. Once you handle the overrides below, you can proceed with the "Running a CKAN instance using the docker-compose environment" section. +**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. After making the changes below, you can continue with the "Running a CKAN instance using the docker-compose environment" section. ### Use traefik.dev.toml From 0034f977c8d239e7df80a66cf953325dbdd7f289 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Tue, 21 Nov 2023 11:39:33 -0500 Subject: [PATCH 4/5] Update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d4d2b6..f568a1b 100644 --- a/README.md +++ b/README.md @@ -178,10 +178,22 @@ Stop the environment: make stop O=vital-strategies ``` +Make a user: + +``` +make user O=vital-strategies U=USER_NAME P=PASSWORD E=EMAIL +``` + +Make a user a sysadmin: + +``` +make sysadmin O=vital-strategies U=USER_NAME +``` + Remove the containers and volumes: ``` -make remote O=vital-strategies +make remove O=vital-strategies ``` Remove the associated images: From 1dac65a4d83a62a0ac5e6743cd3d95579c55d0a2 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Wed, 22 Nov 2023 09:07:19 -0500 Subject: [PATCH 5/5] Fix DB init; Update README; Set default dev URL --- README.md | 11 ++++++++++- ckan/Dockerfile | 3 +-- docker-compose/ckan-secrets.dat | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f568a1b..3552d35 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you're setting up a local environment for development and testing, you can le ## Running locally for development and testing -If you want to run this locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps below and started the environment, you can access CKAN at http://127.0.0.1:5000. +If you want to run this locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps below and started the environment, you can access CKAN at http://ckan:5000 (when creating secrets, don't change the default site URL). **Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. After making the changes below, you can continue with the "Running a CKAN instance using the docker-compose environment" section. @@ -109,6 +109,15 @@ ckan.plugins = image_view googleanalytics # <-- Remove this line ``` +### Hosts file entries + +When using this environment locally, you must add the following entries to your hosts file (`nginx` is mentioned in the next section, but `ckan` is specific to the development and testing setup): + +``` +127.0.0.1 nginx +127.0.0.1 ckan +``` + ## Running a CKAN instance using the docker-compose environment (optional) Clear any existing compose environment to ensure a fresh start diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 0dc2bd8..2acd5ef 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -69,8 +69,7 @@ RUN CKAN_BRANCH="${CKAN_BRANCH:-ckan-2.8.1}" && CKAN_REPO="${CKAN_REPO:-ckan/cka ARG PRE_INSTALL RUN eval "${PRE_INSTALL}" -RUN ckan-pip install psycopg2-binary==2.8.4 -RUN sed -i '/psycopg2/d' $CKAN_VENV/src/ckan/requirements.txt +RUN sed -i 's/psycopg2==2.4.5/psycopg2==2.7.3.2/g' $CKAN_VENV/src/ckan/requirements.txt RUN touch $CKAN_VENV/src/ckan/requirement-setuptools.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt RUN touch $CKAN_VENV/src/ckan/requirements.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt diff --git a/docker-compose/ckan-secrets.dat b/docker-compose/ckan-secrets.dat index 9fce1bf..1df40d3 100644 --- a/docker-compose/ckan-secrets.dat +++ b/docker-compose/ckan-secrets.dat @@ -14,7 +14,7 @@ ckan required SMTP_SERVER mail.example.com Enter SMTP server address ckan required SMTP_USER info Enter SMTP server username ckan optional SMTP_PASSWORD empty Enter SMTP server password ckan optional SMTP_MAIL_FROM empty Enter SMTP mail from -ckan optional CKAN_SITE_URL http://127.0.0.1:5000 Enter Website URL (including https:// or http://) +ckan required CKAN_SITE_URL http://ckan:5000 Enter Website URL (including https:// or http://) ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key (if any) ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key (if any) ckan optional SENTRY_DSN https://@sentry.io/ Enter Sentry DSN URL with token and ID