From d8e699b2f7f4e9e68217edface4b01bd643cb154 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Mon, 18 Nov 2024 11:17:21 -0800 Subject: [PATCH] Migrate from make to just (#1118) * Migrate from make to just * address review from socorro repo justfile --- .github/workflows/build-and-push.yml | 23 ++-- Makefile | 129 ----------------------- bin/run_lint.sh | 11 +- docker-compose.yml | 5 +- docker/config/{my.env.dist => .env.dist} | 4 +- docs/dev.rst | 46 ++++---- justfile | 80 ++++++++++++++ systemtest/README.rst | 9 +- tests/README.rst | 11 +- 9 files changed, 139 insertions(+), 179 deletions(-) delete mode 100644 Makefile rename docker/config/{my.env.dist => .env.dist} (93%) create mode 100644 justfile diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index bca1def9..5c5ad4bc 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -18,7 +18,7 @@ jobs: contents: read deployments: write id-token: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: # Disable docker compose volume mounts in docker-compose.override.yml COMPOSE_FILE: docker-compose.yml @@ -39,24 +39,21 @@ jobs: "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > version.json - name: Output version.json run: cat version.json + - name: Install just + run: sudo apt-get update && sudo apt-get install -y just - name: Build Docker images - run: make build + run: just build - name: Verify requirements.txt contains correct dependencies - run: | - docker compose run --rm --no-deps test shell ./bin/run_verify_reqs.sh + run: just verify-reqs - name: Run lint check - run: | - make my.env - docker compose run --rm --no-deps test shell ./bin/run_lint.sh + run: just lint - name: Run tests - run: | - make my.env - docker compose run --rm test shell ./bin/run_tests.sh + run: just test - name: Run systemtest run: | - make setup - docker compose up --detach --wait --wait-timeout=10 web nginx - docker compose run --rm web shell systemtest/test_env.sh local + just setup + just run --detach --wait --wait-timeout=10 web nginx + just shell systemtest/test_env.sh local - name: Set Docker image tag to "latest" for updates of the main branch if: github.ref == 'refs/heads/main' diff --git a/Makefile b/Makefile deleted file mode 100644 index 115fd080..00000000 --- a/Makefile +++ /dev/null @@ -1,129 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Include my.env and export it so variables set in there are available -# in Makefile. -include my.env -export - -# Set these in the environment to override them. This is helpful for -# development if you have file ownership problems because the user -# in the container doesn't match the user on your host. -ANTENNA_UID ?= 10001 -ANTENNA_GID ?= 10001 - -# Set this in the environment to force --no-cache docker builds. -DOCKER_BUILD_OPTS := -ifeq (1, ${NOCACHE}) -DOCKER_BUILD_OPTS := --no-cache -endif - -DOCKER := $(shell which docker) -DC=${DOCKER} compose - -.DEFAULT_GOAL := help -.PHONY: help -help: - @echo "Usage: make RULE" - @echo "" - @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \ - | grep -v grep \ - | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \ - | column -t -s '|' - @echo "" - @echo "Adjust your my.env file to set configuration." - @echo "" - @echo "See https://antenna.readthedocs.io/ for more documentation." - -# Dev configuration steps -.docker-build: - make build - -.devcontainer-build: - make devcontainerbuild - -my.env: - @if [ ! -f my.env ]; \ - then \ - echo "Copying my.env.dist to my.env..."; \ - cp docker/config/my.env.dist my.env; \ - fi - -.PHONY: build -build: my.env ## | Build docker images. - ${DC} --progress plain build ${DOCKER_BUILD_OPTS} --build-arg userid=${ANTENNA_UID} --build-arg groupid=${ANTENNA_GID} deploy-base - ${DC} --progress plain build fakesentry gcs-emulator statsd nginx - touch .docker-build - -.PHONY: setup -setup: my.env .docker-build ## | Set up services. - ${DC} run --rm web shell ./bin/run_setup.sh - -.PHONY: run -run: my.env .docker-build ## | Run the webapp and services. - ${DC} up \ - --attach web \ - --attach fakesentry \ - --attach nginx \ - web fakesentry nginx - -.PHONY: devcontainerbuild -devcontainerbuild: .env ## | Build VS Code development container. - ${DC} build devcontainer - touch .devcontainer-build - -.PHONY: devcontainer -devcontainer: .env .devcontainer-build ## | Run VS Code development container. - ${DC} up --detach devcontainer - -.PHONY: shell -shell: my.env .docker-build ## | Open a shell in the web image. - ${DC} run --rm web shell - -.PHONY: my.env clean -clean: ## | Remove build, test, and Python artifacts. - # python related things - -rm -rf build/ - -rm -rf dist/ - -rm -rf .eggs/ - find . -name '*.egg-info' -exec rm -rf {} + - find . -name '*.egg' -exec rm -f {} + - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '__pycache__' -exec rm -rf {} + - - # docs files - -rm -rf docs/_build/ - - # state files - -rm .docker-build - -.PHONY: lint -lint: my.env .docker-build ## | Lint code. - ${DC} run --rm --no-deps base shell ./bin/run_lint.sh - -.PHONY: lintfix -lintfix: my.env .docker-build ## | Reformat code. - ${DC} run --rm --no-deps base shell ./bin/run_lint.sh --fix - -.PHONY: test -test: my.env .docker-build ## | Run unit tests. - # Run tests - ${DC} run --rm test shell ./bin/run_tests.sh - -.PHONY: testshell -testshell: my.env .docker-build ## | Open a shell in the test container. - ${DC} run --rm test shell - -.PHONY: docs -docs: my.env .docker-build ## | Generate Sphinx HTML documentation. - ${DC} run -u ${ANTENNA_UID} base shell ./bin/build_docs.sh - -.PHONY: rebuildreqs -rebuildreqs: my.env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. - ${DC} run --rm --no-deps base shell pip-compile --generate-hashes --strip-extras - -.PHONY: updatereqs -updatereqs: my.env .docker-build ## | Update deps in requirements.txt file. - ${DC} run --rm --no-deps base shell pip-compile --generate-hashes --strip-extras --upgrade diff --git a/bin/run_lint.sh b/bin/run_lint.sh index 08070a6a..e1fc0e75 100755 --- a/bin/run_lint.sh +++ b/bin/run_lint.sh @@ -16,11 +16,18 @@ FILES="antenna bin docs testlib tests systemtest" PYTHON_VERSION=$(python --version) -if [[ "${1:-}" == "--fix" ]]; then +if [[ "${1:-}" == "--help" ]]; then + echo "Usage: $0 [OPTIONS]" + echo + echo " Lint code" + echo + echo "Options:" + echo " --help Show this message and exit." + echo " --fix Reformat code." +elif [[ "${1:-}" == "--fix" ]]; then echo ">>> ruff fix (${PYTHON_VERSION})" ruff format $FILES ruff check --fix $FILES - else echo ">>> ruff (${PYTHON_VERSION})" ruff check $FILES diff --git a/docker-compose.yml b/docker-compose.yml index 65bfeed2..e8cff800 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,9 @@ services: build: context: . dockerfile: docker/Dockerfile + args: + userid: ${USE_UID:-10001} + groupid: ${USE_GID:-10001} image: local/antenna_deploy_base # ------------------------------------------------------------------ @@ -56,7 +59,7 @@ services: service: base env_file: - docker/config/local_dev.env - - my.env + - .env command: web links: - fakesentry diff --git a/docker/config/my.env.dist b/docker/config/.env.dist similarity index 93% rename from docker/config/my.env.dist rename to docker/config/.env.dist index 7615c212..b8eed396 100644 --- a/docker/config/my.env.dist +++ b/docker/config/.env.dist @@ -8,5 +8,5 @@ # If you want to set the uid and gid of the app user that we use in the # containers, you can do that with these two variables. -# ANTENNA_UID= -# ANTENNA_GID= +# USE_UID= +# USE_GID= diff --git a/docs/dev.rst b/docs/dev.rst index 9018fd99..e9ac8ae9 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -18,7 +18,7 @@ is also used for local development of Antenna. For more comprehensive documentation or instructions on how to set this up in production, see documentation_. -1. Install required software: Docker, make, and git. +1. Install required software: Docker, just, and git. 2. Clone the repository to your local machine. @@ -32,30 +32,30 @@ production, see documentation_. .. code-block:: shell - $ make my.env + $ just _env - Then edit the file and set the ``ANTENNA_UID`` and ``ANTENNA_GID`` + Then edit the ``.env`` file and set the ``USE_UID`` and ``USE_GID`` variables. These will get used when creating the app user in the base image. - If you ever want different values, change them in ``my.env`` and re-run - ``make build``. + If you ever want different values, change them in ``.env`` and re-run + ``just build``. 4. Download and build Antenna docker containers: .. code-block:: shell - $ make build + $ just build - Anytime you want to update the containers, you can run ``make build``. + Anytime you want to update the containers, you can run ``just build``. 5. Set up local Pub/Sub and GCS services: .. code-block:: shell - $ make setup + $ just setup Anytime you want to wipe service state and recreate them, you can re-run - this make rule. + this just rule. 6. Run with a prod-like fully-functional configuration. @@ -63,7 +63,7 @@ production, see documentation_. .. code-block:: shell - $ make run + $ just run You should see a lot of output. It'll start out with something like this:: @@ -188,7 +188,7 @@ production, see documentation_. .. code-block:: shell - $ make test + $ just test If you need to run specific tests or pass in different arguments, you can run bash in the base container and then run ``pytest`` with whatever args you @@ -196,7 +196,7 @@ production, see documentation_. .. code-block:: shell - $ make shell + $ just shell app@...$ pytest @@ -308,7 +308,7 @@ To lint the code: .. code-block:: shell - $ make lint + $ just lint If you hit issues, use ``# noqa``. @@ -316,7 +316,7 @@ To reformat the code: .. code-block:: shell - $ make lintfix + $ just lint --fix We're using: @@ -353,7 +353,7 @@ For example, to add ``foobar`` version 5: .. code-block:: shell - make rebuildreqs + just rebuild-reqs to apply the updates to ``requirements.txt`` @@ -361,7 +361,7 @@ For example, to add ``foobar`` version 5: .. code-block:: shell - $ make build + $ just build If there are problems, it'll tell you. @@ -370,7 +370,7 @@ dependencies. To do this, run: .. code-block:: shell - $ make updatereqs + $ just rebuild-reqs --update Documentation @@ -384,7 +384,7 @@ To build the docs, run this: .. code-block:: shell - $ make docs + $ just docs Testing @@ -394,7 +394,7 @@ To run the tests, run this: .. code-block:: shell - $ make test + $ just test Tests go in ``tests/``. Data required by tests goes in ``tests/data/``. @@ -405,7 +405,7 @@ For example: .. code-block:: shell - $ make shell + $ just shell app@...$ pytest @@ -429,7 +429,7 @@ development instance. There are a few options: .. code-block:: bash - $ make shell + $ just shell app@c392a11dbfec:/app$ python -m testlib.mini_poster --url URL 2. Use Firefox and set the ``MOZ_CRASHREPORTER_URL`` environment variable: @@ -514,7 +514,7 @@ first run: .. code-block:: shell - $ make devcontainerbuild + $ just build devcontainer Additionally on mac there is the potential that running git from inside any container that mounts the current directory to `/app`, such as the development @@ -534,7 +534,7 @@ pick up changes: .. code-block:: shell - $ make devcontainer + $ just run -d devcontainer Upgrading to a new Python version diff --git a/justfile b/justfile new file mode 100644 index 00000000..8c769c90 --- /dev/null +++ b/justfile @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +_default: + @just --list + +_env: + #!/usr/bin/env sh + if [ ! -f .env ]; then + echo "Copying docker/config/.env.dist to .env..." + cp docker/config/.env.dist .env + fi + +# Build docker images. +build *args='deploy-base fakesentry gcs-emulator statsd nginx': _env + docker compose --progress plain build {{args}} + +# Set up services. +setup: _env + docker compose run --rm web shell ./bin/run_setup.sh + +# Run the webapp and services. +run *args='--attach=web --attach=nginx --attach=fakesentry web nginx': _env + docker compose up {{args}} + +# Stop service containers. +stop *args: + docker compose stop {{args}} + +# Remove service containers and networks. +down *args: + docker compose down {{args}} + +# Open a shell in the web image. +shell *args='/bin/bash': _env + docker compose run --rm --entrypoint= web {{args}} + +# Open a shell in the test container. +test-shell *args='/bin/bash': + docker compose run --rm --entrypoint= test {{args}} + +# Remove build, test, and Python artifacts. +clean: + # python related things + -rm -rf build/ + -rm -rf dist/ + -rm -rf .eggs/ + find . -name '*.egg-info' -exec rm -rf {} + + find . -name '*.egg' -exec rm -f {} + + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '__pycache__' -exec rm -rf {} + + + # docs files + -rm -rf docs/_build/ + +# Lint code, or use --fix to reformat and apply auto-fixes for lint. +lint *args: + docker compose run --rm --no-deps base shell ./bin/run_lint.sh {{args}} + +# Run tests. +test *args: + docker compose run --rm test shell ./bin/run_tests.sh {{args}} + +# Generate Sphinx HTML documentation. +docs: + docker compose run --rm --no-deps base shell ./bin/build_docs.sh + +# Rebuild requirements.txt file after requirements.in changes. +rebuild-reqs *args: + docker compose run --rm --no-deps base shell pip-compile --generate-hashes --strip-extras {{args}} + +# Verify that the requirements file is built by the version of Python that runs in the container. +verify-reqs: + docker compose run --rm --no-deps base shell ./bin/run_verify_reqs.sh + +# Check how far behind different server environments are from main tip. +service-status *args: + docker compose run --rm --no-deps base shell service-status {{args}} diff --git a/systemtest/README.rst b/systemtest/README.rst index 4a0e7c38..0c8742df 100644 --- a/systemtest/README.rst +++ b/systemtest/README.rst @@ -17,20 +17,19 @@ Running tests In a terminal, run:: - $ make shell - app@xxx:/app$ ./systemtest/test_env.sh ENV + $ just setup + $ just shell ./systemtest/test_env.sh ENV Additional arguments will be passed through to pytest. For example:: - $ make shell - app@xxx:/app$ ./systemtest/test_env.sh ENV -- test_dockerflow.py + $ just shell ./systemtest/test_env.sh ENV -- test_dockerflow.py If you're running against ``local``, you'll need to be running antenna in another terminal:: - $ make run + $ just run CI diff --git a/tests/README.rst b/tests/README.rst index d3fa8208..a5b759c2 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -13,13 +13,16 @@ Contents of this directory:: Run these tests from the repository root using:: - make test + $ just test To run a single test or group of tests or with different options, do:: - make testshell + $ just test tests/test_some_file.py -This gives you a bash shell in the docker container where you can more easily -do test runs and debugging. +Or to get a bash shell in the docker container where you can more easily +do test runs and debugging:: + + $ just test-shell + app@test:/app$ tests/test_some_file.py