Merge pull request Aiven-Open#947 from Aiven-Open/jjaakola-aiven-migr…

…ate-to-pyproject-toml

Migrate to pyproject toml

.coveragerc

@@ -1,4 +1,4 @@
 [run]
-branch = True
-relative_files = True
-source = karapace
+branch = true
+relative_files = true
+source = src/karapace

.dockerignore

@@ -3,12 +3,13 @@
 *
 
 # Include source directories and files required for building.
-!karapace
 !go
+!src
 !requirements/*.txt
-!setup.py
-!version.py
 !README.rst
+!LICENSE
+!pyproject.toml
+!setup.py
 !container/start.sh
 !container/healthcheck.py
 

.github/workflows/container-smoke-test.yml

@@ -13,11 +13,23 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+        # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701
+        with:
+          fetch-depth: 0
 
-      - name: Build karapace/version.py
-        run: python version.py
+      - name: Install requirements
+        run: make install
 
-      - name: Build and start services
+      - name: Resolve Karapace version
+        run: |
+          source ./venv/bin/activate
+          KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
+          echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
+
+      - name: Build container
+        run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile .
+
+      - name: Run container
         run: docker compose --file=container/compose.yml up --build --wait --detach
 
       - name: Smoke test registry

.github/workflows/container.yml

@@ -66,7 +66,7 @@ jobs:
           labels: |
             [email protected]
             org.opencontainers.image.url=https://karapace.io
-            org.opencontainers.image.documentation=https://github.com/aiven/karapace/
+            org.opencontainers.image.documentation=https://github.com/Aiven-Open/karapace/
             org.opencontainers.image.vendor=Aiven
             org.opencontainers.image.licenses=Apache-2.0
 

.github/workflows/lint.yml

@@ -26,8 +26,6 @@ jobs:
         python-version: '3.12'
     - name: Install libsnappy-dev
       run: sudo apt install libsnappy-dev
-    # required for pylint
-    - run: make karapace/version.py
     - run: pip install pre-commit
     - uses: actions/cache@v3
       with:
@@ -45,6 +43,11 @@ jobs:
         python-version: '3.12'
     - name: Install libsnappy-dev
       run: sudo apt install libsnappy-dev
-    - run: pip install -r requirements/requirements.txt -r requirements/requirements-typing.txt
-    - run: make karapace/version.py
-    - run: mypy
+    - name: Install requirements and typing requirements
+      run: |
+        python -m venv venv
+        source ./venv/bin/activate
+        pip --require-virtualenv install .[typing]
+    - run: |
+        source ./venv/bin/activate
+        mypy src

.github/workflows/schema.yml

@@ -21,12 +21,17 @@ jobs:
             requirements.txt
       - name: Install libsnappy-dev
         run: sudo apt install libsnappy-dev
-      - run: make install
+      - name: Install requirements
+        run: make install-dev
       # Compare with latest release when running on main.
-      - run: make schema against=$(git describe --abbrev=0 --tags)
+      - run: |
+          source ./venv/bin/activate
+          make schema against=$(git describe --abbrev=0 --tags)
         if: github.ref == 'refs/heads/main'
       # Compare with main when running on branches/PRs.
-      - run: make schema
+      - run: |
+          source ./venv/bin/activate
+          make schema
         if: github.ref != 'refs/heads/main'
       - run: |
           diff=$(git --no-pager diff)

.github/workflows/tests.yml

@@ -40,7 +40,7 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - run: make install version
+    - run: make install-dev
     - run: make unit-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
@@ -73,15 +73,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - run: make install-dev
+
     - name: Download coverage
       id: download_coverage
       uses: actions/download-artifact@v4
       with:
         pattern: coverage-*
         merge-multiple: true
 
-    - run: make karapace/version.py
-
     - name: Post coverage comment
       id: post_coverage_comment
       uses: py-cov-action/python-coverage-comment-action@v3

CONTRIBUTING.md

@@ -12,15 +12,15 @@ Contributions are very welcome on Karapace. When contributing please keep this i
 
 There is very little you need to get started coding for Karapace:
 
-- Use [one of the supported python versions](https://github.com/aiven/karapace/blob/master/setup.py)
-  documented in the `setup.py` classifiers.
+- Use [one of the supported python versions](https://github.com/Aiven-Open/karapace/blob/master/pyproject.toml)
+  documented in the `project:requires-python`.
 - Create [a virtual environment](https://docs.python.org/3/tutorial/venv.html) and install the dev dependencies in it:
 
 ```python
 python -m venv <path_to_venv>
 source <path_to_venv>/bin/activate
-pip install -r ./requirements/requirements-dev.txt
-pip install -e .
+pip install .
+pip install -e .[dev,typing]
 ```
 
 ## Website
@@ -55,7 +55,6 @@ To run the tests use the binary `pytest` available in the virtualenv. It will do
 used in the tests for you:
 
 ```sh
-make karapace/version.py
 pytest tests/unit
 pytest tests/integration
 ```
@@ -75,7 +74,7 @@ The code is statically checked and formatted using [a few tools][requirements-de
 To run these automatically on each commit please enable the [pre-commit](https://pre-commit.com)
 hooks.
 
-[requirements-dev]: https://github.com/aiven/karapace/blob/master/requirements/requirements-dev.txt
+[requirements-dev]: https://github.com/Aiven-Open/karapace/blob/master/requirements/requirements-dev.txt
 
 ## Manual testing
 
@@ -90,7 +89,7 @@ karapace karapace.config.json
 ### Configuration
 
 To see descriptions of configuration keys see our
-[README](https://github.com/aiven/karapace#configuration-keys).
+[README](https://github.com/Aiven-Open/karapace#configuration-keys).
 
 Each configuration key can be overridden with an environment variable prefixed with `KARAPACE_`,
 exception being configuration keys that actually start with the `karapace` string. For example, to

GNUmakefile

@@ -3,14 +3,16 @@ SHELL := /usr/bin/env bash
 VENV_DIR ?= $(CURDIR)/venv
 PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
-ifdef CI
-PYENV    ?= $(PYTHON)
-else
-PYENV    ?= pyenv exec python
-endif
-
 PYTHON_VERSION ?= 3.8
 
+define PIN_VERSIONS_COMMAND
+pip install pip-tools && \
+	python -m piptools compile -o /karapace/requirements/requirements.txt /karapace/pyproject.toml && \
+	python -m piptools compile --extra dev -o /karapace/requirements/requirements-dev.txt /karapace/pyproject.toml && \
+	python -m piptools compile --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml
+endef
+
+
 export PATH   := $(VENV_DIR)/bin:$(PATH)
 export PS4    := \e[0m\e[32m==> \e[0m
 export LC_ALL := C
@@ -28,43 +30,48 @@ all: version
 venv: venv/.make
 venv/.make:
 	rm -fr '$(VENV_DIR)'
-	$(PYENV) -m venv '$(VENV_DIR)'
+	$(PYTHON) -m venv '$(VENV_DIR)'
 	$(PIP) install --upgrade pip
 	touch '$(@)'
 
 .PHONY: install
 install: venv/.deps
-venv/.deps: requirements/requirements-dev.txt requirements/requirements.txt | venv/.make
+venv/.deps: venv/.make
 	set +x
 	source ./bin/get-java
 	source ./bin/get-protoc
 	source ./bin/get-snappy
 	set -x
-	$(PIP) install --use-pep517 -r '$(<)'
 	$(PIP) install --use-pep517 .
 	$(PIP) check
 	touch '$(@)'
 
+.PHONY: install-dev
+install-dev: venv/.deps-dev
+venv/.deps-dev: venv/.make
+	set +x
+	source ./bin/get-java
+	source ./bin/get-protoc
+	source ./bin/get-snappy
+	set -x
+	$(PIP) install -e .[dev]
+	$(PIP) check
+	touch '$(@)'
 
-karapace/version.py:
-	$(PYTHON) version.py
-
-.PHONY: version
-version: venv/.make | karapace/version.py
 
 .PHONY: test
 tests: unit-tests integration-tests
 
 .PHONY: unit-tests
 unit-tests: export PYTEST_ARGS ?=
-unit-tests: karapace/version.py venv/.deps
+unit-tests: venv/.deps-dev
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
 	rm -fr runtime/*
 
 .PHONY: integration-tests
 unit-tests: export PYTEST_ARGS ?=
-integration-tests: karapace/version.py venv/.deps
+integration-tests: venv/.deps-dev
 	rm -fr runtime/*
 	$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
 	rm -fr runtime/*
@@ -82,14 +89,16 @@ cleanest: cleaner
 	rm -fr '$(VENV_DIR)'
 
 .PHONY: requirements
-requirements: export CUSTOM_COMPILE_COMMAND='make requirements'
 requirements:
-	pip install --upgrade pip setuptools pip-tools
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements.in -o requirements.txt
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements-dev.in -o requirements-dev.txt
-	cd requirements && pip-compile --upgrade --resolver=backtracking requirements-typing.in -o requirements-typing.txt
+requirements:
+	$(PIP) install --upgrade pip setuptools pip-tools
+	$(PIP) install .[dev,typing]
 
 .PHONY: schema
 schema: against := origin/main
 schema:
-	python3 -m karapace.backup.backends.v3.schema_tool --against=$(against)
+	$(PYTHON) -m karapace.backup.backends.v3.schema_tool --against=$(against)
+
+.PHONY: pin-requirements
+pin-requirements:
+	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"

MANIFEST.in

@@ -1,14 +1,11 @@
-include karapace/*.py
+graft src
 include karapace.json
 include karapace.unit
 include tests/*.py
 include scripts/*
 include README.rst
-include version.py
-include setup.py
-include setup.cfg
+include pyproject.toml
 include LICENSE
 include MANIFEST.in
-include *.so
 
 recursive-exclude examples *~ *.pyc \.*

README.rst

@@ -3,7 +3,7 @@ Karapace
 
 ``karapace``. Your Apache Kafka® essentials in one tool.
 
-An `open-source <https://github.com/aiven/karapace/blob/master/LICENSE>`_ implementation
+An `open-source <https://github.com/Aiven-Open/karapace/blob/master/LICENSE>`_ implementation
 of `Kafka REST <https://docs.confluent.io/platform/current/kafka-rest/index.html#features>`_ and
 `Schema Registry <https://docs.confluent.io/platform/current/schema-registry/index.html>`_.
 
@@ -81,14 +81,14 @@ override the ``bootstrap_uri`` config value, one would use the environment varia
 ``KARAPACE_BOOTSTRAP_URI``. Here_ you can find an example configuration file to give you an idea
 what you need to change.
 
-.. _`Here`: https://github.com/aiven/karapace/blob/master/karapace.config.json
+.. _`Here`: https://github.com/Aiven-Open/karapace/blob/master/karapace.config.json
 
 Source install
 --------------
 
 Alternatively you can do a source install using::
 
-  python setup.py install
+  pip install .
 
 Quickstart
 ==========
@@ -651,7 +651,7 @@ If you don't need or want to have the Karapace images around you can now proceed
 Installed from Sources
 ----------------------
 
-If you installed Karapace from the sources via ``python setup.py install``, it can be uninstalled with the following ``pip`` command::
+Karapace is installed ``pip install .``, it can be uninstalled with the following ``pip`` command::
 
     pip uninstall karapace
 
@@ -664,12 +664,6 @@ and install the required software for development. Use ``make unit-tests`` and
 ``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the
 execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``).
 
-By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures
-on all platforms that arbitrary Python versions can be used for development. It
-is possible to overwrite this by setting ``PYENV`` to something else (e.g.
-``PYENV=python3 make venv`` to simply use the global Python executable). The
-default Python version is defined in ``.python-version``.
-
 Karapace currently depends on various system software to be installed. The
 installation of these is automated for some operation systems, but not all. At
 the time of writing Java, the Protobuf Compiler, and the Snappy shared library
@@ -695,11 +689,8 @@ targets that correctly clean the ``runtime`` directory without deleting it, but
 keep this in mind whenever you are not using ``make`` (e.g. running tests from
 your IDE).
 
-Note that the pre-commit checks are currently not working with the default
-Python version. This is because isort dropped Python 3.7 support. You have to
-use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or
-… to install pre-commit and use the global installation, there is also no
-dependency on it.
+Use ``pipx`` or ``brew`` to install ``pre-commit`` and use the global installation,
+there is also no dependency on it.
 
 License
 =======
@@ -714,7 +705,7 @@ Contact
 =======
 
 Bug reports and patches are very welcome, please post them as GitHub issues
-and pull requests at https://github.com/aiven/karapace .  Any possible
+and pull requests at https://github.com/Aiven-Open/karapace .  Any possible
 vulnerabilities or other serious issues should be reported directly to the
 maintainers <[email protected]>.
 
@@ -735,6 +726,6 @@ to them for pioneering the concept.
 .. _`Aiven`: https://aiven.io/
 
 Recent contributors are listed on the GitHub project page,
-https://github.com/aiven/karapace/graphs/contributors
+https://github.com/Aiven-Open/karapace/graphs/contributors
 
 Copyright ⓒ 2021 Aiven Ltd.

container/Dockerfile

@@ -26,8 +26,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install -r /build/requirements.txt
 
 COPY . /build/karapace-repo
+WORKDIR /build/karapace-repo
 RUN --mount=type=cache,target=/root/.cache/pip \
-    python3 -m pip install /build/karapace-repo
+    if [ -z "${KARAPACE_VERSION}" ]; then \
+        PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \
+    else \
+        PRETEND_VERSION=$KARAPACE_VERSION; \
+    fi; \
+    SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install --no-deps .
 
 # Karapace image, i.e. production.
 FROM python:3.10.11-slim-bullseye AS karapace