Merge branch 'master' into deps-conda

OCR-D · May 3, 2024 · fb15556 · fb15556
2 parents 619e1a3 + 95fd6e2
commit fb15556
Show file tree

Hide file tree

Showing 304 changed files with 10,851 additions and 5,029 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,20 +16,10 @@ jobs:
       PIP_INSTALL: pip3 install --user
     steps:
       - checkout
-      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash
+      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash opencv
+      - run: $PIP_INSTALL -U pip setuptools
       - run: make install
-      - run: PATH="/Users/distiller/Library/Python/3.9/bin:$PATH" make deps-test test benchmark
-
-  test-python37:
-    docker:
-      - image: cimg/python:3.7
-    working_directory: ~/ocrd-core
-    steps:
-      - checkout
-      - run: sudo apt-get -y update
-      - run: sudo make deps-ubuntu 
-      - run: make install deps-test
-      - run: make test benchmark
+      - run: export PATH="/Users/distiller/Library/Python/3.9/bin:$PATH"; make deps-test test benchmark
 
   test-python38:
     docker:
@@ -41,6 +31,8 @@ jobs:
       - run: sudo make deps-ubuntu
       - run: make install deps-test
       - run: make test benchmark
+      # smoke test to ensure that --editable install works
+      - run: make install-dev; ocrd --version
 
   test-python39:
     docker:
@@ -52,6 +44,8 @@ jobs:
       - run: sudo make deps-ubuntu
       - run: make install deps-test
       - run: make test benchmark
+      # smoke test to ensure that --editable install works
+      - run: make install-dev; ocrd --version
 
   test-python310:
     docker:
@@ -63,6 +57,8 @@ jobs:
       - run: sudo make deps-ubuntu
       - run: make install deps-test
       - run: make test benchmark
+      # smoke test to ensure that --editable install works
+      - run: make install-dev; ocrd --version
 
   test-python311:
     docker:
@@ -74,6 +70,8 @@ jobs:
       - run: sudo make deps-ubuntu
       - run: make install deps-test
       - run: make test benchmark
+      # smoke test to ensure that --editable install works
+      - run: make install-dev; ocrd --version
 
   deploy:
     docker:
@@ -87,8 +85,12 @@ jobs:
       - run:
           name: Login to Docker Hub
           command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
-      - run: docker push ocrd/core
-      - run: docker push ocrd/core-cuda
+      - run: docker push ocrd/core:latest
+      - run: docker push ocrd/core-cuda:latest
+      - run: docker tag ocrd/core ocrd/core:$(git describe --tags --abbrev=0)
+      - run: docker tag ocrd/core-cuda ocrd/core-cuda:$(git describe --tags --abbrev=0)
+      - run: docker push ocrd/core:$(git describe --tags --abbrev=0)
+      - run: docker push ocrd/core-cuda:$(git describe --tags --abbrev=0)
 
 workflows:
   version: 2
@@ -98,11 +100,13 @@ workflows:
           filters:
             branches:
               only: master
+            tags:
+              only: /^v\d\.\d\d\.\d$/
   test-pull-request:
     jobs:
       - test-python37
       - test-python38
       - test-python39
       - test-python310
       - test-python311
-      - test-macos
+      # - test-macos
diff --git a/.dockerignore b/.dockerignore
@@ -1,5 +1,12 @@
 *
-!ocrd*
+!src*
+!pyproject.toml
+!requirements.txt
 !Makefile
+!VERSION
 !LICENSE
 !README.md
+!.git
+!tests
+!requirements_test.txt
+!.gitmodules
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -6,7 +6,8 @@ on:
   workflow_dispatch: # run manually
 
 env:
-  DOCKER_TAG: ghcr.io/ocr-d/core
+  GHCRIO_DOCKER_TAG: ghcr.io/ocr-d/core
+  DOCKERIO_DOCKER_TAG: docker.io/ocrd/core
 
 jobs:
 
@@ -24,18 +25,39 @@ jobs:
       uses: docker/setup-buildx-action@v2
     - name: Build the Docker image
       # default tag uses docker.io, so override on command-line
-      run: make docker DOCKER_TAG=${{ env.DOCKER_TAG }}
+      run: make docker DOCKER_TAG=${{ env.GHCRIO_DOCKER_TAG }}
     - name: Build the Docker image with GPU support
       # default tag uses docker.io, so override on command-line
-      run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }}
+      run: make docker-cuda DOCKER_TAG=${{ env.GHCRIO_DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.GHCRIO_DOCKER_TAG }}
+    - name: Smoke Test that ocrd --help works
+      run: |
+        docker run --rm ${{ env.GHCRIO_DOCKER_TAG }} ocrd --version
+        docker run --rm ${{ env.GHCRIO_DOCKER_TAG }}-cuda ocrd --version
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Log in to Docker Hub
+      uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+      with:
+        username: ${{ secrets.DOCKERIO_USERNAME }}
+        password: ${{ secrets.DOCKERIO_PASSWORD }}
     - name: Push images to Github Container Registry
       run: |
-        docker push ${{ env.DOCKER_TAG }}:latest
-        docker push ${{ env.DOCKER_TAG }}-cuda:latest
+        docker push ${{ env.GHCRIO_DOCKER_TAG }}:latest
+        docker push ${{ env.GHCRIO_DOCKER_TAG }}-cuda:latest
+    - name: Push images to Docker Hub
+      run: |
+        docker tag ${{ env.GHCRIO_DOCKER_TAG }} ${{ env.DOCKERIO_DOCKER_TAG }}
+        docker tag ${{ env.GHCRIO_DOCKER_TAG }}-cuda ${{ env.DOCKERIO_DOCKER_TAG }}-cuda
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}:latest
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}-cuda:latest
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}:latest
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}-cuda:latest
+        docker tag ${{ env.DOCKERIO_DOCKER_TAG }} ${{ env.DOCKERIO_DOCKER_TAG }}:$(git describe --tags --abbrev=0)
+        docker tag ${{ env.DOCKERIO_DOCKER_TAG }}-cuda ${{ env.DOCKERIO_DOCKER_TAG }}-cuda:$(git describe --tags --abbrev=0)
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}:$(git describe --tags --abbrev=0)
+        docker push ${{ env.DOCKERIO_DOCKER_TAG }}-cuda:$(git describe --tags --abbrev=0)
     
diff --git a/.github/workflows/network-testing.yml b/.github/workflows/network-testing.yml
@@ -0,0 +1,60 @@
+name: ocrd network module and integration tests
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch: # run manually
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          # - macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+            sudo apt-get -y update
+            sudo make deps-ubuntu
+          else
+            HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+            HOMEBREW_NO_AUTO_UPDATE=1 \
+            brew install imagemagick geos bash  # opencv
+          fi
+          make install deps-test
+      - name: Install Docker on macOS
+        if: runner.os == 'macos'
+        run: |
+          brew install docker docker-compose
+          colima start
+      - name: Run network modules tests with pytest
+        run: make network-module-test
+      - name: Run network integration tests with pytest
+        run: |
+          if [[ "${{ matrix.os }}" == "macos"* ]];then
+            make network-integration-test-cicd DOCKER_COMPOSE=docker-compose
+          else
+            make network-integration-test-cicd
+          fi
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -0,0 +1,62 @@
+name: Test core installation and run tests
+
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          - ubuntu-20.04
+          # - macos-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+          sudo apt-get -y update
+          sudo make deps-ubuntu
+        else
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+          HOMEBREW_NO_AUTO_UPDATE=1 \
+          brew install imagemagick geos bash  # opencv
+        fi
+        make install deps-test
+    - name: Test with pytest
+      run: |
+        make test benchmark
+    - name: test to ensure that --editable install works
+      run: |
+        make install-dev; ocrd --version
+    - name: Lint with flake8
+      run: |
+        python -m pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 src --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
diff --git a/.gitignore b/.gitignore
@@ -109,7 +109,6 @@ env2/
 
 ocrd.egg-info
 .pytest_cache
-/src
 /profile
 .~lock*
 .pynative
@@ -125,5 +124,6 @@ tests/assets/test.ocrd.zip
 /foo
 sanders*
 ws1
-*.doctree
+*.doctrees
 .vscode
+*.BAK
diff --git a/.pylintrc b/.pylintrc
@@ -6,6 +6,7 @@ ignored-modules=cv2,tesserocr,ocrd.model
 ignore-patterns='.*generateds.*'
 disable =
     fixme,
+    E501,
     trailing-whitespace,
     logging-not-lazy,
     inconsistent-return-statements,