From 4d9df22f93d2e2bbc755c89e2cec0392ae23ea1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 18 Apr 2024 11:55:40 +0200 Subject: [PATCH 1/2] feat: Solrwayback 5.x This commit upgrades solrwayback to version 5.1.0. See the release notes https://github.com/netarchivesuite/solrwayback/releases/tag/5.1.0 The purpose of this repository is to build the container images used to run solrwayback in a containerized environment. In the name of keeping it simple this commit also removes all uneccessary files and abstractions that is not needed for this purpose. --- .gitattributes | 1 - .github/workflows/main.yml | 18 ++---- .github/workflows/release.yml | 15 ++--- .github/workflows/test.yml | 42 ------------- Dockerfile.solr | 32 ---------- Dockerfile.solrwayback | 55 ++++++++--------- Dockerfile.warc-indexer | 25 ++++---- README.md | 107 +++++++++++++++++++++++++++++++++- ensure_arguments_in_sync.py | 91 ----------------------------- favicon.ico | 3 - fetch_solrwayback_bundle.py | 58 ------------------ pyproject.toml | 7 --- test-requirements.in | 1 - test-requirements.txt | 14 ----- tests/test_executable_bit.py | 26 --------- tests/test_values_in_sync.py | 35 ----------- 16 files changed, 155 insertions(+), 375 deletions(-) delete mode 100644 .gitattributes delete mode 100644 .github/workflows/test.yml delete mode 100644 Dockerfile.solr delete mode 100644 ensure_arguments_in_sync.py delete mode 100644 favicon.ico delete mode 100644 fetch_solrwayback_bundle.py delete mode 100644 pyproject.toml delete mode 100644 test-requirements.in delete mode 100644 test-requirements.txt delete mode 100644 tests/test_executable_bit.py delete mode 100644 tests/test_values_in_sync.py diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 361484f..0000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.ico filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02c4a65..3fc1a6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,6 @@ name: Test release -on: - push: +on: push env: REGISTRY: ghcr.io @@ -15,12 +14,11 @@ jobs: packages: write strategy: matrix: - image-type: [solr, solrwayback, warc-indexer] + image: [solrwayback, warc-indexer] steps: - name: Code checkout uses: actions/checkout@v3 - with: - lfs: true + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -28,7 +26,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image-type }} + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image }} tags: | type=semver,pattern={{version}} type=ref,event=branch @@ -40,10 +38,4 @@ jobs: push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile.${{ matrix.image-type }} - build-args: | - SOLRWAYBACK_VERSION=4.4.2 - SOLR_VERSION=7.7.3 - SOLRWAYBACK_TOMCAT_VERSION=8.5.60 - TOMCAT_TAG=8.5-jdk8-temurin-jammy - ECLIPSE_TEMURIN_TAG=8-jre + file: Dockerfile.${{ matrix.image }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec5f20b..19ee09b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,12 +19,11 @@ jobs: packages: write strategy: matrix: - image-type: [solr, solrwayback, warc-indexer] + image: [solrwayback, warc-indexer] steps: - name: Code checkout uses: actions/checkout@v3 - with: - lfs: true + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -32,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image-type }} + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image }} tags: | type=semver,pattern={{version}} type=ref,event=branch @@ -52,10 +51,4 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile.${{ matrix.image-type }} - build-args: | - SOLRWAYBACK_VERSION=4.4.2 - SOLR_VERSION=7.7.3 - SOLRWAYBACK_TOMCAT_VERSION=8.5.60 - TOMCAT_TAG=8.5-jdk8-temurin-jammy - ECLIPSE_TEMURIN_TAG=8-jre + file: Dockerfile.${{ matrix.image }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 34505f4..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Test - -on: - push: - -jobs: - test-ensure-arguments-in-sync: - name: Test ensure arguments in sync - runs-on: ubuntu-latest - - steps: - - name: Code checkout - uses: actions/checkout@v3 - with: - lfs: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - name: Install dependencies - run: | - python -m pip install --requirement test-requirements.txt - - name: Run test - run: | - pytest **/*.py - ensure-arguments-in-sync: - name: Ensure arguments in sync - needs: test-ensure-arguments-in-sync - runs-on: ubuntu-latest - - steps: - - name: Code checkout - uses: actions/checkout@v3 - with: - lfs: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - name: Ensure arguments in sync - run: | - python ensure_arguments_in_sync.py diff --git a/Dockerfile.solr b/Dockerfile.solr deleted file mode 100644 index bcec5aa..0000000 --- a/Dockerfile.solr +++ /dev/null @@ -1,32 +0,0 @@ -# This dockerfile configures a vanilla solr 7.7.3 configured with -# a solrwayback core. -# -# See https://solr.apache.org/guide/7_7/solr-cores-and-solr-xml.html -# for details on how solr cores are configured. - -ARG SOLRWAYBACK_VERSION=4.4.2 -ARG SOLR_VERSION=7.7.3 - -FROM ubuntu:22.04 as solrwayback-bundle - -ARG SOLRWAYBACK_VERSION -ARG SOLRWAYBACK_CHECKSUM - -RUN apt-get update \ - && apt-get install --quiet --assume-yes wget python3 - -WORKDIR /build -COPY fetch_solrwayback_bundle.py . - -RUN python3 fetch_solrwayback_bundle.py \ - --solrwayback-version ${SOLRWAYBACK_VERSION} \ - --destination /app - -FROM solr:${SOLR_VERSION} - -ARG SOLR_VERSION -ARG SOLRWAYBACK_VERSION - -COPY --from=solrwayback-bundle --chown=solr \ - /app/solrwayback_package_${SOLRWAYBACK_VERSION}/solr-${SOLR_VERSION}/server/solr/configsets/netarchivebuilder \ - /opt/solr/server/solr/mycores/netarchivebuilder diff --git a/Dockerfile.solrwayback b/Dockerfile.solrwayback index a54bd6f..d86e32e 100644 --- a/Dockerfile.solrwayback +++ b/Dockerfile.solrwayback @@ -1,51 +1,52 @@ -# This dockerfile configures a vanilla tomcat container -# with solrwayback installed and configured with properties -# from solrwayback bundle. +# This dockerfile builds a tomcat container including the webapps +# from the solrwayback bundle. # -# See https://hub.docker.com/_/tomcat for details on how -# to configure tomcat. +# See https://hub.docker.com/_/tomcat for details on how to configure tomcat. -ARG SOLRWAYBACK_VERSION=4.4.2 -ARG SOLRWAYBACK_TOMCAT_VERSION=8.5.60 -ARG TOMCAT_TAG=8.5-jdk8-temurin-jammy +ARG SOLRWAYBACK_VERSION=5.1.0 +ARG SOLRWAYBACK_TOMCAT_VERSION=9 +ARG TOMCAT_TAG=9-jre17-temurin-jammy FROM ubuntu:22.04 as solrwayback-bundle ARG SOLRWAYBACK_VERSION ARG SOLRWAYBACK_TOMCAT_VERSION -ARG SOLRWAYBACK_VERSION -RUN apt-get update \ - && apt-get install --quiet --assume-yes wget unzip python3 +RUN apt-get update && apt-get install -y \ + unzip \ + wget WORKDIR /build -COPY fetch_solrwayback_bundle.py . -RUN python3 fetch_solrwayback_bundle.py \ - --solrwayback-version ${SOLRWAYBACK_VERSION} \ - --destination /app -RUN unzip /app/solrwayback_package_${SOLRWAYBACK_VERSION}/apache-tomcat-${SOLRWAYBACK_TOMCAT_VERSION}/webapps/solrwayback.war \ - -d /app/solrwayback/ +RUN wget -q https://github.com/netarchivesuite/solrwayback/releases/download/${SOLRWAYBACK_VERSION}/solrwayback_package_${SOLRWAYBACK_VERSION}.zip +RUN unzip solrwayback_package_${SOLRWAYBACK_VERSION}.zip \ +&& mkdir /webapps \ +&& unzip -d /webapps/solrwayback solrwayback_package_${SOLRWAYBACK_VERSION}_MASTER/tomcat-${SOLRWAYBACK_TOMCAT_VERSION}/webapps/solrwayback.war \ +&& cp solrwayback_package_${SOLRWAYBACK_VERSION}_MASTER/tomcat-${SOLRWAYBACK_TOMCAT_VERSION}/webapps/ROOT.war /webapps/ROOT.war -FROM tomcat:${TOMCAT_TAG} -ARG SOLRWAYBACK_TOMCAT_VERSION -ARG SOLRWAYBACK_VERSION +FROM tomcat:${TOMCAT_TAG} -# TODO: install solrwayback dependencies such as ffmpeg, imagemagick, tesseract-ocr, chromium-browser, etc. +# This is not necessary for solrwayback to work. +# It is only necessary if you want to use the page preview feature. +# It increases the size of the image by about 200MB. +RUN apt-get update && apt-get install -y \ + chromium-browser \ + chromium-codecs-ffmpeg \ + && rm -rf /var/lib/apt/lists/* # CATALINA_HOME is the folder where catalina is installed. # The main component of tomcat is called catalina. # CATALINA_HOME is set by the tomcat image. -# Copy the extracted solrwayback.war file and ROOT.war to the webapps folder of tomcat. +# Copy the extracted solrwayback.war file +# We use the extracted solrwayback.war to be able to customize the web +# application (favicon, etc.) at runtime (using overlays). COPY --from=solrwayback-bundle \ - /app/solrwayback/ \ + /webapps/solrwayback \ ${CATALINA_HOME}/webapps/solrwayback +# Copy ROOT.war to the webapps folder of tomcat. COPY --from=solrwayback-bundle \ - /app/solrwayback_package_${SOLRWAYBACK_VERSION}/apache-tomcat-${SOLRWAYBACK_TOMCAT_VERSION}/webapps/ROOT.war \ + /webapps/ROOT.war \ ${CATALINA_HOME}/webapps/ROOT.war - -# Set URL icon for the web application -COPY favicon.ico ${CATALINA_HOME}/webapps/solrwayback/ diff --git a/Dockerfile.warc-indexer b/Dockerfile.warc-indexer index 9285e9c..9b77ec8 100644 --- a/Dockerfile.warc-indexer +++ b/Dockerfile.warc-indexer @@ -1,25 +1,26 @@ -# This Dockerfile creates a vanilla java container -# with warc-indexer from solrwayback bundle. +# This Dockerfile creates a vanilla java container with warc-indexer from the solrwayback bundle. -ARG SOLRWAYBACK_VERSION=4.4.2 -ARG ECLIPSE_TEMURIN_TAG=8-jre +ARG SOLRWAYBACK_VERSION=5.1.0 +ARG ECLIPSE_TEMURIN_TAG=17-jre FROM ubuntu:22.04 as solrwayback-bundle ARG SOLRWAYBACK_VERSION -RUN apt-get update \ - && apt-get install --quiet --assume-yes wget python3 +RUN apt-get update && apt-get install -y \ + unzip \ + wget WORKDIR /build -COPY fetch_solrwayback_bundle.py . -RUN python3 fetch_solrwayback_bundle.py \ - --solrwayback-version ${SOLRWAYBACK_VERSION} \ - --destination /app +RUN wget -q https://github.com/netarchivesuite/solrwayback/releases/download/${SOLRWAYBACK_VERSION}/solrwayback_package_${SOLRWAYBACK_VERSION}.zip +RUN mkdir /app \ + && unzip solrwayback_package_${SOLRWAYBACK_VERSION}.zip \ + && mv solrwayback_package_${SOLRWAYBACK_VERSION}_MASTER/* /app + FROM eclipse-temurin:${ECLIPSE_TEMURIN_TAG} -ARG SOLRWAYBACK_VERSION +COPY --from=solrwayback-bundle /app/indexing /opt/warc-indexer -COPY --from=solrwayback-bundle /app/solrwayback_package_${SOLRWAYBACK_VERSION}/indexing /opt/warc-indexer +ENTRYPOINT ["/opt/warc-indexer/warc-indexer.sh"] diff --git a/README.md b/README.md index d2473b7..56a32d1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,105 @@ -# solrwayback-branding -Contains branding for Solrwayback +# SolrWayback container images + +This repository builds and publishes container images from [SolrWayback releases](https://github.com/netarchivesuite/solrwayback/releases). + +## Images + +### Solrwayback + +The solrwayback container image does not include the configuration files `solrwayback.properties` and `solrwaybackweb.properties`. Use the ones from the official release bundle as a starting point to create your own. + +These files must be placed directly under the `/root` folder using either +overlays at runtime or by building your own image: + +```Dockerfile +FROM github.com/nlnwa/solrwayback-adaption:latest + +COPY solrwayback.properties solrwaybackweb.properties /root +``` + +### Warc indexer + +```shell +$ docker run ghcr.io/nlnwa/warc-indexer -h + +warc-indexer.sh + +Parallel processing of WARC files using webarchive-discovery from UKWA: +https://github.com/ukwa/webarchive-discovery + +The scripts keeps track of already processed WARCs by keeping the output +logs from processing of each WARC. These are stored in the folder +/opt/warc-indexer/status + + +Usage: ./warc-indexer.sh [warc|warc-folder]* + + +Index 2 WARC files: + + ./warc-indexer.sh mywarcfile1.warc.gz mywarcfile2.warc.gz + +Index all WARC files in "folder_with_warc_files" (recursive descend) using +20 threads (this will take 20GB of memory): + + THREADS=20 ./warc-indexer.sh folder_with_warc_files + +Index all WARC files in "folder_with_warc_files" (recursive descend) using +20 threads and with an alternative Solr as receiver: + + THREADS=20 SOLR_URL="http://ourcloud.internal:8123/solr/netarchive" ./warc-indexer.sh folder_with_warc_files + +Note: +Each thread starts its own Java process with -Xmx1024M. +Make sure that there is enough memory on the machine. + +Tweaks: + SOLR_URL: The receiving Solr end point, including collection + Value: http://localhost:8983/solr/netarchivebuilder + + SOLR_CHECK: Check whether Solr is available before processing + Value: true + + SOLR_COMMIT: Whether a Solr commit should be issued after indexing to + flush the buffers and make the changes immediately visible + Value: true + + THREADS: The number of concurrent processes to use for indexing + Value: 2 + + STATUS_ROOT: Where to store log files from processing. The log files are + also used to track which WARCs has been processed + Value: /opt/warc-indexer/status + + TMP_ROOT: Where to store temporary files during processing + Value: /opt/warc-indexer/status/tmp + + INDEXER_JAR: The location of the warc-indexer Java tool + Value: /opt/warc-indexer/warc-indexer-3.3.1-jar-with-dependencies.jar + + INDEXER_MEM: Memory allocation for each builder job + Value: 1024M + + INDEXER_CONFIG: Configuration for the warc-indexer Java tool + Value: /opt/warc-indexer/config3.conf + + INDEXER_CUSTOM: Custom command line options for the warc-indexer tool + Value: "" + Sample: "--collection yearly2020" +``` + +### Solr + +Use the [official image](https://hub.docker.com/_/solr). + +To run Solr in kubernetes see [Official Kubernetes operator for Apache Solr](https://github.com/apache/solr-operator). + +Previously this repository contained a Dockerfile that built a [Solr](https://solr.apache.org/guide/solr/latest/index.html) container image (based on the official image) with the "netarchivebuilder" configset from the [SolrWayback bundle version 4.4.2](https://github.com/netarchivesuite/solrwayback/releases/tag/4.4.2). + +As of SolrWayback version 5.x, Solr is started in cloud mode which store the configsets in [ZooKeeper](https://zookeeper.apache.org/) and the SolrWayback bundle does not include a configset (besides the default). See the [Solr 9 configset in the warc indexer repository](https://github.com/ukwa/webarchive-discovery/tree/master/warc-indexer/src/main/solr/solr9/discovery/conf) for a starting point to create your own. + +## TODO + +- Docker compose file and examples. +- Kubernetes deployment files and examples. +- Rootless versions of the images. diff --git a/ensure_arguments_in_sync.py b/ensure_arguments_in_sync.py deleted file mode 100644 index 00bc547..0000000 --- a/ensure_arguments_in_sync.py +++ /dev/null @@ -1,91 +0,0 @@ -from pathlib import Path -from re import findall -from typing import Sequence, TypeAlias - -ArgumentStruct: TypeAlias = dict[str, dict[str, list[Path]]] - - -def _main() -> None: - all_docker_files = list(Path().rglob("Dockerfile*")) - all_workflow_files = list((Path() / ".github" / "workflows").rglob("*.yml")) - print("Checking that the following paths contain the same arguments") - print(*list(map(str, all_docker_files + all_workflow_files))) - _ensure_arguments_in_sync(all_docker_files, all_workflow_files) - - -def _ensure_arguments_in_sync( - dockerfiles: Sequence[Path], workflows: Sequence[Path] -) -> None: - argument_struct: ArgumentStruct = {} - - for docker_file in dockerfiles: - _dockerfile(docker_file, argument_struct) - - for workflow_file in workflows: - _workflow(workflow_file, argument_struct) - - _validate_argument_struct(argument_struct) - - -def _flatten(non_flat: list): - return [item for sublist in non_flat for item in sublist] - - -def _validate_argument_struct(argument_struct: ArgumentStruct) -> None: - for argument, versions in argument_struct.items(): - if len(versions) > 1: - relevant_files = list(map(str, _flatten(versions.values()))) - raise RuntimeError( - f"Argument: '{argument}' has has inconsistent values for the versions: '{', '.join(versions)}' in the paths '{', '.join(relevant_files)}'" - ) - - -def _dockerfile(dockerfile: Path, argument_struct: ArgumentStruct) -> None: - pattern = r"(ARG\s+)(\S+)(=)(\S*)" # https://regex101.com/r/6tqpqz/1 - _insert_arguments(path=dockerfile, argument_struct=argument_struct, pattern=pattern) - - -def _workflow(workflow_file: Path, argument_struct: ArgumentStruct) -> None: - pattern = r"(\s*)(\S+)(=)(\S*)" # https://regex101.com/r/w3xhV9/1 - ignore_arguments = [ - "type=ref,event", # is not a argument shared between workflows and dockerfiles - ] - _insert_arguments( - path=workflow_file, - argument_struct=argument_struct, - pattern=pattern, - ignore_arguments=ignore_arguments, - ) - - -def _insert_arguments( - path: Path, argument_struct: ArgumentStruct, pattern: str, ignore_arguments=None -) -> None: - content = path.read_text(encoding="utf-8") - if matched := findall(pattern, content): - for match in matched: - argument_name = match[1] - argument_version = match[3] - if ignore_arguments and argument_name in ignore_arguments: - continue - if argument_name not in argument_struct: - argument_struct[argument_name] = { - argument_version: [ - path, - ] - } - else: - if argument_version not in argument_struct[argument_name]: - argument_struct[argument_name][argument_version] = [ - path, - ] - else: - argument_struct[argument_name][argument_version] = argument_struct[ - argument_name - ][argument_version] + [ - path, - ] - - -if __name__ == "__main__": - _main() diff --git a/favicon.ico b/favicon.ico deleted file mode 100644 index 87ed094..0000000 --- a/favicon.ico +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e999907eab50c26cb39d58ea194f8be3223692582c08a096c6eeef75bb568299 -size 100870 diff --git a/fetch_solrwayback_bundle.py b/fetch_solrwayback_bundle.py deleted file mode 100644 index d24babe..0000000 --- a/fetch_solrwayback_bundle.py +++ /dev/null @@ -1,58 +0,0 @@ -from argparse import ArgumentParser, Namespace -from pathlib import Path -from urllib.request import urlopen -from tempfile import TemporaryDirectory -from zipfile import ZipFile -from http import HTTPStatus -from stat import S_IXUSR, S_IXGRP, S_IXOTH - - -def _args() -> Namespace: - parser = ArgumentParser() - parser.add_argument( - "--solrwayback-version", - required=True, - type=str, - help="solrwayback bundle version", - ) - parser.add_argument( - "--destination", - required=True, - type=Path, - help="Directory to download solrwayback bundle to", - ) - return parser.parse_args() - - -def _fetch_bundle(solrwayback_version: str, destination: Path) -> None: - url = f"https://github.com/netarchivesuite/solrwayback/releases/download/{solrwayback_version}/solrwayback_package_{solrwayback_version}.zip" - print(f"Downloading {url} to {destination}", flush=True) - - with urlopen(url) as response: - if response.getcode() != HTTPStatus.OK: - raise RuntimeError( - f"Failed to download '{url}', got response code '{response.getcode()}'" - ) - with TemporaryDirectory() as temp_dir_name: - zip_path = Path(temp_dir_name) / "solrwayback.zip" - with zip_path.open("wb") as zip_file: - zip_file.write(response.read()) - with ZipFile(zip_path, "r") as zip_ref: - zip_ref.extractall(destination) - # set the executable bit for `.sh` files as python removes the original - # executable bits when extracting the zip file - for shell_script in destination.rglob("*.sh"): - mode = shell_script.stat().st_mode - mode |= S_IXUSR | S_IXGRP | S_IXOTH - shell_script.chmod(mode) - - -def _main() -> None: - args = _args() - _fetch_bundle( - solrwayback_version=args.solrwayback_version, destination=args.destination - ) - - -if __name__ == "__main__": - _main() diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index bb14668..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,7 +0,0 @@ -[tool.isort] -profile = "black" - -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "--exitfirst" -pythonpath = "." diff --git a/test-requirements.in b/test-requirements.in deleted file mode 100644 index 55b033e..0000000 --- a/test-requirements.in +++ /dev/null @@ -1 +0,0 @@ -pytest \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 6052851..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile test-requirements.in -# -iniconfig==2.0.0 - # via pytest -packaging==23.1 - # via pytest -pluggy==1.3.0 - # via pytest -pytest==7.4.0 - # via -r test-requirements.in diff --git a/tests/test_executable_bit.py b/tests/test_executable_bit.py deleted file mode 100644 index e7680ba..0000000 --- a/tests/test_executable_bit.py +++ /dev/null @@ -1,26 +0,0 @@ -from pathlib import Path -from stat import S_IXGRP, S_IXOTH, S_IXUSR -from tempfile import TemporaryDirectory - -from fetch_solrwayback_bundle import _fetch_bundle - - -def test_executable_bits_are_set() -> None: - with TemporaryDirectory() as temp_dir_str: - temp_dir = Path(temp_dir_str) - destination = temp_dir / "bundle" - solrwayback_version = "4.4.2" - - _fetch_bundle(solrwayback_version=solrwayback_version, destination=destination) - - sh_files = list(destination.rglob("*.sh")) - for sh_file in sh_files: - assert ( - sh_file.stat().st_mode & S_IXUSR != 0 - ), f"File '{sh_file}' has does not have user executable bit set" - assert ( - sh_file.stat().st_mode & S_IXGRP != 0 - ), f"File '{sh_file}' has does not have group executable bit set" - assert ( - sh_file.stat().st_mode & S_IXOTH != 0 - ), f"File '{sh_file}' has does not have other executable bit set" diff --git a/tests/test_values_in_sync.py b/tests/test_values_in_sync.py deleted file mode 100644 index da2b515..0000000 --- a/tests/test_values_in_sync.py +++ /dev/null @@ -1,35 +0,0 @@ -from pathlib import Path -from tempfile import TemporaryDirectory - -from pytest import raises - -from ensure_arguments_in_sync import _ensure_arguments_in_sync - - -def test_valid() -> None: - dockerfile = """ARG SOLR_VERSION=8.8.2""" - workflow = """SOLR_VERSION=8.8.2""" - - with TemporaryDirectory() as temp_dir_str: - temp_dir = Path(temp_dir_str) - dockerfile_path = temp_dir / "Dockerfile" - workflow_path = temp_dir / "workflow.yml" - dockerfile_path.write_text(dockerfile, encoding="utf-8") - workflow_path.write_text(workflow, encoding="utf-8") - - _ensure_arguments_in_sync([dockerfile_path], [workflow_path]) - - -def test_invalid() -> None: - dockerfile = """ARG SOLR_VERSION=8.8.2""" - workflow = """SOLR_VERSION=8.8.3""" - - with TemporaryDirectory() as temp_dir_str: - temp_dir = Path(temp_dir_str) - dockerfile_path = temp_dir / "Dockerfile" - workflow_path = temp_dir / "workflow.yml" - dockerfile_path.write_text(dockerfile, encoding="utf-8") - workflow_path.write_text(workflow, encoding="utf-8") - - with raises(RuntimeError): - _ensure_arguments_in_sync([dockerfile_path], [workflow_path]) From 4d9e596b1e578a9f483b32ccc8477653dfa560da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 18 Apr 2024 13:47:42 +0200 Subject: [PATCH 2/2] ci: add dependabot configuration This commit adds dependabot configuration to get updates for Dockerfiles and GitHub Actions. --- .github/dependabot.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..df945f8 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + labels: + - "dependencies" + commit-message: + prefix: "build" + include: "scope" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + labels: + - "dependencies" + commit-message: + prefix: "build" + include: "scope"