From b890199dc24f8cff2b326335a2bd0f0699e59c6f Mon Sep 17 00:00:00 2001 From: cpelley Date: Fri, 12 Jul 2024 11:24:12 +0100 Subject: [PATCH] pre-commit hook --- .github/workflows/ci.yml | 84 +++++++++++++++++------------ README.md | 7 +++ copyright_check | 87 ++++++++++++++++++++++++++++++ improver_tests/test_source_code.py | 56 ------------------- init_check | 21 ++++++++ pyproject.toml | 16 +++--- 6 files changed, 173 insertions(+), 98 deletions(-) create mode 100755 copyright_check delete mode 100644 improver_tests/test_source_code.py create mode 100755 init_check diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 574d0e32a5..97e1269a03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ jobs: fail-fast: false matrix: env: [environment_a, environment_b, conda-forge] + steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -18,6 +19,7 @@ jobs: with: path: /usr/share/miniconda/envs/im${{ matrix.env }} key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | @@ -26,29 +28,34 @@ jobs: conda install -q -n base -c conda-forge -c nodefaults mamba rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488 mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} conda info conda list + - name: sphinx documentation run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} make -C doc html SPHINXOPTS="-W --keep-going" + - name: pytest without coverage if: matrix.env == 'conda-forge' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} pytest + - name: pytest with coverage if: matrix.env != 'conda-forge' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} pytest --cov=improver --cov-report xml:coverage.xml + - name: codacy upload if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a' run: | @@ -57,63 +64,66 @@ jobs: python-codacy-coverage -v -r coverage.xml env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + - name: codecov upload uses: codecov/codecov-action@v4 with: name: ${{ matrix.env }} if: matrix.env != 'conda_forge' - Codestyle-and-flake8: + + pre-commit-checks: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - env: [environment_a] steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - id: cache + + - name: improver checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 with: - path: /usr/share/miniconda/envs/im${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - - name: conda env update - if: steps.cache.outputs.cache-hit != 'true' - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda update -q -n base -c defaults conda - conda install -q -n base -c conda-forge -c nodefaults mamba - rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488 - mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - - name: conda info - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate im${{ matrix.env }} - conda info - conda list - - name: isort + python-version: 3.7 + cache: 'pip' + + - name: pip install pre-commit run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate im${{ matrix.env }} - isort --check-only . - - name: black + pip install pre-commit + + - name: Python interpreter version sha (PYSHA) + run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + + - name: Cache pre-commit + uses: actions/cache@v3 + id: pre-commit-cache + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: pre-commit install run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate im${{ matrix.env }} - black --check . - - name: flake8 + pre-commit install + + - name: pre-commit run run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate im${{ matrix.env }} - flake8 improver improver_tests + files_changed=$(git diff --name-status origin/master... | grep -E '^\s*(M|A)' | awk '{print $2}') + echo "Checking the following files:" + echo ${files_changed} + pre-commit run --verbose --color=always --files ${files_changed} + PR-standards: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Check CONTRIBUTING.md uses: cylc/release-actions/check-shortlog@v1 + Safety-Bandit: runs-on: ubuntu-latest strategy: @@ -121,12 +131,15 @@ jobs: matrix: env: [environment_a, environment_b, conda-forge] steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 id: cache with: path: /usr/share/miniconda/envs/im${{ matrix.env }} key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | @@ -135,17 +148,20 @@ jobs: conda install -q -n base -c conda-forge mamba rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488 mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} conda info conda list + - name: safety run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda activate im${{ matrix.env }} safety check || true + - name: bandit run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' diff --git a/README.md b/README.md index 670c475abd..2de74d8d6b 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,10 @@ activate your new improver environment ``` conda activate improver ``` + +## Pre-commit Hook +OPTIONAL: A pre-commit hook can be added to facilitate the development of this code base. +Ensure that you have python available on the path, then install the pre-commit hook by running `pre-commit install` from within your working copy. +pre-commit checks will run against modified files when you commit from then on. + +These pre-commit hooks will run as part of continuous integration to maintain standard in the project. \ No newline at end of file diff --git a/copyright_check b/copyright_check new file mode 100755 index 0000000000..c39fdf8ed6 --- /dev/null +++ b/copyright_check @@ -0,0 +1,87 @@ +#!/bin/bash + +copyright_header="\ +# (C) Crown Copyright, Met Office. All rights reserved.\n\ +#\n\ +# This file is part of IMPROVER and is released under the BSD 3-Clause license.\n\ +# See LICENSE in the root of the repository for full licensing details.\ +" + +show_help() { + echo "info: Check the copyright header in the specified files." + echo " If the non-empty file does not contain the copyright header, it " + echo " will be added. If a file looks like it does contain a copyright " + echo " header, it will verified that it matches exactly." + echo " Hash bangs are preserved." + echo '' + echo "usage: $0 FILEPATH..." + echo '' + echo 'positional arguments:' + echo ' FILEPATH... The file(s) to check' + echo "" + echo "options:" + echo " -h|--help Show this help message and exit" + echo "" +} + +if [[ " $@ " =~ " --help " || " $@ " =~ " -h " ]]; then + show_help + exit 0 +fi + +contains_copyright() { + grep -qi '# .*copyright' $1 +} + + +defines_hash_bang() { + grep -q '^#!' $1 +} + + +correct_copyright() { + python -c 'import sys; sys.exit(0 if "'"$1"'" in open("'"$2"'", "r").read() else 1)' +} + + +# Function to delete temporary files +cleanup() { + for filepath in "$@"; do + tmp_file="${filepath}.tmp" + if [ -f "$tmp_file" ]; then + echo "Deleting temporary file: $tmp_file" + rm -f "$tmp_file" 2&> /dev/null + fi + done +} + +# Trap the EXIT signal to call the cleanup function +trap 'cleanup "$@"' EXIT + +found_error=0 +for filepath in "$@"; do + if contains_copyright "${filepath}"; then + #echo "File '${filepath}' already contains Crown Copyright" + if correct_copyright "${copyright_header}" "${filepath}"; then + continue + else + echo "Incorrect Copyright header in '${filepath}'" + found_error=1 + fi + elif [ -s "${filepath}" ]; then # skip empty files + echo "Adding missing Copyright to '${filepath}'" + tmp_file="${filepath}.tmp" + if defines_hash_bang "${filepath}"; then + head -n 1 ${filepath} > ${tmp_file} + echo -e ${copyright_header} >> ${tmp_file} + tail -n +2 ${filepath} >> ${tmp_file} + else + echo -e ${copyright_header} > ${tmp_file} + cat ${filepath} >> ${tmp_file} + fi + mv ${tmp_file} ${filepath} + found_error=1 + fi +done + +exit ${found_error} \ No newline at end of file diff --git a/improver_tests/test_source_code.py b/improver_tests/test_source_code.py deleted file mode 100644 index 562071db93..0000000000 --- a/improver_tests/test_source_code.py +++ /dev/null @@ -1,56 +0,0 @@ -# (C) Crown copyright, Met Office. All rights reserved. -# -# This file is part of IMPROVER and is released under a BSD 3-Clause license. -# See LICENSE in the root of the repository for full licensing details. -"""Checks on source code files.""" - -from pathlib import Path - -TOP_LEVEL_DIR = (Path(__file__).parent / "..").resolve() -DIRECTORIES_COVERED = [TOP_LEVEL_DIR / "improver", TOP_LEVEL_DIR / "improver_tests"] - - -def self_licence(): - """Collect licence text from this file""" - self_lines = Path(__file__).read_text().splitlines() - licence_lines = list() - for line in self_lines: - if not line.startswith("#"): - break - licence_lines.append(line) - licence = "\n".join(licence_lines) - return licence - - -def test_py_licence(): - """ - Check that non-empty python files contain 3-clause BSD licence text - """ - failed_files = [] - licence_text = self_licence() - for directory in DIRECTORIES_COVERED: - for file in directory.glob("**/*.py"): - contents = file.read_text() - # skip zero-byte empty files such as __init__.py - if len(contents) > 0 and licence_text not in contents: - failed_files.append(str(file)) - assert len(failed_files) == 0, "\n".join(failed_files) - - -def test_init_files_exist(): - """Check for missing __init__.py files.""" - failed_directories = [] - for directory in DIRECTORIES_COVERED: - for path in directory.glob("**"): - if not path.is_dir(): - continue - # ignore hidden directories and their sub-directories - if any([part.startswith(".") for part in path.parts]): - continue - # in-place running will produce pycache directories, these should be ignored - if path.name == "__pycache__": - continue - expected_init = path / "__init__.py" - if not expected_init.exists(): - failed_directories.append(str(path)) - assert len(failed_directories) == 0, "\n".join(failed_directories) diff --git a/init_check b/init_check new file mode 100755 index 0000000000..5990c951c2 --- /dev/null +++ b/init_check @@ -0,0 +1,21 @@ +#!/bin/bash + +found_error=0 +script_root=$(dirname "$0") + +for dirpath in "$@"; do + if [[ "$dirpath" != improver* && "$dirpath" != improver_tests* ]]; then + continue + fi + if [ ! -d "$dirpath" ]; then + dirpath=$(dirname "$dirpath") + fi + if [ ! -f "$dirpath/__init__.py" ]; then + found_error=1 + echo "Missing __init__.py in $dirpath, adding it now..." + touch "$dirpath/__init__.py" + fi +done + +exit ${found_error} + diff --git a/pyproject.toml b/pyproject.toml index 1c8a595819..d6e6b601cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ -[tool.black] -target-version = ['py36'] +[tool.ruff] +target-version = "py36" -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -line_length = 88 +[tool.ruff.lint] +extend-select = ["E", "F", "W", "I"] + +[tool.ruff.format] +# Enable reformatting of code snippets in docstrings. +docstring-code-format = true \ No newline at end of file