Skip to content

Commit

Permalink
pre-commit hook
Browse files Browse the repository at this point in the history
  • Loading branch information
cpelley committed Jul 12, 2024
1 parent 4ae9a3f commit b890199
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 98 deletions.
84 changes: 50 additions & 34 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ jobs:
fail-fast: false
matrix:
env: [environment_a, environment_b, conda-forge]

steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
id: cache
with:
path: /usr/share/miniconda/envs/im${{ matrix.env }}
key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}

- name: conda env update
if: steps.cache.outputs.cache-hit != 'true'
run: |
Expand All @@ -26,29 +28,34 @@ jobs:
conda install -q -n base -c conda-forge -c nodefaults mamba
rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
- name: conda info
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
conda info
conda list
- name: sphinx documentation
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
make -C doc html SPHINXOPTS="-W --keep-going"
- name: pytest without coverage
if: matrix.env == 'conda-forge'
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
pytest
- name: pytest with coverage
if: matrix.env != 'conda-forge'
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
pytest --cov=improver --cov-report xml:coverage.xml
- name: codacy upload
if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
run: |
Expand All @@ -57,76 +64,82 @@ jobs:
python-codacy-coverage -v -r coverage.xml
env:
CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}

- name: codecov upload
uses: codecov/codecov-action@v4
with:
name: ${{ matrix.env }}
if: matrix.env != 'conda_forge'
Codestyle-and-flake8:

pre-commit-checks:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
env: [environment_a]
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
id: cache

- name: improver checkout
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
path: /usr/share/miniconda/envs/im${{ matrix.env }}
key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
- name: conda env update
if: steps.cache.outputs.cache-hit != 'true'
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda update -q -n base -c defaults conda
conda install -q -n base -c conda-forge -c nodefaults mamba
rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
- name: conda info
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
conda info
conda list
- name: isort
python-version: 3.7
cache: 'pip'

- name: pip install pre-commit
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
isort --check-only .
- name: black
pip install pre-commit
- name: Python interpreter version sha (PYSHA)
run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV

- name: Cache pre-commit
uses: actions/cache@v3
id: pre-commit-cache
with:
path: ~/.cache/pre-commit
key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }}

- name: pre-commit install
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
black --check .
- name: flake8
pre-commit install
- name: pre-commit run
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
flake8 improver improver_tests
files_changed=$(git diff --name-status origin/master... | grep -E '^\s*(M|A)' | awk '{print $2}')
echo "Checking the following files:"
echo ${files_changed}
pre-commit run --verbose --color=always --files ${files_changed}
PR-standards:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:

- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Check CONTRIBUTING.md
uses: cylc/release-actions/check-shortlog@v1

Safety-Bandit:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
env: [environment_a, environment_b, conda-forge]
steps:

- uses: actions/checkout@v4

- uses: actions/cache@v4
id: cache
with:
path: /usr/share/miniconda/envs/im${{ matrix.env }}
key: ${{ format('{0}-conda-improver-{1}-{2}', runner.os, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}

- name: conda env update
if: steps.cache.outputs.cache-hit != 'true'
run: |
Expand All @@ -135,17 +148,20 @@ jobs:
conda install -q -n base -c conda-forge mamba
rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
mamba env create -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
- name: conda info
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
conda info
conda list
- name: safety
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
conda activate im${{ matrix.env }}
safety check || true
- name: bandit
run: |
source '/usr/share/miniconda/etc/profile.d/conda.sh'
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,10 @@ activate your new improver environment
```
conda activate improver
```

## Pre-commit Hook
OPTIONAL: A pre-commit hook can be added to facilitate the development of this code base.
Ensure that you have python available on the path, then install the pre-commit hook by running `pre-commit install` from within your working copy.
pre-commit checks will run against modified files when you commit from then on.

These pre-commit hooks will run as part of continuous integration to maintain standard in the project.
87 changes: 87 additions & 0 deletions copyright_check
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/bin/bash

copyright_header="\
# (C) Crown Copyright, Met Office. All rights reserved.\n\
#\n\
# This file is part of IMPROVER and is released under the BSD 3-Clause license.\n\
# See LICENSE in the root of the repository for full licensing details.\
"

show_help() {
echo "info: Check the copyright header in the specified files."
echo " If the non-empty file does not contain the copyright header, it "
echo " will be added. If a file looks like it does contain a copyright "
echo " header, it will verified that it matches exactly."
echo " Hash bangs are preserved."
echo ''
echo "usage: $0 FILEPATH..."
echo ''
echo 'positional arguments:'
echo ' FILEPATH... The file(s) to check'
echo ""
echo "options:"
echo " -h|--help Show this help message and exit"
echo ""
}

if [[ " $@ " =~ " --help " || " $@ " =~ " -h " ]]; then
show_help
exit 0
fi

contains_copyright() {
grep -qi '# .*copyright' $1
}


defines_hash_bang() {
grep -q '^#!' $1
}


correct_copyright() {
python -c 'import sys; sys.exit(0 if "'"$1"'" in open("'"$2"'", "r").read() else 1)'
}


# Function to delete temporary files
cleanup() {
for filepath in "$@"; do
tmp_file="${filepath}.tmp"
if [ -f "$tmp_file" ]; then
echo "Deleting temporary file: $tmp_file"
rm -f "$tmp_file" 2&> /dev/null
fi
done
}

# Trap the EXIT signal to call the cleanup function
trap 'cleanup "$@"' EXIT

found_error=0
for filepath in "$@"; do
if contains_copyright "${filepath}"; then
#echo "File '${filepath}' already contains Crown Copyright"
if correct_copyright "${copyright_header}" "${filepath}"; then
continue
else
echo "Incorrect Copyright header in '${filepath}'"
found_error=1
fi
elif [ -s "${filepath}" ]; then # skip empty files
echo "Adding missing Copyright to '${filepath}'"
tmp_file="${filepath}.tmp"
if defines_hash_bang "${filepath}"; then
head -n 1 ${filepath} > ${tmp_file}
echo -e ${copyright_header} >> ${tmp_file}
tail -n +2 ${filepath} >> ${tmp_file}
else
echo -e ${copyright_header} > ${tmp_file}
cat ${filepath} >> ${tmp_file}
fi
mv ${tmp_file} ${filepath}
found_error=1
fi
done

exit ${found_error}
56 changes: 0 additions & 56 deletions improver_tests/test_source_code.py

This file was deleted.

21 changes: 21 additions & 0 deletions init_check
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash

found_error=0
script_root=$(dirname "$0")

for dirpath in "$@"; do
if [[ "$dirpath" != improver* && "$dirpath" != improver_tests* ]]; then
continue
fi
if [ ! -d "$dirpath" ]; then
dirpath=$(dirname "$dirpath")
fi
if [ ! -f "$dirpath/__init__.py" ]; then
found_error=1
echo "Missing __init__.py in $dirpath, adding it now..."
touch "$dirpath/__init__.py"
fi
done

exit ${found_error}

16 changes: 8 additions & 8 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
[tool.black]
target-version = ['py36']
[tool.ruff]
target-version = "py36"

[tool.isort]
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
line_length = 88
[tool.ruff.lint]
extend-select = ["E", "F", "W", "I"]

[tool.ruff.format]
# Enable reformatting of code snippets in docstrings.
docstring-code-format = true

0 comments on commit b890199

Please sign in to comment.