Skip to content

Commit

Permalink
Merge branch 'develop' into renovate/metabase-metabase-0.x
Browse files Browse the repository at this point in the history
  • Loading branch information
jamil314 authored Oct 31, 2024
2 parents 2662940 + 02b2eee commit d4edcf4
Show file tree
Hide file tree
Showing 117 changed files with 5,471 additions and 4,172 deletions.
84 changes: 76 additions & 8 deletions .github/workflows/build-images-from-branch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,14 @@ on:
description: Branch to build from
default: develop
required: true
push:
pull_request:
branches-ignore:
- 'dependabot/**'
- 'renovate/**'
push:
branches:
- develop
- main
jobs:
base:
runs-on: ubuntu-22.04
Expand All @@ -30,18 +34,29 @@ jobs:

- uses: actions/checkout@v4
if: github.event_name == 'push'
with:
ref: ${{ github.event.pull_request.head.sha }}

- uses: actions/checkout@v4
if: github.event_name == 'pull_request'
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Set version and branch
id: set-version-and-branch
run: |
export VERSION=`git log -1 --pretty=format:%h`
echo "Pushing version $VERSION"
echo "version=$VERSION" >> $GITHUB_OUTPUT
if [ "${{ github.event_name }}" == 'push' ]; then
BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
elif [ "${{ github.event_name }}" == 'pull_request' ]; then
BRANCH=${{ github.event.pull_request.head.ref }}
else
BRANCH=${{ inputs.branch_name }}
BRANCH=${{ inputs.branch_name }}
fi
ESCAPED_BRANCH=$(echo $BRANCH | sed 's/[^a-zA-Z0-9_.-]/-/g')
echo "from branch $BRANCH"
echo "branch=$ESCAPED_BRANCH" >> $GITHUB_OUTPUT
- name: Get list of services
Expand Down Expand Up @@ -90,7 +105,7 @@ jobs:
ref: '${{ github.event.inputs.branch_name }}'

- uses: actions/checkout@v4
if: github.event_name == 'push'
if: github.event_name == 'push' || github.event_name == 'pull_request'

- name: Login to DockerHub
uses: docker/login-action@v3
Expand All @@ -113,11 +128,64 @@ jobs:
cache-from: type=registry,ref=opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
cache-to: type=inline

security-scans-pr:
needs: [build, base]
runs-on: ubuntu-22.04
if: github.event_name == 'pull_request'
strategy:
matrix:
service: ${{ fromJSON(needs.base.outputs.services) }}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: |
trivy.yaml
.trivyignore.yaml
sparse-checkout-cone-mode: false
- name: Gather Trivy output from base branch image
uses: aquasecurity/[email protected]
with:
image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ github.event.pull_request.base.ref }}'
trivy-config: trivy.yaml
format: 'sarif'
output: './trivy-results-base.sarif'
exit-code: '0'

- name: Gather Trivy output from newly build image
uses: aquasecurity/[email protected]
with:
image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
trivy-config: trivy.yaml
format: 'sarif'
output: './trivy-results-branch.sarif'
exit-code: '0'

- name: Remove lines that are always expected to be different
run: |
jq '.runs |= map(del(.originalUriBaseIds, .properties))' ${{ github.workspace }}/trivy-results-base.sarif > ${{ github.workspace }}/trivy-results-base.sarif
jq '.runs |= map(del(.originalUriBaseIds, .properties))' ${{ github.workspace }}/trivy-results-branch.sarif > ${{ github.workspace }}/trivy-results-branch.sarif
- name: Diff Trivy results to catch newly introduced vulnerabilities
run: diff -u ./trivy-results-base.sarif ./trivy-results-branch.sarif

security-scans-develop:
needs: [build, base]
runs-on: ubuntu-22.04
if: ${{ needs.base.outputs.branch == 'develop' }}
strategy:
fail-fast: false
matrix:
service: ${{ fromJSON(needs.base.outputs.services) }}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: |
trivy.yaml
.trivyignore.yaml
sparse-checkout-cone-mode: false

- name: Run Trivy vulnerability scanner
# Skip temporarily for non-develop branches because of
# a TOOMANYREQUESTS error failing the check
# https://github.com/aquasecurity/trivy/discussions/7591
uses: aquasecurity/[email protected]
uses: aquasecurity/[email protected]
with:
image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
trivy-config: trivy.yaml
13 changes: 0 additions & 13 deletions .github/workflows/lint-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,16 +137,3 @@ jobs:
- name: Run Unit Test
if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-test != 'true'
run: cd ${{ matrix.package }} && yarn test

security-scans:
needs: setup
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Run Trivy vulnerability scanner in fs mode
uses: aquasecurity/[email protected]
with:
scan-type: 'fs'
scan-ref: '.'
trivy-config: trivy.yaml
78 changes: 78 additions & 0 deletions .github/workflows/mirror-trivy-db.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
#
# OpenCRVS is also distributed under the terms of the Civil Registration
# & Healthcare Disclaimer located at http://opencrvs.org/license.
#
# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
name: Mirror Docker Images to GHCR

on:
schedule:
# Run 30 mins after trivy DB runs (the trivy job takes 15 mins max)
# https://github.com/aquasecurity/trivy-db/blob/cfa337a1088bbcee598ab93656c83fe6b9acb946/.github/workflows/cron.yml#L5
# https://github.com/aquasecurity/trivy-db/actions
- cron: '30 */6 * * *'

workflow_dispatch: # Allows manual triggering of the workflow

jobs:
mirror-dbs:
runs-on: ubuntu-latest
env:
RETRIES: 100

steps:
- name: Install Skopeo
run: |
sudo apt-get update
sudo apt-get install -y skopeo
- name: Checkout repository
uses: actions/checkout@v3

- name: Log in to GHCR
run: echo "${{ secrets.GITHUB_TOKEN }}" | skopeo login ghcr.io -u ${{ github.actor }} --password-stdin

- name: Create mirror-image.sh script
run: |
cat << 'EOF' > mirror-image.sh
#!/usr/bin/env bash
set -o errexit -o nounset -o pipefail
if [ "$#" -ne 3 ]; then
echo "Usage: $0 <source_image> <destination_image> <retries>"
exit 1
fi
SOURCE_IMAGE=$1
DESTINATION_IMAGE=$2
RETRIES=$3
COUNT=0
until skopeo copy "$SOURCE_IMAGE" "$DESTINATION_IMAGE" || [ $COUNT -ge $RETRIES ]; do
COUNT=$((COUNT+1))
echo "Retry $COUNT/$RETRIES for $SOURCE_IMAGE to $DESTINATION_IMAGE..."
sleep 1
done
if [ $COUNT -ge $RETRIES ]; then
echo "Failed to mirror $SOURCE_IMAGE after $RETRIES attempts."
exit 1
fi
echo "Successfully mirrored $SOURCE_IMAGE to $DESTINATION_IMAGE."
EOF
- name: Make mirror-image.sh executable
run: chmod +x mirror-image.sh

- name: Mirror trivy-db to GHCR
run: ./mirror-image.sh docker://ghcr.io/aquasecurity/trivy-db:2 docker://ghcr.io/${{ github.repository_owner }}/trivy-db:2 ${{ env.RETRIES }}

- name: Mirror trivy-java-db to GHCR
run: ./mirror-image.sh docker://ghcr.io/aquasecurity/trivy-java-db:1 docker://ghcr.io/${{ github.repository_owner }}/trivy-java-db:1 ${{ env.RETRIES }}

- name: Mirror trivy-checks to GHCR
run: ./mirror-image.sh docker://ghcr.io/aquasecurity/trivy-checks:1 docker://ghcr.io/${{ github.repository_owner }}/trivy-checks:1 ${{ env.RETRIES }}
2 changes: 1 addition & 1 deletion .github/workflows/publish-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ jobs:
.trivyignore.yaml
sparse-checkout-cone-mode: false
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.24.0
uses: aquasecurity/trivy-action@0.28.0
with:
image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
trivy-config: trivy.yaml
91 changes: 91 additions & 0 deletions .github/workflows/security-scans.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
#
# OpenCRVS is also distributed under the terms of the Civil Registration
# & Healthcare Disclaimer located at http://opencrvs.org/license.
#
# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
name: Scan files for vulnerabilities

on:
push:
branches:
- develop
pull_request:
schedule:
# Run every day at midnight
- cron: '0 0 * * *'
jobs:
security-scan-scheduled:
if: github.event_name == 'schedule'
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: 'develop'

- name: Run Trivy vulnerability scanner in fs mode
uses: aquasecurity/[email protected]
with:
scan-type: 'fs'
scan-ref: '.'
trivy-config: trivy.yaml

security-scan-development:
if: github.event_name == 'push' && github.ref == 'refs/heads/develop'
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Run Trivy vulnerability scanner in fs mode
uses: aquasecurity/[email protected]
with:
scan-type: 'fs'
scan-ref: '.'
trivy-config: trivy.yaml

security-scan-pull-request:
if: github.event_name == 'pull_request'
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }}
path: 'base'

- name: Checkout code
uses: actions/checkout@v4
with:
path: 'branch'

- name: Gather Trivy output from base branch
uses: aquasecurity/[email protected]
with:
scan-type: 'fs'
scan-ref: 'base'
trivy-config: ./branch/trivy.yaml
format: 'sarif'
output: './trivy-results-base.sarif'
exit-code: '0'

- name: Gather Trivy output from PR branch
uses: aquasecurity/[email protected]
with:
scan-type: 'fs'
scan-ref: 'branch'
trivy-config: ./branch/trivy.yaml
format: 'sarif'
output: './trivy-results-branch.sarif'
exit-code: '0'

- name: Remove lines that are always expected to be different
run: |
jq '.runs |= map(del(.originalUriBaseIds, .properties))' ./trivy-results-base.sarif > ./trivy-results-base.sarif
jq '.runs |= map(del(.originalUriBaseIds, .properties))' ./trivy-results-branch.sarif > ./trivy-results-branch.sarif
- name: Diff Trivy results to catch newly introduced vulnerabilities
run: diff -u ./trivy-results-base.sarif ./trivy-results-branch.sarif
7 changes: 5 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@
- Two new statuses of record are added: `Validated` and `Correction Requested` for advanced search parameters [#6365](https://github.com/opencrvs/opencrvs-core/issues/6365)
- A new field: `Time Period` is added to advanced search [#6365](https://github.com/opencrvs/opencrvs-core/issues/6365)
- Deploy UI-Kit Storybook to [opencrvs.pages.dev](https://opencrvs.pages.dev) to allow extending OpenCRVS using the component library
- Record audit action buttons are moved into action menu [#7390](https://github.com/opencrvs/opencrvs-core/issues/7390)
- Reoder the sytem user add/edit field for surname to be first, also change labels from `Last name` to `User's surname` and lastly remove the NID question from the form [#6830](https://github.com/opencrvs/opencrvs-core/issues/6830)
- Auth now allows exchanging user's token for a new record-specific token [#7728](https://github.com/opencrvs/opencrvs-core/issues/7728)

## Bug fixes

Expand Down Expand Up @@ -70,13 +72,14 @@
- Fix hardcoded placeholder copy of input when saving a query in advanced search
- Handle label params used in form inputs when rendering in action details modal
- **Staged files getting reset on precommit hook failure** We were running lint-staged separately on each package using lerna which potentially created a race condition causing staged changes to get lost on failure. Now we are running lint-staged directly without depending on lerna. **_This is purely a DX improvement without affecting any functionality of the system_**
- Fix `informantType` missing in template object which prevented rendering informant relationship data in the certificates [#5952](https://github.com/opencrvs/opencrvs-core/issues/5952)

### Breaking changes

- Remove informant notification configuration from the UI and read notification configuration settings from `record-notification` endpoint in countryconfig
- **Gateways searchEvents API updated** `operationHistories` only returns `operationType` & `operatedOn` due to the other fields being unused in OpenCRVS
- **Config changes to review/preview and signatures** Core used to provide review/preview section by default which are now removed and need to be provided from countryconfig. The signature field definitions (e.g. informant signature, bride signature etc.) were hard coded in core which also have now been removed. The signatures can now be added through the review/preview sections defined in countryconfig just like any other field. You can use the following section definition as the default which is without any additional fields. We highly recommend checking out our reference country repository which has the signature fields in it's review/preview sections

- **Config changes to review/preview and signatures** Core used to provide review/preview section by default which are now removed and need to be provided from countryconfig. The signature field definitions (e.g. informant signature, bride signature etc.) were hard coded in core which also have now been removed. The signatures can now be added through the review/preview sections defined in countryconfig just like any other field. You can use the following section definition as the default which is without any additional fields. We highly recommend checking out our reference country repository which has the signature fields in its review/preview sections
- `hasChildLocation` query has been removed from gateway. We have created the query `isLeafLevelLocation` instead which is more descriptive on its intended use.
```
{
id: 'preview',
Expand Down
Loading

0 comments on commit d4edcf4

Please sign in to comment.