Release v1.6.0

.dockerignore

@@ -5,4 +5,4 @@
 **/*.log
 .secrets/*
 data
-grafana
+grafana

.eslintrc.js

@@ -48,6 +48,7 @@ module.exports = {
     'import/no-named-as-default': 'off',
     'import/no-relative-parent-imports': 2,
     'import/named': 0,
+    'import/no-duplicates': 'error',
     '@typescript-eslint/explicit-function-return-type': 'off',
     '@typescript-eslint/no-non-null-assertion': 'off',
     '@typescript-eslint/no-unused-vars': 'off',

.github/workflows/auto-pr-to-release.yml

@@ -0,0 +1,172 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+
+name: Auto PR to Release Branch
+
+on:
+  pull_request:
+    types: [closed]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to process'
+        required: true
+        default: ''
+
+jobs:
+  create-pr:
+    if: ${{ github.event_name == 'pull_request' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get PR details from workflow dispatch
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: get_pr_details_dispatch
+        run: |
+          PR_NUMBER=${{ github.event.inputs.pr_number }}
+          PR_DATA=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/opencrvs/opencrvs-core/pulls/$PR_NUMBER)
+          # printf escapes the newlines in the JSON, so we can use jq to parse output such as:
+          # "body": "![image](https://github.com/user-attachments/assets/8eee5bcf-7692-490f-a19f-576623e09961)\r\n",
+          echo "PR_ID=$(printf '%s' $PR_DATA | jq -r '.number')" >> $GITHUB_ENV
+          echo "PR_AUTHOR=$(printf '%s' $PR_DATA | jq -r '.user.login')" >> $GITHUB_ENV
+          echo "PR_MERGER=$(printf '%s' $PR_DATA | jq -r '.merged_by.login')" >> $GITHUB_ENV
+          echo "MILESTONE=$(printf '%s' $PR_DATA | jq -r '.milestone.title')" >> $GITHUB_ENV
+          echo "BASE_BRANCH=$(printf '%s' $PR_DATA | jq -r '.base.ref')" >> $GITHUB_ENV
+          echo "HEAD_BRANCH=$(printf '%s' $PR_DATA | jq -r '.head.ref')" >> $GITHUB_ENV
+          echo "PR_TITLE=$(printf '%s' $PR_DATA | jq -r '.title')" >> $GITHUB_ENV
+          echo "BASE_SHA=$(printf '%s' $PR_DATA | jq -r '.base.sha')" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get PR details from event
+        if: ${{ github.event_name == 'pull_request' }}
+        id: get_pr_details_event
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          echo "PR_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+          echo "PR_AUTHOR=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV
+          echo "MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
+          echo "BASE_BRANCH=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
+          echo "HEAD_BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
+          echo "PR_TITLE=${{ github.event.pull_request.title }}" >> $GITHUB_ENV
+          echo "BASE_SHA=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV
+
+          PR_DETAILS=$(gh pr view $PR_NUMBER --json mergedBy)
+          MERGED_BY_LOGIN=$(echo "$PR_DETAILS" | jq -r '.mergedBy.login')
+          echo "PR_MERGER=$MERGED_BY_LOGIN" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check for milestone and if release branch exists
+        continue-on-error: true
+        id: check_release_branch
+        run: |
+          if [ -z "${{ env.MILESTONE }}" ]; then
+            echo "No milestone set. Exiting."
+            exit 1
+          fi
+
+          RELEASE_BRANCH="release-${{ env.MILESTONE }}"
+
+          # Check if the release branch exists
+          if git ls-remote --heads origin $RELEASE_BRANCH | grep -q "refs/heads/$RELEASE_BRANCH"; then
+            echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV
+          else
+            echo "Release branch $RELEASE_BRANCH does not exist. Exiting."
+            exit 1
+          fi
+
+      - name: Create and push the new branch for the PR
+        if: ${{ steps.check_release_branch.outcome == 'success' }}
+        run: |
+          SEMANTIC_PR_TITLE="${{ env.PR_TITLE }}"
+
+          # Check for semantic prefix
+          if [[ $SEMANTIC_PR_TITLE =~ ^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert|wip|merge)\: ]]; then
+            SEMANTIC_PR_TITLE="${BASH_REMATCH[1]}(${MILESTONE}): ${SEMANTIC_PR_TITLE#*: }"
+          else
+            SEMANTIC_PR_TITLE="🍒 Merge changes from PR #${{ env.PR_ID }} to ${{ env.RELEASE_BRANCH }}"
+          fi
+
+          PR_BODY="Automated PR to merge changes from develop to ${{ env.RELEASE_BRANCH }}"
+
+          # Configure git
+          git config user.name "github-actions"
+          git config user.email "[email protected]"
+          git config advice.mergeConflict false
+
+          # Fetch and checkout the release branch
+          git fetch --all --unshallow
+          git checkout ${{ env.RELEASE_BRANCH }}
+
+          # Create a new branch for the PR
+          NEW_BRANCH="auto-pr-${{ env.RELEASE_BRANCH }}-${{ env.PR_ID }}-$RANDOM"
+          git checkout -b $NEW_BRANCH
+
+          echo "HEAD_BRANCH: ${{ env.HEAD_BRANCH }}"
+          echo "BASE_SHA: ${{ env.BASE_SHA }}"
+
+          COMMIT_RANGE="${{ env.BASE_SHA }}..origin/${{ env.HEAD_BRANCH }}"
+
+          echo "Commit range: ${COMMIT_RANGE}"
+
+          NON_MERGE_COMMITS=$(git log ${COMMIT_RANGE} --reverse --no-merges --pretty=format:"%h" -- | xargs)
+
+          echo "Ordered non-merge commits: $NON_MERGE_COMMITS"
+
+          # Attempt to cherry-pick the commits from the original PR
+          CHERRY_PICK_OUTPUT=$(git cherry-pick ${NON_MERGE_COMMITS} 2>&1) || {
+            git cherry-pick --abort || true
+            # If cherry-pick fails, create a placeholder commit
+            echo "Cherry-pick failed. Creating placeholder commit."
+
+            git reset --hard
+            git commit --allow-empty -m "Placeholder commit for PR #${{ env.PR_ID }}"
+
+            # Add manual cherry-pick commands to the PR body
+            PR_BODY="${PR_BODY}
+
+            **I failed to cherry-pick the changes automatically because of the following:**
+
+            \`\`\`
+            $CHERRY_PICK_OUTPUT
+            \`\`\`
+
+            **To continue manually you can use these commands:**
+            \`\`\`
+            git fetch origin $NEW_BRANCH:$NEW_BRANCH
+            git fetch origin ${{ env.HEAD_BRANCH }}:${{ env.HEAD_BRANCH }}
+
+            git checkout $NEW_BRANCH
+            git reset --hard HEAD~1  # Remove placeholder commit
+            git cherry-pick $NON_MERGE_COMMITS
+            \`\`\`
+
+            "
+          }
+
+          # Push the new branch
+          git push origin $NEW_BRANCH
+
+          # Create a pull request and assign the original PR author as the reviewer
+          AUTHOR=${{ env.PR_AUTHOR }}
+
+          if [[ $AUTHOR == *renovate* ]]; then
+            if [ -z "${{ env.PR_MERGER }}" ]; then
+              AUTHOR=""
+            else
+              AUTHOR=${{ env.PR_MERGER }}
+            fi
+          fi
+          gh pr create --title "$SEMANTIC_PR_TITLE" --body "$PR_BODY" --head "$NEW_BRANCH" --base "${{ env.RELEASE_BRANCH }}" --reviewer "$AUTHOR"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-images-from-branch.yml

@@ -32,28 +32,51 @@ jobs:
       - uses: actions/checkout@v4
         if: github.event_name == 'push'
 
-      - name: Set version
-        id: set-version
+      - name: Set version and branch
+        id: set-version-and-branch
         run: |
           export VERSION=`git log -1 --pretty=format:%h`
           echo "version=$VERSION" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name }}" == 'push' ]; then
+            echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+          else
+            echo "branch=${{ inputs.branch_name }}" >> $GITHUB_OUTPUT
+          fi
 
       - name: Get list of services
         id: get-services
         run: |
-          services=$(grep "^  [^ ]" docker-compose.yml | grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
+          services=$(grep "^  [^ ]" docker-compose.yml | grep -v base| grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
           services_json=$(echo $services | tr '\n' ',' | sed 's/,$//' | jq -R 'split(" ")' | tr -d '\n')
           # Set the list of service names as an output variable
           echo "services=$services_json" >> $GITHUB_OUTPUT
           echo "services=$services_json"
 
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v5
+        with:
+          file: packages/Dockerfile.base
+          context: .
+          push: true
+          tags: 'opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.version }}'
+          cache-from: type=registry,ref=opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.version }}
+          cache-to: type=inline
+
     outputs:
       services: ${{ steps.get-services.outputs.services }}
-      version: ${{ steps.set-version.outputs.version }}
+      version: ${{ steps.set-version-and-branch.outputs.version }}
+      branch: ${{ steps.set-version-and-branch.outputs.branch }}
 
   build:
     needs: base
     strategy:
+      fail-fast: false
       matrix:
         service: ${{ fromJSON(needs.base.outputs.services) }}
     runs-on: ubuntu-22.04
@@ -67,35 +90,25 @@ jobs:
         if: github.event_name == 'push'
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build ${{ matrix.service }}
-        run: |
-          export VERSION="${{ needs.base.outputs.version }}"
-          echo ${{ matrix.service }}
-          docker compose build ${{ matrix.service }}
-          docker compose push ${{ matrix.service }}
-
-  security-scans:
-    needs: [base, build]
-    strategy:
-      fail-fast: false
-      matrix:
-        service: ${{ fromJSON(needs.base.outputs.services) }}
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - name: Build and push
+        uses: docker/build-push-action@v5
         with:
-          sparse-checkout: |
-            trivy.yaml
-            .trivyignore.yaml
-          sparse-checkout-cone-mode: false
+          file: packages/${{ matrix.service }}/Dockerfile
+          build-args: |
+            VERSION=${{ needs.base.outputs.version }}
+          push: true
+          context: .
+          tags: |
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
+
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

.github/workflows/lint-and-test.yml

@@ -26,7 +26,7 @@ jobs:
         run: |
           PACKAGES=$(ls -d packages/* | jq -R -s -c 'split("\n")[:-1]')
           echo "Found packages: $PACKAGES"
-          echo "::set-output name=matrix::${PACKAGES}"
+          echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT
 
   test:
     needs: setup
@@ -45,31 +45,30 @@ jobs:
         run: |
           if [ ! -f "${{ matrix.package }}/package.json" ]; then
             echo "No package.json found for ${{ matrix.package }}. Stopping pipeline."
-            echo "::set-output name=skip::true"
+            echo "skip=true" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=skip::false"
+            echo "skip=false" >> $GITHUB_OUTPUT
 
             if ! grep -q "\"test\":" "${{ matrix.package }}/package.json"; then
               echo "Test not found in ${{ matrix.package }}"
-              echo "::set-output name=skip-test::true"
+              echo "skip-test=true" >> $GITHUB_OUTPUT
             else
-              echo "::set-output name=skip-test::false"
+              echo "skip=false" >> $GITHUB_OUTPUT
             fi
 
             if ! grep -q "\"lint\":" "${{ matrix.package }}/package.json"; then
               echo "Lint scripts not found in ${{ matrix.package }}. Stopping pipeline."
-              echo "::set-output name=skip-lint::true"
+              echo "skip-lint=true" >> $GITHUB_OUTPUT
             else
-              echo "::set-output name=skip-lint::false"
+              echo "skip-lint=false" >> $GITHUB_OUTPUT
             fi
           fi
 
-      - name: Use Node.js 18.19
+      - name: Use Node.js from .nvmrc
         if: steps.check-scripts.outputs.skip != 'true'
         uses: actions/setup-node@v4
         with:
-          node-version: '18.19'
-          cache: 'npm'
+          node-version-file: .nvmrc
 
       - name: Remove other package directories
         if: steps.check-scripts.outputs.skip != 'true'
@@ -85,14 +84,25 @@ jobs:
             fi
           done
 
+      - name: Cache Node.js dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            **/node_modules
+            ~/.cache/yarn/v6
+          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}',matrix.package,'package.json')) }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
       - name: Verify every file has a license header
         if: steps.check-scripts.outputs.skip != 'true'
         run: npx license-check-and-add check -f license-config.json
 
       - name: Runs dependency installation
         if: steps.check-scripts.outputs.skip != 'true'
-        run: CI="" yarn
+        run: CI="" yarn install --frozen-lockfile
 
+      # TODO: Move out of the matrix to be built once and shared
       - name: Build common package
         if: steps.check-scripts.outputs.skip != 'true'
         run: cd packages/commons && yarn build
@@ -104,6 +114,7 @@ jobs:
             cd packages/components && yarn build
           fi
 
+      # TODO: should run parallel to unit tests as can take as much as unit tests
       - name: Run linting
         if: steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-lint != 'true'
         run: cd ${{ matrix.package }} && yarn lint
@@ -119,7 +130,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner in fs mode
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           scan-type: 'fs'
           scan-ref: '.'

.github/workflows/publish-release.yml

@@ -102,7 +102,7 @@ jobs:
             .trivyignore.yaml
           sparse-checkout-cone-mode: false
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

.husky/pre-commit

@@ -0,0 +1,2 @@
+yarn check:license
+yarn lint-staged