opencrvs · rikukissa · Oct 7, 2024 · Oct 7, 2024 · Oct 7, 2024 · Oct 7, 2024
diff --git a/.github/workflows/auto-pr-to-release.yml b/.github/workflows/auto-pr-to-release.yml
@@ -18,36 +18,122 @@ on:
         description: 'PR number to process'
         required: true
         default: ''
+      dry_run:
+        description: 'Dry run'
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  create-pr:
+  resolve-releases:
     if: ${{ github.event_name == 'pull_request' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
-
+    outputs:
+      matrix: ${{ steps.resolve-applicable-versions.outputs.matrix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Get PR details from workflow dispatch
         if: ${{ github.event_name == 'workflow_dispatch' }}
         id: get_pr_details_dispatch
         run: |
           PR_NUMBER=${{ github.event.inputs.pr_number }}
-          PR_DATA=$(gh pr view $PR_NUMBER --json number,headRefName,baseRefName,mergedBy,mergeCommit,author,milestone,title --jq '{number: .number, headRefName: .headRefName, baseRefName: .baseRefName, merger: .mergedBy.login, author: .author.login, milestone: .milestone.title, title: .title}')
-          echo "PR_ID=$(echo $PR_DATA | jq -r '.number')" >> $GITHUB_ENV
-          echo "PR_AUTHOR=$(echo $PR_DATA | jq -r '.author')" >> $GITHUB_ENV
-          echo "PR_MERGER=$(echo $PR_DATA | jq -r '.merger')" >> $GITHUB_ENV
-          echo "MILESTONE=$(echo $PR_DATA | jq -r '.milestone')" >> $GITHUB_ENV
-          echo "BASE_BRANCH=$(echo $PR_DATA | jq -r '.baseRefName')" >> $GITHUB_ENV
-          echo "HEAD_BRANCH=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_ENV
-          echo "PR_TITLE=$(echo $PR_DATA | jq -r '.title')" >> $GITHUB_ENV
+          PR_DATA=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/opencrvs/opencrvs-countryconfig/pulls/$PR_NUMBER)
+          echo "MILESTONE=$(printf '%s' $PR_DATA | jq -r '.milestone.title')" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
-          LATEST_COMMIT_SHA=$(gh pr view $PR_NUMBER --json commits --jq '.commits[-1].oid')
-          FIRST_COMMIT_SHA=$(gh pr view $PR_NUMBER --json commits --jq '.commits[0].oid')
-          echo "LATEST_COMMIT_SHA=${LATEST_COMMIT_SHA}" >> $GITHUB_ENV
-          echo "FIRST_COMMIT_SHA=${FIRST_COMMIT_SHA}" >> $GITHUB_ENV
+      - name: Get PR details from event
+        if: ${{ github.event_name == 'pull_request' }}
+        id: get_pr_details_event
+        run: |
+          echo "MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - name: Check for milestone and if release branch exists
+        continue-on-error: true
+        id: resolve-applicable-versions
+        run: |
+          if [ -z "${{ env.MILESTONE }}" ] || [ "${{ env.MILESTONE }}" = "null" ]; then
+            echo "No milestone set. Exiting."
+            exit 1
+          fi
+
+          filter_versions() {
+              local input_version=$1
+
+              # List remote branches, extract versions, and sort them semantically
+              versions=$(git ls-remote --heads origin 'release-*' | awk -F'release-' '{print $2}' | sort -V)
+
+              # Filter out versions less than the input version
+              filtered_versions=$(echo "$versions" | awk -v input="$input_version" '
+              function compare_versions(v1, v2) {
+                  split(v1, a, /[.v]/);
+                  split(v2, b, /[.v]/);
+                  for (i = 2; i <= 4; i++) {
+                      if (a[i] < b[i]) return -1;
+                      if (a[i] > b[i]) return 1;
+                  }
+                  return 0;
+              }
+              {
+                  if (compare_versions($0, input) >= 0) {
+                      print $0
+                  }
+              }')
+
+              # Keep only the highest patch version for each minor version
+              echo "$filtered_versions" | awk -F. '
+              {
+                  minor = $1 "." $2;
+                  patches[minor] = $0;
+              }
+              END {
+                  for (minor in patches) {
+                      print patches[minor];
+                  }
+              }' | sort -V
+          }
+
+          versions=$(filter_versions "${{ env.MILESTONE }}")
+          json_array=$(echo "$versions" | jq -R -s -c 'split("\n") | map(select(. != ""))')
+          echo "matrix=$json_array" >> $GITHUB_OUTPUT
+
+  create-pr:
+    needs: resolve-releases
+    runs-on: ubuntu-22.04
+    if: ${{ always() && needs.resolve-releases.result == 'success' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.resolve-releases.outputs.matrix)}}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get PR details from workflow dispatch
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: get_pr_details_dispatch
+        run: |
+          PR_NUMBER=${{ github.event.inputs.pr_number }}
+          PR_DATA=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/opencrvs/opencrvs-countryconfig/pulls/$PR_NUMBER)
+          # printf escapes the newlines in the JSON, so we can use jq to parse output such as:
+          # "body": "![image](https://github.com/user-attachments/assets/8eee5bcf-7692-490f-a19f-576623e09961)\r\n",
+          echo "PR_ID=$(printf '%s' $PR_DATA | jq -r '.number')" >> $GITHUB_ENV
+          echo "PR_AUTHOR=$(printf '%s' $PR_DATA | jq -r '.user.login')" >> $GITHUB_ENV
+          echo "PR_MERGER=$(printf '%s' $PR_DATA | jq -r '.merged_by.login')" >> $GITHUB_ENV
+          echo "BASE_BRANCH=$(printf '%s' $PR_DATA | jq -r '.base.ref')" >> $GITHUB_ENV
+          echo "HEAD_BRANCH=$(printf '%s' $PR_DATA | jq -r '.head.ref')" >> $GITHUB_ENV
+          echo "PR_TITLE=$(printf '%s' $PR_DATA | jq -r '.title')" >> $GITHUB_ENV
+          echo "BASE_SHA=$(printf '%s' $PR_DATA | jq -r '.base.sha')" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
       - name: Get PR details from event
         if: ${{ github.event_name == 'pull_request' }}
@@ -56,7 +142,6 @@ jobs:
           PR_NUMBER=${{ github.event.pull_request.number }}
           echo "PR_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
           echo "PR_AUTHOR=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV
-          echo "MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
           echo "BASE_BRANCH=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
           echo "HEAD_BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
           echo "PR_TITLE=${{ github.event.pull_request.title }}" >> $GITHUB_ENV
@@ -70,52 +155,36 @@ jobs:
           MERGED_BY_LOGIN=$(echo "$PR_DETAILS" | jq -r '.mergedBy.login')
           echo "PR_MERGER=$MERGED_BY_LOGIN" >> $GITHUB_ENV
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check for milestone and if release branch exists
-        continue-on-error: true
-        id: check_release_branch
-        run: |
-          if [ -z "${{ env.MILESTONE }}" ]; then
-            echo "No milestone set. Exiting."
-            exit 1
-          fi
-
-          RELEASE_BRANCH="release-${{ env.MILESTONE }}"
-
-          # Check if the release branch exists
-          if git ls-remote --heads origin $RELEASE_BRANCH | grep -q "refs/heads/$RELEASE_BRANCH"; then
-            echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV
-          else
-            echo "Release branch $RELEASE_BRANCH does not exist. Exiting."
-            exit 1
-          fi
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
       - name: Create and push the new branch for the PR
-        if: ${{ steps.check_release_branch.outcome == 'success' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
         run: |
           SEMANTIC_PR_TITLE="${{ env.PR_TITLE }}"
+          RELEASE_BRANCH="release-${{ matrix.version }}"
+          MILESTONE="${{ matrix.version }}"
 
           # Check for semantic prefix
           if [[ $SEMANTIC_PR_TITLE =~ ^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert|wip|merge)\: ]]; then
             SEMANTIC_PR_TITLE="${BASH_REMATCH[1]}(${MILESTONE}): ${SEMANTIC_PR_TITLE#*: }"
           else
-            SEMANTIC_PR_TITLE="🍒 Merge changes from PR #${{ env.PR_ID }} to ${{ env.RELEASE_BRANCH }}"
+            SEMANTIC_PR_TITLE="🍒 Merge changes from PR #${{ env.PR_ID }} to $RELEASE_BRANCH"
           fi
 
-          PR_BODY="Automated PR to merge changes from develop to ${{ env.RELEASE_BRANCH }}"
+          PR_BODY="Automated PR to merge changes from develop to $RELEASE_BRANCH"
 
           # Configure git
           git config user.name "github-actions"
           git config user.email "[email protected]"
           git config advice.mergeConflict false
 
           # Fetch and checkout the release branch
-          git fetch --all
-          git checkout ${{ env.RELEASE_BRANCH }}
+          git fetch --all --unshallow
+          git checkout $RELEASE_BRANCH
 
           # Create a new branch for the PR
-          NEW_BRANCH="auto-pr-${{ env.RELEASE_BRANCH }}-${{ env.PR_ID }}-$RANDOM"
+          NEW_BRANCH="auto-pr-$RELEASE_BRANCH-${{ env.PR_ID }}-$RANDOM"
           git checkout -b $NEW_BRANCH
 
           echo "First commit: ${{ env.FIRST_COMMIT_SHA }}"
@@ -167,6 +236,15 @@ jobs:
             "
           }
 
+          if [ "${{ github.event.inputs.dry_run }}" == "true" ]; then
+            echo "This is a dry run."
+            echo "Would have pushed the new branch $NEW_BRANCH"
+            echo "PR title: $SEMANTIC_PR_TITLE"
+            echo "PR body:"
+            echo "$PR_BODY"
+            exit 0
+          fi
+
           # Push the new branch
           git push origin $NEW_BRANCH
 
@@ -180,6 +258,4 @@ jobs:
               AUTHOR=${{ env.PR_MERGER }}
             fi
           fi
-          gh pr create --title "$SEMANTIC_PR_TITLE" --body "$PR_BODY" --head "$NEW_BRANCH" --base "${{ env.RELEASE_BRANCH }}" --assignee "$AUTHOR" --reviewer "$AUTHOR"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          gh pr create --title "$SEMANTIC_PR_TITLE" --body "$PR_BODY" --head "$NEW_BRANCH" --base "$RELEASE_BRANCH" --assignee "$AUTHOR" --reviewer "$AUTHOR"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -78,6 +78,12 @@ INSERT CSV ROWS IN ENGLISH ONLY
 - Github pipeline dedicated for reading secrets and variables from other environments now checks if GH_TOKEN is still valid before attempting other operations
 - Remove unnecessary UI dividers that add in various sections of the declaration forms(e.g the Death, Birth and Marriage forms) [#244](https://github.com/opencrvs/opencrvs-countryconfig/pull/244)
 
+## 1.5.2 (https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.5.1...v1.5.2)
+
+## Bug fixes
+
+- Broken email alerts from low disk space are now fixed [293](https://github.com/opencrvs/opencrvs-countryconfig/pull/293)
+
 ## 1.5.0 (https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.4.1...v1.5.0)
 
 ### Breaking changes

diff --git a/infrastructure/monitoring/elastalert/rules/alert.yaml b/infrastructure/monitoring/elastalert/rules/alert.yaml
@@ -26,13 +26,13 @@ filter:
       should:
         - term:
             rule.name.keyword:
-              value: 'Available disk space in data partition'
+              value: 'Available disk space in root file system'
         - term:
             rule.name.keyword:
               value: 'CPU under heavy load'
         - term:
             rule.name.keyword:
-              value: 'Low on available disk space'
+              value: 'Low on available disk space in data partition'
       minimum_should_match: 1
 
 alert: post2

diff --git a/package.json b/package.json
@@ -18,7 +18,7 @@
   "scripts": {
     "dev": "yarn start",
     "precommit": "lint-staged",
-    "test": "echo 'no tests, yet'",
+    "test": "vitest",
     "test:compilation": "tsc --noEmit",
     "lint": "eslint -c .eslintrc.js",
     "start": "cross-env NODE_ENV=development NODE_OPTIONS=--dns-result-order=ipv4first nodemon --exec ts-node -r tsconfig-paths/register src/index.ts",
@@ -34,8 +34,8 @@
     "@graphql-codegen/add": "^3.1.1",
     "@graphql-codegen/cli": "^3.3.1",
     "@graphql-codegen/introspection": "^3.0.1",
-    "@graphql-codegen/typescript-operations": "^3.0.4",
     "@graphql-codegen/typescript": "^3.0.4",
+    "@graphql-codegen/typescript-operations": "^3.0.4",
     "@inquirer/editor": "^1.2.13",
     "@octokit/core": "4.2.1",
     "@types/google-libphonenumber": "^7.4.23",
@@ -49,19 +49,21 @@
     "@typescript-eslint/eslint-plugin": "^5.60.1",
     "@typescript-eslint/parser": "^5.60.1",
     "cypress-xpath": "^2.0.1",
+    "eslint": "^8.43.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
-    "eslint": "^8.43.0",
     "husky": "1.0.0-rc.13",
     "inquirer": "^9.2.12",
+    "js-yaml": "^4.1.0",
     "kleur": "^4.1.5",
     "libsodium-wrappers": "^0.7.13",
     "lint-staged": "^7.1.0",
     "node-ssh": "^13.2.0",
     "nodemon": "^2.0.22",
     "pino-pretty": "^11.0.0",
     "prettier": "^2.8.8",
-    "react-intl": "^6.4.3"
+    "react-intl": "^6.4.3",
+    "vitest": "^2.1.2"
   },
   "dependencies": {
     "@faker-js/faker": "^6.0.0-alpha.5",
@@ -76,8 +78,8 @@
     "@types/hapi__hapi": "^20.0.0",
     "@types/jwt-decode": "^2.2.1",
     "@types/lodash": "^4.14.117",
-    "@types/node-fetch": "^2.6.2",
     "@types/node": "^10.12.5",
+    "@types/node-fetch": "^2.6.2",
     "@types/nodemailer": "^6.4.14",
     "app-module-path": "^2.2.0",
     "chalk": "^2.4.1",
@@ -89,8 +91,8 @@
     "dotenv": "^16.4.5",
     "esbuild": "^0.18.9",
     "google-libphonenumber": "^3.2.32",
-    "graphql-tag": "^2.12.6",
     "graphql": "^16.3.0",
+    "graphql-tag": "^2.12.6",
     "handlebars": "^4.7.7",
     "hapi-auth-jwt2": "10.4.0",
     "hapi-pino": "^9.0.0",

diff --git a/tests/verify-elastalert-kibana-alerts-match.test.ts b/tests/verify-elastalert-kibana-alerts-match.test.ts
@@ -0,0 +1,54 @@
+import { readdirSync, readFileSync } from 'fs'
+import yaml from 'js-yaml'
+import { join } from 'path'
+import { expect, it } from 'vitest'
+
+function findAllValuesByKey(obj: unknown, key: string): any[] {
+  const result: any[] = []
+
+  const recurse = (item: unknown) => {
+    if (Array.isArray(item)) {
+      for (const element of item) {
+        recurse(element)
+      }
+    } else if (typeof item === 'object' && item !== null) {
+      for (const k in item) {
+        if (k === key) {
+          result.push(item[k])
+        }
+        recurse(item[k])
+      }
+    }
+  }
+
+  recurse(obj)
+  return result
+}
+
+it('all tests defined in Kibana config are also defined in Elastalert config', () => {
+  const allAlertNames = readFileSync(
+    join(__dirname, '../infrastructure/monitoring/kibana', 'config.ndjson'),
+    'utf8'
+  )
+    .split('\n')
+    .map((str) => JSON.parse(str))
+    .filter((item) => item.type === 'alert')
+    .map((item) => item.attributes.name)
+    .sort()
+    .filter((value, index, self) => self.indexOf(value) === index)
+
+  const ruleNameFilters = readdirSync(
+    join(__dirname, '../infrastructure/monitoring/elastalert/rules')
+  )
+    .map((file) =>
+      join(__dirname, '../infrastructure/monitoring/elastalert/rules', file)
+    )
+    .map((file) => readFileSync(file, 'utf8'))
+    .map((file) => yaml.load(file))
+    .flatMap((rule) => findAllValuesByKey(rule, 'rule.name.keyword'))
+    .map((x) => x.value)
+    .sort()
+    .filter((value, index, self) => self.indexOf(value) === index)
+
+  expect(ruleNameFilters).toEqual(allAlertNames)
+})