Merge pull request #256 from opencrvs/fix/logstash-elastic-8-logging

Fix/logstash elastic 8 logging

infrastructure/monitoring/kibana/config.ndjson

@@ -2,7 +2,7 @@
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"threshold_met","params":{"documents":["{\"@timestamp\":\"2022-04-18T07:05:33.819Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"apm.error_rate","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-06-01T11:30:27.033Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in service","notifyWhen":"onActionGroupChange","params":{"environment":"ENVIRONMENT_ALL","threshold":1,"windowSize":1,"windowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-05T03:00:20.633Z","updatedBy":"opencrvs-admin"},"coreMigrationVersion":"7.17.0","id":"3b6722e0-e19e-11ec-ba8e-51649755648d","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707273006619,214975],"type":"alert","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjAzNCwxOV0="}
 {"attributes":{"buildNum":46534,"defaultIndex":"metricbeat-*"},"coreMigrationVersion":"7.17.0","id":"7.17.0","migrationVersion":{"config":"7.13.0"},"references":[],"sort":[1707273006619,216009],"type":"config","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjQyOCwxOV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"logs.threshold.fired","params":{"documents":["{\"@timestamp\":\"2023-11-22T08:25:47.329Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"logs.alert.document.count","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-22T08:32:38.272Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in backup logs","notifyWhen":"onActionGroupChange","params":{"count":{"comparator":"more than or equals","value":1},"criteria":[{"comparator":"matches","field":"message","value":"error"},{"comparator":"equals","field":"log.file.path","value":"/var/log/opencrvs-backup.log"}],"timeSize":1,"timeUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:22.558Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"b166fcb0-8911-11ee-8111-2f3be9e93efc","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294436464,232766],"type":"alert","updated_at":"2024-02-07T08:27:16.464Z","version":"WzQ5NTQzNCwxOV0="}
-{"attributes":{"anomalyThreshold":50,"description":"","fields":{"container":"container.id","host":"host.name","message":["message","@message"],"pod":"kubernetes.pod.uid","tiebreaker":"_doc","timestamp":"@timestamp"},"inventoryDefaultView":"0","logColumns":[{"timestampColumn":{"id":"5e7f964a-be8a-40d8-88d2-fbcfbdca0e2f"}},{"fieldColumn":{"field":"event.dataset","id":" eb9777a8-fcd3-420e-ba7d-172fff6da7a2"}},{"messageColumn":{"id":"b645d6da-824b-4723-9a2a-e8cece1645c0"}}],"logIndices":{"indexName":"logs-*,filebeat-*,kibana_sample_data_logs*,logstash*","type":"index_name"},"metricAlias":"metrics-*,metricbeat-*","metricsExplorerDefaultView":"0","name":"Default"},"coreMigrationVersion":"7.17.0","id":"default","migrationVersion":{"infrastructure-ui-source":"7.16.2"},"references":[],"sort":[1707273006619,217714],"type":"infrastructure-ui-source","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MzAyNywxOV0="}
+{"attributes":{"anomalyThreshold":50,"description":"","fields":{"container":"container.id","host":"host.name","message":["message","@message"],"pod":"kubernetes.pod.uid","tiebreaker":"_doc","timestamp":"@timestamp"},"inventoryDefaultView":"0","logColumns":[{"timestampColumn":{"id":"5e7f964a-be8a-40d8-88d2-fbcfbdca0e2f"}},{"fieldColumn":{"field":"event.dataset","id":" eb9777a8-fcd3-420e-ba7d-172fff6da7a2"}},{"messageColumn":{"id":"b645d6da-824b-4723-9a2a-e8cece1645c0"}}],"logIndices":{"indexName":"logs-*,filebeat-*,kibana_sample_data_logs*,logstash*,ecs-logstash*","type":"index_name"},"metricAlias":"metrics-*,metricbeat-*","metricsExplorerDefaultView":"0","name":"Default"},"coreMigrationVersion":"7.17.0","id":"default","migrationVersion":{"infrastructure-ui-source":"7.16.2"},"references":[],"sort":[1707273006619,217714],"type":"infrastructure-ui-source","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MzAyNywxOV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"query matched","params":{"documents":["{\"@timestamp\":\"2023-11-20T10:19:30.521Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-20T09:12:19.237Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Successful SSH login","notifyWhen":"onActionGroupChange","params":{"esQuery":"{ \"query\": { \"bool\": { \"must\": [ \n  { \"term\": { \"log.file.path\": \"/var/log/auth.log\" } },\n  { \"term\": { \"event.outcome\": \"success\" }}\n  ] } } }","index":["filebeat-*"],"size":100,"threshold":[1],"thresholdComparator":">=","timeField":"@timestamp","timeWindowSize":1,"timeWindowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:19.537Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"e79aaa90-8784-11ee-b9ba-89bbe73df7ff","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294457367,232778],"type":"alert","updated_at":"2024-02-07T08:27:37.367Z","version":"WzQ5NTQ0MCwxOV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"metrics.inventory_threshold.fired","params":{"documents":["{\"@timestamp\":\"2022-06-20T06:16:33.414Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"metrics.alert.inventory.threshold","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-05-31T10:10:47.084Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"CPU under heavy load","notifyWhen":"onActionGroupChange","params":{"criteria":[{"comparator":">","customMetric":{"aggregation":"avg","field":"","id":"alert-custom-metric","type":"custom"},"metric":"cpu","threshold":[70],"timeSize":1,"timeUnit":"m"}],"nodeType":"host","sourceId":"default"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":["infra","opencrvs-builtin"],"throttle":null,"updatedAt":"2024-02-07T02:27:30.573Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"f022bee0-e0c9-11ec-99b8-dbfd54551fda","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294457362,232774],"type":"alert","updated_at":"2024-02-07T08:27:37.362Z","version":"WzQ5NTQzOCwxOV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"metrics.inventory_threshold.fired","params":{"documents":["{\"@timestamp\":\"2023-11-17T13:17:52.791Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"metrics.alert.inventory.threshold","apiKey":null,"apiKeyOwner":null,"consumer":"infrastructure","createdAt":"2022-05-31T10:10:47.080Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Low on available disk space in data partition","notifyWhen":"onActionGroupChange","params":{"alertOnNoData":true,"criteria":[{"comparator":">=","customMetric":{"aggregation":"max","field":"system.filesystem.used.pct","id":"alert-custom-metric","label":"","type":"custom"},"metric":"custom","threshold":[0.7],"timeSize":1,"timeUnit":"h","warningComparator":">=","warningThreshold":[0.5]}],"filterQuery":"{\"bool\":{\"should\":[{\"match_phrase\":{\"system.filesystem.mount_point\":\"/hostfs/data\"}}],\"minimum_should_match\":1}}","filterQueryText":"system.filesystem.mount_point : \"/hostfs/data\"","nodeType":"host","sourceId":"default"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":["infra","opencrvs-builtin"],"throttle":null,"updatedAt":"2024-02-07T02:27:20.542Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"f023d050-e0c9-11ec-99b8-dbfd54551fda","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294448366,232768],"type":"alert","updated_at":"2024-02-07T08:27:28.366Z","version":"WzQ5NTQzNSwxOV0="}

infrastructure/monitoring/logstash/logstash.conf

@@ -9,7 +9,10 @@ filter {
         # Docker's GELF driver want's to write into a field named
         # container_id, but Kibana and other tools read from container.id by default
         rename => {"container_id" => "container.id"}
+        # As of V8, Elasticsearch JSON logs now comply with ECS. [host] becomes object
+        rename => {"[host]" => "[host][name]"}
     }
+
     truncate {
         length_bytes => 12
         fields => ["container.id"]