From 1db45683303240c88076855e890fe992f547c376 Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 30 Jul 2024 07:47:26 +0300 Subject: [PATCH] move restart to deploy script --- infrastructure/deployment/deploy.sh | 9 +++++ .../elasticsearch/setup-elastalert-indices.sh | 35 +++++++++++++------ infrastructure/elasticsearch/setup-helpers.sh | 32 ----------------- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh index 5d8725915..ba3eda77d 100755 --- a/infrastructure/deployment/deploy.sh +++ b/infrastructure/deployment/deploy.sh @@ -386,6 +386,15 @@ echo echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios." echo +echo 'Setting up elastalert indices' + +while true; do + if configured_ssh "/opt/opencrvs/infrastructure/elasticsearch/setup-elastalert-indices.sh"; then + break + fi + sleep 5 +done + echo "Setting up Kibana config & alerts" while true; do diff --git a/infrastructure/elasticsearch/setup-elastalert-indices.sh b/infrastructure/elasticsearch/setup-elastalert-indices.sh index e53e4ce4a..699ec03de 100755 --- a/infrastructure/elasticsearch/setup-elastalert-indices.sh +++ b/infrastructure/elasticsearch/setup-elastalert-indices.sh @@ -9,21 +9,36 @@ # # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. -set -eu -set -o pipefail +# Upgrading from 7 to 8 requires deleting elastalert indices. https://elastalert2.readthedocs.io/en/latest/recipes/faq.html#does-elastalert-2-support-elasticsearch-8 -source "$(dirname "${BASH_SOURCE[0]}")/setup-helpers.sh" +set -e -echo "-------- $(date) --------" +docker_command="docker run --rm --network=opencrvs_overlay_net curlimages/curl" -log 'Waiting for availability of Elasticsearch' -wait_for_elasticsearch +echo 'Waiting for availability of Elasticsearch' +ping_status_code=$($docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -o /dev/null -w '%{http_code}' "http://elasticsearch:9200") +if [ "$ping_status_code" -ne 200 ]; then + echo "Elasticsearch is not ready. API returned status code: $ping_status_code" + exit 1 +fi + + + +echo 'Scaling down Elastalert' -log 'Scaling down Elastalert' docker service scale opencrvs_elastalert=0 -log 'Deleting Elastalert indices' -delete_elastalert_indices -log 'Scaling up Elastalert' + +echo 'Deleting Elastalert indices' +indices='elastalert_status,elastalert_status_error,elastalert_status_past,elastalert_status_silence,elastalert_status_status' + +delete_status_code=$($docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -o /dev/null -w '%{http_code}' "http://elasticsearch:9200/${indices}" -X DELETE) + +if [ "$delete_status_code" -ne 200 ]; then + echo "Could not delete indices. API returned status code: $delete_status_code" + exit 1 +fi + +echo 'Scaling up Elastalert' docker service scale opencrvs_elastalert=1 diff --git a/infrastructure/elasticsearch/setup-helpers.sh b/infrastructure/elasticsearch/setup-helpers.sh index b058a2658..1b3380f0f 100755 --- a/infrastructure/elasticsearch/setup-helpers.sh +++ b/infrastructure/elasticsearch/setup-helpers.sh @@ -230,35 +230,3 @@ function ensure_settings { return $result } - -# Upgrading from 7 to 8 requires deleting elastalert indices. https://elastalert2.readthedocs.io/en/latest/recipes/faq.html#does-elastalert-2-support-elasticsearch-8 -# Elastalert depends on kibana/beat indices, so we delete can elastalert indices during each deployment. -function delete_elastalert_indices { - # Opt for explicity over wildcard since we are deleting indices - local indices='elastalert_status,elastalert_status_error,elastalert_status_past,elastalert_status_silence,elastalert_status_status' - - local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" - - local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' - "http://${elasticsearch_host}:9200/${indices}" - '-X' 'DELETE' - ) - - if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then - args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) - fi - - local -i result=1 - local output - - output="$(curl "${args[@]}")" - if [[ "${output: -3}" -eq 200 ]]; then - result=0 - fi - - if ((result)); then - echo -e "\n${output::-3}\n" - fi - - return $result -} \ No newline at end of file