From 3a923204d3e3680933a68fd8f4da051dfda5bc85 Mon Sep 17 00:00:00 2001 From: Vikas Agarwal Date: Wed, 24 Oct 2018 13:02:56 +0530 Subject: [PATCH 1/5] Trying to run event bus consumers and api in separate services to be more fault tolerant --- connect/connectNotificationServer.js | 16 +++++++------- deploy.sh | 28 ++++++++++++++++--------- index-api.js | 12 +++++++++++ index.js | 18 ++++++++++++---- package.json | 1 + src/app.js | 31 ++++++++++++++-------------- 6 files changed, 69 insertions(+), 37 deletions(-) create mode 100644 index-api.js diff --git a/connect/connectNotificationServer.js b/connect/connectNotificationServer.js index 51ecb71..e6ed1f4 100644 --- a/connect/connectNotificationServer.js +++ b/connect/connectNotificationServer.js @@ -361,13 +361,13 @@ if (config.ENABLE_EMAILS) { } // init database, it will clear and re-create all tables -notificationServer - .initDatabase() - .then(() => notificationServer.start()) - .catch((e) => { - console.log(e); // eslint-disable-line no-console - notificationServer.logger.error('Notification server errored out'); - }); +// notificationServer +// .initDatabase() +// .then(() => notificationServer.startKafkaConsumers()) +// .catch((e) => { +// console.log(e); // eslint-disable-line no-console +// notificationServer.logger.error('Notification server errored out'); +// }); // if no need to init database, then directly start the server: -// notificationServer.start(); +notificationServer.startKafkaConsumers(); diff --git a/deploy.sh b/deploy.sh index 31d85e0..1bb5128 100755 --- a/deploy.sh +++ b/deploy.sh @@ -29,7 +29,8 @@ AWS_SECRET_ACCESS_KEY=$(eval "echo \$${ENV}_AWS_SECRET_ACCESS_KEY") AWS_ACCOUNT_ID=$(eval "echo \$${ENV}_AWS_ACCOUNT_ID") AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY") AWS_ECS_CLUSTER=$(eval "echo \$${ENV}_AWS_ECS_CLUSTER") -AWS_ECS_SERVICE=$(eval "echo \$${ENV}_AWS_ECS_SERVICE") +AWS_ECS_SERVICE_API=$(eval "echo \$${ENV}_AWS_ECS_SERVICE") +AWS_ECS_SERVICE_CONSUMERS=$(eval "echo \$${ENV}_AWS_ECS_SERVICE_CONSUMERS") KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT") KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY") @@ -98,9 +99,9 @@ deploy_cluster() { #family="nginx-api-dev-task" - make_task_def - register_definition - update_result=$(aws ecs update-service --cluster $AWS_ECS_CLUSTER --service $AWS_ECS_SERVICE --task-definition $revision ) + make_task_def $1 $2 $3 $4 + register_definition $1 + update_result=$(aws ecs update-service --cluster $AWS_ECS_CLUSTER --service $1 --task-definition $revision ) #echo $update_result result=$(echo $update_result | $JQ '.service.taskDefinition' ) echo $result @@ -121,6 +122,7 @@ make_task_def(){ "essential": true, "memory": 1536, "cpu": 768, + "entryPoint": ["%s", "%s", "%s"], "environment": [ { "name": "ENV", @@ -261,11 +263,11 @@ make_task_def(){ } ]' - task_def=$(printf "$task_template" $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $CONNECT_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV) + task_def=$(printf "$task_template" $1 $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $2 $3 $4 $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $CONNECT_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV) } register_definition() { - if revision=$(aws ecs register-task-definition --container-definitions "$task_def" --family $family | $JQ '.taskDefinition.taskDefinitionArn'); then + if revision=$(aws ecs register-task-definition --container-definitions "$task_def" --family $1 2> /dev/null | $JQ '.taskDefinition.taskDefinitionArn'); then echo "Revision: $revision" else echo "Failed to register task definition" @@ -277,13 +279,13 @@ register_definition() { check_service_status() { counter=0 sleep 60 - servicestatus=`aws ecs describe-services --service $AWS_ECS_SERVICE --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'` + servicestatus=`aws ecs describe-services --service $1 --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'` while [[ $servicestatus != *"steady state"* ]] do echo "Current event message : $servicestatus" echo "Waiting for 30 seconds to check the service status...." sleep 30 - servicestatus=`aws ecs describe-services --service $AWS_ECS_SERVICE --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'` + servicestatus=`aws ecs describe-services --service $1 --cluster $AWS_ECS_CLUSTER | $JQ '.services[].events[0].message'` counter=`expr $counter + 1` if [[ $counter -gt $COUNTER_LIMIT ]] ; then echo "Service does not reach steady state within 10 minutes. Please check" @@ -295,5 +297,11 @@ check_service_status() { configure_aws_cli push_ecr_image -deploy_cluster -check_service_status + +deploy_cluster AWS_ECS_SERVICE_API "npm" "run" "startAPI" + +deploy_cluster AWS_ECS_SERVICE_CONSUMERS "npm" "run" "start" + +check_service_status AWS_ECS_SERVICE_API + +check_service_status AWS_ECS_SERVICE_CONSUMERS diff --git a/index-api.js b/index-api.js new file mode 100644 index 0000000..3301805 --- /dev/null +++ b/index-api.js @@ -0,0 +1,12 @@ +const notificationServer = require('./index'); + +// // init database, it will clear and re-create all tables +// notificationServer +// .initDatabase() +// .then(() => notificationServer.startAPI()) +// .catch((e) => { +// console.log(e); // eslint-disable-line no-console +// notificationServer.logger.error('Notification API server errored out'); +// }); + +module.exports = notificationServer.startAPI(); diff --git a/index.js b/index.js index c13c164..e9e821f 100644 --- a/index.js +++ b/index.js @@ -95,15 +95,24 @@ function getAllHandlers() { } /** - * Start the notification server. + * Start the notification API server. */ -function start() { +function startAPI() { + // load app only after config is set + const app = require('./src/app'); + app.start(); +} + +/** + * Start the event bus consumer. + */ +function startKafkaConsumers() { if (_.isEmpty(handlers)) { throw new errors.ValidationError('Missing handler(s).'); } // load app only after config is set const app = require('./src/app'); - app.start(handlers, notificationServiceHandlers); + app.startKafkaConsumer(handlers, notificationServiceHandlers); } /** @@ -122,7 +131,8 @@ module.exports = { addTopicHandler, removeTopicHandler, getAllHandlers, - start, + startAPI, + startKafkaConsumers, initDatabase, addNotificationServiceHandler, diff --git a/package.json b/package.json index c4b8036..5e44e27 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "./index.js", "scripts": { "start": "node connect/connectNotificationServer", + "startAPI": "node index-api", "lint": "eslint *.js src config test connect || true", "lint:fix": "eslint *.js --fix src config test connect || true", "postinstall": "npm run build", diff --git a/src/app.js b/src/app.js index 540983f..6243b88 100644 --- a/src/app.js +++ b/src/app.js @@ -17,7 +17,7 @@ const models = require('./models'); const Kafka = require('no-kafka'); /** - * Start Kafka consumer. + * Start Kafka consumer for event bus events. * @param {Object} handlers the handlers * @param {Array} notificationServiceHandlers list of notification service handlers */ @@ -74,6 +74,7 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { }); }); + consumer .init() .then(() => _.each(_.keys(handlers), @@ -85,11 +86,9 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { } /** - * Start the notification server. - * @param {Object} handlers the handlers - * @param {Array} notificationServiceHandlers list of notification service handlers + * Start the notifications API server. */ -function start(handlers, notificationServiceHandlers) { +function start() { const app = express(); app.set('port', config.PORT); @@ -157,19 +156,21 @@ function start(handlers, notificationServiceHandlers) { } }); - models - .init() - .then(() => { - app.listen(app.get('port'), () => { - logger.info(`Express server listening on port ${app.get('port')}`); - }); - - startKafkaConsumer(handlers, notificationServiceHandlers); - }) - .catch((err) => logger.error(err)); + // models + // .init() + // .then(() => { + // app.listen(app.get('port'), () => { + // logger.info(`Express server listening on port ${app.get('port')}`); + // }); + // }) + // .catch((err) => logger.error(err)); + app.listen(app.get('port'), () => { + logger.info(`Express server listening on port ${app.get('port')}`); + }); } // Exports module.exports = { start, + startKafkaConsumer, }; From 6d1667c4383aa41f6805984792998d89a8dd87a3 Mon Sep 17 00:00:00 2001 From: Vikas Agarwal Date: Wed, 24 Oct 2018 13:03:39 +0530 Subject: [PATCH 2/5] Temporary making feature branch deployable --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2eb485..756057c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ workflows: - "build-dev": filters: branches: - only: [dev, architectt1-feature/emailTemplatesClean] + only: [dev, 'feature/multi-service-deployment'] - "build-prod": filters: branches: From 81b6e4e40da48310dd1d4dd2194d59b6c8d0e129 Mon Sep 17 00:00:00 2001 From: Vikas Agarwal Date: Wed, 24 Oct 2018 14:21:25 +0530 Subject: [PATCH 3/5] Fixed typo --- deploy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy.sh b/deploy.sh index 1bb5128..b887dcb 100755 --- a/deploy.sh +++ b/deploy.sh @@ -298,10 +298,10 @@ check_service_status() { configure_aws_cli push_ecr_image -deploy_cluster AWS_ECS_SERVICE_API "npm" "run" "startAPI" +deploy_cluster $AWS_ECS_SERVICE_API "npm" "run" "startAPI" -deploy_cluster AWS_ECS_SERVICE_CONSUMERS "npm" "run" "start" +deploy_cluster $AWS_ECS_SERVICE_CONSUMERS "npm" "run" "start" -check_service_status AWS_ECS_SERVICE_API +check_service_status $AWS_ECS_SERVICE_API -check_service_status AWS_ECS_SERVICE_CONSUMERS +check_service_status $AWS_ECS_SERVICE_CONSUMERS From 6da8b5b68c0ad816fe8984e585fc6a123c89163f Mon Sep 17 00:00:00 2001 From: Vikas Agarwal Date: Wed, 24 Oct 2018 15:26:22 +0530 Subject: [PATCH 4/5] reducing the resource limit for containers so that we can spawn 2 instances per task while deploying --- deploy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy.sh b/deploy.sh index b887dcb..7b06708 100755 --- a/deploy.sh +++ b/deploy.sh @@ -120,8 +120,8 @@ make_task_def(){ "name": "%s", "image": "%s.dkr.ecr.%s.amazonaws.com/%s:%s", "essential": true, - "memory": 1536, - "cpu": 768, + "memory": 768, + "cpu": 512, "entryPoint": ["%s", "%s", "%s"], "environment": [ { From 013abf1bad1c5bc11a219594fb02a7f2dfbfc8cb Mon Sep 17 00:00:00 2001 From: Vikas Agarwal Date: Wed, 24 Oct 2018 16:19:40 +0530 Subject: [PATCH 5/5] Removing feature branch from deployable branches. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 756057c..51fe260 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ workflows: - "build-dev": filters: branches: - only: [dev, 'feature/multi-service-deployment'] + only: [dev] - "build-prod": filters: branches: