From b99993fbd5d704f607df3552ff59e4bea0fd0c90 Mon Sep 17 00:00:00 2001 From: Shruthi Date: Fri, 12 Mar 2021 17:23:56 +0530 Subject: [PATCH] Pass env variable as argument Signed-off-by: Shruthi --- acmeair/README.md | 3 +- acmeair/scripts/acmeair-common.sh | 8 +- acmeair/scripts/acmeair-deploy-openshift.sh | 11 +- acmeair/scripts/perf/README.md | 3 +- acmeair/scripts/perf/run_acmeair_openshift.sh | 13 +- galaxies/README.md | 48 +- galaxies/manifests/galaxies.yaml | 1 + galaxies/scripts/galaxies-common.sh | 73 +- galaxies/scripts/galaxies-deploy-openshift.sh | 27 +- galaxies/scripts/galaxies-load.sh | 171 +++++ galaxies/scripts/perf/README.md | 43 ++ galaxies/scripts/perf/getstats-openshift.sh | 348 ++++++++++ .../scripts/perf/run-galaxies-openshift.sh | 629 ++++++++++++++++++ galaxies/scripts/runWrk.sh | 32 - spring-petclinic/README.md | 3 +- spring-petclinic/manifests/petclinic.yaml | 1 + spring-petclinic/scripts/perf/README.md | 3 +- .../scripts/perf/run-petclinic-openshift.sh | 11 +- spring-petclinic/scripts/petclinic-common.sh | 6 +- .../scripts/petclinic-deploy-openshift.sh | 11 +- 20 files changed, 1376 insertions(+), 69 deletions(-) create mode 100755 galaxies/scripts/galaxies-load.sh create mode 100755 galaxies/scripts/perf/README.md create mode 100755 galaxies/scripts/perf/getstats-openshift.sh create mode 100755 galaxies/scripts/perf/run-galaxies-openshift.sh delete mode 100755 galaxies/scripts/runWrk.sh diff --git a/acmeair/README.md b/acmeair/README.md index 82c76333..bf1c2ef1 100644 --- a/acmeair/README.md +++ b/acmeair/README.md @@ -82,7 +82,7 @@ service/acmeair-service-1 created # Openshift To deploy the benchmark use `acmeair-deploy-openshift.sh` -`./scripts/acmeair-deploy-openshift.sh -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM]` +`./scripts/acmeair-deploy-openshift.sh -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]` - **BENCHMARK_SERVER**: Name of the cluster you are using - **SERVER_INSTANCES**: Number of acmeair instances to be deployed. It is optional, if is not specified then by default it will be considered as 1 instance. @@ -92,6 +92,7 @@ To deploy the benchmark use `acmeair-deploy-openshift.sh` - **MEM_REQ**: Memory request - **CPU_LIM**: CPU limit - **MEM_LIM**: Memory limit +- **ENV_VAR**: Environment variable Example to deploy and run acmeair application on openshift cluster diff --git a/acmeair/scripts/acmeair-common.sh b/acmeair/scripts/acmeair-common.sh index c8c10246..d182dc30 100755 --- a/acmeair/scripts/acmeair-common.sh +++ b/acmeair/scripts/acmeair-common.sh @@ -18,8 +18,8 @@ # CURRENT_DIR="$(dirname "$(realpath "$0")")" -pushd "${CURRENT_DIR}" >> setup.log -pushd ".." >> setup.log +pushd "${CURRENT_DIR}" >> /dev/null +pushd ".." >> /dev/null # Set the defaults for the app JMX_FILE="${PWD}/jmeter-driver/acmeair-jmeter/scripts/AcmeAir.jmx" @@ -70,7 +70,7 @@ function get_ip() { # output:build the application from scratch and create the acmeair docker image function build_acmeair() { # Build acmeair monolithic application docker image - pushd acmeair >>${LOGFILE} + pushd acmeair >> /dev/null # Build the application docker run --rm -v "${PWD}":/home/gradle/project -w /home/gradle/project dinogun/gradle:5.5.0-jdk8-openj9 gradle build 2>>${LOGFILE} >>${LOGFILE} err_exit "Error: gradle build of acmeair monolithic application failed." @@ -84,7 +84,7 @@ function build_acmeair() { # Build the acmeair driver application function build_acmeair_driver() { # Build acmeair driver - pushd jmeter-driver >>${LOGFILE} + pushd jmeter-driver >> /dev/null docker run --rm -v "${PWD}":/home/gradle/project -w /home/gradle/project dinogun/gradle:5.5.0-jdk8-openj9 gradle build 2>>${LOGFILE} >>${LOGFILE} err_exit "Error: gradle build of acmeair driver failed." popd >>${LOGFILE} diff --git a/acmeair/scripts/acmeair-deploy-openshift.sh b/acmeair/scripts/acmeair-deploy-openshift.sh index 4be432ee..14648750 100755 --- a/acmeair/scripts/acmeair-deploy-openshift.sh +++ b/acmeair/scripts/acmeair-deploy-openshift.sh @@ -28,7 +28,7 @@ CLUSTER_TYPE="openshift" # Describes the usage of the script function usage() { echo - echo "Usage: $0 -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM]" + echo "Usage: $0 -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]" echo " " echo "Example: $0 -s rouging.os.fyre.ibm.com -i 2 -a dinogun/acmeair-monolithic:latest --cpulim=4 --cpureq=2 --memlim=1024Mi --memreq=512Mi" exit -1 @@ -78,6 +78,9 @@ do memlim=*) MEM_LIM=${OPTARG#*=} ;; + env=*) + ENV_VAR=${OPTARG#*=} + ;; *) esac ;; @@ -159,6 +162,12 @@ function createInstances() { sed -i '/limits:/a \ \ \ \ \ \ \ \ \ \ cpu: '${CPU_LIM}'' ${MANIFESTS_DIR}/acmeair-${inst}.yaml fi + # Pass environment variables + if [ ! -z ${ENV_VAR} ]; then + sed -i '/env:/a \ \ \ \ \ \ \ \ - name: "JVM_ARGS"' ${MANIFESTS_DIR}/acmeair-${inst}.yaml + sed -i '/- name: "JVM_ARGS"/a \ \ \ \ \ \ \ \ \ \ value: '${ENV_VAR}'' ${MANIFESTS_DIR}/acmeair-${inst}.yaml + fi + oc create -f ${MANIFESTS_DIR}/acmeair-${inst}.yaml -n ${NAMESPACE} err_exit "Error: Issue in deploying." ((ACMEAIR_PORT=ACMEAIR_PORT+1)) diff --git a/acmeair/scripts/perf/README.md b/acmeair/scripts/perf/README.md index 01d0b8c7..3859bbb7 100644 --- a/acmeair/scripts/perf/README.md +++ b/acmeair/scripts/perf/README.md @@ -1,6 +1,6 @@ # Test with multiple instances -`./scripts/perf/run_acmeair_openshift.sh -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM]` +`./scripts/perf/run_acmeair_openshift.sh -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]` - **BENCHMARK_SERVER_NAME** : Name of the cluster you are using - **RESULTS_DIR_PATH** : Location where you want to store the results @@ -18,6 +18,7 @@ - **MEM_REQ**: Memory request - **CPU_LIM**: CPU limit - **MEM_LIM**: Memory limit +- **ENV_VAR**: Environment variable Example to test with multiple instances diff --git a/acmeair/scripts/perf/run_acmeair_openshift.sh b/acmeair/scripts/perf/run_acmeair_openshift.sh index 79ddf63a..8a9bc8ab 100755 --- a/acmeair/scripts/perf/run_acmeair_openshift.sh +++ b/acmeair/scripts/perf/run_acmeair_openshift.sh @@ -21,10 +21,10 @@ # Ex of ARGS : -s wobbled.os.fyre.ibm.com default -e /acmeair/results -u 400 -d 300 -w 5 -m 3 CURRENT_DIR="$(dirname "$(realpath "$0")")" -pushd "${CURRENT_DIR}" >> setup.log -pushd ".." >> setup.log +pushd "${CURRENT_DIR}" >> /dev/null +pushd ".." >> /dev/null SCRIPT_REPO=${PWD} -pushd ".." >> setup.log +pushd ".." >> /dev/null JMX_FILE="${PWD}/jmeter-driver/acmeair-jmeter/scripts/AcmeAir.jmx" ACMEAIR_DEFAULT_IMAGE="dinogun/acmeair-monolithic" LOG_FILE="${PWD}/logs/jmeter.log" @@ -33,7 +33,7 @@ LOGFILE="${PWD}/logs/setup.log" # Describes the usage of the script function usage() { echo - echo "Usage: $0 -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] " + echo "Usage: $0 -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-a ACMEAIR_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]" exit -1 } @@ -58,6 +58,9 @@ do memlim=*) MEM_LIM=${OPTARG#*=} ;; + env=*) + ENV_VAR=${OPTARG#*=} + ;; *) esac ;; @@ -553,7 +556,7 @@ function runIterations() { for (( itr=0; itr<${TOTAL_ITR}; itr++ )) do if [ ${RE_DEPLOY} == "true" ]; then - ${SCRIPT_REPO}/acmeair-deploy-openshift.sh -s ${BENCHMARK_SERVER} -n ${NAMESPACE} -i ${SCALING} -a ${ACMEAIR_IMAGE} --cpureq=${CPU_REQ} --memreq=${MEM_REQ} --cpulim=${CPU_LIM} --memlim=${MEM_LIM} >> setup.log + ${SCRIPT_REPO}/acmeair-deploy-openshift.sh -s ${BENCHMARK_SERVER} -n ${NAMESPACE} -i ${SCALING} -a ${ACMEAIR_IMAGE} --cpureq=${CPU_REQ} --memreq=${MEM_REQ} --cpulim=${CPU_LIM} --memlim=${MEM_LIM} --env=${ENV_VAR} >> setup.log err_exit "Error: acmeair deployment failed" fi # Start the load diff --git a/galaxies/README.md b/galaxies/README.md index f024a41f..967fbdf7 100644 --- a/galaxies/README.md +++ b/galaxies/README.md @@ -75,7 +75,7 @@ service/galaxies-service-1 created # Openshift To deploy the benchmark use `galaxies-deploy-openshift.sh` -`./scripts/galaxies-deploy-openshift.sh -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM]` +`./scripts/galaxies-deploy-openshift.sh -s BENCHMARK_SERVER [-n NAMESPACE] [-i SERVER_INSTANCES] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]` - **BENCHMARK_SERVER**: Name of the cluster you are using - **SERVER_INSTANCES**: Number of galaxies instances to be deployed. It is optional, if is not specified then by default it will be considered as 1 instance. @@ -85,6 +85,7 @@ To deploy the benchmark use `galaxies-deploy-openshift.sh` - **MEM_REQ**: Memory request - **CPU_LIM**: CPU limit - **MEM_LIM**: Memory limit +- **ENV_VAR**: Environment variable Example to deploy and run galaxies application on openshift cluster @@ -103,6 +104,51 @@ route.route.openshift.io/galaxies-service-1 exposed ``` +# Run the load +Simulating the load on galaxies benchmarks using hyperfoil/wrk +`./scripts/galaxies-load.sh -c CLUSTER_TYPE [-i SERVER_INSTANCES] [--iter MAX_LOOP] [-n NAMESPACE] [-a IP_ADDR] [-t THREAD] [-R REQUEST_RATE] [-d DURATION] [--connection=CONNECTIONS]` + +- **CLUSTER_TYPE**: docker|minikube|openshift +- **SERVER_INSTANCES**: Number of galaxies instances to which you want to run the load. It is optional, if is not specified then by default it will be considered as 1 instance. +- **MAX_LOOP**: Number of times you want to run the load. It is optional, if is not specified then by default it will be considered as 5 iterations. +- **NAMESPACE**: Namespace in which galaxies application is deployed(Required only in the case of openshift cluster and if the application is deployed in other namespaces except `openshift-monitoring`) +- **IP_ADDR**: IP address of the machine. It is optional, if it is not specified then the get_ip function written inside the script will get the IP address of the machine. +- **THREAD**: Number of threads +- **REQUEST_RATE**: Requests rate +- **DURATION**: Test duration +- **CONNECTIONS**: Number of connections + +Example to run the load on 1 galaxies instances for 2 iterations in openshift cluster + +**`$./scripts/galaxies-load.sh -c docker --iter=2`** +``` +######################################################################################### + Starting Iteration 1 +######################################################################################### + +Running wrk load for galaxies instance 1 with the following parameters +CMD=./wrk2.sh --threads=10 --connections=700 --duration=60s --rate=2000 http://192.168.122.105:32000/galaxies +wrk logs Dir : /home/shruthi/galaxies-12-mar/benchmarks/galaxies/logs/galaxies-2021-03-12:12:30 + +######################################################################################### + Starting Iteration 2 +######################################################################################### + +Running wrk load for galaxies instance 1 with the following parameters +CMD=./wrk2.sh --threads=20 --connections=700 --duration=60s --rate=2000 http://192.168.122.105:32000/galaxies +wrk logs Dir : /home/shruthi/galaxies-12-mar/benchmarks/galaxies/logs/galaxies-2021-03-12:12:30 +######################################################################################### + Displaying the results +######################################################################################### +RUN, THROUGHPUT, RESPONSE_TIME, MAX_RESPONSE_TIME, STDDEV_RESPONSE_TIME, ERRORS +1, 2008.90, 1.31, 150.99, 4.40, 0 +2, 2001.05, 939.76, 84.93, 1.84, 0 + +``` +Above image shows the logs of the load run, it processes and displays the output for each run. See Displaying the results section of the log for information about throughput, Number of pages it has retreived, average response time and errors if any. + +To run the load for multiple instances in case of openshift cluster follow [README.md](/galaxies/scripts/perf/README.md) + # Cleanup `$ ./scripts/galaxies-cleanup.sh -c CLUSTER_TYPE[docker|minikube|openshift] [-n NAMESPACE]` diff --git a/galaxies/manifests/galaxies.yaml b/galaxies/manifests/galaxies.yaml index 03d578c1..9aee6a81 100644 --- a/galaxies/manifests/galaxies.yaml +++ b/galaxies/manifests/galaxies.yaml @@ -25,6 +25,7 @@ spec: - name: galaxies image: dinogun/galaxies:1.2-jdk-11.0.10_9 imagePullPolicy: IfNotPresent + env: resources: requests: limits: diff --git a/galaxies/scripts/galaxies-common.sh b/galaxies/scripts/galaxies-common.sh index aa1f4d04..a0f985a8 100755 --- a/galaxies/scripts/galaxies-common.sh +++ b/galaxies/scripts/galaxies-common.sh @@ -17,8 +17,8 @@ ### Script containing common functions ### CURRENT_DIR="$(dirname "$(realpath "$0")")" -pushd "${CURRENT_DIR}" >> setup.log -pushd ".." >> setup.log +pushd "${CURRENT_DIR}" > /dev/null +pushd ".." > /dev/null # Set the defaults for the app export GALAXIES_PORT="32000" @@ -42,6 +42,30 @@ function err_exit() { fi } +# Check if java is installed and it is of version 11 or newer +function check_load_prereq() { + echo + echo -n "Info: Checking prerequisites..." >> ${LOGFILE} + # check if java exists + if [ ! `which java` ]; then + echo " " + echo "Error: java is not installed." + exit 1 + else + JAVA_VER=$(java -version 2>&1 >/dev/null | egrep "\S+\s+version" | awk '{print $3}' | tr -d '"') + case "${JAVA_VER}" in + 1[1-9].*.*) + echo "done" >> ${LOGFILE} + ;; + *) + echo " " + echo "Error: Hyperfoil requires Java 11 or newer and current java version is ${JAVA_VER}" + exit 1 + ;; + esac + fi +} + # Check for all the prereqs function check_prereq() { docker version > ${LOGFILE} 2>&1 @@ -69,7 +93,7 @@ function check_app() { for status in "${CMD[@]}" do if [ -z "${status}" ]; then - echo "Application pod did not come up" + echo "Application pod did not come up" exit -1; fi done @@ -84,11 +108,11 @@ function build_galaxies() { err_exit "Error: Building of docker image of galaxies." } -# Run the galaxies application +# Run the galaxies application # input:galaxies image to be used and JVM arguments if any # output:Create network bridge "kruize-network" and run galaxies application container on the same network function run_galaxies() { - GALAXIES_IMAGE=$1 + GALAXIES_IMAGE=$1 INST=$2 # Create docker network bridge "kruize-network" @@ -98,14 +122,47 @@ function run_galaxies() { echo "Creating Kruize network: ${NETWORK}..." docker network create --driver bridge ${NETWORK} 2>>${LOGFILE} >>${LOGFILE} else - echo "${NETWORK} already exists..." + echo "${NETWORK} already exists..." fi # Run the galaxies app container on "kruize-network" - cmd="docker run -d --name=galaxies-app-${INST} -p ${GALAXIES_PORT}:8080 --network=${NETWORK} ${GALAXIES_IMAGE} " - ${cmd}s 2>>${LOGFILE} >>${LOGFILE} + cmd="docker run -d --name=galaxies-app-${INST} -p ${GALAXIES_PORT}:8080 --network=${NETWORK} ${GALAXIES_IMAGE}" + ${cmd} 2>>${LOGFILE} >>${LOGFILE} err_exit "Error: Unable to start galaxies container" ((GALAXIES_PORT=GALAXIES_PORT+1)) # Check if the application is running check_app } + +# Parse the Throughput Results +# input:result directory , Number of iterations of the wrk load +# output:Throughput log file (tranfer per sec, Number of requests per second, average latency and errors if any) +function parse_galaxies_results() { + RESULTS_DIR=$1 + TOTAL_LOGS=$2 + echo "RUN, THROUGHPUT, RESPONSE_TIME, MAX_RESPONSE_TIME, STDDEV_RESPONSE_TIME, ERRORS" > ${RESULTS_DIR}/Throughput.log + for (( iteration=1 ; iteration<=${TOTAL_LOGS} ;iteration++)) + do + RESULT_LOG=${RESULTS_DIR}/wrk-${inst}-${iteration}.log + throughput=`cat ${RESULT_LOG} | grep "Requests" | cut -d ":" -f2 ` + responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f2 ` + max_responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f6 ` + stddev_responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f4 ` + weberrors=`cat ${RESULT_LOG} | grep "Non-2xx" | cut -d ":" -f2` + if [ "${weberrors}" == "" ]; then + weberrors="0" + fi + echo "${iteration}, ${throughput}, ${responsetime}, ${max_responsetime}, ${stddev_responsetime}, ${weberrors}" >> ${RESULTS_DIR}/Throughput.log + done +} + +# Download the required dependencies +# output: Check if the hyperfoil/wrk dependencies is already present, If not download the required dependencies to apply the load +function load_setup(){ + if [ ! -d "${PWD}/hyperfoil-0.13" ]; then + wget https://github.com/Hyperfoil/Hyperfoil/releases/download/release-0.13/hyperfoil-0.13.zip >> ${LOGFILE} 2>&1 + unzip hyperfoil-0.13.zip + fi + pushd hyperfoil-0.13/bin > /dev/null +} + diff --git a/galaxies/scripts/galaxies-deploy-openshift.sh b/galaxies/scripts/galaxies-deploy-openshift.sh index 763d5d86..25b0e00b 100755 --- a/galaxies/scripts/galaxies-deploy-openshift.sh +++ b/galaxies/scripts/galaxies-deploy-openshift.sh @@ -29,9 +29,9 @@ CLUSTER_TYPE="openshift" # Describes the usage of the script function usage() { echo - echo "Usage: $0 -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] " + echo "Usage: $0 -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]" echo " " - echo "Example: $0 -s rouging.os.fyre.ibm.com -i 2 -g dinogun/galaxies:1.1-jdk-11.0.10_9 --cpulim=4 --cpureq=2 --memlim=1024Mi --memreq=512Mi" + echo "Example: $0 -s rouging.os.fyre.ibm.com -i 2 -g dinogun/galaxies:1.2-jdk-11.0.10_9 --cpulim=4 --cpureq=2 --memlim=1024Mi --memreq=512Mi" exit -1 } @@ -72,16 +72,15 @@ do ;; memreq=*) MEM_REQ=${OPTARG#*=} - # check memory request for unit - check_memory_unit ${MEM_REQ} ;; cpulim=*) CPU_LIM=${OPTARG#*=} ;; memlim=*) MEM_LIM=${OPTARG#*=} - # check memory limit for unit - check_memory_unit ${MEM_LIM} + ;; + env=*) + ENV_VAR=${OPTARG#*=} ;; *) esac @@ -119,6 +118,16 @@ if [ -z "${NAMESPACE}" ]; then NAMESPACE="${DEFAULT_NAMESPACE}" fi +# check memory limit for unit +if [ ! -z "${MEM_LIM}" ]; then + check_memory_unit ${MEM_LIM} +fi + +# check memory request for unit +if [ ! -z "${MEM_REQ}" ]; then + check_memory_unit ${MEM_REQ} +fi + # Create multiple yamls based on instances and Update the template yamls with names and create multiple files # input:galaxies and service-monitor yaml file function createInstances() { @@ -154,6 +163,12 @@ function createInstances() { sed -i '/limits:/a \ \ \ \ \ \ \ \ \ \ cpu: '${CPU_LIM}'' ${MANIFESTS_DIR}/galaxies-${inst}.yaml fi + # Pass environment variables + if [ ! -z ${ENV_VAR} ]; then + sed -i '/env:/a \ \ \ \ \ \ \ \ - name: "JVM_ARGS"' ${MANIFESTS_DIR}/galaxies-${inst}.yaml + sed -i '/- name: "JVM_ARGS"/a \ \ \ \ \ \ \ \ \ \ value: '${ENV_VAR}'' ${MANIFESTS_DIR}/galaxies-${inst}.yaml + fi + oc create -f ${MANIFESTS_DIR}/galaxies-${inst}.yaml -n ${NAMESPACE} err_exit "Error: Issue in deploying." ((GALAXIES_PORT=GALAXIES_PORT+1)) diff --git a/galaxies/scripts/galaxies-load.sh b/galaxies/scripts/galaxies-load.sh new file mode 100755 index 00000000..a9060b20 --- /dev/null +++ b/galaxies/scripts/galaxies-load.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020, 2021 Red Hat, IBM Corporation and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### Script to load test galaxies application on docker,minikube or openshift### +# + +CURRENT_DIR="$(dirname "$(realpath "$0")")" +source ${CURRENT_DIR}/galaxies-common.sh + +function usage() { + echo + echo "Usage: -c CLUSTER_TYPE[docker|minikube|openshift] [-i SERVER_INSTANCES] [--iter=MAX_LOOP] [-n NAMESPACE] [-a IP_ADDR] [-t THREAD] [-R REQUEST_RATE] [-d DURATION] [--connection=CONNECTIONS]" + exit -1 +} + +while getopts c:i:a:n:t:R:d:-: gopts +do + case ${gopts} in + -) + case "${OPTARG}" in + iter=*) + MAX_LOOP=${OPTARG#*=} + ;; + connection=*) + CONNECTIONS=${OPTARG#*=} + ;; + esac + ;; + c) + CLUSTER_TYPE=${OPTARG} + ;; + i) + SERVER_INSTANCES="${OPTARG}" + ;; + a) + IP_ADDR="${OPTARG}" + ;; + n) + NAMESPACE="${OPTARG}" + ;; + t) + THREAD="${OPTARG}" + ;; + R) + REQUEST_RATE="${OPTARG}" + ;; + d) + DURATION="${OPTARG}" + ;; + esac +done + +if [ -z "${CLUSTER_TYPE}" ]; then + usage +fi + +if [ -z "${SERVER_INSTANCES}" ]; then + SERVER_INSTANCES=1 +fi + +if [ -z "${MAX_LOOP}" ]; then + MAX_LOOP=5 +fi + +if [ -z "${NAMESPACE}" ]; then + NAMESPACE="${DEFAULT_NAMESPACE}" +fi + +if [ -z "${THREAD}" ]; then + THREAD="10" +fi + +if [ -z "${REQUEST_RATE}" ]; then + REQUEST_RATE="2000" +fi + +if [ -z "${DURATION}" ]; then + DURATION="60" +fi + +if [ -z "${CONNECTIONS}" ]; then + CONNECTIONS="700" +fi + +case ${CLUSTER_TYPE} in +docker) + if [ -z "${IP_ADDR}" ]; then + get_ip + fi + ;; +icp|minikube) + if [ -z "${IP_ADDR}" ]; then + IP_ADDR=$(minikube service list | grep "galaxies" | awk '{print $8}' | tr '\r\n' ' ') + IFS=' ' read -r -a IP_ADDR <<< ${IP_ADDR} + fi + ;; +openshift) + if [ -z "${IP_ADDR}" ]; then + IP_ADDR=($(oc status --namespace=${NAMESPACE} | grep "galaxies" | grep port | cut -d " " -f1 | cut -d "/" -f3)) + fi + ;; +*) + echo "Load is not determined" + ;; +esac + +LOG_DIR="${PWD}/logs/galaxies-$(date +%Y-%m-%d:%H:%M)" +mkdir -p ${LOG_DIR} + +# Check if java is installed and it is of version 11 or newer +check_load_prereq >> setup.log + +# Create the required load setup +load_setup >> setup.log + +for(( inst=1; inst<=${SERVER_INSTANCES}; inst++ )) +do + for iter in `seq 1 ${MAX_LOOP}` + do + echo + echo "#########################################################################################" + echo " Starting Iteration ${iter} " + echo "#########################################################################################" + echo + + # Change these appropriately to vary load + USERS=$(( THREAD*iter )) + + case "${CLUSTER_TYPE}" in + docker) + cmd="./wrk2.sh --threads=${USERS} --connections=${CONNECTIONS} --duration=${DURATION}s --rate=${REQUEST_RATE} http://${IP_ADDR}:${GALAXIES_PORT}/galaxies" + ;; + minikube) + cmd="./wrk2.sh --threads=${USERS} --connections=${CONNECTIONS} --duration=${DURATION}s --rate=${REQUEST_RATE} ${IP_ADDR[inst-1]}/galaxies" + ;; + openshift) + cmd="./wrk2.sh --threads=${USERS} --connections=${CONNECTIONS} --duration=${DURATION}s --rate=${REQUEST_RATE} http://${IP_ADDR[inst-1]}/galaxies" + ;; + esac + + # Check if the application is running + check_app + # Run the wrk load + echo "Running wrk load for galaxies instance ${inst} with the following parameters" | tee -a ${LOG_DIR}/wrk-${inst}-${iter}.log + echo "CMD=${cmd}" | tee -a ${LOG_DIR}/wrk-${inst}-${iter}.log + echo "wrk logs Dir : ${LOG_DIR}" + sleep 20 + ${cmd} >> ${LOG_DIR}/wrk-${inst}-${iter}.log + err_exit "can not execute the command" + done + ((GALAXIES_PORT=GALAXIES_PORT+1)) + # Parse the results + parse_galaxies_results ${LOG_DIR} ${MAX_LOOP} + echo "#########################################################################################" + echo " Displaying the results " + echo "#########################################################################################" + cat ${LOG_DIR}/Throughput.log +done diff --git a/galaxies/scripts/perf/README.md b/galaxies/scripts/perf/README.md new file mode 100755 index 00000000..fa0b2b20 --- /dev/null +++ b/galaxies/scripts/perf/README.md @@ -0,0 +1,43 @@ +# Test with multiple instances + +`./scripts/perf/run_galaxies_openshift.sh -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq MEM_REQ] [--cpulim=CPU_LIM] [--memlim MEM_LIM] [-t THREAD] [-R REQUEST_RATE] [-d DURATION] [--connection=CONNECTIONS] [--env=ENV_VAR]` + +- **BENCHMARK_SERVER_NAME** : Name of the cluster you are using +- **RESULTS_DIR_PATH** : Location where you want to store the results +- **JMETER_LOAD_USERS** : Number of users +- **JMETER_LOAD_DURATION** : Load duration +- **WARMUPS** : Number of warmups +- **MEASURES** : Number of measures + +- **TOTAL_INST**: Number of instances +- **TOTAL_ITR**: Number of iterations you want to do the benchmarking +- **RE_DEPLOY**: true +- **NAMESPACE** : Namespace in which application to be deployed. It is optional, if not specified then `openshift-monitoring` will be considered as the namespace. +- **GALAXIES_IMAGE**: galaxies image to be used during deployment. It is optional, if not specified then the default image `dinogun/galaxies:1.2-jdk-11.0.10_9` will be used for the deployment. +- **CPU_REQ**: CPU request +- **MEM_REQ**: Memory request +- **CPU_LIM**: CPU limit +- **MEM_LIM**: Memory limit +- **THREAD**: Number of threads +- **REQUEST_RATE**: Requests rate +- **DURATION**: Test duration +- **CONNECTIONS**: Number of connections +- **ENV_VAR**: Environment variable + +Example to test with multiple instances + +**`$./scripts/perf/run-galaxies-openshift.sh -s rouging.os.fyre.ibm.com -e result -u 10 -d 10 -w 1 -m 1 -i 1 --iter=1 -r `** + +``` +Instances , Throughput , Responsetime , MEM_MEAN , CPU_MEAN , CPU_MIN , CPU_MAX , MEM_MIN , MEM_MAX , CLUSTER_MEM% , CLUSTER_CPU% , CPU_REQ , MEM_REQ , CPU_LIM , MEM_LIM , WEB_ERRORS , RESPONSETIME_MAX , STDEV_RESPTIME_MAX +1 , 585.023 , 622.883 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , , , , , , , 619.44 , , +Run , CPU_REQ , MEM_REQ , CPU_LIM , MEM_LIM , Throughput , Responsetime , WEB_ERRORS , Responsetime_MAX , stdev_responsetime_max , CPU , CPU_MIN , CPU_MAX , MEM , MEM_MIN , MEM_MAX +0 , , , , , 858.80 , 444.16 , 122600 , 2.28 , 313.66 ,0 , 0 , 0 , 0 , 0 , 0 +1 , , , , , 586.11 , 859.73 , 63260 , 4.60 , 619.44 ,0 , 0 , 0 , 0 , 0 , 0 +2 , , , , , 310.16 , 564.76 , 35740 , 1.94 , 280.60 ,0 , 0 , 0 , 0 , 0 , 0 + +``` +Above image shows the log of the load run i.e, throghput, response time, total memory used by the pod, total cpu used by the pod, minimum cpu, maximum cpu, minimum memory, maximum memory, cluster memory usage in percentage, cluster cpu in percentage, web errors if any. + +For CPU and Memory details refer Metrics-cpu.log and Metrics-mem.log . And for individual informations look into the logs generated during the run. + diff --git a/galaxies/scripts/perf/getstats-openshift.sh b/galaxies/scripts/perf/getstats-openshift.sh new file mode 100755 index 00000000..19a894ad --- /dev/null +++ b/galaxies/scripts/perf/getstats-openshift.sh @@ -0,0 +1,348 @@ +#!/bin/bash +# +# Copyright (c) 2020, 2021 IBM Corporation, RedHat and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### Script to get pod and cluster information through prometheus queries### +# + +# checks if the previous command is executed successfully +# input:Return value of previous command +# output:Prompts the error message if the return value is not zero +function err_exit() +{ + if [ $? != 0 ]; then + printf "$*" + echo + exit -1 + fi +} + +# Get the memory details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory information for pod through prometheus query and store it in json file +function get_pod_mem_rss() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + echo "APP_NAME is ..." ${APP_NAME} >> setup.log + rm -rf ${RESULTS_DIR}/${node}_mem-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_memory_rss{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_mem-${ITER}.json + err_exit "Error: could not get memory details of the pod" >>setup.log + done +} + +# Get the memory usage details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory usage information for pod through prometheus query and store it in json file +function get_pod_mem_usage() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_memusage-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_memory_working_set_bytes{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_memusage-${ITER}.json + err_exit "Error: could not get memory usage details of pod" >>setup.log + done +} + +# Get the memory request details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory request information for pod through prometheus query and store it in json file +function get_pod_mem_requests() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_memrequests-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_requests_memory_bytes{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_memrequests-${ITER}.json + err_exit "Error: could not get memory request details of pod" >>setup.log + done +} + +# Get the memory usage details for pod in percentage +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory usage information in percentage through prometheus query and store it in json file +function get_pod_mem_requests_in_p() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_memreq_in_p-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_memory_working_set_bytes{node='\"${node}\"'}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_memreq_in_p-${ITER}.json + err_exit "Error: could not get memory request details of pod in percentage" >>setup.log + done +} + +# Get the memory limit details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory limit information for pod through prometheus query and store it in json file +function get_pod_mem_limits() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_memlimits-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_limits_memory_bytes{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_memlimits-${ITER}.json + err_exit "Error: could not get memory limit details of pod" >>setup.log + done +} + +# Get the memory limit details for pod in percentage +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the memory percentage information in percentage through prometheus query and store it in json file +function get_pod_mem_limits_in_p() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_memlimit_in_p-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_memory_working_set_bytes{node='\"${node}\"'}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_memlimit_in_p-${ITER}.json + err_exit "Error: could not get memory limit details of pod in percentage" >>setup.log + done +} + +# Get the cpu usage details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cpu usage information for pod through prometheus query and store it in json file +function get_pod_cpu_usage() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_cpu-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + #Get all pods data from 1 node using single command + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_cpu-${ITER}.json + err_exit "Error: could not get CPU usage details of pod" >>setup.log + done +} + +# Get the cpu request details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cpu request information for pod through prometheus query and store it in json file +function get_pod_cpu_requests() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_cpurequests-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + #Get all pods data from 1 node using single command + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_requests_cpu_cores{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_cpurequests-${ITER}.json + err_exit "Error: could not get CPU request details of pod" >>setup.log + done +} + +# Get the cpu request details for pod in percentage +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cpu request information in percentage through prometheus query and store it in json file +function get_pod_cpu_requests_in_p() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_cpureq_in_p-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + #Get all pods data from 1 node using single command + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{node='\"${node}\"'}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_cpureq_in_p-${ITER}.json + err_exit "Error: could not get CPU request details of pod in percentage" >>setup.log + done +} + +# Get the cpu limits details for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cpu limits information for pod through prometheus query and store it in json file +function get_pod_cpu_limits() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_cpulimits-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + #Get all pods data from 1 node using single command + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_limits_cpu_cores{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_cpulimits-${ITER}.json + err_exit "Error: could not get CPU limit details of pod" >>setup.log + done +} + +# Get the cpu limits details for pod in percentage +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cpu limits information in percentage through prometheus query and store it in json file +function get_pod_cpu_limits_in_p() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + # Delete the old json file if any + rm -rf ${RESULTS_DIR}/${node}_cpulimits_in_p-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + #Get all pods data from 1 node using single command + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{node='\"${node}\"'}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{node='\"${node}\"'}) by (pod)' ${URL} | jq '[ .data.result[] | [ .value[0], .metric.pod, .value[1]|tostring] | join(";") ]' | grep "${APP_NAME}" >> ${RESULTS_DIR}/${node}_cpulimits_in_p-${ITER}.json + err_exit "Error: could not get CPU limit details of pod in percentage" >>setup.log + done +} + +# Get the cluster information for pod +# input:worker node, prometheus url, authorization token, result directory, application name +# output:generate the cluster information for pod through prometheus query and store it in json file +function get_cluster_info() +{ + node=$1 + URL=$2 + TOKEN=$3 + RESULTS_DIR=$4 + ITER=$5 + APP_NAME=$6 + rm -rf ${RESULTS_DIR}/cluster_cpu-${ITER}.json ${RESULTS_DIR}/cluster_mem-${ITER}.json ${RESULTS_DIR}/cluster_cpurequests-${ITER}.json ${RESULTS_DIR}/cluster_cpulimits-${ITER}.json ${RESULTS_DIR}/cluster_memrequests-${ITER}.json ${RESULTS_DIR}/cluster_memlimits-${ITER}.json + while true + do + # Processing curl output "timestamp value" using jq tool. + # Cluster MEm Usage + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=(1-sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable_memory_bytes))' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_mem-${ITER}.json + err_exit "Error: could not get cluster memory usage details" >>setup.log + + # Cluster CPU Usage + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=(1-avg(rate(node_cpu_seconds_total{mode="idle"}[1m])))' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_cpu-${ITER}.json + err_exit "Error: could not get cluster CPU usage details" >>setup.log + + # CPU Requests COmmited + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_requests_cpu_cores) / sum(kube_node_status_allocatable_cpu_cores)' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_cpurequests-${ITER}.json + err_exit "Error: could not get cluster CPU request details" >>setup.log + + # CPU Limits Commited + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_limits_cpu_cores) / sum(kube_node_status_allocatable_cpu_cores)' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_cpulimits-${ITER}.json + err_exit "Error: could not get cluster CPU limits details" >>setup.log + + # Mem Requests Commited + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_requests_memory_bytes) / sum(kube_node_status_allocatable_memory_bytes)' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_memrequests-${ITER}.json + err_exit "Error: could not get cluster memory request details" >>setup.log + + # Mem Limits Commited + curl --silent -G -kH "Authorization: Bearer ${TOKEN}" --data-urlencode 'query=sum(kube_pod_container_resource_limits_memory_bytes) / sum(kube_node_status_allocatable_memory_bytes)' ${URL} | jq '[ .data.result[] | [ .value[0] , .value[1]|tostring ] | join(";") ]' >> ${RESULTS_DIR}/c_memlimits-${ITER}.json + err_exit "Error: could not get cluster memory limits details" >>setup.log + done +} + +ITER=$1 +TIMEOUT=$2 +RESULTS_DIR=$3 +mkdir -p ${RESULTS_DIR} + +PROMETHEUS_APP=prometheus-k8s-openshift-monitoring.apps +BENCHMARK_SERVER=$4 +APP_NAME=$5 + +URL=https://${PROMETHEUS_APP}.${BENCHMARK_SERVER}/api/v1/query +#URL=https://prometheus-k8s-openshift-monitoring.apps.${BENCHMARK_SERVER}/api/v1/query +TOKEN=`oc whoami --show-token` + +worker_nodes=($(oc get nodes | grep worker | cut -d " " -f1)) +#Need to export the function to enable timeout +export -f err_exit +export -f get_pod_mem_rss get_pod_mem_usage get_pod_mem_requests get_pod_mem_requests_in_p get_pod_mem_limits get_pod_mem_limits_in_p +export -f get_pod_cpu_usage get_pod_cpu_requests get_pod_cpu_requests_in_p get_pod_cpu_limits get_pod_cpu_limits_in_p +export -f get_cluster_info + +for i in "${worker_nodes[@]}" +do + echo "Collecting CPU & MEM details of nodes ${i} and cluster" >> setup.log + timeout ${TIMEOUT} bash -c "get_pod_mem_rss ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_cpu_usage ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_cluster_info ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + + timeout ${TIMEOUT} bash -c "get_pod_mem_usage ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_mem_requests ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_mem_requests_in_p ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_mem_limits ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_mem_limits_in_p ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + + timeout ${TIMEOUT} bash -c "get_pod_cpu_requests ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_cpu_requests_in_p ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_cpu_limits ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & + timeout ${TIMEOUT} bash -c "get_pod_cpu_limits_in_p ${i} ${URL} ${TOKEN} ${RESULTS_DIR} ${ITER} ${APP_NAME}" & +done + diff --git a/galaxies/scripts/perf/run-galaxies-openshift.sh b/galaxies/scripts/perf/run-galaxies-openshift.sh new file mode 100755 index 00000000..6447c084 --- /dev/null +++ b/galaxies/scripts/perf/run-galaxies-openshift.sh @@ -0,0 +1,629 @@ +#!/bin/bash +# +# Copyright (c) 2020, 2021 Red Hat, IBM Corporation and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### Script to perform load test on multiple instances of galaxies on openshift### +# + +CURRENT_DIR="$(dirname "$(realpath "$0")")" +pushd "${CURRENT_DIR}" > /dev/null +pushd ".." > /dev/null +SCRIPT_REPO=${PWD} + +CLUSTER_TYPE="openshift" +pushd ".." > /dev/null +HYPERFOIL_DIR="${PWD}/hyperfoil-0.13/bin" +GALAXIES_DEFAULT_IMAGE="dinogun/galaxies:1.2-jdk-11.0.10_9" + +# checks if the previous command is executed successfully +# input:Return value of previous command +# output:Prompts the error message if the return value is not zero +function err_exit() { + if [ $? != 0 ]; then + printf "$*" + echo + echo "See setup.log for more details" + exit -1 + fi +} + +# Run the benchmark as +# SCRIPT BENCHMARK_SERVER_NAME NAMESPACE RESULTS_DIR_PATH WRK_LOAD_USERS WRK_LOAD_DURATION WARMUPS MEASURES +# Ex of ARGS : -s wobbled.os.fyre.ibm.com -e /galaxies/results -u 400 -d 300 -w 5 -m 3 + +# Describes the usage of the script +function usage() { + echo + echo "Usage: $0 -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u WRK_LOAD_USERS] [-d WRK_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-g GALAXIES_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [-t THREAD] [-R REQUEST_RATE] [-d DURATION] [--connection=CONNECTIONS] [--env=ENV_VAR]" + exit -1 +} + +# Check if java is installed and it is of version 11 or newer +function check_load_prereq() { + echo + echo -n "Info: Checking prerequisites..." + # check if java exists + if [ ! `which java` ]; then + echo " " + echo "Error: java is not installed." + exit 1 + else + JAVA_VER=$(java -version 2>&1 >/dev/null | egrep "\S+\s+version" | awk '{print $3}' | tr -d '"') + case "${JAVA_VER}" in + 1[1-9].*.*) + echo "done" + ;; + *) + echo " " + echo "Error: Hyperfoil requires Java 11 or newer and current java version is ${JAVA_VER}" + exit 1 + ;; + esac + fi +} + +# Iterate through the commandline options +while getopts s:e:w:m:i:rg:n:t:R:d:-: gopts +do + case ${gopts} in + -) + case "${OPTARG}" in + iter=*) + TOTAL_ITR=${OPTARG#*=} + ;; + cpureq=*) + CPU_REQ=${OPTARG#*=} + ;; + memreq=*) + MEM_REQ=${OPTARG#*=} + ;; + cpulim=*) + CPU_LIM=${OPTARG#*=} + ;; + memlim=*) + MEM_LIM=${OPTARG#*=} + ;; + connection=*) + CONNECTIONS=${OPTARG#*=} + ;; + env=*) + ENV_VAR=${OPTARG#*=} + ;; + *) + esac + ;; + s) + BENCHMARK_SERVER="${OPTARG}" + ;; + e) + RESULTS_DIR_PATH="${OPTARG}" + ;; + w) + WARMUPS="${OPTARG}" + ;; + m) + MEASURES="${OPTARG}" + ;; + i) + TOTAL_INST="${OPTARG}" + ;; + r) + RE_DEPLOY="true" + ;; + g) + GALAXIES_IMAGE="${OPTARG}" + ;; + n) + NAMESPACE="${OPTARG}" + ;; + t) + THREAD="${OPTARG}" + ;; + R) + REQUEST_RATE="${OPTARG}" + ;; + d) + DURATION="${OPTARG}" + ;; + esac +done + +if [[ -z "${BENCHMARK_SERVER}" || -z "${RESULTS_DIR_PATH}" ]]; then + echo "Do set the variables - BENCHMARK_SERVER and RESULTS_DIR_PATH " + usage +fi + +if [ -z "${WARMUPS}" ]; then + WARMUPS=5 +fi + +if [ -z "${MEASURES}" ]; then + MEASURES=3 +fi + +if [ -z "${TOTAL_INST}" ]; then + TOTAL_INST=1 +fi + +if [ -z "${TOTAL_ITR}" ]; then + TOTAL_ITR=1 +fi + +if [ -z "${RE_DEPLOY}" ]; then + RE_DEPLOY=false +fi + +if [ -z "${GALAXIES_IMAGE}" ]; then + GALAXIES_IMAGE="${GALAXIES_DEFAULT_IMAGE}" +fi + +if [ -z "${NAMESPACE}" ]; then + NAMESPACE="openshift-monitoring" +fi + +if [ -z "${REQUEST_RATE}" ]; then + REQUEST_RATE="2000" +fi + +if [ -z "${THREAD}" ]; then + THREAD="40" +fi + +if [ -z "${DURATION}" ]; then + DURATION="60" +fi + +if [ -z "${CONNECTIONS}" ]; then + CONNECTIONS="700" +fi + +# Check if the dependencies required to apply the load is present +check_load_prereq + +# Check if the application is running +# output: Returns 1 if the application is running else returns 0 +function check_app() { + CMD=$(oc get pods --namespace=${NAMESPACE} | grep "galaxies" | grep "Running" | cut -d " " -f1) + for status in "${CMD[@]}" + do + if [ -z "${status}" ]; then + echo "Application pod did not come up" + exit -1; + fi + done +} + +RESULTS_DIR_ROOT=${RESULTS_DIR_PATH}/galaxies-$(date +%Y%m%d%H%M) +mkdir -p ${RESULTS_DIR_ROOT} + +#Adding 10 secs buffer to retrieve CPU and MEM info +CPU_MEM_DURATION=`expr ${DURATION} + 10` + +throughputlogs=(throughput responsetime weberrorresponsetime_max stdev_resptime_max) +podcpulogs=(cpu cpurequests cpulimits cpureq_in_p cpulimits_in_p) +podmemlogs=(mem memusage memrequests memlimits memreq_in_p memlimit_in_p) +clusterlogs=(c_mem c_cpu c_cpulimits c_cpurequests c_memlimits c_memrequests) +total_logs=(${throughputlogs[@]} ${podcpulogs[@]} ${podmemlogs[@]} ${clusterlogs[@]} cpu_min cpu_max mem_min mem_max) + +# Download the required dependencies +# output: Check if the hyperfoil/wrk dependencies is already present, If not download the required dependencies to apply the load +function load_setup(){ + if [ ! -d "${PWD}/hyperfoil-0.13" ]; then + wget https://github.com/Hyperfoil/Hyperfoil/releases/download/release-0.13/hyperfoil-0.13.zip >> setup.log 2>&1 + err_exit "Error: Could not download the dependencies" + unzip hyperfoil-0.13.zip >> setup.log + fi +} + +# Run the wrk load +# input: machine IP address, Result log file +# output: Run the wrk load on galaxies application and store the result in log file +function run_wrk_workload() { + # Store results in this file + IP_ADDR=$1 + RESULTS_LOG=$2 + # Run the wrk load + echo "Running wrk load with the following parameters" >> setup.log + cmd="${HYPERFOIL_DIR}/wrk2.sh --threads=${THREAD} --connections=${CONNECTIONS} --duration=${DURATION}s --rate=${REQUEST_RATE} http://${IP_ADDR}/galaxies" + echo "CMD = ${cmd}" >> setup.log + sleep 30 + ${cmd} > ${RESULTS_LOG} + sleep 20 +} + +# Run the wrk load on each instace of the application +# input: Result directory, Type of run(warmup|measure), iteration number +# output: call the run_wrk_workload for each application service +function run_wrk_with_scaling() +{ + RESULTS_DIR_J=$1 + TYPE=$2 + RUN=$3 + svc_apis=($(oc status --namespace=${NAMESPACE} | grep "galaxies" | grep port | cut -d " " -f1 | cut -d "/" -f3)) + load_setup + for svc_api in "${svc_apis[@]}" + do + RESULT_LOG=${RESULTS_DIR_J}/wrk-${svc_api}-${TYPE}-${RUN}.log + run_wrk_workload ${svc_api} ${RESULT_LOG} & + done +} + +# Parse the result log files +# input:Type of run(warmup|measure), total number of runs, total number of iteration +# output:Throughput log file(throghput, response time, total memory used by the pod, total cpu used by the pod, cluster memory usage in percentage,cluster cpu in percentage and web errors if any) +function parseData() { + TYPE=$1 + TOTAL_RUNS=$2 + ITR=$3 + echo "${TYPE} Runs" >> ${RESULTS_DIR_J}/Throughput-${itr}.log + for (( run=0 ; run<${TOTAL_RUNS} ;run++)) + do + thrp_sum=0 + resp_sum=0 + wer_sum=0 + svc_apis=($(oc status --namespace=${NAMESPACE} | grep "galaxies" | grep port | cut -d " " -f1 | cut -d "/" -f3)) + for svc_api in "${svc_apis[@]}" + do + RESULT_LOG=${RESULTS_DIR_P}/wrk-${svc_api}-${TYPE}-${run}.log + throughput=`cat ${RESULT_LOG} | grep "Requests" | cut -d ":" -f2 ` + responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f2 ` + max_responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f6 ` + stddev_responsetime=`cat ${RESULT_LOG} | grep "Latency" | cut -d ":" -f2 | tr -s " " | cut -d " " -f4 ` + weberrors=`cat ${RESULT_LOG} | grep "Non-2xx" | cut -d ":" -f2` + thrp_sum=$(echo ${thrp_sum}+${throughput} | bc) + resp_sum=$(echo ${resp_sum}+${responsetime} | bc) + if [ "${weberrors}" != "" ]; then + wer_sum=`expr ${wer_sum} + ${weberrors}` + fi + done + echo "${run},${thrp_sum},${resp_sum},${wer_sum},${max_responsetime},${stddev_responsetime}" >> ${RESULTS_DIR_J}/Throughput-${TYPE}-${itr}.log + echo "${run} , ${CPU_REQ} , ${MEM_REQ} , ${CPU_LIM} , ${MEM_LIM} , ${thrp_sum} , ${responsetime} , ${wer_sum} , ${max_responsetime} , ${stddev_responsetime}" >> ${RESULTS_DIR_J}/Throughput-${TYPE}-raw.log + done +} + +# Parse CPU, memeory and cluster information +# input:type of run(warmup|measure), total number of runs, iteration number +# output:Creates cpu, memory and cluster information in the form of log files for each run +function parseCpuMem() { + TYPE=$1 + TOTAL_RUNS=$2 + ITR=$3 + + echo "${TYPE} Runs" >> ${RESULTS_DIR_J}/Mem-${itr}.log + for (( run=0 ; run<${TOTAL_RUNS} ;run++)) + do + for podcpulog in "${podcpulogs[@]}" + do + # Parsing CPU logs for pod + parsePodCpuLog ${podcpulog} ${TYPE} ${run} ${ITR} + done + for podmemlog in "${podmemlogs[@]}" + do + # Parsing Mem logs for pod + parsePodMemLog ${podmemlog} ${TYPE} ${run} ${ITR} + done + for clusterlog in "${clusterlogs[@]}" + do + # Parsing Cluster logs + parseClusterLog ${clusterlog} ${RESULTS_DIR_P}/${clusterlog}-${TYPE}-${run}.json ${clusterlog}-${TYPE}-${itr}.log + done + done +} + +# Parsing CPU logs for pod +# input: podcpulogs array element, type of run(warmup|measure), run(warmup|measure) number, iteration number +# output:creates cpu log for pod +function parsePodCpuLog() +{ + MODE=$1 + TYPE=$2 + run=$3 + ITR=$4 + RESULTS_LOG=${MODE}-${TYPE}-${ITR}.log + cpu_sum=0 + cpu_min=0 + cpu_max=0 + t_nodes=($(oc get nodes | grep worker | cut -d " " -f1)) + for t_node in "${t_nodes[@]}" + do + CPU_LOG=${RESULTS_DIR_P}/${t_node}_${MODE}-${TYPE}-${run}.json + run_pods=($(cat ${CPU_LOG} | cut -d ";" -f2 | sort | uniq)) + for run_pod in "${run_pods[@]}" + do + cat ${CPU_LOG} | grep ${run_pod} | cut -d ";" -f3 | cut -d '"' -f1 > ${RESULTS_DIR_P}/temp-cpu.log + each_pod_cpu_avg=$( echo `calcAvg ${RESULTS_DIR_P}/temp-cpu.log | cut -d "=" -f2` ) + each_pod_cpu_min=$( echo `calcMin ${RESULTS_DIR_P}/temp-cpu.log` ) + each_pod_cpu_max=$( echo `calcMax ${RESULTS_DIR_P}/temp-cpu.log` ) + cpu_sum=$(echo ${cpu_sum}+${each_pod_cpu_avg}| bc) + cpu_min=$(echo ${cpu_min}+${each_pod_cpu_min}| bc) + cpu_max=$(echo ${cpu_max}+${each_pod_cpu_max} | bc) + done + done + echo "${run} , ${cpu_sum}, ${cpu_min} , ${cpu_max}" >> ${RESULTS_DIR_J}/${RESULTS_LOG} + echo ",${cpu_sum} , ${cpu_min} , ${cpu_max}" >> ${RESULTS_DIR_J}/${MODE}-${TYPE}-raw.log +} + +# Parsing memory logs for pod +# input: podmemlogs array element, type of run(warmup|measure), run(warmup|measure) number, iteration number +# output:creates memory log for pod +function parsePodMemLog() +{ + MODE=$1 + TYPE=$2 + run=$3 + ITR=$4 + RESULTS_LOG=${MODE}-${TYPE}-${ITR}.log + mem_sum=0 + mem_min=0 + mem_max=0 + t_nodes=($(oc get nodes | grep worker | cut -d " " -f1)) + for t_node in "${t_nodes[@]}" + do + MEM_LOG=${RESULTS_DIR_P}/${t_node}_${MODE}-${TYPE}-${run}.json + mem_pods=($(cat ${MEM_LOG} | cut -d ";" -f2 | sort | uniq)) + for mem_pod in "${mem_pods[@]}" + do + cat ${MEM_LOG} | grep ${mem_pod} | cut -d ";" -f3 | cut -d '"' -f1 > ${RESULTS_DIR_P}/temp-mem.log + if [ ${MODE} == "memreq_in_p" ] || [ ${MODE} == "memlimit_in_p" ]; then + each_pod_mem_avg=$( echo `calcAvg_in_p ${RESULTS_DIR_P}/temp-mem.log | cut -d "=" -f2` ) + else + each_pod_mem_avg=$( echo `calcAvg_inMB ${RESULTS_DIR_P}/temp-mem.log | cut -d "=" -f2` ) + each_pod_mem_min=$( echo `calcMin ${RESULTS_DIR_P}/temp-mem.log` ) + each_pod_mem_min_inMB=$(echo ${each_pod_mem_min}/1024/1024 | bc) + each_pod_mem_max=$( echo `calcMax ${RESULTS_DIR_P}/temp-mem.log` ) + each_pod_mem_max_inMB=$(echo ${each_pod_mem_max}/1024/1024 | bc) + fi + mem_sum=$(echo ${mem_sum}+${each_pod_mem_avg} | bc) + mem_min=$(echo ${mem_min}+${each_pod_mem_min_inMB} | bc) + mem_max=$(echo ${mem_max}+${each_pod_mem_max_inMB} | bc) + done + done + echo "${run} , ${mem_sum}, ${mem_min} , ${mem_max} " >> ${RESULTS_DIR_J}/${RESULTS_LOG} + echo ", ${mem_sum} , ${mem_min} , ${mem_max} " >> ${RESULTS_DIR_J}/${MODE}-${TYPE}-raw.log +} + +# Parsing memory logs for pod +# input: clusterlogs array element, json file with cluster information, result log file +# output:creates clsuter log file +function parseClusterLog() { + MODE=$1 + CLUSTER_LOG=$2 + CLUSTER_RESULTS_LOG=$3 + cat ${CLUSTER_LOG}| cut -d ";" -f2 | cut -d '"' -f1 | grep -Eo '[0-9\.]+' > C_temp.log + cluster_cpumem=$( echo `calcAvg_in_p C_temp.log | cut -d "=" -f2` ) + echo "${run} , ${cluster_cpumem}" >> ${RESULTS_DIR_J}/${CLUSTER_RESULTS_LOG} +} + +# Parse the results of wrk load for each instance of application +# input: total number of iterations, result directory, Total number of instances +# output: Parse the results and generate the Metrics log files +function parseResults() { + TOTAL_ITR=$1 + RESULTS_DIR_J=$2 + sca=$3 + for (( itr=0 ; itr<${TOTAL_ITR} ;itr++)) + do + RESULTS_DIR_P=${RESULTS_DIR_J}/ITR-${itr} + parseData warmup ${WARMUPS} ${itr} + parseData measure ${MEASURES} ${itr} + parseCpuMem warmup ${WARMUPS} ${itr} + parseCpuMem measure ${MEASURES} ${itr} + + #Calculte Average and Median of Throughput, Memory and CPU scores + cat ${RESULTS_DIR_J}/Throughput-measure-${itr}.log | cut -d "," -f2 >> ${RESULTS_DIR_J}/throughput-measure-temp.log + cat ${RESULTS_DIR_J}/Throughput-measure-${itr}.log | cut -d "," -f3 >> ${RESULTS_DIR_J}/responsetime-measure-temp.log + cat ${RESULTS_DIR_J}/Throughput-measure-${itr}.log | cut -d "," -f4 >> ${RESULTS_DIR_J}/weberror-measure-temp.log + cat ${RESULTS_DIR_J}/Throughput-measure-${itr}.log | cut -d "," -f5 >> ${RESULTS_DIR_J}/responsetime_max-measure-temp.log + cat ${RESULTS_DIR_J}/Throughput-measure-${itr}.log | cut -d "," -f6 >> ${RESULTS_DIR_J}/stdev_resptime_max-measure-temp.log + + for podcpulog in "${podcpulogs[@]}" + do + cat ${RESULTS_DIR_J}/${podcpulog}-measure-${itr}.log | cut -d "," -f2 >> ${RESULTS_DIR_J}/${podcpulog}-measure-temp.log + cat ${RESULTS_DIR_J}/${podcpulog}-measure-${itr}.log | cut -d "," -f3 >> ${RESULTS_DIR_J}/${podcpulog}_min-measure-temp.log + cat ${RESULTS_DIR_J}/${podcpulog}-measure-${itr}.log | cut -d "," -f4 >> ${RESULTS_DIR_J}/${podcpulog}_max-measure-temp.log + done + for podmemlog in "${podmemlogs[@]}" + do + cat ${RESULTS_DIR_J}/${podmemlog}-measure-${itr}.log | cut -d "," -f2 >> ${RESULTS_DIR_J}/${podmemlog}-measure-temp.log + cat ${RESULTS_DIR_J}/${podmemlog}-measure-${itr}.log | cut -d "," -f3 >> ${RESULTS_DIR_J}/${podmemlog}_min-measure-temp.log + cat ${RESULTS_DIR_J}/${podmemlog}-measure-${itr}.log | cut -d "," -f4 >> ${RESULTS_DIR_J}/${podmemlog}_max-measure-temp.log + done + for clusterlog in "${clusterlogs[@]}" + do + cat ${RESULTS_DIR_J}/${clusterlog}-measure-${itr}.log | cut -d "," -f2 >> ${RESULTS_DIR_J}/${clusterlog}-measure-temp.log + done + + ###### Add different raw logs we want to merge + #Cumulative raw data + paste ${RESULTS_DIR_J}/Throughput-measure-raw.log ${RESULTS_DIR_J}/cpu-measure-raw.log ${RESULTS_DIR_J}/mem-measure-raw.log >> ${RESULTS_DIR_J}/../Metrics-raw.log + done + + for metric in "${total_logs[@]}" + do + if [ ${metric} == "cpu_min" ] || [ ${metric} == "mem_min" ]; then + minval=$(echo `calcMin ${RESULTS_DIR_J}/${metric}-measure-temp.log`) + eval total_${metric}=${minval} + elif [ ${metric} == "cpu_max" ] || [ ${metric} == "mem_max" ] || [ ${metric} == "responsetime_max" ] || [ ${metric} == "stdev_resptime_max" ]; then + maxval=$(echo `calcMax ${RESULTS_DIR_J}/${metric}-measure-temp.log`) + eval total_${metric}=${maxval} + else + val=$(echo `calcAvg ${RESULTS_DIR_J}/${metric}-measure-temp.log | cut -d "=" -f2`) + eval total_${metric}_avg=${val} + fi + done + + echo "${sca} , ${total_throughput_avg} , ${total_responsetime_avg} , ${total_mem_avg} , ${total_cpu_avg} , ${total_cpu_min} , ${total_cpu_max} , ${total_mem_min} , ${total_mem_max} , ${total_c_mem_avg} , ${total_c_cpu_avg} , ${CPU_REQ} , ${MEM_REQ} , ${CPU_LIM} , ${MEM_LIM} , ${total_weberror_avg}, ${total_responsetime_max} , ${total_stdev_resptime_max} , ${quarkus_thread_pool_core_threads} , ${quarkus_thread_pool_queue_size=} " >> ${RESULTS_DIR_J}/../Metrics.log + echo "${sca} , ${total_mem_avg} , ${total_memusage_avg} , ${total_memrequests_avg} , ${total_memlimits_avg} , ${total_memreq_in_p_avg} , ${total_memlimit_in_p_avg} " >> ${RESULTS_DIR_J}/../Metrics-mem.log + echo "${sca} , ${total_cpu_avg} , ${total_cpurequests_avg} , ${total_cpulimits_avg} , ${total_cpureq_in_p_avg} , ${total_cpulimits_in_p_avg} " >> ${RESULTS_DIR_J}/../Metrics-cpu.log + echo "${sca} , ${total_c_cpu_avg} , ${total_c_cpurequests_avg} , ${total_c_cpulimits_avg} , ${total_c_mem_avg} , ${total_c_memrequests_avg} , ${total_c_memlimits_avg} " >> ${RESULTS_DIR_J}/../Metrics-cluster.log +} + +# Calculate average in MB +# input: Result directory +# output: Average in MB +function calcAvg_inMB() +{ + LOG=$1 + if [ -s ${LOG} ]; then + awk '{sum+=$1} END { print " Average =",sum/NR/1024/1024}' ${LOG} ; + fi +} + +# Calculate average in percentage +# input: Result directory +# output: Average in percentage +function calcAvg_in_p() +{ + LOG=$1 + if [ -s ${LOG} ]; then + awk '{sum+=$1} END { print " % Average =",sum/NR*100}' ${LOG} ; + fi +} + +# Calculate average +# input: Result directory +# output: Average +function calcAvg() +{ + LOG=$1 + if [ -s ${LOG} ]; then + awk '{sum+=$1} END { print " Average =",sum/NR}' ${LOG} ; + fi +} + +#Calculate Median +# input: Result directory +# output: Median +function calcMedian() +{ + LOG=$1 + if [ -s ${LOG} ]; then + sort -n ${LOG} | awk ' { a[i++]=$1; } END { x=int((i+1)/2); if (x < (i+1)/2) print " Median =",(a[x-1]+a[x])/2; else print " Median =",a[x-1]; }' + fi +} + +# Calculate minimum +# input: Result directory +# output: Minimum value +function calcMin() +{ + LOG=$1 + if [ -s ${LOG} ]; then + sort -n ${LOG} | head -1 + fi +} + +# Calculate maximum +# input: Result directory +# output: Maximum value +function calcMax() { + LOG=$1 + if [ -s ${LOG} ]; then + sort -n ${LOG} | tail -1 + fi +} + +# Perform warmup and measure runs +# input: number of runs(warmup|measure), result directory +# output: Cpu info, memory info, node info, wrk load for each runs(warmup|measure) in the form of jason files +function runItr() +{ + TYPE=$1 + RUNS=$2 + RESULTS_runItr=$3 + for (( run=0; run<${RUNS}; run++ )) + do + # Check if the application is running + check_app + echo "##### ${TYPE} ${run}" >> setup.log + # Get CPU and MEM info through prometheus queries + ${SCRIPT_REPO}/perf/getstats-openshift.sh ${TYPE}-${run} ${CPU_MEM_DURATION} ${RESULTS_runItr} ${BENCHMARK_SERVER} galaxies & + # Run the wrk workload + run_wrk_with_scaling ${RESULTS_runItr} ${TYPE} ${run} + # Sleep till the wrk load completes + sleep ${WRK_LOAD_DURATION} + sleep 20 + done +} + +# get the kruize recommendation for galaxies application +# input: result directory +# output: kruize recommendations for galaxies +function get_recommendations_from_kruize() +{ + TOKEN=`oc whoami --show-token` + app_list=($(oc get deployments --namespace=${NAMESPACE} | grep "galaxies" | cut -d " " -f1)) + for app in "${app_list[@]}" + do + curl --silent -k -H "Authorization: Bearer ${TOKEN}" http://kruize-openshift-monitoring.apps.${BENCHMARK_SERVER}/recommendations?application_name=${app} > ${RESULTS_DIR_I}/${app}-recommendations.log + err_exit "Error: could not generate the recommendations for galaxies" + done +} + +# Perform warmup and measure runs +# input: scaling instance, total number of iterations, warmups , measures , result directory +# output: Deploy the application if required, perform the runs and get the recommendations +function runIterations() { + SCALING=$1 + TOTAL_ITR=$2 + WARMUPS=$3 + MEASURES=$4 + RESULTS_DIR_ITR=$5 + #IF we want to use array of users we can use this variable. + USERS=${WRK_LOAD_USERS} + for (( itr=0; itr<${TOTAL_ITR}; itr++ )) + do + if [ ${RE_DEPLOY} == "true" ]; then + ${SCRIPT_REPO}/galaxies-deploy-openshift.sh -s ${BENCHMARK_SERVER} -i ${SCALING} -g ${GALAXIES_IMAGE} --cpureq=${CPU_REQ} --memreq=${MEM_REQ} --cpulim=${CPU_LIM} --memlim=${MEM_LIM} --env=${ENV_VAR}>> setup.log + err_exit "Error: galaxies deployment failed" + fi + # Start the load + RESULTS_DIR_I=${RESULTS_DIR_ITR}/ITR-${itr} + echo "Running ${WARMUPS} warmups for ${USERS} users" >> setup.log + # Perform warmup runs + runItr warmup ${WARMUPS} ${RESULTS_DIR_I} + echo "Running ${MEASURES} measures for ${USERS} users" >> setup.log + # Perform measure runs + runItr measure ${MEASURES} ${RESULTS_DIR_I} + sleep 60 + # get the kruize recommendation for galaxies application + # commenting for now as it is not required in all cases + #get_recommendations_from_kruize ${RESULTS_DIR_I} + done +} + +#TODO Create a function on how many DB inst required for a server. For now,defaulting it to 1 +# Scale the instances and run the iterations +echo "Instances , Throughput , Responsetime , MEM_MEAN , CPU_MEAN , CPU_MIN , CPU_MAX , MEM_MIN , MEM_MAX , CLUSTER_MEM% , CLUSTER_CPU% , CPU_REQ , MEM_REQ , CPU_LIM , MEM_LIM , WEB_ERRORS , RESPONSETIME_MAX , STDEV_RESPTIME_MAX" > ${RESULTS_DIR_ROOT}/Metrics.log +echo "Instances , MEM_RSS , MEM_USAGE , MEM_REQ , MEM_LIM , MEM_REQ_IN_P , MEM_LIM_IN_P " > ${RESULTS_DIR_ROOT}/Metrics-mem.log +echo "Instances , CPU_USAGE , CPU_REQ , CPU_LIM , CPU_REQ_IN_P , CPU_LIM_IN_P " > ${RESULTS_DIR_ROOT}/Metrics-cpu.log +echo "Instances , CLUSTER_CPU% , C_CPU_REQ% , C_CPU_LIM% , CLUSTER_MEM% , C_MEM_REQ% , C_MEM_LIM% " > ${RESULTS_DIR_ROOT}/Metrics-cluster.log +echo "Run , CPU_REQ , MEM_REQ , CPU_LIM , MEM_LIM , Throughput , Responsetime , WEB_ERRORS , Responsetime_MAX , stdev_responsetime_max , CPU , CPU_MIN , CPU_MAX , MEM , MEM_MIN , MEM_MAX" > ${RESULTS_DIR_ROOT}/Metrics-raw.log + +for (( scale=1; scale<=${TOTAL_INST}; scale++ )) +do + RESULTS_SC=${RESULTS_DIR_ROOT}/scale_${scale} + echo "RESULTS DIRECTORY is " ${RESULTS_DIR_ROOT} >> setup.log + echo "Running the benchmark with ${scale} instances with ${TOTAL_ITR} iterations having ${WARMUPS} warmups and ${MEASURES} measurements" >> setup.log + # Perform warmup and measure runs + runIterations ${scale} ${TOTAL_ITR} ${WARMUPS} ${MEASURES} ${RESULTS_SC} + echo "Parsing results for ${scale} instances" >> setup.log + # Parse the results + parseResults ${TOTAL_ITR} ${RESULTS_SC} ${scale} +done + +# Display the Metrics log file +cat ${RESULTS_DIR_ROOT}/Metrics.log +cat ${RESULTS_DIR_ROOT}/Metrics-raw.log diff --git a/galaxies/scripts/runWrk.sh b/galaxies/scripts/runWrk.sh deleted file mode 100755 index c9329e74..00000000 --- a/galaxies/scripts/runWrk.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020, 2021 Red Hat, IBM Corporation and others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -if [ $# -ne 2 ]; then - echo "Usage: $0 IP PORT" - exit 1 -fi - -export IP=$1 -export PORT=$2 - -for USERS in 1 5 10 15 20 25 30 35 40 -do - echo "Runnning with ${USERS} users" - for run in {1..2} - do - wrk --threads=${USERS} --connections=${USERS} -d60s http://${IP}:${PORT}/galaxies; - done -done diff --git a/spring-petclinic/README.md b/spring-petclinic/README.md index 8111fdab..fce01c88 100644 --- a/spring-petclinic/README.md +++ b/spring-petclinic/README.md @@ -85,7 +85,7 @@ service/petclinic-service-1 created ## Openshift To deploy the benchmark use `petclinic-deploy-openshift.sh` -`./scripts/petclinic-deploy-openshift.sh -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq MEM_REQ] [--cpulim=CPU_LIM] [--memlim MEM_LIM]` +`./scripts/petclinic-deploy-openshift.sh -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq MEM_REQ] [--cpulim=CPU_LIM] [--memlim MEM_LIM] [--env=ENV_VAR]` - **BENCHMARK_SERVER**: Name of the cluster you are using - **SERVER_INSTANCES**: Number of petclinic instances to be deployed. It is optional, if is not specified then by default it will be considered as 1 instance. @@ -95,6 +95,7 @@ To deploy the benchmark use `petclinic-deploy-openshift.sh` - **MEM_REQ**: Memory request - **CPU_LIM**: CPU limit - **MEM_LIM**: Memory limit +- **ENV_VAR**: Environment variable Example to deploy and run petclinic application on openshift cluster diff --git a/spring-petclinic/manifests/petclinic.yaml b/spring-petclinic/manifests/petclinic.yaml index df882436..47df00de 100644 --- a/spring-petclinic/manifests/petclinic.yaml +++ b/spring-petclinic/manifests/petclinic.yaml @@ -24,6 +24,7 @@ spec: - name: petclinic-tomcat image: kruize/spring_petclinic:2.2.0-jdk-11.0.8-openj9-0.21.0 imagePullPolicy: IfNotPresent + env: resources: requests: limits: diff --git a/spring-petclinic/scripts/perf/README.md b/spring-petclinic/scripts/perf/README.md index 48c5c936..cd18780b 100755 --- a/spring-petclinic/scripts/perf/README.md +++ b/spring-petclinic/scripts/perf/README.md @@ -1,6 +1,6 @@ # Test with multiple instances -`./scripts/perf/run_petclinic_openshift.sh -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq MEM_REQ] [--cpulim=CPU_LIM] [--memlim MEM_LIM]` +`./scripts/perf/run_petclinic_openshift.sh -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq MEM_REQ] [--cpulim=CPU_LIM] [--memlim MEM_LIM] [--env=ENV_VAR]` - **BENCHMARK_SERVER_NAME** : Name of the cluster you are using - **RESULTS_DIR_PATH** : Location where you want to store the results @@ -18,6 +18,7 @@ - **MEM_REQ**: Memory request - **CPU_LIM**: CPU limit - **MEM_LIM**: Memory limit +- **ENV_VAR**: Environment variable Example to test with multiple instances diff --git a/spring-petclinic/scripts/perf/run-petclinic-openshift.sh b/spring-petclinic/scripts/perf/run-petclinic-openshift.sh index 5f88b4e8..4c6d80f8 100755 --- a/spring-petclinic/scripts/perf/run-petclinic-openshift.sh +++ b/spring-petclinic/scripts/perf/run-petclinic-openshift.sh @@ -18,8 +18,8 @@ # CURRENT_DIR="$(dirname "$(realpath "$0")")" -pushd "${CURRENT_DIR}" >> setup.log -pushd ".." >> setup.log +pushd "${CURRENT_DIR}" >> /dev/null +pushd ".." >> /dev/null SCRIPT_REPO=${PWD} CLUSTER_TYPE="openshift" @@ -43,7 +43,7 @@ function err_exit() { # Describes the usage of the script function usage() { echo - echo "Usage: $0 -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] " + echo "Usage: $0 -s BENCHMARK_SERVER -e RESULTS_DIR_PATH [-u JMETER_LOAD_USERS] [-d JMETER_LOAD_DURATION] [-w WARMUPS] [-m MEASURES] [-i TOTAL_INST] [--iter=TOTAL_ITR] [-r= set redeploy to true] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]" exit -1 } @@ -68,6 +68,9 @@ do memlim=*) MEM_LIM=${OPTARG#*=} ;; + env=*) + ENV_VAR=${OPTARG#*=} + ;; *) esac ;; @@ -523,7 +526,7 @@ function runIterations() { for (( itr=0; itr<${TOTAL_ITR}; itr++ )) do if [ ${RE_DEPLOY} == "true" ]; then - ${SCRIPT_REPO}/petclinic-deploy-openshift.sh -s ${BENCHMARK_SERVER} -i ${SCALING} -p ${PETCLINIC_IMAGE} --cpureq=${CPU_REQ} --memreq=${MEM_REQ} --cpulim=${CPU_LIM} --memlim=${MEM_LIM} >> setup.log + ${SCRIPT_REPO}/petclinic-deploy-openshift.sh -s ${BENCHMARK_SERVER} -i ${SCALING} -p ${PETCLINIC_IMAGE} --cpureq=${CPU_REQ} --memreq=${MEM_REQ} --cpulim=${CPU_LIM} --memlim=${MEM_LIM} --env=${ENV_VAR}>> setup.log err_exit "Error: petclinic deployment failed" fi # Start the load diff --git a/spring-petclinic/scripts/petclinic-common.sh b/spring-petclinic/scripts/petclinic-common.sh index 22b2dee7..7a6b1943 100755 --- a/spring-petclinic/scripts/petclinic-common.sh +++ b/spring-petclinic/scripts/petclinic-common.sh @@ -17,8 +17,8 @@ ### Script containing common functions ### # CURRENT_DIR="$(dirname "$(realpath "$0")")" -pushd "${CURRENT_DIR}" >> setup.log -pushd ".." >> setup.log +pushd "${CURRENT_DIR}" >> /dev/null +pushd ".." >> /dev/null # Set the defaults for the app export PETCLINIC_PORT="32334" @@ -66,7 +66,7 @@ function build_petclinic() { # Build the application from git clone. Requires git , JAVA compiler on your machine to work. git clone https://github.com/spring-projects/spring-petclinic.git 2>>${LOGFILE} >>${LOGFILE} err_exit "Error: Unable to clone the git repo" - pushd spring-petclinic >>${LOGFILE} + pushd spring-petclinic >> /dev/null # Change the server port in application.properties sed -i '1 s/^/server.port=8081\n/' src/main/resources/application.properties sed -i '19imanagement.endpoints.web.base-path=/manage\n' src/main/resources/application.properties diff --git a/spring-petclinic/scripts/petclinic-deploy-openshift.sh b/spring-petclinic/scripts/petclinic-deploy-openshift.sh index a8948be2..33dfc036 100755 --- a/spring-petclinic/scripts/petclinic-deploy-openshift.sh +++ b/spring-petclinic/scripts/petclinic-deploy-openshift.sh @@ -29,7 +29,7 @@ CLUSTER_TYPE="openshift" # Describes the usage of the script function usage() { echo - echo "Usage: $0 -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] " + echo "Usage: $0 -s BENCHMARK_SERVER [-i SERVER_INSTANCES] [-n NAMESPACE] [-p PETCLINIC_IMAGE] [--cpureq=CPU_REQ] [--memreq=MEM_REQ] [--cpulim=CPU_LIM] [--memlim=MEM_LIM] [--env=ENV_VAR]" echo " " echo "Example: $0 -s rouging.os.fyre.ibm.com -i 2 -p kruize/spring_petclinic:2.2.0-jdk-11.0.8-openj9-0.21.0 --cpulim=4 --cpureq=2 --memlim=1024Mi --memreq=512Mi" exit -1 @@ -79,6 +79,9 @@ do memlim=*) MEM_LIM=${OPTARG#*=} ;; + env=*) + ENV_VAR=${OPTARG#*=} + ;; *) esac ;; @@ -164,6 +167,12 @@ function createInstances() { sed -i '/limits:/a \ \ \ \ \ \ \ \ \ \ cpu: '${CPU_LIM}'' ${MANIFESTS_DIR}/petclinic-${inst}.yaml fi + # Pass environment variables + if [ ! -z ${ENV_VAR} ]; then + sed -i '/env:/a \ \ \ \ \ \ \ \ - name: "JVM_ARGS"' ${MANIFESTS_DIR}/petclinic-${inst}.yaml + sed -i '/- name: "JVM_ARGS"/a \ \ \ \ \ \ \ \ \ \ value: '${ENV_VAR}'' ${MANIFESTS_DIR}/petclinic-${inst}.yaml + fi + oc create -f ${MANIFESTS_DIR}/petclinic-${inst}.yaml -n ${NAMESPACE} err_exit "Error: Issue in deploying." ((PETCLINIC_PORT=PETCLINIC_PORT+1))