Merge pull request #1014 from it-novum/development

openITCOCKPIT 4.0.4 Stable
it-novum · Jul 22, 2020 · 077f411 · 077f411
2 parents a47682f + 0fbb638
commit 077f411
Show file tree

Hide file tree

Showing 32 changed files with 21,342 additions and 1,086 deletions.
diff --git a/README.md b/README.md
@@ -10,7 +10,14 @@ This is the **BETA VERSION** of openITCOCKPIT 4. [Click here](https://github.com
 [![Chat on Matrix](https://img.shields.io/badge/style-matrix-blue.svg?style=flat&label=chat)](https://riot.im/app/#/room/#openitcockpit:matrix.org)
 [![IRC: #openitcockpit on chat.freenode.net](https://img.shields.io/badge/%23openitcockpit-freenode-blue.svg)](https://kiwiirc.com/client/chat.freenode.net/#openitcockpit)
 
+openITCOCKPIT 4.x 
+| Distribution | Stable                                                                                                      | Nightly                                                                                                      |
+|--------------|-------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------|
+| Focal       | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-focal-stable&style=flat-square)  | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-focal&style=flat-square)  |
+| Bionic       | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-bionic-stable&style=flat-square)  | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-bionic&style=flat-square)  |
+| Buster      | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-buster-stable&style=flat-square) | ![status](https://drone.openitcockpit.io/buildStatus/icon?job=openitcockpit-packages-buster&style=flat-square) |
 
+openITCOCKPIT 3.x 
 | Distribution | Stable                                                                                                      | Nightly                                                                                                      |
 |--------------|-------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------|
 | Bionic       | ![status](https://jenkins.it-novum.com/buildStatus/icon?job=bionic-openitcockpit-stable&style=flat-square)  | ![status](https://jenkins.it-novum.com/buildStatus/icon?job=bionic-openitcockpit-nightly&style=flat-square)  |
@@ -23,7 +30,7 @@ openITCOCKPIT is an Open Source system monitoring tool built for different monit
 
 So easy that everyone can use it: create your entire monitoring configuration with a few clicks due to our smart interface written in PHP
 
-![openITCOCKPIT](screenshots/openITC_3.7.1.png?raw=true "openITCOCKPIT")
+![openITCOCKPIT](screenshots/dashboard_v4.png?raw=true "openITCOCKPIT")
 
 # Demo
 Play around with our [Demo](https://demo.openitcockpit.io/) system. Its equipped with the majority of modules that you will get with the community license
@@ -101,11 +108,9 @@ After successful registration you can install the free community modules at Syst
 
 # Screenshots
 
-![openITCOCKPIT](screenshots/timeline_oitc.png?raw=true "Timeline")
-
-![openITCOCKPIT](screenshots/map1.png?raw=true "Maps")
+![openITCOCKPIT](screenshots/timeline.png?raw=true "Timeline")
 
-![openITCOCKPIT](screenshots/map2.png?raw=true "Maps")
+![openITCOCKPIT](screenshots/mapmodule.png?raw=true "Maps")
 
 ![openITCOCKPIT](screenshots/event_correlation.png?raw=true "Event correlation")
 

diff --git a/UPDATE.sh b/UPDATE.sh
@@ -122,7 +122,8 @@ mysqldump --defaults-extra-file=${DUMPINIFILE} --databases $dbc_dbname --flush-p
 echo "---------------------------------------------------------------"
 echo "Convert MySQL Tables from utf8_general_ci to utf8mb4_general_ci..."
 
-mysql --defaults-extra-file=${INIFILE} -e "ALTER DATABASE ${dbc_dbname} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"
+# Disabled - this takes ages!
+#mysql --defaults-extra-file=${INIFILE} -e "ALTER DATABASE ${dbc_dbname} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"
 
 mysql --defaults-extra-file=${INIFILE} --batch --skip-column-names -e "SELECT TABLE_NAME FROM \`information_schema\`.\`TABLES\` WHERE \`TABLE_SCHEMA\`='${dbc_dbname}' AND \`TABLE_NAME\` NOT LIKE 'nagios_%' AND \`TABLE_COLLATION\`='utf8_general_ci'" | while read TABLE_NAME; do
     echo "ALTER TABLE \`${TABLE_NAME}\` CONVERT TO CHARACTER SET utf8mb4; ✔"
@@ -201,7 +202,8 @@ done
 echo "---------------------------------------------------------------"
 echo "Convert MySQL Tables from utf8_general_ci to utf8mb4_general_ci..."
 
-mysql --defaults-extra-file=${INIFILE} -e "ALTER DATABASE ${dbc_dbname} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"
+# Disabled - this takes ages!
+#mysql --defaults-extra-file=${INIFILE} -e "ALTER DATABASE ${dbc_dbname} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"
 
 mysql --defaults-extra-file=${INIFILE} --batch --skip-column-names -e "SELECT TABLE_NAME FROM \`information_schema\`.\`TABLES\` WHERE \`TABLE_SCHEMA\`='${dbc_dbname}' AND \`TABLE_NAME\` NOT LIKE 'nagios_%' AND \`TABLE_COLLATION\`='utf8_general_ci'" | while read TABLE_NAME; do
     echo "ALTER TABLE \`${TABLE_NAME}\` CONVERT TO CHARACTER SET utf8mb4; ✔"
@@ -328,15 +330,17 @@ systemctl enable sudo_server.service\
  push_notification.service\
  nodejs_server.service\
  openitcockpit-graphing.service\
- oitc_cronjobs.timer
+ oitc_cronjobs.timer\
+ statusengine.service
 
 systemctl restart\
  sudo_server.service\
  oitc_cmd.service\
  gearman_worker.service\
  push_notification.service\
  nodejs_server.service\
- oitc_cronjobs.timer
+ oitc_cronjobs.timer\
+ statusengine.service
 
 PHPVersion=$(php -r "echo substr(PHP_VERSION, 0, 3);")
 echo "Detected PHP Version: ${PHPVersion} try to restart php-fpm"

diff --git a/bin/scripts/UPGRADE.sh b/bin/scripts/UPGRADE.sh
@@ -5,6 +5,12 @@ if ! [ $(id -u) = 0 ]; then
     exit 1
 fi
 
+# Stop phpnsta-client after upgrading from V3 to V4
+if systemctl is-active phpnsta.service; then
+    systemctl stop phpnsta.service
+fi
+
+
 APPDIR="/opt/openitc/frontend"
 
 OLDINIFILE=/etc/openitcockpit/mysql.cnf
@@ -40,6 +46,12 @@ cp ${APPDIR}/system/nginx/ssl_cert.conf /etc/nginx/openitc/ssl_cert.conf
 echo "# This file will NOT be overwritten during an update" >> /etc/nginx/openitc/custom.conf
 chmod +x /usr/bin/oitc
 
+echo "Delete all tmp files"
+rm -rf /opt/openitc/frontend/tmp
+mkdir -p /opt/openitc/frontend/tmp
+chown www-data:www-data /opt/openitc/frontend/tmp
+chmod 777 /opt/openitc/frontend/tmp
+
 echo "Create required system folders"
 mkdir -p /opt/openitc/etc/{mysql,grafana,carbon,frontend,nagios,nsta,statusengine} /opt/openitc/etc/statusengine/Config
 
@@ -166,6 +178,12 @@ for PLUGIN in $(ls -1 "${APPDIR}/plugins"); do
     fi
 done
 
+echo "Delete all tmp files"
+rm -rf /opt/openitc/frontend/tmp
+mkdir -p /opt/openitc/frontend/tmp
+chown www-data:www-data /opt/openitc/frontend/tmp
+chmod 777 /opt/openitc/frontend/tmp
+
 echo "---------------------------------------------------------------"
 echo "Create new WebSocket Key"
 WEBSOCKET_KEY=$(php -r "echo bin2hex(openssl_random_pseudo_bytes(80, \$cstrong));")
@@ -320,6 +338,9 @@ mysql --defaults-extra-file=${INIFILE} -e "TRUNCATE TABLE changelogs;"
 mysql --defaults-extra-file=${INIFILE} -e "TRUNCATE TABLE changelogs_to_containers;"
 
 mysql --defaults-extra-file=${INIFILE} -e "UPDATE commands SET command_line = REPLACE(command_line, '/usr/share/openitcockpit/app/Console/cake', '/opt/openitc/frontend/bin/cake');"
+#Replace EVC command with missing quotes
+mysql --defaults-extra-file=${INIFILE} -e "UPDATE commands SET command_line = REPLACE(command_line, 'EventcorrelationModule.evc_plugin \$HOSTNAME\$', 'EventcorrelationModule.evc_plugin --uuid \"\$HOSTNAME\$\"');"
+#Replace EVC command with surrounding quotes
 mysql --defaults-extra-file=${INIFILE} -e "UPDATE commands SET command_line = REPLACE(command_line, 'EventcorrelationModule.evc_plugin \"\$HOSTNAME\$\"', 'EventcorrelationModule.evc_plugin --uuid \"\$HOSTNAME\$\"');"
 
 mysql --defaults-extra-file=${INIFILE} -e "UPDATE widgets SET icon = REPLACE(icon, 'fa-', 'fas fa-');"
@@ -347,6 +368,8 @@ oitc update3_to4 --migrate-statehistory
 oitc update3_to4 --migrate-acknowledgements
 oitc update3_to4 --migrate-downtimes
 
+oitc update3_to4 --migrate-satellites
+
 echo "---------------------------------------------------------------"
 echo "Convert MySQL Tables from utf8_swedish_ci to utf8mb4_general_ci..."
 
@@ -357,7 +380,6 @@ mysql --defaults-extra-file=${INIFILE} --batch --skip-column-names -e "SELECT TA
     mysql --defaults-extra-file=${INIFILE} -e "ALTER TABLE \`${TABLE_NAME}\` CONVERT TO CHARACTER SET utf8mb4;"
 done
 
-#oitc setup
 
 oitc nagios_export
 
@@ -425,6 +447,24 @@ rm -rf /usr/share/openitcockpit
 rm -rf /etc/openitcockpit
 rm -rf /var/lib/openitcockpit
 
+echo "Delete all tmp files"
+rm -rf /opt/openitc/frontend/tmp
+mkdir -p /opt/openitc/frontend/tmp
+chown www-data:www-data /opt/openitc/frontend/tmp
+chmod 777 /opt/openitc/frontend/tmp
+mkdir -p /opt/openitc/frontend/tmp/nagios
+chown nagios:nagios /opt/openitc/frontend/tmp/nagios
+
+# Set filesystem permissions after all is done - again
+chown www-data:www-data /opt/openitc/logs/frontend
+oitc rights
+chown nagios:nagios /opt/openitc/logs/frontend/nagios
+chown www-data:www-data /opt/openitc/frontend/
+chmod 775 /opt/openitc/logs/frontend
+chmod 775 /opt/openitc/logs/frontend/nagios
+chown www-data:www-data /opt/openitc/frontend/tmp
+chown nagios:nagios /opt/openitc/frontend/tmp/nagios
+
 echo ""
 echo ""
 echo "Upgrade to openITCOCKPIT 4 done."

diff --git a/bin/scripts/oitc3to4.sh b/bin/scripts/oitc3to4.sh
@@ -31,6 +31,9 @@ Red=$'\e[1;31m'
 Green=$'\e[1;32m'
 Yellow=$'\e[1;33m'
 Blue=$'\e[1;34m'
+Cyan=$'\e[1;96m'
+BgBlack=$'\e[40m';
+Reset=$'\e[0m';
 
 isAptWorking=0
 satellitesCount=0
@@ -52,6 +55,8 @@ fi
 
 MYSQL_DATABASE=$(php -r "echo parse_ini_file('/etc/openitcockpit/mysql.cnf')['database'];")
 
+LICENSE=$(mysql "--defaults-extra-file=$v3MysqlIni" -e "SELECT license FROM registers LIMIT 1;" -B -s 2>/dev/null)
+
 debug_log(){
     if [ $enableLog == 1 ]; then
         echo "${Blue}${1}" | tr '___' ' '
@@ -77,7 +82,7 @@ print_results(){
         echo "${Red}${errorSign} ${msg}" | tr '___' ' '
     done
 
-    echo -e "\n${Green}Oks: $okCount | ${Blue}Hints: $hintCount | ${Yellow}Warnings: $warningCount | ${Red}Errors: $errorCount"
+    echo -e "\n${Green}Oks: $okCount | ${Blue}Hints: $hintCount | ${BgBlack}${Yellow}Warnings: ${warningCount}${Reset} | ${Red}Errors: $errorCount"
 }
 
 get_configured_satellites(){
@@ -186,8 +191,13 @@ check_mysql_version(){
         mysqlVersion=$(echo $mysqlVersion | cut -c1-3);
 
         if [[ "$mysqlVersion" -lt "103" ]]; then
-          errors+=($(echo "openITCOCKPIT Version 4 requires at least MariaDB version 10.3" | tr ' ' '___'))
-          ((errorCount++))
+          if [ "$VERSION_CODENAME" == "stretch" ]; then
+            oks+=($(echo "Your version of MariaDB gets upgraded by the upgrade from Debian Stretch to Debian Buster" | tr ' ' '___'))
+            ((okCount++))
+          else
+            errors+=($(echo "openITCOCKPIT Version 4 requires at least MariaDB version 10.3" | tr ' ' '___'))
+            ((errorCount++))
+          fi
         else
           oks+=($(echo "MariaDB Version >= 10.3" | tr ' ' '___'))
           ((okCount++))
@@ -311,10 +321,16 @@ fi
 
 print_results
 
+echo "${BgBlack}${Yellow}"
+echo "#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+echo "# Please ensure that you have a valid backup of the system BEFORE your continue!"
+echo "${Reset}"
+echo ""
+
 if [ "$VERSION_CODENAME" == "stretch" ]; then
 
     openitcockpit_upd=$(apt-mark showmanual | grep openitcockpit | grep -v -e openitcockpit-message -e openitcockpit-statusengine-naemon -e openitcockpit-module-nrpe -e openitcockpit-module-mk | xargs echo)
-    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
+    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/ {print $2}' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
     php_upd=$(while read pkg; do echo "$pkg-"; if [ "$pkg" != "php7.0-mcrypt" ]; then echo "$pkg"|sed 's/php7.0/php7.3/'; fi; done< <(dpkg -l | awk '$2 ~ /php7.0/ {print $2}') | xargs echo)
     always="openitcockpit wkhtmltox"
 
@@ -326,76 +342,111 @@ if [ "$VERSION_CODENAME" == "stretch" ]; then
         always="$always mariadb-server-10.3 mariadb-client-10.3 mariadb-client-core-10.3 mariadb-server-core-10.3"
     fi
 
-    echo "${Yellow}"
+    echo "${BgBlack}${Cyan}"
     echo "###############################################################################"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "We recommend a combined distribution and openITCOCKPIT upgrade"
     echo "You can run the upgrade with the following commands:"
     echo ""
     echo "# Update sources.list to Debian 10"
     echo "sed -i 's/stretch/buster/g' /etc/apt/sources.list"
+    echo ""
     echo "# Add openITCOCKPIT 4 sources"
-    echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/buster/stable buster main' > /etc/apt/sources.list.d/openitcockpit.list"
+
+    if [ -z "$LICENSE" ] ; then
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/buster/stable buster main' > /etc/apt/sources.list.d/openitcockpit.list"
+    else
+        echo "mkdir -p /etc/apt/auth.conf.d"
+        echo "echo 'machine packages.openitcockpit.io login secret password ${LICENSE}' > /etc/apt/auth.conf.d/openitcockpit.conf"
+
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/buster/stable buster main' > /etc/apt/sources.list.d/openitcockpit.list"
+    fi
+
     echo "curl https://packages.openitcockpit.io/repokey.txt | apt-key add -"
+    echo ""
     echo "# Upgrade the distribution and openITCOCKPIT"
     echo "apt-get update"
     echo "apt-get dist-upgrade $php_upd $openitcockpit_upd $openitcockpit_rem $always"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "###############################################################################"
+    echo "${Reset}"
 fi
 
 if [ "$VERSION_CODENAME" == "xenial" ]; then
 
     openitcockpit_upd=$(apt-mark showmanual | grep openitcockpit | grep -v -e openitcockpit-message -e openitcockpit-statusengine-naemon -e openitcockpit-module-nrpe -e openitcockpit-module-mk | xargs echo)
-    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
+    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/ {print $2}' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
     php_upd=$(while read pkg; do echo "$pkg-"; if [ "$pkg" != "php7.0-mcrypt" ]; then echo "$pkg"|sed 's/php7.0/php7.2/'; fi; done< <(dpkg -l | awk '$2 ~ /php7.0/ {print $2}') | xargs echo)
     always="openitcockpit wkhtmltox"
 
     if [ ! -z "$(dpkg -l | awk '$2 ~ /openitcockpit-module-distribute/')" ]; then
         always="$always openitcockpit-nsta"
     fi
 
-    echo "${Yellow}"
+    echo "${BgBlack}${Cyan}"
     echo "###############################################################################"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "We recommend a combined distribution and openITCOCKPIT upgrade"
     echo "You can run the upgrade with the following commands:"
     echo ""
     echo "# Update sources.list to Ubuntu 18.04"
     echo "sed -i 's/xenial/bionic/g' /etc/apt/sources.list"
+    echo ""
     echo "# Add openITCOCKPIT 4 sources"
-    echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+
+    if [ -z "$LICENSE" ] ; then
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+    else
+        echo "mkdir -p /etc/apt/auth.conf.d"
+        echo "echo 'machine packages.openitcockpit.io login secret password ${LICENSE}' > /etc/apt/auth.conf.d/openitcockpit.conf"
+
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+    fi
+
     echo "curl https://packages.openitcockpit.io/repokey.txt | apt-key add -"
+    echo ""
     echo "# Upgrade the distribution and openITCOCKPIT"
     echo "apt-get update"
     echo "apt-get dist-upgrade $php_upd $openitcockpit_upd $openitcockpit_rem $always"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "###############################################################################"
+    echo "${Reset}"
 fi
 
 if [ "$VERSION_CODENAME" == "bionic" ]; then
 
     openitcockpit_upd=$(apt-mark showmanual | grep openitcockpit | grep -v -e openitcockpit-message -e openitcockpit-statusengine-naemon -e openitcockpit-module-nrpe -e openitcockpit-module-mk | xargs echo)
-    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
+    openitcockpit_rem=$(while read pkg; do echo "$pkg-"; done< <(dpkg -l | awk '$2 ~ /openitcockpit-/ {print $2} $2 ~ /phpnsta/ {print $2}' | grep -e 'openitcockpit-wkhtmltopdf' -e 'phpnsta') | xargs echo)
     always="openitcockpit wkhtmltox"
 
     if [ ! -z "$(dpkg -l | awk '$2 ~ /openitcockpit-module-distribute/')" ]; then
         always="$always openitcockpit-nsta"
     fi
 
-    echo "${Yellow}"
+    echo "${BgBlack}${Cyan}"
     echo "###############################################################################"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "You can run the upgrade with the following commands:"
     echo ""
     echo "# Add openITCOCKPIT 4 sources"
-    echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+
+    if [ -z "$LICENSE" ] ; then
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+    else
+        echo "mkdir -p /etc/apt/auth.conf.d"
+        echo "echo 'machine packages.openitcockpit.io login secret password ${LICENSE}' > /etc/apt/auth.conf.d/openitcockpit.conf"
+
+        echo "echo 'deb https://packages.openitcockpit.io/openitcockpit/bionic/stable bionic main' > /etc/apt/sources.list.d/openitcockpit.list"
+    fi
+
     echo "curl https://packages.openitcockpit.io/repokey.txt | apt-key add -"
+    echo ""
     echo "# Upgrade the distribution and openITCOCKPIT"
     echo "apt-get update"
     echo "apt-get dist-upgrade $openitcockpit_upd $openitcockpit_rem $always"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "###############################################################################"
+    echo "${Reset}"
 fi
 
 tput sgr0