diff --git a/.gitignore b/.gitignore index 14e6c4a79..7135c1b32 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ tools/__pycache__/ externals/ .env .vagrant -api/docs/api-docs.html \ No newline at end of file +api/docs/api-docs.html +*.code-workspace diff --git a/conf/fail2ban/filter.d/miab-munin.conf b/conf/fail2ban/filter.d/miab-munin.conf index b254cc62f..da257b3e5 100644 --- a/conf/fail2ban/filter.d/miab-munin.conf +++ b/conf/fail2ban/filter.d/miab-munin.conf @@ -3,5 +3,5 @@ before = common.conf [Definition] -failregex= - .*GET /admin/munin/.* HTTP/1.1\" 401.* +failregex= - .*GET /admin/munin/.* HTTP/\d+\.\d+\" 401.* ignoreregex = diff --git a/management/mail_log.py b/management/mail_log.py index 0462a5c5c..e2349598e 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -497,7 +497,7 @@ def add_login(user, date, protocol_name, host, collector): data["totals_by_protocol"][protocol_name] += 1 data["totals_by_protocol_and_host"][(protocol_name, host)] += 1 - if True: + if host not in {"127.0.0.1", "::1"} or True: data["activity-by-hour"][protocol_name][date.hour] += 1 collector["logins"][user] = data diff --git a/management/status_checks.py b/management/status_checks.py index e9e564472..f55aa5bd3 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -293,15 +293,15 @@ def run_network_checks(env, output): if zen is None: output.print_ok("IP address is not blacklisted by zen.spamhaus.org.") elif zen == "[timeout]": - output.print_warning("Connection to zen.spamhaus.org timed out. We could not determine whether your server's IP address is blacklisted. Please try again later.") + output.print_warning("Connection to zen.spamhaus.org timed out. Could not determine whether this box's IP address is blacklisted. Please try again later.") elif zen == "[Not Set]": - output.print_warning("Could not connect to zen.spamhaus.org. We could not determine whether your server's IP address is blacklisted. Please try again later.") + output.print_warning("Could not connect to zen.spamhaus.org. Could not determine whether this box's IP address is blacklisted. Please try again later.") elif zen == "127.255.255.252": - output.print_warning("Incorrect spamhaus query: %s We could not determine whether your server's IP address is blacklisted." % (rev_ip4+'.zen.spamhaus.org')) + output.print_warning("Incorrect spamhaus query: %s. Could not determine whether this box's IP address is blacklisted." % (rev_ip4+'.zen.spamhaus.org')) elif zen == "127.255.255.254": - output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. We could not determine whether your server's IP address is blacklisted.") + output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether this box's IP address is blacklisted.") elif zen == "127.255.255.255": - output.print_warning("Too many queries have been performed on the spamhaus server. We could not determine whether your server's IP address is blacklisted.") + output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether this box's IP address is blacklisted.") else: output.print_error("""The IP address of this machine {} is listed in the Spamhaus Block List (code {}), which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/{}.""".format(env['PUBLIC_IP'], zen, env['PUBLIC_IP'])) @@ -439,7 +439,7 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles): if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and ipv6 != normalize_ip(env['PUBLIC_IPV6'])): output.print_ok("Domain resolves to box's IP address. [{} ↦ {}]".format(env['PRIMARY_HOSTNAME'], my_ips)) else: - output.print_error("""This domain must resolve to your box's IP address ({}) in public DNS but it currently resolves + output.print_error("""This domain must resolve to this box's IP address ({}) in public DNS but it currently resolves to {}. It may take several hours for public DNS to update after a change. This problem may result from other issues listed above.""".format(my_ips, ip + ((" / " + ipv6) if ipv6 is not None else ""))) @@ -451,11 +451,11 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles): if existing_rdns_v4 == domain and existing_rdns_v6 in {None, domain}: output.print_ok("Reverse DNS is set correctly at ISP. [{} ↦ {}]".format(my_ips, env['PRIMARY_HOSTNAME'])) elif existing_rdns_v4 == existing_rdns_v6 or existing_rdns_v6 is None: - output.print_error(f"""Your box's reverse DNS is currently {existing_rdns_v4}, but it should be {domain}. Your ISP or cloud provider will have instructions - on setting up reverse DNS for your box.""" ) + output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4}, but it should be {domain}. Your ISP or cloud provider will have instructions + on setting up reverse DNS for this box.""" ) else: - output.print_error(f"""Your box's reverse DNS is currently {existing_rdns_v4} (IPv4) and {existing_rdns_v6} (IPv6), but it should be {domain}. Your ISP or cloud provider will have instructions - on setting up reverse DNS for your box.""" ) + output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4} (IPv4) and {existing_rdns_v6} (IPv6), but it should be {domain}. Your ISP or cloud provider will have instructions + on setting up reverse DNS for this box.""" ) # Check the TLSA record. tlsa_qname = "_25._tcp." + domain @@ -757,19 +757,19 @@ def check_mail_domain(domain, env, output): # See https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now. for # information on spamhaus return codes - dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None, retry=False) + dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None) if dbl is None: output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.") elif dbl == "[timeout]": - output.print_warning(f"Connection to dbl.spamhaus.org timed out. We could not determine whether the domain {domain} is blacklisted. Please try again later.") + output.print_warning(f"Connection to dbl.spamhaus.org timed out. Could not determine whether the domain {domain} is blacklisted. Please try again later.") elif dbl == "[Not Set]": - output.print_warning(f"Could not connect to dbl.spamhaus.org. We could not determine whether the domain {domain} is blacklisted. Please try again later.") + output.print_warning(f"Could not connect to dbl.spamhaus.org. Could not determine whether the domain {domain} is blacklisted. Please try again later.") elif dbl == "127.255.255.252": - output.print_warning("Incorrect spamhaus query: %s . We could not determine whether the domain %s is blacklisted." % (domain+'.dbl.spamhaus.org', domain)) + output.print_warning("Incorrect spamhaus query: %s. Could not determine whether the domain %s is blacklisted." % (domain+'.dbl.spamhaus.org', domain)) elif dbl == "127.255.255.254": - output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. We could not determine whether the domain {} is blacklisted.".format(domain)) + output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether the domain {} is blacklisted.".format(domain)) elif dbl == "127.255.255.255": - output.print_warning("Too many queries have been performed on the spamhaus server. We could not determine whether the domain {} is blacklisted.".format(domain)) + output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether the domain {} is blacklisted.".format(domain)) else: output.print_error(f"""This domain is listed in the Spamhaus Domain Block List (code {dbl}), which may prevent recipients from receiving your mail. @@ -787,7 +787,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output): if value == normalize_ip(expected): ok_values.append(value) else: - output.print_error(f"""This domain should resolve to your box's IP address ({rtype} {expected}) if you would like the box to serve + output.print_error(f"""This domain should resolve to this box's IP address ({rtype} {expected}) if you would like the box to serve webmail or a website on this domain. The domain currently resolves to {value} in public DNS. It may take several hours for public DNS to update after a change. This problem may result from other issues listed here.""") return diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index e71e4b029..ded432a05 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -26,13 +26,16 @@ nextcloud_hash=3869f55b9d2ba186e214dcb7729f1684d39fb759 # Nextcloud apps # -------------- -# * Find the most recent tag that is compatible with the Nextcloud version above by -# consulting the ... node at: -# https://github.com/nextcloud-releases/contacts/blob/main/appinfo/info.xml -# https://github.com/nextcloud-releases/calendar/blob/main/appinfo/info.xml -# https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml -# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and -# copying it from the error message when it doesn't match what is below. +# * Find the most recent tag that is compatible with the Nextcloud version above by: +# https://github.com/nextcloud-releases/contacts/tags +# https://github.com/nextcloud-releases/calendar/tags +# https://github.com/nextcloud/user_external/tags +# +# * For these three packages, contact, calendar and user_external, the hash is the SHA1 hash of +# the ZIP package, which you can find by just running this script and copying it from +# the error message when it doesn't match what is below: + +# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/contacts contacts_ver=5.5.1 contacts_hash=546f53f65f54b8b3f356afd6150715fcd20ffd31 @@ -40,12 +43,26 @@ contacts_hash=546f53f65f54b8b3f356afd6150715fcd20ffd31 calendar_ver=4.6.5 calendar_hash=b9c6039286e53fd816aeea7ca8a733407b91d881 -# And https://apps.nextcloud.com/apps/user_external +# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/user_external user_external_ver=3.2.0 user_external_hash=67ce8cbf8990b9d6517523d7236dcfb7f74b0201 -# Clear prior packages and install dependencies from apt. +# Developer advice (test plan) +# ---------------------------- +# When upgrading above versions, how to test? +# +# 1. Enter your server instance (or on the Vagrant image) +# 1. Git clone +# 2. Git checkout +# 3. Run `sudo ./setup/nextcloud.sh` +# 4. Ensure the installation completes. If any hashes mismatch, correct them. +# 5. Enter nextcloud web, run following tests: +# 5.1 You still can create, edit and delete contacts +# 5.2 You still can create, edit and delete calendar events +# 5.3 You still can create, edit and delete users +# 5.4 Go to Administration > Logs and ensure no new errors are shown +# Clear prior packages and install dependencies from apt. apt-get purge -qq -y owncloud* # we used to use the package manager apt_install curl php php-fpm \ @@ -146,7 +163,7 @@ InstallNextcloud() { # Current Nextcloud Version, #1623 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB -# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than +# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than # version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud # application version than the database. @@ -200,6 +217,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc return 0 fi + # Hint: whenever you bump, remember this: + # - Run a server with the previous version + # - On a new if-else block, copy the versions/hashes from the previous version + # - Run sudo ./setup/start.sh on the new machine. Upon completion, test its basic functionalities. + if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then # Version 20 is the latest version from the 18.04 version of miab. To upgrade to version 21, install php8.0. This is # not supported by version 20, but that does not matter, as the InstallNextcloud function only runs the version 21 code. diff --git a/setup/preflight.sh b/setup/preflight.sh index e921f298a..459ca93eb 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Are we running as root? if [[ $EUID -ne 0 ]]; then echo "This script must be run as root. Please re-run like this:" @@ -26,16 +27,16 @@ fi # # Skip the check if we appear to be running inside of Vagrant, because that's really just for testing. TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}') -if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then +if [ "$TOTAL_PHYSICAL_MEM" -lt 490000 ]; then if [ ! -d /vagrant ]; then - TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000) + TOTAL_PHYSICAL_MEM=$(( TOTAL_PHYSICAL_MEM * 1024 / 1000 / 1000 )) echo "Your Mail-in-a-Box needs more memory (RAM) to function properly." echo "Please provision a machine with at least 512 MB, 1 GB recommended." echo "This machine has $TOTAL_PHYSICAL_MEM MB memory." exit fi fi -if [ $TOTAL_PHYSICAL_MEM -lt 750000 ]; then +if [ "$TOTAL_PHYSICAL_MEM" -lt 750000 ]; then echo "WARNING: Your Mail-in-a-Box has less than 768 MB of memory." echo " It might run unreliably when under heavy load." fi diff --git a/setup/webmail.sh b/setup/webmail.sh index 32c06633b..46202e1bc 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -226,7 +226,7 @@ sed -i.miabold 's/^[^#]\+.\+PRAGMA journal_mode = WAL.\+$/#&/' \ # Because Roundcube wants to set the PRAGMA we just deleted from the source, we apply it here # to the roundcube database (see https://github.com/roundcube/roundcubemail/issues/8035) # Database should exist, created by migration script -sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite 'PRAGMA journal_mode=WAL;' > /dev/null +sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite 'PRAGMA journal_mode=WAL;' | 2>&1 # Enable PHP modules. phpenmod -v php imap