diff --git a/Containers/borgbackup/Dockerfile b/Containers/borgbackup/Dockerfile index 2f54145faea..706a9c7d4e7 100644 --- a/Containers/borgbackup/Dockerfile +++ b/Containers/borgbackup/Dockerfile @@ -11,11 +11,13 @@ RUN set -ex; \ rsync \ fuse \ py3-llfuse \ - jq + jq \ + openssh-client VOLUME /root COPY --chmod=770 *.sh / +COPY borg_excludes / ENTRYPOINT ["/start.sh"] # hadolint ignore=DL3002 diff --git a/Containers/borgbackup/backupscript.sh b/Containers/borgbackup/backupscript.sh index 739d5f16a5f..f86edb01508 100644 --- a/Containers/borgbackup/backupscript.sh +++ b/Containers/borgbackup/backupscript.sh @@ -34,19 +34,23 @@ for volume in "${DEFAULT_VOLUMES[@]}"; do done # Check if target is mountpoint -if ! mountpoint -q /mnt/borgbackup; then - echo "/mnt/borgbackup is not a mountpoint which is not allowed." +if [ -z "$BORG_REMOTE_REPO" ] && ! mountpoint -q "$MOUNT_DIR"; then + echo "$MOUNT_DIR is not a mountpoint which is not allowed." exit 1 fi -# Check if target is empty -if [ "$BORG_MODE" != backup ] && [ "$BORG_MODE" != test ] && ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then - echo "The repository is empty. Cannot perform check or restore." +# Check if repo is uninitialized +if [ "$BORG_MODE" != backup ] && [ "$BORG_MODE" != test ] && ! borg info > /dev/null; then + if [ -n "$BORG_REMOTE_REPO" ]; then + echo "The repository is uninitialized or cannot connect to remote. Cannot perform check or restore." + else + echo "The repository is uninitialized. Cannot perform check or restore." + fi exit 1 fi # Do not continue if this file exists (needed for simple external blocking) -if [ -f "$BORG_BACKUP_DIRECTORY/aio-lockfile" ]; then +if [ -z "$BORG_REMOTE_REPO" ] && [ -f "$BORG_BACKUP_DIRECTORY/aio-lockfile" ]; then echo "Not continuing because aio-lockfile exists – it seems like a script is externally running which is locking the backup archive." echo "If this should not be the case, you can fix this by deleting the 'aio-lockfile' file from the backup archive directory." exit 1 @@ -57,6 +61,13 @@ if [ "$BORG_MODE" = backup ] || [ "$BORG_MODE" = restore ]; then touch "/nextcloud_aio_volumes/nextcloud_aio_database_dump/backup-is-running" fi +if [ -n "$BORG_REMOTE_REPO" ] && ! [ -f "$BORGBACKUP_KEY" ]; then + echo "First run, creating borg ssh key" + ssh-keygen -f "$BORGBACKUP_KEY" -N "" + echo "You should configure the remote to accept this public key" +fi +echo "Your public ssh key for borgbackup is: $(cat "$BORGBACKUP_KEY.pub")" + # Do the backup if [ "$BORG_MODE" = backup ]; then @@ -100,15 +111,22 @@ if [ "$BORG_MODE" = backup ]; then exit 1 fi - # Create backup folder - mkdir -p "$BORG_BACKUP_DIRECTORY" + if [ -z "$BORG_REMOTE_REPO" ]; then + # Create backup folder + mkdir -p "$BORG_BACKUP_DIRECTORY" + fi - # Initialize the repository if the target is empty - if ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then + # Initialize the repository if can't get info from target + if ! borg info > /dev/null; then # Don't initialize if already initialized if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config" ]; then - echo "No borg config file was found in the targeted directory." - echo "This might happen if the targeted directory is located on an external drive and the drive not connected anymore. You should check this." + if [ -n "$BORG_REMOTE_REPO" ]; then + echo "Borg could not get info from the remote repo." + echo "This might be a failure to connect to the remote server. See the above borg info output for details." + else + echo "Borg could not get info from the targeted directory." + echo "This might happen if the targeted directory is located on an external drive and the drive not connected anymore. You should check this." + fi echo "If you instead want to initialize a new backup repository, you may delete the 'borg.config' file that is stored in the mastercontainer volume manually, which will allow you to initialize a new borg repository in the chosen directory:" echo "sudo docker exec nextcloud-aio-mastercontainer rm /mnt/docker-aio-config/data/borg.config" exit 1 @@ -116,28 +134,44 @@ if [ "$BORG_MODE" = backup ]; then echo "Initializing repository..." NEW_REPOSITORY=1 - if ! borg init --debug --encryption=repokey-blake2 "$BORG_BACKUP_DIRECTORY"; then + if ! borg init --debug --encryption=repokey-blake2; then echo "Could not initialize borg repository." - rm -f "$BORG_BACKUP_DIRECTORY/config" + if [ -z "$BORG_REMOTE_REPO" ]; then + # Originally we checked for presence of the config file instead of calling `borg info`. Likely `borg info` + # will error on a partially initialized repo, so this line is probably no longer necessary + rm -f "$BORG_BACKUP_DIRECTORY/config" + fi exit 1 fi - borg config "$BORG_BACKUP_DIRECTORY" additional_free_space 2G - # Fix too large Borg cache - # https://borgbackup.readthedocs.io/en/stable/faq.html#the-borg-cache-eats-way-too-much-disk-space-what-can-i-do - BORG_ID="$(borg config "$BORG_BACKUP_DIRECTORY" id)" - rm -r "/root/.cache/borg/$BORG_ID/chunks.archive.d" - touch "/root/.cache/borg/$BORG_ID/chunks.archive.d" + if [ -z "$BORG_REMOTE_REPO" ]; then + # borg config only works for local repos; it's up to the remote to ensure the disk isn't full + borg config :: additional_free_space 2G + + # Fix too large Borg cache + # https://borgbackup.readthedocs.io/en/stable/faq.html#the-borg-cache-eats-way-too-much-disk-space-what-can-i-do + BORG_ID="$(borg config :: id)" + rm -r "/root/.cache/borg/$BORG_ID/chunks.archive.d" + touch "/root/.cache/borg/$BORG_ID/chunks.archive.d" + fi - # Make a backup from the borg config file - if ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then - echo "The borg config file wasn't created. Something is wrong." + if ! borg info > /dev/null; then + echo "Borg can't get info from the repo it created. Something is wrong." exit 1 fi + rm -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config" - if ! cp "$BORG_BACKUP_DIRECTORY/config" "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"; then - echo "Could not copy config file to second place. Cannot perform backup." - exit 1 + if [ -n "$BORG_REMOTE_REPO" ]; then + # `borg config` does not support remote repos so instead create a dummy file and rely on the remote to avoid + # corruption of the config file (which contains the encryption key). We don't actually use the contents of + # this file anywhere, so a touch is all we need so we remember we already initialized the repo. + touch "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config" + else + # Make a backup from the borg config file + if ! cp "$BORG_BACKUP_DIRECTORY/config" "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"; then + echo "Could not copy config file to second place. Cannot perform backup." + exit 1 + fi fi echo "Repository successfully initialized." @@ -167,9 +201,9 @@ if [ "$BORG_MODE" = backup ]; then # Create the backup echo "Starting the backup..." get_start_time - if ! borg create "${BORG_OPTS[@]}" "${BORG_EXCLUDE[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/"; then + if ! borg create "${BORG_OPTS[@]}" "${BORG_EXCLUDE[@]}" "::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/" --exclude-from /borg_excludes; then echo "Deleting the failed backup archive..." - borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-nextcloud-aio" + borg delete --stats "::$CURRENT_DATE-nextcloud-aio" echo "Backup failed!" echo "You might want to check the backup integrity via the AIO interface." if [ "$NEW_REPOSITORY" = 1 ]; then @@ -188,14 +222,14 @@ if [ "$BORG_MODE" = backup ]; then # Prune archives echo "Pruning the archives..." - if ! borg prune --stats --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then + if ! borg prune --stats --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}"; then echo "Failed to prune archives!" exit 1 fi # Compact archives echo "Compacting the archives..." - if ! borg compact "$BORG_BACKUP_DIRECTORY"; then + if ! borg compact; then echo "Failed to compact archives!" exit 1 fi @@ -212,19 +246,19 @@ if [ "$BORG_MODE" = backup ]; then fi done echo "Starting the backup for additional volumes..." - if ! borg create "${BORG_OPTS[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes" "/docker_volumes/"; then + if ! borg create "${BORG_OPTS[@]}" "::$CURRENT_DATE-additional-docker-volumes" "/docker_volumes/"; then echo "Deleting the failed backup archive..." - borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes" + borg delete --stats "::$CURRENT_DATE-additional-docker-volumes" echo "Backup of additional docker-volumes failed!" exit 1 fi echo "Pruning additional volumes..." - if ! borg prune --stats --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then + if ! borg prune --stats --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}"; then echo "Failed to prune additional docker-volumes archives!" exit 1 fi echo "Compacting additional volumes..." - if ! borg compact "$BORG_BACKUP_DIRECTORY"; then + if ! borg compact; then echo "Failed to compact additional docker-volume archives!" exit 1 fi @@ -242,19 +276,19 @@ if [ "$BORG_MODE" = backup ]; then EXCLUDE_DIRS+=(--exclude "/host_mounts/$directory/") done echo "Starting the backup for additional host mounts..." - if ! borg create "${BORG_OPTS[@]}" "${EXCLUDE_DIRS[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts" "/host_mounts/"; then + if ! borg create "${BORG_OPTS[@]}" "${EXCLUDE_DIRS[@]}" "::$CURRENT_DATE-additional-host-mounts" "/host_mounts/"; then echo "Deleting the failed backup archive..." - borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts" + borg delete --stats "::$CURRENT_DATE-additional-host-mounts" echo "Backup of additional host-mounts failed!" exit 1 fi echo "Pruning additional host mounts..." - if ! borg prune --stats --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then + if ! borg prune --stats --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}"; then echo "Failed to prune additional host-mount archives!" exit 1 fi echo "Compacting additional host mounts..." - if ! borg compact "$BORG_BACKUP_DIRECTORY"; then + if ! borg compact; then echo "Failed to compact additional host-mount archives!" exit 1 fi @@ -276,18 +310,13 @@ fi if [ "$BORG_MODE" = restore ]; then get_start_time - # Perform the restore + # Pick archive to restore if [ -n "$SELECTED_RESTORE_TIME" ]; then - SELECTED_ARCHIVE="$(borg list "$BORG_BACKUP_DIRECTORY" | grep "nextcloud-aio" | grep "$SELECTED_RESTORE_TIME" | awk -F " " '{print $1}' | head -1)" + SELECTED_ARCHIVE="$(borg list | grep "nextcloud-aio" | grep "$SELECTED_RESTORE_TIME" | awk -F " " '{print $1}' | head -1)" else - SELECTED_ARCHIVE="$(borg list "$BORG_BACKUP_DIRECTORY" | grep "nextcloud-aio" | awk -F " " '{print $1}' | sort -r | head -1)" + SELECTED_ARCHIVE="$(borg list | grep "nextcloud-aio" | awk -F " " '{print $1}' | sort -r | head -1)" fi echo "Restoring '$SELECTED_ARCHIVE'..." - mkdir -p /tmp/borg - if ! borg mount "$BORG_BACKUP_DIRECTORY::$SELECTED_ARCHIVE" /tmp/borg; then - echo "Could not mount the backup!" - exit 1 - fi # Save Additional Backup dirs if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories" ]; then @@ -299,27 +328,12 @@ if [ "$BORG_MODE" = restore ]; then DAILY_BACKUPTIME="$(cat /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_time)" fi - # Restore everything except the configuration file - if ! rsync --stats --archive --human-readable -vv --delete \ - --exclude "nextcloud_aio_apache/caddy/**" \ - --exclude "nextcloud_aio_mastercontainer/caddy/**" \ - --exclude "nextcloud_aio_nextcloud/data/nextcloud.log*" \ - --exclude "nextcloud_aio_nextcloud/data/audit.log" \ - --exclude "nextcloud_aio_mastercontainer/certs/**" \ - --exclude "nextcloud_aio_mastercontainer/data/configuration.json" \ - --exclude "nextcloud_aio_mastercontainer/data/daily_backup_running" \ - --exclude "nextcloud_aio_mastercontainer/data/session_date_file" \ - --exclude "nextcloud_aio_mastercontainer/session/**" \ - /tmp/borg/nextcloud_aio_volumes/ /nextcloud_aio_volumes/; then - RESTORE_FAILED=1 - echo "Something failed while restoring from backup." - fi - # Save current aio password AIO_PASSWORD="$(jq '.password' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" - # Save current path + # Save current backup location vars BORG_LOCATION="$(jq '.borg_backup_host_location' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" + REMOTE_REPO="$(jq '.borg_remote_repo' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" # Save current nextcloud datadir if grep -q '"nextcloud_datadir":' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then @@ -328,21 +342,111 @@ if [ "$BORG_MODE" = restore ]; then NEXTCLOUD_DATADIR='""' fi - # Restore the configuration file - if ! rsync --archive --human-readable -vv \ - /tmp/borg/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json \ - /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then - RESTORE_FAILED=1 - echo "Something failed while restoring the configuration.json." + if [ -z "$BORG_REMOTE_REPO" ]; then + mkdir -p /tmp/borg + if ! borg mount "::$SELECTED_ARCHIVE" /tmp/borg; then + echo "Could not mount the backup!" + exit 1 + fi + + # Restore everything except the configuration file + # + # These exclude patterns need to be kept in sync with the borg_excludes file and the find excludes in this file, + # which use a different syntax (patterns appear in 3 places in total) + if ! rsync --stats --archive --human-readable -vv --delete \ + --exclude "nextcloud_aio_apache/caddy/**" \ + --exclude "nextcloud_aio_mastercontainer/caddy/**" \ + --exclude "nextcloud_aio_nextcloud/data/nextcloud.log*" \ + --exclude "nextcloud_aio_nextcloud/data/audit.log" \ + --exclude "nextcloud_aio_mastercontainer/certs/**" \ + --exclude "nextcloud_aio_mastercontainer/data/configuration.json" \ + --exclude "nextcloud_aio_mastercontainer/data/daily_backup_running" \ + --exclude "nextcloud_aio_mastercontainer/data/session_date_file" \ + --exclude "nextcloud_aio_mastercontainer/session/**" \ + /tmp/borg/nextcloud_aio_volumes/ /nextcloud_aio_volumes/; then + RESTORE_FAILED=1 + echo "Something failed while restoring from backup." + fi + + # Restore the configuration file + if ! rsync --archive --human-readable -vv \ + /tmp/borg/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json \ + /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then + RESTORE_FAILED=1 + echo "Something failed while restoring the configuration.json." + fi + + if ! umount /tmp/borg; then + echo "Failed to unmount the borg archive but should still be able to restore successfully" + fi + else + # Restore nearly everything + # + # borg mount is really slow for remote repos (did not check whether it's slow for local repos too), + # using extract to /tmp would require temporarily storing a second copy of the data. + # So instead extract directly on top of the destination with exclude patterns for the config, but + # then we do still need to delete local files which are not present in the archive. + # + # Older backups may still contain files we've since excluded, so we have to exclude on extract as well. + cd / # borg extract has no destination arg and extracts to CWD + if ! borg extract "::$SELECTED_ARCHIVE" --progress --exclude-from /borg_excludes --pattern '+nextcloud_aio_volumes/**' + then + RESTORE_FAILED=1 + echo "Failed to extract backup archive." + else + # Delete files/dirs present locally, but not in the backup archive, excluding conf files + # https://unix.stackexchange.com/a/759341 + # This comm does not support -z, but I doubt any file names would have \n in them + echo "Deleting local files which do not exist in the backup" + # These find patterns need to be kept in sync with the borg_excludes file and the rsync excludes in this + # file, which use a different syntax (patterns appear in 3 places in total) + if ! find nextcloud_aio_volumes \ + -not \( \ + -path nextcloud_aio_volumes/nextcloud_aio_apache/caddy \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_apache/caddy/*" \ + -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/caddy \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/caddy/*" \ + -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/certs \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/certs/*" \ + -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/session \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/session/*" \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/nextcloud.log*" \ + -o -path nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/audit.log \ + -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_running \ + -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/session_date_file \ + -o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/id_borg*" \ + \) \ + | LC_ALL=C sort \ + | LC_ALL=C comm -23 - \ + <(borg list "::$SELECTED_ARCHIVE" --short --exclude-from /borg_excludes --pattern '+nextcloud_aio_volumes/**' | LC_ALL=C sort) \ + > /tmp/local_files_not_in_backup + then + RESTORE_FAILED=1 + echo "Failed to delete local files not in backup archive." + else + # More robust than e.g. xargs as I got a ~"args line too long" error while testing that, but it's slower + # https://stackoverflow.com/a/21848934 + while IFS= read -r file + do rm -vrf -- "$file" || DELETE_FAILED=1 + done < /tmp/local_files_not_in_backup + + if [ "$DELETE_FAILED" = 1 ]; then + RESTORE_FAILED=1 + echo "Failed to delete (some) local files not in backup archive." + fi + fi + fi fi # Set backup-mode to restore since it was a restore CONTENTS="$(jq '."backup-mode" = "restore"' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json - # Reset the backup path to the currently used one + # Reset the backup location vars to the currently used one CONTENTS="$(jq ".borg_backup_host_location = $BORG_LOCATION" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json + CONTENTS="$(jq ".borg_remote_repo = $REMOTE_REPO" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" + echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json # Reset the AIO password to the currently used one CONTENTS="$(jq ".password = $AIO_PASSWORD" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)" @@ -366,8 +470,6 @@ if [ "$BORG_MODE" = restore ]; then chmod 770 "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_time" fi - umount /tmp/borg - if [ "$RESTORE_FAILED" = 1 ]; then exit 1 fi @@ -394,7 +496,7 @@ if [ "$BORG_MODE" = check ]; then echo "Checking the backup integrity..." # Perform the check - if ! borg check -v --verify-data "$BORG_BACKUP_DIRECTORY"; then + if ! borg check -v --verify-data; then echo "Some errors were found while checking the backup integrity!" echo "Check the AIO interface for advices on how to proceed now!" exit 1 @@ -412,7 +514,7 @@ if [ "$BORG_MODE" = "check-repair" ]; then echo "Checking the backup integrity and repairing it..." # Perform the check-repair - if ! echo YES | borg check -v --repair "$BORG_BACKUP_DIRECTORY"; then + if ! echo YES | borg check -v --repair; then echo "Some errors were found while checking and repairing the backup integrity!" exit 1 fi @@ -425,19 +527,29 @@ fi # Do the backup test if [ "$BORG_MODE" = test ]; then - if ! [ -d "$BORG_BACKUP_DIRECTORY" ]; then - echo "No 'borg' directory in the given backup directory found!" - echo "Only the files/folders below have been found in the given directory." - ls -a "$MOUNT_DIR" - echo "Please adjust the directory so that the borg archive is positioned in a folder named 'borg' inside the given directory!" - exit 1 - elif ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then - echo "A 'borg' directory was found but could not find the borg archive." - echo "Only the files/folders below have been found in the borg directory." - ls -a "$BORG_BACKUP_DIRECTORY" - echo "The archive and most importantly the config file must be positioned directly in the 'borg' subfolder." - exit 1 - elif ! borg list "$BORG_BACKUP_DIRECTORY"; then + if [ -n "$BORG_REMOTE_REPO" ]; then + if ! borg info > /dev/null; then + echo "Borg could not get info from the remote repo." + echo "See the above borg info output for details." + exit 1 + fi + else + if ! [ -d "$BORG_BACKUP_DIRECTORY" ]; then + echo "No 'borg' directory in the given backup directory found!" + echo "Only the files/folders below have been found in the given directory." + ls -a "$MOUNT_DIR" + echo "Please adjust the directory so that the borg archive is positioned in a folder named 'borg' inside the given directory!" + exit 1 + elif ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then + echo "A 'borg' directory was found but could not find the borg archive." + echo "Only the files/folders below have been found in the borg directory." + ls -a "$BORG_BACKUP_DIRECTORY" + echo "The archive and most importantly the config file must be positioned directly in the 'borg' subfolder." + exit 1 + fi + fi + + if ! borg list; then echo "The entered path seems to be valid but could not open the backup archive." echo "Most likely the entered password was wrong so please adjust it accordingly!" exit 1 diff --git a/Containers/borgbackup/borg_excludes b/Containers/borgbackup/borg_excludes new file mode 100644 index 00000000000..bbe6adaa0a3 --- /dev/null +++ b/Containers/borgbackup/borg_excludes @@ -0,0 +1,11 @@ +# These patterns need to be kept in sync with rsync and find excludes in backupscript.sh, +# which use a different syntax (patterns appear in 3 places in total) +nextcloud_aio_volumes/nextcloud_aio_apache/caddy/ +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/caddy/ +nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/nextcloud.log* +nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/audit.log +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/certs/ +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_running +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/session_date_file +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/session/ +nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/id_borg* \ No newline at end of file diff --git a/Containers/borgbackup/start.sh b/Containers/borgbackup/start.sh index e8d93f58943..9da0d840535 100644 --- a/Containers/borgbackup/start.sh +++ b/Containers/borgbackup/start.sh @@ -2,7 +2,7 @@ # Variables export MOUNT_DIR="/mnt/borgbackup" -export BORG_BACKUP_DIRECTORY="$MOUNT_DIR/borg" +export BORG_BACKUP_DIRECTORY="$MOUNT_DIR/borg" # necessary even when remote to store the aio-lockfile # Validate BORG_PASSWORD if [ -z "$BORG_PASSWORD" ] && [ -z "$BACKUP_RESTORE_PASSWORD" ]; then @@ -18,6 +18,18 @@ else fi export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes +if [ -n "$BORG_REMOTE_REPO" ]; then + export BORG_REPO="$BORG_REMOTE_REPO" + + # Location to create the borg ssh pub/priv key + export BORGBACKUP_KEY="/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/id_borg" + + # Accept any host key the first time connecting to the remote. Strictly speaking should be provided by user but you'd + # have to be very unlucky to get MitM'ed on your first connection. + export BORG_RSH="ssh -o StrictHostKeyChecking=accept-new -i $BORGBACKUP_KEY" +else + export BORG_REPO="$BORG_BACKUP_DIRECTORY" +fi # Validate BORG_MODE if [ "$BORG_MODE" != backup ] && [ "$BORG_MODE" != restore ] && [ "$BORG_MODE" != check ] && [ "$BORG_MODE" != "check-repair" ] && [ "$BORG_MODE" != test ]; then @@ -36,8 +48,8 @@ fi rm -f "/nextcloud_aio_volumes/nextcloud_aio_database_dump/backup-is-running" # Get a list of all available borg archives -if borg list "$BORG_BACKUP_DIRECTORY" &>/dev/null; then - borg list "$BORG_BACKUP_DIRECTORY" | grep "nextcloud-aio" | awk -F " " '{print $1","$3,$4}' > "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/backup_archives.list" +if borg list &>/dev/null; then + borg list | grep "nextcloud-aio" | awk -F " " '{print $1","$3,$4}' > "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/backup_archives.list" else echo "" > "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/backup_archives.list" fi diff --git a/Containers/mastercontainer/Dockerfile b/Containers/mastercontainer/Dockerfile index 1ba92044c2c..b0ec438b483 100644 --- a/Containers/mastercontainer/Dockerfile +++ b/Containers/mastercontainer/Dockerfile @@ -64,7 +64,7 @@ RUN set -ex; \ wget https://getcomposer.org/installer -O - | php -- --install-dir=/usr/local/bin --filename=composer; \ chmod +x /usr/local/bin/composer; \ cd /var/www/docker-aio; \ - git clone https://github.com/nextcloud-releases/all-in-one.git --depth 1 .; \ + git clone https://github.com/timdiels/nextcloud-aio.git --depth 1 --branch feature/remote-borg-backup3 .; \ find ./ -maxdepth 1 -mindepth 1 -not -path ./php -not -path ./community-containers -exec rm -r {} \; ; \ chown www-data:www-data -R /var/www/docker-aio; \ cd php; \ diff --git a/php/containers.json b/php/containers.json index a107df84a40..2146e584f23 100644 --- a/php/containers.json +++ b/php/containers.json @@ -458,6 +458,7 @@ "image": "nextcloud/aio-borgbackup", "init": true, "environment": [ + "BORG_REMOTE_REPO=%BORGBACKUP_REMOTE_REPO%", "BORG_PASSWORD=%BORGBACKUP_PASSWORD%", "BORG_MODE=%BORGBACKUP_MODE%", "SELECTED_RESTORE_TIME=%SELECTED_RESTORE_TIME%", diff --git a/php/public/index.php b/php/public/index.php index 9614419eb0d..30e39246c46 100644 --- a/php/public/index.php +++ b/php/public/index.php @@ -86,6 +86,8 @@ 'domain' => $configurationManager->GetDomain(), 'apache_port' => $configurationManager->GetApachePort(), 'borg_backup_host_location' => $configurationManager->GetBorgBackupHostLocation(), + 'borg_remote_repo' => $configurationManager->GetBorgRemoteRepo(), + 'borg_public_key' => $configurationManager->GetBorgPublicKey(), 'nextcloud_password' => $configurationManager->GetAndGenerateSecret('NEXTCLOUD_PASSWORD'), 'containers' => (new \AIO\ContainerDefinitionFetcher($container->get(\AIO\Data\ConfigurationManager::class), $container))->FetchDefinition(), 'borgbackup_password' => $configurationManager->GetAndGenerateSecret('BORGBACKUP_PASSWORD'), diff --git a/php/src/Controller/ConfigurationController.php b/php/src/Controller/ConfigurationController.php index c38c1bc1998..67463ab14c7 100644 --- a/php/src/Controller/ConfigurationController.php +++ b/php/src/Controller/ConfigurationController.php @@ -28,15 +28,17 @@ public function SetConfig(Request $request, Response $response, array $args) : R $this->configurationManager->ChangeMasterPassword($currentMasterPassword, $newMasterPassword); } - if (isset($request->getParsedBody()['borg_backup_host_location'])) { + if (isset($request->getParsedBody()['borg_backup_host_location']) || isset($request->getParsedBody()['borg_remote_repo'])) { $location = $request->getParsedBody()['borg_backup_host_location'] ?? ''; - $this->configurationManager->SetBorgBackupHostLocation($location); + $borgRemoteRepo = $request->getParsedBody()['borg_remote_repo'] ?? ''; + $this->configurationManager->SetBorgLocationVars($location, $borgRemoteRepo); } - if (isset($request->getParsedBody()['borg_restore_host_location']) || isset($request->getParsedBody()['borg_restore_password'])) { + if (isset($request->getParsedBody()['borg_restore_host_location']) || isset($request->getParsedBody()['borg_restore_remote_repo']) || isset($request->getParsedBody()['borg_restore_password'])) { $restoreLocation = $request->getParsedBody()['borg_restore_host_location'] ?? ''; + $borgRemoteRepo = $request->getParsedBody()['borg_restore_remote_repo'] ?? ''; $borgPassword = $request->getParsedBody()['borg_restore_password'] ?? ''; - $this->configurationManager->SetBorgRestoreHostLocationAndPassword($restoreLocation, $borgPassword); + $this->configurationManager->SetBorgRestoreLocationVarsAndPassword($restoreLocation, $borgRemoteRepo, $borgPassword); } if (isset($request->getParsedBody()['daily_backup_time'])) { @@ -132,8 +134,8 @@ public function SetConfig(Request $request, Response $response, array $args) : R $this->configurationManager->SetCollaboraDictionaries($collaboraDictionaries); } - if (isset($request->getParsedBody()['delete_borg_backup_host_location'])) { - $this->configurationManager->DeleteBorgBackupHostLocation(); + if (isset($request->getParsedBody()['delete_borg_backup_location_vars'])) { + $this->configurationManager->DeleteBorgBackupLocationVars(); } return $response->withStatus(201)->withHeader('Location', '/'); diff --git a/php/src/Data/ConfigurationManager.php b/php/src/Data/ConfigurationManager.php index 527904fbfed..6404042978e 100644 --- a/php/src/Data/ConfigurationManager.php +++ b/php/src/Data/ConfigurationManager.php @@ -439,48 +439,61 @@ public function GetAIOURL() : string { /** * @throws InvalidSettingConfigurationException */ - public function SetBorgBackupHostLocation(string $location) : void { - $isValidPath = false; - if (str_starts_with($location, '/') && !str_ends_with($location, '/')) { - $isValidPath = true; - } elseif ($location === 'nextcloud_aio_backupdir') { - $isValidPath = true; - } - - if (!$isValidPath) { - throw new InvalidSettingConfigurationException("The path must start with '/', and must not end with '/'!"); - } - + public function SetBorgLocationVars(string $location, string $repo) : void { + $this->ValidateBorgLocationVars($location, $repo); $config = $this->GetConfig(); $config['borg_backup_host_location'] = $location; + $config['borg_remote_repo'] = $repo; $this->WriteConfig($config); } - public function DeleteBorgBackupHostLocation() : void { + private function ValidateBorgLocationVars(string $location, string $repo) : void { + if ($location === '' && $repo === '') { + throw new InvalidSettingConfigurationException("Please enter a path or a remote repo url!"); + } elseif ($location !== '' && $repo !== '') { + throw new InvalidSettingConfigurationException("Location and remote repo url are mutually exclusive!"); + } + + if ($location !== '') { + $isValidPath = false; + if (str_starts_with($location, '/') && !str_ends_with($location, '/')) { + $isValidPath = true; + } elseif ($location === 'nextcloud_aio_backupdir') { + $isValidPath = true; + } + + if (!$isValidPath) { + throw new InvalidSettingConfigurationException("The path must start with '/', and must not end with '/'!"); + } + } else { + $this->ValidateBorgRemoteRepo($repo); + } + } + + private function ValidateBorgRemoteRepo(string $repo) : void { + $commonMsg = "For valid urls, see the remote examples at https://borgbackup.readthedocs.io/en/stable/usage/general.html#repository-urls"; + if ($repo === "") { + // Ok, remote repo is optional + } elseif (!str_contains($repo, "@")) { + throw new InvalidSettingConfigurationException("The remote repo must contain '@'. $commonMsg"); + } elseif (!str_contains($repo, ":")) { + throw new InvalidSettingConfigurationException("The remote repo must contain ':'. $commonMsg"); + } + } + + public function DeleteBorgBackupLocationVars() : void { $config = $this->GetConfig(); $config['borg_backup_host_location'] = ''; + $config['borg_remote_repo'] = ''; $this->WriteConfig($config); } - /** + /** * @throws InvalidSettingConfigurationException */ - public function SetBorgRestoreHostLocationAndPassword(string $location, string $password) : void { - if ($location === '') { - throw new InvalidSettingConfigurationException("Please enter a path!"); - } - - $isValidPath = false; - if (str_starts_with($location, '/') && !str_ends_with($location, '/')) { - $isValidPath = true; - } elseif ($location === 'nextcloud_aio_backupdir') { - $isValidPath = true; - } - - if (!$isValidPath) { - throw new InvalidSettingConfigurationException("The path must start with '/', and must not end with '/'!"); - } + public function SetBorgRestoreLocationVarsAndPassword(string $location, string $repo, string $password) : void { + $this->ValidateBorgLocationVars($location, $repo); if ($password === '') { throw new InvalidSettingConfigurationException("Please enter the password!"); @@ -488,6 +501,7 @@ public function SetBorgRestoreHostLocationAndPassword(string $location, string $ $config = $this->GetConfig(); $config['borg_backup_host_location'] = $location; + $config['borg_remote_repo'] = $repo; $config['borg_restore_password'] = $password; $config['instance_restore_attempt'] = 1; $this->WriteConfig($config); @@ -582,6 +596,23 @@ public function GetBorgBackupHostLocation() : string { return $config['borg_backup_host_location']; } + public function GetBorgRemoteRepo() : string { + $config = $this->GetConfig(); + if(!isset($config['borg_remote_repo'])) { + $config['borg_remote_repo'] = ''; + } + + return $config['borg_remote_repo']; + } + + public function GetBorgPublicKey() : string { + if (!file_exists(DataConst::GetBackupPublicKey())) { + return ""; + } + + return trim(file_get_contents(DataConst::GetBackupPublicKey())); + } + public function GetBorgRestorePassword() : string { $config = $this->GetConfig(); if(!isset($config['borg_restore_password'])) { diff --git a/php/src/Data/DataConst.php b/php/src/Data/DataConst.php index 2512b7fb679..4f69732540e 100644 --- a/php/src/Data/DataConst.php +++ b/php/src/Data/DataConst.php @@ -23,6 +23,10 @@ public static function GetConfigFile() : string { return self::GetDataDirectory() . '/configuration.json'; } + public static function GetBackupPublicKey() : string { + return self::GetDataDirectory() . '/id_borg.pub'; + } + public static function GetBackupSecretFile() : string { return self::GetDataDirectory() . '/backupsecret'; } diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index 12a641e0869..6c7f716a512 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -265,6 +265,8 @@ public function CreateContainer(Container $container) : void { $replacements[1] = $this->configurationManager->GetBaseDN(); } elseif ($out[1] === 'AIO_TOKEN') { $replacements[1] = $this->configurationManager->GetToken(); + } elseif ($out[1] === 'BORGBACKUP_REMOTE_REPO') { + $replacements[1] = $this->configurationManager->GetBorgRemoteRepo(); } elseif ($out[1] === 'BORGBACKUP_MODE') { $replacements[1] = $this->configurationManager->GetBackupMode(); } elseif ($out[1] === 'AIO_URL') { diff --git a/php/templates/containers.twig b/php/templates/containers.twig index d1ddb499dec..2c02a64cd61 100644 --- a/php/templates/containers.twig +++ b/php/templates/containers.twig @@ -25,6 +25,7 @@ {# timezone-prefill #} + {% set hasBackupLocation = borg_backup_host_location or borg_remote_repo %} {% set isAnyRunning = false %} {% set isAnyRestarting = false %} {% set isWatchtowerRunning = false %} @@ -90,7 +91,7 @@ {% else %} - {% if borg_backup_host_location == '' and borg_restore_password == '' %} + {% if not hasBackupLocation %}
The official Nextcloud installation method. Nextcloud All-in-One provides easy deployment and maintenance with most features included in this one Nextcloud instance.
You can either create a new AIO instance or restore a former AIO instance from backup. See the two sections below.
{{ include('includes/aio-config.twig') }} @@ -130,7 +131,7 @@ {% endif %} {% if is_instance_restore_attempt == false %} - {% if borg_backup_host_location != '' and borg_restore_password != '' %} + {% if hasBackupLocation %} {% if borg_backup_mode in ['test', 'check'] %} {% if backup_exit_code > 0 %}Last {{ borg_backup_mode }} failed! (Logs)
@@ -179,11 +180,23 @@ {% endif %} {% endif %} - {% if borg_backup_host_location == '' or borg_restore_password == '' or borg_backup_mode not in ['test', 'check', ''] or backup_exit_code > 0 %} -Please enter the location of the backup archive on your host and the encryption password of the backup archive below:
+ {% if not hasBackupLocation or borg_backup_mode not in ['test', 'check', ''] or backup_exit_code > 0 %} + {% if borg_remote_repo and backup_exit_code > 0 %} +
+ You may still need to authorize this pubkey on your borg remote:
{{ borg_public_key }}
+ To try again, resubmit your location and rerun the test.
+
+ Please enter the location of the backup archive on your host or a + remote borg repo url + if stored remotely; and the encryption password of the backup archive below: +
{% if has_backup_run_once == false %} -If the configured backup host location {{ borg_backup_host_location }} is wrong, you can reset it by clicking on the button below.
++ If the configured backup host location {{ borg_backup_host_location }} + {% if borg_remote_repo %}or the remote repo {{ borg_remote_repo }} is wrong{% endif %}, + you can reset it by clicking on the button below. +