From 28d004e26b8e0fd51a38844940004959ea5d7904 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 17:43:34 -0500 Subject: [PATCH 01/23] minor fixes to zfs-replicate.sh --- zfs-replicate.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/zfs-replicate.sh b/zfs-replicate.sh index 26ae894..a998934 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -41,14 +41,14 @@ pruneLogs() { ## check count and delete old logs if [[ "${#logs[@]}" -gt "$LOG_KEEP" ]]; then printf "pruning logs %s\n" "${logs[*]:${LOG_KEEP}}" - rm -rf "${logs[@]:${LOG_KEEP}}" + rm -f "${logs[@]:${LOG_KEEP}}" fi } ## delete lock files clearLock() { local lockFile=$1 - if [ -f "$lockFile" ]; then + if [[ -f "$lockFile" ]]; then printf "deleting lockfile %s\n" "$lockFile" rm "$lockFile" fi @@ -92,18 +92,17 @@ checkLock() { local ps if ps=$(pgrep -lx -F "$lockFile"); then ## looks like it's still running - printf "ERROR: script is already running as: %s\n" "$ps" + printf "ERROR: script is already running as: %d\n" "$ps" else ## stale lock file? printf "ERROR: stale lockfile %s\n" "$lockFile" fi ## cleanup and exit exitClean 128 "confirm script is not running and delete lockfile $lockFile" - else - ## well no lockfile..let's make a new one - printf "creating lockfile %s\n" "$lockFile" - printf "%d\n" "$$" > "$lockFile" fi + ## well no lockfile..let's make a new one + printf "creating lockfile %s\n" "$lockFile" + printf "%d\n" "$$" > "$lockFile" } ## check remote host status @@ -275,7 +274,7 @@ snapCreate() { fi done ## set our base snap for incremental generation if src contains a sufficient - ## number of snapshots and the base source snapshot exists in destination data set. + ## number of snapshots and the base source snapshot exists in destination dataset local base if [[ ${#srcSnaps[@]} -ge 1 ]]; then ## set source snap base candidate @@ -284,7 +283,7 @@ snapCreate() { read -r -a tempa <<< "${ss//@/ }" sn="${tempa[1]}" sn="${sn%"${sn##*[![:space:]]}"}" - ## loop over base snaps and check for a match + ## loop over destination snaps and check for a match for snap in "${dstSnaps[@]}"; do read -r -a tempa <<< "${snap//@/ }" dn="${tempa[1]}" @@ -327,7 +326,7 @@ snapCreate() { unset 'srcSnaps[idx]' fi done - ## come on already...make that snapshot + ## come on already...take that snapshot if [[ -n "$srcHost" ]]; then read -r -a cmd <<< "$SSH" cmd+=("$srcHost") @@ -392,7 +391,7 @@ showStatus() { if [[ -n "${logs[0]}" ]]; then printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${logs[0]}")" else - printf "Unable to find most recent log file, cannot print status." + printf "Unable to find most recent log file, cannot print status.\n" fi exit 0 } From 937320a8855c9fe8f1c8a4995d6a95a21f8c2958 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 17:44:14 -0500 Subject: [PATCH 02/23] add new posix compliant script --- zfs-replicate-posix.sh | 502 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100755 zfs-replicate-posix.sh diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh new file mode 100755 index 0000000..af241c2 --- /dev/null +++ b/zfs-replicate-posix.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env sh +## zfs-replicate.sh +set -e + +############################################ +##### warning gremlins live below here ##### +############################################ + +## output log files in decreasing age order +sortLogs() { + ## check if file logging is enabled + if [ -z "$LOG_BASE" ] || [ ! -d "$LOG_BASE" ]; then + return + fi + ## find existing logs + logs=$(find "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') + ## get file change time via stat (platform specific) + if [ "$(uname -s)" = "Linux" ] || [ "$(uname -s)" = "SunOS" ]; then + fstat='stat -c %Z' + else + fstat='stat -f %c' + fi + ## output logs in descending age order + for log in $logs; do + printf "%s\t%s\n" "$($fstat "$log")" "$log" + done | sort -rn | cut -f2 +} + +## check log count and delete old logs +pruneLogs() { + logs=$(sortLogs) + logCount=$(echo "$logs" | wc -l) + if [ "$logCount" -gt "$LOG_KEEP" ]; then + prune="$(echo "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" + printf "pruning logs %s\n" "$prune" + echo "$prune" | xargs rm -f + fi +} + +## delete lock files +clearLock() { + lockFile=$1 + if [ -f "$lockFile" ]; then + printf "deleting lockfile %s\n" "$lockFile" + rm "$lockFile" + fi +} + +## exit and cleanup +exitClean() { + exitCode=${1:-0} + extraMsg=$2 + status="success" + ## set status to warning if we skipped any datasets + if [ "$__SKIP_COUNT" -gt 0 ]; then + status="WARNING" + fi + logMsg=$(printf "%s total sets %d skipped %d" "$status" "$__PAIR_COUNT" "$__SKIP_COUNT") + ## build and print error message + if [ "$exitCode" -ne 0 ]; then + status="ERROR" + logMsg=$(printf "%s: operation exited unexpectedly: code=%d" "$status" "$exitCode") + if [ -n "$extraMsg" ]; then + logMsg=$(printf "%s msg=%s" "$logMsg" "$extraMsg") + fi + fi + ## append extra message if available + if [ "$exitCode" -eq 0 ] && [ -n "$extraMsg" ]; then + logMsg=$(printf "%s: %s" "$logMsg" "$extraMsg") + fi + ## cleanup old logs and clear locks + pruneLogs + clearLock "${TMPDIR}/.replicate.snapshot.lock" + clearLock "${TMPDIR}/.replicate.send.lock" + ## print log message and exit + printf "%s\n" "$logMsg" + exit "$exitCode" +} + +## lockfile creation and maintenance +checkLock() { + lockFile=$1 + ## check our lockfile status + if [ -f "$lockFile" ]; then + ## see if this pid is still running + if ps -p "$(cat "$lockFile")" > /dev/null 2>&1; then + ## looks like it's still running + printf "ERROR: script is already running as: %d\n" "$(cat "$lockFile")" + else + ## stale lock file? + printf "ERROR: stale lockfile %s\n" "$lockFile" + fi + ## cleanup and exit + exitClean 128 "confirm script is not running and delete lockfile $lockFile" + fi + ## well no lockfile..let's make a new one + printf "creating lockfile %s\n" "$lockFile" + printf "%d\n" "$$" > "$lockFile" +} + +## check remote host status +checkHost() { + ## do we have a host check defined + if [ -z "$HOST_CHECK" ]; then + return + fi + host=$1 + cmd=$(echo "$HOST_CHECK" | sed "s/%HOST%/$host/g") + printf "checking host cmd=%s\n" "$cmd" + ## run the check + if ! sh -c "$cmd" > /dev/null 2>&1; then + exitClean 128 "host check failed" + fi +} + +## ensure dataset exists +checkDataset() { + set=$1 + host=$2 + cmd="" + ## build command + if [ -n "$host" ]; then + cmd="$SSH $host " + fi + cmd="$cmd$ZFS list -H -o name $set" + printf "checking dataset cmd=%s\n" "$cmd" + ## execute command + if ! sh -c "$cmd"; then + exitClean 128 "failed to list dataset: $set" + fi +} + +## small wrapper around zfs destroy +snapDestroy() { + snap=$1 + host=$2 + cmd="" + ## build command + if [ -n "$host" ]; then + cmd="$SSH $host " + fi + cmd="$cmd$ZFS destroy" + if [ "$RECURSE_CHILDREN" -eq 1 ]; then + cmd="$cmd -r" + fi + cmd="$cmd $snap" + printf "destroying snapshot cmd=%s\n" "$cmd" + ## ignore error from destroy and count on logging to alert the end-user + ## destroying recursive snapshots can lead to "snapshot not found" errors + sh -c "$cmd" || true +} + +## main replication function +snapSend() { + base=$1 + snap=$2 + src=$3 + srcHost=$4 + dst=$5 + dstHost=$6 + ## check our send lockfile + checkLock "${TMPDIR}/.replicate.send.lock" + ## begin building send command + if [ -n "$srcHost" ]; then + cmd="$SSH $srcHost " + fi + cmd="$cmd$ZFS send -Rs" + ## if first snap name is not empty generate an incremental + if [ -n "$base" ]; then + cmd="$cmd -I $base" + fi + cmd="$cmd ${src}@${snap}" + ## set destination pipe based on destination host + pipe="$DEST_PIPE_WITHOUT_HOST" + if [ -n "$dstHost" ]; then + pipe=$(echo "$DEST_PIPE_WITH_HOST" | sed "s/%HOST%/$dstHost/g") + fi + pipe="$pipe $dst" + printf "sending snapshot cmd=%s | %s\n" "$cmd" "$pipe" + ## execute send and check return + if ! sh -c "$cmd" | eval "$pipe"; then + snapDestroy "${src}@${name}" "$srcHost" + exitClean 128 "failed to send snapshot: ${src}@${name}" + fi + ## clear lockfile + clearLock "${TMPDIR}/.replicate.send.lock" +} + +## list replication snapshots +snapList() { + set=$1 + host=$2 + depth=${3:-0} + cmd="" + ## build send command + if [ -n "$host" ]; then + cmd="$SSH $host " + fi + cmd="$cmd$ZFS list -Hr -o name -s creation -t snapshot" + if [ "$depth" -gt 0 ]; then + cmd="$cmd -d $depth" + fi + cmd="$cmd $set" + ## get snapshots from host + if ! snaps=$(sh -c "$cmd"); then + exitClean 128 "failed to list snapshots for dataset: $set" + fi + ## filter snaps matching our pattern + echo "$snaps" | grep "@autorep-" +} + +## helper function to check if substring is within string +contains() { + string="$1" + substring="$2" + if [ "${string#*"$substring"}" != "$string" ]; then + return 0 + fi + return 1 +} + +## create and manage source snapshots +snapCreate() { + ## make sure we aren't ever creating simultaneous snapshots + checkLock "${TMPDIR}/.replicate.snapshot.lock" + ## set our snap name + name="autorep-${TAG}" + ## generate snapshot list and cleanup old snapshots + for pair in $REPLICATE_SETS; do + __PAIR_COUNT=$((__PAIR_COUNT + 1)) + ## split dataset into source and destination parts and trim any trailing space + src=$(echo "$pair" | cut -f1 -d: | sed 's/[[:space:]]*$//') + dst=$(echo "$pair" | cut -f2 -d: | sed 's/[[:space:]]*$//') + ## check for root dataset destination + if [ "$ALLOW_ROOT_DATASETS" -ne 1 ]; then + if [ "$dst" = "$(basename "$dst")" ] || [ "$dst" = "$(basename "$dst")/" ]; then + temps="replicating root datasets can lead to data loss - set ALLOW_ROOT_DATASETS=1 to override" + printf "WARNING: %s\n" "$temps" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) + continue + fi + fi + ## look for host options on source + if contains "$src" "@"; then + srcHost=$(echo "$src" | cut -f2 -d@) + src=$(echo "$src" | cut -f1 -d@) + fi + ## look for host options on destination + if contains "$dst" "@"; then + dstHost=$(echo "$dst" | cut -f2 -d@) + dst=$(echo "$dst" | cut -f1 -d@) + fi + ## check source and destination datasets + checkDataset "$src" "$srcHost" + checkDataset "$dst" "$dstHost" + ## get source and destination snapshots + srcSnaps=$(snapList "$src" "$srcHost" 1) + dstSnaps=$(snapList "$dst" "$dstHost" 0) + for snap in $srcSnaps; do + ## while we are here...check for our current snap name + if [ "$snap" = "${src}@${name}" ]; then + ## looks like it's here...we better kill it + printf "destroying duplicate snapshot: %s@%s\n" "$src" "$name" + snapDestroy "${src}@${name}" "$srcHost" + fi + done + ## get source and destination snap count + srcSnapCount=$(echo "$srcSnaps" | wc -l) + dstSnapCount=$(echo "$dstSnaps" | wc -l) + ## set our base snap for incremental generation if src contains a sufficient + ## number of snapshots and the base source snapshot exists in destination dataset + base="" + if [ "$srcSnapCount" -ge 1 ]; then + ## get most recent source snapshot + ss=$(printf "%s\n" "$srcSnaps" | tail -n 1) + ## get source snapshot name + sn=$(echo "$ss" | cut -f2 -d@) + ## loop over destinations snaps and look for a match + for ds in $dstSnaps; do + dn=$(echo "$ds" | cut -f2 -d@) + if [ "$dn" = "$sn" ]; then + base="$ss" + fi + done + ## no matching base, are we allowed to fallback? + if [ -z "$base" ] && [ "$dstSnapCount" -ge 1 ] && [ "$ALLOW_RECONCILIATION" -ne 1 ]; then + temps=$(printf "source snapshot '%s' not in destination dataset: %s" "$ss" "$dst") + temps=$(printf "%s - set 'ALLOW_RECONCILIATION=1' to fallback to a full send" "$temps") + printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) + continue + fi + fi + ## without a base snapshot, the destination must be clean + if [ -z "$base" ] && [ "$dstSnapCount" -gt 0 ]; then + ## allowed to prune remote dataset? + if [ "$ALLOW_RECONCILIATION" -ne 1 ]; then + temps="destination contains snapshots not in source - set 'ALLOW_RECONCILIATION=1' to prune snapshots" + printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) + continue + fi + ## prune destination snapshots + printf "pruning destination snapshots: %s\n" "$dstSnaps" + for snap in $dstSnaps; do + snapDestroy "$snap" "$dstHost" + done + fi + ## cleanup old snapshots + if [ "$srcSnapCount" -ge "$SNAP_KEEP" ]; then + ## snaps are sorted above by creation in ascending order + echo "$srcSnaps" | sed -n "1,$((srcSnapCount - SNAP_KEEP))p" | while read -r snap; do + printf "found old snapshot %s\n" "$snap" + snapDestroy "$snap" "$srcHost" + done + fi + ## build snapshot create command + cmd="" + if [ -n "$srcHost" ]; then + cmd="$SSH $srcHost " + fi + cmd="$cmd$ZFS snapshot" + ## check if we are supposed to be recursive + if [ "$RECURSE_CHILDREN" -eq 1 ]; then + cmd="$cmd -r" + fi + cmd="$cmd ${src}@${name}" + ## come on already...take that snapshot + printf "creating snapshot cmd=%s\n" "$cmd" + if ! sh -c "$cmd"; then + snapDestroy "${src}@${name}" "$srcHost" + exitClean 128 "failed to create snapshot: ${src}@${name}" + fi + ## send snapshot to destination + snapSend "$base" "$name" "$src" "$srcHost" "$dst" "$dstHost" + done + ## clear snapshot lockfile + clearLock "${TMPDIR}/.replicate.snapshot.lock" +} + +## handle logging to file or syslog +writeLog() { + line=$1 + logf="/dev/null" + ## if a log base and file has been configured set them + if [ -n "$LOG_BASE" ] && [ -n "$LOG_FILE" ]; then + logf="${LOG_BASE}/${LOG_FILE}" + fi + ## always print to stdout and copy to logfile if set + printf "%s %s[%d]: %s\n" "$(date '+%b %d %T')" "$SCRIPT" "$$" "$line" | tee -a "$logf" + ## if syslog has been enabled write to syslog via logger + if [ -n "$SYSLOG" ] && [ "$SYSLOG" -eq 1 ] && [ -n "$LOGGER" ]; then + $LOGGER -p "${SYSLOG_FACILITY}.info" -t "$SCRIPT" "$line" + fi +} + +## read from stdin till script exit +captureOutput() { + while IFS= read -r line; do + writeLog "$line" + done +} + +## perform macro substitution for tags +subTags() { + m=$1 + ## do the substitutions + m=$(echo "$m" | sed "s/%DOW%/${__DOW}/g") + m=$(echo "$m" | sed "s/%DOM%/${__DOM}/g") + m=$(echo "$m" | sed "s/%MOY%/${__MOY}/g") + m=$(echo "$m" | sed "s/%CYR%/${__CYR}/g") + m=$(echo "$m" | sed "s/%NOW%/${__NOW}/g") + m=$(echo "$m" | sed "s/%TAG%/${TAG}/g") + printf "%s\n" "$m" +} + +## show last replication status +showStatus() { + log=$(sortLogs | head -n 1) + if [ -n "$log" ]; then + printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${log}")" + else + printf "Unable to find most recent log file, cannot print status.\n" + fi + exit 0 +} + +## show usage and exit +showHelp() { + printf "Usage: %s [options] [config]\n\n" "${SCRIPT}" + printf "Bash script to automate ZFS Replication\n\n" + printf "Options:\n" + printf " -c, --config bash configuration file\n" + printf " -s, --status print most recent log messages to stdout\n" + printf " -h, --help show this message\n" + exit 0 +} + +## read and load config file +loadConfig() { + ## set SCRIPT used by writeLog and showStatus + readonly SCRIPT="${0##*/}" + readonly SCRIPT_PATH="${0%/*}" + configFile="" + status=0 + ## process command-line options + while [ $# -gt 0 ]; do + case "$1" in + -c | --config) + configFile=$2 + shift + ;; + -s | --status) + showStatus + ;; + -h | --help) + showHelp + ;; + *) # bad long option + writeLog "ERROR: illegal option ${1}" && exit 1 + ;; + esac + shift + done + ## attempt to load configuration + if [ -f "$configFile" ]; then + writeLog "sourcing config file $configFile" + # shellcheck disable=SC1090 + . "$configFile" + elif configFile="${SCRIPT_PATH}/config.sh" && [ -f "$configFile" ]; then + writeLog "sourcing config file $configFile" + # shellcheck disable=SC1090 + . "$configFile" + else + writeLog "loading configuration from defaults and environmental settings." + fi + ## set date substitutions for macros + __DOW=$(date "+%a") + readonly __DOW + __DOM=$(date "+%d") + readonly __DOM + __MOY=$(date "+%m") + readonly __MOY + __CYR=$(date "+%Y") + readonly __CYR + __NOW=$(date "+%s") + readonly __NOW + ## complete configuration with values from environment or set defaults + readonly TMPDIR="${TMPDIR:-"/tmp"}" + readonly REPLICATE_SETS ## no default value + readonly ALLOW_ROOT_DATASETS="${ALLOW_ROOT_DATASETS:-0}" + readonly ALLOW_RECONCILIATION="${ALLOW_RECONCILIATION:-0}" + readonly RECURSE_CHILDREN="${RECURSE_CHILDREN:-0}" + readonly SNAP_KEEP="${SNAP_KEEP:-2}" + readonly SYSLOG="${SYSLOG:-1}" + readonly SYSLOG_FACILITY="${SYSLOG_FACILITY:-"user"}" + TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" + TAG="$(subTags "$TAG")" + readonly TAG + LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" + LOG_FILE="$(subTags "$LOG_FILE")" + readonly LOG_FILE + readonly LOG_KEEP="${LOG_KEEP:-5}" + readonly LOG_BASE ## no default value + readonly LOGGER="${LOGGER:-$(which logger)}" + readonly FIND="${FIND:-$(which find)}" + readonly ZFS="${ZFS:-$(which zfs)}" + readonly SSH="${SSH:-$(which ssh)}" + readonly DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" + readonly DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" + readonly HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" + ## init values used in snapCreate and exitClean + __PAIR_COUNT=0 + __SKIP_COUNT=0 + ## check configuration + if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then + mkdir -p "$LOG_BASE" + fi + if [ -z "$REPLICATE_SETS" ]; then + writeLog "ERROR: missing required setting REPLICATE_SETS" && exit 1 + fi + if [ -z "$ZFS" ]; then + writeLog "ERROR: unable to locate system zfs binary" && exit 1 + fi + if [ "$SNAP_KEEP" -lt 2 ]; then + writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 + fi + ## show status if toggled + if [ "$status" -eq 1 ]; then + showStatus + fi +} + +main() { + ## do snapshots and send + snapCreate + ## that's it, sending is called from doSnap + exitClean 0 +} + +## here we go ... +loadConfig "$@" && main 2>&1 | captureOutput From 13478c985dc39df709fcc820f967cd40958ce8db Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 17:47:36 -0500 Subject: [PATCH 03/23] note posix script in README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index acc5b7e..d3edf64 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ A Bash script to automate ZFS Replication. - Includes a `--status` option for XigmaNAS that can be used to email the last log output at your preferred schedule. Simply add it as a custom script in the email settings under "System > Advanced > Email Reports" + +## Experimental POSIX Support + +There is a new script called `zfs-replicate-posix.sh` that is POSIX compliant (sh|dash). This script was converted from +the existing script with ChatGPT and then hand reviewed line-by-line for correctness and completion of parts that were +left out from the ChatGPT conversion. This script is currently considered experimental, but feedback would +be appreciated. + ## FreeBSD Package This script is available in the FreeBSD [package and ports tree](https://www.freshports.org/sysutils/zfs-replicate/). From 09d982491e388ad83fabf0a97d06db057d118f89 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 19:24:31 -0500 Subject: [PATCH 04/23] bugfix for variables not being reset --- .github/workflows/status-checks.yaml | 4 ++-- test.sh => test-bash.sh | 0 zfs-replicate-posix.sh | 19 +++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) rename test.sh => test-bash.sh (100%) diff --git a/.github/workflows/status-checks.yaml b/.github/workflows/status-checks.yaml index c7bdf40..2325999 100644 --- a/.github/workflows/status-checks.yaml +++ b/.github/workflows/status-checks.yaml @@ -54,7 +54,7 @@ jobs: level: info filter_mode: nofilter fail_on_error: true - shfmt_flags: '-ln bash -ci -sr -i 2' + shfmt_flags: '-ci -sr -i 2' shellcheck: name: runner / shellcheck @@ -76,6 +76,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: bash ./test.sh + - run: bash ./test-bash.sh env: TMPDIR: ${{ runner.temp }} diff --git a/test.sh b/test-bash.sh similarity index 100% rename from test.sh rename to test-bash.sh diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh index af241c2..2f1e300 100755 --- a/zfs-replicate-posix.sh +++ b/zfs-replicate-posix.sh @@ -108,7 +108,7 @@ checkHost() { cmd=$(echo "$HOST_CHECK" | sed "s/%HOST%/$host/g") printf "checking host cmd=%s\n" "$cmd" ## run the check - if ! sh -c "$cmd" > /dev/null 2>&1; then + if ! $cmd > /dev/null 2>&1; then exitClean 128 "host check failed" fi } @@ -125,7 +125,7 @@ checkDataset() { cmd="$cmd$ZFS list -H -o name $set" printf "checking dataset cmd=%s\n" "$cmd" ## execute command - if ! sh -c "$cmd"; then + if ! $cmd; then exitClean 128 "failed to list dataset: $set" fi } @@ -147,7 +147,7 @@ snapDestroy() { printf "destroying snapshot cmd=%s\n" "$cmd" ## ignore error from destroy and count on logging to alert the end-user ## destroying recursive snapshots can lead to "snapshot not found" errors - sh -c "$cmd" || true + $cmd || true } ## main replication function @@ -161,6 +161,7 @@ snapSend() { ## check our send lockfile checkLock "${TMPDIR}/.replicate.send.lock" ## begin building send command + cmd="" if [ -n "$srcHost" ]; then cmd="$SSH $srcHost " fi @@ -178,7 +179,7 @@ snapSend() { pipe="$pipe $dst" printf "sending snapshot cmd=%s | %s\n" "$cmd" "$pipe" ## execute send and check return - if ! sh -c "$cmd" | eval "$pipe"; then + if ! $cmd | $pipe; then snapDestroy "${src}@${name}" "$srcHost" exitClean 128 "failed to send snapshot: ${src}@${name}" fi @@ -202,7 +203,7 @@ snapList() { fi cmd="$cmd $set" ## get snapshots from host - if ! snaps=$(sh -c "$cmd"); then + if ! snaps=$($cmd); then exitClean 128 "failed to list snapshots for dataset: $set" fi ## filter snaps matching our pattern @@ -327,7 +328,7 @@ snapCreate() { cmd="$cmd ${src}@${name}" ## come on already...take that snapshot printf "creating snapshot cmd=%s\n" "$cmd" - if ! sh -c "$cmd"; then + if ! $cmd; then snapDestroy "${src}@${name}" "$srcHost" exitClean 128 "failed to create snapshot: ${src}@${name}" fi @@ -498,5 +499,7 @@ main() { exitClean 0 } -## here we go ... -loadConfig "$@" && main 2>&1 | captureOutput +## process config and start main if we weren't sourced +if [ "${0##*/}" != "sh" ] && [ "${0##*/}" != "dash" ] && [ "${0##*/}" != "-bash" ]; then + loadConfig "$@" && main 2>&1 | captureOutput +fi From ba34f75e221be5495a86b719b8925d659d50f87c Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 20:46:25 -0500 Subject: [PATCH 05/23] variable default fixes --- zfs-replicate-posix.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh index 2f1e300..9c040db 100755 --- a/zfs-replicate-posix.sh +++ b/zfs-replicate-posix.sh @@ -29,11 +29,14 @@ sortLogs() { ## check log count and delete old logs pruneLogs() { logs=$(sortLogs) - logCount=$(echo "$logs" | wc -l) + logCount=0 + if [ -n "$logs" ]; then + logCount=$(printf "%s" "$logs" | wc -l) + fi if [ "$logCount" -gt "$LOG_KEEP" ]; then prune="$(echo "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" - printf "pruning logs %s\n" "$prune" - echo "$prune" | xargs rm -f + printf "pruning %d logs\n" "$((logCount - LOG_KEEP + 1))" + echo "$prune" | xargs rm -vf fi } @@ -207,7 +210,7 @@ snapList() { exitClean 128 "failed to list snapshots for dataset: $set" fi ## filter snaps matching our pattern - echo "$snaps" | grep "@autorep-" + echo "$snaps" | grep "@autorep-" || true } ## helper function to check if substring is within string @@ -266,8 +269,14 @@ snapCreate() { fi done ## get source and destination snap count - srcSnapCount=$(echo "$srcSnaps" | wc -l) - dstSnapCount=$(echo "$dstSnaps" | wc -l) + srcSnapCount=0 + dstSnapCount=0 + if [ -n "$srcSnaps" ]; then + srcSnapCount=$(printf "%s\n" "$srcSnaps" | wc -l) + fi + if [ -n "$dstSnaps" ]; then + dstSnapCount=$(printf "%s\n" "$dstSnaps" | wc -l) + fi ## set our base snap for incremental generation if src contains a sufficient ## number of snapshots and the base source snapshot exists in destination dataset base="" From d13234fb005259543dfd1e33c9958a2dc3e41c69 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Thu, 29 Aug 2024 20:58:15 -0500 Subject: [PATCH 06/23] make config use POSIX sh --- README.md | 2 +- config.sample.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d3edf64..341031d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Options: ### Config File and Environment Variable Reference ```bash -#!/usr/bin/env bash +#!/usr/bin/env sh ## zfs-replicate configuration file # shellcheck disable=SC2034 diff --git a/config.sample.sh b/config.sample.sh index afadb6d..11b0ffd 100644 --- a/config.sample.sh +++ b/config.sample.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh ## zfs-replicate configuration file # shellcheck disable=SC2034 From b85ad7198f3c1b20f657334cfddea221b5ea4eda Mon Sep 17 00:00:00 2001 From: tschettervictor <85497460+tschettervictor@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:58:34 -0600 Subject: [PATCH 07/23] clear host var to prevent mix up when doing multiple sets --- zfs-replicate-posix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh index 9c040db..50af598 100755 --- a/zfs-replicate-posix.sh +++ b/zfs-replicate-posix.sh @@ -244,6 +244,8 @@ snapCreate() { continue fi fi + srcHost="" + dstHost="" ## look for host options on source if contains "$src" "@"; then srcHost=$(echo "$src" | cut -f2 -d@) From 220dca2bd4f20c72e23310b351527ee0c1325d08 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 07:38:46 -0500 Subject: [PATCH 08/23] error on unbound variables and set pipefail if available --- zfs-replicate-posix.sh | 125 ++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 53 deletions(-) diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh index 50af598..f3ca531 100755 --- a/zfs-replicate-posix.sh +++ b/zfs-replicate-posix.sh @@ -1,10 +1,49 @@ #!/usr/bin/env sh ## zfs-replicate.sh -set -e +set -eu -############################################ -##### warning gremlins live below here ##### -############################################ +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +## set date substitutions for macros +__DOW=$(date "+%a") +readonly __DOW +__DOM=$(date "+%d") +readonly __DOM +__MOY=$(date "+%m") +readonly __MOY +__CYR=$(date "+%Y") +readonly __CYR +__NOW=$(date "+%s") +readonly __NOW + +## init configuration with values from environment or set defaults +REPLICATE_SETS=${REPLICATE_SETS:-""} ## default empty +ALLOW_ROOT_DATASETS="${ALLOW_ROOT_DATASETS:-0}" +ALLOW_RECONCILIATION="${ALLOW_RECONCILIATION:-0}" +RECURSE_CHILDREN="${RECURSE_CHILDREN:-0}" +SNAP_KEEP="${SNAP_KEEP:-2}" +SYSLOG="${SYSLOG:-1}" +SYSLOG_FACILITY="${SYSLOG_FACILITY:-"user"}" +TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" +LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" +LOG_KEEP="${LOG_KEEP:-5}" +LOG_BASE=${LOG_BASE:-""} ## default empty +LOGGER="${LOGGER:-$(which logger)}" +FIND="${FIND:-$(which find)}" +ZFS="${ZFS:-$(which zfs)}" +SSH="${SSH:-$(which ssh)}" +DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" +DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" +HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" + +## temp path used for lock files +TMPDIR="${TMPDIR:-"/tmp"}" + +## init values used in snapCreate and exitClean +__PAIR_COUNT=0 +__SKIP_COUNT=0 ## output log files in decreasing age order sortLogs() { @@ -52,7 +91,7 @@ clearLock() { ## exit and cleanup exitClean() { exitCode=${1:-0} - extraMsg=$2 + extraMsg=${2:-""} status="success" ## set status to warning if we skipped any datasets if [ "$__SKIP_COUNT" -gt 0 ]; then @@ -213,16 +252,6 @@ snapList() { echo "$snaps" | grep "@autorep-" || true } -## helper function to check if substring is within string -contains() { - string="$1" - substring="$2" - if [ "${string#*"$substring"}" != "$string" ]; then - return 0 - fi - return 1 -} - ## create and manage source snapshots snapCreate() { ## make sure we aren't ever creating simultaneous snapshots @@ -244,15 +273,16 @@ snapCreate() { continue fi fi + ## init source and destination host in each loop iteration srcHost="" dstHost="" ## look for host options on source - if contains "$src" "@"; then + if [ "${src#*"@"}" != "$src" ]; then srcHost=$(echo "$src" | cut -f2 -d@) src=$(echo "$src" | cut -f1 -d@) fi ## look for host options on destination - if contains "$dst" "@"; then + if [ "${dst#*"@"}" != "$dst" ]; then dstHost=$(echo "$dst" | cut -f2 -d@) dst=$(echo "$dst" | cut -f1 -d@) fi @@ -408,13 +438,16 @@ showHelp() { exit 0 } -## read and load config file +## read config file if present, process flags, validate, and lock config variables loadConfig() { ## set SCRIPT used by writeLog and showStatus readonly SCRIPT="${0##*/}" readonly SCRIPT_PATH="${0%/*}" configFile="" status=0 + ## sub macros for logging + TAG="$(subTags "$TAG")" + LOG_FILE="$(subTags "$LOG_FILE")" ## process command-line options while [ $# -gt 0 ]; do case "$1" in @@ -446,44 +479,29 @@ loadConfig() { else writeLog "loading configuration from defaults and environmental settings." fi - ## set date substitutions for macros - __DOW=$(date "+%a") - readonly __DOW - __DOM=$(date "+%d") - readonly __DOM - __MOY=$(date "+%m") - readonly __MOY - __CYR=$(date "+%Y") - readonly __CYR - __NOW=$(date "+%s") - readonly __NOW - ## complete configuration with values from environment or set defaults - readonly TMPDIR="${TMPDIR:-"/tmp"}" - readonly REPLICATE_SETS ## no default value - readonly ALLOW_ROOT_DATASETS="${ALLOW_ROOT_DATASETS:-0}" - readonly ALLOW_RECONCILIATION="${ALLOW_RECONCILIATION:-0}" - readonly RECURSE_CHILDREN="${RECURSE_CHILDREN:-0}" - readonly SNAP_KEEP="${SNAP_KEEP:-2}" - readonly SYSLOG="${SYSLOG:-1}" - readonly SYSLOG_FACILITY="${SYSLOG_FACILITY:-"user"}" - TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" + ## perform final substitution TAG="$(subTags "$TAG")" - readonly TAG - LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" LOG_FILE="$(subTags "$LOG_FILE")" + ## lock configuration + readonly REPLICATE_SETS + readonly ALLOW_ROOT_DATASETS + readonly ALLOW_RECONCILIATION + readonly RECURSE_CHILDREN + readonly SNAP_KEEP + readonly SYSLOG + readonly SYSLOG_FACILITY + readonly TAG readonly LOG_FILE - readonly LOG_KEEP="${LOG_KEEP:-5}" - readonly LOG_BASE ## no default value - readonly LOGGER="${LOGGER:-$(which logger)}" - readonly FIND="${FIND:-$(which find)}" - readonly ZFS="${ZFS:-$(which zfs)}" - readonly SSH="${SSH:-$(which ssh)}" - readonly DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" - readonly DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" - readonly HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" - ## init values used in snapCreate and exitClean - __PAIR_COUNT=0 - __SKIP_COUNT=0 + readonly LOG_KEEP + readonly LOG_BASE + readonly LOGGER + readonly FIND + readonly ZFS + readonly SSH + readonly DEST_PIPE_WITH_HOST + readonly DEST_PIPE_WITHOUT_HOST + readonly HOST_CHECK + readonly TMPDIR ## check configuration if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" @@ -503,6 +521,7 @@ loadConfig() { fi } +## main function, not much here main() { ## do snapshots and send snapCreate From 7e34b0e69749d9f850c7b914403b113219edab4e Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 15:01:52 -0500 Subject: [PATCH 09/23] final tests for POSIX compliant version --- .github/workflows/status-checks.yaml | 2 +- README.md | 24 +- config.sample.sh | 10 +- test-bash.sh | 138 ------- test/find.sh | 14 + test/ssh.sh | 25 ++ test/test.sh | 212 ++++++++++ test/zfs.sh | 34 ++ zfs-replicate-posix.sh | 535 ------------------------- zfs-replicate.sh | 557 ++++++++++++++------------- 10 files changed, 593 insertions(+), 958 deletions(-) delete mode 100755 test-bash.sh create mode 100755 test/find.sh create mode 100755 test/ssh.sh create mode 100755 test/test.sh create mode 100755 test/zfs.sh delete mode 100755 zfs-replicate-posix.sh diff --git a/.github/workflows/status-checks.yaml b/.github/workflows/status-checks.yaml index 2325999..4c7c3b3 100644 --- a/.github/workflows/status-checks.yaml +++ b/.github/workflows/status-checks.yaml @@ -76,6 +76,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: bash ./test-bash.sh + - run: cd test && ./test.sh env: TMPDIR: ${{ runner.temp }} diff --git a/README.md b/README.md index 341031d..1d01454 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # zfs-replicate -A Bash script to automate ZFS Replication. +A POSIX shell script to automate ZFS Replication. ## Features +- The script follows strict POSIX standards and should be usable on any host with a POSIX compliant shell. - Source pools and datasets are always authoritative, the script will always defer to the source. - Supports push and pull replication with local and remote datasets. - Supports multiple pool/dataset pairs to replicate. @@ -12,12 +13,11 @@ A Bash script to automate ZFS Replication. - Includes a well documented `config.sh` file that may be used as configuration or as reference for environment variables passed to the script. - May be run on any schedule using cron or similar mechanism. -- May be sourced and/or leveraged by/in other Bash scripts. -- Test coverage of core functions via mocks in the test.sh script. +- Fully source compliant and may be used by other scripts. +- Test coverage of core functions via mocks in the test/test.sh script. - Includes a `--status` option for XigmaNAS that can be used to email the last log output at your preferred schedule. Simply add it as a custom script in the email settings under "System > Advanced > Email Reports" - ## Experimental POSIX Support There is a new script called `zfs-replicate-posix.sh` that is POSIX compliant (sh|dash). This script was converted from @@ -62,17 +62,17 @@ is not met. ```text Usage: ./zfs-replicate.sh [options] [config] -Bash script to automate ZFS Replication +POSIX shell script to automate ZFS Replication Options: - -c, --config bash configuration file + -c, --config configuration file -s, --status print most recent log messages to stdout -h, --help show this message ``` ### Config File and Environment Variable Reference -```bash +```sh #!/usr/bin/env sh ## zfs-replicate configuration file # shellcheck disable=SC2034 @@ -218,11 +218,6 @@ Options: ## #FIND=$(which find) -## Path to the system "zfs" binary. The default uses the first "zfs" -## executable found in $PATH. -## -#ZFS=$(which zfs) - ## Path to the system "ssh" binary. You may also include custom arguments ## to SSH here or in the "DEST_PIPE_WITH_HOST" option above. ## Example: SSH="ssh -l root" to login as root to target host. @@ -230,6 +225,11 @@ Options: ## #SSH=$(which ssh) +## Path to the system "zfs" binary. The default uses the first "zfs" +## executable found in $PATH. +## +#ZFS=$(which zfs) + ## Set the pipe to the destination pool. But DO NOT INCLUDE the pipe (|) ## character in this setting. Filesystem names from the source will be ## sent to the destination. For increased transfer speed to remote hosts you diff --git a/config.sample.sh b/config.sample.sh index 11b0ffd..da579d7 100644 --- a/config.sample.sh +++ b/config.sample.sh @@ -143,11 +143,6 @@ ## #FIND=$(which find) -## Path to the system "zfs" binary. The default uses the first "zfs" -## executable found in $PATH. -## -#ZFS=$(which zfs) - ## Path to the system "ssh" binary. You may also include custom arguments ## to SSH here or in the "DEST_PIPE_WITH_HOST" option above. ## Example: SSH="ssh -l root" to login as root to target host. @@ -155,6 +150,11 @@ ## #SSH=$(which ssh) +## Path to the system "zfs" binary. The default uses the first "zfs" +## executable found in $PATH. +## +#ZFS=$(which zfs) + ## Set the pipe to the destination pool. But DO NOT INCLUDE the pipe (|) ## character in this setting. Filesystem names from the source will be ## sent to the destination. For increased transfer speed to remote hosts you diff --git a/test-bash.sh b/test-bash.sh deleted file mode 100755 index a479daa..0000000 --- a/test-bash.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2317 -## test.sh contains zfs-replicate test cases -set -e -o pipefail - -_fakeLogger() { - return 0 -} - -_fakeFind() { - return 0 -} - -_fakeSSH() { - return 0 -} - -_fakeZFS() { - local cmd=$1 - local args=("$@") - local target="${args[-1]}" - - case "$cmd" in - list) - printf "%s@autorep-test1\n" "${target}" - printf "%s@autorep-test2\n" "${target}" - printf "%s@autorep-test3\n" "${target}" - ;; - esac - return 0 -} - -_fakeCheck() { - return 0 -} - -_fail() { - local line=$1 match=$2 - printf "test failed: '%s' != '*%s*'\n" "$line" "$match" - exit 1 -} - -_testSimpleSetNoConfig() { - ## define test conditions - export FIND=_fakeFind - export ZFS=_fakeZFS - export DEST_PIPE_WITHOUT_HOST="echo receive -vFd" - export SYSLOG=0 - export REPLICATE_SETS="srcPool/srcFS:dstPool/dstFS" - - ## set output - local configOut snapOut exitOut line idx match - - ## source script and run test - # shellcheck source=/dev/null - . zfs-replicate.sh || true - - ## test loadConfig - printf "_testSimpleSetNoConfig/loadConfig\n" - mapfile -t configOut < <(loadConfig) - line="${configOut[0]}" - printf "%d %s\n" 0 "$line" - match="loading configuration from defaults" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - - ## init test environment - loadConfig > /dev/null 2>&1 - - ## test snapCreate - printf "_testSimpleSetNoConfig/snapCreate\n" - mapfile -t snapOut < <(snapCreate) - for idx in "${!snapOut[@]}"; do - line="${snapOut[idx]}" - printf "%d %s\n" "$idx" "$line" - case $idx in - 0) - match="creating lockfile ${TMPDIR}/.replicate.snapshot.lock" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 1) - match="cmd=_fakeZFS list -H -o name srcPool/srcFS" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 5) - match="cmd=_fakeZFS list -H -o name dstPool/dstFS" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 10) - match="cmd=_fakeZFS destroy srcPool/srcFS@autorep-test1" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 12) - match="cmd=_fakeZFS destroy srcPool/srcFS@autorep-test2" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 13) - match="cmd=_fakeZFS snapshot srcPool/srcFS@autorep-" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 15) - match="cmd=_fakeZFS send -Rs -I srcPool/srcFS@autorep-test3 srcPool/srcFS@autorep-${TAG} | " - match+="echo receive -vFd dstPool/dstFS" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 16) - match="receive -vFd dstPool/dstFS" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 17) - match="deleting lockfile ${TMPDIR}/.replicate.send.lock" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - 18) - match="deleting lockfile ${TMPDIR}/.replicate.snapshot.lock" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - esac - done - - ## test exitClean - printf "_testSimpleSetNoConfig/exitClean\n" - mapfile -t exitOut < <(exitClean 0 "test message") - for idx in "${!exitOut[@]}"; do - line="${exitOut[idx]}" - printf "%d %s\n" "$idx" "$line" - case $idx in - 10) - match="success total sets 0 skipped 0: test message" - [[ ! "$line" == *"$match"* ]] && _fail "$line" "$match" - ;; - esac - done - - ## yay, tests completed! - return 0 -} - -_testSimpleSetNoConfig diff --git a/test/find.sh b/test/find.sh new file mode 100755 index 0000000..b4768f1 --- /dev/null +++ b/test/find.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +## test/zfs.sh +set -eu + +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +_fakeFIND() { + printf "find %s\n" "$@" + return 0 +} + +_fakeZFS "$@" diff --git a/test/ssh.sh b/test/ssh.sh new file mode 100755 index 0000000..4d22893 --- /dev/null +++ b/test/ssh.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh +## test/zfs.sh +set -eu + +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +_fakeSSH() { + host=$1 + shift + cmd=$1 + shift + case "$cmd" in + *zfs*) + ./zfs.sh "$@" + ;; + *) + printf "ssh %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + ;; + esac + return 0 +} + +_fakeSSH "$@" diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 0000000..aad8ffe --- /dev/null +++ b/test/test.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env dash +## test.sh contains zfs-replicate test cases +set -eu ## fail on errors and undefined variabels + +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +## set self identification values +SCRIPT="${0##*/}" +SCRIPT_PATH="${0%/*}" + +## check line against match and exit on failure +_fail() { + local line=$1 match=$2 + case "$line" in + *"$match"*) ;; + *) printf "FAILED '%s' != '*%s*'\n" "$line" "$match" && exit 1 ;; + esac + return 0 +} + +_testZFSReplicate() { + ## wrapper for easy matching + export ECHO="echo" + ## define test conditions + export FIND="${SCRIPT_PATH}/find.sh" + export ZFS="${SCRIPT_PATH}/zfs.sh" + export SSH="${SCRIPT_PATH}/ssh.sh" + export HOST_CHECK="${ECHO} %HOST%" + export SYSLOG=0 + REPLICATE_SETS="srcPool0/srcFS0:dstPool0/dstFS0" + REPLICATE_SETS="${REPLICATE_SETS} srcPool1/srcFS1/subFS1:dstPool1/dstFS1@dstHost1" + REPLICATE_SETS="${REPLICATE_SETS} srcPool2/srcFS2:dstPool2/dstFS2@dstHost2" + REPLICATE_SETS="${REPLICATE_SETS} srcPool3/srcFS3@srcHost3:dstPool3/dstFS3" + REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" + + ## source script functions + . ../zfs-replicate.sh + + ## test loadConfig + ( + printf "_testSimpleSetNoConfig/loadConfig\n" + loadConfig | awk '{ print NR-1, $0 }' | while read idx line; do + printf "%d %s\n" "$idx" "$line" + case $idx in + 0) + match="loading configuration from defaults" + _fail "$line" "$match" + ;; + esac + done + ) + + ## test snapCreate + ( + loadConfig 2>&1 > /dev/null + printf "_testSimpleSetNoConfig/snapCreate\n" + snapCreate | awk '{ print NR-1, $0 }' | while read idx line; do + match="" + printf "%d %s\n" "$idx" "$line" + case $idx in + 0) + match="creating lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; + 1) + match="cmd=${ZFS} list -H -o name srcPool0/srcFS0" + ;; + 5) + match="cmd=${ZFS} list -H -o name dstPool0/dstFS0" + ;; + 10) + match="cmd=${ZFS} destroy srcPool0/srcFS0@autorep-test1" + ;; + 11) + match="cmd=${ZFS} snapshot srcPool0/srcFS0@autorep-" + ;; + 12) + match="creating lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 13) + match="cmd=${ZFS} send -Rs -I srcPool0/srcFS0@autorep-test3 srcPool0/srcFS0@autorep-${TAG} |" + match="$match ${DEST_PIPE_WITHOUT_HOST} dstPool0/dstFS0" + ;; + 14) + match="receive -vFd dstPool0/dstFS0" + ;; + 15) + match="deleting lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 16) + match="cmd=${ECHO} dstHost1" + ;; + 17) + match="cmd=${ZFS} list -H -o name srcPool1/srcFS1/subFS1" + ;; + 21) + match="cmd=${SSH} dstHost1 ${ZFS} list -H -o name dstPool1/dstFS1" + ;; + 26) + match="cmd=${ZFS} destroy srcPool1/srcFS1/subFS1@autorep-test1" + ;; + 27) + match="cmd=${ZFS} snapshot srcPool1/srcFS1/subFS1@autorep-${TAG}" + ;; + 28) + match="creating lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 29) + match="cmd=${ZFS} send -Rs -I srcPool1/srcFS1/subFS1@autorep-test3 srcPool1/srcFS1/subFS1@autorep-${TAG} |" + match="$match ${SSH} dstHost1 ${ZFS} receive -vFd dstPool1/dstFS1" + ;; + 31) + match="deleting lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 33) + match="cmd=${ZFS} list -H -o name srcPool2/srcFS2" + ;; + 37) + match="cmd=${SSH} dstHost2 ${ZFS} list -H -o name dstPool2/dstFS2" + ;; + 42) + match="cmd=${ZFS} destroy srcPool2/srcFS2@autorep-test1" + ;; + 43) + match="cmd=${ZFS} snapshot srcPool2/srcFS2@autorep-${TAG}" + ;; + 44) + match="creating lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 45) + match="cmd=${ZFS} send -Rs -I srcPool2/srcFS2@autorep-test3 srcPool2/srcFS2@autorep-${TAG} |" + match="$match ${SSH} dstHost2 ${ZFS} receive -vFd dstPool2/dstFS2" + ;; + 47) + match="deleting lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 48) + match="cmd=${ECHO} srcHost3" + ;; + 49) + match=" cmd=${SSH} srcHost3 ${ZFS} list -H -o name srcPool3/srcFS3" + ;; + 53) + match="cmd=${ZFS} list -H -o name dstPool3/dstFS3" + ;; + 58) + match="cmd=${SSH} srcHost3 ${ZFS} destroy srcPool3/srcFS3@autorep-test1" + ;; + 59) + match="cmd=${SSH} srcHost3 ${ZFS} snapshot srcPool3/srcFS3@autorep-${TAG}" + ;; + 60) + match="creating lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 61) + match="cmd=${SSH} srcHost3 ${ZFS} send -Rs -I srcPool3/srcFS3@autorep-test3 srcPool3/srcFS3@autorep-${TAG} |" + match="$match ${ZFS} receive -vFd dstPool3/dstFS3" + ;; + 63) + match="deleting lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 66) + match="cmd=${SSH} srcHost4 ${ZFS} list -H -o name srcPool4/srcFS4" + ;; + 70) + match="cmd=${SSH} dstHost4 ${ZFS} list -H -o name dstPool4/dstFS4" + ;; + 75) + match="cmd=${SSH} srcHost4 ${ZFS} destroy srcPool4/srcFS4@autorep-test1" + ;; + 76) + match="cmd=${SSH} srcHost4 ${ZFS} snapshot srcPool4/srcFS4@autorep-${TAG}" + ;; + 77) + match="creating lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 78) + match="cmd=${SSH} srcHost4 ${ZFS} send -Rs -I srcPool4/srcFS4@autorep-test3 srcPool4/srcFS4@autorep-${TAG} |" + match="$match ${SSH} dstHost4 ${ZFS} receive -vFd dstPool4/dstFS4" + ;; + 80) + match="deleting lockfile ${TMPDIR}/.replicate.send.lock" + ;; + 81) + match="deleting lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; + esac + _fail "$line" "$match" + done + ) + + ## test exitClean + ( + printf "_testSimpleSetNoConfig/exitClean\n" + exitOut=$(exitClean 0 "test message") + echo "$exitOut" | awk '{ print NR-1, $0 }' | while read idx line; do + printf "%d %s\n" "$idx" "$line" + case $idx in + 0) + match="success total sets 0 skipped 0: test message" + _fail "$line" "$match" + ;; + esac + done + ) + + ## yay, tests completed! + return 0 +} + +_testZFSReplicate diff --git a/test/zfs.sh b/test/zfs.sh new file mode 100755 index 0000000..32b3716 --- /dev/null +++ b/test/zfs.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env sh +## test/zfs.sh +set -eu + +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +_fakeZFS() { + cmd=$1 + + for arg in $@; do + target=$arg + done + + case "$cmd" in + list) + ## this should probably check for dataset or snapshot list, but it works for testing + printf "%s@autorep-test1\n" "${target}" + printf "%s@autorep-test2\n" "${target}" + printf "%s@autorep-test3\n" "${target}" + ;; + receive) + printf "%s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + ;; + destroy | snapshot) ;; + *) + printf "zfs %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + ;; + esac + return 0 +} + +_fakeZFS "$@" diff --git a/zfs-replicate-posix.sh b/zfs-replicate-posix.sh deleted file mode 100755 index f3ca531..0000000 --- a/zfs-replicate-posix.sh +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env sh -## zfs-replicate.sh -set -eu - -# check pipefail in a subshell and set if supported -# shellcheck disable=SC3040 -(set -o pipefail 2> /dev/null) && set -o pipefail - -## set date substitutions for macros -__DOW=$(date "+%a") -readonly __DOW -__DOM=$(date "+%d") -readonly __DOM -__MOY=$(date "+%m") -readonly __MOY -__CYR=$(date "+%Y") -readonly __CYR -__NOW=$(date "+%s") -readonly __NOW - -## init configuration with values from environment or set defaults -REPLICATE_SETS=${REPLICATE_SETS:-""} ## default empty -ALLOW_ROOT_DATASETS="${ALLOW_ROOT_DATASETS:-0}" -ALLOW_RECONCILIATION="${ALLOW_RECONCILIATION:-0}" -RECURSE_CHILDREN="${RECURSE_CHILDREN:-0}" -SNAP_KEEP="${SNAP_KEEP:-2}" -SYSLOG="${SYSLOG:-1}" -SYSLOG_FACILITY="${SYSLOG_FACILITY:-"user"}" -TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" -LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" -LOG_KEEP="${LOG_KEEP:-5}" -LOG_BASE=${LOG_BASE:-""} ## default empty -LOGGER="${LOGGER:-$(which logger)}" -FIND="${FIND:-$(which find)}" -ZFS="${ZFS:-$(which zfs)}" -SSH="${SSH:-$(which ssh)}" -DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" -DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" -HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" - -## temp path used for lock files -TMPDIR="${TMPDIR:-"/tmp"}" - -## init values used in snapCreate and exitClean -__PAIR_COUNT=0 -__SKIP_COUNT=0 - -## output log files in decreasing age order -sortLogs() { - ## check if file logging is enabled - if [ -z "$LOG_BASE" ] || [ ! -d "$LOG_BASE" ]; then - return - fi - ## find existing logs - logs=$(find "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') - ## get file change time via stat (platform specific) - if [ "$(uname -s)" = "Linux" ] || [ "$(uname -s)" = "SunOS" ]; then - fstat='stat -c %Z' - else - fstat='stat -f %c' - fi - ## output logs in descending age order - for log in $logs; do - printf "%s\t%s\n" "$($fstat "$log")" "$log" - done | sort -rn | cut -f2 -} - -## check log count and delete old logs -pruneLogs() { - logs=$(sortLogs) - logCount=0 - if [ -n "$logs" ]; then - logCount=$(printf "%s" "$logs" | wc -l) - fi - if [ "$logCount" -gt "$LOG_KEEP" ]; then - prune="$(echo "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" - printf "pruning %d logs\n" "$((logCount - LOG_KEEP + 1))" - echo "$prune" | xargs rm -vf - fi -} - -## delete lock files -clearLock() { - lockFile=$1 - if [ -f "$lockFile" ]; then - printf "deleting lockfile %s\n" "$lockFile" - rm "$lockFile" - fi -} - -## exit and cleanup -exitClean() { - exitCode=${1:-0} - extraMsg=${2:-""} - status="success" - ## set status to warning if we skipped any datasets - if [ "$__SKIP_COUNT" -gt 0 ]; then - status="WARNING" - fi - logMsg=$(printf "%s total sets %d skipped %d" "$status" "$__PAIR_COUNT" "$__SKIP_COUNT") - ## build and print error message - if [ "$exitCode" -ne 0 ]; then - status="ERROR" - logMsg=$(printf "%s: operation exited unexpectedly: code=%d" "$status" "$exitCode") - if [ -n "$extraMsg" ]; then - logMsg=$(printf "%s msg=%s" "$logMsg" "$extraMsg") - fi - fi - ## append extra message if available - if [ "$exitCode" -eq 0 ] && [ -n "$extraMsg" ]; then - logMsg=$(printf "%s: %s" "$logMsg" "$extraMsg") - fi - ## cleanup old logs and clear locks - pruneLogs - clearLock "${TMPDIR}/.replicate.snapshot.lock" - clearLock "${TMPDIR}/.replicate.send.lock" - ## print log message and exit - printf "%s\n" "$logMsg" - exit "$exitCode" -} - -## lockfile creation and maintenance -checkLock() { - lockFile=$1 - ## check our lockfile status - if [ -f "$lockFile" ]; then - ## see if this pid is still running - if ps -p "$(cat "$lockFile")" > /dev/null 2>&1; then - ## looks like it's still running - printf "ERROR: script is already running as: %d\n" "$(cat "$lockFile")" - else - ## stale lock file? - printf "ERROR: stale lockfile %s\n" "$lockFile" - fi - ## cleanup and exit - exitClean 128 "confirm script is not running and delete lockfile $lockFile" - fi - ## well no lockfile..let's make a new one - printf "creating lockfile %s\n" "$lockFile" - printf "%d\n" "$$" > "$lockFile" -} - -## check remote host status -checkHost() { - ## do we have a host check defined - if [ -z "$HOST_CHECK" ]; then - return - fi - host=$1 - cmd=$(echo "$HOST_CHECK" | sed "s/%HOST%/$host/g") - printf "checking host cmd=%s\n" "$cmd" - ## run the check - if ! $cmd > /dev/null 2>&1; then - exitClean 128 "host check failed" - fi -} - -## ensure dataset exists -checkDataset() { - set=$1 - host=$2 - cmd="" - ## build command - if [ -n "$host" ]; then - cmd="$SSH $host " - fi - cmd="$cmd$ZFS list -H -o name $set" - printf "checking dataset cmd=%s\n" "$cmd" - ## execute command - if ! $cmd; then - exitClean 128 "failed to list dataset: $set" - fi -} - -## small wrapper around zfs destroy -snapDestroy() { - snap=$1 - host=$2 - cmd="" - ## build command - if [ -n "$host" ]; then - cmd="$SSH $host " - fi - cmd="$cmd$ZFS destroy" - if [ "$RECURSE_CHILDREN" -eq 1 ]; then - cmd="$cmd -r" - fi - cmd="$cmd $snap" - printf "destroying snapshot cmd=%s\n" "$cmd" - ## ignore error from destroy and count on logging to alert the end-user - ## destroying recursive snapshots can lead to "snapshot not found" errors - $cmd || true -} - -## main replication function -snapSend() { - base=$1 - snap=$2 - src=$3 - srcHost=$4 - dst=$5 - dstHost=$6 - ## check our send lockfile - checkLock "${TMPDIR}/.replicate.send.lock" - ## begin building send command - cmd="" - if [ -n "$srcHost" ]; then - cmd="$SSH $srcHost " - fi - cmd="$cmd$ZFS send -Rs" - ## if first snap name is not empty generate an incremental - if [ -n "$base" ]; then - cmd="$cmd -I $base" - fi - cmd="$cmd ${src}@${snap}" - ## set destination pipe based on destination host - pipe="$DEST_PIPE_WITHOUT_HOST" - if [ -n "$dstHost" ]; then - pipe=$(echo "$DEST_PIPE_WITH_HOST" | sed "s/%HOST%/$dstHost/g") - fi - pipe="$pipe $dst" - printf "sending snapshot cmd=%s | %s\n" "$cmd" "$pipe" - ## execute send and check return - if ! $cmd | $pipe; then - snapDestroy "${src}@${name}" "$srcHost" - exitClean 128 "failed to send snapshot: ${src}@${name}" - fi - ## clear lockfile - clearLock "${TMPDIR}/.replicate.send.lock" -} - -## list replication snapshots -snapList() { - set=$1 - host=$2 - depth=${3:-0} - cmd="" - ## build send command - if [ -n "$host" ]; then - cmd="$SSH $host " - fi - cmd="$cmd$ZFS list -Hr -o name -s creation -t snapshot" - if [ "$depth" -gt 0 ]; then - cmd="$cmd -d $depth" - fi - cmd="$cmd $set" - ## get snapshots from host - if ! snaps=$($cmd); then - exitClean 128 "failed to list snapshots for dataset: $set" - fi - ## filter snaps matching our pattern - echo "$snaps" | grep "@autorep-" || true -} - -## create and manage source snapshots -snapCreate() { - ## make sure we aren't ever creating simultaneous snapshots - checkLock "${TMPDIR}/.replicate.snapshot.lock" - ## set our snap name - name="autorep-${TAG}" - ## generate snapshot list and cleanup old snapshots - for pair in $REPLICATE_SETS; do - __PAIR_COUNT=$((__PAIR_COUNT + 1)) - ## split dataset into source and destination parts and trim any trailing space - src=$(echo "$pair" | cut -f1 -d: | sed 's/[[:space:]]*$//') - dst=$(echo "$pair" | cut -f2 -d: | sed 's/[[:space:]]*$//') - ## check for root dataset destination - if [ "$ALLOW_ROOT_DATASETS" -ne 1 ]; then - if [ "$dst" = "$(basename "$dst")" ] || [ "$dst" = "$(basename "$dst")/" ]; then - temps="replicating root datasets can lead to data loss - set ALLOW_ROOT_DATASETS=1 to override" - printf "WARNING: %s\n" "$temps" - __SKIP_COUNT=$((__SKIP_COUNT + 1)) - continue - fi - fi - ## init source and destination host in each loop iteration - srcHost="" - dstHost="" - ## look for host options on source - if [ "${src#*"@"}" != "$src" ]; then - srcHost=$(echo "$src" | cut -f2 -d@) - src=$(echo "$src" | cut -f1 -d@) - fi - ## look for host options on destination - if [ "${dst#*"@"}" != "$dst" ]; then - dstHost=$(echo "$dst" | cut -f2 -d@) - dst=$(echo "$dst" | cut -f1 -d@) - fi - ## check source and destination datasets - checkDataset "$src" "$srcHost" - checkDataset "$dst" "$dstHost" - ## get source and destination snapshots - srcSnaps=$(snapList "$src" "$srcHost" 1) - dstSnaps=$(snapList "$dst" "$dstHost" 0) - for snap in $srcSnaps; do - ## while we are here...check for our current snap name - if [ "$snap" = "${src}@${name}" ]; then - ## looks like it's here...we better kill it - printf "destroying duplicate snapshot: %s@%s\n" "$src" "$name" - snapDestroy "${src}@${name}" "$srcHost" - fi - done - ## get source and destination snap count - srcSnapCount=0 - dstSnapCount=0 - if [ -n "$srcSnaps" ]; then - srcSnapCount=$(printf "%s\n" "$srcSnaps" | wc -l) - fi - if [ -n "$dstSnaps" ]; then - dstSnapCount=$(printf "%s\n" "$dstSnaps" | wc -l) - fi - ## set our base snap for incremental generation if src contains a sufficient - ## number of snapshots and the base source snapshot exists in destination dataset - base="" - if [ "$srcSnapCount" -ge 1 ]; then - ## get most recent source snapshot - ss=$(printf "%s\n" "$srcSnaps" | tail -n 1) - ## get source snapshot name - sn=$(echo "$ss" | cut -f2 -d@) - ## loop over destinations snaps and look for a match - for ds in $dstSnaps; do - dn=$(echo "$ds" | cut -f2 -d@) - if [ "$dn" = "$sn" ]; then - base="$ss" - fi - done - ## no matching base, are we allowed to fallback? - if [ -z "$base" ] && [ "$dstSnapCount" -ge 1 ] && [ "$ALLOW_RECONCILIATION" -ne 1 ]; then - temps=$(printf "source snapshot '%s' not in destination dataset: %s" "$ss" "$dst") - temps=$(printf "%s - set 'ALLOW_RECONCILIATION=1' to fallback to a full send" "$temps") - printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" - __SKIP_COUNT=$((__SKIP_COUNT + 1)) - continue - fi - fi - ## without a base snapshot, the destination must be clean - if [ -z "$base" ] && [ "$dstSnapCount" -gt 0 ]; then - ## allowed to prune remote dataset? - if [ "$ALLOW_RECONCILIATION" -ne 1 ]; then - temps="destination contains snapshots not in source - set 'ALLOW_RECONCILIATION=1' to prune snapshots" - printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" - __SKIP_COUNT=$((__SKIP_COUNT + 1)) - continue - fi - ## prune destination snapshots - printf "pruning destination snapshots: %s\n" "$dstSnaps" - for snap in $dstSnaps; do - snapDestroy "$snap" "$dstHost" - done - fi - ## cleanup old snapshots - if [ "$srcSnapCount" -ge "$SNAP_KEEP" ]; then - ## snaps are sorted above by creation in ascending order - echo "$srcSnaps" | sed -n "1,$((srcSnapCount - SNAP_KEEP))p" | while read -r snap; do - printf "found old snapshot %s\n" "$snap" - snapDestroy "$snap" "$srcHost" - done - fi - ## build snapshot create command - cmd="" - if [ -n "$srcHost" ]; then - cmd="$SSH $srcHost " - fi - cmd="$cmd$ZFS snapshot" - ## check if we are supposed to be recursive - if [ "$RECURSE_CHILDREN" -eq 1 ]; then - cmd="$cmd -r" - fi - cmd="$cmd ${src}@${name}" - ## come on already...take that snapshot - printf "creating snapshot cmd=%s\n" "$cmd" - if ! $cmd; then - snapDestroy "${src}@${name}" "$srcHost" - exitClean 128 "failed to create snapshot: ${src}@${name}" - fi - ## send snapshot to destination - snapSend "$base" "$name" "$src" "$srcHost" "$dst" "$dstHost" - done - ## clear snapshot lockfile - clearLock "${TMPDIR}/.replicate.snapshot.lock" -} - -## handle logging to file or syslog -writeLog() { - line=$1 - logf="/dev/null" - ## if a log base and file has been configured set them - if [ -n "$LOG_BASE" ] && [ -n "$LOG_FILE" ]; then - logf="${LOG_BASE}/${LOG_FILE}" - fi - ## always print to stdout and copy to logfile if set - printf "%s %s[%d]: %s\n" "$(date '+%b %d %T')" "$SCRIPT" "$$" "$line" | tee -a "$logf" - ## if syslog has been enabled write to syslog via logger - if [ -n "$SYSLOG" ] && [ "$SYSLOG" -eq 1 ] && [ -n "$LOGGER" ]; then - $LOGGER -p "${SYSLOG_FACILITY}.info" -t "$SCRIPT" "$line" - fi -} - -## read from stdin till script exit -captureOutput() { - while IFS= read -r line; do - writeLog "$line" - done -} - -## perform macro substitution for tags -subTags() { - m=$1 - ## do the substitutions - m=$(echo "$m" | sed "s/%DOW%/${__DOW}/g") - m=$(echo "$m" | sed "s/%DOM%/${__DOM}/g") - m=$(echo "$m" | sed "s/%MOY%/${__MOY}/g") - m=$(echo "$m" | sed "s/%CYR%/${__CYR}/g") - m=$(echo "$m" | sed "s/%NOW%/${__NOW}/g") - m=$(echo "$m" | sed "s/%TAG%/${TAG}/g") - printf "%s\n" "$m" -} - -## show last replication status -showStatus() { - log=$(sortLogs | head -n 1) - if [ -n "$log" ]; then - printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${log}")" - else - printf "Unable to find most recent log file, cannot print status.\n" - fi - exit 0 -} - -## show usage and exit -showHelp() { - printf "Usage: %s [options] [config]\n\n" "${SCRIPT}" - printf "Bash script to automate ZFS Replication\n\n" - printf "Options:\n" - printf " -c, --config bash configuration file\n" - printf " -s, --status print most recent log messages to stdout\n" - printf " -h, --help show this message\n" - exit 0 -} - -## read config file if present, process flags, validate, and lock config variables -loadConfig() { - ## set SCRIPT used by writeLog and showStatus - readonly SCRIPT="${0##*/}" - readonly SCRIPT_PATH="${0%/*}" - configFile="" - status=0 - ## sub macros for logging - TAG="$(subTags "$TAG")" - LOG_FILE="$(subTags "$LOG_FILE")" - ## process command-line options - while [ $# -gt 0 ]; do - case "$1" in - -c | --config) - configFile=$2 - shift - ;; - -s | --status) - showStatus - ;; - -h | --help) - showHelp - ;; - *) # bad long option - writeLog "ERROR: illegal option ${1}" && exit 1 - ;; - esac - shift - done - ## attempt to load configuration - if [ -f "$configFile" ]; then - writeLog "sourcing config file $configFile" - # shellcheck disable=SC1090 - . "$configFile" - elif configFile="${SCRIPT_PATH}/config.sh" && [ -f "$configFile" ]; then - writeLog "sourcing config file $configFile" - # shellcheck disable=SC1090 - . "$configFile" - else - writeLog "loading configuration from defaults and environmental settings." - fi - ## perform final substitution - TAG="$(subTags "$TAG")" - LOG_FILE="$(subTags "$LOG_FILE")" - ## lock configuration - readonly REPLICATE_SETS - readonly ALLOW_ROOT_DATASETS - readonly ALLOW_RECONCILIATION - readonly RECURSE_CHILDREN - readonly SNAP_KEEP - readonly SYSLOG - readonly SYSLOG_FACILITY - readonly TAG - readonly LOG_FILE - readonly LOG_KEEP - readonly LOG_BASE - readonly LOGGER - readonly FIND - readonly ZFS - readonly SSH - readonly DEST_PIPE_WITH_HOST - readonly DEST_PIPE_WITHOUT_HOST - readonly HOST_CHECK - readonly TMPDIR - ## check configuration - if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then - mkdir -p "$LOG_BASE" - fi - if [ -z "$REPLICATE_SETS" ]; then - writeLog "ERROR: missing required setting REPLICATE_SETS" && exit 1 - fi - if [ -z "$ZFS" ]; then - writeLog "ERROR: unable to locate system zfs binary" && exit 1 - fi - if [ "$SNAP_KEEP" -lt 2 ]; then - writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 - fi - ## show status if toggled - if [ "$status" -eq 1 ]; then - showStatus - fi -} - -## main function, not much here -main() { - ## do snapshots and send - snapCreate - ## that's it, sending is called from doSnap - exitClean 0 -} - -## process config and start main if we weren't sourced -if [ "${0##*/}" != "sh" ] && [ "${0##*/}" != "dash" ] && [ "${0##*/}" != "-bash" ]; then - loadConfig "$@" && main 2>&1 | captureOutput -fi diff --git a/zfs-replicate.sh b/zfs-replicate.sh index a998934..1f8bc7f 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -1,54 +1,92 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh ## zfs-replicate.sh -set -e -o pipefail +set -eu -############################################ -##### warning gremlins live below here ##### -############################################ +# check pipefail in a subshell and set if supported +# shellcheck disable=SC3040 +(set -o pipefail 2> /dev/null) && set -o pipefail + +## set self identification values +readonly SCRIPT="${0##*/}" +readonly SCRIPT_PATH="${0%/*}" + +## set date substitutions for macros +__DOW=$(date "+%a") +readonly __DOW +__DOM=$(date "+%d") +readonly __DOM +__MOY=$(date "+%m") +readonly __MOY +__CYR=$(date "+%Y") +readonly __CYR +__NOW=$(date "+%s") +readonly __NOW + +## init configuration with values from environment or set defaults +REPLICATE_SETS=${REPLICATE_SETS:-""} ## default empty +ALLOW_ROOT_DATASETS="${ALLOW_ROOT_DATASETS:-0}" +ALLOW_RECONCILIATION="${ALLOW_RECONCILIATION:-0}" +RECURSE_CHILDREN="${RECURSE_CHILDREN:-0}" +SNAP_KEEP="${SNAP_KEEP:-2}" +SYSLOG="${SYSLOG:-1}" +SYSLOG_FACILITY="${SYSLOG_FACILITY:-"user"}" +TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" +LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" +LOG_KEEP="${LOG_KEEP:-5}" +LOG_BASE=${LOG_BASE:-""} ## default empty +LOGGER="${LOGGER:-$(which logger)}" +FIND="${FIND:-$(which find)}" +SSH="${SSH:-$(which ssh)}" +ZFS="${ZFS:-$(which zfs)}" +DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" +DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" +HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" + +## temp path used for lock files +TMPDIR="${TMPDIR:-"/tmp"}" + +## init values used in snapCreate and exitClean +__PAIR_COUNT=0 +__SKIP_COUNT=0 ## output log files in decreasing age order sortLogs() { ## check if file logging is enabled - if [[ -z "$LOG_BASE" ]] || [[ ! -d "$LOG_BASE" ]]; then + if [ -z "$LOG_BASE" ] || [ ! -d "$LOG_BASE" ]; then return fi ## find existing logs - local logs=() - for log in $("$FIND" "$LOG_BASE" -maxdepth 1 -type f -name autorep-\*); do - ## get file change time via stat (platform specific) - local fstat - case "$(uname -s)" in - Linux | SunOS) - fstat=$(stat -c %Z "$log") - ;; - *) - fstat=$(stat -f %c "$log") - ;; - esac - ## append logs to array with creation time - logs+=("${fstat}\t${log}") - done + logs=$(find "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') + ## get file change time via stat (platform specific) + if [ "$(uname -s)" = "Linux" ] || [ "$(uname -s)" = "SunOS" ]; then + fstat='stat -c %Z' + else + fstat='stat -f %c' + fi ## output logs in descending age order - for log in $(printf "%b\n" "${logs[@]}" | sort -rn | cut -f2); do - printf "%s\n" "$log" - done + for log in $logs; do + printf "%s\t%s\n" "$($fstat "$log")" "$log" + done | sort -rn | cut -f2 } ## check log count and delete old logs pruneLogs() { - local logs - mapfile -t logs < <(sortLogs) - ## check count and delete old logs - if [[ "${#logs[@]}" -gt "$LOG_KEEP" ]]; then - printf "pruning logs %s\n" "${logs[*]:${LOG_KEEP}}" - rm -f "${logs[@]:${LOG_KEEP}}" + logs=$(sortLogs) + logCount=0 + if [ -n "$logs" ]; then + logCount=$(printf "%s" "$logs" | wc -l) + fi + if [ "$logCount" -gt "$LOG_KEEP" ]; then + prune="$(echo "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" + printf "pruning %d logs\n" "$((logCount - LOG_KEEP + 1))" + echo "$prune" | xargs rm -vf fi } ## delete lock files clearLock() { - local lockFile=$1 - if [[ -f "$lockFile" ]]; then + lockFile=$1 + if [ -f "$lockFile" ]; then printf "deleting lockfile %s\n" "$lockFile" rm "$lockFile" fi @@ -56,28 +94,30 @@ clearLock() { ## exit and cleanup exitClean() { - local exitCode=${1:-0} extraMsg=$2 logMsg status="success" + exitCode=${1:-0} + extraMsg=${2:-""} + status="success" ## set status to warning if we skipped any datasets - if [[ $__SKIP_COUNT -gt 0 ]]; then + if [ "$__SKIP_COUNT" -gt 0 ]; then status="WARNING" fi - printf -v logMsg "%s total sets %d skipped %d" "$status" "$__PAIR_COUNT" "$__SKIP_COUNT" + logMsg=$(printf "%s total sets %d skipped %d" "$status" "$__PAIR_COUNT" "$__SKIP_COUNT") ## build and print error message - if [[ $exitCode -ne 0 ]]; then + if [ "$exitCode" -ne 0 ]; then status="ERROR" - printf -v logMsg "%s: operation exited unexpectedly: code=%d" "$status" "$exitCode" - if [[ -n "$extraMsg" ]]; then - printf -v logMsg "%s msg=%s" "$logMsg" "$extraMsg" + logMsg=$(printf "%s: operation exited unexpectedly: code=%d" "$status" "$exitCode") + if [ -n "$extraMsg" ]; then + logMsg=$(printf "%s msg=%s" "$logMsg" "$extraMsg") fi fi ## append extra message if available - if [[ $exitCode -eq 0 ]] && [[ -n "$extraMsg" ]]; then - printf -v logMsg "%s: %s" "$logMsg" "$extraMsg" + if [ "$exitCode" -eq 0 ] && [ -n "$extraMsg" ]; then + logMsg=$(printf "%s: %s" "$logMsg" "$extraMsg") fi ## cleanup old logs and clear locks pruneLogs - clearLock "${TMPDIR}"/.replicate.snapshot.lock - clearLock "${TMPDIR}"/.replicate.send.lock + clearLock "${TMPDIR}/.replicate.snapshot.lock" + clearLock "${TMPDIR}/.replicate.send.lock" ## print log message and exit printf "%s\n" "$logMsg" exit "$exitCode" @@ -85,14 +125,13 @@ exitClean() { ## lockfile creation and maintenance checkLock() { - local lockFile=$1 + lockFile=$1 ## check our lockfile status - if [[ -f "$lockFile" ]]; then + if [ -f "$lockFile" ]; then ## see if this pid is still running - local ps - if ps=$(pgrep -lx -F "$lockFile"); then + if ps -p "$(cat "$lockFile")" > /dev/null 2>&1; then ## looks like it's still running - printf "ERROR: script is already running as: %d\n" "$ps" + printf "ERROR: script is already running as: %d\n" "$(cat "$lockFile")" else ## stale lock file? printf "ERROR: stale lockfile %s\n" "$lockFile" @@ -108,79 +147,85 @@ checkLock() { ## check remote host status checkHost() { ## do we have a host check defined - if [[ -z "$HOST_CHECK" ]]; then + if [ -z "$HOST_CHECK" ]; then return fi - local host=$1 cmd=() - ## substitute host - read -r -a cmd <<< "${HOST_CHECK//%HOST%/$host}" - printf "checking host cmd=%s\n" "${cmd[*]}" + host=$1 + cmd=$(echo "$HOST_CHECK" | sed "s/%HOST%/$host/g") + printf "checking host cmd=%s\n" "$cmd" ## run the check - if ! "${cmd[@]}" > /dev/null 2>&1; then + if ! $cmd > /dev/null 2>&1; then exitClean 128 "host check failed" fi } ## ensure dataset exists checkDataset() { - local set=$1 host=$2 cmd=() + set=$1 + host=$2 + cmd="" ## build command - if [[ -n "$host" ]]; then - read -r -a cmd <<< "$SSH" - cmd+=("$host") + if [ -n "$host" ]; then + cmd="$SSH $host " fi - cmd+=("$ZFS" "list" "-H" "-o" "name" "$set") - printf "checking dataset cmd=%s\n" "${cmd[*]}" + cmd="$cmd$ZFS list -H -o name $set" + printf "checking dataset cmd=%s\n" "$cmd" ## execute command - if ! "${cmd[@]}"; then - exitClean 128 "failed to list dataset: ${set}" + if ! $cmd; then + exitClean 128 "failed to list dataset: $set" fi } ## small wrapper around zfs destroy snapDestroy() { - local snap=$1 host=$2 cmd=() + snap=$1 + host=$2 + cmd="" ## build command - if [[ -n "$host" ]]; then - read -r -a cmd <<< "$SSH" - cmd+=("$host") + if [ -n "$host" ]; then + cmd="$SSH $host " fi - cmd+=("$ZFS" "destroy") - if [[ $RECURSE_CHILDREN -eq 1 ]]; then - cmd+=("-r") + cmd="$cmd$ZFS destroy" + if [ "$RECURSE_CHILDREN" -eq 1 ]; then + cmd="$cmd -r" fi - cmd+=("$snap") - printf "destroying snapshot cmd=%s\n" "${cmd[*]}" + cmd="$cmd $snap" + printf "destroying snapshot cmd=%s\n" "$cmd" ## ignore error from destroy and count on logging to alert the end-user ## destroying recursive snapshots can lead to "snapshot not found" errors - "${cmd[@]}" || true + $cmd || true } ## main replication function snapSend() { - local base=$1 snap=$2 src=$3 srcHost=$4 dst=$5 dstHost=$6 cmd=() pipe=() + base=$1 + snap=$2 + src=$3 + srcHost=$4 + dst=$5 + dstHost=$6 ## check our send lockfile checkLock "${TMPDIR}/.replicate.send.lock" ## begin building send command - if [[ -n "$srcHost" ]]; then - read -r -a cmd <<< "$SSH" - cmd+=("$srcHost") + cmd="" + if [ -n "$srcHost" ]; then + cmd="$SSH $srcHost " fi - cmd+=("$ZFS" "send" "-Rs") + cmd="$cmd$ZFS send -Rs" ## if first snap name is not empty generate an incremental if [ -n "$base" ]; then - cmd+=("-I" "$base") + cmd="$cmd -I $base" fi - cmd+=("${src}@${snap}") + cmd="$cmd ${src}@${snap}" ## set destination pipe based on destination host - read -r -a pipe <<< "$DEST_PIPE_WITHOUT_HOST" - if [[ -n "$dstHost" ]]; then - read -r -a pipe <<< "${DEST_PIPE_WITH_HOST//%HOST%/$dstHost}" + pipe="$DEST_PIPE_WITHOUT_HOST" + if [ -n "$dstHost" ]; then + pipe=$(echo "$DEST_PIPE_WITH_HOST" | sed "s/%HOST%/$dstHost/g") fi - pipe+=("$dst") - printf "sending snapshot cmd=%s | %s\n" "${cmd[*]}" "${pipe[*]}" + pipe="$pipe $dst" + printf "sending snapshot cmd=%s | %s\n" "$cmd" "$pipe" ## execute send and check return - if ! "${cmd[@]}" | "${pipe[@]}"; then + if ! $cmd | $pipe; then snapDestroy "${src}@${name}" "$srcHost" exitClean 128 "failed to send snapshot: ${src}@${name}" fi @@ -190,27 +235,25 @@ snapSend() { ## list replication snapshots snapList() { - local set=$1 host=$2 depth=${3:-0} cmd=() snaps snap + set=$1 + host=$2 + depth=$3 + cmd="" ## build send command - if [[ -n "$host" ]]; then - read -r -a cmd <<< "$SSH" - cmd+=("$host") + if [ -n "$host" ]; then + cmd="$SSH $host " fi - cmd+=("$ZFS" "list" "-Hr" "-o" "name" "-s" "creation" "-t" "snapshot") - if [[ $depth -gt 0 ]]; then - cmd+=("-d" "$depth") + cmd="$cmd$ZFS list -Hr -o name -s creation -t snapshot" + if [ "$depth" -gt 0 ]; then + cmd="$cmd -d $depth" fi - cmd+=("$set") + cmd="$cmd $set" ## get snapshots from host - if ! snaps="$("${cmd[@]}")"; then - exitClean 128 "failed to list snapshots for dataset: ${set}" + if ! snaps=$($cmd); then + exitClean 128 "failed to list snapshots for dataset: $set" fi ## filter snaps matching our pattern - for snap in $snaps; do - if [[ "$snap" == *"@autorep-"* ]]; then - printf "%s\n" "$snap" - fi - done + echo "$snaps" | grep "@autorep-" || true } ## create and manage source snapshots @@ -218,154 +261,150 @@ snapCreate() { ## make sure we aren't ever creating simultaneous snapshots checkLock "${TMPDIR}/.replicate.snapshot.lock" ## set our snap name - local name="autorep-${TAG}" temps="" tempa=() src dst pair + name="autorep-${TAG}" ## generate snapshot list and cleanup old snapshots - __PAIR_COUNT=0 __SKIP_COUNT=0 ## these are used in exitClean for pair in $REPLICATE_SETS; do - ((__PAIR_COUNT++)) || true + __PAIR_COUNT=$((__PAIR_COUNT + 1)) ## split dataset into source and destination parts and trim any trailing space - read -r -a tempa <<< "${pair//:/ }" - src="${tempa[0]}" - src="${src%"${src##*[![:space:]]}"}" - dst="${tempa[1]}" - dst="${dst%"${dst##*[![:space:]]}"}" + src=$(echo "$pair" | cut -f1 -d: | sed 's/[[:space:]]*$//') + dst=$(echo "$pair" | cut -f2 -d: | sed 's/[[:space:]]*$//') ## check for root dataset destination - if [[ "$ALLOW_ROOT_DATASETS" -ne 1 ]]; then - if [[ "$dst" == "$(basename "$dst")" ]] || [[ "$dst" == "$(basename "$dst")/" ]]; then - temps="replicating root datasets can lead to data loss - set 'ALLOW_ROOT_DATASETS=1' to disable warning" - printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" - ((__SKIP_COUNT++)) || true + if [ "$ALLOW_ROOT_DATASETS" -ne 1 ]; then + if [ "$dst" = "$(basename "$dst")" ] || [ "$dst" = "$(basename "$dst")/" ]; then + temps="replicating root datasets can lead to data loss - set ALLOW_ROOT_DATASETS=1 to override" + printf "WARNING: %s\n" "$temps" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) continue fi fi - ## look for host options on source and destination - local srcHost dstHost - if [[ "$src" == *@* ]]; then - ## split and trim trailing spaces - read -r -a tempa <<< "${src//@/ }" - src="${tempa[0]}" - src="${src%"${src##*[![:space:]]}"}" - srcHost="${tempa[1]}" - srcHost="${srcHost%"${srcHost##*[![:space:]]}"}" - checkHost "$srcHost" ## we only check the host once per set + ## init source and destination host in each loop iteration + srcHost="" + dstHost="" + ## look for host options on source and check host if found + if [ "${src#*"@"}" != "$src" ]; then + srcHost=$(echo "$src" | cut -f2 -d@) + checkHost "$srcHost" + src=$(echo "$src" | cut -f1 -d@) fi - if [[ "$dst" == *@* ]]; then - ## split and trim trailing spaces - read -r -a tempa <<< "${dst//@/ }" - dst="${tempa[0]}" - dst="${dst%"${dst##*[![:space:]]}"}" - dstHost="${tempa[1]}" - dstHost="${dstHost%"${dstHost##*[![:space:]]}"}" - checkHost "$dstHost" ## we only check the host once per set + ## look for host options on destination and check host if found + if [ "${dst#*"@"}" != "$dst" ]; then + dstHost=$(echo "$dst" | cut -f2 -d@) + checkHost "$dstHost" + dst=$(echo "$dst" | cut -f1 -d@) fi - ## ensure datasets exist + ## check source and destination datasets checkDataset "$src" "$srcHost" checkDataset "$dst" "$dstHost" ## get source and destination snapshots - local srcSnaps dstSnaps - mapfile -t srcSnaps < <(snapList "$src" "$srcHost" 1) - mapfile -t dstSnaps < <(snapList "$dst" "$dstHost" 0) - for snap in "${srcSnaps[@]}"; do + srcSnaps=$(snapList "$src" "$srcHost" 1) + dstSnaps=$(snapList "$dst" "$dstHost" 0) + for snap in $srcSnaps; do ## while we are here...check for our current snap name - if [[ "$snap" == "${src}@${name}" ]]; then + if [ "$snap" = "${src}@${name}" ]; then ## looks like it's here...we better kill it printf "destroying duplicate snapshot: %s@%s\n" "$src" "$name" snapDestroy "${src}@${name}" "$srcHost" fi done + ## get source and destination snap count + srcSnapCount=0 + dstSnapCount=0 + if [ -n "$srcSnaps" ]; then + srcSnapCount=$(printf "%s\n" "$srcSnaps" | wc -l) + fi + if [ -n "$dstSnaps" ]; then + dstSnapCount=$(printf "%s\n" "$dstSnaps" | wc -l) + fi ## set our base snap for incremental generation if src contains a sufficient ## number of snapshots and the base source snapshot exists in destination dataset - local base - if [[ ${#srcSnaps[@]} -ge 1 ]]; then - ## set source snap base candidate - ss="${srcSnaps[-1]}" - ## split snap into fs and snap name - read -r -a tempa <<< "${ss//@/ }" - sn="${tempa[1]}" - sn="${sn%"${sn##*[![:space:]]}"}" - ## loop over destination snaps and check for a match - for snap in "${dstSnaps[@]}"; do - read -r -a tempa <<< "${snap//@/ }" - dn="${tempa[1]}" - dn="${dn%"${dn##*[![:space:]]}"}" - if [[ "$dn" == "$sn" ]]; then + base="" + if [ "$srcSnapCount" -ge 1 ] && [ "$dstSnapCount" -ge 1 ]; then + ## get most recent source snapshot + ss=$(printf "%s\n" "$srcSnaps" | tail -n 1) + ## get source snapshot name + sn=$(echo "$ss" | cut -f2 -d@) + ## loop over destinations snaps and look for a match + for ds in $dstSnaps; do + dn=$(echo "$ds" | cut -f2 -d@) + if [ "$dn" = "$sn" ]; then base="$ss" + break fi done ## no matching base, are we allowed to fallback? - if [[ -z "$base" ]] && [[ ${#dstSnaps[@]} -ge 1 ]] && [[ $ALLOW_RECONCILIATION -ne 1 ]]; then - printf -v temps "source snapshot '%s' not in destination dataset: %s" "${srcSnaps[-1]}" "$dst" - printf -v temps "%s - set 'ALLOW_RECONCILIATION=1' to fallback to a full send" "$temps" + if [ -z "$base" ] && [ "$ALLOW_RECONCILIATION" -ne 1 ]; then + temps=$(printf "source snapshot '%s' not in destination dataset: %s" "$ss" "$dst") + temps=$(printf "%s - set 'ALLOW_RECONCILIATION=1' to fallback to a full send" "$temps") printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" - ((__SKIP_COUNT++)) || true + __SKIP_COUNT=$((__SKIP_COUNT + 1)) continue fi fi ## without a base snapshot, the destination must be clean - if [[ -z "$base" ]] && [[ ${#dstSnaps[@]} -gt 0 ]]; then + if [ -z "$base" ] && [ "$dstSnapCount" -gt 0 ]; then ## allowed to prune remote dataset? - if [[ $ALLOW_RECONCILIATION -ne 1 ]]; then + if [ "$ALLOW_RECONCILIATION" -ne 1 ]; then temps="destination contains snapshots not in source - set 'ALLOW_RECONCILIATION=1' to prune snapshots" printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" - ((__SKIP_COUNT++)) || true + __SKIP_COUNT=$((__SKIP_COUNT + 1)) continue fi ## prune destination snapshots - printf "pruning destination snapshots: %s\n" "${dstSnaps[*]}" - for snap in "${dstSnaps[@]}"; do + printf "pruning destination snapshots: %s\n" "$dstSnaps" + for snap in $dstSnaps; do snapDestroy "$snap" "$dstHost" done fi ## cleanup old snapshots - local idx - for idx in "${!srcSnaps[@]}"; do - if [[ ${#srcSnaps[@]} -ge $SNAP_KEEP ]]; then - ## snaps are sorted above by creation in ascending order - printf "found old snapshot %s\n" "${srcSnaps[idx]}" - snapDestroy "${srcSnaps[idx]}" "$srcHost" - unset 'srcSnaps[idx]' - fi - done - ## come on already...take that snapshot - if [[ -n "$srcHost" ]]; then - read -r -a cmd <<< "$SSH" - cmd+=("$srcHost") + if [ "$srcSnapCount" -ge "$SNAP_KEEP" ]; then + ## snaps are sorted above by creation in ascending order + echo "$srcSnaps" | sed -n "1,$((srcSnapCount - SNAP_KEEP))p" | while read -r snap; do + printf "found old snapshot %s\n" "$snap" + snapDestroy "$snap" "$srcHost" + done + fi + ## build snapshot create command + cmd="" + if [ -n "$srcHost" ]; then + cmd="$SSH $srcHost " fi - cmd+=("$ZFS" "snapshot") + cmd="$cmd$ZFS snapshot" ## check if we are supposed to be recursive - if [[ $RECURSE_CHILDREN -eq 1 ]]; then - cmd+=("-r") + if [ "$RECURSE_CHILDREN" -eq 1 ]; then + cmd="$cmd -r" fi - cmd+=("$src@$name") - printf "taking snapshot cmd=%s\n" "${cmd[*]}" - if ! "${cmd[@]}"; then + cmd="$cmd ${src}@${name}" + ## come on already...take that snapshot + printf "creating snapshot cmd=%s\n" "$cmd" + if ! $cmd; then + snapDestroy "${src}@${name}" "$srcHost" exitClean 128 "failed to create snapshot: ${src}@${name}" fi ## send snapshot to destination snapSend "$base" "$name" "$src" "$srcHost" "$dst" "$dstHost" done - ## clear our lockfile + ## clear snapshot lockfile clearLock "${TMPDIR}/.replicate.snapshot.lock" } ## handle logging to file or syslog writeLog() { - local line=$1 logf="/dev/null" + line=$1 + logf="/dev/null" ## if a log base and file has been configured set them - if [[ -n "$LOG_BASE" ]] && [[ -n "$LOG_FILE" ]]; then + if [ -n "$LOG_BASE" ] && [ -n "$LOG_FILE" ]; then logf="${LOG_BASE}/${LOG_FILE}" fi ## always print to stdout and copy to logfile if set printf "%s %s[%d]: %s\n" "$(date '+%b %d %T')" "$SCRIPT" "$$" "$line" | tee -a "$logf" ## if syslog has been enabled write to syslog via logger - if [[ -n "$SYSLOG" ]] && [[ "$SYSLOG" -eq 1 ]] && [[ -n "$LOGGER" ]]; then + if [ -n "$SYSLOG" ] && [ "$SYSLOG" -eq 1 ] && [ -n "$LOGGER" ]; then $LOGGER -p "${SYSLOG_FACILITY}.info" -t "$SCRIPT" "$line" fi } ## read from stdin till script exit captureOutput() { - local line while IFS= read -r line; do writeLog "$line" done @@ -373,23 +412,22 @@ captureOutput() { ## perform macro substitution for tags subTags() { - local m=$1 + m=$1 ## do the substitutions - m=${m//%DOW%/${DATE_MACROS[DOW]}} - m=${m//%DOM%/${DATE_MACROS[DOM]}} - m=${m//%MOY%/${DATE_MACROS[MOY]}} - m=${m//%CYR%/${DATE_MACROS[CYR]}} - m=${m//%NOW%/${DATE_MACROS[NOW]}} - m=${m//%TAG%/$TAG} + m=$(echo "$m" | sed "s/%DOW%/${__DOW}/g") + m=$(echo "$m" | sed "s/%DOM%/${__DOM}/g") + m=$(echo "$m" | sed "s/%MOY%/${__MOY}/g") + m=$(echo "$m" | sed "s/%CYR%/${__CYR}/g") + m=$(echo "$m" | sed "s/%NOW%/${__NOW}/g") + m=$(echo "$m" | sed "s/%TAG%/${TAG}/g") printf "%s\n" "$m" } -## dump latest log to stdout and exit +## show last replication status showStatus() { - local logs - mapfile -t logs < <(sortLogs) - if [[ -n "${logs[0]}" ]]; then - printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${logs[0]}")" + log=$(sortLogs | head -n 1) + if [ -n "$log" ]; then + printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${log}")" else printf "Unable to find most recent log file, cannot print status.\n" fi @@ -398,114 +436,96 @@ showStatus() { ## show usage and exit showHelp() { - printf "Usage: %s [options] [config]\n\n" "${BASH_SOURCE[0]}" - printf "Bash script to automate ZFS Replication\n\n" + printf "Usage: %s [options] [config]\n\n" "${SCRIPT}" + printf "POSIX shell script to automate ZFS Replication\n\n" printf "Options:\n" - printf " -c, --config bash configuration file\n" + printf " -c, --config configuration file\n" printf " -s, --status print most recent log messages to stdout\n" printf " -h, --help show this message\n" exit 0 } -## load configuration defaults, parse flags, config, and environment -## captureOutput is not running yet, so use writeLog directly in loadConfig +## read config file if present, process flags, validate, and lock config variables loadConfig() { - ## set SCRIPT used by writeLog and showStatus - SCRIPT="$(basename "${BASH_SOURCE[0]}")" - readonly SCRIPT - ## local variables only used in loadConfig - local status=0 configFile opt OPTARG OPTIND line - ## read command line flags - while getopts ":shc:-:" opt; do - if [[ "$opt" == "-" ]]; then - opt="${OPTARG%%=*}" # extract long option name - opt="${opt#"${opt%%[![:space:]]*}"}" # remove leading whitespace characters - opt="${opt%"${opt##*[![:space:]]}"}" # remove trailing whitespace characters - OPTARG="${OPTARG#"$opt"}" # extract long option argument (may be empty) - OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` - fi - case "$opt" in - c | config) - configFile="${OPTARG}" + configFile="" + status=0 + ## sub macros for logging + TAG="$(subTags "$TAG")" + LOG_FILE="$(subTags "$LOG_FILE")" + ## process command-line options + while [ $# -gt 0 ]; do + case "$1" in + -c | --config) + configFile=$2 + shift ;; - s | status) - status=1 + -s | --status) + showStatus ;; - h | help) + -h | --help) showHelp ;; - \?) # bad short option - writeLog "ERROR: illegal option -${OPTARG}" && exit 1 - ;; *) # bad long option - writeLog "ERROR: illegal option --${opt}" && exit 1 + writeLog "ERROR: illegal option ${1}" && exit 1 ;; esac + shift done - # remove parsed options and args from $@ list - shift $((OPTIND - 1)) - ## allow config file to be passed as argument without a flag for backwards compat - [[ -z "$configFile" ]] && configFile=$1 ## attempt to load configuration - if [[ -f "$configFile" ]]; then + if [ -f "$configFile" ]; then writeLog "sourcing config file $configFile" # shellcheck disable=SC1090 - source "$configFile" - elif configFile="$(dirname "${BASH_SOURCE[0]}")/config.sh" && [[ -f "$configFile" ]]; then + . "$configFile" + elif configFile="${SCRIPT_PATH}/config.sh" && [ -f "$configFile" ]; then writeLog "sourcing config file $configFile" # shellcheck disable=SC1090 - source "$configFile" + . "$configFile" else writeLog "loading configuration from defaults and environmental settings." fi - declare -A DATE_MACROS=( - ["DOW"]=$(date "+%a") ["DOM"]=$(date "+%d") ["MOY"]=$(date "+%m") - ["CYR"]=$(date "+%Y") ["NOW"]=$(date "+%s") - ) - readonly DATE_MACROS - readonly TMPDIR=${TMPDIR:-"/tmp"} - readonly REPLICATE_SETS ## no default value - readonly ALLOW_ROOT_DATASETS=${ALLOW_ROOT_DATASETS:-0} - readonly ALLOW_RECONCILIATION=${ALLOW_RECONCILIATION:-0} - readonly RECURSE_CHILDREN=${RECURSE_CHILDREN:-0} - readonly SNAP_KEEP=${SNAP_KEEP:-2} - readonly SYSLOG=${SYSLOG:-1} - readonly SYSLOG_FACILITY=${SYSLOG_FACILITY:-"user"} - TAG=${TAG:-"%MOY%%DOM%%CYR%_%NOW%"} - TAG=$(subTags "$TAG") + ## perform final substitution + TAG="$(subTags "$TAG")" + LOG_FILE="$(subTags "$LOG_FILE")" + ## lock configuration + readonly REPLICATE_SETS + readonly ALLOW_ROOT_DATASETS + readonly ALLOW_RECONCILIATION + readonly RECURSE_CHILDREN + readonly SNAP_KEEP + readonly SYSLOG + readonly SYSLOG_FACILITY readonly TAG - LOG_FILE=${LOG_FILE:-"autorep-%TAG%.log"} - LOG_FILE=$(subTags "$LOG_FILE") readonly LOG_FILE - readonly LOG_KEEP=${LOG_KEEP:-5} - readonly LOG_BASE ## no default value - readonly LOGGER=${LOGGER:-$(which logger)} - readonly FIND=${FIND:-$(which find)} - readonly ZFS=${ZFS:-$(which zfs)} - readonly SSH=${SSH:-$(which ssh)} - readonly DEST_PIPE_WITH_HOST=${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"} - readonly DEST_PIPE_WITHOUT_HOST=${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"} - readonly HOST_CHECK=${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"} + readonly LOG_KEEP + readonly LOG_BASE + readonly LOGGER + readonly FIND + readonly SSH + readonly ZFS + readonly DEST_PIPE_WITH_HOST + readonly DEST_PIPE_WITHOUT_HOST + readonly HOST_CHECK + readonly TMPDIR ## check configuration - if [[ -n "$LOG_BASE" ]] && [[ ! -d "$LOG_BASE" ]]; then + if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" fi - if [[ -z "$REPLICATE_SETS" ]]; then + if [ -z "$REPLICATE_SETS" ]; then writeLog "ERROR: missing required setting REPLICATE_SETS" && exit 1 fi - if [[ -z "$ZFS" ]]; then + if [ -z "$ZFS" ]; then writeLog "ERROR: unable to locate system zfs binary" && exit 1 fi - if [[ $SNAP_KEEP -lt 2 ]]; then + if [ "$SNAP_KEEP" -lt 2 ]; then writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 fi ## show status if toggled - if [[ $status -eq 1 ]]; then + if [ "$status" -eq 1 ]; then showStatus fi } -## it all starts here... +## main function, not much here main() { ## do snapshots and send snapCreate @@ -513,5 +533,8 @@ main() { exitClean 0 } -## start main if we weren't sourced -[[ "$0" == "${BASH_SOURCE[0]}" ]] && loadConfig "$@" && main 2>&1 | captureOutput +## process config and start main if we weren't sourced +if [ "$SCRIPT" != "sh" ] && [ "$SCRIPT" != "dash" ] && [ "$SCRIPT" != "-bash" ] && + [ $(expr "$SCRIPT" : 'zfs-replicate') -gt 0 ]; then + loadConfig "$@" && main 2>&1 | captureOutput +fi From 71051ae7ef085903a248d775af2eb11aa3315c86 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 15:22:56 -0500 Subject: [PATCH 10/23] cleanup tests --- test/ssh.sh | 2 +- test/test.sh | 12 ++++++------ test/zfs.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/ssh.sh b/test/ssh.sh index 4d22893..30c33b0 100755 --- a/test/ssh.sh +++ b/test/ssh.sh @@ -16,7 +16,7 @@ _fakeSSH() { ./zfs.sh "$@" ;; *) - printf "ssh %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + printf "ssh $host $cmd %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" ;; esac return 0 diff --git a/test/test.sh b/test/test.sh index aad8ffe..78b5c31 100755 --- a/test/test.sh +++ b/test/test.sh @@ -7,7 +7,6 @@ set -eu ## fail on errors and undefined variabels (set -o pipefail 2> /dev/null) && set -o pipefail ## set self identification values -SCRIPT="${0##*/}" SCRIPT_PATH="${0%/*}" ## check line against match and exit on failure @@ -36,12 +35,13 @@ _testZFSReplicate() { REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" ## source script functions + # shellcheck source=/dev/null . ../zfs-replicate.sh ## test loadConfig ( printf "_testSimpleSetNoConfig/loadConfig\n" - loadConfig | awk '{ print NR-1, $0 }' | while read idx line; do + loadConfig | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in 0) @@ -54,9 +54,9 @@ _testZFSReplicate() { ## test snapCreate ( - loadConfig 2>&1 > /dev/null + loadConfig > /dev/null 2>&1 printf "_testSimpleSetNoConfig/snapCreate\n" - snapCreate | awk '{ print NR-1, $0 }' | while read idx line; do + snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do match="" printf "%d %s\n" "$idx" "$line" case $idx in @@ -192,9 +192,9 @@ _testZFSReplicate() { ## test exitClean ( + loadConfig > /dev/null 2>&1 printf "_testSimpleSetNoConfig/exitClean\n" - exitOut=$(exitClean 0 "test message") - echo "$exitOut" | awk '{ print NR-1, $0 }' | while read idx line; do + exitClean 0 "test message" | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in 0) diff --git a/test/zfs.sh b/test/zfs.sh index 32b3716..adf55c8 100755 --- a/test/zfs.sh +++ b/test/zfs.sh @@ -9,7 +9,7 @@ set -eu _fakeZFS() { cmd=$1 - for arg in $@; do + for arg in "$@"; do target=$arg done From b05205c8ff6f2fec7b2a02a10004d275927cff97 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 15:27:38 -0500 Subject: [PATCH 11/23] fix typo --- test/test.sh | 2 +- zfs-replicate.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index 78b5c31..7eb1013 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env dash ## test.sh contains zfs-replicate test cases -set -eu ## fail on errors and undefined variabels +set -eu ## fail on errors and undefined variables # check pipefail in a subshell and set if supported # shellcheck disable=SC3040 diff --git a/zfs-replicate.sh b/zfs-replicate.sh index 1f8bc7f..fb7d4df 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh ## zfs-replicate.sh -set -eu +set -eu ## fail on errors and undefined variables # check pipefail in a subshell and set if supported # shellcheck disable=SC3040 From 64a43febc962cf3bdf6ec0024aee36873a9784fc Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 15:38:50 -0500 Subject: [PATCH 12/23] just a few more sanity checks and defaults --- zfs-replicate.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/zfs-replicate.sh b/zfs-replicate.sh index fb7d4df..f95c450 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -34,10 +34,10 @@ TAG="${TAG:-"%MOY%%DOM%%CYR%_%NOW%"}" LOG_FILE="${LOG_FILE:-"autorep-%TAG%.log"}" LOG_KEEP="${LOG_KEEP:-5}" LOG_BASE=${LOG_BASE:-""} ## default empty -LOGGER="${LOGGER:-$(which logger)}" -FIND="${FIND:-$(which find)}" -SSH="${SSH:-$(which ssh)}" -ZFS="${ZFS:-$(which zfs)}" +LOGGER="${LOGGER:-$(which logger || true)}" +FIND="${FIND:-$(which find || true)}" +SSH="${SSH:-$(which ssh || true)}" +ZFS="${ZFS:-$(which zfs || true)}" DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" @@ -398,7 +398,7 @@ writeLog() { ## always print to stdout and copy to logfile if set printf "%s %s[%d]: %s\n" "$(date '+%b %d %T')" "$SCRIPT" "$$" "$line" | tee -a "$logf" ## if syslog has been enabled write to syslog via logger - if [ -n "$SYSLOG" ] && [ "$SYSLOG" -eq 1 ] && [ -n "$LOGGER" ]; then + if [ "$SYSLOG" -eq 1 ] && [ -n "$LOGGER" ]; then $LOGGER -p "${SYSLOG_FACILITY}.info" -t "$SCRIPT" "$line" fi } @@ -510,15 +510,24 @@ loadConfig() { if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" fi + if [ "$SYSLOG" -eq 1 ] && [ -z "$LOGGER" ]; then + writeLog "ERROR: unable to locate system logger binary and SYSLOG is enabled" && exit 1 + fi if [ -z "$REPLICATE_SETS" ]; then writeLog "ERROR: missing required setting REPLICATE_SETS" && exit 1 fi - if [ -z "$ZFS" ]; then - writeLog "ERROR: unable to locate system zfs binary" && exit 1 - fi if [ "$SNAP_KEEP" -lt 2 ]; then writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 fi + if [ -z "$FIND" ]; then + writeLog "ERROR: unable to locate system find binary" && exit 1 + fi + if [ -z "$SSH" ]; then + writeLog "ERROR: unable to locate system ssh binary" && exit 1 + fi + if [ -z "$ZFS" ]; then + writeLog "ERROR: unable to locate system zfs binary" && exit 1 + fi ## show status if toggled if [ "$status" -eq 1 ]; then showStatus @@ -535,6 +544,6 @@ main() { ## process config and start main if we weren't sourced if [ "$SCRIPT" != "sh" ] && [ "$SCRIPT" != "dash" ] && [ "$SCRIPT" != "-bash" ] && - [ $(expr "$SCRIPT" : 'zfs-replicate') -gt 0 ]; then + [ "$(expr "$SCRIPT" : 'zfs-replicate')" -gt 0 ]; then loadConfig "$@" && main 2>&1 | captureOutput fi From 96cf8849f8b7759c21a956ff2b85c59bd3e44fc4 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 15:48:00 -0500 Subject: [PATCH 13/23] support bare config for backwards compatibility --- zfs-replicate.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zfs-replicate.sh b/zfs-replicate.sh index f95c450..c3033a0 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -453,10 +453,11 @@ loadConfig() { TAG="$(subTags "$TAG")" LOG_FILE="$(subTags "$LOG_FILE")" ## process command-line options + opt=${1:-""} optArg=${2:-""} while [ $# -gt 0 ]; do - case "$1" in + case "$opt" in -c | --config) - configFile=$2 + configFile="$optArg" shift ;; -s | --status) @@ -465,7 +466,14 @@ loadConfig() { -h | --help) showHelp ;; - *) # bad long option + *) + ## check for config file for backwards compatibility + if [ -z "$configFile" ] && [ -f "$opt" ]; then + configFile="$opt" + shift + continue + fi + ## nothing left, error out writeLog "ERROR: illegal option ${1}" && exit 1 ;; esac From d527aabcbd292f95b837a8d1d5ed49f3738e39f5 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 16:48:52 -0500 Subject: [PATCH 14/23] printf everywhere, fix flags --- test/test.sh | 9 ++-- zfs-replicate.sh | 109 ++++++++++++++++++++++------------------------- 2 files changed, 56 insertions(+), 62 deletions(-) diff --git a/test/test.sh b/test/test.sh index 7eb1013..af2467d 100755 --- a/test/test.sh +++ b/test/test.sh @@ -12,6 +12,10 @@ SCRIPT_PATH="${0%/*}" ## check line against match and exit on failure _fail() { local line=$1 match=$2 + ## hack to match blank lines + if [ "$match" = "null" ] && [ -n "$line" ]; then + printf "FAILED '%s' != ''\n" "$line" && exit 1 + fi case "$line" in *"$match"*) ;; *) printf "FAILED '%s' != '*%s*'\n" "$line" "$match" && exit 1 ;; @@ -44,9 +48,8 @@ _testZFSReplicate() { loadConfig | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in - 0) - match="loading configuration from defaults" - _fail "$line" "$match" + *) + _fail "$line" "null" ;; esac done diff --git a/zfs-replicate.sh b/zfs-replicate.sh index c3033a0..4a0993f 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -77,9 +77,9 @@ pruneLogs() { logCount=$(printf "%s" "$logs" | wc -l) fi if [ "$logCount" -gt "$LOG_KEEP" ]; then - prune="$(echo "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" + prune="$(printf "%s\n" "$logs" | sed -n "$((LOG_KEEP + 1)),\$p")" printf "pruning %d logs\n" "$((logCount - LOG_KEEP + 1))" - echo "$prune" | xargs rm -vf + printf "%s\n" "$prune" | xargs rm -vf fi } @@ -151,7 +151,7 @@ checkHost() { return fi host=$1 - cmd=$(echo "$HOST_CHECK" | sed "s/%HOST%/$host/g") + cmd=$(printf "%s\n" "$HOST_CHECK" | sed "s/%HOST%/$host/g") printf "checking host cmd=%s\n" "$cmd" ## run the check if ! $cmd > /dev/null 2>&1; then @@ -220,7 +220,7 @@ snapSend() { ## set destination pipe based on destination host pipe="$DEST_PIPE_WITHOUT_HOST" if [ -n "$dstHost" ]; then - pipe=$(echo "$DEST_PIPE_WITH_HOST" | sed "s/%HOST%/$dstHost/g") + pipe=$(printf "%s\n" "$DEST_PIPE_WITH_HOST" | sed "s/%HOST%/$dstHost/g") fi pipe="$pipe $dst" printf "sending snapshot cmd=%s | %s\n" "$cmd" "$pipe" @@ -253,7 +253,7 @@ snapList() { exitClean 128 "failed to list snapshots for dataset: $set" fi ## filter snaps matching our pattern - echo "$snaps" | grep "@autorep-" || true + printf "%s\n" "$snaps" | grep "@autorep-" || true } ## create and manage source snapshots @@ -266,8 +266,8 @@ snapCreate() { for pair in $REPLICATE_SETS; do __PAIR_COUNT=$((__PAIR_COUNT + 1)) ## split dataset into source and destination parts and trim any trailing space - src=$(echo "$pair" | cut -f1 -d: | sed 's/[[:space:]]*$//') - dst=$(echo "$pair" | cut -f2 -d: | sed 's/[[:space:]]*$//') + src=$(printf "%s\n" "$pair" | cut -f1 -d: | sed 's/[[:space:]]*$//') + dst=$(printf "%s\n" "$pair" | cut -f2 -d: | sed 's/[[:space:]]*$//') ## check for root dataset destination if [ "$ALLOW_ROOT_DATASETS" -ne 1 ]; then if [ "$dst" = "$(basename "$dst")" ] || [ "$dst" = "$(basename "$dst")/" ]; then @@ -282,15 +282,15 @@ snapCreate() { dstHost="" ## look for host options on source and check host if found if [ "${src#*"@"}" != "$src" ]; then - srcHost=$(echo "$src" | cut -f2 -d@) + srcHost=$(printf "%s\n" "$src" | cut -f2 -d@) checkHost "$srcHost" - src=$(echo "$src" | cut -f1 -d@) + src=$(printf "%s\n" "$src" | cut -f1 -d@) fi ## look for host options on destination and check host if found if [ "${dst#*"@"}" != "$dst" ]; then - dstHost=$(echo "$dst" | cut -f2 -d@) + dstHost=$(printf "%s\n" "$dst" | cut -f2 -d@) checkHost "$dstHost" - dst=$(echo "$dst" | cut -f1 -d@) + dst=$(printf "%s\n" "$dst" | cut -f1 -d@) fi ## check source and destination datasets checkDataset "$src" "$srcHost" @@ -322,10 +322,10 @@ snapCreate() { ## get most recent source snapshot ss=$(printf "%s\n" "$srcSnaps" | tail -n 1) ## get source snapshot name - sn=$(echo "$ss" | cut -f2 -d@) + sn=$(printf "%s\n" "$ss" | cut -f2 -d@) ## loop over destinations snaps and look for a match for ds in $dstSnaps; do - dn=$(echo "$ds" | cut -f2 -d@) + dn=$(printf "%s\n" "$ds" | cut -f2 -d@) if [ "$dn" = "$sn" ]; then base="$ss" break @@ -358,7 +358,7 @@ snapCreate() { ## cleanup old snapshots if [ "$srcSnapCount" -ge "$SNAP_KEEP" ]; then ## snaps are sorted above by creation in ascending order - echo "$srcSnaps" | sed -n "1,$((srcSnapCount - SNAP_KEEP))p" | while read -r snap; do + printf "%s\n" "$srcSnaps" | sed -n "1,$((srcSnapCount - SNAP_KEEP))p" | while read -r snap; do printf "found old snapshot %s\n" "$snap" snapDestroy "$snap" "$srcHost" done @@ -414,12 +414,12 @@ captureOutput() { subTags() { m=$1 ## do the substitutions - m=$(echo "$m" | sed "s/%DOW%/${__DOW}/g") - m=$(echo "$m" | sed "s/%DOM%/${__DOM}/g") - m=$(echo "$m" | sed "s/%MOY%/${__MOY}/g") - m=$(echo "$m" | sed "s/%CYR%/${__CYR}/g") - m=$(echo "$m" | sed "s/%NOW%/${__NOW}/g") - m=$(echo "$m" | sed "s/%TAG%/${TAG}/g") + m=$(printf "%s\n" "$m" | sed "s/%DOW%/${__DOW}/g") + m=$(printf "%s\n" "$m" | sed "s/%DOM%/${__DOM}/g") + m=$(printf "%s\n" "$m" | sed "s/%MOY%/${__MOY}/g") + m=$(printf "%s\n" "$m" | sed "s/%CYR%/${__CYR}/g") + m=$(printf "%s\n" "$m" | sed "s/%NOW%/${__NOW}/g") + m=$(printf "%s\n" "$m" | sed "s/%TAG%/${TAG}/g") printf "%s\n" "$m" } @@ -427,11 +427,10 @@ subTags() { showStatus() { log=$(sortLogs | head -n 1) if [ -n "$log" ]; then - printf "Last output from %s:\n%s\n" "$SCRIPT" "$(cat "${log}")" - else - printf "Unable to find most recent log file, cannot print status.\n" + printf "%s" "$(cat "${log}")" && exit 0 fi - exit 0 + ## not found, log error and exit + writeLog "ERROR: unable to find most recent log file, cannot print status" && exit 1 } ## show usage and exit @@ -453,43 +452,34 @@ loadConfig() { TAG="$(subTags "$TAG")" LOG_FILE="$(subTags "$LOG_FILE")" ## process command-line options - opt=${1:-""} optArg=${2:-""} while [ $# -gt 0 ]; do - case "$opt" in - -c | --config) - configFile="$optArg" - shift - ;; - -s | --status) - showStatus - ;; - -h | --help) - showHelp - ;; - *) - ## check for config file for backwards compatibility - if [ -z "$configFile" ] && [ -f "$opt" ]; then - configFile="$opt" - shift - continue - fi - ## nothing left, error out - writeLog "ERROR: illegal option ${1}" && exit 1 - ;; - esac - shift + if [ "$1" = "-c" ] || [ "$1" = "--config" ]; then + shift + configFile="$1" + shift + continue + fi + if [ "$1" = "-s" ] || [ "$1" = "--status" ]; then + status=1 + shift + continue + fi + ## unknown option - check for config file for backwards compatibility + if [ -z "$configFile" ] && [ -f "$1" ]; then + configFile="$1" + shift + continue + fi + ## nothing left, error out + writeLog "ERROR: illegal option ${1}" && exit 1 done ## attempt to load configuration if [ -f "$configFile" ]; then - writeLog "sourcing config file $configFile" # shellcheck disable=SC1090 . "$configFile" elif configFile="${SCRIPT_PATH}/config.sh" && [ -f "$configFile" ]; then - writeLog "sourcing config file $configFile" # shellcheck disable=SC1090 . "$configFile" - else - writeLog "loading configuration from defaults and environmental settings." fi ## perform final substitution TAG="$(subTags "$TAG")" @@ -518,6 +508,14 @@ loadConfig() { if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" fi + if [ -z "$FIND" ]; then + writeLog "ERROR: unable to locate system find binary" && exit 1 + fi + ## we have all we need for status + if [ "$status" -eq 1 ]; then + showStatus + fi + ## continue validating config if [ "$SYSLOG" -eq 1 ] && [ -z "$LOGGER" ]; then writeLog "ERROR: unable to locate system logger binary and SYSLOG is enabled" && exit 1 fi @@ -527,19 +525,12 @@ loadConfig() { if [ "$SNAP_KEEP" -lt 2 ]; then writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 fi - if [ -z "$FIND" ]; then - writeLog "ERROR: unable to locate system find binary" && exit 1 - fi if [ -z "$SSH" ]; then writeLog "ERROR: unable to locate system ssh binary" && exit 1 fi if [ -z "$ZFS" ]; then writeLog "ERROR: unable to locate system zfs binary" && exit 1 fi - ## show status if toggled - if [ "$status" -eq 1 ]; then - showStatus - fi } ## main function, not much here From 4f1b0e745e7cab57d3b6a166eae42d50ebb4d036 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 17:01:59 -0500 Subject: [PATCH 15/23] posix no longer experimental --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 1d01454..0fff434 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,6 @@ A POSIX shell script to automate ZFS Replication. - Includes a `--status` option for XigmaNAS that can be used to email the last log output at your preferred schedule. Simply add it as a custom script in the email settings under "System > Advanced > Email Reports" -## Experimental POSIX Support - -There is a new script called `zfs-replicate-posix.sh` that is POSIX compliant (sh|dash). This script was converted from -the existing script with ChatGPT and then hand reviewed line-by-line for correctness and completion of parts that were -left out from the ChatGPT conversion. This script is currently considered experimental, but feedback would -be appreciated. - ## FreeBSD Package This script is available in the FreeBSD [package and ports tree](https://www.freshports.org/sysutils/zfs-replicate/). From 43cbd0889cd43959d8853d0d61a6434326e4b849 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 21:03:40 -0500 Subject: [PATCH 16/23] use FIND setting, add config tests --- test/test.sh | 41 +++++++++++++++++++++++++++++++---------- zfs-replicate.sh | 2 +- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/test/test.sh b/test/test.sh index af2467d..bf5f3a7 100755 --- a/test/test.sh +++ b/test/test.sh @@ -11,7 +11,9 @@ SCRIPT_PATH="${0%/*}" ## check line against match and exit on failure _fail() { - local line=$1 match=$2 + line=$1 + match=$2 + ## verbose testing ## hack to match blank lines if [ "$match" = "null" ] && [ -n "$line" ]; then printf "FAILED '%s' != ''\n" "$line" && exit 1 @@ -38,13 +40,12 @@ _testZFSReplicate() { REPLICATE_SETS="${REPLICATE_SETS} srcPool3/srcFS3@srcHost3:dstPool3/dstFS3" REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" - ## source script functions - # shellcheck source=/dev/null - . ../zfs-replicate.sh - ## test loadConfig ( - printf "_testSimpleSetNoConfig/loadConfig\n" + ## source script functions + # shellcheck source=/dev/null + . ../zfs-replicate.sh + printf "_testSetsNoConfig/loadConfig\n" ## we expect no output and clean exit loadConfig | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in @@ -55,10 +56,28 @@ _testZFSReplicate() { done ) + ## test config override + ( + ## source script functions + # shellcheck source=/dev/null + . ../zfs-replicate.sh + printf "_testSetsNoConfig/loadConfigOverrideDefaults\n" + _fail "./ssh.sh %HOST% ./zfs.sh receive -vFd" "$DEST_PIPE_WITH_HOST" + _fail "./zfs.sh receive -vFd" "$DEST_PIPE_WITHOUT_HOST" + config="$(mktemp)" + printf "DEST_PIPE_WITH_HOST=\"pipe with host\"\n" | tee -a "$config" + printf "DEST_PIPE_WITHOUT_HOST=\"pipe without host\"\n" | tee -a "$config" + loadConfig "$config" > /dev/null 2>&1 + _fail "pipe with host" "$DEST_PIPE_WITH_HOST" + _fail "pipe without host" "$DEST_PIPE_WITHOUT_HOST" + ) + ## test snapCreate ( - loadConfig > /dev/null 2>&1 - printf "_testSimpleSetNoConfig/snapCreate\n" + ## source script functions + # shellcheck source=/dev/null + . ../zfs-replicate.sh && loadConfig + printf "_testSetsNoConfig/snapCreate\n" snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do match="" printf "%d %s\n" "$idx" "$line" @@ -195,8 +214,10 @@ _testZFSReplicate() { ## test exitClean ( - loadConfig > /dev/null 2>&1 - printf "_testSimpleSetNoConfig/exitClean\n" + ## source script functions + # shellcheck source=/dev/null + . ../zfs-replicate.sh && loadConfig + printf "_testSetsNoConfig/exitClean\n" exitClean 0 "test message" | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in diff --git a/zfs-replicate.sh b/zfs-replicate.sh index 4a0993f..3b9874f 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -56,7 +56,7 @@ sortLogs() { return fi ## find existing logs - logs=$(find "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') + logs=$($FIND "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') ## get file change time via stat (platform specific) if [ "$(uname -s)" = "Linux" ] || [ "$(uname -s)" = "SunOS" ]; then fstat='stat -c %Z' From 6b56c3035d0d5a89fa21910541c1fdb47618b2f4 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 21:19:10 -0500 Subject: [PATCH 17/23] ensure proper sub order and reflect in test --- test/test.sh | 20 +++++++++++++------- zfs-replicate.sh | 12 ++++++------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/test/test.sh b/test/test.sh index bf5f3a7..046ee20 100755 --- a/test/test.sh +++ b/test/test.sh @@ -58,18 +58,24 @@ _testZFSReplicate() { ## test config override ( + ## likely default values at script load time + ZFS="/sbin/zfs" + SSH="/usr/sbin/ssh" ## source script functions # shellcheck source=/dev/null . ../zfs-replicate.sh printf "_testSetsNoConfig/loadConfigOverrideDefaults\n" - _fail "./ssh.sh %HOST% ./zfs.sh receive -vFd" "$DEST_PIPE_WITH_HOST" - _fail "./zfs.sh receive -vFd" "$DEST_PIPE_WITHOUT_HOST" + _fail "/usr/sbin/ssh %HOST% /sbin/zfs receive -vFd" "$DEST_PIPE_WITH_HOST" + _fail "/sbin/zfs receive -vFd" "$DEST_PIPE_WITHOUT_HOST" + ## generate config config="$(mktemp)" - printf "DEST_PIPE_WITH_HOST=\"pipe with host\"\n" | tee -a "$config" - printf "DEST_PIPE_WITHOUT_HOST=\"pipe without host\"\n" | tee -a "$config" - loadConfig "$config" > /dev/null 2>&1 - _fail "pipe with host" "$DEST_PIPE_WITH_HOST" - _fail "pipe without host" "$DEST_PIPE_WITHOUT_HOST" + printf "ZFS=\"myZFS\"\n" >> "$config" + ## set SSH via environment + SSH="mySSH" + loadConfig "$config" && rm -f "$config" + ## values should match config and environment + _fail "mySSH %HOST% myZFS receive -vFd" "$DEST_PIPE_WITH_HOST" + _fail "myZFS receive -vFd" "$DEST_PIPE_WITHOUT_HOST" ) ## test snapCreate diff --git a/zfs-replicate.sh b/zfs-replicate.sh index 3b9874f..aaaabb2 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -38,13 +38,12 @@ LOGGER="${LOGGER:-$(which logger || true)}" FIND="${FIND:-$(which find || true)}" SSH="${SSH:-$(which ssh || true)}" ZFS="${ZFS:-$(which zfs || true)}" -DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" -DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" HOST_CHECK="${HOST_CHECK:-"ping -c1 -q -W2 %HOST%"}" - +## we default these after config is loaded +DEST_PIPE_WITH_HOST= +DEST_PIPE_WITHOUT_HOST= ## temp path used for lock files TMPDIR="${TMPDIR:-"/tmp"}" - ## init values used in snapCreate and exitClean __PAIR_COUNT=0 __SKIP_COUNT=0 @@ -500,10 +499,11 @@ loadConfig() { readonly FIND readonly SSH readonly ZFS - readonly DEST_PIPE_WITH_HOST - readonly DEST_PIPE_WITHOUT_HOST readonly HOST_CHECK readonly TMPDIR + ## set pipes after configuration to ensure proper $SSH and $ZFS subs + readonly DEST_PIPE_WITH_HOST="${DEST_PIPE_WITH_HOST:-"$SSH %HOST% $ZFS receive -vFd"}" + readonly DEST_PIPE_WITHOUT_HOST="${DEST_PIPE_WITHOUT_HOST:-"$ZFS receive -vFd"}" ## check configuration if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" From 03053989a9df78f0a460e2c4d5039cba435f67ce Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Fri, 30 Aug 2024 21:27:46 -0500 Subject: [PATCH 18/23] make shellcheck happy --- test/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test.sh b/test/test.sh index 046ee20..8c48f14 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,4 +1,7 @@ #!/usr/bin/env dash +# shellcheck disable=SC2030,SC2031 +## ^^ tests are intentionally run in subshells + ## test.sh contains zfs-replicate test cases set -eu ## fail on errors and undefined variables From 4ca06843d2f71915de4a142677db465e394ab2fb Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Sat, 31 Aug 2024 08:27:23 -0500 Subject: [PATCH 19/23] remove silly call to tr --- test/find.sh | 2 +- test/ssh.sh | 2 +- test/zfs.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/find.sh b/test/find.sh index b4768f1..d10f69d 100755 --- a/test/find.sh +++ b/test/find.sh @@ -7,7 +7,7 @@ set -eu (set -o pipefail 2> /dev/null) && set -o pipefail _fakeFIND() { - printf "find %s\n" "$@" + printf "find %s\n" "$*" return 0 } diff --git a/test/ssh.sh b/test/ssh.sh index 30c33b0..fe22902 100755 --- a/test/ssh.sh +++ b/test/ssh.sh @@ -16,7 +16,7 @@ _fakeSSH() { ./zfs.sh "$@" ;; *) - printf "ssh $host $cmd %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + printf "ssh $host $cmd %s\n" "$*" ;; esac return 0 diff --git a/test/zfs.sh b/test/zfs.sh index adf55c8..e2ea25a 100755 --- a/test/zfs.sh +++ b/test/zfs.sh @@ -21,11 +21,11 @@ _fakeZFS() { printf "%s@autorep-test3\n" "${target}" ;; receive) - printf "%s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + printf "%s\n" "$*" ;; destroy | snapshot) ;; *) - printf "zfs %s\n" "$(printf "%s\n" "$@" | tr "\n" " ")" + printf "zfs %s\n" "$*" ;; esac return 0 From 9accb0424a2313448050860d0e6f9b62203cf817 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Sat, 31 Aug 2024 09:22:53 -0500 Subject: [PATCH 20/23] warn on check failure - fixes #43 --- test/test.sh | 116 ++++++++++++++++++++++++++++++++++++----------- zfs-replicate.sh | 40 ++++++++++------ 2 files changed, 115 insertions(+), 41 deletions(-) diff --git a/test/test.sh b/test/test.sh index 8c48f14..806d3cf 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env dash -# shellcheck disable=SC2030,SC2031 +# shellcheck disable=SC2030,SC2031,SC2034 ## ^^ tests are intentionally run in subshells +## variables that appear unused here are used by main script ## test.sh contains zfs-replicate test cases set -eu ## fail on errors and undefined variables @@ -16,7 +17,6 @@ SCRIPT_PATH="${0%/*}" _fail() { line=$1 match=$2 - ## verbose testing ## hack to match blank lines if [ "$match" = "null" ] && [ -n "$line" ]; then printf "FAILED '%s' != ''\n" "$line" && exit 1 @@ -30,44 +30,35 @@ _fail() { _testZFSReplicate() { ## wrapper for easy matching - export ECHO="echo" - ## define test conditions - export FIND="${SCRIPT_PATH}/find.sh" - export ZFS="${SCRIPT_PATH}/zfs.sh" - export SSH="${SCRIPT_PATH}/ssh.sh" - export HOST_CHECK="${ECHO} %HOST%" - export SYSLOG=0 - REPLICATE_SETS="srcPool0/srcFS0:dstPool0/dstFS0" - REPLICATE_SETS="${REPLICATE_SETS} srcPool1/srcFS1/subFS1:dstPool1/dstFS1@dstHost1" - REPLICATE_SETS="${REPLICATE_SETS} srcPool2/srcFS2:dstPool2/dstFS2@dstHost2" - REPLICATE_SETS="${REPLICATE_SETS} srcPool3/srcFS3@srcHost3:dstPool3/dstFS3" - REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" + ECHO="echo" + ## disable syslog for tests + SYSLOG=0 ## test loadConfig ( - ## source script functions # shellcheck source=/dev/null . ../zfs-replicate.sh - printf "_testSetsNoConfig/loadConfig\n" ## we expect no output and clean exit + printf "_testZFSReplicate/loadConfigWithError\n" loadConfig | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in - *) - _fail "$line" "null" + 0) + _fail "$line" "missing required setting REPLICATE_SETS" ;; esac done ) - ## test config override + ## test config override of script defaults ( ## likely default values at script load time + FIND="/usr/bin/find" ZFS="/sbin/zfs" SSH="/usr/sbin/ssh" - ## source script functions + REPLICATE_SETS="fakeSource:fakeDest" # shellcheck source=/dev/null . ../zfs-replicate.sh - printf "_testSetsNoConfig/loadConfigOverrideDefaults\n" + printf "_testZFSReplicate/loadConfigOverrideDefaults\n" _fail "/usr/sbin/ssh %HOST% /sbin/zfs receive -vFd" "$DEST_PIPE_WITH_HOST" _fail "/sbin/zfs receive -vFd" "$DEST_PIPE_WITHOUT_HOST" ## generate config @@ -81,12 +72,21 @@ _testZFSReplicate() { _fail "myZFS receive -vFd" "$DEST_PIPE_WITHOUT_HOST" ) - ## test snapCreate + ## test snapCreate with different set combinations ( - ## source script functions + ## configure test parameters + FIND="${SCRIPT_PATH}/find.sh" + ZFS="${SCRIPT_PATH}/zfs.sh" + SSH="${SCRIPT_PATH}/ssh.sh" + HOST_CHECK="${ECHO} %HOST%" + REPLICATE_SETS="srcPool0/srcFS0:dstPool0/dstFS0" + REPLICATE_SETS="${REPLICATE_SETS} srcPool1/srcFS1/subFS1:dstPool1/dstFS1@dstHost1" + REPLICATE_SETS="${REPLICATE_SETS} srcPool2/srcFS2:dstPool2/dstFS2@dstHost2" + REPLICATE_SETS="${REPLICATE_SETS} srcPool3/srcFS3@srcHost3:dstPool3/dstFS3" + REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig - printf "_testSetsNoConfig/snapCreate\n" + printf "_testZFSReplicate/snapCreateWithoutErrors\n" snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do match="" printf "%d %s\n" "$idx" "$line" @@ -221,17 +221,78 @@ _testZFSReplicate() { done ) - ## test exitClean + ## test snapCreate with host check errors ( + ## configure test parameters + FIND="${SCRIPT_PATH}/find.sh" + ZFS="${SCRIPT_PATH}/zfs.sh" + SSH="${SCRIPT_PATH}/ssh.sh" + HOST_CHECK="false" + REPLICATE_SETS="srcPool0/srcFS0:dstPool0/dstFS0" + REPLICATE_SETS="${REPLICATE_SETS} srcPool1/srcFS1/subFS1:dstPool1/dstFS1@dstHost1" + REPLICATE_SETS="${REPLICATE_SETS} srcPool2/srcFS2:dstPool2/dstFS2@dstHost2" + REPLICATE_SETS="${REPLICATE_SETS} srcPool3/srcFS3@srcHost3:dstPool3/dstFS3" + REPLICATE_SETS="${REPLICATE_SETS} srcPool4/srcFS4@srcHost4:dstPool4/dstFS4@dstHost4" + # shellcheck source=/dev/null + . ../zfs-replicate.sh && loadConfig + printf "_testZFSReplicate/snapCreateWithHostCheckErrors\n" + snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do + match="" + printf "%d %s\n" "$idx" "$line" + case $idx in + 17) + match="source or destination host check failed" + ;; + 19) + match="source or destination host check failed" + ;; + 21) + match="source or destination host check failed" + ;; + 23) + match="source or destination host check failed" + ;; + esac + _fail "$line" "$match" + done + ) + + ## test exitClean code=0 and extra message + ( + FIND="fakeFIND" + ZFS="fakeZFS" + SSH="fakeSSH" + REPLICATE_SETS="fakeSource:fakeDest" ## source script functions # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig - printf "_testSetsNoConfig/exitClean\n" + printf "_testZFSReplicate/exitCleanSuccess\n" exitClean 0 "test message" | awk '{ print NR-1, $0 }' | while read -r idx line; do printf "%d %s\n" "$idx" "$line" case $idx in 0) - match="success total sets 0 skipped 0: test message" + match="success total sets 0 skipped 0: test message" ## counts in test are always zero + _fail "$line" "$match" + ;; + esac + done + ) + + ## test exitClean code=99 with error message + ( + FIND="fakeFIND" + ZFS="fakeZFS" + SSH="fakeSSH" + REPLICATE_SETS="fakeSource:fakeDest" + ## source script functions + # shellcheck source=/dev/null + . ../zfs-replicate.sh && loadConfig + printf "_testZFSReplicate/exitCleanError\n" + exitClean 99 "error message" | awk '{ print NR-1, $0 }' | while read -r idx line; do + printf "%d %s\n" "$idx" "$line" + case $idx in + 0) + match="operation exited unexpectedly: code=99 msg=error message" _fail "$line" "$match" ;; esac @@ -239,6 +300,7 @@ _testZFSReplicate() { ) ## yay, tests completed! + printf "Tests Complete: No Error!\n" return 0 } diff --git a/zfs-replicate.sh b/zfs-replicate.sh index aaaabb2..d4741ba 100755 --- a/zfs-replicate.sh +++ b/zfs-replicate.sh @@ -52,7 +52,7 @@ __SKIP_COUNT=0 sortLogs() { ## check if file logging is enabled if [ -z "$LOG_BASE" ] || [ ! -d "$LOG_BASE" ]; then - return + return 0 fi ## find existing logs logs=$($FIND "$LOG_BASE" -maxdepth 1 -type f -name 'autorep-*') @@ -147,15 +147,19 @@ checkLock() { checkHost() { ## do we have a host check defined if [ -z "$HOST_CHECK" ]; then - return + return 0 fi host=$1 + if [ -z "$host" ]; then + return 0 + fi cmd=$(printf "%s\n" "$HOST_CHECK" | sed "s/%HOST%/$host/g") printf "checking host cmd=%s\n" "$cmd" ## run the check if ! $cmd > /dev/null 2>&1; then - exitClean 128 "host check failed" + return 1 fi + return 0 } ## ensure dataset exists @@ -171,8 +175,9 @@ checkDataset() { printf "checking dataset cmd=%s\n" "$cmd" ## execute command if ! $cmd; then - exitClean 128 "failed to list dataset: $set" + return 1 fi + return 0 } ## small wrapper around zfs destroy @@ -271,7 +276,7 @@ snapCreate() { if [ "$ALLOW_ROOT_DATASETS" -ne 1 ]; then if [ "$dst" = "$(basename "$dst")" ] || [ "$dst" = "$(basename "$dst")/" ]; then temps="replicating root datasets can lead to data loss - set ALLOW_ROOT_DATASETS=1 to override" - printf "WARNING: %s\n" "$temps" + printf "WARNING: skipping replication set '%s' - %s\n" "$pair" "$temps" __SKIP_COUNT=$((__SKIP_COUNT + 1)) continue fi @@ -279,21 +284,28 @@ snapCreate() { ## init source and destination host in each loop iteration srcHost="" dstHost="" - ## look for host options on source and check host if found + ## look for source host option if [ "${src#*"@"}" != "$src" ]; then srcHost=$(printf "%s\n" "$src" | cut -f2 -d@) - checkHost "$srcHost" src=$(printf "%s\n" "$src" | cut -f1 -d@) fi - ## look for host options on destination and check host if found + ## look for destination host option if [ "${dst#*"@"}" != "$dst" ]; then dstHost=$(printf "%s\n" "$dst" | cut -f2 -d@) - checkHost "$dstHost" dst=$(printf "%s\n" "$dst" | cut -f1 -d@) fi + ## check source and destination hosts + if ! checkHost "$srcHost" || ! checkHost "$dstHost"; then + printf "WARNING: skipping replication set '%s' - source or destination host check failed\n" "$pair" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) + continue + fi ## check source and destination datasets - checkDataset "$src" "$srcHost" - checkDataset "$dst" "$dstHost" + if ! checkDataset "$src" "$srcHost" || ! checkDataset "$dst" "$dstHost"; then + printf "WARNING: skipping replication set '%s' - source or destination dataset check failed\n" "$pair" + __SKIP_COUNT=$((__SKIP_COUNT + 1)) + continue + fi ## get source and destination snapshots srcSnaps=$(snapList "$src" "$srcHost" 1) dstSnaps=$(snapList "$dst" "$dstHost" 0) @@ -508,9 +520,6 @@ loadConfig() { if [ -n "$LOG_BASE" ] && [ ! -d "$LOG_BASE" ]; then mkdir -p "$LOG_BASE" fi - if [ -z "$FIND" ]; then - writeLog "ERROR: unable to locate system find binary" && exit 1 - fi ## we have all we need for status if [ "$status" -eq 1 ]; then showStatus @@ -525,6 +534,9 @@ loadConfig() { if [ "$SNAP_KEEP" -lt 2 ]; then writeLog "ERROR: a minimum of 2 snapshots are required for incremental sending" && exit 1 fi + if [ -z "$FIND" ]; then + writeLog "ERROR: unable to locate system find binary" && exit 1 + fi if [ -z "$SSH" ]; then writeLog "ERROR: unable to locate system ssh binary" && exit 1 fi From 0b7fe25667b9db4e48184bd545069c8348ff4fb3 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Sat, 31 Aug 2024 10:09:18 -0500 Subject: [PATCH 21/23] more test improvements --- test/test.sh | 61 ++++++++++++++++++++++++++-------------------------- test/zfs.sh | 4 ++-- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/test/test.sh b/test/test.sh index 806d3cf..c24d351 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env dash +#!/usr/bin/env sh # shellcheck disable=SC2030,SC2031,SC2034 ## ^^ tests are intentionally run in subshells ## variables that appear unused here are used by main script @@ -18,8 +18,11 @@ _fail() { line=$1 match=$2 ## hack to match blank lines - if [ "$match" = "null" ] && [ -n "$line" ]; then - printf "FAILED '%s' != ''\n" "$line" && exit 1 + if [ "$match" = "null" ]; then + if [ -n "$line" ]; then + printf "FAILED '%s' != ''\n" "$line" && exit 1 + fi + return 0 fi case "$line" in *"$match"*) ;; @@ -34,19 +37,29 @@ _testZFSReplicate() { ## disable syslog for tests SYSLOG=0 - ## test loadConfig + ## test loadConfig without error ( + FIND="fakeFIND" + ZFS="fakeZFS" + SSH="fakeSSH" + REPLICATE_SETS="fakeSource:fakeDest" + # shellcheck source=/dev/null + . ../zfs-replicate.sh + printf "_testZFSReplicate/loadConfigWithoutError\n" + line=$(loadConfig) + _fail "$line" "null" ## we expect no output here + ) + + ## test loadConfig with missing values + ( + FIND="fakeFIND" + ZFS="fakeZFS" + SSH="fakeSSH" # shellcheck source=/dev/null . ../zfs-replicate.sh printf "_testZFSReplicate/loadConfigWithError\n" - loadConfig | awk '{ print NR-1, $0 }' | while read -r idx line; do - printf "%d %s\n" "$idx" "$line" - case $idx in - 0) - _fail "$line" "missing required setting REPLICATE_SETS" - ;; - esac - done + ! line=$(loadConfig) && true ## prevent tests from exiting + _fail "$line" "missing required setting REPLICATE_SETS" ) ## test config override of script defaults @@ -267,15 +280,9 @@ _testZFSReplicate() { # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig printf "_testZFSReplicate/exitCleanSuccess\n" - exitClean 0 "test message" | awk '{ print NR-1, $0 }' | while read -r idx line; do - printf "%d %s\n" "$idx" "$line" - case $idx in - 0) - match="success total sets 0 skipped 0: test message" ## counts in test are always zero - _fail "$line" "$match" - ;; - esac - done + line=$(exitClean 0 "test message") + match="success total sets 0 skipped 0: test message" ## counts are modified in snapCreate + _fail "$line" "$match" ) ## test exitClean code=99 with error message @@ -288,15 +295,9 @@ _testZFSReplicate() { # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig printf "_testZFSReplicate/exitCleanError\n" - exitClean 99 "error message" | awk '{ print NR-1, $0 }' | while read -r idx line; do - printf "%d %s\n" "$idx" "$line" - case $idx in - 0) - match="operation exited unexpectedly: code=99 msg=error message" - _fail "$line" "$match" - ;; - esac - done + ! line=$(exitClean 99 "error message") && true ## prevent tests from exiting + match="operation exited unexpectedly: code=99 msg=error message" + _fail "$line" "$match" ) ## yay, tests completed! diff --git a/test/zfs.sh b/test/zfs.sh index e2ea25a..8cc0e3e 100755 --- a/test/zfs.sh +++ b/test/zfs.sh @@ -23,9 +23,9 @@ _fakeZFS() { receive) printf "%s\n" "$*" ;; - destroy | snapshot) ;; + destroy | snapshot | send) ;; *) - printf "zfs %s\n" "$*" + printf "%s\n" "$*" ;; esac return 0 From eb9396ee337fa495af2a30c2e8a8fcc8792f8b23 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Sat, 31 Aug 2024 12:39:19 -0500 Subject: [PATCH 22/23] add test cases for dataset check failures --- test/test.sh | 160 +++++++++++++++++++++++++++++++++++++-------------- test/zfs.sh | 37 ++++++++++-- 2 files changed, 149 insertions(+), 48 deletions(-) diff --git a/test/test.sh b/test/test.sh index c24d351..502fe6b 100755 --- a/test/test.sh +++ b/test/test.sh @@ -110,123 +110,132 @@ _testZFSReplicate() { 1) match="cmd=${ZFS} list -H -o name srcPool0/srcFS0" ;; - 5) + 3) match="cmd=${ZFS} list -H -o name dstPool0/dstFS0" ;; - 10) + 6) match="cmd=${ZFS} destroy srcPool0/srcFS0@autorep-test1" ;; - 11) + 7) match="cmd=${ZFS} snapshot srcPool0/srcFS0@autorep-" ;; - 12) + 8) match="creating lockfile ${TMPDIR}/.replicate.send.lock" ;; - 13) + 9) match="cmd=${ZFS} send -Rs -I srcPool0/srcFS0@autorep-test3 srcPool0/srcFS0@autorep-${TAG} |" match="$match ${DEST_PIPE_WITHOUT_HOST} dstPool0/dstFS0" ;; - 14) + 10) match="receive -vFd dstPool0/dstFS0" ;; - 15) + 11) match="deleting lockfile ${TMPDIR}/.replicate.send.lock" ;; - 16) + 12) match="cmd=${ECHO} dstHost1" ;; - 17) + 13) match="cmd=${ZFS} list -H -o name srcPool1/srcFS1/subFS1" ;; - 21) + 15) match="cmd=${SSH} dstHost1 ${ZFS} list -H -o name dstPool1/dstFS1" ;; - 26) + 18) match="cmd=${ZFS} destroy srcPool1/srcFS1/subFS1@autorep-test1" ;; - 27) + 19) match="cmd=${ZFS} snapshot srcPool1/srcFS1/subFS1@autorep-${TAG}" ;; - 28) + 20) match="creating lockfile ${TMPDIR}/.replicate.send.lock" ;; - 29) + 21) match="cmd=${ZFS} send -Rs -I srcPool1/srcFS1/subFS1@autorep-test3 srcPool1/srcFS1/subFS1@autorep-${TAG} |" match="$match ${SSH} dstHost1 ${ZFS} receive -vFd dstPool1/dstFS1" ;; - 31) + 23) match="deleting lockfile ${TMPDIR}/.replicate.send.lock" ;; - 33) + 24) + match="cmd=${ECHO} dstHost2" + ;; + 25) match="cmd=${ZFS} list -H -o name srcPool2/srcFS2" ;; - 37) + 27) match="cmd=${SSH} dstHost2 ${ZFS} list -H -o name dstPool2/dstFS2" ;; - 42) + 30) match="cmd=${ZFS} destroy srcPool2/srcFS2@autorep-test1" ;; - 43) + 31) match="cmd=${ZFS} snapshot srcPool2/srcFS2@autorep-${TAG}" ;; - 44) + 32) match="creating lockfile ${TMPDIR}/.replicate.send.lock" ;; - 45) + 33) match="cmd=${ZFS} send -Rs -I srcPool2/srcFS2@autorep-test3 srcPool2/srcFS2@autorep-${TAG} |" match="$match ${SSH} dstHost2 ${ZFS} receive -vFd dstPool2/dstFS2" ;; - 47) + 35) match="deleting lockfile ${TMPDIR}/.replicate.send.lock" ;; - 48) + 36) match="cmd=${ECHO} srcHost3" ;; - 49) + 37) match=" cmd=${SSH} srcHost3 ${ZFS} list -H -o name srcPool3/srcFS3" ;; - 53) + 39) match="cmd=${ZFS} list -H -o name dstPool3/dstFS3" ;; - 58) + 42) match="cmd=${SSH} srcHost3 ${ZFS} destroy srcPool3/srcFS3@autorep-test1" ;; - 59) + 43) match="cmd=${SSH} srcHost3 ${ZFS} snapshot srcPool3/srcFS3@autorep-${TAG}" ;; - 60) + 44) match="creating lockfile ${TMPDIR}/.replicate.send.lock" ;; - 61) + 45) match="cmd=${SSH} srcHost3 ${ZFS} send -Rs -I srcPool3/srcFS3@autorep-test3 srcPool3/srcFS3@autorep-${TAG} |" match="$match ${ZFS} receive -vFd dstPool3/dstFS3" ;; - 63) + 47) match="deleting lockfile ${TMPDIR}/.replicate.send.lock" ;; - 66) + 48) + match="cmd=${ECHO} srcHost4" + ;; + 49) + match="cmd=${ECHO} dstHost4" + ;; + 50) match="cmd=${SSH} srcHost4 ${ZFS} list -H -o name srcPool4/srcFS4" ;; - 70) + 52) match="cmd=${SSH} dstHost4 ${ZFS} list -H -o name dstPool4/dstFS4" ;; - 75) + 55) match="cmd=${SSH} srcHost4 ${ZFS} destroy srcPool4/srcFS4@autorep-test1" ;; - 76) + 56) match="cmd=${SSH} srcHost4 ${ZFS} snapshot srcPool4/srcFS4@autorep-${TAG}" ;; - 77) + 57) match="creating lockfile ${TMPDIR}/.replicate.send.lock" ;; - 78) + 58) match="cmd=${SSH} srcHost4 ${ZFS} send -Rs -I srcPool4/srcFS4@autorep-test3 srcPool4/srcFS4@autorep-${TAG} |" match="$match ${SSH} dstHost4 ${ZFS} receive -vFd dstPool4/dstFS4" ;; - 80) + 60) match="deleting lockfile ${TMPDIR}/.replicate.send.lock" ;; - 81) + 61) match="deleting lockfile ${TMPDIR}/.replicate.snapshot.lock" ;; esac @@ -253,18 +262,85 @@ _testZFSReplicate() { match="" printf "%d %s\n" "$idx" "$line" case $idx in - 17) + 0) + match="creating lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; + 13) match="source or destination host check failed" ;; - 19) + 15) match="source or destination host check failed" ;; - 21) + 17) match="source or destination host check failed" ;; - 23) + 19) match="source or destination host check failed" ;; + 20) + match="deleting lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; + esac + _fail "$line" "$match" + done + ) + + ## test snapCreate with dataset check errors + ( + ## configure test parameters + FIND="${SCRIPT_PATH}/find.sh" + ZFS="${SCRIPT_PATH}/zfs.sh" + SSH="${SCRIPT_PATH}/ssh.sh" + HOST_CHECK="${ECHO} %HOST%" + REPLICATE_SETS="failPool0/srcFS0:dstPool0/dstFS0" + REPLICATE_SETS="${REPLICATE_SETS} srcPool1/srcFS1:failPool1/dstFS1@dstHost1" + REPLICATE_SETS="${REPLICATE_SETS} failPool2/srcFS2@srcHost2:dstPool2/dstFS2" + # shellcheck source=/dev/null + . ../zfs-replicate.sh && loadConfig + printf "_testZFSReplicate/snapCreateWithDatasetCheckErrors\n" + snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do + match="" + printf "%d %s\n" "$idx" "$line" + case $idx in + 0) + match="creating lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; + 1) + match="cmd=${ZFS} list -H -o name failPool0/srcFS0" + ;; + 2) + match="dataset does not exist" + ;; + 3) + match="source or destination dataset check failed" + ;; + 5) + match="cmd=${ZFS} list -H -o name srcPool1/srcFS1" + ;; + 6) + match="srcPool1/srcFS1" + ;; + 7) + match="cmd=${SSH} dstHost1 ${ZFS} list -H -o name failPool1/dstFS1" + ;; + 8) + match="dataset does not exist" + ;; + 9) + match="source or destination dataset check failed" + ;; + 11) + match="cmd=${SSH} srcHost2 ${ZFS} list -H -o name failPool2/srcFS2" + ;; + 12) + match="dataset does not exist" + ;; + 13) + match="source or destination dataset check failed" + ;; + 14) + match="deleting lockfile ${TMPDIR}/.replicate.snapshot.lock" + ;; esac _fail "$line" "$match" done diff --git a/test/zfs.sh b/test/zfs.sh index 8cc0e3e..022f7cb 100755 --- a/test/zfs.sh +++ b/test/zfs.sh @@ -8,24 +8,49 @@ set -eu _fakeZFS() { cmd=$1 + shift + showSnaps=0 + ## check arguments for arg in "$@"; do + case "$arg" in + -H) + ## nothing for now + ;; + -o) + ## nothing for now + ;; + -t) + ## assume snapshots for tests + showSnaps=1 + ;; + esac + ## cheap way to get the last arg target=$arg done case "$cmd" in list) - ## this should probably check for dataset or snapshot list, but it works for testing - printf "%s@autorep-test1\n" "${target}" - printf "%s@autorep-test2\n" "${target}" - printf "%s@autorep-test3\n" "${target}" + if [ $showSnaps -eq 1 ]; then + printf "%s@autorep-test1\n" "$target" + printf "%s@autorep-test2\n" "$target" + printf "%s@autorep-test3\n" "$target" + return 0 + fi + ## allow selective failures in tests + if [ "$(expr "$target" : 'fail')" -gt 0 ]; then + printf "cannot open '%s': dataset does not exist\n" "$target" + return 1 + fi + ## just print target + printf "%s\n" "$target" ;; receive) - printf "%s\n" "$*" + printf "%s %s\n" "$cmd" "$*" ;; destroy | snapshot | send) ;; *) - printf "%s\n" "$*" + printf "%s %s\n" "$cmd" "$*" ;; esac return 0 From d6eba594d07678d52a537aacc83213a7a65430b3 Mon Sep 17 00:00:00 2001 From: Aaron Hurt Date: Sat, 31 Aug 2024 13:38:03 -0500 Subject: [PATCH 23/23] simplify test loop, add simulated receive delay --- test/test.sh | 12 +++++++++--- test/zfs.sh | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/test.sh b/test/test.sh index 502fe6b..e6510ae 100755 --- a/test/test.sh +++ b/test/test.sh @@ -100,7 +100,8 @@ _testZFSReplicate() { # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig printf "_testZFSReplicate/snapCreateWithoutErrors\n" - snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do + idx=0 + snapCreate | while IFS= read -r line; do match="" printf "%d %s\n" "$idx" "$line" case $idx in @@ -240,6 +241,7 @@ _testZFSReplicate() { ;; esac _fail "$line" "$match" + idx=$((idx + 1)) done ) @@ -258,7 +260,8 @@ _testZFSReplicate() { # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig printf "_testZFSReplicate/snapCreateWithHostCheckErrors\n" - snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do + idx=0 + snapCreate | while IFS= read -r line; do match="" printf "%d %s\n" "$idx" "$line" case $idx in @@ -282,6 +285,7 @@ _testZFSReplicate() { ;; esac _fail "$line" "$match" + idx=$((idx + 1)) done ) @@ -298,7 +302,8 @@ _testZFSReplicate() { # shellcheck source=/dev/null . ../zfs-replicate.sh && loadConfig printf "_testZFSReplicate/snapCreateWithDatasetCheckErrors\n" - snapCreate | awk '{ print NR-1, $0 }' | while read -r idx line; do + idx=0 + snapCreate | while IFS= read -r line; do match="" printf "%d %s\n" "$idx" "$line" case $idx in @@ -343,6 +348,7 @@ _testZFSReplicate() { ;; esac _fail "$line" "$match" + idx=$((idx + 1)) done ) diff --git a/test/zfs.sh b/test/zfs.sh index 022f7cb..b2c6c4b 100755 --- a/test/zfs.sh +++ b/test/zfs.sh @@ -46,6 +46,7 @@ _fakeZFS() { printf "%s\n" "$target" ;; receive) + sleep 2 ## simulate transfer wait printf "%s %s\n" "$cmd" "$*" ;; destroy | snapshot | send) ;;