Suggested improvements for scripts/lb-wrapper [refactor logging & subprocess substitution, etc]

scripts/lb-wrapper

@@ -2,47 +2,76 @@
 
 LOG_FILE="/var/log/xklb.log"
 XKLB_EXECUTABLE="${XKLB_EXECUTABLE:-lb}"
+VERBOSITY="-vv"
 TMP_DOWNLOADS_DIR="/library/downloads/calibre-web"
 SURVEY_DB_FILE="${TMP_DOWNLOADS_DIR}/survey.db"
 URL="$1"
+
+# Or download largest possible HD-style / UltraHD videos, to try to force
+# out-of-memory "502 Bad Gateway" for testing of issues like #37 and #79
+# FORMAT_OPTIONS="--format-sort size"
+
 FORMAT_OPTIONS="--format best --format-sort 'tbr~1000'"
-XKLB_FULL_CMD="${XKLB_EXECUTABLE} tubeadd ${SURVEY_DB_FILE} ${URL} --verbose && ${XKLB_EXECUTABLE} dl ${SURVEY_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --write-thumbnail ${FORMAT_OPTIONS} --video ${URL} --verbose"
+XKLB_FULL_CMD="${XKLB_EXECUTABLE} tubeadd ${SURVEY_DB_FILE} ${URL} ${VERBOSITY} \
+&& ${XKLB_EXECUTABLE} dl ${SURVEY_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} \
+--video ${URL} ${FORMAT_OPTIONS} --write-thumbnail ${VERBOSITY}"
+
 
 mkdir -p ${TMP_DOWNLOADS_DIR}
 
-# Function to log messages (and errors especially!)
+# Function to log messages e.g. with level "Info" or "Error"
 log() {
-    echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" | tee -a ${LOG_FILE}
+    local level=$1
+    local message=$2
+    if [[ $message == downloading* ]]; then
+        echo "$message"
+    else
+        echo "$(date +'%Y-%m-%d %H:%M:%S') - [$level] $message" | tee -a ${LOG_FILE}
+    fi
 }
 
-if ! command -v "${XKLB_EXECUTABLE}"; then
-    log "xklb could not be found. Please install xklb and try again."
-    exit 1
-fi
-
 if [ $# -eq 0 ]; then
-    log "No arguments provided. Please provide a URL to download."
+    log "Error" "No arguments provided. Please provide a URL to download."
     exit 1
 fi
 
-# URL validation already taken care of by cps/static/js/main.js Lines 194-195
+# URL validation already taken care of by cps/static/js/main.js Lines 167-170
 # which (1) trims outer whitespace, and (2) prepends https:// if URL doesn't
-# already begin with http:// or https://
-# (Test below meant well, enforcing much the same things with logging, but
-# let's avoid duplicate code insofar as possible!)
+# already begin with http:// or https://              (Test below means well,
+# enforcing much the same and with logging, but let's avoid duplicate code!)
 # if [[ ! ${URL} =~ ^http[s]?:// ]]; then
 #     log 'Invalid URL: xklb commands require URLs begin with "http://" or "https://"'
 #     exit 1
 # fi
 
+if ! command -v "${XKLB_EXECUTABLE}"; then
+    log "Error" "xklb could not be found. Please install xklb and try again."
+    exit 1
+fi
+
+log "Info" "xklb version: $(${XKLB_EXECUTABLE} --version)"
+
 if mv ${SURVEY_DB_FILE} ${SURVEY_DB_FILE}.$(date +%F_%T_%Z) 2> /dev/null; then
-    log "Old ${SURVEY_DB_FILE} moved aside."
+    log "Info" "Old ${SURVEY_DB_FILE} moved aside."
 fi
 
-log "Running command: ${XKLB_FULL_CMD}"
-if OUTPUT=$(eval "${XKLB_FULL_CMD}"); then
-    log "Download completed successfully."
+log "Info" "Running xklb commands: ${XKLB_FULL_CMD}"
+
+# >(...) "process substitution" explained at https://unix.stackexchange.com/a/324170
+# 1>&2 redirect back-to-STDERR explained at https://stackoverflow.com/a/15936384
+eval "${XKLB_FULL_CMD}" \
+     > >(while read -r line; do log "Info" "$line"; done) \
+     2> >(while read -r line; do log "Error" "$line" 1>&2; done) &
+pid=$!
+
+# Wait for background process to complete
+wait $pid
+rc=$?
+
+# Check return code (exit status)
+if [ $rc -eq 0 ]; then
+    log "Info" "lb-wrapper's xklb commands (download) completed successfully."
 else
-    log "An error occurred while running the command. Output: ${OUTPUT}"
+    log "Error" "Error $rc occurred while running lb-wrapper's xklb commands."
     exit 1
 fi