Sync Repo

2022-10-26 15:47:07 -07:00
commit e8258e87fb
17 changed files with 1615 additions and 0 deletions
@@ -0,0 +1,50 @@
+# Run a command in the background and record its PID so we can wait for its completion
+function run_in_bg {
+    ($1) &
+    local PID=$!
+    BG_JOBS["$2"]=${PID}
+    debug "Started $2 (PID=${PID})"
+}
+
+# Wait for all tracked background jobs (i.e. jobs recorded in 'BG_JOBS') to finish. If one or more jobs return a
+# non-zero exit code, we log an error for each and return a non-zero value to fail the backup.
+function wait_for_bg_jobs {
+    for bg_job_name in "${!BG_JOBS[@]}"; do
+        local PID=${BG_JOBS[${bg_job_name}]}
+        debug "Waiting for ${bg_job_name} (PID=${PID})"
+        {
+            wait ${PID}
+        } && {
+            debug "${bg_job_name} finished successfully (PID=${PID})"
+            COMPLETED_BG_JOBS+=("${bg_job_name}")
+            update_backup_progress 50
+        } || {
+            FAILED_BG_JOBS["${bg_job_name}"]=$?
+        }
+    done
+
+    if ((${#FAILED_BG_JOBS[@]})); then
+        for bg_job_name in "${!FAILED_BG_JOBS[@]}"; do
+            error "${bg_job_name} failed with status ${FAILED_BG_JOBS[${bg_job_name}]} (PID=${PID})"
+        done
+        return 1
+    fi
+}
+
+# Clean up after a failed backup
+function cleanup_incomplete_backup {
+    debug "Cleaning up after failed backup"
+    for bg_job_name in "${COMPLETED_BG_JOBS[@]}"; do
+        case "$bg_job_name" in
+        "$DB_BACKUP_JOB_NAME")
+            cleanup_incomplete_db_backup
+            ;;
+        "$DISK_BACKUP_JOB_NAME")
+            cleanup_incomplete_disk_backup
+            ;;
+        *)
+            error "No cleanup task defined for backup type: $bg_job_name"
+            ;;
+        esac
+    done
+}