Improved db backup and restore

.gitignore

@@ -62,6 +62,7 @@ vagrant.log
 # And no need to share individual SQL files with each other
 *.sql
 *.sql.gz
+*.sql.gz.sha256
 
 # BUT....
 

CHANGELOG.md

@@ -13,6 +13,7 @@ permalink: /docs/en-US/changelog/
 * Upgraded Ubuntu boxes from 20.04 to 24.04 for docker provider ( #2739 )
 * Github action improvements ( #2739 )
 * Improved noroot to check for vagrant users and fallback if not present ( #2739 )
+* Greatly improved database backup and import script checks and output ( #2759 )
 
 ### Bug Fixes
 

config/homebin/db_backup

@@ -6,68 +6,113 @@
 set -eo pipefail
 set -u
 
+SAVEIFS=$IFS
+
 if [ -z "${VVV_LOG+x}" ]; then
   export VVV_LOG=""
 fi
 
-trap 'rm -rf $TMPFIFODIR' EXIT; TMPFIFODIR=$(mktemp -d); mkfifo "${TMPFIFODIR}/dbnames"
+trap 'rm -rf $TMPFIFODIR' EXIT
+TMPFIFODIR=$(mktemp -d) || { echo "Failed to create temp dir"; exit 1; }
+mkfifo "${TMPFIFODIR}/dbnames" || { echo "Failed to create FIFO"; exit 1; }
 
+if [ ! -f /srv/provision/provision-helpers.sh ]; then
+  echo "Missing provision-helpers.sh"
+  exit 1
+fi
 source /srv/provision/provision-helpers.sh
 
+DEFAULT_SKIP_DBS=(
+  "mysql"
+  "information_schema"
+  "performance_schema"
+  "sys"
+  "test"
+  "wordpress_unit_tests"
+  "phpmyadmin"
+  "innodb"
+  "mysql_innodb_cluster_metadata"
+)
+
+skipped=()
+backedup=()
+failed=()
+
 mkdir -p /srv/database/backups
 vvv_info " * Performing Database Backups"
 databases=()
 gzip=$(get_config_value "general.db_backup.gzip")
 exclude_list=$(get_config_values "general.db_backup.exclude")
 
+ALL_EXCLUDES=("${DEFAULT_SKIP_DBS[@]}" "${exclude_list[@]}")
+
+should_skip_db() {
+  local db="${1}"
+  if [[ " ${ALL_EXCLUDES[*]} " =~ $db ]]; then
+    return 0
+  fi
+  return 1
+}
+
 vvv_info " * Fetching Database names"
 mysql -e 'show databases' | grep -v -F "Database" > "${TMPFIFODIR}/dbnames" &
-while read db_name; do
-    # skip these databases
-    [ "${db_name}" == "mysql" ] && vvv_info "   - skipped <b>${db_name}</b>" && continue;
-    [ "${db_name}" == "information_schema" ] && vvv_info "   - skipped <b>${db_name}</b>" && continue;
-    [ "${db_name}" == "performance_schema" ] && vvv_info "   - skipped <b>${db_name}</b>" && continue;
-    [ "${db_name}" == "sys" ] && vvv_info "   - skipped <b>${db_name}</b>" && continue;
-    [ "${db_name}" == "test" ] && vvv_info "   - skipped ${db_name}" && continue;
-    [ "${db_name}" == "wordpress_unit_tests" ] && vvv_info "   - skipped <b>${db_name}</b>" && continue;
-
-    skip="false"
-    for exclude in "${exclude_list[@]}"; do
-      if [ "${exclude}" == "${db_name}" ]; then
-        skip="true"
+while read -r db_name; do
+  if should_skip_db "${db_name}"; then
+    skipped+=("${db_name}")
+    vvv_info "   - skipped <b>${db_name}</b>" && continue;
+  fi
+  # don't back up databases with no tables
+  mysql_cmd="SHOW TABLES FROM \`${db_name}\`" # Required to support hyphens in database names
+  db_exist=$(mysql --skip-column-names -e "${mysql_cmd}")
+  if [ $? -eq 0 ]; then
+      if [ "" == "${db_exist}" ]; then
+          vvv_info "   - skipped <b>${db_name}</b><info>, no tables in database to back up</info>" && continue;
       fi
-    done
-
-    if [ ${skip} == "true" ]; then
-        vvv_info "   - excluded <b>${db_name}</b>" && continue;
-    fi
-
-    # don't back up databases with no tables
-    mysql_cmd="SHOW TABLES FROM \`${db_name}\`" # Required to support hyphens in database names
-    db_exist=$(mysql --skip-column-names -e "${mysql_cmd}")
-    if [ "$?" == "0" ]; then
-        if [ "" == "${db_exist}" ]; then
-            vvv_info "   - skipped <b>${db_name}</b><info>, no tables in database to back up</info>" && continue;
-        fi
-    fi
-    databases+=( "${db_name}" )
+  fi
+  databases+=( "${db_name}" )
 done < "${TMPFIFODIR}/dbnames"
 
 ext=".sql"
 if [[ "${gzip}" == "True" ]]; then
     ext=".sql.gz"
 fi
-count=1
-for db in "${databases[@]}"
-do
-    OUTPUT=$(printf "<info>   - %2s/%s Backing up </info><b>%-23s</b><info> to </info><b>'database/backups/%s${ext}'</b>" "${count}" "${#databases[@]}" "'${db}'" "${db}")
-    vvv_output "${OUTPUT}"
-    if [[ "${gzip}" == "True" ]]; then
-      mysqldump "${db}" | gzip > "/srv/database/backups/${db}.sql.gz"
-    else
-      mysqldump "${db}" > "/srv/database/backups/${db}.sql";
+count=0
+for db in "${databases[@]}"; do
+  count=$((count+1))
+  OUTPUT=$(printf "<info>   - %2s/%s Backing up </info><b>%-23s</b><info> to </info><b>'database/backups/%s${ext}'</b>" "${count}" "${#databases[@]}" "'${db}'" "${db}")
+  vvv_output "${OUTPUT}"
+  if [[ "${gzip}" == "True" ]]; then
+    if ! mysqldump --databases "${db}" | gzip > "/srv/database/backups/${db}.sql.gz"; then
+      failed+=("${db}")
+      vvv_error "   - Error backing up ${db} to /srv/database/backups/${db}.sql.gz"
+      continue
+    fi
+    backedup+=("${db}")
+    sha256sum "/srv/database/backups/${db}.sql.gz" > "/srv/database/backups/${db}.sql.gz.sha256"
+  else
+    if ! mysqldump --databases "${db}" > "/srv/database/backups/${db}.sql"; then
+      failed+=("${db}")
+      vvv_error "   - Error backing up ${db} to /srv/database/backups/${db}.sql"
+      continue
     fi
-    count=$((count+1))
+    backedup+=("${db}")
+    sha256sum "/srv/database/backups/${db}.sql" > "/srv/database/backups/${db}.sql.sha256"
+  fi
 done
 
-vvv_success " * Finished backing up databases to the <b>database/sql/backups</b><success> folder</success>"
+vvv_success " * Finished backing up ${count} databases to the <b>database/sql/backups</b><success> folder</success>"
+
+IFS=', '
+if [ ${#backedup[@]} -gt 0 ]; then
+  vvv_success " * Backed up databases: <b>${backedup[*]}</>"
+fi
+
+if [ ${#skipped[@]} -gt 0 ]; then
+  vvv_info " * Skipped databases: <b>${skipped[*]}</>"
+fi
+
+if [ ${#failed[@]} -gt 0 ]; then
+  vvv_error " * Failed databases: <b>${failed[*]}</>"
+fi
+
+IFS=$SAVEIFS

database/sql/import-sql.sh

@@ -26,6 +26,16 @@ fi
 
 source /srv/provision/provision-helpers.sh
 
+DEFAULT_SKIP_DBS=(
+  "mysql"
+  "information_schema"
+  "performance_schema"
+  "sys"
+  "test"
+  "phpmyadmin"
+  "wordpress_unit_tests"
+)
+
 VVV_CONFIG=/srv/config/default-config.yml
 if [[ -f /srv/config/config.yml ]]; then
 	VVV_CONFIG=/srv/config/config.yml
@@ -55,6 +65,25 @@ run_restore=$(shyaml get-value general.db_restore 2> /dev/null < ${VVV_CONFIG})
 exclude_list=$(get_config_values "general.db_restore.exclude")
 include_list=$(get_config_values "general.db_restore.include")
 restore_by_default=$(get_config_values "general.db_restore.restore_by_default")
+ALL_EXCLUDES=("${DEFAULT_SKIP_DBS[@]}" "${exclude_list[@]}")
+
+should_skip_db() {
+  local db="${1}"
+  if [[ " ${ALL_EXCLUDES[*]} " =~ $db ]]; then
+    return 0
+  fi
+  if [[ "$restore_by_default" == "true" ]]; then
+    return 0
+  fi
+  for include in "${include_list[@]}"; do
+    [[ "${include}" == "${db}" ]] && return 1
+  done
+  return 1
+}
+
+skipped=()
+imported=()
+failed=()
 
 if [[ $run_restore == "False" ]]
 then
@@ -75,24 +104,34 @@ IFS=$(echo -en "\n\b")
 # import the SQL file into the database of the same name
 sql_count=$(ls -1 ./*.sql* 2>/dev/null | wc -l)
 vvv_info " * Found ${sql_count} database dumps"
-if [ "$sql_count" != 0 ]
-then
+if [ "$sql_count" != 0 ]; then
 	for file in $( ls ./*.sql* )
 	do
 		# get rid of the extension
 		db_name=$(basename "${file}" .sql)
-		if [ "${file: -3}" == ".gz" ]; then
-			db_name=$(basename "${file}" .sql.gz)
-		fi
-
-		# skip these databases
-		[ "${db_name}" == "mysql" ] && continue;
-		[ "${db_name}" == "information_schema" ] && continue;
-		[ "${db_name}" == "performance_schema" ] && continue;
-		[ "${db_name}" == "sys" ] && continue;
-		[ "${db_name}" == "test" ] && continue;
-
-    vvv_info " * Processing ${db_name} dump"
+    if [[ "$file" == *.sql.gz ]]; then
+      db_name=$(basename "$file" .sql.gz)
+      if ! file "$file" | grep -q 'gzip compressed'; then
+        vvv_warning " ! ${file} has a gzip extension but gunzip reports it is not a compressed gzip, proceed with caution."
+      fi
+    elif [[ "$file" == *.sql ]]; then
+      head -n 1 "${file}" | grep -qE '^(--|CREATE|INSERT|DROP|USE|SET)' || {
+        vvv_warning " * Skipping suspicious file: ${file}"
+        continue
+      }
+      db_name=$(basename "$file" .sql)
+    else
+      vvv_info " * Skipping unrecognized file format: ${file}"
+      continue
+    fi
+
+    # Skip if db is in skip list
+    if should_skip_db "${db_name}"; then
+      skipped+=("${db_name}")
+      vvv_info " * skipped <b>${db_name}</b>" && continue;
+    fi
+
+    vvv_info " * Processing <b>${db_name}</b><info>:"
 
 		# if we specified databases, only restore specified ones
 		if [[ "${#@}" -gt 0 ]]; then
@@ -104,68 +143,68 @@ then
 				fi
 			done
 			if [[ "${FOUND}" -eq 0 ]]; then
+        skipped+=("${db_name}")
 				continue;
 			fi
 		fi
 
 		if [ "1" == "${FORCE_RESTORE}" ]; then
-			vvv_info " * Forcing restore of <b>${db_name}</b><info> database, and granting the wp user access"
+			vvv_info "   - Forcing restore of <b>${db_name}</b><info> database, and granting the wp user access"
 			mysql -e "DROP DATABASE IF EXISTS \`${db_name}\`"
 		else
-			vvv_info " * Creating the <b>${db_name}</b><info> database if it doesn't already exist, and granting the wp user access"
+			vvv_info "   - Creating the <b>${db_name}</b><info> database if it doesn't already exist, and granting the wp user access"
 		fi
 
-		skip="false"
-
-		if [ "${restore_by_default}" == "true" ]; then
-				skip="true"
-		fi
-
-		for exclude in ${exclude_list[@]}; do
-			if [ "${exclude}" == "${db_name}" ]; then
-				skip="true"
-			fi
-		done
-
-		for include in ${include_list[@]}; do
-			if [ "${include}" == "${db_name}" ]; then
-				skip="false"
-			fi
-		done
-
 		mysql -e "CREATE DATABASE IF NOT EXISTS \`${db_name}\`"
 		mysql -e "GRANT ALL PRIVILEGES ON \`${db_name}\`.* TO wp@localhost IDENTIFIED BY 'wp';"
 
-		[ "${db_name}" == "wordpress_unit_tests" ] && continue;
-
-		if [ ${skip} == "true" ]; then
-			vvv_info "   - skipped <b>${db_name}</b>" && continue;
-		fi
-
 		mysql_cmd="SHOW TABLES FROM \`${db_name}\`" # Required to support hyphens in database names
-		db_exist=$(mysql --skip-column-names -e "${mysql_cmd}")
-		if [ "$?" != "0" ]
-		then
-			vvv_error " * Error - Create the <b>${db_name}</b><error> database via init-custom.sql before attempting import"
+		db_has_tables=$(mysql --skip-column-names -e "${mysql_cmd}")
+		if [ $? -gt 0 ]; then
+      failed+=("${db_name}")
+      vvv_info "${db_has_tables}"
+			vvv_error "   ! Error - Checking if the <b>${db_name}</b><error> database already contains tables failed!"
 		else
-			if [ "" == "${db_exist}" ]; then
-				vvv_info " * Importing <b>${db_name}</b><info> from <b>${file}</b>"
+			if [ "" == "${db_has_tables}" ]; then
+				vvv_info "   - Importing <b>${db_name}</b><info> from <b>${file}</b>"
 				if [ "${file: -3}" == ".gz" ]; then
-					gunzip < "${file}" | mysql "${db_name}"
+          if ! gunzip < "${file}" | mysql "${db_name}"; then
+            failed+=("${db_name}")
+            vvv_error "   ! Import failed for ${db_name} from ${file}"
+            continue
+          fi
 				else
-					mysql "${db_name}" < "${file}"
+          if ! mysql "${db_name}" < "${file}"; then
+            failed+=("${db_name}")
+            vvv_error "   ! Import failed for ${db_name} from ${file}"
+            continue
+          fi
 				fi
-				vvv_success " * Import of <b>'${db_name}'</b><success> successful</success>"
+				vvv_success "   - Import of <b>'${db_name}'</b><success> successful</success>"
+        imported+=("${db_name}")
 			else
-				vvv_info " * Skipped import of <b>\`${db_name}\`</b><info> - tables already exist"
+				vvv_info "   - Skipped import of <b>\`${db_name}\`</b><info> - tables already exist"
+        skipped+=("${db_name}")
 			fi
 		fi
 	done
-	vvv_success " * Databases imported"
 else
 	vvv_success " * No custom databases to import"
 fi
 
-vvv_success " * Database importing finished"
+vvv_success " * Database import script finished"
+
+IFS=","
+if [ ${#imported[@]} -gt 0 ]; then
+  vvv_success " * Imported databases: <b>${imported[*]}</>"
+fi
+
+if [ ${#skipped[@]} -gt 0 ]; then
+  vvv_info " * Skipped databases: <b>${skipped[*]}</>"
+fi
+
+if [ ${#failed[@]} -gt 0 ]; then
+  vvv_error " * Failed databases: <b>${failed[*]}</>"
+fi
 
 IFS=$SAVEIFS