Add capabilitity to backup only a single database

This PS adds the capability to Mariadb and Postgresql to backup a single database (as an optional parameter to the backup script). Change-Id: I9bc1eb0173063638b2cf58465c063f602ed20bc1
2020-08-13 16:44:39 +00:00 · 2020-08-13 16:44:39 +00:00 · 233197fc0b
commit 233197fc0b
parent ba601e0cba
5 changed files with 126 additions and 66 deletions
--- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@ -9,10 +9,12 @@
 #   source /tmp/backup_main.sh
 #
 # Then the script should call the main backup function (backup_databases):
-#   backup_databases
+#   backup_databases [scope]
+#       [scope] is an optional parameter, defaulted to "all". If only one specific
+#               database is required to be backed up then this parameter will
+#               contain the name of the database; otherwise all are backed up.
 #
-#       No arguments required. However, the framework will require the
-#       following variables to be exported:
+#       The framework will require the following variables to be exported:
 #
 #         export DB_NAMESPACE          Namespace where the database(s) reside
 #         export DB_NAME               Name of the database system
@ -44,12 +46,15 @@
 #                                      is 1800 (30 minutes).
 #
 # The database-specific functions that need to be implemented are:
-#   dump_databases_to_directory <directory> <err_logfile>
+#   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
 #         <directory>   is the full directory path to dump the database files
 #                       into. This is a temporary directory for this backup only.
 #         <err_logfile> is the full directory path where error logs are to be
 #                       written by the application.
+#         [scope]       set to "all" if all databases are to be backed up; or
+#                       set to the name of a specific database to be backed up.
+#                       This optional parameter is defaulted to "all".
 #       returns: 0 if no errors; 1 if any errors occurred
 #
 #       This function is expected to dump the database file(s) to the specified
@ -285,7 +290,11 @@ remove_old_remote_archives() {
 #  1) The directory where the final backup will be kept after it is compressed.
 #  2) A temporary directory to use for placing database files to be compressed.
 #     Note: this temp directory will be deleted after backup is done.
+#  3) Optional "scope" parameter indicating what database to back up. Defaults
+#     to "all".
 backup_databases() {
+  SCOPE=${1:-"all"}
+
  # Create necessary directories if they do not exist.
  mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
  export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
@ -294,7 +303,7 @@ backup_databases() {
  export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"

  # It is expected that this function will dump the database files to the $TMP_DIR
-  dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE
+  dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE

  # If successful, there should be at least one file in the TMP_DIR
  if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
@ -305,7 +314,7 @@ backup_databases() {
  log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..."

  NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
-  TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.all.${NOW}.tar.gz"
+  TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz"

  cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"

--- a/mariadb/templates/bin/_backup_mariadb.sh.tpl
+++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl
@ -1,5 +1,7 @@
 #!/bin/bash

+SCOPE=${1:-"all"}
+
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@ -28,6 +30,7 @@ export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/arch
 dump_databases_to_directory() {
  TMP_DIR=$1
  LOG_FILE=$2
+  SCOPE=${3:-"all"}

  MYSQL="mysql \
     --defaults-file=/etc/mysql/admin_user.cnf \
@ -36,9 +39,18 @@ dump_databases_to_directory() {
  MYSQLDUMP="mysqldump \
     --defaults-file=/etc/mysql/admin_user.cnf"

+  if [[ "${SCOPE}" == "all" ]]; then
    MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
       "show databases;" | \
       egrep -vi 'information_schema|performance_schema|mysql') )
+  else
+    if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" ]]; then
+      MYSQL_DBNAMES=( ${SCOPE} )
+    else
+      log ERROR "It is not allowed to backup database ${SCOPE}."
+      return 1
+    fi
+  fi

  #check if there is a database to backup, otherwise exit
  if [[ -z "${MYSQL_DBNAMES// }" ]]
@ -50,6 +62,7 @@ dump_databases_to_directory() {
  #Create a list of Databases
  printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list

+  if [[ "${SCOPE}" == "all" ]]; then
    #Retrieve and create the GRANT file for all the users
 {{- if .Values.manifests.certificates }}
    SSL_DSN=";mysql_ssl=1"
@ -64,6 +77,7 @@ dump_databases_to_directory() {
      log ERROR "Failed to create GRANT for all the users"
      return 1
    fi
+  fi

  #Retrieve and create the GRANT files per DB
  for db in "${MYSQL_DBNAMES[@]}"
@ -82,22 +96,20 @@ dump_databases_to_directory() {
  done

  #Dumping the database
-  DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all
-  TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE}

  $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}"  \
            > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE
  if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]]
  then
-    log INFO "Databases dumped successfully."
+    log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
    return 0
  else
-    log ERROR "Backup failed and need attention."
+    log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})"
    return 1
  fi
 }

 # Call main program to start the database backup
-backup_databases
+backup_databases ${SCOPE}
--- a/mariadb/templates/bin/_restore_mariadb.sh.tpl
+++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl
@ -83,9 +83,9 @@ get_tables() {
  TMP_DIR=$2
  TABLE_FILE=$3

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    current_db_desc ${DATABASE} ${SQL_FILE} \
        | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \
        > $TABLE_FILE
  else
@ -103,9 +103,9 @@ get_rows() {
  TMP_DIR=$3
  ROW_FILE=$4

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    current_db_desc ${DATABASE} ${SQL_FILE} \
        | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE
    return 0
  else
@ -123,10 +123,10 @@ get_schema() {
  TMP_DIR=$3
  SCHEMA_FILE=$4

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
    DB_FILE=$(mktemp -p /tmp)
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE}
+    current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE}
    sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE}
    if [[ ! (-s ${SCHEMA_FILE}) ]]; then
      sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \
@ -193,8 +193,8 @@ restore_single_db() {
    return 1
  fi

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]
  then
    # Restoring a single database requires us to create a temporary user
    # which has capability to only restore that ONE database. One gotcha
@ -208,7 +208,7 @@ restore_single_db() {
      echo "Restore $SINGLE_DB_NAME failed create restore user."
      return 1
    fi
-    $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG
+    $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG
    if [[ "$?" -eq 0 ]]
    then
      echo "Database $SINGLE_DB_NAME Restore successful."
@ -254,10 +254,20 @@ restore_single_db() {
 restore_all_dbs() {
  TMP_DIR=$1

-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
-  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]
  then
-    $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG
+    # Check the scope of the archive.
+    SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
+    if [[ "${SCOPE}" != "all" ]]; then
+      # This is just a single database backup. The user should
+      # instead use the single database restore option.
+      echo "Cannot use the restore all option for an archive containing only a single database."
+      echo "Please use the single database restore option."
+      return 1
+    fi
+
+    $MYSQL < $SQL_FILE 2>$RESTORE_LOG
    if [[ "$?" -eq 0 ]]
    then
      echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
--- a/postgresql/templates/bin/_backup_postgresql.sh.tpl
+++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl
@ -1,5 +1,7 @@
 #!/bin/bash

+SCOPE=${1:-"all"}
+
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@ -40,27 +42,43 @@ export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/a
 dump_databases_to_directory() {
  TMP_DIR=$1
  LOG_FILE=$2
+  SCOPE=${3:-"all"}

-  PG_DUMPALL_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
+  PG_DUMP_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
+  PG_DUMP="pg_dump \
+             $PG_DUMP_OPTIONS --create \
+             -U $POSTGRESQL_ADMIN_USER \
+             -h $POSTGRESQL_SERVICE_HOST"
  PG_DUMPALL="pg_dumpall \
-                $PG_DUMPALL_OPTIONS \
+                $PG_DUMP_OPTIONS \
                -U $POSTGRESQL_ADMIN_USER \
                -h $POSTGRESQL_SERVICE_HOST"

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all
+  SQL_FILE=postgres.${POSTGRESQL_POD_NAMESPACE}.${SCOPE}

  cd $TMP_DIR

+  if [[ "${SCOPE}" == "all" ]]; then
    # Dump all databases
-  $PG_DUMPALL --file=${TMP_DIR}/${SQL_FILE}.sql 2>>$LOG_FILE
+    ${PG_DUMPALL} --file=${TMP_DIR}/${SQL_FILE}.sql 2>>${LOG_FILE}
+  else
+    if [[ "${SCOPE}" != "postgres" && "${SCOPE}" != "template0" && "${SCOPE}" != "template1" ]]; then
+      # Dump the specified database
+      ${PG_DUMP} --file=${TMP_DIR}/${SQL_FILE}.sql ${SCOPE} 2>>${LOG_FILE}
+    else
+      log ERROR "It is not allowed to backup up the ${SCOPE} database."
+      return 1
+    fi
+  fi
+
  if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then
-    log INFO postgresql_backup "Databases dumped successfully."
+    log INFO postgresql_backup "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
    return 0
  else
-    log ERROR "Backup of the postgresql database failed and needs attention."
+    log ERROR "Backup of the postgresql database(s) failed and needs attention. (SCOPE = ${SCOPE})"
    return 1
  fi
 }

 # Call main program to start the database backup
-backup_databases
+backup_databases ${SCOPE}
--- a/postgresql/templates/bin/_restore_postgresql.sh.tpl
+++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl
@ -38,9 +38,9 @@ get_databases() {
  TMP_DIR=$1
  DB_FILE=$2

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    grep 'CREATE DATABASE' $TMP_DIR/$SQL_FILE | awk '{ print $3 }' > $DB_FILE
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    grep 'CREATE DATABASE' $SQL_FILE | awk '{ print $3 }' > $DB_FILE
  else
    # Error, cannot report the databases
    echo "No SQL file found - cannot extract the databases"
@ -55,9 +55,9 @@ get_tables() {
  TMP_DIR=$2
  TABLE_FILE=$3

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    cat $SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
  else
    # Error, cannot report the tables
    echo "No SQL file found - cannot extract the tables"
@ -73,9 +73,9 @@ get_rows() {
  TMP_DIR=$3
  ROW_FILE=$4

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
    cat /tmp/db.sql | grep "INSERT INTO public.${TABLE} VALUES" > $ROW_FILE
    rm /tmp/db.sql
  else
@ -93,10 +93,10 @@ get_schema() {
  TMP_DIR=$3
  SCHEMA_FILE=$4

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
    DB_FILE=$(mktemp -p /tmp)
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
+    cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
    cat ${DB_FILE} | sed -n /'CREATE TABLE public.'${TABLE}' ('/,/'--'/p > ${SCHEMA_FILE}
    cat ${DB_FILE} | sed -n /'CREATE SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
    cat ${DB_FILE} | sed -n /'ALTER TABLE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
@ -239,9 +239,9 @@ restore_single_db() {
    return 1
  fi

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -f $TMP_DIR/$SQL_FILE ]]; then
-    extract_single_db_dump $TMP_DIR/$SQL_FILE $SINGLE_DB_NAME $TMP_DIR
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+    extract_single_db_dump $SQL_FILE $SINGLE_DB_NAME $TMP_DIR
    if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then
      # Drop connections first
      drop_connections ${SINGLE_DB_NAME}
@ -308,15 +308,26 @@ restore_all_dbs() {
  rm -rf ${LOG_FILE}
  touch ${LOG_FILE}

-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
-  if [[ -f $TMP_DIR/$SQL_FILE ]]; then
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
+  if [ -f $SQL_FILE ]; then
+
+    # Check the scope of the archive.
+    SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
+    if [[ "${SCOPE}" != "all" ]]; then
+      # This is just a single database backup. The user should
+      # instead use the single database restore option.
+      echo "Cannot use the restore all option for an archive containing only a single database."
+      echo "Please use the single database restore option."
+      return 1
+    fi
+
    # First drop all connections on all databases
    drop_connections_on_all_dbs
    if [[ "$?" -ne 0 ]]; then
      return 1
    fi

-    $PSQL postgres -f $TMP_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
+    $PSQL postgres -f $SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
    if [[ "$?" -eq 0 ]]; then
      if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then
        cat ${LOG_FILE}