Add capabilitity to backup only a single database

This PS adds the capability to Mariadb and Postgresql to backup a
single database (as an optional parameter to the backup script).

Change-Id: I9bc1eb0173063638b2cf58465c063f602ed20bc1

helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl

@@ -9,10 +9,12 @@
 #   source /tmp/backup_main.sh
 #
 # Then the script should call the main backup function (backup_databases):
-#   backup_databases
+#   backup_databases [scope]
+#       [scope] is an optional parameter, defaulted to "all". If only one specific
+#               database is required to be backed up then this parameter will
+#               contain the name of the database; otherwise all are backed up.
 #
-#       No arguments required. However, the framework will require the
+#       The framework will require the following variables to be exported:
-#       following variables to be exported:
 #
 #         export DB_NAMESPACE          Namespace where the database(s) reside
 #         export DB_NAME               Name of the database system
@@ -44,12 +46,15 @@
 #                                      is 1800 (30 minutes).
 #
 # The database-specific functions that need to be implemented are:
-#   dump_databases_to_directory <directory> <err_logfile>
+#   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
 #         <directory>   is the full directory path to dump the database files
 #                       into. This is a temporary directory for this backup only.
 #         <err_logfile> is the full directory path where error logs are to be
 #                       written by the application.
+#         [scope]       set to "all" if all databases are to be backed up; or
+#                       set to the name of a specific database to be backed up.
+#                       This optional parameter is defaulted to "all".
 #       returns: 0 if no errors; 1 if any errors occurred
 #
 #       This function is expected to dump the database file(s) to the specified
@@ -285,7 +290,11 @@ remove_old_remote_archives() {
 #  1) The directory where the final backup will be kept after it is compressed.
 #  2) A temporary directory to use for placing database files to be compressed.
 #     Note: this temp directory will be deleted after backup is done.
+#  3) Optional "scope" parameter indicating what database to back up. Defaults
+#     to "all".
 backup_databases() {
+  SCOPE=${1:-"all"}
+
   # Create necessary directories if they do not exist.
   mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
   export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
@@ -294,7 +303,7 @@ backup_databases() {
   export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"
 
   # It is expected that this function will dump the database files to the $TMP_DIR
-  dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE
+  dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE
 
   # If successful, there should be at least one file in the TMP_DIR
   if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
@@ -305,7 +314,7 @@ backup_databases() {
   log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..."
 
   NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")
-  TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.all.${NOW}.tar.gz"
+  TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz"
 
   cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"
 

mariadb/templates/bin/_backup_mariadb.sh.tpl

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+SCOPE=${1:-"all"}
+
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -28,6 +30,7 @@ export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/arch
 dump_databases_to_directory() {
   TMP_DIR=$1
   LOG_FILE=$2
+  SCOPE=${3:-"all"}
 
   MYSQL="mysql \
      --defaults-file=/etc/mysql/admin_user.cnf \
@@ -36,9 +39,18 @@ dump_databases_to_directory() {
   MYSQLDUMP="mysqldump \
      --defaults-file=/etc/mysql/admin_user.cnf"
 
-  MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
+  if [[ "${SCOPE}" == "all" ]]; then
-     "show databases;" | \
+    MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \
-     egrep -vi 'information_schema|performance_schema|mysql') )
+       "show databases;" | \
+       egrep -vi 'information_schema|performance_schema|mysql') )
+  else
+    if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" ]]; then
+      MYSQL_DBNAMES=( ${SCOPE} )
+    else
+      log ERROR "It is not allowed to backup database ${SCOPE}."
+      return 1
+    fi
+  fi
 
   #check if there is a database to backup, otherwise exit
   if [[ -z "${MYSQL_DBNAMES// }" ]]
@@ -50,19 +62,21 @@ dump_databases_to_directory() {
   #Create a list of Databases
   printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list
 
-  #Retrieve and create the GRANT file for all the users
+  if [[ "${SCOPE}" == "all" ]]; then
+    #Retrieve and create the GRANT file for all the users
 {{- if .Values.manifests.certificates }}
-  SSL_DSN=";mysql_ssl=1"
+    SSL_DSN=";mysql_ssl=1"
-  SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key"
+    SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key"
-  SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt"
+    SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt"
-  SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt"
+    SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt"
-  if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \
+    if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \
 {{- else }}
-  if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \
+    if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \
 {{- end }}
-       2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then
+         2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then
-    log ERROR "Failed to create GRANT for all the users"
+      log ERROR "Failed to create GRANT for all the users"
-    return 1
+      return 1
+    fi
   fi
 
   #Retrieve and create the GRANT files per DB
@@ -82,22 +96,20 @@ dump_databases_to_directory() {
   done
 
   #Dumping the database
-  DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all
+  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE}
-  TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz
 
   $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}"  \
             > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE
   if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]]
   then
-    log INFO "Databases dumped successfully."
+    log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
     return 0
   else
-    log ERROR "Backup failed and need attention."
+    log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})"
     return 1
   fi
 }
 
 # Call main program to start the database backup
-backup_databases
+backup_databases ${SCOPE}

mariadb/templates/bin/_restore_mariadb.sh.tpl

@@ -83,9 +83,9 @@ get_tables() {
   TMP_DIR=$2
   TABLE_FILE=$3
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
+    current_db_desc ${DATABASE} ${SQL_FILE} \
         | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \
         > $TABLE_FILE
   else
@@ -103,9 +103,9 @@ get_rows() {
   TMP_DIR=$3
   ROW_FILE=$4
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \
+    current_db_desc ${DATABASE} ${SQL_FILE} \
         | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE
     return 0
   else
@@ -123,10 +123,10 @@ get_schema() {
   TMP_DIR=$3
   SCHEMA_FILE=$4
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
     DB_FILE=$(mktemp -p /tmp)
-    current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE}
+    current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE}
     sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE}
     if [[ ! (-s ${SCHEMA_FILE}) ]]; then
       sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \
@@ -193,8 +193,8 @@ restore_single_db() {
     return 1
   fi
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
-  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
+  if [ -f $SQL_FILE ]
   then
     # Restoring a single database requires us to create a temporary user
     # which has capability to only restore that ONE database. One gotcha
@@ -208,7 +208,7 @@ restore_single_db() {
       echo "Restore $SINGLE_DB_NAME failed create restore user."
       return 1
     fi
-    $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG
+    $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG
     if [[ "$?" -eq 0 ]]
     then
       echo "Database $SINGLE_DB_NAME Restore successful."
@@ -254,10 +254,20 @@ restore_single_db() {
 restore_all_dbs() {
   TMP_DIR=$1
 
-  SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql
-  if [[ -f ${TMP_DIR}/$SQL_FILE ]]
+  if [ -f $SQL_FILE ]
   then
-    $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG
+    # Check the scope of the archive.
+    SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
+    if [[ "${SCOPE}" != "all" ]]; then
+      # This is just a single database backup. The user should
+      # instead use the single database restore option.
+      echo "Cannot use the restore all option for an archive containing only a single database."
+      echo "Please use the single database restore option."
+      return 1
+    fi
+
+    $MYSQL < $SQL_FILE 2>$RESTORE_LOG
     if [[ "$?" -eq 0 ]]
     then
       echo "Databases $( echo $DBS | tr -d '\n') Restore successful."

postgresql/templates/bin/_backup_postgresql.sh.tpl

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+SCOPE=${1:-"all"}
+
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -40,27 +42,43 @@ export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/a
 dump_databases_to_directory() {
   TMP_DIR=$1
   LOG_FILE=$2
+  SCOPE=${3:-"all"}
 
-  PG_DUMPALL_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
+  PG_DUMP_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g')
+  PG_DUMP="pg_dump \
+             $PG_DUMP_OPTIONS --create \
+             -U $POSTGRESQL_ADMIN_USER \
+             -h $POSTGRESQL_SERVICE_HOST"
   PG_DUMPALL="pg_dumpall \
-                $PG_DUMPALL_OPTIONS \
+                $PG_DUMP_OPTIONS \
                 -U $POSTGRESQL_ADMIN_USER \
                 -h $POSTGRESQL_SERVICE_HOST"
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all
+  SQL_FILE=postgres.${POSTGRESQL_POD_NAMESPACE}.${SCOPE}
 
   cd $TMP_DIR
 
-  #Dump all databases
+  if [[ "${SCOPE}" == "all" ]]; then
-  $PG_DUMPALL --file=${TMP_DIR}/${SQL_FILE}.sql 2>>$LOG_FILE
+    # Dump all databases
+    ${PG_DUMPALL} --file=${TMP_DIR}/${SQL_FILE}.sql 2>>${LOG_FILE}
+  else
+    if [[ "${SCOPE}" != "postgres" && "${SCOPE}" != "template0" && "${SCOPE}" != "template1" ]]; then
+      # Dump the specified database
+      ${PG_DUMP} --file=${TMP_DIR}/${SQL_FILE}.sql ${SCOPE} 2>>${LOG_FILE}
+    else
+      log ERROR "It is not allowed to backup up the ${SCOPE} database."
+      return 1
+    fi
+  fi
+
   if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then
-    log INFO postgresql_backup "Databases dumped successfully."
+    log INFO postgresql_backup "Database(s) dumped successfully. (SCOPE = ${SCOPE})"
     return 0
   else
-    log ERROR "Backup of the postgresql database failed and needs attention."
+    log ERROR "Backup of the postgresql database(s) failed and needs attention. (SCOPE = ${SCOPE})"
     return 1
   fi
 }
 
 # Call main program to start the database backup
-backup_databases
+backup_databases ${SCOPE}

postgresql/templates/bin/_restore_postgresql.sh.tpl

@@ -38,9 +38,9 @@ get_databases() {
   TMP_DIR=$1
   DB_FILE=$2
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    grep 'CREATE DATABASE' $TMP_DIR/$SQL_FILE | awk '{ print $3 }' > $DB_FILE
+    grep 'CREATE DATABASE' $SQL_FILE | awk '{ print $3 }' > $DB_FILE
   else
     # Error, cannot report the databases
     echo "No SQL file found - cannot extract the databases"
@@ -55,9 +55,9 @@ get_tables() {
   TMP_DIR=$2
   TABLE_FILE=$3
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
+    cat $SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE
   else
     # Error, cannot report the tables
     echo "No SQL file found - cannot extract the tables"
@@ -73,9 +73,9 @@ get_rows() {
   TMP_DIR=$3
   ROW_FILE=$4
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
+    cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql
     cat /tmp/db.sql | grep "INSERT INTO public.${TABLE} VALUES" > $ROW_FILE
     rm /tmp/db.sql
   else
@@ -93,10 +93,10 @@ get_schema() {
   TMP_DIR=$3
   SCHEMA_FILE=$4
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -e $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
     DB_FILE=$(mktemp -p /tmp)
-    cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
+    cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE}
     cat ${DB_FILE} | sed -n /'CREATE TABLE public.'${TABLE}' ('/,/'--'/p > ${SCHEMA_FILE}
     cat ${DB_FILE} | sed -n /'CREATE SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
     cat ${DB_FILE} | sed -n /'ALTER TABLE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE}
@@ -239,9 +239,9 @@ restore_single_db() {
     return 1
   fi
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -f $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
-    extract_single_db_dump $TMP_DIR/$SQL_FILE $SINGLE_DB_NAME $TMP_DIR
+    extract_single_db_dump $SQL_FILE $SINGLE_DB_NAME $TMP_DIR
     if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then
       # Drop connections first
       drop_connections ${SINGLE_DB_NAME}
@@ -308,15 +308,26 @@ restore_all_dbs() {
   rm -rf ${LOG_FILE}
   touch ${LOG_FILE}
 
-  SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql
+  SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql
-  if [[ -f $TMP_DIR/$SQL_FILE ]]; then
+  if [ -f $SQL_FILE ]; then
+
+    # Check the scope of the archive.
+    SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}')
+    if [[ "${SCOPE}" != "all" ]]; then
+      # This is just a single database backup. The user should
+      # instead use the single database restore option.
+      echo "Cannot use the restore all option for an archive containing only a single database."
+      echo "Please use the single database restore option."
+      return 1
+    fi
+
     # First drop all connections on all databases
     drop_connections_on_all_dbs
     if [[ "$?" -ne 0 ]]; then
       return 1
     fi
 
-    $PSQL postgres -f $TMP_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
+    $PSQL postgres -f $SQL_FILE 2>>$LOG_FILE >> $LOG_FILE
     if [[ "$?" -eq 0 ]]; then
       if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then
         cat ${LOG_FILE}