From 233197fc0b8f327c9dd61e03bbfc06a709d9135e Mon Sep 17 00:00:00 2001 From: "Parsons, Cliff (cp769u)" Date: Thu, 13 Aug 2020 16:44:39 +0000 Subject: [PATCH] Add capabilitity to backup only a single database This PS adds the capability to Mariadb and Postgresql to backup a single database (as an optional parameter to the backup script). Change-Id: I9bc1eb0173063638b2cf58465c063f602ed20bc1 --- .../db-backup-restore/_backup_main.sh.tpl | 21 +++++--- mariadb/templates/bin/_backup_mariadb.sh.tpl | 50 ++++++++++++------- mariadb/templates/bin/_restore_mariadb.sh.tpl | 40 +++++++++------ .../templates/bin/_backup_postgresql.sh.tpl | 34 ++++++++++--- .../templates/bin/_restore_postgresql.sh.tpl | 47 ++++++++++------- 5 files changed, 126 insertions(+), 66 deletions(-) diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl index 8f6fa5bc0..a3105cda7 100755 --- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl +++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl @@ -9,10 +9,12 @@ # source /tmp/backup_main.sh # # Then the script should call the main backup function (backup_databases): -# backup_databases +# backup_databases [scope] +# [scope] is an optional parameter, defaulted to "all". If only one specific +# database is required to be backed up then this parameter will +# contain the name of the database; otherwise all are backed up. # -# No arguments required. However, the framework will require the -# following variables to be exported: +# The framework will require the following variables to be exported: # # export DB_NAMESPACE Namespace where the database(s) reside # export DB_NAME Name of the database system @@ -44,12 +46,15 @@ # is 1800 (30 minutes). # # The database-specific functions that need to be implemented are: -# dump_databases_to_directory +# dump_databases_to_directory [scope] # where: # is the full directory path to dump the database files # into. This is a temporary directory for this backup only. # is the full directory path where error logs are to be # written by the application. +# [scope] set to "all" if all databases are to be backed up; or +# set to the name of a specific database to be backed up. +# This optional parameter is defaulted to "all". # returns: 0 if no errors; 1 if any errors occurred # # This function is expected to dump the database file(s) to the specified @@ -285,7 +290,11 @@ remove_old_remote_archives() { # 1) The directory where the final backup will be kept after it is compressed. # 2) A temporary directory to use for placing database files to be compressed. # Note: this temp directory will be deleted after backup is done. +# 3) Optional "scope" parameter indicating what database to back up. Defaults +# to "all". backup_databases() { + SCOPE=${1:-"all"} + # Create necessary directories if they do not exist. mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!" export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!" @@ -294,7 +303,7 @@ backup_databases() { export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!" # It is expected that this function will dump the database files to the $TMP_DIR - dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE + dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE # If successful, there should be at least one file in the TMP_DIR if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then @@ -305,7 +314,7 @@ backup_databases() { log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..." NOW=$(date +"%Y-%m-%dT%H:%M:%SZ") - TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.all.${NOW}.tar.gz" + TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz" cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR" diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl index 3c31121b5..face534e0 100644 --- a/mariadb/templates/bin/_backup_mariadb.sh.tpl +++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl @@ -1,5 +1,7 @@ #!/bin/bash +SCOPE=${1:-"all"} + # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -28,6 +30,7 @@ export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/arch dump_databases_to_directory() { TMP_DIR=$1 LOG_FILE=$2 + SCOPE=${3:-"all"} MYSQL="mysql \ --defaults-file=/etc/mysql/admin_user.cnf \ @@ -36,9 +39,18 @@ dump_databases_to_directory() { MYSQLDUMP="mysqldump \ --defaults-file=/etc/mysql/admin_user.cnf" - MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ - "show databases;" | \ - egrep -vi 'information_schema|performance_schema|mysql') ) + if [[ "${SCOPE}" == "all" ]]; then + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + egrep -vi 'information_schema|performance_schema|mysql') ) + else + if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" ]]; then + MYSQL_DBNAMES=( ${SCOPE} ) + else + log ERROR "It is not allowed to backup database ${SCOPE}." + return 1 + fi + fi #check if there is a database to backup, otherwise exit if [[ -z "${MYSQL_DBNAMES// }" ]] @@ -50,19 +62,21 @@ dump_databases_to_directory() { #Create a list of Databases printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list - #Retrieve and create the GRANT file for all the users + if [[ "${SCOPE}" == "all" ]]; then + #Retrieve and create the GRANT file for all the users {{- if .Values.manifests.certificates }} - SSL_DSN=";mysql_ssl=1" - SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key" - SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt" - SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt" - if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \ + SSL_DSN=";mysql_ssl=1" + SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key" + SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt" + SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt" + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \ {{- else }} - if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \ + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \ {{- end }} - 2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then - log ERROR "Failed to create GRANT for all the users" - return 1 + 2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then + log ERROR "Failed to create GRANT for all the users" + return 1 + fi fi #Retrieve and create the GRANT files per DB @@ -82,22 +96,20 @@ dump_databases_to_directory() { done #Dumping the database - DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all - TARBALL_FILE=${SQL_FILE}.${DATE}.tar.gz + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE} $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] then - log INFO "Databases dumped successfully." + log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})" return 0 else - log ERROR "Backup failed and need attention." + log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})" return 1 fi } # Call main program to start the database backup -backup_databases +backup_databases ${SCOPE} diff --git a/mariadb/templates/bin/_restore_mariadb.sh.tpl b/mariadb/templates/bin/_restore_mariadb.sh.tpl index 5b4462a3f..d35c6a2ad 100755 --- a/mariadb/templates/bin/_restore_mariadb.sh.tpl +++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl @@ -83,9 +83,9 @@ get_tables() { TMP_DIR=$2 TABLE_FILE=$3 - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then - current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ > $TABLE_FILE else @@ -103,9 +103,9 @@ get_rows() { TMP_DIR=$3 ROW_FILE=$4 - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then - current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} \ + SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE return 0 else @@ -123,10 +123,10 @@ get_schema() { TMP_DIR=$3 SCHEMA_FILE=$4 - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then + SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then DB_FILE=$(mktemp -p /tmp) - current_db_desc ${DATABASE} ${TMP_DIR}/${SQL_FILE} > ${DB_FILE} + current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE} sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} if [[ ! (-s ${SCHEMA_FILE}) ]]; then sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ @@ -193,8 +193,8 @@ restore_single_db() { return 1 fi - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql - if [[ -f ${TMP_DIR}/$SQL_FILE ]] + SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ] then # Restoring a single database requires us to create a temporary user # which has capability to only restore that ONE database. One gotcha @@ -208,7 +208,7 @@ restore_single_db() { echo "Restore $SINGLE_DB_NAME failed create restore user." return 1 fi - $RESTORE_CMD --force < ${TMP_DIR}/$SQL_FILE 2>>$RESTORE_LOG + $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG if [[ "$?" -eq 0 ]] then echo "Database $SINGLE_DB_NAME Restore successful." @@ -254,10 +254,20 @@ restore_single_db() { restore_all_dbs() { TMP_DIR=$1 - SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.all.sql - if [[ -f ${TMP_DIR}/$SQL_FILE ]] + SQL_FILE=$TMP_DIR/mariadb.$MARIADB_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ] then - $MYSQL < ${TMP_DIR}/$SQL_FILE 2>$RESTORE_LOG + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + + $MYSQL < $SQL_FILE 2>$RESTORE_LOG if [[ "$?" -eq 0 ]] then echo "Databases $( echo $DBS | tr -d '\n') Restore successful." diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl index 41f1ab1a3..cae73978c 100755 --- a/postgresql/templates/bin/_backup_postgresql.sh.tpl +++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl @@ -1,5 +1,7 @@ #!/bin/bash +SCOPE=${1:-"all"} + # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -40,27 +42,43 @@ export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/a dump_databases_to_directory() { TMP_DIR=$1 LOG_FILE=$2 + SCOPE=${3:-"all"} - PG_DUMPALL_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g') + PG_DUMP_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g') + PG_DUMP="pg_dump \ + $PG_DUMP_OPTIONS --create \ + -U $POSTGRESQL_ADMIN_USER \ + -h $POSTGRESQL_SERVICE_HOST" PG_DUMPALL="pg_dumpall \ - $PG_DUMPALL_OPTIONS \ + $PG_DUMP_OPTIONS \ -U $POSTGRESQL_ADMIN_USER \ -h $POSTGRESQL_SERVICE_HOST" - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all + SQL_FILE=postgres.${POSTGRESQL_POD_NAMESPACE}.${SCOPE} cd $TMP_DIR - #Dump all databases - $PG_DUMPALL --file=${TMP_DIR}/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ "${SCOPE}" == "all" ]]; then + # Dump all databases + ${PG_DUMPALL} --file=${TMP_DIR}/${SQL_FILE}.sql 2>>${LOG_FILE} + else + if [[ "${SCOPE}" != "postgres" && "${SCOPE}" != "template0" && "${SCOPE}" != "template1" ]]; then + # Dump the specified database + ${PG_DUMP} --file=${TMP_DIR}/${SQL_FILE}.sql ${SCOPE} 2>>${LOG_FILE} + else + log ERROR "It is not allowed to backup up the ${SCOPE} database." + return 1 + fi + fi + if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then - log INFO postgresql_backup "Databases dumped successfully." + log INFO postgresql_backup "Database(s) dumped successfully. (SCOPE = ${SCOPE})" return 0 else - log ERROR "Backup of the postgresql database failed and needs attention." + log ERROR "Backup of the postgresql database(s) failed and needs attention. (SCOPE = ${SCOPE})" return 1 fi } # Call main program to start the database backup -backup_databases +backup_databases ${SCOPE} diff --git a/postgresql/templates/bin/_restore_postgresql.sh.tpl b/postgresql/templates/bin/_restore_postgresql.sh.tpl index 5817d4bed..ed9702de3 100755 --- a/postgresql/templates/bin/_restore_postgresql.sh.tpl +++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl @@ -38,9 +38,9 @@ get_databases() { TMP_DIR=$1 DB_FILE=$2 - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then - grep 'CREATE DATABASE' $TMP_DIR/$SQL_FILE | awk '{ print $3 }' > $DB_FILE + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + grep 'CREATE DATABASE' $SQL_FILE | awk '{ print $3 }' > $DB_FILE else # Error, cannot report the databases echo "No SQL file found - cannot extract the databases" @@ -55,9 +55,9 @@ get_tables() { TMP_DIR=$2 TABLE_FILE=$3 - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then - cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + cat $SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE else # Error, cannot report the tables echo "No SQL file found - cannot extract the tables" @@ -73,9 +73,9 @@ get_rows() { TMP_DIR=$3 ROW_FILE=$4 - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then - cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql cat /tmp/db.sql | grep "INSERT INTO public.${TABLE} VALUES" > $ROW_FILE rm /tmp/db.sql else @@ -93,10 +93,10 @@ get_schema() { TMP_DIR=$3 SCHEMA_FILE=$4 - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -e $TMP_DIR/$SQL_FILE ]]; then + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then DB_FILE=$(mktemp -p /tmp) - cat $TMP_DIR/$SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE} + cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE} cat ${DB_FILE} | sed -n /'CREATE TABLE public.'${TABLE}' ('/,/'--'/p > ${SCHEMA_FILE} cat ${DB_FILE} | sed -n /'CREATE SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} cat ${DB_FILE} | sed -n /'ALTER TABLE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} @@ -239,9 +239,9 @@ restore_single_db() { return 1 fi - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -f $TMP_DIR/$SQL_FILE ]]; then - extract_single_db_dump $TMP_DIR/$SQL_FILE $SINGLE_DB_NAME $TMP_DIR + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + extract_single_db_dump $SQL_FILE $SINGLE_DB_NAME $TMP_DIR if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then # Drop connections first drop_connections ${SINGLE_DB_NAME} @@ -308,15 +308,26 @@ restore_all_dbs() { rm -rf ${LOG_FILE} touch ${LOG_FILE} - SQL_FILE=postgres.$POSTGRESQL_POD_NAMESPACE.all.sql - if [[ -f $TMP_DIR/$SQL_FILE ]]; then + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + # First drop all connections on all databases drop_connections_on_all_dbs if [[ "$?" -ne 0 ]]; then return 1 fi - $PSQL postgres -f $TMP_DIR/$SQL_FILE 2>>$LOG_FILE >> $LOG_FILE + $PSQL postgres -f $SQL_FILE 2>>$LOG_FILE >> $LOG_FILE if [[ "$?" -eq 0 ]]; then if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then cat ${LOG_FILE}