Merge "Fix postgresql database backup issue"
This commit is contained in:
Zuul
Gerrit Code Review
commit
4a770bb908
@ -14,26 +14,19 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
|
||||
| grep postgres | awk -F: '{print $5}')
|
||||
|
||||
set -x
|
||||
export PGPASSFILE=/etc/postgresql/admin_user.conf
|
||||
|
||||
PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
|
||||
BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
|
||||
ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
|
||||
POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
|
||||
PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
|
||||
|
||||
#Delete files
|
||||
delete_files() {
|
||||
files_to_delete=("$@")
|
||||
for f in "${files_to_delete[@]}"
|
||||
do
|
||||
if [ -f $f ]
|
||||
then
|
||||
echo "Deleting file $f."
|
||||
rm -rf $f
|
||||
fi
|
||||
done
|
||||
}
|
||||
LOG_FILE=/tmp/dberror.log
|
||||
PG_DUMPALL="pg_dumpall \
|
||||
$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS \
|
||||
-U $POSTGRESQL_BACKUP_USER \
|
||||
-h $POSTGRESQL_SERVICE_HOST"
|
||||
|
||||
#Get the day delta since the archive file backup
|
||||
seconds_difference() {
|
||||
@ -56,8 +49,7 @@ mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
|
||||
|
||||
#Dump all databases
|
||||
DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")
|
||||
pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \
|
||||
-h $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log
|
||||
$PG_DUMPALL --file=$BACKUPS_DIR/postgres.all.sql 2>>$LOG_FILE
|
||||
if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]]
|
||||
then
|
||||
#Archive the current databases files
|
||||
@ -73,26 +65,27 @@ then
|
||||
else
|
||||
#TODO: This can be convert into mail alert of alert send to a monitoring system
|
||||
echo "Backup of postgresql failed and need attention."
|
||||
cat dberror.log
|
||||
cat $LOG_FILE
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#Only delete the old archive after a successful archive
|
||||
if [ $ARCHIVE_RET -eq 0 ]
|
||||
then
|
||||
if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
|
||||
then
|
||||
if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
|
||||
echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
|
||||
if [ -d $ARCHIVE_DIR ]
|
||||
then
|
||||
echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
|
||||
if [ -d $ARCHIVE_DIR ]
|
||||
then
|
||||
for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
|
||||
do
|
||||
archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
|
||||
if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
|
||||
then
|
||||
rm -rf $archive_file
|
||||
fi
|
||||
done
|
||||
fi
|
||||
for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
|
||||
do
|
||||
archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
|
||||
if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
|
||||
then
|
||||
rm -rf $archive_file
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@ -14,23 +14,31 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
#set -x
|
||||
export PGPASSFILE=/etc/postgresql/admin_user.conf
|
||||
export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
|
||||
| grep postgres | awk -F: '{print $5}')
|
||||
|
||||
log_error() {
|
||||
echo $1
|
||||
exit 1
|
||||
}
|
||||
|
||||
ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
|
||||
RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
|
||||
POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
|
||||
LIST_OPTIONS=(list_archives list_databases)
|
||||
LOG_FILE=/tmp/dbrestore.log
|
||||
ARGS=("$@")
|
||||
PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
|
||||
|
||||
usage() {
|
||||
ret_val=$1
|
||||
echo "Usage:"
|
||||
echo "$0 options"
|
||||
echo "Restore command options"
|
||||
echo "============================="
|
||||
echo "options: "
|
||||
echo "help"
|
||||
echo "list_archives"
|
||||
echo "list_databases archive_filename"
|
||||
echo "restore archive_filename [DB_NAME or ALL/all]"
|
||||
echo "list_databases <archive_filename>"
|
||||
echo "restore <archive_filename> [<db_name> | ALL]"
|
||||
exit $ret_val
|
||||
}
|
||||
|
||||
#Delete file
|
||||
@ -63,10 +71,8 @@ list_archives() {
|
||||
do
|
||||
echo $archive | cut -d '/' -f 8
|
||||
done
|
||||
exit 0
|
||||
else
|
||||
echo "Archive directory is not available."
|
||||
exit 1
|
||||
log_error "Archive directory is not available."
|
||||
fi
|
||||
}
|
||||
|
||||
@ -122,8 +128,7 @@ restore_single_db() {
|
||||
single_db_name=$1
|
||||
if [ -z "$single_db_name" ]
|
||||
then
|
||||
usage
|
||||
exit 1
|
||||
usage 1
|
||||
fi
|
||||
if [ -f ${ARCHIVE_DIR}/${archive_file} ]
|
||||
then
|
||||
@ -136,21 +141,21 @@ restore_single_db() {
|
||||
if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]]
|
||||
then
|
||||
create_db_if_not_exist $single_db_name
|
||||
$PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log
|
||||
$PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>>$LOG_FILE
|
||||
if [ "$?" -eq 0 ]
|
||||
then
|
||||
echo "Database Restore Successful."
|
||||
else
|
||||
echo "Database Restore Failed."
|
||||
log_error "Database Restore Failed."
|
||||
fi
|
||||
else
|
||||
echo "Database Dump For $single_db_name is empty or not available."
|
||||
log_error "Database Dump For $single_db_name is empty or not available."
|
||||
fi
|
||||
else
|
||||
echo "Database file for dump_all not available to restore from"
|
||||
log_error "Database file for dump_all not available to restore from"
|
||||
fi
|
||||
else
|
||||
echo "Archive does not exist"
|
||||
log_error "Archive does not exist"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -163,18 +168,18 @@ restore_all_dbs() {
|
||||
tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
|
||||
if [ -f ${RESTORE_DIR}/postgres.all.sql ]
|
||||
then
|
||||
$PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log
|
||||
$PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>>$LOG_FILE
|
||||
if [ "$?" -eq 0 ]
|
||||
then
|
||||
echo "Database Restore successful."
|
||||
else
|
||||
echo "Database Restore failed."
|
||||
log_error "Database Restore failed."
|
||||
fi
|
||||
else
|
||||
echo "There is no database file available to restore from"
|
||||
log_error "There is no database file available to restore from"
|
||||
fi
|
||||
else
|
||||
echo "Archive does not exist"
|
||||
log_error "Archive does not exist"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -198,52 +203,48 @@ is_Option() {
|
||||
mkdir -p $RESTORE_DIR
|
||||
if [ ${#ARGS[@]} -gt 3 ]
|
||||
then
|
||||
usage
|
||||
exit
|
||||
usage 0
|
||||
elif [ ${#ARGS[@]} -eq 1 ]
|
||||
then
|
||||
if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
|
||||
if [ "${ARGS[0]}" == "list_archives" ]
|
||||
then
|
||||
${ARGS[0]}
|
||||
exit
|
||||
list_archives
|
||||
elif [ "${ARGS[0]}" == "help" ]
|
||||
then
|
||||
usage 0
|
||||
else
|
||||
usage
|
||||
exit
|
||||
usage 1
|
||||
fi
|
||||
elif [ ${#ARGS[@]} -eq 2 ]
|
||||
then
|
||||
if [ "${ARGS[0]}" == "list_databases" ]
|
||||
then
|
||||
list_databases ${ARGS[1]}
|
||||
exit 0
|
||||
else
|
||||
usage
|
||||
exit
|
||||
usage 1
|
||||
fi
|
||||
elif [ ${#ARGS[@]} -eq 3 ]
|
||||
then
|
||||
if [ "${ARGS[0]}" != "restore" ]
|
||||
then
|
||||
usage
|
||||
exit 1
|
||||
usage 1
|
||||
else
|
||||
if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
|
||||
then
|
||||
#Get all the databases in that archive
|
||||
get_databases ${ARGS[1]}
|
||||
|
||||
#check if the requested database is available in the archive
|
||||
if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
|
||||
then
|
||||
echo "Restoring Database ${ARGS[2]} And Grants"
|
||||
restore_single_db ${ARGS[2]}
|
||||
echo "Tail dbrestore.log for restore log."
|
||||
exit 0
|
||||
echo "Tail ${LOG_FILE} for restore log."
|
||||
elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
|
||||
then
|
||||
echo "Restoring All The Database."
|
||||
restore_all_dbs
|
||||
echo "Tail dbrestore.log for restore log."
|
||||
exit 0
|
||||
echo "Tail ${LOG_FILE} for restore log."
|
||||
else
|
||||
echo "There is no database with that name"
|
||||
fi
|
||||
@ -252,7 +253,7 @@ then
|
||||
fi
|
||||
fi
|
||||
else
|
||||
usage
|
||||
exit
|
||||
usage 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
@ -41,13 +41,15 @@ spec:
|
||||
labels:
|
||||
{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 999
|
||||
serviceAccountName: {{ $serviceAccountName }}
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector:
|
||||
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
|
||||
containers:
|
||||
- name: postgresql-backup
|
||||
{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
|
||||
command:
|
||||
- /tmp/backup_postgresql.sh
|
||||
|
||||
@ -126,7 +126,6 @@ images:
|
||||
image_repo_sync: docker.io/docker:17.07.0
|
||||
prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
|
||||
prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
|
||||
postgresql_backup: "docker.io/postgres:9.5"
|
||||
pull_policy: "IfNotPresent"
|
||||
local_registry:
|
||||
active: false
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user