Merge "Fix postgresql database backup issue"

postgresql/templates/bin/_backup_postgresql.sh.tpl

@@ -14,26 +14,19 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
+                    | grep postgres | awk -F: '{print $5}')
+
 set -x
-export PGPASSFILE=/etc/postgresql/admin_user.conf
+
 PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
 BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
 ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
-POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
+LOG_FILE=/tmp/dberror.log
-PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
+PG_DUMPALL="pg_dumpall \
-
+              $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS \
-#Delete files
+              -U $POSTGRESQL_BACKUP_USER \
-delete_files() {
+              -h $POSTGRESQL_SERVICE_HOST"
-  files_to_delete=("$@")
-  for f in "${files_to_delete[@]}"
-  do
-    if [ -f $f ]
-    then
-      echo "Deleting file $f."
-      rm -rf $f
-    fi
-  done
-}
 
 #Get the day delta since the archive file backup
 seconds_difference() {
@@ -56,8 +49,7 @@ mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
 
 #Dump all databases
 DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")
-pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \
+$PG_DUMPALL --file=$BACKUPS_DIR/postgres.all.sql 2>>$LOG_FILE
-   -h  $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log
 if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]]
 then
   #Archive the current databases files
@@ -73,26 +65,27 @@ then
 else
   #TODO: This can be convert into mail alert of alert send to a monitoring system
   echo "Backup of postgresql failed and need attention."
-  cat dberror.log
+  cat $LOG_FILE
   exit 1
 fi
 
 #Only delete the old archive after a successful archive
 if [ $ARCHIVE_RET -eq 0 ]
+then
+  if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
   then
-    if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
+    echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
+    if [ -d $ARCHIVE_DIR ]
     then
-      echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
+      for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
-      if [ -d $ARCHIVE_DIR ]
+      do
-      then
+        archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
-        for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
+        if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
-        do
+        then
-          archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
+          rm -rf $archive_file
-          if [ "$(seconds_difference $archive_date)" -gt "$(($POSTGRESQL_BACKUP_DAYS_TO_KEEP*86400))" ]
+        fi
-          then
+      done
-            rm -rf $archive_file
-          fi
-        done
-      fi
     fi
+  fi
 fi
+

postgresql/templates/bin/_restore_postgresql.sh.tpl

@@ -14,23 +14,31 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-#set -x
+export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \
-export PGPASSFILE=/etc/postgresql/admin_user.conf
+                    | grep postgres | awk -F: '{print $5}')
+
+log_error() {
+  echo $1
+  exit 1
+}
+
 ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
 RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
 POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
-LIST_OPTIONS=(list_archives list_databases)
+LOG_FILE=/tmp/dbrestore.log
 ARGS=("$@")
 PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
 
 usage() {
+  ret_val=$1
   echo "Usage:"
-  echo "$0 options"
+  echo "Restore command options"
   echo "============================="
-  echo "options: "
+  echo "help"
   echo "list_archives"
-  echo "list_databases archive_filename"
+  echo "list_databases <archive_filename>"
-  echo "restore archive_filename [DB_NAME or ALL/all]"
+  echo "restore <archive_filename> [<db_name> | ALL]"
+  exit $ret_val
 }
 
 #Delete file
@@ -63,10 +71,8 @@ list_archives() {
     do
       echo $archive | cut -d '/' -f 8
     done
-    exit 0
   else
-    echo "Archive directory is not available."
+    log_error "Archive directory is not available."
-    exit 1
   fi
 }
 
@@ -122,8 +128,7 @@ restore_single_db() {
   single_db_name=$1
   if [ -z "$single_db_name" ]
   then
-    usage
+    usage 1
-    exit 1
   fi
   if [ -f ${ARCHIVE_DIR}/${archive_file} ]
   then
@@ -136,21 +141,21 @@ restore_single_db() {
       if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]]
       then
         create_db_if_not_exist $single_db_name
-        $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log
+        $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>>$LOG_FILE
         if [ "$?" -eq 0 ]
         then
           echo "Database Restore Successful."
         else
-          echo "Database Restore Failed."
+          log_error "Database Restore Failed."
         fi
       else
-        echo "Database Dump For $single_db_name is empty or not available."
+        log_error "Database Dump For $single_db_name is empty or not available."
       fi
     else
-      echo "Database file for dump_all not available to restore from"
+      log_error "Database file for dump_all not available to restore from"
     fi
   else
-    echo "Archive does not exist"
+    log_error "Archive does not exist"
   fi
 }
 
@@ -163,18 +168,18 @@ restore_all_dbs() {
     tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
     if [ -f ${RESTORE_DIR}/postgres.all.sql ]
     then
-      $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log
+      $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>>$LOG_FILE
       if [ "$?" -eq 0 ]
       then
         echo "Database Restore successful."
       else
-        echo "Database Restore failed."
+        log_error "Database Restore failed."
       fi
     else
-      echo "There is no database file available to restore from"
+      log_error "There is no database file available to restore from"
     fi
   else
-    echo "Archive does not exist"
+    log_error "Archive does not exist"
  fi
 }
 
@@ -198,52 +203,48 @@ is_Option() {
 mkdir -p $RESTORE_DIR
 if [ ${#ARGS[@]} -gt 3 ]
 then
-  usage
+  usage 0
-  exit
 elif [ ${#ARGS[@]} -eq 1 ]
 then
-  if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
+  if [ "${ARGS[0]}" == "list_archives" ]
   then
-    ${ARGS[0]}
+    list_archives
-    exit
+  elif [ "${ARGS[0]}" == "help" ]
+  then
+    usage 0
   else
-    usage
+    usage 1
-    exit
   fi
 elif [ ${#ARGS[@]} -eq 2 ]
 then
   if [ "${ARGS[0]}" == "list_databases" ]
   then
     list_databases ${ARGS[1]}
-    exit 0
   else
-    usage
+    usage 1
-    exit
   fi
 elif [ ${#ARGS[@]} -eq 3 ]
 then
   if [ "${ARGS[0]}" != "restore" ]
   then
-    usage
+    usage 1
-    exit 1
   else
     if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
     then
       #Get all the databases in that archive
       get_databases ${ARGS[1]}
+
       #check if the requested database is available in the archive
       if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
       then
         echo "Restoring Database ${ARGS[2]} And Grants"
         restore_single_db ${ARGS[2]}
-        echo "Tail dbrestore.log for restore log."
+        echo "Tail ${LOG_FILE} for restore log."
-        exit 0
       elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
       then
         echo "Restoring All The Database."
         restore_all_dbs
-        echo "Tail dbrestore.log for restore log."
+        echo "Tail ${LOG_FILE} for restore log."
-        exit 0
       else
         echo "There is no database with that name"
       fi
@@ -252,7 +253,7 @@ then
     fi
   fi
 else
-  usage
+  usage 1
-  exit
 fi
 
+exit 0

postgresql/templates/cron-job-backup-postgres.yaml

@@ -41,13 +41,15 @@ spec:
           labels:
 {{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
         spec:
+          securityContext:
+            fsGroup: 999
           serviceAccountName: {{ $serviceAccountName }}
           restartPolicy: OnFailure
           nodeSelector:
             {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
           containers:
             - name: postgresql-backup
-{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 14 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
               command:
                 - /tmp/backup_postgresql.sh

postgresql/values.yaml

@@ -126,7 +126,6 @@ images:
     image_repo_sync: docker.io/docker:17.07.0
     prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
     prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
-    postgresql_backup: "docker.io/postgres:9.5"
   pull_policy: "IfNotPresent"
   local_registry:
     active: false