diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl
new file mode 100755
index 000000000..8c9d6e294
--- /dev/null
+++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -x
+export PGPASSFILE=/etc/postgresql/admin_user.conf
+PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
+BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current
+ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
+POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
+PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
+
+#Delete files
+delete_files() {
+ files_to_delete=("$@")
+ for f in "${files_to_delete[@]}"
+ do
+ if [ -f $f ]
+ then
+ echo "Deleting file $f."
+ rm -rf $f
+ fi
+ done
+}
+
+#Get the day delta since the archive file backup
+days_difference() {
+ archive_date=$( date --date="$1" +%s )
+ if [ "$?" -ne 0 ]
+ then
+ day_delta=0
+ fi
+ current_date=$( date +%s )
+ date_delta=$(($current_date-$archive_date))
+ if [ "$date_delta" -lt 0 ]
+ then
+ day_delta=0
+ else
+ day_delta=$(($date_delta/86400))
+ fi
+ echo $day_delta
+}
+
+#Create backups directory if it does not exists.
+mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
+
+#Dump all databases
+DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")
+pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \
+ -h $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log
+if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]]
+then
+ #Archive the current databases files
+ pushd $BACKUPS_DIR 1>/dev/null
+ tar zcvf $ARCHIVE_DIR/postgres.all.${DATE}.tar.gz *
+ ARCHIVE_RET=$?
+ popd 1>/dev/null
+ #Remove the current backup
+ if [ -d $BACKUPS_DIR ]
+ then
+ rm -rf $BACKUPS_DIR/*.sql
+ fi
+else
+ #TODO: This can be convert into mail alert of alert send to a monitoring system
+ echo "Backup of postgresql failed and need attention."
+ cat dberror.log
+ exit 1
+fi
+
+#Only delete the old archive after a successful archive
+if [ $ARCHIVE_RET -eq 0 ]
+ then
+ if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ]
+ then
+ echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days"
+ if [ -d $ARCHIVE_DIR ]
+ then
+ for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
+ do
+ archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
+ if [ "$(days_difference $archive_date)" -gt "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" ]
+ then
+ rm -rf $archive_file
+ fi
+ done
+ fi
+ fi
+fi
diff --git a/postgresql/templates/bin/_restore_postgresql.sh.tpl b/postgresql/templates/bin/_restore_postgresql.sh.tpl
new file mode 100755
index 000000000..7b833e385
--- /dev/null
+++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl
@@ -0,0 +1,249 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+#set -x
+export PGPASSFILE=/etc/postgresql/admin_user.conf
+ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive
+RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore
+POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1)
+LIST_OPTIONS=(list_archives list_databases)
+ARGS=("$@")
+PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST"
+
+usage() {
+ echo "Usage:"
+ echo "$0 options"
+ echo "============================="
+ echo "options: "
+ echo "list_archives"
+ echo "list_databases archive_filename"
+ echo "restore archive_filename [DB_NAME or ALL/all]"
+}
+
+#Delete file
+delete_files() {
+ files_to_delete=("$@")
+ for f in "${files_to_delete[@]}"
+ do
+ if [ -f $f ]
+ then
+ echo "Deleting file $f."
+ rm -rf $f
+ fi
+ done
+}
+
+#Extract Single Database SQL Dump from pg_dumpall dump file
+extract_single_db_dump() {
+ sed "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d" > \
+ ${RESTORE_DIR}/$2.sql
+}
+
+#Display all archives
+list_archives() {
+ if [ -d ${ARCHIVE_DIR} ]
+ then
+ archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
+ echo "All Archives"
+ echo "=================================="
+ for archive in $archives
+ do
+ echo $archive | cut -d '/' -f 8
+ done
+ exit 0
+ else
+ echo "Archive directory is not available."
+ exit 1
+ fi
+}
+
+#Return all databases from an archive
+get_databases() {
+ archive_file=$1
+ if [ -e ${ARCHIVE_DIR}/${archive_file} ]
+ then
+ files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+ delete_files $files_to_purge
+ tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+ if [ -e ${RESTORE_DIR}/postgres.all.sql ]
+ then
+ DBS=$( grep 'CREATE DATABASE' ${RESTORE_DIR}/postgres.all.sql | awk '{ print $3 }' )
+ else
+ DBS=" "
+ fi
+ else
+ DBS=" "
+ fi
+}
+
+#Display all databases from an archive
+list_databases() {
+ archive_file=$1
+ get_databases $archive_file
+ #echo $DBS
+ if [ -n "$DBS" ]
+ then
+ echo " "
+ echo "Databases in the archive $archive_file"
+ echo "================================================================="
+ for db in $DBS
+ do
+ echo $db
+ done
+ else
+ echo "There is no database in the archive."
+ fi
+
+}
+
+#Restore a single database dump from pg_dumpall dump.
+restore_single_db() {
+ single_db_name=$1
+ if [ -z "$single_db_name" ]
+ then
+ usage
+ exit 1
+ fi
+ if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+ then
+ files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+ delete_files $files_to_purge
+ tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+ if [ -f ${RESTORE_DIR}/postgres.all.sql ]
+ then
+ extract_single_db_dump ${RESTORE_DIR}/postgres.all.sql $single_db_name
+ if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]]
+ then
+ $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log
+ if [ "$?" -eq 0 ]
+ then
+ echo "Database Restore Successful."
+ else
+ echo "Database Restore Failed."
+ fi
+ else
+ echo "Database Dump For $single_db_name is empty or not available."
+ fi
+ else
+ echo "Database file for dump_all not available to restore from"
+ fi
+ else
+ echo "Archive does not exist"
+ fi
+}
+
+#Restore all the databases
+restore_all_dbs() {
+ if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+ then
+ files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+ delete_files $files_to_purge
+ tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+ if [ -f ${RESTORE_DIR}/postgres.all.sql ]
+ then
+ $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log
+ if [ "$?" -eq 0 ]
+ then
+ echo "Database Restore successful."
+ else
+ echo "Database Restore failed."
+ fi
+ else
+ echo "There is no database file available to restore from"
+ fi
+ else
+ echo "Archive does not exist"
+ fi
+}
+
+
+is_Option() {
+ opts=$1
+ param=$2
+ find=0
+ for opt in $opts
+ do
+ if [ "$opt" == "$param" ]
+ then
+ find=1
+ fi
+ done
+ echo $find
+}
+
+#Main
+#Create Restore Directory
+mkdir -p $RESTORE_DIR
+if [ ${#ARGS[@]} -gt 3 ]
+then
+ usage
+ exit
+elif [ ${#ARGS[@]} -eq 1 ]
+then
+ if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
+ then
+ ${ARGS[0]}
+ exit
+ else
+ usage
+ exit
+ fi
+elif [ ${#ARGS[@]} -eq 2 ]
+then
+ if [ "${ARGS[0]}" == "list_databases" ]
+ then
+ list_databases ${ARGS[1]}
+ exit 0
+ else
+ usage
+ exit
+ fi
+elif [ ${#ARGS[@]} -eq 3 ]
+then
+ if [ "${ARGS[0]}" != "restore" ]
+ then
+ usage
+ exit 1
+ else
+ if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
+ then
+ #Get all the databases in that archive
+ get_databases ${ARGS[1]}
+ #check if the requested database is available in the archive
+ if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
+ then
+ echo "Restoring Database ${ARGS[2]} And Grants"
+ restore_single_db ${ARGS[2]}
+ echo "Tail dbrestore.log for restore log."
+ exit 0
+ elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
+ then
+ echo "Restoring All The Database."
+ restore_all_dbs
+ echo "Tail dbrestore.log for restore log."
+ exit 0
+ else
+ echo "There is no database with that name"
+ fi
+ else
+ echo "Archive file not found"
+ fi
+ fi
+else
+ usage
+ exit
+fi
+
diff --git a/postgresql/templates/configmap-bin.yaml b/postgresql/templates/configmap-bin.yaml
index 4feb64229..64011fa00 100644
--- a/postgresql/templates/configmap-bin.yaml
+++ b/postgresql/templates/configmap-bin.yaml
@@ -33,4 +33,8 @@ data:
{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
db_test.sh: |
{{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- if .Values.conf.backup.enabled }}
+ backup_postgresql.sh: |
+{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
{{- end }}
diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml
new file mode 100644
index 000000000..32d716815
--- /dev/null
+++ b/postgresql/templates/cron-job-backup-postgres.yaml
@@ -0,0 +1,109 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_postgresql_backup }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "postgresql-backup" }}
+{{ tuple $envAll "postgresql_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+ name: postgresql-backup
+ annotations:
+ {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+ schedule: {{ .Values.jobs.backup_postgresql.cron | quote }}
+ successfulJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.success }}
+ failedJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.failed }}
+ concurrencyPolicy: Forbid
+ jobTemplate:
+ metadata:
+ labels:
+{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+ spec:
+ template:
+ metadata:
+ labels:
+{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+ spec:
+ serviceAccountName: {{ $serviceAccountName }}
+ restartPolicy: OnFailure
+ nodeSelector:
+ {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+ containers:
+ - command:
+ - /tmp/backup_postgresql.sh
+ env:
+ - name: POSTGRESQL_BACKUP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: POSTGRES_PASSWORD
+ name: postgresql-admin
+ - name: POSTGRESQL_BACKUP_USER
+ valueFrom:
+ secretKeyRef:
+ key: POSTGRES_USER
+ name: postgresql-admin
+ - name: POSTGRESQL_BACKUP_BASE_DIR
+ value: {{ .Values.conf.backup.base_path }}
+ - name: POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
+ value: {{ .Values.conf.backup.pg_dumpall_options }}
+ - name: POSTGRESQL_BACKUP_DAYS_TO_KEEP
+ value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
+ - name: POSTGRESQL_POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 12 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }}
+ name: postgresql-backup
+ volumeMounts:
+ - mountPath: /tmp/backup_postgresql.sh
+ name: postgresql-bin
+ readOnly: true
+ subPath: backup_postgresql.sh
+ - mountPath: {{ .Values.conf.backup.base_path }}
+ name: postgresql-backup-dir
+ - name: postgresql-secrets
+ mountPath: /etc/postgresql/admin_user.conf
+ subPath: admin_user.conf
+ readOnly: true
+ restartPolicy: OnFailure
+ securityContext: {}
+ serviceAccount: {{ $serviceAccountName }}
+ serviceAccountName: {{ $serviceAccountName }}
+ volumes:
+ - name: postgresql-secrets
+ secret:
+ secretName: postgresql-secrets
+ defaultMode: 0600
+ - configMap:
+ defaultMode: 365
+ name: postgresql-bin
+ name: postgresql-bin
+ {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
+ - name: postgresql-backup-dir
+ persistentVolumeClaim:
+ claimName: postgresql-backup-data
+ {{- else }}
+ - hostPath:
+ path: {{ .Values.conf.backup.base_path }}
+ type: DirectoryOrCreate
+ name: postgresql-backup-dir
+ {{- end }}
+{{- end }}
diff --git a/postgresql/templates/postgresql-backup-pvc.yaml b/postgresql/templates/postgresql-backup-pvc.yaml
new file mode 100644
index 000000000..8d0fe525f
--- /dev/null
+++ b/postgresql/templates/postgresql-backup-pvc.yaml
@@ -0,0 +1,30 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: postgresql-backup-data
+spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: {{ .Values.volume.backup.size }}
+ storageClassName: {{ .Values.volume.backup.class_name }}
+{{- end }}
+
diff --git a/postgresql/templates/secrets-etc.yaml b/postgresql/templates/secrets-etc.yaml
new file mode 100644
index 000000000..c1c9b51cd
--- /dev/null
+++ b/postgresql/templates/secrets-etc.yaml
@@ -0,0 +1,28 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_etc }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: postgresql-secrets
+type: Opaque
+data:
+ admin_user.conf: {{ tuple "secrets/_admin_user.conf.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
+{{- end }}
+
diff --git a/postgresql/templates/secrets/_admin_user.conf.tpl b/postgresql/templates/secrets/_admin_user.conf.tpl
new file mode 100644
index 000000000..9b945ea67
--- /dev/null
+++ b/postgresql/templates/secrets/_admin_user.conf.tpl
@@ -0,0 +1,17 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}:*:*:{{ .Values.endpoints.postgresql.auth.admin.username }}:{{ .Values.endpoints.postgresql.auth.admin.password }}
diff --git a/postgresql/values.yaml b/postgresql/values.yaml
index 1c1850702..76b05514c 100644
--- a/postgresql/values.yaml
+++ b/postgresql/values.yaml
@@ -86,6 +86,13 @@ pod:
requests:
memory: "128Mi"
cpu: "100m"
+ postgresql_backup:
+ requests:
+ memory: "128Mi"
+ cpu: "100m"
+ limits:
+ memory: "1024Mi"
+ cpu: "2000m"
# using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/
images:
@@ -95,6 +102,7 @@ images:
image_repo_sync: docker.io/docker:17.07.0
prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5"
+ postgresql_backup: "docker.io/postgres:9.5"
pull_policy: "IfNotPresent"
local_registry:
active: false
@@ -124,6 +132,9 @@ labels:
prometheus_postgresql_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
+ job:
+ node_selector_key: openstack-control-plane
+ node_selector_value: enabled
dependencies:
dynamic:
@@ -155,6 +166,10 @@ dependencies:
service: postgresql
jobs:
- prometheus-postgresql-exporter-create-user
+ postgresql-backup:
+ services:
+ - endpoint: internal
+ service: postgresql
monitoring:
prometheus:
@@ -162,11 +177,29 @@ monitoring:
postgresql_exporter:
scrape: true
+volume:
+ backup:
+ enabled: true
+ class_name: general
+ size: 5Gi
+
+jobs:
+ backup_postgresql:
+ cron: "0 0 * * *"
+ history:
+ success: 3
+ failed: 1
+
conf:
debug: false
postgresql:
max_connections: 100
shared_buffers: 128MB
+ backup:
+ enabled: true
+ base_path: /var/backup
+ days_of_backup_to_keep: 3
+ pg_dumpall_options: null
secrets:
postgresql:
@@ -222,8 +255,11 @@ manifests:
configmap_bin: true
job_image_repo_sync: true
secret_admin: true
+ secret_etc: true
service: true
statefulset: true
+ cron_job_postgresql_backup: false
+ pvc_backup: false
monitoring:
prometheus:
configmap_bin: true