diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl new file mode 100755 index 000000000..8c9d6e294 --- /dev/null +++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -x +export PGPASSFILE=/etc/postgresql/admin_user.conf +PG_DUMPALL_OPTIONS=$POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS +BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current +ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive +POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1) +PG_DUMPALL="pg_dumpall -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST" + +#Delete files +delete_files() { + files_to_delete=("$@") + for f in "${files_to_delete[@]}" + do + if [ -f $f ] + then + echo "Deleting file $f." + rm -rf $f + fi + done +} + +#Get the day delta since the archive file backup +days_difference() { + archive_date=$( date --date="$1" +%s ) + if [ "$?" -ne 0 ] + then + day_delta=0 + fi + current_date=$( date +%s ) + date_delta=$(($current_date-$archive_date)) + if [ "$date_delta" -lt 0 ] + then + day_delta=0 + else + day_delta=$(($date_delta/86400)) + fi + echo $day_delta +} + +#Create backups directory if it does not exists. +mkdir -p $BACKUPS_DIR $ARCHIVE_DIR + +#Dump all databases +DATE=$(date +"%Y-%m-%dT%H:%M:%SZ") +pg_dumpall $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS -U $POSTGRESQL_BACKUP_USER \ + -h $POSTGRESQL_HOST --file=$BACKUPS_DIR/postgres.all.sql 2>dberror.log +if [[ $? -eq 0 && -s "$BACKUPS_DIR/postgres.all.sql" ]] +then + #Archive the current databases files + pushd $BACKUPS_DIR 1>/dev/null + tar zcvf $ARCHIVE_DIR/postgres.all.${DATE}.tar.gz * + ARCHIVE_RET=$? + popd 1>/dev/null + #Remove the current backup + if [ -d $BACKUPS_DIR ] + then + rm -rf $BACKUPS_DIR/*.sql + fi +else + #TODO: This can be convert into mail alert of alert send to a monitoring system + echo "Backup of postgresql failed and need attention." + cat dberror.log + exit 1 +fi + +#Only delete the old archive after a successful archive +if [ $ARCHIVE_RET -eq 0 ] + then + if [ "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" -gt 0 ] + then + echo "Deleting backups older than $POSTGRESQL_BACKUP_DAYS_TO_KEEP days" + if [ -d $ARCHIVE_DIR ] + then + for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) + do + archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) + if [ "$(days_difference $archive_date)" -gt "$POSTGRESQL_BACKUP_DAYS_TO_KEEP" ] + then + rm -rf $archive_file + fi + done + fi + fi +fi diff --git a/postgresql/templates/bin/_restore_postgresql.sh.tpl b/postgresql/templates/bin/_restore_postgresql.sh.tpl new file mode 100755 index 000000000..7b833e385 --- /dev/null +++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl @@ -0,0 +1,249 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#set -x +export PGPASSFILE=/etc/postgresql/admin_user.conf +ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive +RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore +POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1) +LIST_OPTIONS=(list_archives list_databases) +ARGS=("$@") +PSQL="psql -U $POSTGRESQL_BACKUP_USER -h $POSTGRESQL_HOST" + +usage() { + echo "Usage:" + echo "$0 options" + echo "=============================" + echo "options: " + echo "list_archives" + echo "list_databases archive_filename" + echo "restore archive_filename [DB_NAME or ALL/all]" +} + +#Delete file +delete_files() { + files_to_delete=("$@") + for f in "${files_to_delete[@]}" + do + if [ -f $f ] + then + echo "Deleting file $f." + rm -rf $f + fi + done +} + +#Extract Single Database SQL Dump from pg_dumpall dump file +extract_single_db_dump() { + sed "/connect.*$2/,\$!d" $1 | sed "/PostgreSQL database dump complete/,\$d" > \ + ${RESTORE_DIR}/$2.sql +} + +#Display all archives +list_archives() { + if [ -d ${ARCHIVE_DIR} ] + then + archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print) + echo "All Archives" + echo "==================================" + for archive in $archives + do + echo $archive | cut -d '/' -f 8 + done + exit 0 + else + echo "Archive directory is not available." + exit 1 + fi +} + +#Return all databases from an archive +get_databases() { + archive_file=$1 + if [ -e ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -e ${RESTORE_DIR}/postgres.all.sql ] + then + DBS=$( grep 'CREATE DATABASE' ${RESTORE_DIR}/postgres.all.sql | awk '{ print $3 }' ) + else + DBS=" " + fi + else + DBS=" " + fi +} + +#Display all databases from an archive +list_databases() { + archive_file=$1 + get_databases $archive_file + #echo $DBS + if [ -n "$DBS" ] + then + echo " " + echo "Databases in the archive $archive_file" + echo "=================================================================" + for db in $DBS + do + echo $db + done + else + echo "There is no database in the archive." + fi + +} + +#Restore a single database dump from pg_dumpall dump. +restore_single_db() { + single_db_name=$1 + if [ -z "$single_db_name" ] + then + usage + exit 1 + fi + if [ -f ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -f ${RESTORE_DIR}/postgres.all.sql ] + then + extract_single_db_dump ${RESTORE_DIR}/postgres.all.sql $single_db_name + if [[ -f ${RESTORE_DIR}/${single_db_name}.sql && -s ${RESTORE_DIR}/${single_db_name}.sql ]] + then + $PSQL -d $single_db_name -f ${RESTORE_DIR}/${single_db_name}.sql 2>dbrestore.log + if [ "$?" -eq 0 ] + then + echo "Database Restore Successful." + else + echo "Database Restore Failed." + fi + else + echo "Database Dump For $single_db_name is empty or not available." + fi + else + echo "Database file for dump_all not available to restore from" + fi + else + echo "Archive does not exist" + fi +} + +#Restore all the databases +restore_all_dbs() { + if [ -f ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -f ${RESTORE_DIR}/postgres.all.sql ] + then + $PSQL postgres -f ${RESTORE_DIR}/postgres.all.sql 2>dbrestore.log + if [ "$?" -eq 0 ] + then + echo "Database Restore successful." + else + echo "Database Restore failed." + fi + else + echo "There is no database file available to restore from" + fi + else + echo "Archive does not exist" + fi +} + + +is_Option() { + opts=$1 + param=$2 + find=0 + for opt in $opts + do + if [ "$opt" == "$param" ] + then + find=1 + fi + done + echo $find +} + +#Main +#Create Restore Directory +mkdir -p $RESTORE_DIR +if [ ${#ARGS[@]} -gt 3 ] +then + usage + exit +elif [ ${#ARGS[@]} -eq 1 ] +then + if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ] + then + ${ARGS[0]} + exit + else + usage + exit + fi +elif [ ${#ARGS[@]} -eq 2 ] +then + if [ "${ARGS[0]}" == "list_databases" ] + then + list_databases ${ARGS[1]} + exit 0 + else + usage + exit + fi +elif [ ${#ARGS[@]} -eq 3 ] +then + if [ "${ARGS[0]}" != "restore" ] + then + usage + exit 1 + else + if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ] + then + #Get all the databases in that archive + get_databases ${ARGS[1]} + #check if the requested database is available in the archive + if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ] + then + echo "Restoring Database ${ARGS[2]} And Grants" + restore_single_db ${ARGS[2]} + echo "Tail dbrestore.log for restore log." + exit 0 + elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ] + then + echo "Restoring All The Database." + restore_all_dbs + echo "Tail dbrestore.log for restore log." + exit 0 + else + echo "There is no database with that name" + fi + else + echo "Archive file not found" + fi + fi +else + usage + exit +fi + diff --git a/postgresql/templates/configmap-bin.yaml b/postgresql/templates/configmap-bin.yaml index 4feb64229..64011fa00 100644 --- a/postgresql/templates/configmap-bin.yaml +++ b/postgresql/templates/configmap-bin.yaml @@ -33,4 +33,8 @@ data: {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} db_test.sh: | {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.conf.backup.enabled }} + backup_postgresql.sh: | +{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} {{- end }} diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml new file mode 100644 index 000000000..32d716815 --- /dev/null +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -0,0 +1,109 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_postgresql_backup }} +{{- $envAll := . }} + +{{- $serviceAccountName := "postgresql-backup" }} +{{ tuple $envAll "postgresql_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: postgresql-backup + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.backup_postgresql.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.backup_postgresql.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "postgresql-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + containers: + - command: + - /tmp/backup_postgresql.sh + env: + - name: POSTGRESQL_BACKUP_PASSWORD + valueFrom: + secretKeyRef: + key: POSTGRES_PASSWORD + name: postgresql-admin + - name: POSTGRESQL_BACKUP_USER + valueFrom: + secretKeyRef: + key: POSTGRES_USER + name: postgresql-admin + - name: POSTGRESQL_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path }} + - name: POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS + value: {{ .Values.conf.backup.pg_dumpall_options }} + - name: POSTGRESQL_BACKUP_DAYS_TO_KEEP + value: "{{ .Values.conf.backup.days_of_backup_to_keep }}" + - name: POSTGRESQL_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{ tuple $envAll "postgresql_backup" | include "helm-toolkit.snippets.image" | indent 12 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }} + name: postgresql-backup + volumeMounts: + - mountPath: /tmp/backup_postgresql.sh + name: postgresql-bin + readOnly: true + subPath: backup_postgresql.sh + - mountPath: {{ .Values.conf.backup.base_path }} + name: postgresql-backup-dir + - name: postgresql-secrets + mountPath: /etc/postgresql/admin_user.conf + subPath: admin_user.conf + readOnly: true + restartPolicy: OnFailure + securityContext: {} + serviceAccount: {{ $serviceAccountName }} + serviceAccountName: {{ $serviceAccountName }} + volumes: + - name: postgresql-secrets + secret: + secretName: postgresql-secrets + defaultMode: 0600 + - configMap: + defaultMode: 365 + name: postgresql-bin + name: postgresql-bin + {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} + - name: postgresql-backup-dir + persistentVolumeClaim: + claimName: postgresql-backup-data + {{- else }} + - hostPath: + path: {{ .Values.conf.backup.base_path }} + type: DirectoryOrCreate + name: postgresql-backup-dir + {{- end }} +{{- end }} diff --git a/postgresql/templates/postgresql-backup-pvc.yaml b/postgresql/templates/postgresql-backup-pvc.yaml new file mode 100644 index 000000000..8d0fe525f --- /dev/null +++ b/postgresql/templates/postgresql-backup-pvc.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgresql-backup-data +spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.backup.size }} + storageClassName: {{ .Values.volume.backup.class_name }} +{{- end }} + diff --git a/postgresql/templates/secrets-etc.yaml b/postgresql/templates/secrets-etc.yaml new file mode 100644 index 000000000..c1c9b51cd --- /dev/null +++ b/postgresql/templates/secrets-etc.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-secrets +type: Opaque +data: + admin_user.conf: {{ tuple "secrets/_admin_user.conf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} + diff --git a/postgresql/templates/secrets/_admin_user.conf.tpl b/postgresql/templates/secrets/_admin_user.conf.tpl new file mode 100644 index 000000000..9b945ea67 --- /dev/null +++ b/postgresql/templates/secrets/_admin_user.conf.tpl @@ -0,0 +1,17 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}:*:*:{{ .Values.endpoints.postgresql.auth.admin.username }}:{{ .Values.endpoints.postgresql.auth.admin.password }} diff --git a/postgresql/values.yaml b/postgresql/values.yaml index 1c1850702..76b05514c 100644 --- a/postgresql/values.yaml +++ b/postgresql/values.yaml @@ -86,6 +86,13 @@ pod: requests: memory: "128Mi" cpu: "100m" + postgresql_backup: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" # using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/ images: @@ -95,6 +102,7 @@ images: image_repo_sync: docker.io/docker:17.07.0 prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6 prometheus_postgresql_exporter_create_user: "docker.io/postgres:9.5" + postgresql_backup: "docker.io/postgres:9.5" pull_policy: "IfNotPresent" local_registry: active: false @@ -124,6 +132,9 @@ labels: prometheus_postgresql_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled dependencies: dynamic: @@ -155,6 +166,10 @@ dependencies: service: postgresql jobs: - prometheus-postgresql-exporter-create-user + postgresql-backup: + services: + - endpoint: internal + service: postgresql monitoring: prometheus: @@ -162,11 +177,29 @@ monitoring: postgresql_exporter: scrape: true +volume: + backup: + enabled: true + class_name: general + size: 5Gi + +jobs: + backup_postgresql: + cron: "0 0 * * *" + history: + success: 3 + failed: 1 + conf: debug: false postgresql: max_connections: 100 shared_buffers: 128MB + backup: + enabled: true + base_path: /var/backup + days_of_backup_to_keep: 3 + pg_dumpall_options: null secrets: postgresql: @@ -222,8 +255,11 @@ manifests: configmap_bin: true job_image_repo_sync: true secret_admin: true + secret_etc: true service: true statefulset: true + cron_job_postgresql_backup: false + pvc_backup: false monitoring: prometheus: configmap_bin: true