diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl new file mode 100644 index 000000000..ef3a6a74b --- /dev/null +++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl @@ -0,0 +1,132 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -x +BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current +ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive + +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST \ + --connect-timeout 10" + +MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST" + +delete_files() { + files_to_delete=("$@") + for f in "${files_to_delete[@]}" + do + if [ -f $f ] + then + echo "Deleting file $f." + rm -rf $f + fi + done +} + +days_difference() { + archive_date=$( date --date="$1" +%s ) + if [ "$?" -ne 0 ] + then + day_delta=0 + fi + current_date=$( date +%s ) + date_delta=$(($current_date-$archive_date)) + if [ "$date_delta" -lt 0 ] + then + day_delta=0 + else + day_delta=$(($date_delta/86400)) + fi + echo $day_delta +} + +DBNAME=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + egrep -vi 'information_schema|performance_schema|mysql') ) + +#check if there is a database to backup, otherwise exit +if [[ -z "${DBNAME// }" ]] +then + echo "There is no database to backup" + exit 0 +fi + +#Create archive and backup directories. +mkdir -p $BACKUPS_DIR $ARCHIVE_DIR + +#Create a list of Databases +printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list + +#Retrieve and create the GRANT files per DB +for db in "${DBNAME[@]}" +do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + $MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql + else + cat grant_err.log + fi +done + +#Dumping the database +#DATE=$(date +"%Y_%m_%d_%H_%M_%S") +DATE=$(date +'%Y-%m-%dT%H:%M:%SZ') +$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}" \ + > $BACKUPS_DIR/mariadb.all.sql 2>dberror.log +if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]] +then + #Archive the current db files + pushd $BACKUPS_DIR 1>/dev/null + tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz * + ARCHIVE_RET=$? + popd 1>/dev/null +else + #TODO: This can be convert into mail alert of alert send to a monitoring system + echo "Backup failed and need attention." + cat dberror.log + exit 1 +fi + +#Remove the current backup +if [ -d $BACKUPS_DIR ] +then + rm -rf $BACKUPS_DIR/*.sql +fi + +#Only delete the old archive after a successful archive +if [ $ARCHIVE_RET -eq 0 ] + then + if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ] + then + echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days" + if [ -d $ARCHIVE_DIR ] + then + for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz) + do + archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3) + if [ "$(days_difference $archive_date)" -gt "$MARIADB_BACKUP_DAYS_TO_KEEP" ] + then + rm -rf $archive_file + fi + done + fi + fi +fi diff --git a/mariadb/templates/bin/_restore_mariadb.sh.tpl b/mariadb/templates/bin/_restore_mariadb.sh.tpl new file mode 100755 index 000000000..51b801c33 --- /dev/null +++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl @@ -0,0 +1,269 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive +RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore +ARGS=("$@") +LIST_OPTIONS=(list_archives list_databases) + +#Create Restore Directory +mkdir -p $RESTORE_DIR + +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST \ + --connect-timeout 10" + +#Delete file +delete_files() { + files_to_delete=("$@") + for f in "${files_to_delete[@]}" + do + if [ -f $f ] + then + rm -rf $f + fi + done +} + +#Display all archives +list_archives() { + if [ -d ${ARCHIVE_DIR} ] + then + archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print) + echo "All Archives" + echo "==================================" + for archive in $archives + do + echo $archive | cut -d '/' -f 8 + done + exit 0 + else + echo "Archive directory is not available." + exit 1 + fi +} + +#Return all database from an archive +get_databases() { + archive_file=$1 + if [ -e ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -e ${RESTORE_DIR}/db.list ] + then + DBS=$(cat ${RESTORE_DIR}/db.list ) + else + DBS=" " + fi + else + DBS=" " + fi +} + +#Display all database from an archive +list_databases() { + archive_file=$1 + get_databases $archive_file + #echo $DBS + if [ -n "$DBS" ] + then + echo " " + echo "Databases in the archive $archive_file" + echo "=================================================================" + for db in $DBS + do + echo $db + done + fi + +} + +#Restore a single database +restore_single_db() { + single_db_name=$1 + if [ -z "$single_db_name" ] + then + usage + exit 1 + fi + if [ -f ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + then + $MYSQL --one-database $single_db_name < ${RESTORE_DIR}/mariadb.all.sql + if [ "$?" -eq 0 ] + then + echo "Database $single_db_name Restore successful." + else + echo "Database $single_db_name Restore failed." + fi + if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ] + then + $MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql + if [ "$?" -eq 0 ] + then + echo "Database $single_db_name Permission Restore successful." + else + echo "Database $single_db_name Permission Restore failed." + fi + else + echo "There is no permission file available for $single_db_name" + fi + else + echo "There is no database file available to restore from" + fi + else + echo "Archive does not exist" + fi +} + +#Restore all the databases +restore_all_dbs() { + if [ -f ${ARCHIVE_DIR}/${archive_file} ] + then + files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print) + delete_files $files_to_purge + tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null + if [ -f ${RESTORE_DIR}/mariadb.all.sql ] + then + $MYSQL < ${RESTORE_DIR}/mariadb.all.sql + if [ "$?" -eq 0 ] + then + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." + else + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + fi + if [ -n "$DBS" ] + then + for db in $DBS + do + if [ -f ${RESTORE_DIR}/${db}_grant.sql ] + then + $MYSQL < ${RESTORE_DIR}/${db}_grant.sql + if [ "$?" -eq 0 ] + then + echo "Database $db Permission Restore successful." + else + echo "Database $db Permission Restore failed." + fi + else + echo "There is no permission file available for $db" + fi + done + else + echo "There is no database file available to restore from" + fi + else + echo "Archive does not exist" + fi + fi +} + +usage() { + echo "Usage:" + echo "$0 options" + echo "=============================" + echo "options: " + echo "list_archives" + echo "list_databases archive_filename" + echo "restore archive_filename [DB_NAME or ALL/all]" +} + +is_Option() { + opts=$1 + param=$2 + find=0 + for opt in $opts + do + if [ "$opt" == "$param" ] + then + find=1 + fi + done + echo $find +} + +#Main +if [ ${#ARGS[@]} -gt 3 ] +then + usage + exit +elif [ ${#ARGS[@]} -eq 1 ] +then + if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ] + then + ${ARGS[0]} + exit + else + usage + exit + fi +elif [ ${#ARGS[@]} -eq 2 ] +then + if [ "${ARGS[0]}" == "list_databases" ] + then + list_databases ${ARGS[1]} + exit 0 + else + usage + exit + fi +elif [ ${#ARGS[@]} -eq 3 ] +then + if [ "${ARGS[0]}" != "restore" ] + then + usage + exit 1 + else + if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ] + then + #Get all the databases in that archive + get_databases ${ARGS[1]} + #check if the requested database is available in the archive + if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ] + then + echo "Restoring Database ${ARGS[2]} And Grants" + echo "Creating Database ${ARGS[2]} if it does not exist" + $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`" + restore_single_db ${ARGS[2]} + exit 0 + elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ] + then + echo "Restoring All The Database." + echo "Creating Database if it does not exist" + for db in $DBS + do + $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`" + done + restore_all_dbs + exit 0 + else + echo "Database ${ARGS[2]} does not exist." + fi + else + echo "Archive file not found" + fi + fi +else + usage + exit +fi diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml index 62fb5a5bb..1ce90a52f 100644 --- a/mariadb/templates/configmap-bin.yaml +++ b/mariadb/templates/configmap-bin.yaml @@ -36,4 +36,10 @@ data: {{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} stop.sh: | {{ tuple "bin/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.conf.backup.enabled }} + backup_mariadb.sh: | +{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + restore_mariadb.sh: | +{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} {{- end }} diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml new file mode 100644 index 000000000..b438af3e4 --- /dev/null +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -0,0 +1,98 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_mariadb_backup }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-backup" }} +{{ tuple $envAll "mariadb_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: mariadb-backup + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.backup_mariadb.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + containers: + - command: + - /tmp/backup_mariadb.sh + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path }} + - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS + value: {{ .Values.conf.backup.mysqldump_options }} + - name: MARIADB_BACKUP_DAYS_TO_KEEP + value: "{{ .Values.conf.backup.days_of_backup_to_keep }}" + - name: MARIADB_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 12 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }} + name: mariadb-backup + volumeMounts: + - mountPath: /tmp/backup_mariadb.sh + name: mariadb-bin + readOnly: true + subPath: backup_mariadb.sh + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + restartPolicy: OnFailure + serviceAccount: {{ $serviceAccountName }} + serviceAccountName: {{ $serviceAccountName }} + volumes: + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 384 + - configMap: + defaultMode: 365 + name: mariadb-bin + name: mariadb-bin + {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} + - name: mariadb-backup-dir + persistentVolumeClaim: + claimName: mariadb-backup-data + {{- else }} + - hostPath: + path: {{ .Values.conf.backup.base_path }} + type: DirectoryOrCreate + name: mariadb-backup-dir + {{- end }} +{{- end }} diff --git a/mariadb/templates/mariadb-backup-pvc.yaml b/mariadb/templates/mariadb-backup-pvc.yaml new file mode 100644 index 000000000..f7e5883ab --- /dev/null +++ b/mariadb/templates/mariadb-backup-pvc.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-backup-data +spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.backup.size }} + storageClassName: {{ .Values.volume.backup.class_name }} +{{- end }} + diff --git a/mariadb/values.yaml b/mariadb/values.yaml index 046ed9cbb..9bfbe6959 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -29,6 +29,7 @@ images: prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 image_repo_sync: docker.io/docker:17.07.0 + mariadb_backup: docker.io/openstackhelm/mariadb:10.2.18 pull_policy: "IfNotPresent" local_registry: active: false @@ -49,6 +50,9 @@ labels: error_server: node_selector_key: openstack-control-plane node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled pod: user: @@ -119,11 +123,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" - -jobs: - exporter_create_sql_user: - backoffLimit: 87600 - activeDeadlineSeconds: 3600 + mariadb_backup: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" dependencies: dynamic: @@ -165,6 +171,10 @@ dependencies: services: - endpoint: internal service: local_image_registry + mariadb_backup: + services: + - endpoint: internal + service: oslo_db force_bootstrap: false @@ -173,9 +183,30 @@ volume: enabled: true class_name: general size: 5Gi + backup: + enabled: true + class_name: general + size: 5Gi + +jobs: + exporter_create_sql_user: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + backup_mariadb: + cron: "0 0 * * *" + history: + success: 3 + failed: 1 conf: ingress: null + backup: + enabled: true + base_path: /var/backup + mysqldump_options: > + --single-transaction --quick --skip-opt + --add-drop-database --add-drop-table --databases + days_of_backup_to_keep: 3 database: config_override: null @@ -292,6 +323,8 @@ manifests: deployment_error: true deployment_ingress: true job_image_repo_sync: true + cron_job_mariadb_backup: false + pvc_backup: false monitoring: prometheus: configmap_bin: true