Add mariadb backup capability to mariadb chart

* backup script for mariadb
 * restore script for mariadb
 * cronjob to control backup automation for mariadb
 * add parameters to values.yaml

Change-Id: Ib0b6744f519ea1919507df0c2a50b45b27ac86ab

mariadb/templates/bin/_backup_mariadb.sh.tpl

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -x
+BACKUPS_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/current
+ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
+
+MYSQL="mysql \
+   --defaults-file=/etc/mysql/admin_user.cnf \
+   --host=$MARIADB_SERVER_SERVICE_HOST \
+   --connect-timeout 10"
+
+MYSQLDUMP="mysqldump \
+   --defaults-file=/etc/mysql/admin_user.cnf \
+   --host=$MARIADB_SERVER_SERVICE_HOST"
+
+delete_files() {
+  files_to_delete=("$@")
+  for f in "${files_to_delete[@]}"
+  do
+    if [ -f $f ]
+    then
+      echo "Deleting file $f."
+      rm -rf $f
+    fi
+  done
+}
+
+days_difference() {
+  archive_date=$( date --date="$1" +%s )
+  if [ "$?" -ne 0 ]
+  then
+    day_delta=0
+  fi
+  current_date=$( date +%s )
+  date_delta=$(($current_date-$archive_date))
+  if [ "$date_delta" -lt 0 ]
+  then
+    day_delta=0
+  else
+    day_delta=$(($date_delta/86400))
+  fi
+  echo $day_delta
+}
+
+DBNAME=( $($MYSQL --silent --skip-column-names -e \
+   "show databases;" | \
+   egrep -vi 'information_schema|performance_schema|mysql') )
+
+#check if there is a database to backup, otherwise exit
+if [[ -z "${DBNAME// }" ]]
+then
+  echo "There is no database to backup"
+  exit 0
+fi
+
+#Create archive and backup directories.
+mkdir -p $BACKUPS_DIR $ARCHIVE_DIR
+
+#Create a list of Databases
+printf "%s\n" "${DBNAME[@]}" > $BACKUPS_DIR/db.list
+
+#Retrieve and create the GRANT files per DB
+for db in "${DBNAME[@]}"
+do
+  echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \
+        from mysql.db where ucase(db)=ucase('$db');") | \
+        $MYSQL --silent --skip-column-names 2>grant_err.log > $BACKUPS_DIR/${db}_grant.sql
+  if [ "$?" -eq 0 ]
+  then
+    sed -i 's/$/;/' $BACKUPS_DIR/${db}_grant.sql
+  else
+    cat grant_err.log
+  fi
+done
+
+#Dumping the database
+#DATE=$(date +"%Y_%m_%d_%H_%M_%S")
+DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
+$MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${DBNAME[@]}"  \
+          > $BACKUPS_DIR/mariadb.all.sql 2>dberror.log
+if [[ $? -eq 0 && -s $BACKUPS_DIR/mariadb.all.sql ]]
+then
+  #Archive the current db files
+  pushd $BACKUPS_DIR 1>/dev/null
+  tar zcvf $ARCHIVE_DIR/mariadb.all.${DATE}.tar.gz *
+  ARCHIVE_RET=$?
+  popd 1>/dev/null
+else
+  #TODO: This can be convert into mail alert of alert send to a monitoring system
+  echo "Backup failed and need attention."
+  cat dberror.log
+  exit 1
+fi
+
+#Remove the current backup
+if [ -d $BACKUPS_DIR ]
+then
+  rm -rf $BACKUPS_DIR/*.sql
+fi
+
+#Only delete the old archive after a successful archive
+if [ $ARCHIVE_RET -eq 0 ]
+  then
+    if [ "$MARIADB_BACKUP_DAYS_TO_KEEP" -gt 0 ]
+    then
+      echo "Deleting backups older than $MARIADB_BACKUP_DAYS_TO_KEEP days"
+      if [ -d $ARCHIVE_DIR ]
+      then
+        for archive_file in $(ls -1 $ARCHIVE_DIR/*.gz)
+        do
+          archive_date=$( echo $archive_file | awk -F/ '{print $NF}' | cut -d'.' -f 3)
+          if [ "$(days_difference $archive_date)" -gt "$MARIADB_BACKUP_DAYS_TO_KEEP" ]
+          then
+            rm -rf $archive_file
+          fi
+        done
+      fi
+    fi
+fi

mariadb/templates/bin/_restore_mariadb.sh.tpl

@@ -0,0 +1,269 @@
+#!/bin/bash
+
+# Copyright 2018 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/archive
+RESTORE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/mariadb/restore
+ARGS=("$@")
+LIST_OPTIONS=(list_archives list_databases)
+
+#Create Restore Directory
+mkdir -p $RESTORE_DIR
+
+MYSQL="mysql \
+   --defaults-file=/etc/mysql/admin_user.cnf \
+   --host=$MARIADB_SERVER_SERVICE_HOST \
+   --connect-timeout 10"
+
+#Delete file
+delete_files() {
+  files_to_delete=("$@")
+  for f in "${files_to_delete[@]}"
+  do
+    if [ -f $f ]
+    then
+      rm -rf $f
+    fi
+  done
+}
+
+#Display all archives
+list_archives() {
+  if [ -d ${ARCHIVE_DIR} ]
+  then
+    archives=$(find ${ARCHIVE_DIR}/ -iname "*.gz" -print)
+    echo "All Archives"
+    echo "=================================="
+    for archive in $archives
+    do
+      echo $archive | cut -d '/' -f 8
+    done
+    exit 0
+  else
+    echo "Archive directory is not available."
+    exit 1
+  fi
+}
+
+#Return all database from an archive
+get_databases() {
+  archive_file=$1
+  if [ -e ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -e ${RESTORE_DIR}/db.list ]
+    then
+      DBS=$(cat ${RESTORE_DIR}/db.list )
+    else
+      DBS=" "
+    fi
+  else
+    DBS=" "
+  fi
+}
+
+#Display all database from an archive
+list_databases() {
+  archive_file=$1
+  get_databases $archive_file
+  #echo $DBS
+  if [ -n "$DBS" ]
+  then
+    echo " "
+    echo "Databases in the archive $archive_file"
+    echo "================================================================="
+    for db in $DBS
+    do
+      echo $db
+    done
+  fi
+
+}
+
+#Restore a single database
+restore_single_db() {
+  single_db_name=$1
+  if [ -z "$single_db_name" ]
+  then
+    usage
+    exit 1
+  fi
+  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
+    then
+      $MYSQL --one-database $single_db_name < ${RESTORE_DIR}/mariadb.all.sql
+      if [ "$?" -eq 0 ]
+      then
+        echo "Database $single_db_name Restore successful."
+      else
+        echo "Database $single_db_name Restore failed."
+      fi
+      if [ -f ${RESTORE_DIR}/${single_db_name}_grant.sql ]
+      then
+        $MYSQL < ${RESTORE_DIR}/${single_db_name}_grant.sql
+        if [ "$?" -eq 0 ]
+        then
+          echo "Database $single_db_name Permission Restore successful."
+        else
+          echo "Database $single_db_name Permission Restore failed."
+        fi
+      else
+        echo "There is no permission file available for $single_db_name"
+      fi
+    else
+      echo "There is no database file available to restore from"
+    fi
+  else
+    echo "Archive does not exist"
+  fi
+}
+
+#Restore all the databases
+restore_all_dbs() {
+  if [ -f ${ARCHIVE_DIR}/${archive_file} ]
+  then
+    files_to_purge=$(find $RESTORE_DIR/ -iname "*.sql" -print)
+    delete_files $files_to_purge
+    tar zxvf ${ARCHIVE_DIR}/${archive_file} -C ${RESTORE_DIR} 1>/dev/null
+    if [ -f ${RESTORE_DIR}/mariadb.all.sql ]
+    then
+      $MYSQL < ${RESTORE_DIR}/mariadb.all.sql
+      if [ "$?" -eq 0 ]
+      then
+        echo "Databases $( echo $DBS | tr -d '\n') Restore successful."
+      else
+        echo "Databases $( echo $DBS | tr -d '\n') Restore failed."
+      fi
+      if [ -n "$DBS" ]
+      then
+        for db in $DBS
+        do
+          if [ -f ${RESTORE_DIR}/${db}_grant.sql ]
+          then
+            $MYSQL < ${RESTORE_DIR}/${db}_grant.sql
+            if [ "$?" -eq 0 ]
+            then
+              echo "Database $db Permission Restore successful."
+            else
+              echo "Database $db Permission Restore failed."
+            fi
+          else
+            echo "There is no permission file available for $db"
+          fi
+        done
+    else
+      echo "There is no database file available to restore from"
+    fi
+  else
+    echo "Archive does not exist"
+  fi
+ fi
+}
+
+usage() {
+  echo "Usage:"
+  echo "$0 options"
+  echo "============================="
+  echo "options: "
+  echo "list_archives"
+  echo "list_databases archive_filename"
+  echo "restore archive_filename [DB_NAME or ALL/all]"
+}
+
+is_Option() {
+  opts=$1
+  param=$2
+  find=0
+  for opt in $opts
+  do
+    if [ "$opt" == "$param" ]
+    then
+      find=1
+    fi
+  done
+  echo $find
+}
+
+#Main
+if [ ${#ARGS[@]} -gt 3 ]
+then
+  usage
+  exit
+elif [ ${#ARGS[@]} -eq 1 ]
+then
+  if [ $(is_Option "$LIST_OPTIONS" ${ARGS[0]}) -eq 1 ]
+  then
+    ${ARGS[0]}
+    exit
+  else
+    usage
+    exit
+  fi
+elif [ ${#ARGS[@]} -eq 2 ]
+then
+  if [ "${ARGS[0]}" == "list_databases" ]
+  then
+    list_databases ${ARGS[1]}
+    exit 0
+  else
+    usage
+    exit
+  fi
+elif [ ${#ARGS[@]} -eq 3 ]
+then
+  if [ "${ARGS[0]}" != "restore" ]
+  then
+    usage
+    exit 1
+  else
+    if [ -f ${ARCHIVE_DIR}/${ARGS[1]} ]
+    then
+      #Get all the databases in that archive
+      get_databases ${ARGS[1]}
+      #check if the requested database is available in the archive
+      if [ $(is_Option "$DBS" ${ARGS[2]}) -eq 1 ]
+      then
+        echo "Restoring Database ${ARGS[2]} And Grants"
+        echo "Creating Database ${ARGS[2]} if it does not exist"
+        $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`${ARGS[2]}\`"
+        restore_single_db ${ARGS[2]}
+        exit 0
+      elif [ "$( echo ${ARGS[2]} | tr '[a-z]' '[A-Z]')" == "ALL" ]
+      then
+        echo "Restoring All The Database."
+        echo "Creating Database if it does not exist"
+        for db in $DBS
+        do
+          $MYSQL -e "CREATE DATABASE IF NOT EXISTS \`$db\`"
+        done
+        restore_all_dbs
+        exit 0
+      else
+        echo "Database ${ARGS[2]} does not exist."
+      fi
+    else
+      echo "Archive file not found"
+    fi
+  fi
+else
+  usage
+  exit
+fi

mariadb/templates/configmap-bin.yaml

@@ -36,4 +36,10 @@ data:
 {{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   stop.sh: |
 {{ tuple "bin/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- if .Values.conf.backup.enabled }}
+  backup_mariadb.sh: |
+{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  restore_mariadb.sh: |
+{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
 {{- end }}

mariadb/templates/cron-job-backup-mariadb.yaml

@@ -0,0 +1,98 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_mariadb_backup }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "mariadb-backup" }}
+{{ tuple $envAll "mariadb_account" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: mariadb-backup
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  schedule: {{ .Values.jobs.backup_mariadb.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.backup_mariadb.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          containers:
+          - command:
+            - /tmp/backup_mariadb.sh
+            env:
+            - name: MARIADB_BACKUP_BASE_DIR
+              value: {{ .Values.conf.backup.base_path }}
+            - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS
+              value: {{ .Values.conf.backup.mysqldump_options }}
+            - name: MARIADB_BACKUP_DAYS_TO_KEEP
+              value: "{{ .Values.conf.backup.days_of_backup_to_keep }}"
+            - name: MARIADB_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 12 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 12 }}
+            name: mariadb-backup
+            volumeMounts:
+            - mountPath: /tmp/backup_mariadb.sh
+              name: mariadb-bin
+              readOnly: true
+              subPath: backup_mariadb.sh
+            - mountPath: {{ .Values.conf.backup.base_path }}
+              name: mariadb-backup-dir
+            - name: mariadb-secrets
+              mountPath: /etc/mysql/admin_user.cnf
+              subPath: admin_user.cnf
+              readOnly: true
+          restartPolicy: OnFailure
+          serviceAccount: {{ $serviceAccountName }}
+          serviceAccountName: {{ $serviceAccountName }}
+          volumes:
+          - name: mariadb-secrets
+            secret:
+              secretName: mariadb-secrets
+              defaultMode: 384
+          - configMap:
+              defaultMode: 365
+              name: mariadb-bin
+            name: mariadb-bin
+          {{- if and .Values.volume.backup.enabled  .Values.manifests.pvc_backup  }}
+          - name: mariadb-backup-dir
+            persistentVolumeClaim:
+              claimName: mariadb-backup-data
+          {{- else }}
+          - hostPath:
+              path: {{ .Values.conf.backup.base_path }}
+              type: DirectoryOrCreate
+            name: mariadb-backup-dir
+          {{- end }}
+{{- end }}

mariadb/templates/mariadb-backup-pvc.yaml

@@ -0,0 +1,30 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: mariadb-backup-data
+spec:
+  accessModes: [ "ReadWriteOnce" ]
+  resources:
+    requests:
+      storage: {{ .Values.volume.backup.size }}
+  storageClassName: {{ .Values.volume.backup.class_name }}
+{{- end }}
+

mariadb/values.yaml

@@ -29,6 +29,7 @@ images:
     prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     image_repo_sync: docker.io/docker:17.07.0
+    mariadb_backup: docker.io/openstackhelm/mariadb:10.2.18
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -49,6 +50,9 @@ labels:
   error_server:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 pod:
   user:
@@ -119,11 +123,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
-
+      mariadb_backup:
-jobs:
+        requests:
-  exporter_create_sql_user:
+          memory: "128Mi"
-    backoffLimit: 87600
+          cpu: "100m"
-    activeDeadlineSeconds: 3600
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 dependencies:
   dynamic:
@@ -165,6 +171,10 @@ dependencies:
       services:
         - endpoint: internal
           service: local_image_registry
+    mariadb_backup:
+      services:
+        - endpoint: internal
+          service: oslo_db
 
 force_bootstrap: false
 
@@ -173,9 +183,30 @@ volume:
   enabled: true
   class_name: general
   size: 5Gi
+  backup:
+    enabled: true
+    class_name: general
+    size: 5Gi
+
+jobs:
+  exporter_create_sql_user:
+    backoffLimit: 87600
+    activeDeadlineSeconds: 3600
+  backup_mariadb:
+    cron: "0 0 * * *"
+    history:
+      success: 3
+      failed: 1
 
 conf:
   ingress: null
+  backup:
+    enabled: true
+    base_path: /var/backup
+    mysqldump_options: >
+      --single-transaction --quick --skip-opt
+      --add-drop-database --add-drop-table --databases
+    days_of_backup_to_keep: 3
 
 database:
   config_override: null
@@ -292,6 +323,8 @@ manifests:
   deployment_error: true
   deployment_ingress: true
   job_image_repo_sync: true
+  cron_job_mariadb_backup: false
+  pvc_backup: false
   monitoring:
     prometheus:
       configmap_bin: true