[backups] Mariadb backups improvements

This PS removes mariadb-verify-server sidecar container from
mariadb-backup cronjob in order to make backup process more resilient.

Change-Id: I2517c2de435ead34397ca0483610f511c8035bdf

helm-toolkit/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.57
+version: 0.2.58
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:

helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl

@@ -95,7 +95,7 @@ log_backup_error_exit() {
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +104,7 @@ log_verify_backup_exit() {
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 

mariadb-backup/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.14
 description: OpenStack-Helm MariaDB backups
 name: mariadb-backup
-version: 0.0.2
+version: 0.0.3
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb-backup/templates/bin/_backup_mariadb.sh.tpl

@@ -441,8 +441,8 @@ verify_databases_backup_archives() {
   export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
   export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
   export MYSQL_OPTS="--silent --skip-column-names"
-  export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
+  export MYSQL_LIVE="mysql ${MYSQL_OPTS}"
-  export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
+  export MYSQL_LOCAL_OPTS=""
   export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
   export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
   export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"

mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl

@@ -25,4 +25,5 @@ log () {
 }
 
 log "Starting Mariadb server for backup verification..."
-MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
+mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1
+MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1

mariadb-backup/templates/cron-job-backup-mariadb.yaml

@@ -50,12 +50,12 @@ spec:
 {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
         spec:
 {{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
-          restartPolicy: OnFailure
           serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
           shareProcessNamespace: true
-{{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
+{{- if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
-{{ end }}
+{{- end }}
 {{- if $envAll.Values.pod.affinity }}
 {{- if $envAll.Values.pod.affinity.mariadb_backup }}
           affinity:
@@ -104,8 +104,8 @@ spec:
               args:
                 - -c
                 - >-
-                    /tmp/backup_mariadb.sh;
+                    ( /tmp/start_verification_server.sh ) &
-                    /usr/bin/pkill mysqld
+                    /tmp/backup_mariadb.sh
               env:
                 - name: MARIADB_BACKUP_BASE_DIR
                   value: {{ .Values.conf.backup.base_path | quote }}
@@ -156,47 +156,15 @@ spec:
                   mountPath: /etc/mysql/admin_user.cnf
                   subPath: admin_user.cnf
                   readOnly: true
-{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
-            - name: mariadb-verify-server
-{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
-{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
-{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
-              env:
-                {{- if $envAll.Values.manifests.certificates }}
-                - name: MARIADB_X509
-                  value: "REQUIRE X509"
-                {{- end }}
-                - name: MYSQL_HISTFILE
-                  value: /dev/null
-                - name: MARIADB_BACKUP_BASE_DIR
-                  value: {{ .Values.conf.backup.base_path | quote }}
-              ports:
-                - name: mysql
-                  protocol: TCP
-                  containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-              command:
-                - /tmp/start_verification_server.sh
-              volumeMounts:
-                - name: pod-tmp
-                  mountPath: /tmp
-                - name: var-run
-                  mountPath: /var/run/mysqld
-                - name: mycnfd
-                  mountPath: /etc/mysql/conf.d
-                - name: mariadb-backup-etc
-                  mountPath: /etc/mysql/my.cnf
-                  subPath: my.cnf
-                  readOnly: true
-                - name: mariadb-backup-secrets
-                  mountPath: /etc/mysql/admin_user.cnf
-                  subPath: admin_user.cnf
-                  readOnly: true
-                - name: mysql-data
-                  mountPath: /var/lib/mysql
                 - name: mariadb-backup-bin
                   mountPath: /tmp/start_verification_server.sh
                   readOnly: true
                   subPath: start_verification_server.sh
+                - name: mysql-data
+                  mountPath: /var/lib/mysql
+                - name: var-run
+                  mountPath: /run/mysqld
+{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
           volumes:
             - name: pod-tmp
               emptyDir: {}

mariadb-backup/values.yaml

@@ -73,10 +73,6 @@ pod:
           runAsUser: 65534
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
-        mariadb_verify_server:
-          runAsUser: 65534
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
     tests:
       pod:
         runAsUser: 999

mariadb/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.36
+version: 0.2.37
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb/templates/bin/_backup_mariadb.sh.tpl

@@ -441,8 +441,8 @@ verify_databases_backup_archives() {
   export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
   export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
   export MYSQL_OPTS="--silent --skip-column-names"
-  export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
+  export MYSQL_LIVE="mysql ${MYSQL_OPTS}"
-  export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
+  export MYSQL_LOCAL_OPTS=""
   export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
   export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
   export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"

mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl

@@ -25,4 +25,5 @@ log () {
 }
 
 log "Starting Mariadb server for backup verification..."
-MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
+mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1
+MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1

mariadb/templates/cron-job-backup-mariadb.yaml

@@ -104,8 +104,8 @@ spec:
               args:
                 - -c
                 - >-
-                    /tmp/backup_mariadb.sh;
+                    ( /tmp/start_verification_server.sh ) &
-                    /usr/bin/pkill mysqld
+                    /tmp/backup_mariadb.sh
               env:
                 - name: MARIADB_BACKUP_BASE_DIR
                   value: {{ .Values.conf.backup.base_path | quote }}
@@ -157,46 +157,14 @@ spec:
                   subPath: admin_user.cnf
                   readOnly: true
 {{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
-            - name: mariadb-verify-server
-{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
-{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
-{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
-              env:
-                {{- if $envAll.Values.manifests.certificates }}
-                - name: MARIADB_X509
-                  value: "REQUIRE X509"
-                {{- end }}
-                - name: MYSQL_HISTFILE
-                  value: /dev/null
-                - name: MARIADB_BACKUP_BASE_DIR
-                  value: {{ .Values.conf.backup.base_path | quote }}
-              ports:
-                - name: mysql
-                  protocol: TCP
-                  containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-              command:
-                - /tmp/start_verification_server.sh
-              volumeMounts:
-                - name: pod-tmp
-                  mountPath: /tmp
-                - name: var-run
-                  mountPath: /var/run/mysqld
-                - name: mycnfd
-                  mountPath: /etc/mysql/conf.d
-                - name: mariadb-etc
-                  mountPath: /etc/mysql/my.cnf
-                  subPath: my.cnf
-                  readOnly: true
-                - name: mariadb-secrets
-                  mountPath: /etc/mysql/admin_user.cnf
-                  subPath: admin_user.cnf
-                  readOnly: true
-                - name: mysql-data
-                  mountPath: /var/lib/mysql
                 - name: mariadb-bin
                   mountPath: /tmp/start_verification_server.sh
                   readOnly: true
                   subPath: start_verification_server.sh
+                - name: mysql-data
+                  mountPath: /var/lib/mysql
+                - name: var-run
+                  mountPath: /run/mysqld
           volumes:
             - name: pod-tmp
               emptyDir: {}

mariadb/values.yaml

@@ -28,7 +28,7 @@ images:
     prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/library/docker:17.07.0
-    mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
+    mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal
     ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
     scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
   pull_policy: "IfNotPresent"
@@ -129,10 +129,6 @@ pod:
           runAsUser: 65534
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
-        mariadb_verify_server:
-          runAsUser: 65534
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
     tests:
       pod:
         runAsUser: 999

releasenotes/notes/helm-toolkit.yaml

@@ -64,4 +64,5 @@ helm-toolkit:
   - 0.2.55 Updated deprecated IngressClass annotation
   - 0.2.56 Expose S3 credentials from Rook bucket CRD secret
   - 0.2.57 Safer file removal
+  - 0.2.58 Backups verification improvements
 ...

releasenotes/notes/mariadb-backup.yaml

@@ -2,4 +2,5 @@
 mariadb-backup:
   - 0.0.1 Initial Chart
   - 0.0.2 Added staggered backups support
+  - 0.0.3 Backups verification improvements
 ...

releasenotes/notes/mariadb.yaml

@@ -52,4 +52,5 @@ mariadb:
   - 0.2.34 Uplift ingress controller image to 1.8.2
   - 0.2.35 Update apparmor override
   - 0.2.36 Added staggered backups support
+  - 0.2.37 Backups verification improvements
 ...