[backups] Mariadb backups improvements

This PS removes mariadb-verify-server sidecar container from mariadb-backup cronjob in order to make backup process more resilient. Change-Id: I2517c2de435ead34397ca0483610f511c8035bdf
2023-12-13 01:01:07 +00:00 · 2023-12-13 01:01:07 +00:00 · f66c924b2f
commit f66c924b2f
parent a656fc4875
15 changed files with 33 additions and 100 deletions
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.57
+version: 0.2.58
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
--- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@ -95,7 +95,7 @@ log_backup_error_exit() {
  log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
  rm -f $ERR_LOG_FILE
  rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }

 log_verify_backup_exit() {
@ -104,7 +104,7 @@ log_verify_backup_exit() {
  log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
  rm -f $ERR_LOG_FILE
  # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }


--- a/mariadb-backup/Chart.yaml
+++ b/mariadb-backup/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.14
 description: OpenStack-Helm MariaDB backups
 name: mariadb-backup
-version: 0.0.2
+version: 0.0.3
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:
--- a/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl
+++ b/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl
@ -441,8 +441,8 @@ verify_databases_backup_archives() {
  export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
  export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
  export MYSQL_OPTS="--silent --skip-column-names"
-  export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
-  export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
+  export MYSQL_LIVE="mysql ${MYSQL_OPTS}"
+  export MYSQL_LOCAL_OPTS=""
  export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
  export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
  export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"
--- a/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl
+++ b/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl
@ -25,4 +25,5 @@ log () {
 }

 log "Starting Mariadb server for backup verification..."
-MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
+mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1
+MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1
--- a/mariadb-backup/templates/cron-job-backup-mariadb.yaml
+++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml
@ -50,12 +50,12 @@ spec:
 {{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
        spec:
 {{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
-          restartPolicy: OnFailure
          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
          shareProcessNamespace: true
-{{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
+{{- if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
-{{ end }}
+{{- end }}
 {{- if $envAll.Values.pod.affinity }}
 {{- if $envAll.Values.pod.affinity.mariadb_backup }}
          affinity:
@ -104,8 +104,8 @@ spec:
              args:
                - -c
                - >-
-                    /tmp/backup_mariadb.sh;
-                    /usr/bin/pkill mysqld
+                    ( /tmp/start_verification_server.sh ) &
+                    /tmp/backup_mariadb.sh
              env:
                - name: MARIADB_BACKUP_BASE_DIR
                  value: {{ .Values.conf.backup.base_path | quote }}
@ -156,47 +156,15 @@ spec:
                  mountPath: /etc/mysql/admin_user.cnf
                  subPath: admin_user.cnf
                  readOnly: true
-{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
-            - name: mariadb-verify-server
-{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
-{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
-{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
-              env:
-                {{- if $envAll.Values.manifests.certificates }}
-                - name: MARIADB_X509
-                  value: "REQUIRE X509"
-                {{- end }}
-                - name: MYSQL_HISTFILE
-                  value: /dev/null
-                - name: MARIADB_BACKUP_BASE_DIR
-                  value: {{ .Values.conf.backup.base_path | quote }}
-              ports:
-                - name: mysql
-                  protocol: TCP
-                  containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-              command:
-                - /tmp/start_verification_server.sh
-              volumeMounts:
-                - name: pod-tmp
-                  mountPath: /tmp
-                - name: var-run
-                  mountPath: /var/run/mysqld
-                - name: mycnfd
-                  mountPath: /etc/mysql/conf.d
-                - name: mariadb-backup-etc
-                  mountPath: /etc/mysql/my.cnf
-                  subPath: my.cnf
-                  readOnly: true
-                - name: mariadb-backup-secrets
-                  mountPath: /etc/mysql/admin_user.cnf
-                  subPath: admin_user.cnf
-                  readOnly: true
-                - name: mysql-data
-                  mountPath: /var/lib/mysql
                - name: mariadb-backup-bin
                  mountPath: /tmp/start_verification_server.sh
                  readOnly: true
                  subPath: start_verification_server.sh
+                - name: mysql-data
+                  mountPath: /var/lib/mysql
+                - name: var-run
+                  mountPath: /run/mysqld
+{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
          volumes:
            - name: pod-tmp
              emptyDir: {}
--- a/mariadb-backup/values.yaml
+++ b/mariadb-backup/values.yaml
@ -73,10 +73,6 @@ pod:
          runAsUser: 65534
          readOnlyRootFilesystem: true
          allowPrivilegeEscalation: false
-        mariadb_verify_server:
-          runAsUser: 65534
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
    tests:
      pod:
        runAsUser: 999
--- a/mariadb/Chart.yaml
+++ b/mariadb/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.36
+version: 0.2.37
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:
--- a/mariadb/templates/bin/_backup_mariadb.sh.tpl
+++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl
@ -441,8 +441,8 @@ verify_databases_backup_archives() {
  export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
  export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
  export MYSQL_OPTS="--silent --skip-column-names"
-  export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
-  export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
+  export MYSQL_LIVE="mysql ${MYSQL_OPTS}"
+  export MYSQL_LOCAL_OPTS=""
  export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
  export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
  export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"
--- a/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl
+++ b/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl
@ -25,4 +25,5 @@ log () {
 }

 log "Starting Mariadb server for backup verification..."
-MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
+mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1
+MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1
--- a/mariadb/templates/cron-job-backup-mariadb.yaml
+++ b/mariadb/templates/cron-job-backup-mariadb.yaml
@ -104,8 +104,8 @@ spec:
              args:
                - -c
                - >-
-                    /tmp/backup_mariadb.sh;
-                    /usr/bin/pkill mysqld
+                    ( /tmp/start_verification_server.sh ) &
+                    /tmp/backup_mariadb.sh
              env:
                - name: MARIADB_BACKUP_BASE_DIR
                  value: {{ .Values.conf.backup.base_path | quote }}
@ -157,46 +157,14 @@ spec:
                  subPath: admin_user.cnf
                  readOnly: true
 {{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
-            - name: mariadb-verify-server
-{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
-{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
-{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
-              env:
-                {{- if $envAll.Values.manifests.certificates }}
-                - name: MARIADB_X509
-                  value: "REQUIRE X509"
-                {{- end }}
-                - name: MYSQL_HISTFILE
-                  value: /dev/null
-                - name: MARIADB_BACKUP_BASE_DIR
-                  value: {{ .Values.conf.backup.base_path | quote }}
-              ports:
-                - name: mysql
-                  protocol: TCP
-                  containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-              command:
-                - /tmp/start_verification_server.sh
-              volumeMounts:
-                - name: pod-tmp
-                  mountPath: /tmp
-                - name: var-run
-                  mountPath: /var/run/mysqld
-                - name: mycnfd
-                  mountPath: /etc/mysql/conf.d
-                - name: mariadb-etc
-                  mountPath: /etc/mysql/my.cnf
-                  subPath: my.cnf
-                  readOnly: true
-                - name: mariadb-secrets
-                  mountPath: /etc/mysql/admin_user.cnf
-                  subPath: admin_user.cnf
-                  readOnly: true
-                - name: mysql-data
-                  mountPath: /var/lib/mysql
                - name: mariadb-bin
                  mountPath: /tmp/start_verification_server.sh
                  readOnly: true
                  subPath: start_verification_server.sh
+                - name: mysql-data
+                  mountPath: /var/lib/mysql
+                - name: var-run
+                  mountPath: /run/mysqld
          volumes:
            - name: pod-tmp
              emptyDir: {}
--- a/mariadb/values.yaml
+++ b/mariadb/values.yaml
@ -28,7 +28,7 @@ images:
    prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
    dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
    image_repo_sync: docker.io/library/docker:17.07.0
-    mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
+    mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal
    ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
    scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
  pull_policy: "IfNotPresent"
@ -129,10 +129,6 @@ pod:
          runAsUser: 65534
          readOnlyRootFilesystem: true
          allowPrivilegeEscalation: false
-        mariadb_verify_server:
-          runAsUser: 65534
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
    tests:
      pod:
        runAsUser: 999
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@ -64,4 +64,5 @@ helm-toolkit:
  - 0.2.55 Updated deprecated IngressClass annotation
  - 0.2.56 Expose S3 credentials from Rook bucket CRD secret
  - 0.2.57 Safer file removal
+  - 0.2.58 Backups verification improvements
 ...
--- a/releasenotes/notes/mariadb-backup.yaml
+++ b/releasenotes/notes/mariadb-backup.yaml
@ -2,4 +2,5 @@
 mariadb-backup:
  - 0.0.1 Initial Chart
  - 0.0.2 Added staggered backups support
+  - 0.0.3 Backups verification improvements
 ...
--- a/releasenotes/notes/mariadb.yaml
+++ b/releasenotes/notes/mariadb.yaml
@ -52,4 +52,5 @@ mariadb:
  - 0.2.34 Uplift ingress controller image to 1.8.2
  - 0.2.35 Update apparmor override
  - 0.2.36 Added staggered backups support
+  - 0.2.37 Backups verification improvements
 ...