From 29f2b616ccc2ce4f8be0e203210d669e2785abd1 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Tue, 17 Oct 2023 04:04:37 +0000 Subject: [PATCH] [mariadb-operator] Mariadb-cluster chart This PS adds mariadb-cluster chart based on mariadb-operator. Also for some backward compartibility this PS adds mariadb-backup chart and prometheus-mysql-exporter chart as a separate ones. Change-Id: I3f652375cce2e3b45e095e08d2e6f4ae73b8d8f0 --- .gitignore | 1 + mariadb-backup/Chart.yaml | 26 + mariadb-backup/README.rst | 19 + mariadb-backup/requirements.yaml | 18 + .../templates/bin/_backup_mariadb.sh.tpl | 584 ++++++++++++++++++ .../templates/bin/_restore_mariadb.sh.tpl | 328 ++++++++++ .../bin/_start_mariadb_verify_server.sh.tpl | 28 + mariadb-backup/templates/configmap-bin.yaml | 45 ++ mariadb-backup/templates/configmap-etc.yaml | 24 + .../templates/cron-job-backup-mariadb.yaml | 226 +++++++ .../templates/job-image-repo-sync.yaml | 22 + mariadb-backup/templates/job-ks-user.yaml | 24 + .../templates/mariadb-backup-pvc.yaml | 29 + .../templates/secret-backup-restore.yaml | 30 + mariadb-backup/templates/secret-registry.yaml | 17 + mariadb-backup/templates/secret-rgw.yaml | 78 +++ mariadb-backup/templates/secrets-etc.yaml | 26 + .../templates/secrets/_admin_user.cnf.tpl | 24 + .../secrets/_admin_user_internal.cnf.tpl | 24 + mariadb-backup/values.yaml | 383 ++++++++++++ .../values_overrides/2023.1-ubuntu_focal.yaml | 18 + .../values_overrides/2023.2-ubuntu_jammy.yaml | 18 + mariadb-backup/values_overrides/apparmor.yaml | 15 + mariadb-backup/values_overrides/backups.yaml | 15 + mariadb-backup/values_overrides/tls.yaml | 13 + .../values_overrides/ubuntu_focal.yaml | 19 + mariadb-cluster/.helmignore | 21 + mariadb-cluster/Chart.yaml | 27 + mariadb-cluster/README.rst | 18 + mariadb-cluster/requirements.yaml | 18 + .../templates/bin/_liveness.sh.tpl | 68 ++ .../templates/bin/_readiness.sh.tpl | 60 ++ mariadb-cluster/templates/bin/_test.sh.tpl | 27 + mariadb-cluster/templates/certificates.yaml | 17 + mariadb-cluster/templates/configmap-bin.yaml | 41 ++ mariadb-cluster/templates/configmap-etc.yaml | 24 + .../templates/job-image-repo-sync.yaml | 22 + .../templates/job-refresh-statefulset.yaml | 105 ++++ mariadb-cluster/templates/mariadb.yaml | 225 +++++++ mariadb-cluster/templates/network_policy.yaml | 17 + mariadb-cluster/templates/pod-test.yaml | 86 +++ .../templates/secret-dbadmin-password.yaml | 25 + .../templates/secret-dbaudit-password.yaml | 25 + .../templates/secret-registry.yaml | 17 + .../templates/secret-sst-password.yaml | 25 + mariadb-cluster/templates/secrets-etc.yaml | 26 + .../templates/secrets/_admin_user.cnf.tpl | 24 + .../secrets/_admin_user_internal.cnf.tpl | 24 + mariadb-cluster/values.yaml | 581 +++++++++++++++++ .../values_overrides/2023.1-ubuntu_focal.yaml | 18 + .../values_overrides/2023.2-ubuntu_jammy.yaml | 18 + .../values_overrides/apparmor.yaml | 21 + .../values_overrides/downscaled.yaml | 8 + .../values_overrides/local-storage.yaml | 11 + mariadb-cluster/values_overrides/netpol.yaml | 84 +++ .../values_overrides/prometheus.yaml | 14 + mariadb-cluster/values_overrides/tls.yaml | 13 + .../values_overrides/ubuntu_focal.yaml | 20 + .../values_overrides/upscaled.yaml | 8 + mariadb/Chart.yaml | 2 +- mariadb/values_overrides/apparmor.yaml | 1 + prometheus-mysql-exporter/.helmignore | 21 + prometheus-mysql-exporter/Chart.yaml | 26 + prometheus-mysql-exporter/README.rst | 18 + prometheus-mysql-exporter/requirements.yaml | 18 + .../templates/bin/_create-mysql-user.sh.tpl | 50 ++ .../templates/bin/_mysqld-exporter.sh.tpl | 57 ++ .../templates/exporter-configmap-bin.yaml | 27 + .../templates/exporter-deployment.yaml | 103 +++ .../templates/exporter-job-create-user.yaml | 98 +++ .../templates/exporter-network-policy.yaml | 18 + .../templates/exporter-secrets-etc.yaml | 33 + .../templates/exporter-service.yaml | 35 ++ .../templates/secrets/_exporter_user.cnf.tpl | 24 + .../value_overrides/2023.1-ubuntu_focal.yaml | 18 + .../value_overrides/2023.2-ubuntu_jammy.yaml | 18 + .../value_overrides/apparmor.yaml | 37 ++ .../value_overrides/prometheus.yaml | 14 + .../value_overrides/tls.yaml | 13 + prometheus-mysql-exporter/values.yaml | 329 ++++++++++ releasenotes/notes/mariadb-backup.yaml | 4 + releasenotes/notes/mariadb-cluster.yaml | 4 + releasenotes/notes/mariadb.yaml | 1 + .../notes/prometheus-mysql-exporter.yaml | 4 + tools/deployment/common/prepare-k8s.sh | 2 +- .../000-prepare-k8s.sh | 1 + .../010-deploy-docker-registry.sh | 1 + .../012-setup-client.sh | 1 + .../mariadb-operator-cluster/020-ingress.sh | 1 + .../030-nfs-provisioner.sh | 1 + .../mariadb-operator-cluster/040-rabbitmq.sh | 1 + .../045-mariadb-operator-cluster.sh | 71 +++ .../mariadb-operator-cluster/050-memcached.sh | 1 + .../mariadb-operator-cluster/070-keystone.sh | 48 ++ .../090-mariadb-backup-test.sh | 40 ++ .../095-mariadb-prometheus-mysql-exporter.sh | 36 ++ zuul.d/jobs.yaml | 27 + zuul.d/project.yaml | 1 + 98 files changed, 4995 insertions(+), 2 deletions(-) create mode 100644 mariadb-backup/Chart.yaml create mode 100644 mariadb-backup/README.rst create mode 100644 mariadb-backup/requirements.yaml create mode 100644 mariadb-backup/templates/bin/_backup_mariadb.sh.tpl create mode 100755 mariadb-backup/templates/bin/_restore_mariadb.sh.tpl create mode 100644 mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl create mode 100644 mariadb-backup/templates/configmap-bin.yaml create mode 100644 mariadb-backup/templates/configmap-etc.yaml create mode 100644 mariadb-backup/templates/cron-job-backup-mariadb.yaml create mode 100644 mariadb-backup/templates/job-image-repo-sync.yaml create mode 100644 mariadb-backup/templates/job-ks-user.yaml create mode 100644 mariadb-backup/templates/mariadb-backup-pvc.yaml create mode 100644 mariadb-backup/templates/secret-backup-restore.yaml create mode 100644 mariadb-backup/templates/secret-registry.yaml create mode 100644 mariadb-backup/templates/secret-rgw.yaml create mode 100644 mariadb-backup/templates/secrets-etc.yaml create mode 100644 mariadb-backup/templates/secrets/_admin_user.cnf.tpl create mode 100644 mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl create mode 100644 mariadb-backup/values.yaml create mode 100644 mariadb-backup/values_overrides/2023.1-ubuntu_focal.yaml create mode 100644 mariadb-backup/values_overrides/2023.2-ubuntu_jammy.yaml create mode 100644 mariadb-backup/values_overrides/apparmor.yaml create mode 100644 mariadb-backup/values_overrides/backups.yaml create mode 100644 mariadb-backup/values_overrides/tls.yaml create mode 100644 mariadb-backup/values_overrides/ubuntu_focal.yaml create mode 100644 mariadb-cluster/.helmignore create mode 100644 mariadb-cluster/Chart.yaml create mode 100644 mariadb-cluster/README.rst create mode 100644 mariadb-cluster/requirements.yaml create mode 100644 mariadb-cluster/templates/bin/_liveness.sh.tpl create mode 100644 mariadb-cluster/templates/bin/_readiness.sh.tpl create mode 100644 mariadb-cluster/templates/bin/_test.sh.tpl create mode 100644 mariadb-cluster/templates/certificates.yaml create mode 100644 mariadb-cluster/templates/configmap-bin.yaml create mode 100644 mariadb-cluster/templates/configmap-etc.yaml create mode 100644 mariadb-cluster/templates/job-image-repo-sync.yaml create mode 100644 mariadb-cluster/templates/job-refresh-statefulset.yaml create mode 100644 mariadb-cluster/templates/mariadb.yaml create mode 100644 mariadb-cluster/templates/network_policy.yaml create mode 100644 mariadb-cluster/templates/pod-test.yaml create mode 100644 mariadb-cluster/templates/secret-dbadmin-password.yaml create mode 100644 mariadb-cluster/templates/secret-dbaudit-password.yaml create mode 100644 mariadb-cluster/templates/secret-registry.yaml create mode 100644 mariadb-cluster/templates/secret-sst-password.yaml create mode 100644 mariadb-cluster/templates/secrets-etc.yaml create mode 100644 mariadb-cluster/templates/secrets/_admin_user.cnf.tpl create mode 100644 mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl create mode 100644 mariadb-cluster/values.yaml create mode 100644 mariadb-cluster/values_overrides/2023.1-ubuntu_focal.yaml create mode 100644 mariadb-cluster/values_overrides/2023.2-ubuntu_jammy.yaml create mode 100644 mariadb-cluster/values_overrides/apparmor.yaml create mode 100644 mariadb-cluster/values_overrides/downscaled.yaml create mode 100644 mariadb-cluster/values_overrides/local-storage.yaml create mode 100644 mariadb-cluster/values_overrides/netpol.yaml create mode 100644 mariadb-cluster/values_overrides/prometheus.yaml create mode 100644 mariadb-cluster/values_overrides/tls.yaml create mode 100644 mariadb-cluster/values_overrides/ubuntu_focal.yaml create mode 100644 mariadb-cluster/values_overrides/upscaled.yaml create mode 100644 prometheus-mysql-exporter/.helmignore create mode 100644 prometheus-mysql-exporter/Chart.yaml create mode 100644 prometheus-mysql-exporter/README.rst create mode 100644 prometheus-mysql-exporter/requirements.yaml create mode 100644 prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl create mode 100644 prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl create mode 100644 prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml create mode 100644 prometheus-mysql-exporter/templates/exporter-deployment.yaml create mode 100644 prometheus-mysql-exporter/templates/exporter-job-create-user.yaml create mode 100644 prometheus-mysql-exporter/templates/exporter-network-policy.yaml create mode 100644 prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml create mode 100644 prometheus-mysql-exporter/templates/exporter-service.yaml create mode 100644 prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl create mode 100644 prometheus-mysql-exporter/value_overrides/2023.1-ubuntu_focal.yaml create mode 100644 prometheus-mysql-exporter/value_overrides/2023.2-ubuntu_jammy.yaml create mode 100644 prometheus-mysql-exporter/value_overrides/apparmor.yaml create mode 100644 prometheus-mysql-exporter/value_overrides/prometheus.yaml create mode 100644 prometheus-mysql-exporter/value_overrides/tls.yaml create mode 100644 prometheus-mysql-exporter/values.yaml create mode 100644 releasenotes/notes/mariadb-backup.yaml create mode 100644 releasenotes/notes/mariadb-cluster.yaml create mode 100644 releasenotes/notes/prometheus-mysql-exporter.yaml create mode 120000 tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh create mode 120000 tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh create mode 120000 tools/deployment/mariadb-operator-cluster/012-setup-client.sh create mode 120000 tools/deployment/mariadb-operator-cluster/020-ingress.sh create mode 120000 tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh create mode 120000 tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh create mode 100755 tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh create mode 120000 tools/deployment/mariadb-operator-cluster/050-memcached.sh create mode 100755 tools/deployment/mariadb-operator-cluster/070-keystone.sh create mode 100755 tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh create mode 100755 tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh diff --git a/.gitignore b/.gitignore index d8b3b05a6..0bc6f588a 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ releasenotes/build # Dev tools .idea/ .vscode/ +.devcontainer/ **/.vagrant **/*.log diff --git a/mariadb-backup/Chart.yaml b/mariadb-backup/Chart.yaml new file mode 100644 index 000000000..a34b23fc2 --- /dev/null +++ b/mariadb-backup/Chart.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +appVersion: v10.6.14 +description: OpenStack-Helm MariaDB backups +name: mariadb-backup +version: 0.0.1 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +... diff --git a/mariadb-backup/README.rst b/mariadb-backup/README.rst new file mode 100644 index 000000000..4a5c7340b --- /dev/null +++ b/mariadb-backup/README.rst @@ -0,0 +1,19 @@ +openstack-helm/mariadb-backup +====================== + +By default, this chart creates a mariadb-backup cronjob that runs in a schedule +in order to create mysql backups. + +This chart depends on mariadb-cluster chart. + +The backups are stored in a PVC and also are possible to upload then to a remote +RGW container. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/mariadb-backup/requirements.yaml b/mariadb-backup/requirements.yaml new file mode 100644 index 000000000..84f0affae --- /dev/null +++ b/mariadb-backup/requirements.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl b/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl new file mode 100644 index 000000000..dba8ddb56 --- /dev/null +++ b/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl @@ -0,0 +1,584 @@ +#!/bin/bash + +SCOPE=${1:-"all"} + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +source /tmp/backup_main.sh + +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already +# exported. +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export DB_NAME="mariadb" +export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP} +export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP} +export REMOTE_BACKUP_RETRIES=${NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE} +export MIN_DELAY_SEND_REMOTE=${MIN_DELAY_SEND_BACKUP_TO_REMOTE} +export MAX_DELAY_SEND_REMOTE=${MAX_DELAY_SEND_BACKUP_TO_REMOTE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + SCOPE=${3:-"all"} + + + MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --connect-timeout 10" + + MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf" + + if [[ "${SCOPE}" == "all" ]]; then + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + grep -ivE 'information_schema|performance_schema|mysql|sys') ) + else + if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" && "${SCOPE}" != "sys" ]]; then + MYSQL_DBNAMES=( ${SCOPE} ) + else + log ERROR "It is not allowed to backup database ${SCOPE}." + return 1 + fi + fi + + #check if there is a database to backup, otherwise exit + if [[ -z "${MYSQL_DBNAMES// }" ]] + then + log INFO "There is no database to backup" + return 0 + fi + + #Create a list of Databases + printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list + + if [[ "${SCOPE}" == "all" ]]; then + #Retrieve and create the GRANT file for all the users +{{- if .Values.manifests.certificates }} + SSL_DSN=";mysql_ssl=1" + SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key" + SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt" + SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt" + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \ +{{- else }} + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \ +{{- end }} + 2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then + log ERROR "Failed to create GRANT for all the users" + return 1 + fi + fi + + #Retrieve and create the GRANT files per DB + for db in "${MYSQL_DBNAMES[@]}" + do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/g" | \ + $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql + else + log ERROR "Failed to create GRANT files for ${db}" + return 1 + fi + done + + #Dumping the database + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE} + + $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ + > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] + then + log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})" + return 0 + else + log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})" + return 1 + fi +} + +# functions from mariadb-verifier chart + +get_time_delta_secs () { + second_delta=0 + input_date_second=$( date --date="$1" +%s ) + if [ -n "$input_date_second" ]; then + current_date=$( date +"%Y-%m-%dT%H:%M:%SZ" ) + current_date_second=$( date --date="$current_date" +%s ) + ((second_delta=current_date_second-input_date_second)) + if [ "$second_delta" -lt 0 ]; then + second_delta=0 + fi + fi + echo $second_delta +} + + +check_data_freshness () { + archive_file=$(basename "$1") + archive_date=$(echo "$archive_file" | cut -d'.' -f 4) + SCOPE=$2 + + if [[ "${SCOPE}" != "all" ]]; then + log "Data freshness check is skipped for individual database." + return 0 + fi + + log "Checking for data freshness in the backups..." + # Get some idea of which database.table has changed in the last 30m + # Excluding the system DBs and aqua_test_database + # + changed_tables=$(${MYSQL_LIVE} -e "select TABLE_SCHEMA,TABLE_NAME from \ +information_schema.tables where UPDATE_TIME >= SUBTIME(now(),'00:30:00') AND TABLE_SCHEMA \ +NOT IN('information_schema', 'mysql', 'performance_schema', 'sys', 'aqua_test_database');" | \ +awk '{print $1 "." $2}') + + if [ -n "${changed_tables}" ]; then + delta_secs=$(get_time_delta_secs "$archive_date") + age_offset={{ .Values.conf.backup.validateData.ageOffset }} + ((age_threshold=delta_secs+age_offset)) + + data_freshness=false + skipped_freshness=false + + for table in ${changed_tables}; do + tab_schema=$(echo "$table" | awk -F. '{print $1}') + tab_name=$(echo "$table" | awk -F. '{print $2}') + + local_table_existed=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select TABLE_SCHEMA,TABLE_NAME from \ +INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA=\"${tab_schema}\" AND TABLE_NAME=\"${tab_name}\";") + + if [ -n "$local_table_existed" ]; then + # TODO: If last updated field of a table structure has different + # patterns (updated/timstamp), it may be worth to parameterize the patterns. + datetime=$(${MYSQL_LOCAL_SHORT_SILENT} -e "describe ${table};" | \ + awk '(/updated/ || /timestamp/) && /datetime/ {print $1}') + + if [ -n "${datetime}" ]; then + data_ages=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select \ +time_to_sec(timediff(now(),${datetime})) from ${table} where ${datetime} is not null order by 1 limit 10;") + + for age in $data_ages; do + if [ "$age" -le $age_threshold ]; then + data_freshness=true + break + fi + done + + # As long as there is an indication of data freshness, no need to check further + if [ "$data_freshness" = true ] ; then + break + fi + else + skipped_freshness=true + log "No indicator to determine data freshness for table $table. Skipped data freshness check." + + # Dumping out table structure to determine if enhancement is needed to include this table + debug_info=$(${MYSQL_LOCAL} --skip-column-names -e "describe ${table};" | awk '{print $2 " " $1}') + log "$debug_info" "DEBUG" + fi + else + log "Table $table doesn't exist in local database" + skipped_freshness=true + fi + done + + if [ "$data_freshness" = true ] ; then + log "Database passed integrity (data freshness) check." + else + if [ "$skipped_freshness" = false ] ; then + log "Local backup database restore failed integrity check." "ERROR" + log "The backup may not have captured the up-to-date data." "INFO" + return 1 + fi + fi + else + log "No tables changed in this backup. Skipped data freshness check as the" + log "check should have been performed by previous validation runs." + fi + + return 0 +} + + +cleanup_local_databases () { + old_local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' || true) + + for db in $old_local_dbs; do + ${MYSQL_LOCAL_SHORT_SILENT} -e "drop database $db;" + done +} + +list_archive_dir () { + archive_dir_content=$(ls -1R "$ARCHIVE_DIR") + if [ -n "$archive_dir_content" ]; then + log "Content of $ARCHIVE_DIR" + log "${archive_dir_content}" + fi +} + +remove_remote_archive_file () { + archive_file=$(basename "$1") + token_req_file=$(mktemp --suffix ".json") + header_file=$(mktemp) + resp_file=$(mktemp --suffix ".json") + http_resp="404" + + HEADER_CONTENT_TYPE="Content-Type: application/json" + HEADER_ACCEPT="Accept: application/json" + + cat << JSON_EOF > "$token_req_file" +{ + "auth": { + "identity": { + "methods": [ + "password" + ], + "password": { + "user": { + "domain": { + "name": "${OS_USER_DOMAIN_NAME}" + }, + "name": "${OS_USERNAME}", + "password": "${OS_PASSWORD}" + } + } + }, + "scope": { + "project": { + "domain": { + "name": "${OS_PROJECT_DOMAIN_NAME}" + }, + "name": "${OS_PROJECT_NAME}" + } + } + } +} +JSON_EOF + + http_resp=$(curl -s -X POST "$OS_AUTH_URL/auth/tokens" -H "${HEADER_CONTENT_TYPE}" \ + -H "${HEADER_ACCEPT}" -d @"${token_req_file}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + + if [ "$http_resp" = "201" ]; then + OS_TOKEN=$(grep -i "x-subject-token" "$header_file" | cut -d' ' -f2 | tr -d "\r") + + if [ -n "$OS_TOKEN" ]; then + OS_OBJ_URL=$(python3 -c "import json,sys;print([[ep['url'] for ep in obj['endpoints'] if ep['interface']=='public'] for obj in json.load(sys.stdin)['token']['catalog'] if obj['type']=='object-store'][0][0])" < "$resp_file") + + if [ -n "$OS_OBJ_URL" ]; then + http_resp=$(curl -s -X DELETE "$OS_OBJ_URL/$CONTAINER_NAME/$archive_file" \ + -H "${HEADER_CONTENT_TYPE}" -H "${HEADER_ACCEPT}" \ + -H "X-Auth-Token: ${OS_TOKEN}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + fi + fi + fi + + if [ "$http_resp" == "404" ] ; then + log "Failed to cleanup remote backup. Container object $archive_file is not on RGW." + return 1 + fi + + if [ "$http_resp" != "204" ] ; then + log "Failed to cleanup remote backup. Cannot delete container object $archive_file" "ERROR" + cat "$header_file" + cat "$resp_file" + fi + return 0 +} + +handle_bad_archive_file () { + archive_file=$1 + + if [ ! -d "$BAD_ARCHIVE_DIR" ]; then + mkdir -p "$BAD_ARCHIVE_DIR" + fi + + # Move the file to quarantine directory such that + # file won't be used for restore in case of recovery + # + log "Moving $i to $BAD_ARCHIVE_DIR..." + mv "$i" "$BAD_ARCHIVE_DIR" + log "Removing $i from remote RGW..." + if remove_remote_archive_file "$i"; then + log "File $i has been successfully removed from RGW." + else + log "FIle $i cannot be removed form RGW." "ERROR" + return 1 + fi + + # Atmost only three bad files are kept. Deleting the oldest if + # number of files exceeded the threshold. + # + bad_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l) + if [ "$bad_files" -gt 3 ]; then + ((bad_files=bad_files-3)) + delete_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | sort | head --lines=$bad_files) + for b in $delete_files; do + log "Deleting $b..." + rm -f "${b}" + done + fi + return 0 +} + +cleanup_old_validation_result_file () { + clean_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.passed" 2>/dev/null) + for d in $clean_files; do + archive_file=${d/.passed} + if [ ! -f "$archive_file" ]; then + log "Deleting $d as its associated archive file $archive_file nolonger existed." + rm -f "${d}" + fi + done +} + +validate_databases_backup () { + archive_file=$1 + SCOPE=${2:-"all"} + + restore_log='/tmp/restore_error.log' + tmp_dir=$(mktemp -d) + + rm -f $restore_log + cd "$tmp_dir" + log "Decompressing archive $archive_file..." + if ! tar zxvf - < "$archive_file" 1>/dev/null; then + log "Database restore from local backup failed. Archive decompression failed." "ERROR" + return 1 + fi + + db_list_file="$tmp_dir/db.list" + if [[ -e "$db_list_file" ]]; then + dbs=$(sort < "$db_list_file" | grep -ivE sys | tr '\n' ' ') + else + dbs=" " + fi + + sql_file="${tmp_dir}/mariadb.${MARIADB_POD_NAMESPACE}.${SCOPE}.sql" + + if [[ "${SCOPE}" == "all" ]]; then + grant_file="${tmp_dir}/grants.sql" + else + grant_file="${tmp_dir}/${SCOPE}_grant.sql" + fi + + if [[ -f $sql_file ]]; then + if $MYSQL_LOCAL < "$sql_file" 2>$restore_log; then + local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' | sort | tr '\n' ' ') + + if [ "$dbs" = "$local_dbs" ]; then + log "Databases restored successful." + else + log "Database restore from local backup failed. Database mismatched between local backup and local server" "ERROR" + log "Databases restored on local server: $local_dbs" "DEBUG" + log "Databases in the local backup: $dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. $dbs" "ERROR" + cat $restore_log + return 1 + fi + + if [[ -f $grant_file ]]; then + if $MYSQL_LOCAL < "$grant_file" 2>$restore_log; then + if ! $MYSQL_LOCAL -e 'flush privileges;'; then + log "Database restore from local backup failed. Failed to flush privileges." "ERROR" + return 1 + fi + log "Databases permission restored successful." + else + log "Database restore from local backup failed. Databases permission failed to restore." "ERROR" + cat "$restore_log" + cat "$grant_file" + log "Local DBs: $local_dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. There is no permission file available" "ERROR" + return 1 + fi + + if ! check_data_freshness "$archive_file" ${SCOPE}; then + # Log has already generated during check data freshness + return 1 + fi + else + log "Database restore from local backup failed. There is no database file available to restore from" "ERROR" + return 1 + fi + + return 0 +} + +# end of functions form mariadb verifier chart + +# Verify all the databases backup archives +verify_databases_backup_archives() { + SCOPE=${1:-"all"} + + # verification code + export DB_NAME="mariadb" + export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive + export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine + export MYSQL_OPTS="--silent --skip-column-names" + export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}" + export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1" + export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2" + export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}" + export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10" + + max_wait={{ .Values.conf.mariadb_server.setup_wait.iteration }} + duration={{ .Values.conf.mariadb_server.setup_wait.duration }} + counter=0 + dbisup=false + + log "Waiting for Mariadb backup verification server to start..." + + # During Mariadb init/startup process, a temporary server is startup + # and shutdown prior to starting up the normal server. + # To avoid prematurely determine server availability, lets snooze + # a bit to give time for the process to complete prior to issue + # mysql commands. + # + + + while [ $counter -lt $max_wait ]; do + if ! $MYSQL_LOCAL_SHORT -e 'select 1' > /dev/null 2>&1 ; then + sleep $duration + ((counter=counter+1)) + else + # Lets sleep for an additional duration just in case async + # init takes a bit more time to complete. + # + sleep $duration + dbisup=true + counter=$max_wait + fi + done + + if ! $dbisup; then + log "Mariadb backup verification server is not running" "ERROR" + return 1 + fi + + # During Mariadb init process, a test database will be briefly + # created and deleted. Adding to the exclusion list for some + # edge cases + # + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + cleanup_local_databases + log "Old databases found on the Mariadb backup verification server were cleaned." + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + log "Cannot clean old databases on verification server." "ERROR" + return 1 + fi + log "The server is ready for verification." + fi + + # Starting with 10.4.13, new definer mariadb.sys was added. However, mariadb.sys was deleted + # during init mariadb as it was not on the exclusion list. This corrupted the view of mysql.user. + # Insert the tuple back to avoid other similar issues with error i.e + # The user specified as a definer ('mariadb.sys'@'localhost') does not exist + # + # Before insert the tuple mentioned above, we should make sure that the MariaDB version is 10.4.+ + mariadb_version=$($MYSQL_LOCAL_SHORT -e "status" | grep -E '^Server\s+version:') + log "Current database ${mariadb_version}" + if [[ ! -z ${mariadb_version} && -z $(grep '10.2' <<< ${mariadb_version}}) ]]; then + if [[ -z $(grep 'mariadb.sys' <<< $($MYSQL_LOCAL_SHORT mysql -e "select * from global_priv where user='mariadb.sys'")) ]]; then + $MYSQL_LOCAL_SHORT -e "insert into mysql.global_priv values ('localhost','mariadb.sys',\ + '{\"access\":0,\"plugin\":\"mysql_native_password\",\"authentication_string\":\"\",\"account_locked\":true,\"password_last_changed\":0}');" + $MYSQL_LOCAL_SHORT -e 'flush privileges;' + fi + fi + + # Ensure archive dir existed + if [ -d "$ARCHIVE_DIR" ]; then + # List archive dir before + list_archive_dir + + # Ensure the local databases are clean for each restore validation + # + cleanup_local_databases + + if [[ "${SCOPE}" == "all" ]]; then + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "$i"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + else + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | grep "${SCOPE}" | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "${i}" "${SCOPE}"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + fi + + + # Cleanup passed files if its archive file nolonger existed + cleanup_old_validation_result_file + + # List archive dir after + list_archive_dir + fi + + + return 0 +} + +# Call main program to start the database backup +backup_databases ${SCOPE} diff --git a/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl b/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl new file mode 100755 index 000000000..334ba85bc --- /dev/null +++ b/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl @@ -0,0 +1,328 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +{{- $envAll := . }} + +# Capture the user's command line arguments +ARGS=("$@") + +if [[ -s /tmp/restore_main.sh ]]; then + source /tmp/restore_main.sh +else + echo "File /tmp/restore_main.sh does not exist." + exit 1 +fi + +# Export the variables needed by the framework +export DB_NAME="mariadb" +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +RESTORE_USER='restoreuser' +RESTORE_PW=$(pwgen 16 1) +RESTORE_LOG='/tmp/restore_error.log' +rm -f $RESTORE_LOG + +# This is for commands which require admin access +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST \ + --connect-timeout 10" + +# This is for commands which we want the temporary "restore" user +# to execute +RESTORE_CMD="mysql \ + --user=${RESTORE_USER} \ + --password=${RESTORE_PW} \ + --host=$MARIADB_SERVER_SERVICE_HOST \ +{{- if .Values.manifests.certificates }} + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 10" + +# Get a single database data from the SQL file. +# $1 - database name +# $2 - sql file path +current_db_desc() { + PATTERN="-- Current Database:" + sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2 +} + +#Return all database from an archive +get_databases() { + TMP_DIR=$1 + DB_FILE=$2 + + if [[ -e ${TMP_DIR}/db.list ]] + then + DBS=$(cat ${TMP_DIR}/db.list | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' ) + else + DBS=" " + fi + + echo $DBS > $DB_FILE +} + +# Determine sql file from 2 options - current and legacy one +# if current is not found check that there is no other namespaced dump file +# before falling back to legacy one +_get_sql_file() { + TMP_DIR=$1 + SQL_FILE="${TMP_DIR}/mariadb.${MARIADB_POD_NAMESPACE}.*.sql" + LEGACY_SQL_FILE="${TMP_DIR}/mariadb.*.sql" + INVALID_SQL_FILE="${TMP_DIR}/mariadb.*.*.sql" + if [ -f ${SQL_FILE} ] + then + echo "Found $(ls ${SQL_FILE})" > /dev/stderr + printf ${SQL_FILE} + elif [ -f ${INVALID_SQL_FILE} ] + then + echo "Expected to find ${SQL_FILE} or ${LEGACY_SQL_FILE}, but found $(ls ${INVALID_SQL_FILE})" > /dev/stderr + elif [ -f ${LEGACY_SQL_FILE} ] + then + echo "Falling back to legacy naming ${LEGACY_SQL_FILE}. Found $(ls ${LEGACY_SQL_FILE})" > /dev/stderr + printf ${LEGACY_SQL_FILE} + fi +} + +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ + > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put +# them in the requested file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE + return 0 + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to +# the archive file found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + DB_FILE=$(mktemp -p /tmp) + current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE} + sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} + if [[ ! (-s ${SCHEMA_FILE}) ]]; then + sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ + > ${SCHEMA_FILE} + fi + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 + fi +} + +# Create temporary user for restoring specific databases. +create_restore_user() { + restore_db=$1 + + # Ensure any old restore user is removed first, if it exists. + # If it doesn't exist it may return error, so do not exit the + # script if that's the case. + delete_restore_user "dont_exit_on_error" + + $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + cat $RESTORE_LOG + echo "Failed to grant restore user ALL permissions on database ${restore_db}" + return 1 + fi + else + cat $RESTORE_LOG + echo "Failed to grant restore user select permissions on all databases" + return 1 + fi +} + +# Delete temporary restore user +delete_restore_user() { + error_handling=$1 + + $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + if [ "$error_handling" == "exit_on_error" ] + then + cat $RESTORE_LOG + echo "Failed to delete temporary restore user - needs attention to avoid a security hole" + return 1 + fi + fi +} + +#Restore a single database +restore_single_db() { + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + if [[ -z "$SINGLE_DB_NAME" ]] + then + echo "Restore single DB called but with wrong parameter." + return 1 + fi + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Restoring a single database requires us to create a temporary user + # which has capability to only restore that ONE database. One gotcha + # is that the mysql command to restore the database is going to throw + # errors because of all the other databases that it cannot access. So + # because of this reason, the --force option is used to prevent the + # command from stopping on an error. + create_restore_user $SINGLE_DB_NAME + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed create restore user." + return 1 + fi + $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Database $SINGLE_DB_NAME Restore successful." + else + cat $RESTORE_LOG + delete_restore_user "exit_on_error" + echo "Database $SINGLE_DB_NAME Restore failed." + return 1 + fi + delete_restore_user "exit_on_error" + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed delete restore user." + return 1 + fi + if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ] + then + $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges for $SINGLE_DB_NAME." + return 1 + fi + echo "Database $SINGLE_DB_NAME Permission Restore successful." + else + cat $RESTORE_LOG + echo "Database $SINGLE_DB_NAME Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available for $SINGLE_DB_NAME" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +#Restore all the databases +restore_all_dbs() { + TMP_DIR=$1 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + + $MYSQL < $SQL_FILE 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." + else + cat $RESTORE_LOG + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + return 1 + fi + if [[ -f ${TMP_DIR}/grants.sql ]] + then + $MYSQL < ${TMP_DIR}/grants.sql 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges." + return 1 + fi + echo "Databases Permission Restore successful." + else + cat $RESTORE_LOG + echo "Databases Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl b/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl new file mode 100644 index 000000000..dce67fa15 --- /dev/null +++ b/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl @@ -0,0 +1,28 @@ +#!/bin/bash -ex + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +log () { + msg_default="Need some text to log" + level_default="INFO" + component_default="Mariadb Backup Verifier" + + msg=${1:-$msg_default} + level=${2:-$level_default} + component=${3:-"$component_default"} + + echo "$(date +'%Y-%m-%d %H:%M:%S,%3N') - ${component} - ${level} - ${msg}" +} + +log "Starting Mariadb server for backup verification..." +MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1 diff --git a/mariadb-backup/templates/configmap-bin.yaml b/mariadb-backup/templates/configmap-bin.yaml new file mode 100644 index 000000000..2c8b1cc5b --- /dev/null +++ b/mariadb-backup/templates/configmap-bin.yaml @@ -0,0 +1,45 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{ if eq .Values.endpoints.oslo_db.auth.admin.username .Values.endpoints.oslo_db.auth.sst.username }} +{{ fail "the DB admin username should not match the sst user username" }} +{{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-backup-bin +data: + backup_mariadb.sh: | +{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start_verification_server.sh: | +{{ tuple "bin/_start_mariadb_verify_server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + restore_mariadb.sh: | +{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }} + restore_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }} +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} +{{- end }} +... diff --git a/mariadb-backup/templates/configmap-etc.yaml b/mariadb-backup/templates/configmap-etc.yaml new file mode 100644 index 000000000..1f792ab38 --- /dev/null +++ b/mariadb-backup/templates/configmap-etc.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License" ); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-backup-etc +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "my" ) "key" "my.cnf" ) | indent 2 }} +{{- end }} diff --git a/mariadb-backup/templates/cron-job-backup-mariadb.yaml b/mariadb-backup/templates/cron-job-backup-mariadb.yaml new file mode 100644 index 000000000..18dd3e0fd --- /dev/null +++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml @@ -0,0 +1,226 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_mariadb_backup }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-backup" }} +{{ tuple $envAll "mariadb_backup" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mariadb-backup + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.mariadb_backup.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{- if .Values.jobs.mariadb_backup.backoffLimit }} + backoffLimit: {{ .Values.jobs.mariadb_backup.backoffLimit }} +{{- end }} +{{- if .Values.jobs.mariadb_backup.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.jobs.mariadb_backup.activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + restartPolicy: OnFailure + serviceAccountName: {{ $serviceAccountName }} + shareProcessNamespace: true +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} +{{ end }} + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + - name: backup-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - $(MARIADB_BACKUP_BASE_DIR) + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: verify-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - /var/lib/mysql + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /var/lib/mysql + name: mysql-data + containers: + - name: mariadb-backup + command: + - /bin/sh + args: + - -c + - >- + /tmp/backup_mariadb.sh; + /usr/bin/pkill mysqld + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS + value: {{ .Values.conf.backup.mysqldump_options | quote }} + - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.days_to_keep | quote }} + - name: MARIADB_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REMOTE_BACKUP_ENABLED + value: "{{ .Values.conf.backup.remote_backup.enabled }}" +{{- if .Values.conf.backup.remote_backup.enabled }} + - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }} + - name: CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.container_name | quote }} + - name: STORAGE_POLICY + value: "{{ .Values.conf.backup.remote_backup.storage_policy }}" + - name: NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.number_of_retries | quote }} + - name: MIN_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }} + - name: MAX_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }} +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.mariadb }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }} +{{- end }} +{{- end }} +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/backup_mariadb.sh + name: mariadb-backup-bin + readOnly: true + subPath: backup_mariadb.sh + - mountPath: /tmp/backup_main.sh + name: mariadb-backup-bin + readOnly: true + subPath: backup_main.sh + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: mariadb-backup-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }} + - name: mariadb-verify-server +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + env: + {{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" + {{- end }} + - name: MYSQL_HISTFILE + value: /dev/null + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + ports: + - name: mysql + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + command: + - /tmp/start_verification_server.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: var-run + mountPath: /var/run/mysqld + - name: mycnfd + mountPath: /etc/mysql/conf.d + - name: mariadb-backup-etc + mountPath: /etc/mysql/my.cnf + subPath: my.cnf + readOnly: true + - name: mariadb-backup-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + - name: mysql-data + mountPath: /var/lib/mysql + - name: mariadb-backup-bin + mountPath: /tmp/start_verification_server.sh + readOnly: true + subPath: start_verification_server.sh + volumes: + - name: pod-tmp + emptyDir: {} + - name: mycnfd + emptyDir: {} + - name: var-run + emptyDir: {} + - name: mariadb-backup-etc + configMap: + name: mariadb-backup-etc + defaultMode: 0444 + - name: mysql-data + emptyDir: {} + - name: mariadb-backup-secrets + secret: + secretName: mariadb-backup-secrets + defaultMode: 420 + - configMap: + defaultMode: 365 + name: mariadb-backup-bin + name: mariadb-backup-bin + {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} + - name: mariadb-backup-dir + persistentVolumeClaim: + claimName: mariadb-backup-data + {{- else }} + - hostPath: + path: {{ .Values.conf.backup.base_path }} + type: DirectoryOrCreate + name: mariadb-backup-dir + {{- end }} +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }} +{{- end }} diff --git a/mariadb-backup/templates/job-image-repo-sync.yaml b/mariadb-backup/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..2f59221ad --- /dev/null +++ b/mariadb-backup/templates/job-image-repo-sync.yaml @@ -0,0 +1,22 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" $serviceName -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mariadb-backup/templates/job-ks-user.yaml b/mariadb-backup/templates/job-ks-user.yaml new file mode 100644 index 000000000..bc7befa38 --- /dev/null +++ b/mariadb-backup/templates/job-ks-user.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }} +{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $ksUserJob := dict "envAll" . "serviceName" $serviceName "configMapBin" "mariadb-backup-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $ksUserJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/mariadb-backup/templates/mariadb-backup-pvc.yaml b/mariadb-backup/templates/mariadb-backup-pvc.yaml new file mode 100644 index 000000000..e2b582765 --- /dev/null +++ b/mariadb-backup/templates/mariadb-backup-pvc.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-backup-data +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.volume.backup.size }} + storageClassName: {{ .Values.volume.backup.class_name }} +... +{{- end }} + diff --git a/mariadb-backup/templates/secret-backup-restore.yaml b/mariadb-backup/templates/secret-backup-restore.yaml new file mode 100644 index 000000000..c3ed882f3 --- /dev/null +++ b/mariadb-backup/templates/secret-backup-restore.yaml @@ -0,0 +1,30 @@ +{{/* +This manifest results a secret being created which has the key information +needed for backing up and restoring the Mariadb databases. +*/}} + +{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }} + +{{- $envAll := . }} +{{- $userClass := "backup_restore" }} +{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }} + BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }} + LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }} + MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }} + REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }} + REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }} + REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }} + REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }} + REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }} +... +{{- end }} diff --git a/mariadb-backup/templates/secret-registry.yaml b/mariadb-backup/templates/secret-registry.yaml new file mode 100644 index 000000000..da979b322 --- /dev/null +++ b/mariadb-backup/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mariadb-backup/templates/secret-rgw.yaml b/mariadb-backup/templates/secret-rgw.yaml new file mode 100644 index 000000000..bdb9ca098 --- /dev/null +++ b/mariadb-backup/templates/secret-rgw.yaml @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +This manifest results in two secrets being created: + 1) Keystone "mariadb" secret, which is needed to access the cluster + (remote or same cluster) for storing mariadb backups. If the + cluster is remote, the auth_url would be non-null. + 2) Keystone "admin" secret, which is needed to create the + "mariadb" keystone account mentioned above. This may not + be needed if the account is in a remote cluster (auth_url is non-null + in that case). +*/}} + +{{- if .Values.conf.backup.remote_backup.enabled }} + +{{- $envAll := . }} +{{- $userClass := "mariadb-server" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- if .Values.manifests.job_ks_user }} +{{- $userClass := "admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- end }} +{{- end }} diff --git a/mariadb-backup/templates/secrets-etc.yaml b/mariadb-backup/templates/secrets-etc.yaml new file mode 100644 index 000000000..de2925847 --- /dev/null +++ b/mariadb-backup/templates/secrets-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-backup-secrets +type: Opaque +data: + admin_user.cnf: {{ tuple "secrets/_admin_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + admin_user_internal.cnf: {{ tuple "secrets/_admin_user_internal.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb-backup/templates/secrets/_admin_user.cnf.tpl b/mariadb-backup/templates/secrets/_admin_user.cnf.tpl new file mode 100644 index 000000000..0031a4bd7 --- /dev/null +++ b/mariadb-backup/templates/secrets/_admin_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl b/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl new file mode 100644 index 000000000..fa0d09a55 --- /dev/null +++ b/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-backup/values.yaml b/mariadb-backup/values.yaml new file mode 100644 index 000000000..65bef4eb8 --- /dev/null +++ b/mariadb-backup/values.yaml @@ -0,0 +1,383 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + server: + pod: + runAsUser: 999 + container: + perms: + runAsUser: 0 + readOnlyRootFilesystem: true + init: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + agent: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb_backup: + pod: + runAsUser: 65534 + container: + backup_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + verify_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb_backup: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + mariadb_verify_server: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + tests: + pod: + runAsUser: 999 + container: + test: + runAsUser: 999 + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + mariadb: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 3 + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + error_pages: + timeout: 10 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + mariadb_backup: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mariadb-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + mariadb_server_ks_user: + services: + - endpoint: internal + service: oslo_db + mariadb_backup: + services: + - endpoint: internal + service: oslo_db + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + services: + - endpoint: internal + service: oslo_db + +volume: + backup: + enabled: true + class_name: general + size: 5Gi + +jobs: + mariadb_backup: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + cron: "0 0 * * *" + history: + success: 3 + failed: 1 + ks_user: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + +conf: + mariadb_server: + setup_wait: + iteration: 30 + duration: 5 + database: + my: | + [mysqld] + datadir=/var/lib/mysql + basedir=/usr + ignore-db-dirs=lost+found + + [client-server] + !includedir /etc/mysql/conf.d/ + backup: + enabled: false + base_path: /var/backup + validateData: + ageOffset: 120 + mysqldump_options: > + --single-transaction --quick --add-drop-database + --add-drop-table --add-locks --databases + days_to_keep: 3 + remote_backup: + enabled: false + container_name: mariadb + days_to_keep: 14 + storage_policy: default-placement + number_of_retries: 5 + delay_range: + min: 30 + max: 60 + +secrets: + identity: + admin: keystone-admin-user + mariadb-server: mariadb-backup-user + mariadb: + backup_restore: mariadb-backup-restore + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + mariadb: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + job_ks_user: false + cron_job_mariadb_backup: true + pvc_backup: true + network_policy: false + pod_test: true + secret_dbadmin_password: true + secret_sst_password: true + secret_dbaudit_password: true + secret_backup_restore: true + secret_etc: true + +... diff --git a/mariadb-backup/values_overrides/2023.1-ubuntu_focal.yaml b/mariadb-backup/values_overrides/2023.1-ubuntu_focal.yaml new file mode 100644 index 000000000..4c9e14ecc --- /dev/null +++ b/mariadb-backup/values_overrides/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/mariadb-backup/values_overrides/2023.2-ubuntu_jammy.yaml b/mariadb-backup/values_overrides/2023.2-ubuntu_jammy.yaml new file mode 100644 index 000000000..e234a9e0a --- /dev/null +++ b/mariadb-backup/values_overrides/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/mariadb-backup/values_overrides/apparmor.yaml b/mariadb-backup/values_overrides/apparmor.yaml new file mode 100644 index 000000000..fa458fa55 --- /dev/null +++ b/mariadb-backup/values_overrides/apparmor.yaml @@ -0,0 +1,15 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-backup: + init: runtime/default + mariadb-backup: runtime/default + mariadb-verify-server: runtime/default + create-sql-user: + init: runtime/default + exporter-create-sql-user: runtime/default + +manifests: + cron_job_mariadb_backup: true +... diff --git a/mariadb-backup/values_overrides/backups.yaml b/mariadb-backup/values_overrides/backups.yaml new file mode 100644 index 000000000..5a7de206c --- /dev/null +++ b/mariadb-backup/values_overrides/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/mariadb-backup/values_overrides/tls.yaml b/mariadb-backup/values_overrides/tls.yaml new file mode 100644 index 000000000..d50f732bf --- /dev/null +++ b/mariadb-backup/values_overrides/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/mariadb-backup/values_overrides/ubuntu_focal.yaml b/mariadb-backup/values_overrides/ubuntu_focal.yaml new file mode 100644 index 000000000..0a2b32775 --- /dev/null +++ b/mariadb-backup/values_overrides/ubuntu_focal.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal +... diff --git a/mariadb-cluster/.helmignore b/mariadb-cluster/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/mariadb-cluster/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/mariadb-cluster/Chart.yaml b/mariadb-cluster/Chart.yaml new file mode 100644 index 000000000..222bb5620 --- /dev/null +++ b/mariadb-cluster/Chart.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +appVersion: v10.6.14 +description: OpenStack-Helm MariaDB controlled by mariadb-operator +name: mariadb-cluster +version: 0.0.1 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://github.com/mariadb-operator/mariadb-operator + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +... diff --git a/mariadb-cluster/README.rst b/mariadb-cluster/README.rst new file mode 100644 index 000000000..1615a9065 --- /dev/null +++ b/mariadb-cluster/README.rst @@ -0,0 +1,18 @@ +openstack-helm/mariadb +====================== + +By default, this chart creates a 3-member mariadb galera cluster. + +This chart depends on mariadb-operator chart. + +The StatefulSets all leverage PVCs to provide stateful storage to +``/var/lib/mysql``. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/mariadb-cluster/requirements.yaml b/mariadb-cluster/requirements.yaml new file mode 100644 index 000000000..84f0affae --- /dev/null +++ b/mariadb-cluster/requirements.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/mariadb-cluster/templates/bin/_liveness.sh.tpl b/mariadb-cluster/templates/bin/_liveness.sh.tpl new file mode 100644 index 000000000..ca1df1d9c --- /dev/null +++ b/mariadb-cluster/templates/bin/_liveness.sh.tpl @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +MYSQL="mariadb \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=localhost \ +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 2" + +mysql_status_query () { + STATUS=$1 + $MYSQL -e "show status like \"${STATUS}\"" | \ + awk "/${STATUS}/ { print \$NF; exit }" +} + +{{- if eq (int .Values.pod.replicas.server) 1 }} +if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then + exit 1 +fi + +{{- else }} +# if [ -f /var/lib/mysql/sst_in_progress ]; then +# # SST in progress, with this node receiving a snapshot. +# # MariaDB won't be up yet; avoid killing. +# exit 0 +# fi + +if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then + # WSREP says the node can receive queries + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then + # WSREP connected + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then + # Not in primary cluster + exit 1 +fi + +wsrep_local_state_comment=$(mysql_status_query wsrep_local_state_comment) +if [ "x${wsrep_local_state_comment}" != "xSynced" ] && [ "x${wsrep_local_state_comment}" != "xDonor/Desynced" ]; then + # WSREP not synced or not sending SST + exit 1 +fi +{{- end }} diff --git a/mariadb-cluster/templates/bin/_readiness.sh.tpl b/mariadb-cluster/templates/bin/_readiness.sh.tpl new file mode 100644 index 000000000..0ee233adb --- /dev/null +++ b/mariadb-cluster/templates/bin/_readiness.sh.tpl @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +MYSQL="mariadb \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=localhost \ +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 2" + +mysql_status_query () { + STATUS=$1 + $MYSQL -e "show status like \"${STATUS}\"" | \ + awk "/${STATUS}/ { print \$NF; exit }" +} + +if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then + exit 1 +fi + +{{- if gt (int .Values.pod.replicas.server) 1 }} +if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then + # WSREP says the node can receive queries + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then + # WSREP connected + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then + # Not in primary cluster + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_local_state_comment)" != "xSynced" ]; then + # WSREP not synced + exit 1 +fi +{{- end }} diff --git a/mariadb-cluster/templates/bin/_test.sh.tpl b/mariadb-cluster/templates/bin/_test.sh.tpl new file mode 100644 index 000000000..536a4213e --- /dev/null +++ b/mariadb-cluster/templates/bin/_test.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +rm -f /tmp/test-success + +mysqlslap \ + --defaults-file=/etc/mysql/test-params.cnf \ + {{ include "helm-toolkit.utils.joinListWithSpace" $.Values.conf.tests.params }} -vv \ + --post-system="touch /tmp/test-success" + +if ! [ -f /tmp/test-success ]; then + exit 1 +fi diff --git a/mariadb-cluster/templates/certificates.yaml b/mariadb-cluster/templates/certificates.yaml new file mode 100644 index 000000000..200f974ac --- /dev/null +++ b/mariadb-cluster/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "oslo_db" "type" "default" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/mariadb-cluster/templates/configmap-bin.yaml b/mariadb-cluster/templates/configmap-bin.yaml new file mode 100644 index 000000000..6fac66a70 --- /dev/null +++ b/mariadb-cluster/templates/configmap-bin.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{ if eq .Values.endpoints.oslo_db.auth.admin.username .Values.endpoints.oslo_db.auth.sst.username }} +{{ fail "the DB admin username should not match the sst user username" }} +{{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "init_script" ) "key" "init.sh" ) | indent 2 }} + readiness.sh: | +{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + liveness.sh: | +{{ tuple "bin/_liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + test.sh: | +{{ tuple "bin/_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} +{{- end }} diff --git a/mariadb-cluster/templates/configmap-etc.yaml b/mariadb-cluster/templates/configmap-etc.yaml new file mode 100644 index 000000000..dc52daddc --- /dev/null +++ b/mariadb-cluster/templates/configmap-etc.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License" ); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-etc +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "my" ) "key" "my.cnf" ) | indent 2 }} +{{- end }} diff --git a/mariadb-cluster/templates/job-image-repo-sync.yaml b/mariadb-cluster/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..2f59221ad --- /dev/null +++ b/mariadb-cluster/templates/job-image-repo-sync.yaml @@ -0,0 +1,22 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" $serviceName -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mariadb-cluster/templates/job-refresh-statefulset.yaml b/mariadb-cluster/templates/job-refresh-statefulset.yaml new file mode 100644 index 000000000..b16a73035 --- /dev/null +++ b/mariadb-cluster/templates/job-refresh-statefulset.yaml @@ -0,0 +1,105 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.mariadb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-cluster-refresh-statefulset" }} +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + resources: + - statefulsets + verbs: + - get + - list + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mariadb-cluster-refresh-statefulset + labels: +{{ tuple $envAll "mariadb-cluster" "refresh-statefulset" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if .Values.helm3_hook }} + annotations: + "helm.sh/hook": "post-upgrade" + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": "before-hook-creation" +{{- end }} +spec: + backoffLimit: {{ .Values.jobs.mariadb_cluster_refresh_statefulset.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb-cluster" "refresh-statefulset" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "" "containerNames" (list "init" "exporter-create-sql-user") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "job" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + activeDeadlineSeconds: {{ .Values.jobs.mariadb_cluster_refresh_statefulset.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: refresh-statefulset +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mariadb_cluster_refresh_statefulset" "container" "main" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_cluster_refresh_statefulset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: ["/bin/sh", "-c"] + args: ["kubectl delete statefulset ${STATEFULSET_NAME} --namespace=${NAMESPACE}"] + env: + - name: STATEFULSET_NAME + value: {{ tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: pod-tmp + mountPath: /tmp + volumes: + - name: pod-tmp + emptyDir: {} +{{- end }} diff --git a/mariadb-cluster/templates/mariadb.yaml b/mariadb-cluster/templates/mariadb.yaml new file mode 100644 index 000000000..82b9d11b5 --- /dev/null +++ b/mariadb-cluster/templates/mariadb.yaml @@ -0,0 +1,225 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "mariadbReadinessProbe" }} +exec: + command: + - /tmp/readiness.sh +{{- end }} +{{- define "mariadbLivenessProbe" }} +exec: + command: + - /tmp/liveness.sh +{{- end }} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.mariadb }} +{{- $envAll := . }} + +--- +apiVersion: mariadb.mmontes.io/v1alpha1 +kind: MariaDB +metadata: + # NOTE(portdirect): the statefulset name must match the POD_NAME_PREFIX env var for discovery to work + name: {{ tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + mariadb-dbadmin-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + rootPasswordSecretKeyRef: + name: mariadb-dbadmin-password + key: MYSQL_DBADMIN_PASSWORD + +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 2 }} + + initContainers: + - command: + - /tmp/init.sh +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ dict "envAll" $envAll "application" "server" "container" "perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + +{{ if $envAll.Values.conf.galera.enabled }} + galera: + enabled: true + primary: + podIndex: {{ .Values.conf.galera.primary.podIndex }} + automaticFailover: {{ .Values.conf.galera.primary.automaticFailover }} + sst: {{ .Values.conf.galera.sst }} + replicaThreads: {{ .Values.conf.galera.replicaThreads }} + agent: +{{ tuple $envAll "agent" | include "helm-toolkit.snippets.image" | indent 6 }} +{{- dict "envAll" $envAll "application" "server" "container" "agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + args: + - '--graceful-shutdown-timeout=5s' + - '--recovery-timeout=5m0s' + - '-log-dev' + - '-log-level=debug' + port: {{ .Values.conf.galera.agent.port }} + {{- if $envAll.Values.conf.galera.agent.kubernetesAuth.enabled }} + kubernetesAuth: + enabled: true + {{- end }} + gracefulShutdownTimeout: {{ .Values.conf.galera.agent.gracefulShutdownTimeout }} + {{- if $envAll.Values.conf.galera.recovery.enabled }} + recovery: + enabled: true + clusterHealthyTimeout: {{ .Values.conf.galera.recovery.clusterHealthyTimeout }} + clusterBootstrapTimeout: {{ .Values.conf.galera.recovery.clusterBootstrapTimeout }} + podRecoveryTimeout: {{ .Values.conf.galera.recovery.podRecoveryTimeout }} + podSyncTimeout: {{ .Values.conf.galera.recovery.podSyncTimeout }} + {{- end }} + initContainer: +{{ tuple $envAll "initContainer" | include "helm-toolkit.snippets.image" | indent 6 }} +{{- dict "envAll" $envAll "application" "server" "container" "init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + args: + - '-log-dev' + - '-log-level=debug' + # galera volume templates + volumeClaimTemplate: + resources: + requests: + storage: {{ .Values.volume.galera.size }} + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.volume.galera.class_name }} +{{ end }} + +{{ include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "galera" ) "key" "myCnf" ) | indent 2 }} + + replicas: {{ .Values.pod.replicas.server }} + + affinity: +{{- tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 4 }} + +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{- tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{- end }} + + updateStrategy: + type: {{ .Values.pod.lifecycle.upgrades.deployments.pod_replacement_strategy }} + +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 2 }} +{{ dict "envAll" $envAll "application" "server" "container" "mariadb" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 2 }} + + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} + + podAnnotations: +{{- dict "envAll" $envAll "podName" "mariadb-server" "containerNames" (list "init-0" "init" "agent" "mariadb") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} + + podDisruptionBudget: + minAvailable: {{ .Values.pod.lifecycle.disruption_budget.mariadb.min_available }} + +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 2 }} + +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbLivenessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 2 }} + +{{ if .Values.monitoring.prometheus.enabled }} + metrics: + exporter: +{{ tuple $envAll "prometheus_mysql_exporter" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" "container" "exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_mysql_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + port: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if $envAll.Values.manifests.certificates }} + volumeMounts: +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} +{{- end }} + serviceMonitor: + prometheusRelease: prometheus-mysql-exporter + interval: 10s + scrapeTimeout: 10s +{{ end }} + + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" + {{- end }} + - name: MARIADB_REPLICAS + value: {{ .Values.pod.replicas.server | quote }} + - name: POD_NAME_PREFIX + value: {{ tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: DISCOVERY_DOMAIN + value: {{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + - name: DIRECT_SVC_NAME + value: {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: MYSQL_DBADMIN_USERNAME + value: {{ .Values.endpoints.oslo_db.auth.admin.username }} + - name: MYSQL_DBADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-dbadmin-password + key: MYSQL_DBADMIN_PASSWORD + - name: MYSQL_HISTFILE + value: {{ .Values.conf.database.mysql_histfile }} + + + volumeMounts: + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + - name: mariadb-bin + mountPath: /tmp/init.sh + subPath: init.sh + - name: mariadb-bin + mountPath: /tmp/readiness.sh + subPath: readiness.sh + readOnly: true + - name: mariadb-bin + mountPath: /tmp/liveness.sh + subPath: liveness.sh + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 4 }} + + volumes: + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-etc + configMap: + name: mariadb-etc + defaultMode: 0444 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 + - name: pod-tmp + emptyDir: {} +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} + + # storage volume templates + volumeClaimTemplate: + resources: + requests: + storage: {{ .Values.volume.size }} + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.volume.class_name }} + +{{- end }} diff --git a/mariadb-cluster/templates/network_policy.yaml b/mariadb-cluster/templates/network_policy.yaml new file mode 100644 index 000000000..78ecc07bd --- /dev/null +++ b/mariadb-cluster/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "mariadb" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/mariadb-cluster/templates/pod-test.yaml b/mariadb-cluster/templates/pod-test.yaml new file mode 100644 index 000000000..c8b3c29c3 --- /dev/null +++ b/mariadb-cluster/templates/pod-test.yaml @@ -0,0 +1,86 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.pod_test }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.static.tests }} + +{{- $serviceAccountName := print .deployment_name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.deployment_name}}-test" + labels: +{{ tuple $envAll "mariadb" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "mariadb-test" "containerNames" (list "init" "mariadb-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "tests" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{ end }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: mariadb-test +{{ dict "envAll" $envAll "application" "tests" "container" "test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 6 }} + command: + - /tmp/test.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mariadb-bin + mountPath: /tmp/test.sh + subPath: test.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/test-params.cnf + {{ if eq $envAll.Values.conf.tests.endpoint "internal" }} + subPath: admin_user_internal.cnf + {{ else if eq $envAll.Values.conf.tests.endpoint "direct" }} + subPath: admin_user.cnf + {{ else }} + {{ fail "Either 'direct' or 'internal' should be specified for .Values.conf.tests.endpoint" }} + {{ end }} + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-dbadmin-password.yaml b/mariadb-cluster/templates/secret-dbadmin-password.yaml new file mode 100644 index 000000000..c9f8c4e26 --- /dev/null +++ b/mariadb-cluster/templates/secret-dbadmin-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbadmin_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbadmin-password +type: Opaque +data: + MYSQL_DBADMIN_PASSWORD: {{ .Values.endpoints.oslo_db.auth.admin.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-dbaudit-password.yaml b/mariadb-cluster/templates/secret-dbaudit-password.yaml new file mode 100644 index 000000000..7733da7dd --- /dev/null +++ b/mariadb-cluster/templates/secret-dbaudit-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbaudit_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbaudit-password +type: Opaque +data: + MYSQL_DBAUDIT_PASSWORD: {{ .Values.endpoints.oslo_db.auth.audit.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-registry.yaml b/mariadb-cluster/templates/secret-registry.yaml new file mode 100644 index 000000000..da979b322 --- /dev/null +++ b/mariadb-cluster/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-sst-password.yaml b/mariadb-cluster/templates/secret-sst-password.yaml new file mode 100644 index 000000000..c49c0ff9b --- /dev/null +++ b/mariadb-cluster/templates/secret-sst-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_sst_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbsst-password +type: Opaque +data: + MYSQL_DBSST_PASSWORD: {{ .Values.endpoints.oslo_db.auth.sst.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secrets-etc.yaml b/mariadb-cluster/templates/secrets-etc.yaml new file mode 100644 index 000000000..9dac3eb1b --- /dev/null +++ b/mariadb-cluster/templates/secrets-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-secrets +type: Opaque +data: + admin_user.cnf: {{ tuple "secrets/_admin_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + admin_user_internal.cnf: {{ tuple "secrets/_admin_user_internal.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl b/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl new file mode 100644 index 000000000..0031a4bd7 --- /dev/null +++ b/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl b/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl new file mode 100644 index 000000000..fa0d09a55 --- /dev/null +++ b/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-cluster/values.yaml b/mariadb-cluster/values.yaml new file mode 100644 index 000000000..170ab9987 --- /dev/null +++ b/mariadb-cluster/values.yaml @@ -0,0 +1,581 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + agent: ghcr.io/mariadb-operator/agent:v0.0.3 + initContainer: ghcr.io/mariadb-operator/init:v0.0.6 + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1 + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + image_repo_sync: docker.io/library/docker:17.07.0 + scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415 + mariadb_cluster_refresh_statefulset: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + probes: + server: + mariadb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 15 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 15 + security_context: + server: + pod: + runAsUser: 0 + container: + init-0: + runAsUser: 0 + readOnlyRootFilesystem: true + init: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + agent: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb_cluster_refresh_statefulset: + pod: + runAsUser: 0 + container: + main: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + tests: + pod: + runAsUser: 999 + container: + test: + runAsUser: 999 + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + mariadb: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 3 + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + mariadb_cluster_refresh_statefulset: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mariadb-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + services: + - endpoint: internal + service: oslo_db + +volume: + enabled: true + class_name: general + size: 5Gi + backup: + enabled: true + class_name: general + size: 5Gi + galera: + enabled: true + class_name: general + size: 300Mi + +jobs: + mariadb_cluster_refresh_statefulset: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + +conf: + galera: + enabled: true + primary: + podIndex: 0 + automaticFailover: true + sst: mariabackup + replicaThreads: 1 + agent: + port: 5555 + kubernetesAuth: + enabled: true + gracefulShutdownTimeout: 5s + recovery: + enabled: true + clusterHealthyTimeout: 3m + clusterBootstrapTimeout: 10m + podRecoveryTimeout: 5m + podSyncTimeout: 5m + tests: + # This may either be: + # * internal: which will hit the endpoint exposed by the ingress controller + # * direct: which will hit the backends directly via a k8s service ip + # Note, deadlocks and failure are to be expected with concurrency if + # hitting the `direct` endpoint. + endpoint: internal + # This is a list of tuning params passed to mysqlslap: + params: + - --auto-generate-sql + - --concurrency=100 + - --number-of-queries=1000 + - --number-char-cols=1 + - --number-int-cols=1 + mariadb_server: + setup_wait: + iteration: 30 + duration: 5 + database: + mysql_histfile: "/dev/null" + init_script: | + #!/usr/bin/env bash + + {{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */}} + + set -x + + chown -R "mysql:mysql" /var/lib/mysql; + chmod 771 /var/lib/mysql; + galera: | + [mariadb] + bind-address=0.0.0.0 + default_storage_engine=InnoDB + binlog_format=row + innodb_autoinc_lock_mode=2 + max_allowed_packet=256M + ######################## + # + ######################## + ignore-db-dirs=lost+found + + # Charset + character_set_server=utf8 + collation_server=utf8_general_ci + skip-character-set-client-handshake + + # Logging + slow_query_log=off + slow_query_log_file=/var/log/mysql/mariadb-slow.log + log_warnings=2 + + # General logging has huge performance penalty therefore is disabled by default + general_log=off + general_log_file=/var/log/mysql/mariadb-error.log + + long_query_time=3 + log_queries_not_using_indexes=on + + # Networking + bind_address=0.0.0.0 + port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + # When a client connects, the server will perform hostname resolution, + # and when DNS is slow, establishing the connection will become slow as well. + # It is therefore recommended to start the server with skip-name-resolve to + # disable all DNS lookups. The only limitation is that the GRANT statements + # must then use IP addresses only. + skip_name_resolve + + # Tuning + user=mysql + max_allowed_packet=256M + open_files_limit=10240 + max_connections=8192 + max-connect-errors=1000000 + + # General security settings + # Reference: https://dev.mysql.com/doc/mysql-security-excerpt/8.0/en/general-security-issues.html + # secure_file_priv is set to '/home' because it is read-only, which will + # disable this feature completely. + secure_file_priv=/home + local_infile=0 + symbolic_links=0 + sql_mode="STRICT_ALL_TABLES,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" + + + ## Generally, it is unwise to set the query cache to be larger than 64-128M + ## as the costs associated with maintaining the cache outweigh the performance + ## gains. + ## The query cache is a well known bottleneck that can be seen even when + ## concurrency is moderate. The best option is to disable it from day 1 + ## by setting query_cache_size=0 (now the default on MySQL 5.6) + ## and to use other ways to speed up read queries: good indexing, adding + ## replicas to spread the read load or using an external cache. + query_cache_size=0 + query_cache_type=0 + + sync_binlog=0 + thread_cache_size=16 + table_open_cache=2048 + table_definition_cache=1024 + + # + # InnoDB + # + # The buffer pool is where data and indexes are cached: having it as large as possible + # will ensure you use memory and not disks for most read operations. + # Typical values are 50..75% of available RAM. + # TODO(tomasz.paszkowski): This needs to by dynamic based on available RAM. + innodb_buffer_pool_size=1024M + innodb_doublewrite=0 + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_io_capacity=500 + innodb_log_file_size=128M + innodb_old_blocks_time=1000 + innodb_read_io_threads=8 + innodb_write_io_threads=8 + + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + {{ end }} + + + [mysqldump] + max-allowed-packet=16M + + [client] + default_character_set=utf8 + protocol=tcp + port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + ssl-verify-server-cert + {{ end }} + + my: | + [mysqld] + datadir=/var/lib/mysql + basedir=/usr + ignore-db-dirs=lost+found + + [client-server] + !includedir /etc/mysql/conf.d/ + + config_override: null + # Any configuration here will override the base config. + # config_override: |- + # [mysqld] + # wsrep_slave_threads=1 + +monitoring: + prometheus: + enabled: false + mysqld_exporter: + scrape: true + +secrets: + identity: + admin: keystone-admin-user + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + prometheus_mysql_exporter: + namespace: null + hosts: + default: mysql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9104 + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb-server: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + mariadb: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + job_image_repo_sync: true + network_policy: false + pod_test: true + secret_dbadmin_password: true + secret_sst_password: true + secret_dbaudit_password: true + secret_etc: true + secret_registry: true + service_primary: true + mariadb: true +... diff --git a/mariadb-cluster/values_overrides/2023.1-ubuntu_focal.yaml b/mariadb-cluster/values_overrides/2023.1-ubuntu_focal.yaml new file mode 100644 index 000000000..4c9e14ecc --- /dev/null +++ b/mariadb-cluster/values_overrides/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/mariadb-cluster/values_overrides/2023.2-ubuntu_jammy.yaml b/mariadb-cluster/values_overrides/2023.2-ubuntu_jammy.yaml new file mode 100644 index 000000000..e234a9e0a --- /dev/null +++ b/mariadb-cluster/values_overrides/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/mariadb-cluster/values_overrides/apparmor.yaml b/mariadb-cluster/values_overrides/apparmor.yaml new file mode 100644 index 000000000..c0fb0d381 --- /dev/null +++ b/mariadb-cluster/values_overrides/apparmor.yaml @@ -0,0 +1,21 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-server: + init-0: runtime/default + agent: runtime/default + init: runtime/default + metrics: runtime/default + mariadb: runtime/default + mariadb-test: + init: runtime/default + mariadb-test: runtime/default + refresh-statefulset: + init: runtime/default + mariadb-refresh-statefulset: runtime/default + +monitoring: + prometheus: + enabled: true +... diff --git a/mariadb-cluster/values_overrides/downscaled.yaml b/mariadb-cluster/values_overrides/downscaled.yaml new file mode 100644 index 000000000..e536d1304 --- /dev/null +++ b/mariadb-cluster/values_overrides/downscaled.yaml @@ -0,0 +1,8 @@ +--- +conf: + galera: + enabled: false +pod: + replicas: + server: 1 +... diff --git a/mariadb-cluster/values_overrides/local-storage.yaml b/mariadb-cluster/values_overrides/local-storage.yaml new file mode 100644 index 000000000..2346728ca --- /dev/null +++ b/mariadb-cluster/values_overrides/local-storage.yaml @@ -0,0 +1,11 @@ +--- +pod: + replicas: + server: 1 +volume: + size: 1Gi + class_name: local-storage +monitoring: + prometheus: + enabled: false +... diff --git a/mariadb-cluster/values_overrides/netpol.yaml b/mariadb-cluster/values_overrides/netpol.yaml new file mode 100644 index 000000000..7c2ba1f8e --- /dev/null +++ b/mariadb-cluster/values_overrides/netpol.yaml @@ -0,0 +1,84 @@ +--- +manifests: + network_policy: true +network_policy: + mariadb: + egress: + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + - podSelector: + matchLabels: + application: heat + - podSelector: + matchLabels: + application: glance + - podSelector: + matchLabels: + application: cinder + - podSelector: + matchLabels: + application: aodh + - podSelector: + matchLabels: + application: barbican + - podSelector: + matchLabels: + application: ceilometer + - podSelector: + matchLabels: + application: designate + - podSelector: + matchLabels: + application: horizon + - podSelector: + matchLabels: + application: ironic + - podSelector: + matchLabels: + application: magnum + - podSelector: + matchLabels: + application: mistral + - podSelector: + matchLabels: + application: nova + - podSelector: + matchLabels: + application: neutron + - podSelector: + matchLabels: + application: rally + - podSelector: + matchLabels: + application: senlin + - podSelector: + matchLabels: + application: placement + - podSelector: + matchLabels: + application: prometheus-mysql-exporter + - podSelector: + matchLabels: + application: mariadb + - podSelector: + matchLabels: + application: mariadb-backup + ports: + - protocol: TCP + port: 3306 + - protocol: TCP + port: 4567 + - protocol: TCP + port: 80 + - protocol: TCP + port: 8080 +... diff --git a/mariadb-cluster/values_overrides/prometheus.yaml b/mariadb-cluster/values_overrides/prometheus.yaml new file mode 100644 index 000000000..91093da70 --- /dev/null +++ b/mariadb-cluster/values_overrides/prometheus.yaml @@ -0,0 +1,14 @@ +--- +monitoring: + prometheus: + enabled: true +manifests: + monitoring: + prometheus: + configmap_bin: true + deployment_exporter: true + job_user_create: true + secret_etc: true + service_exporter: true + network_policy_exporter: true +... diff --git a/mariadb-cluster/values_overrides/tls.yaml b/mariadb-cluster/values_overrides/tls.yaml new file mode 100644 index 000000000..d50f732bf --- /dev/null +++ b/mariadb-cluster/values_overrides/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/mariadb-cluster/values_overrides/ubuntu_focal.yaml b/mariadb-cluster/values_overrides/ubuntu_focal.yaml new file mode 100644 index 000000000..0b69fb00f --- /dev/null +++ b/mariadb-cluster/values_overrides/ubuntu_focal.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415 + mariadb_cluster_refresh_statefulset: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal +... diff --git a/mariadb-cluster/values_overrides/upscaled.yaml b/mariadb-cluster/values_overrides/upscaled.yaml new file mode 100644 index 000000000..b35f91550 --- /dev/null +++ b/mariadb-cluster/values_overrides/upscaled.yaml @@ -0,0 +1,8 @@ +--- +conf: + galera: + enabled: true +pod: + replicas: + server: 3 +... diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 43644b3e9..101e14257 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.7 description: OpenStack-Helm MariaDB name: mariadb -version: 0.2.34 +version: 0.2.35 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/values_overrides/apparmor.yaml b/mariadb/values_overrides/apparmor.yaml index ffde96e81..09acc7bd6 100644 --- a/mariadb/values_overrides/apparmor.yaml +++ b/mariadb/values_overrides/apparmor.yaml @@ -32,4 +32,5 @@ monitoring: manifests: cron_job_mariadb_backup: true + job_ks_user: false ... diff --git a/prometheus-mysql-exporter/.helmignore b/prometheus-mysql-exporter/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/prometheus-mysql-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/prometheus-mysql-exporter/Chart.yaml b/prometheus-mysql-exporter/Chart.yaml new file mode 100644 index 000000000..85c9b4c45 --- /dev/null +++ b/prometheus-mysql-exporter/Chart.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +appVersion: v0.12.1 +description: OpenStack-Helm Prometheus mysql-exporter +name: prometheus-mysql-exporter +version: 0.0.1 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +... diff --git a/prometheus-mysql-exporter/README.rst b/prometheus-mysql-exporter/README.rst new file mode 100644 index 000000000..1615a9065 --- /dev/null +++ b/prometheus-mysql-exporter/README.rst @@ -0,0 +1,18 @@ +openstack-helm/mariadb +====================== + +By default, this chart creates a 3-member mariadb galera cluster. + +This chart depends on mariadb-operator chart. + +The StatefulSets all leverage PVCs to provide stateful storage to +``/var/lib/mysql``. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/prometheus-mysql-exporter/requirements.yaml b/prometheus-mysql-exporter/requirements.yaml new file mode 100644 index 000000000..84f0affae --- /dev/null +++ b/prometheus-mysql-exporter/requirements.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl b/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl new file mode 100644 index 000000000..bf6e733cb --- /dev/null +++ b/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl @@ -0,0 +1,50 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + + # SLAVE MONITOR + # Grants ability to SHOW SLAVE STATUS, SHOW REPLICA STATUS, + # SHOW ALL SLAVES STATUS, SHOW ALL REPLICAS STATUS, SHOW RELAYLOG EVENTS. + # New privilege added in MariaDB Enterprise Server 10.5.8-5. Alias for REPLICA MONITOR. + # + # REPLICATION CLIENT + # Grants ability to SHOW MASTER STATUS, SHOW SLAVE STATUS, SHOW BINARY LOGS. In ES10.5, + # is an alias for BINLOG MONITOR and the capabilities have changed. BINLOG MONITOR grants + # ability to SHOW MASTER STATUS, SHOW BINARY LOGS, SHOW BINLOG EVENTS, and SHOW BINLOG STATUS. + + mariadb_version=$(mysql --defaults-file=/etc/mysql/admin_user.cnf -e "status" | grep -E '^Server\s+version:') + echo "Current database ${mariadb_version}" + + if [[ ! -z ${mariadb_version} && -z $(grep -E '10.2|10.3|10.4' <<< ${mariadb_version}) ]]; then + # In case MariaDB version is 10.2.x-10.4.x - we use old privileges definitions + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT PROCESS, BINLOG MONITOR, SLAVE MONITOR, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + else + # here we use new MariaDB privileges definitions defines since version 10.5 + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + fi diff --git a/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl b/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl new file mode 100644 index 000000000..d794be374 --- /dev/null +++ b/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl @@ -0,0 +1,57 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +compareVersions() { +echo $1 $2 | \ +awk '{ split($1, a, "."); + split($2, b, "."); + res = -1; + for (i = 1; i <= 3; i++){ + if (a[i] < b[i]) { + res =-1; + break; + } else if (a[i] > b[i]) { + res = 1; + break; + } else if (a[i] == b[i]) { + if (i == 3) { + res = 0; + break; + } else { + continue; + } + } + } + print res; + }' +} + +MYSQL_EXPORTER_VER=`/bin/mysqld_exporter --version 2>&1 | grep "mysqld_exporter" | awk '{print $3}'` + +#in versions greater than 0.10.0 different configuration flags are used: +#https://github.com/prometheus/mysqld_exporter/commit/66c41ac7eb90a74518a6ecf6c6bb06464eb68db8 +compverResult=`compareVersions "${MYSQL_EXPORTER_VER}" "0.10.0"` +CONFIG_FLAG_PREFIX='-' +if [ ${compverResult} -gt 0 ]; then + CONFIG_FLAG_PREFIX='--' +fi + +exec /bin/mysqld_exporter \ + ${CONFIG_FLAG_PREFIX}config.my-cnf=/etc/mysql/mysql_user.cnf \ + ${CONFIG_FLAG_PREFIX}web.listen-address="${POD_IP}:${LISTEN_PORT}" \ + ${CONFIG_FLAG_PREFIX}web.telemetry-path="$TELEMETRY_PATH" diff --git a/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml b/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml new file mode 100644 index 000000000..94bafc0ba --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-exporter-bin +data: + create-mysql-user.sh: | +{{ tuple "bin/_create-mysql-user.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mysqld-exporter.sh: | +{{ tuple "bin/_mysqld-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-deployment.yaml b/prometheus-mysql-exporter/templates/exporter-deployment.yaml new file mode 100644 index 000000000..b2ac8242f --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-deployment.yaml @@ -0,0 +1,103 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "prometheus-mysql-exporter" }} +{{ tuple $envAll "prometheus_mysql_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-mysql-exporter + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.prometheus_mysql_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + namespace: {{ .Values.endpoints.prometheus_mysql_exporter.namespace }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "prometheus-mysql-exporter" "containerNames" (list "init" "mysql-exporter") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + nodeSelector: + {{ .Values.labels.prometheus_mysql_exporter.node_selector_key }}: {{ .Values.labels.prometheus_mysql_exporter.node_selector_value }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_mysql_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus_mysql_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: mysql-exporter +{{ tuple $envAll "prometheus_mysql_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" "container" "exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_mysql_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/mysqld-exporter.sh + ports: + - name: metrics + containerPort: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LISTEN_PORT + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: TELEMETRY_PATH + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" | quote }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-secrets + mountPath: /etc/mysql/mysql_user.cnf + subPath: mysql_user.cnf + readOnly: true + - name: mysql-exporter-bin + mountPath: /tmp/mysqld-exporter.sh + subPath: mysqld-exporter.sh + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mysql-exporter-secrets + secret: + secretName: mysql-exporter-secrets + defaultMode: 0444 + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml b/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml new file mode 100644 index 000000000..3352ab8d6 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml @@ -0,0 +1,98 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.job_user_create .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "exporter-create-sql-user" }} +{{ tuple $envAll "prometheus_create_mysql_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: exporter-create-sql-user + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if .Values.helm3_hook }} + annotations: + "helm.sh/hook": "post-install,post-upgrade" + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": "before-hook-creation" +{{- end }} +spec: + backoffLimit: {{ .Values.jobs.exporter_create_sql_user.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "create-sql-user" "containerNames" (list "init" "exporter-create-sql-user") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + activeDeadlineSeconds: {{ .Values.jobs.exporter_create_sql_user.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.prometheus_mysql_exporter.node_selector_key }}: {{ .Values.labels.prometheus_mysql_exporter.node_selector_value }} + initContainers: +{{ tuple $envAll "prometheus_create_mysql_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: exporter-create-sql-user +{{ tuple $envAll "prometheus_create_mysql_user" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" "container" "main" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.prometheus_create_mysql_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/create-mysql-user.sh + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-bin + mountPath: /tmp/create-mysql-user.sh + subPath: create-mysql-user.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-network-policy.yaml b/prometheus-mysql-exporter/templates/exporter-network-policy.yaml new file mode 100644 index 000000000..3769506e7 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-mysql-exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml b/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml new file mode 100644 index 000000000..99f01f8e2 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.secret_etc .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $exporter_user := .Values.endpoints.oslo_db.auth.exporter.username }} +{{- $exporter_password := .Values.endpoints.oslo_db.auth.exporter.password }} +{{- $db_host := tuple "oslo_db" "direct" "mysql" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $data_source_name := printf "%s:%s@(%s)/" $exporter_user $exporter_password $db_host }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mysql-exporter-secrets +type: Opaque +data: + DATA_SOURCE_NAME: {{ $data_source_name | b64enc }} + EXPORTER_USER: {{ .Values.endpoints.oslo_db.auth.exporter.username | b64enc }} + EXPORTER_PASSWORD: {{ .Values.endpoints.oslo_db.auth.exporter.password | b64enc }} + mysql_user.cnf: {{ tuple "secrets/_exporter_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-service.yaml b/prometheus-mysql-exporter/templates/exporter-service.yaml new file mode 100644 index 000000000..a7166358a --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-service.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.mysqld_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_mysql_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: metrics + port: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl b/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl new file mode 100644 index 000000000..c86fc01f2 --- /dev/null +++ b/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.exporter.username }} +password = {{ .Values.endpoints.oslo_db.auth.exporter.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/prometheus-mysql-exporter/value_overrides/2023.1-ubuntu_focal.yaml b/prometheus-mysql-exporter/value_overrides/2023.1-ubuntu_focal.yaml new file mode 100644 index 000000000..4c9e14ecc --- /dev/null +++ b/prometheus-mysql-exporter/value_overrides/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/prometheus-mysql-exporter/value_overrides/2023.2-ubuntu_jammy.yaml b/prometheus-mysql-exporter/value_overrides/2023.2-ubuntu_jammy.yaml new file mode 100644 index 000000000..e234a9e0a --- /dev/null +++ b/prometheus-mysql-exporter/value_overrides/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/prometheus-mysql-exporter/value_overrides/apparmor.yaml b/prometheus-mysql-exporter/value_overrides/apparmor.yaml new file mode 100644 index 000000000..fc86fbf8b --- /dev/null +++ b/prometheus-mysql-exporter/value_overrides/apparmor.yaml @@ -0,0 +1,37 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-ingress-error-pages: + init: runtime/default + ingress-error-pages: runtime/default + mariadb-ingress: + init: runtime/default + ingress: runtime/default + mariadb-server: + init-0: runtime/default + agent: runtime/default + init: runtime/default + mariadb-perms: runtime/default + mariadb: runtime/default + mariadb-backup: + init: runtime/default + mariadb-backup: runtime/default + mariadb-verify-server: runtime/default + mariadb-test: + init: runtime/default + mariadb-test: runtime/default + prometheus-mysql-exporter: + init: runtime/default + mysql-exporter: runtime/default + create-sql-user: + init: runtime/default + exporter-create-sql-user: runtime/default + +monitoring: + prometheus: + enabled: true + +manifests: + cron_job_mariadb_backup: true +... diff --git a/prometheus-mysql-exporter/value_overrides/prometheus.yaml b/prometheus-mysql-exporter/value_overrides/prometheus.yaml new file mode 100644 index 000000000..91093da70 --- /dev/null +++ b/prometheus-mysql-exporter/value_overrides/prometheus.yaml @@ -0,0 +1,14 @@ +--- +monitoring: + prometheus: + enabled: true +manifests: + monitoring: + prometheus: + configmap_bin: true + deployment_exporter: true + job_user_create: true + secret_etc: true + service_exporter: true + network_policy_exporter: true +... diff --git a/prometheus-mysql-exporter/value_overrides/tls.yaml b/prometheus-mysql-exporter/value_overrides/tls.yaml new file mode 100644 index 000000000..d50f732bf --- /dev/null +++ b/prometheus-mysql-exporter/value_overrides/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/prometheus-mysql-exporter/values.yaml b/prometheus-mysql-exporter/values.yaml new file mode 100644 index 000000000..4af38359c --- /dev/null +++ b/prometheus-mysql-exporter/values.yaml @@ -0,0 +1,329 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + prometheus_create_mysql_user: docker.io/library/mariadb:10.5.9-focal + prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1 + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + prometheus_mysql_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + prometheus_mysql_exporter: + pod: + runAsUser: 99 + container: + exporter: + runAsUser: 99 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + prometheus_create_mysql_user: + pod: + runAsUser: 0 + container: + main: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + resources: + enabled: false + prometheus_mysql_exporter: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + prometheus_create_mysql_user: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mysql-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + prometheus_create_mysql_user: + services: + - endpoint: internal + service: oslo_db + prometheus_mysql_exporter: + jobs: + - exporter-create-sql-user + services: + - endpoint: internal + service: oslo_db + prometheus_mysql_exporter_tests: + services: + - endpoint: internal + service: prometheus_mysql_exporter + - endpoint: internal + service: monitoring + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +jobs: + exporter_create_sql_user: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + +monitoring: + prometheus: + enabled: false + mysqld_exporter: + scrape: true + +secrets: + identity: + admin: keystone-admin-user + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + prometheus_mysql_exporter: + namespace: null + hosts: + default: mysql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9104 + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb-server: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + prometheus-mysql-exporter: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + job_image_repo_sync: true + monitoring: + prometheus: + configmap_bin: false + deployment_exporter: false + job_user_create: false + secret_etc: false + service_exporter: false + network_policy_exporter: false + network_policy: false + secret_etc: true + secret_registry: true +... diff --git a/releasenotes/notes/mariadb-backup.yaml b/releasenotes/notes/mariadb-backup.yaml new file mode 100644 index 000000000..e8bc615c6 --- /dev/null +++ b/releasenotes/notes/mariadb-backup.yaml @@ -0,0 +1,4 @@ +--- +mariadb-backup: + - 0.0.1 Initial Chart +... diff --git a/releasenotes/notes/mariadb-cluster.yaml b/releasenotes/notes/mariadb-cluster.yaml new file mode 100644 index 000000000..0588f8eea --- /dev/null +++ b/releasenotes/notes/mariadb-cluster.yaml @@ -0,0 +1,4 @@ +--- +mariadb-cluster: + - 0.0.1 Initial Chart +... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index 31afa5eb0..d7ac44994 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -50,4 +50,5 @@ mariadb: - 0.2.32 Prevent liveness probe from killing pods during SST - 0.2.33 Add 2023.1 Ubuntu Focal overrides - 0.2.34 Uplift ingress controller image to 1.8.2 + - 0.2.35 Update apparmor override ... diff --git a/releasenotes/notes/prometheus-mysql-exporter.yaml b/releasenotes/notes/prometheus-mysql-exporter.yaml new file mode 100644 index 000000000..87e954361 --- /dev/null +++ b/releasenotes/notes/prometheus-mysql-exporter.yaml @@ -0,0 +1,4 @@ +--- +prometheus-mysql-exporter: + - 0.0.1 Initial Chart +... diff --git a/tools/deployment/common/prepare-k8s.sh b/tools/deployment/common/prepare-k8s.sh index a4d3724cf..f2ebe30ae 100755 --- a/tools/deployment/common/prepare-k8s.sh +++ b/tools/deployment/common/prepare-k8s.sh @@ -31,7 +31,7 @@ kubectl label --overwrite nodes --all ceph-mgr=enabled # and we don't need L2 overlay (will be implemented later). kubectl label --overwrite nodes -l "node-role.kubernetes.io/control-plane" l3-agent=enabled -for NAMESPACE in ceph openstack osh-infra; do +for NAMESPACE in ceph mariadb-operator openstack osh-infra; do tee /tmp/${NAMESPACE}-ns.yaml << EOF apiVersion: v1 kind: Namespace diff --git a/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh b/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh new file mode 120000 index 000000000..aa9807064 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh @@ -0,0 +1 @@ +../common/prepare-k8s.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh b/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh new file mode 120000 index 000000000..b1dde55a7 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh @@ -0,0 +1 @@ +../osh-infra-monitoring/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/012-setup-client.sh b/tools/deployment/mariadb-operator-cluster/012-setup-client.sh new file mode 120000 index 000000000..b2416e5e9 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/012-setup-client.sh @@ -0,0 +1 @@ +../common/setup-client.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/020-ingress.sh b/tools/deployment/mariadb-operator-cluster/020-ingress.sh new file mode 120000 index 000000000..2a7183040 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/020-ingress.sh @@ -0,0 +1 @@ +../keystone-auth/020-ingress.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh b/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh new file mode 120000 index 000000000..2d0231b7f --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh @@ -0,0 +1 @@ +../osh-infra-monitoring/030-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh b/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh new file mode 120000 index 000000000..a5eca6ee5 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh @@ -0,0 +1 @@ +../keystone-auth/040-rabbitmq.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh b/tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh new file mode 100755 index 000000000..e50b6dbac --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +# Specify the Rook release tag to use for the Rook operator here +: ${MARIADB_OPERATOR_RELEASE:="0.22.0"} + +# install mariadb-operator +helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator +helm install mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh mariadb-operator + + +#NOTE: Lint and package chart +make mariadb-cluster + +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER:="$(./tools/deployment/common/get-values-overrides.sh mariadb-cluster)"} + +#NOTE: Deploy command +# Deploying downscaled cluster +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +helm upgrade --install mariadb-cluster ./mariadb-cluster \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + --values mariadb-cluster/values_overrides/downscaled.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} + + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +kubectl get pods --namespace=openstack -o wide + +#NOTE: Deploy command +# Upscaling the cluster to 3 instances +# mariadb-operator is not handinling changes in appropriate statefulset +# so a special job has to delete the statefulset in order +# to let mariadb-operator to re-create the sts with new params +helm upgrade --install mariadb-cluster ./mariadb-cluster \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + --values mariadb-cluster/values_overrides/upscaled.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +kubectl get pods --namespace=openstack -o wide + +# Delete the test pod if it still exists +kubectl delete pods -l application=mariadb,release_group=mariadb-cluster,component=test --namespace=openstack --ignore-not-found +#NOTE: Validate the deployment +helm test mariadb-cluster --namespace openstack diff --git a/tools/deployment/mariadb-operator-cluster/050-memcached.sh b/tools/deployment/mariadb-operator-cluster/050-memcached.sh new file mode 120000 index 000000000..3c3fa1821 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/050-memcached.sh @@ -0,0 +1 @@ +../keystone-auth/050-memcached.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/070-keystone.sh b/tools/deployment/mariadb-operator-cluster/070-keystone.sh new file mode 100755 index 000000000..bafe63241 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/070-keystone.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_PATH:="../openstack-helm"} +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +: ${OSH_EXTRA_HELM_ARGS:=""} +: ${OSH_EXTRA_HELM_ARGS_KEYSTONE:="$(./tools/deployment/common/get-values-overrides.sh keystone)"} + +# Install LDAP +make ldap +helm upgrade --install ldap ./ldap \ + --namespace=openstack \ + --set pod.replicas.server=1 \ + --set bootstrap.enabled=true \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_LDAP} + +# Install Keystone +cd ${OSH_PATH} +make keystone +cd - +helm upgrade --install keystone ${OSH_PATH}/keystone \ + --namespace=openstack \ + --values=${OSH_PATH}/keystone/values_overrides/ldap.yaml \ + --set network.api.ingress.classes.namespace=nginx-openstack \ + --set endpoints.oslo_db.hosts.default=mariadb-server-primary \ + ${OSH_EXTRA_HELM_ARGS} \ + ${OSH_EXTRA_HELM_ARGS_KEYSTONE} + +./tools/deployment/common/wait-for-pods.sh openstack + +# Testing basic functionality +export OS_CLOUD=openstack_helm +sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and restart Nginx +openstack endpoint list diff --git a/tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh b/tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh new file mode 100755 index 000000000..cd99e05e6 --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make mariadb-backup + +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP:="$(./tools/deployment/common/get-values-overrides.sh mariadb-backup)"} + +#NOTE: Deploy command +# Deploying downscaled cluster +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +helm upgrade --install mariadb-backup ./mariadb-backup \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP} + + +./tools/deployment/common/wait-for-pods.sh openstack + + +kubectl create job --from=cronjob/mariadb-backup mariadb-backup-manual-001 -n openstack + +./tools/deployment/common/wait-for-pods.sh openstack + +kubectl logs jobs/mariadb-backup-manual-001 -n openstack diff --git a/tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh b/tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh new file mode 100755 index 000000000..ba03e36be --- /dev/null +++ b/tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make prometheus-mysql-exporter + +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-mysql-exporter)"} + +#NOTE: Deploy command +# Deploying downscaled cluster +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +helm upgrade --install prometheus-mysql-exporter ./prometheus-mysql-exporter \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER} + + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +kubectl get pods --namespace=openstack -o wide diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 643ee8bb3..3a56c8199 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -267,4 +267,31 @@ - ./tools/deployment/openstack-support/110-openstack-exporter.sh - ./tools/deployment/openstack-support/120-powerdns.sh - ./tools/deployment/openstack-support/130-cinder.sh + + +- job: + name: openstack-helm-infra-mariadb-operator + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-3nodes-ubuntu_focal + vars: + osh_params: + openstack_release: "2023.1" + container_distro_name: ubuntu + container_distro_version: focal + feature_gates: "prometheus,backups" + gate_scripts: + - ./tools/deployment/openstack-support/000-prepare-k8s.sh + - ./tools/deployment/openstack-support/007-namespace-config.sh + - ./tools/deployment/openstack-support/010-ingress.sh + - ./tools/deployment/ceph/ceph.sh + - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh + - ./tools/deployment/mariadb-operator-cluster/012-setup-client.sh + - ./tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh + - ./tools/deployment/mariadb-operator-cluster/050-memcached.sh + - ./tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh + - ./tools/deployment/mariadb-operator-cluster/070-keystone.sh + - ./tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh + - ./tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh + + ... diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index b1f1b318d..0361c2cbf 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -28,6 +28,7 @@ - openstack-helm-infra-openstack-support-rook - openstack-helm-infra-openstack-support-ssl - openstack-helm-infra-metacontroller + - openstack-helm-infra-mariadb-operator gate: jobs: - openstack-helm-lint