[mariadb] Add mariadb controller support

This patch adds mairadb controller that is responsible to mark one
ready pod as mariadb_role: primary to forward all traffic to it.
This will allow to drop nginx ingress controller which adds extra
hops between client and server and uses heavy customized nginx templates.

Change-Id: I3b29bc2029bfd39754516e73a09e4e14c52ccc99

mariadb/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.44
+version: 0.2.45
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb/templates/bin/_mariadb_controller.py.tpl

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""
+Mariadb controller
+
+The script is responsible for set mariadb_role: primary to first
+active pod in mariadb deployment.
+
+Env variables:
+MARIADB_CONTROLLER_DEBUG: Flag to enable debug when set to 1.
+MARIADB_CONTROLLER_CHECK_PODS_DELAY: The delay between check pod attempts.
+MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: The timeout for kubernetes http session
+MARIADB_CONTROLLER_PODS_NAMESPACE: The namespace to look for mariadb pods.
+MARIADB_MASTER_SERVICE_NAME: The name of master service for mariadb.
+
+Changelog:
+0.1.0: Initial varsion
+"""
+
+
+import logging
+import os
+import sys
+import time
+
+import pykube
+
+MARIADB_CONTROLLER_DEBUG = os.getenv("MARIADB_CONTROLLER_DEBUG")
+MARIADB_CONTROLLER_CHECK_PODS_DELAY = int(
+    os.getenv("MARIADB_CONTROLLER_CHECK_PODS_DELAY", 10)
+)
+MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT = int(
+    os.getenv("MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT", 60)
+)
+MARIADB_CONTROLLER_PODS_NAMESPACE = os.getenv(
+    "MARIADB_CONTROLLER_PODS_NAMESPACE", "openstack"
+)
+MARIADB_MASTER_SERVICE_NAME = os.getenv(
+    "MARIADB_MASTER_SERVICE_NAME", "mariadb"
+)
+
+log_level = "DEBUG" if MARIADB_CONTROLLER_DEBUG else "INFO"
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s %(levelname)s %(name)s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+LOG = logging.getLogger("mariadb-controller")
+
+LOG.setLevel(log_level)
+
+
+def login():
+    config = pykube.KubeConfig.from_env()
+    client = pykube.HTTPClient(
+        config=config, timeout=MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT
+    )
+    LOG.info(f"Created k8s api client from context {config.current_context}")
+    return client
+
+
+api = login()
+
+
+def resource_list(klass, selector, namespace=None):
+    return klass.objects(api).filter(namespace=namespace, selector=selector)
+
+
+def get_mariadb_pods():
+    sorted_pods = sorted(
+        resource_list(
+            pykube.Pod,
+            {"application": "mariadb", "component": "server"},
+            MARIADB_CONTROLLER_PODS_NAMESPACE,
+        ).iterator(),
+        key=lambda i: i.name,
+    )
+    return sorted_pods
+
+
+def get_mariadb_master_service(namespace):
+    return pykube.Service.objects(api).filter(namespace=namespace).get(name=MARIADB_MASTER_SERVICE_NAME)
+
+
+def link_master_service(pod):
+    svc = get_mariadb_master_service(MARIADB_CONTROLLER_PODS_NAMESPACE)
+    svc.reload()
+    if svc.obj['spec']['selector'].get('statefulset.kubernetes.io/pod-name') == pod.name:
+        LOG.debug(f"Nothing to do, master service points to {pod.name}")
+    else:
+        svc.obj['spec']['selector']['statefulset.kubernetes.io/pod-name'] = pod.name
+        svc.update()
+        LOG.info(f"Link master service with {pod.name}")
+
+
+def is_ready(pod):
+    if pod.ready and "deletionTimestamp" not in pod.metadata:
+        return True
+
+
+def main():
+    while True:
+        for pod in get_mariadb_pods():
+            pod.reload()
+            if is_ready(pod):
+                link_master_service(pod)
+                break
+        LOG.debug(f"Sleeping for {MARIADB_CONTROLLER_CHECK_PODS_DELAY}")
+        time.sleep(MARIADB_CONTROLLER_CHECK_PODS_DELAY)
+
+
+main()

mariadb/templates/configmap-bin.yaml

@@ -53,4 +53,8 @@ data:
   ks-user.sh: |
 {{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
 {{- end }}
+{{- if .Values.manifests.deployment_controller }}
+  mariadb_controller.py: |
+{{ tuple "bin/_mariadb_controller.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
 {{- end }}

mariadb/templates/deployment-controller.yaml

@@ -0,0 +1,119 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.deployment_controller }}
+{{- if .Values.manifests.deployment_ingress }}
+{{- fail ".Values.manifests.deployment_ingress and .Values.manifests.deployment_controlle are mutually exclusive" }}
+{{- end }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "mariadb-controller" }}
+{{ tuple $envAll "controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod
+  namespace: {{ $envAll.Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - update
+      - patch
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod
+  namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb-controller
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  replicas: {{ .Values.pod.replicas.controller }}
+  selector:
+    matchLabels:
+{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+{{ dict "envAll" $envAll "application" "controller" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      affinity:
+{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      nodeSelector:
+        {{ .Values.labels.controller.node_selector_key }}: {{ .Values.labels.controller.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: controller
+{{ tuple $envAll "mariadb_controller" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/mariadb_controller.py
+          env:
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.mariadb_controller | indent 12 }}
+            - name: MARIADB_CONTROLLER_PODS_NAMESPACE
+              value: {{ $envAll.Release.Namespace }}
+            - name: MARIADB_MASTER_SERVICE_NAME
+              value: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - mountPath: /tmp/mariadb_controller.py
+              name: mariadb-bin
+              readOnly: true
+              subPath: mariadb_controller.py
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: mariadb-bin
+          configMap:
+            name: mariadb-bin
+            defaultMode: 365
+{{- end }}

mariadb/templates/service-discovery.yaml

@@ -25,8 +25,13 @@ spec:
       port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     - name: wsrep
       port: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    - name: ist
+      port: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    - name: sst
+      port: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   clusterIP: None
   publishNotReadyAddresses: true
   selector:
 {{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ .Values.network.mariadb_discovery | include "helm-toolkit.snippets.service_params" | indent 2 }}
 {{- end }}

mariadb/templates/service-master.yaml

@@ -0,0 +1,33 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service_master }}
+{{- if .Values.manifests.service_ingress }}
+{{- fail ".Values.manifests.service_ingress and .Values.manifests.service_master are mutually exclusive" }}
+{{- end }}
+
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+    - name: mysql
+      port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  selector:
+{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ .Values.network.mariadb_master | include "helm-toolkit.snippets.service_params" | indent 2 }}
+{{- end }}

mariadb/templates/service.yaml

@@ -25,4 +25,5 @@ spec:
       port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
 {{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ .Values.network.mariadb | include "helm-toolkit.snippets.service_params" | indent 2 }}
 {{- end }}

mariadb/templates/statefulset.yaml

@@ -226,6 +226,12 @@ spec:
             - name: wsrep
               protocol: TCP
               containerPort: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: ist
+              protocol: TCP
+              containerPort: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: sst
+              protocol: TCP
+              containerPort: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           command:
             - /tmp/start.py
           lifecycle:

mariadb/values.yaml

@@ -31,6 +31,7 @@ images:
     mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal
     ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
     scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
+    mariadb_controller: docker.io/openstackhelm/mariadb:latest-ubuntu_focal
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -57,8 +58,16 @@ labels:
   test:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  controller:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 pod:
+  env:
+    mariadb_controller:
+      MARIADB_CONTROLLER_DEBUG: 0
+      MARIADB_CONTROLLER_CHECK_PODS_DELAY: 10
+      MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: 60
   probes:
     server:
       mariadb:
@@ -136,6 +145,13 @@ pod:
         test:
           runAsUser: 999
           readOnlyRootFilesystem: true
+    controller:
+      pod:
+        runAsUser: 65534
+      container:
+        controller:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
   affinity:
     anti:
       type:
@@ -159,6 +175,7 @@ pod:
     ingress: 2
     error_page: 1
     prometheus_mysql_exporter: 1
+    controller: 1
   lifecycle:
     upgrades:
       deployments:
@@ -282,7 +299,8 @@ dependencies:
       services:
         - endpoint: internal
           service: oslo_db
-
+    controller:
+      services: null
 volume:
   # this value is used for single pod deployments of mariadb to prevent losing all data
   # if the pod is restarted
@@ -615,6 +633,10 @@ endpoints:
         default: 3306
       wsrep:
         default: 4567
+      ist:
+        default: 4568
+      sst:
+        default: 4444
   kube_dns:
     namespace: kube-system
     name: kubernetes-dns
@@ -670,6 +692,13 @@ endpoints:
         default: 80
         internal: 5000
 
+network:
+  mariadb: {}
+  mariadb_discovery: {}
+  mariadb_ingress: {}
+  mariadb_ingress_error_pages: {}
+  mariadb_master: {}
+
 network_policy:
   mariadb:
     ingress:
@@ -721,4 +750,6 @@ manifests:
   service_error: false
   service: true
   statefulset: true
+  deployment_controller: false
+  service_master: false
 ...

releasenotes/notes/mariadb.yaml

@@ -60,4 +60,5 @@ mariadb:
   - 0.2.42 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default
   - 0.2.43 Add 2024.1 Ubuntu Jammy overrides
   - 0.2.44 Uplift ingress controller image to 1.11.2
+  - 0.2.45 Add mariadb controller support
 ...