Merge "[mariadb] Add mariadb-server-primary service"

mariadb/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.39
+version: 0.2.40
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:

mariadb/templates/bin/_start.py.tpl

@@ -80,6 +80,10 @@ if check_env_var("STATE_CONFIGMAP"):
     state_configmap_name = os.environ['STATE_CONFIGMAP']
     logger.info("Will use \"{0}\" configmap for cluster state info".format(
         state_configmap_name))
+if check_env_var("PRIMARY_SERVICE_NAME"):
+    primary_service_name = os.environ['PRIMARY_SERVICE_NAME']
+    logger.info("Will use \"{0}\" service as primary".format(
+        primary_service_name))
 if check_env_var("POD_NAMESPACE"):
     pod_namespace = os.environ['POD_NAMESPACE']
 if check_env_var("DIRECT_SVC_NAME"):
@@ -92,6 +96,8 @@ if check_env_var("DISCOVERY_DOMAIN"):
     discovery_domain = os.environ['DISCOVERY_DOMAIN']
 if check_env_var("WSREP_PORT"):
     wsrep_port = os.environ['WSREP_PORT']
+if check_env_var("MARIADB_PORT"):
+    mariadb_port = int(os.environ['MARIADB_PORT'])
 if check_env_var("MYSQL_DBADMIN_USERNAME"):
     mysql_dbadmin_username = os.environ['MYSQL_DBADMIN_USERNAME']
 if check_env_var("MYSQL_DBADMIN_PASSWORD"):
@@ -115,7 +121,8 @@ if mysql_dbadmin_username == mysql_dbsst_username:
     sys.exit(1)
 
 # Set some variables for tuneables
-cluster_leader_ttl = 120
+if check_env_var("CLUSTER_LEADER_TTL"):
+    cluster_leader_ttl = int(os.environ['CLUSTER_LEADER_TTL'])
 state_configmap_update_period = 10
 default_sleep = 20
 
@@ -138,6 +145,25 @@ def ensure_state_configmap(pod_namespace, configmap_name, configmap_body):
 
         return False
 
+def ensure_primary_service(pod_namespace, service_name, service_body):
+    """Ensure the primary service exists.
+
+    Keyword arguments:
+    pod_namespace -- the namespace to house the service
+    service_name -- the service name
+    service_body -- the service body
+    """
+    try:
+        k8s_api_instance.read_namespaced_service(
+            name=service_name, namespace=pod_namespace)
+        return True
+    except:
+        k8s_api_instance.create_namespaced_service(
+            namespace=pod_namespace, body=service_body)
+
+        return False
+
+
 
 def run_cmd_with_logging(popenargs,
                          logger,
@@ -388,6 +414,60 @@ def set_configmap_data(key, value):
     return safe_update_configmap(
         configmap_dict=configmap_dict, configmap_patch=configmap_patch)
 
+def safe_update_service(service_dict, service_patch):
+    """Update a service with locking.
+
+    Keyword arguments:
+    service_dict -- a dict representing the service to be patched
+    service_patch -- a dict containign the patch
+    """
+    logger.debug("Safe Patching service")
+    # NOTE(portdirect): Explictly set the resource version we are patching to
+    # ensure nothing else has modified the service since we read it.
+    service_patch['metadata']['resourceVersion'] = service_dict[
+        'metadata']['resource_version']
+
+    # Retry up to 8 times in case of 409 only.  Each retry has a ~1 second
+    # sleep in between so do not want to exceed the roughly 10 second
+    # write interval per cm update.
+    for i in range(8):
+        try:
+            api_response = k8s_api_instance.patch_namespaced_service(
+                name=primary_service_name,
+                namespace=pod_namespace,
+                body=service_patch)
+            return True
+        except kubernetes.client.rest.ApiException as error:
+            if error.status == 409:
+                # This status code indicates a collision trying to write to the
+                # service while another instance is also trying the same.
+                logger.warning("Collision writing service: {0}".format(error))
+                # This often happens when the replicas were started at the same
+                # time, and tends to be persistent. Sleep with some random
+                # jitter value briefly to break the synchronization.
+                naptime = secretsGen.uniform(0.8,1.2)
+                time.sleep(naptime)
+            else:
+                logger.error("Failed to set service: {0}".format(error))
+                return error
+        logger.info("Retry writing service attempt={0} sleep={1}".format(
+            i+1, naptime))
+    return True
+
+def set_primary_service_spec(key, value):
+    """Update a service's endpoint via patching.
+
+    Keyword arguments:
+    key -- the key to be patched
+    value -- the value to give the key
+    """
+    logger.debug("Setting service spec.selector key={0} to value={1}".format(key, value))
+    service_dict = k8s_api_instance.read_namespaced_service(
+        name=primary_service_name, namespace=pod_namespace).to_dict()
+    service_patch = {'spec': {'selector': {}}, 'metadata': {}}
+    service_patch['spec']['selector'][key] = value
+    return safe_update_service(
+        service_dict=service_dict, service_patch=service_patch)
 
 def get_configmap_value(key, type='data'):
     """Get a configmap's key's value.
@@ -469,6 +549,35 @@ def get_cluster_state():
                 pod_namespace=pod_namespace,
                 configmap_name=state_configmap_name,
                 configmap_body=initial_configmap_body)
+
+
+            initial_primary_service_body = {
+                "apiVersion": "v1",
+                "kind": "Service",
+                "metadata": {
+                    "name": primary_service_name,
+                },
+                "spec": {
+                    "ports": [
+                        {
+                            "name": "mysql",
+                            "port": mariadb_port
+                        }
+                    ],
+                    "selector": {
+                        "application": "mariadb",
+                        "component": "server",
+                        "statefulset.kubernetes.io/pod-name": leader
+                    }
+                }
+            }
+            if ensure_primary_service(
+                    pod_namespace=pod_namespace,
+                    service_name=primary_service_name,
+                    service_body=initial_primary_service_body):
+                logger.info("Service {0} already exists".format(primary_service_name))
+            else:
+                logger.info("Service {0} has been successfully created".format(primary_service_name))
     return state
 
 
@@ -480,6 +589,38 @@ def declare_myself_cluster_leader():
     leader_expiry = "{0}Z".format(leader_expiry_raw.isoformat("T"))
     set_configmap_annotation(
         key='openstackhelm.openstack.org/leader.node', value=local_hostname)
+    logger.info("Setting primary_service's spec.selector to {0}".format(local_hostname))
+    try:
+        set_primary_service_spec(
+            key='statefulset.kubernetes.io/pod-name', value=local_hostname)
+    except:
+        initial_primary_service_body = {
+            "apiVersion": "v1",
+            "kind": "Service",
+            "metadata": {
+                "name": primary_service_name,
+            },
+            "spec": {
+                "ports": [
+                    {
+                        "name": "mysql",
+                        "port": mariadb_port
+                    }
+                ],
+                "selector": {
+                    "application": "mariadb",
+                    "component": "server",
+                    "statefulset.kubernetes.io/pod-name": local_hostname
+                }
+            }
+        }
+        if ensure_primary_service(
+                pod_namespace=pod_namespace,
+                service_name=primary_service_name,
+                service_body=initial_primary_service_body):
+            logger.info("Service {0} already exists".format(primary_service_name))
+        else:
+            logger.info("Service {0} has been successfully created".format(primary_service_name))
     set_configmap_annotation(
         key='openstackhelm.openstack.org/leader.expiry', value=leader_expiry)
 

mariadb/templates/statefulset.yaml

@@ -47,6 +47,29 @@ rules:
       - configmaps
     verbs:
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resourceNames:
+      - {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+    resources:
+      - services
+    verbs:
+      - get
+      - patch
+  - apiGroups:
+      - ""
+    resourceNames:
+      - {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+    resources:
+      - endpoints
+    verbs:
+      - get
   - apiGroups:
       - ""
     resourceNames:
@@ -165,6 +188,12 @@ spec:
               value: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: STATE_CONFIGMAP
               value: {{ printf "%s-%s" .deployment_name "mariadb-state" | quote }}
+            - name: PRIMARY_SERVICE_NAME
+              value: {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+            - name: CLUSTER_LEADER_TTL
+              value: {{ .Values.conf.galera.cluster_leader_ttl | quote }}
+            - name: MARIADB_PORT
+              value: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: MYSQL_DBADMIN_USERNAME
               value: {{ .Values.endpoints.oslo_db.auth.admin.username }}
             - name: MYSQL_DBADMIN_PASSWORD

mariadb/values.yaml

@@ -362,6 +362,8 @@ conf:
         lock_expire_after: 7200
         retry_after: 3600
         container_name: throttle-backups-manager
+  galera:
+    cluster_leader_ttl: 120
   database:
     mysql_histfile: "/dev/null"
     my: |
@@ -603,6 +605,7 @@ endpoints:
       direct: mariadb-server
       discovery: mariadb-discovery
       error_pages: mariadb-ingress-error-pages
+      primary: mariadb-server-primary
     host_fqdn_override:
       default: null
     path: null

mariadb/values_overrides/primary-service.yaml

@@ -0,0 +1,21 @@
+---
+manifests:
+  deployment_ingress: false
+  deployment_error: false
+  service_ingress: false
+  configmap_ingress_conf: false
+  configmap_ingress_etc: false
+  service_error: false
+volume:
+  size: 1Gi
+  backup:
+    size: 1Gi
+conf:
+  galera:
+    cluster_leader_ttl: 10
+endpoints:
+  oslo_db:
+    hosts:
+      default: mariadb
+      primary: mariadb
+...

releasenotes/notes/mariadb.yaml

@@ -55,4 +55,5 @@ mariadb:
   - 0.2.37 Backups verification improvements
   - 0.2.38 Added throttling remote backups
   - 0.2.39 Template changes for image 1.9 compatibility
+  - 0.2.40 Start.py allows to create mariadb-service-primary service and endpoint
 ...

zuul.d/jobs.yaml

@@ -266,6 +266,34 @@
         - ./tools/deployment/openstack-support/120-powerdns.sh
         - ./tools/deployment/openstack-support/130-cinder.sh
 
+- job:
+    name: openstack-helm-infra-openstack-support-mariadb-service-primary
+    parent: openstack-helm-infra-deploy
+    nodeset: openstack-helm-1node-ubuntu_focal
+    vars:
+      osh_params:
+        openstack_release: "2023.1"
+        container_distro_name: ubuntu
+        container_distro_version: focal
+        feature_gates: "ssl,primary-service"
+      gate_scripts:
+        - ./tools/deployment/openstack-support/000-prepare-k8s.sh
+        - ./tools/deployment/openstack-support/007-namespace-config.sh
+        - ./tools/deployment/openstack-support/010-ingress.sh
+        - ./tools/deployment/ceph/ceph.sh
+        - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh
+        - ./tools/deployment/openstack-support/030-rabbitmq.sh
+        - ./tools/deployment/openstack-support/070-mariadb.sh
+        - ./tools/deployment/openstack-support/040-memcached.sh
+        - ./tools/deployment/openstack-support/051-libvirt-ssl.sh
+        - ./tools/deployment/openstack-support/060-openvswitch.sh
+        - ./tools/deployment/common/setup-client.sh
+        - ./tools/deployment/openstack-support/090-keystone.sh
+        - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
+        - ./tools/deployment/openstack-support/110-openstack-exporter.sh
+        - ./tools/deployment/openstack-support/120-powerdns.sh
+        - ./tools/deployment/openstack-support/130-cinder.sh
+
 
 - job:
     name: openstack-helm-infra-mariadb-operator

zuul.d/project.yaml

@@ -29,6 +29,7 @@
         - openstack-helm-infra-openstack-support-ssl
         - openstack-helm-infra-metacontroller
         - openstack-helm-infra-mariadb-operator
+        - openstack-helm-infra-openstack-support-mariadb-service-primary
     gate:
       jobs:
         - openstack-helm-lint
@@ -38,6 +39,7 @@
         - openstack-helm-infra-openstack-support
         - openstack-helm-infra-openstack-support-rook
         - openstack-helm-infra-openstack-support-ssl
+        - openstack-helm-infra-openstack-support-mariadb-service-primary
     post:
       jobs:
         - publish-openstack-helm-charts