Patroni inclusion work for HA Postgres

This patchset aims to add HA Clustering support for Postgres. HA Clustering
provides automatic failover in the event of the database going down in addition
to keeping replicas of the database for rebuilding in the event of a node
going down. To achieve this clustering we use
[Patroni](https://github.com/zalando/patroni) which offers HA clustering
support for Postgres.

Patroni is a daemon that runs in the background and keeps track of which
node in your cluster is currently the leader node and routes all traffic
on the Postgresql endpoint to that node. If the leader node goes down,
Patroni holds an election to chose a new leader and updates the endpoint
to route traffic accordingly. All communication between nodes is done by
a Patroni created endpoint, seperate from the externally facing Postgres
endpoint.

Note that, although the postgresql helm chart can be upgraded from
non-patroni to patroni clustering, the previous `postgresql`
endpoints object (which is not directly managed by helm) must be
deleted via an out-of-band mechanism so that it may be replaced by the
patroni-managed endpoints.  If Postgres itself is leveraged for the
deployment process, this must be done with careful timing.  Note that
the old endpoints had a port named "db", and the new endpoints has
a port named "postgresql".

- Picking up patchset: https://review.openstack.org/#/c/591663

Co-authored-by: Tony Sorrentino <as1413@att.com>
Co-authored-by: Randeep Jalli <rj2083@att.com>
Co-authored-by: Pete Birley <pete@port.direct>
Co-authored-by: Matt McEuen <mm9745@att.com>

Change-Id: I721b745017dc1ea7ae05dfd9f8d5dd08d0965985

postgresql/templates/bin/_patroni_conversion.sh.tpl

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This script creates the patroni replication user if it doesn't exist.
+# This is only needed for brownfield upgrade scenarios, on top of sites that
+# were greenfield-deployed with a pre-patroni version of postgres.
+#
+# For greenfield deployments, the patroni-enabled postgresql chart will
+# create this user automatically.
+#
+# If any additional conversion steps are found to be needed, they can go here.
+
+set -e
+
+function patroni_started() {
+    HOST=$1
+    PORT=$2
+    STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT};
+      echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3;
+      cat <&3 | tail -n1 | grep -o \"running\"")
+
+    [[ x${STATUS} == "xrunning" ]]
+}
+
+PGDATABASE=${PGDATABASE:-'postgres'}
+PGHOST=${PGHOST:-'127.0.0.1'}
+PGPORT={{- tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+PSQL="psql -h ${PGHOST} -p ${PGPORT} -d ${PGDATABASE}"
+
+PVC_MNT={{- .Values.storage.mount.path }}
+FILE_MADE_BY_POSTGRES=${PVC_MNT}/pgdata/pg_xlog
+FILE_MADE_BY_PATRONI=${PVC_MNT}/pgdata/patroni.dynamic.json
+
+TIMEOUT=0
+
+# Only need to add the user once, on the first replica
+if [ "x${POD_NAME}" != "xpostgresql-0" ]; then
+  echo "Nothing to do on ${POD_NAME}"
+  exit 0
+fi
+
+# Look for a file-based clue that we're migrating from vanilla pg to patroni.
+# This is lighter-weight than checking in the database for the user, since
+# we have to fire up the database at this point to do the check.
+if [[ -e "${FILE_MADE_BY_POSTGRES}" && ! -e "${FILE_MADE_BY_PATRONI}" ]]
+then
+  echo "We are upgrading to Patroni -- checking for replication user"
+
+  # Fire up a temporary postgres
+  /docker-entrypoint.sh postgres &
+  while ! $PSQL -c "select 1;"; do
+    sleep 1
+    if [[ $TIMEOUT -gt 120 ]]; then
+      exit 1
+    fi
+    TIMEOUT=$((TIMEOUT+1))
+  done
+  TIMEOUT=0
+
+  # Add the replication user if it doesn't exist
+  USER_COUNT=$(${PSQL} -qt -c \
+                  "SELECT COUNT(*) FROM pg_roles \
+                   WHERE rolname='${PATRONI_REPLICATION_USERNAME}'")
+
+  if [ ${USER_COUNT} -eq 0 ]; then
+    echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:"
+    ${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \
+                WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';"
+    echo "done."
+  else
+    echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do."
+  fi
+
+  # Start Patroni to assimilate the postgres
+  sed "s/POD_IP_PATTERN/${PATRONI_KUBERNETES_POD_IP}/g" \
+      /tmp/patroni-templated.yaml > /tmp/patroni.yaml
+
+  READY_FLAG="i am the leader with the lock"
+  PATRONI_LOG=/tmp/patroni_conversion.log
+  /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni-templated.yaml &> ${PATRONI_LOG} &
+
+  # Sleep until patroni is running
+  while ! grep -q "${READY_FLAG}" ${PATRONI_LOG}; do
+    sleep 5
+    if [[ $TIMEOUT -gt 24 ]]; then
+      echo "A timeout occurred.  Patroni logs:"
+      cat ${PATRONI_LOG}
+      exit 1
+    fi
+    TIMEOUT=$((TIMEOUT+1))
+  done
+  TIMEOUT=0
+
+  # Gracefully stop postgres and patroni
+  while pkill INT --uid postgres; do
+    sleep 5
+    if [[ $TIMEOUT -gt 24 ]]; then
+      echo "A timeout occurred.  Patroni logs:"
+      cat ${PATRONI_LOG}
+      exit 1
+    fi
+    TIMEOUT=$((TIMEOUT+1))
+  done
+else
+  echo "Patroni is already in place: nothing to do."
+fi

postgresql/templates/bin/_readiness.sh.tpl

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 {{/*
-Copyright 2017 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,4 +18,8 @@ limitations under the License.
 
 set -ex
 
-pg_isready -U ${POSTGRES_USER}
+if [ -f /tmp/postgres-disable-liveness-probe ]; then
+  exit 0
+else
+  pg_isready -U ${PATRONI_SUPERUSER_USERNAME}
+fi

postgresql/templates/bin/_set_password.sh.tpl

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+PGDATABASE=${PGDATABASE:-'postgres'}
+PGHOST=${PGHOST:-'127.0.0.1'}
+PGPORT={{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+
+# These are passed in via the Patroni callback interface
+action="$1"
+role="$2"
+cluster="$3"
+
+# Note: this script when rendered is stored in a secret and encrypted to disk.
+PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }}
+PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }}
+PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }}
+PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }}
+
+if [[ x${role} == "xmaster" ]]; then
+  echo "I have become the patroni master: updating superuser and replication passwords"
+
+  # It can take a few seconds for a freshly promoted leader to become read/write.
+  sleep 10
+  if [[ ! -z "$PATRONI_SUPERUSER_PASSWORD" && ! -z "$PATRONI_SUPERUSER_USERNAME" ]]; then
+    psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_SUPERUSER_USERNAME WITH PASSWORD '$PATRONI_SUPERUSER_PASSWORD';"
+  else
+    echo "WARNING: Did not set superuser password!!!"
+  fi
+
+  if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then
+    psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE"  -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';"
+  else
+    echo "WARNING: Did not set replication user password!!!"
+  fi
+
+  echo "password update complete"
+fi

postgresql/templates/bin/_start.sh.tpl

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 {{/*
-Copyright 2017 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,25 +16,62 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-# Disable echo mode while setting the password
+set -ex
-# unless we are in debug mode
-{{- if .Values.conf.debug }}
-set -x
-{{- end }}
-set -e
 
-POSTGRES_DB=${POSTGRES_DB:-"postgres"}
+function patroni_started() {
+  HOST=$1
+  PORT=$2
+  STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT};
+    echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3;
+    cat <&3 | tail -n1 | grep -o \"running\"")
 
-# Check if the Postgres data directory exists before attempting to
+  [[ x${STATUS} == "xrunning" ]]
-# set the password
+}
-if [[ -d "$PGDATA" && -s "$PGDATA/PG_VERSION" ]]
+SVC_FQDN='{{ tuple "postgresql-restapi" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}'
-then
+SVC_PORT='{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}'
-  postgres --single -D "$PGDATA" "$POSTGRES_DB" <<EOF
-ALTER ROLE $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD'
-EOF
 
+# This is required because the declarative values.yaml config doesn't
+# know the dynamic podIP.  TODO: something more elegant.
+sed "s/POD_IP_PATTERN/${PATRONI_KUBERNETES_POD_IP}/g" \
+    /tmp/patroni-templated.yaml > \
+    /tmp/patroni.yaml
+
+FILE_MADE_BY_PATRONI=${PGDATA}/patroni.dynamic.json
+if [[ ! $POD_NAME -eq "postgresql-0" ]]; then
+
+  echo "I am not postgresql pod zero: disabling liveness probe temporarily"
+  # disable liveness probe as it may take some time for the pod to come online
+  touch /tmp/postgres-disable-liveness-probe
+
+  # During normal upgrades, we just need to turn liveness probes off temporarily
+  # for the sake of password rotation - need to bounce all pods at once
+  # (overriding RollingUpdate) to avoid deadlock.  This accounts for that.
+  sleep 60
+
+  # During initial bootstrapping, we need to sequence 0,1,2
+  if [[ ! -e "${FILE_MADE_BY_PATRONI}" ]]; then
+    echo "patroni has not been initialized on this node"
+    # NOTE: this boolean forces a second check after a delay.  This accounts for a
+    # scenario during initial vanilla postgres -> patroni conversion, where
+    # a temporary master is brought up, killed off, and then restarted.
+    # This can be safely removed in the future, once all clusters are converted.
+    WAITED_EXTRA="false"
+
+    while [ ${WAITED_EXTRA} = "false" ]; do
+      while ! patroni_started "${SVC_FQDN}" "${SVC_PORT}"; do
+        echo "Waiting until a Leader is elected..."
+        sleep 5
+      done
+      # See note above: this code can be removed once all clusters are Patroni.
+      if [ ${WAITED_EXTRA} = "false" ]; then
+        echo "Leader is up; sleeping to ensure it gets through restarts..."
+        sleep 10
+        WAITED_EXTRA="true"
+      fi
+    done
   fi
 
-set -x
+  rm -fv /tmp/postgres-disable-liveness-probe
+fi
 
-exec /docker-entrypoint.sh postgres -N {{ .Values.conf.postgresql.max_connections | quote }} -B {{ .Values.conf.postgresql.shared_buffers | quote }}
+exec /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni.yaml

postgresql/templates/configmap-bin.yaml

@@ -19,24 +19,22 @@ limitations under the License.
 {{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin"  }}
 ---
 apiVersion: v1
-kind: ConfigMap
+{{/* Note: this is a secret because credentials must be rendered into the password script. */}}
+kind: Secret
 metadata:
   name: postgresql-bin
+type: Opaque
 data:
 {{- if .Values.images.local_registry.active }}
-  image-repo-sync.sh: |
+  image-repo-sync.sh: {{- include "helm-toolkit.scripts.image_repo_sync" . | b64enc }}
-{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}
-  start.sh: |
+  start.sh: {{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
-{{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  readiness.sh: {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
-  readiness.sh: |
+  db_test.sh: {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
-{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  db_test.sh: |
-{{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- if .Values.conf.backup.enabled }}
-  backup_postgresql.sh: |
+  backup_postgresql.sh: {{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
-{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  restore_postgresql.sh: {{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
-  restore_postgresql.sh: |
-{{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
+  set_password.sh: {{ tuple "bin/_set_password.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
+  patroni_conversion.sh: {{ tuple "bin/_patroni_conversion.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
 {{- end }}

postgresql/templates/configmap-etc.yaml

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_etc }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgresql-etc
+type: Opaque
+data:
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.patroni "key" "patroni.yaml" "format" "Secret") | indent 2 }}
+{{- end }}

postgresql/templates/cron-job-backup-postgres.yaml

@@ -96,10 +96,10 @@ spec:
               secret:
                 secretName: postgresql-secrets
                 defaultMode: 0600
-            - configMap:
+            - name: postgresql-bin
+              secret:
+                secretName: postgresql-bin
                 defaultMode: 365
-                name: postgresql-bin
-              name: postgresql-bin
             {{- if and .Values.volume.backup.enabled  .Values.manifests.pvc_backup }}
             - name: postgresql-backup-dir
               persistentVolumeClaim:

postgresql/templates/pod-test.yaml

@@ -70,8 +70,8 @@ spec:
     - name: pod-tmp
       emptyDir: {}
     - name: postgresql-bin
-      configMap:
+      secret:
-        name: postgresql-bin
+        secretName: postgresql-bin
         defaultMode: 0555
 ...
 {{- end }}

postgresql/templates/secret-replica.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_replica }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secrets.postgresql.replica }}
+type: Opaque
+data:
+  REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }}
+  REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }}
+{{- end }}

postgresql/templates/service-postgres.yaml

@@ -1,5 +1,5 @@
 {{/*
-Copyright 2017 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -23,8 +23,6 @@ metadata:
   name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
   ports:
-    - name: db
+    - name: postgresql
       port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-  selector:
-{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
 {{- end }}

postgresql/templates/service-restapi.yaml

@@ -0,0 +1,30 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "postgresql-restapi" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+    - name: restapi
+      port: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  selector:
+{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{- end }}

postgresql/templates/statefulset.yaml

@@ -1,5 +1,5 @@
 {{/*
-Copyright 2017 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +20,81 @@ limitations under the License.
 {{- $serviceAccountName := "postgresql" }}
 {{ tuple $envAll "postgresql" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ $envAll.Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      # delete and deletecollection are required only for 'patronictl remove'
+      - delete
+      - deletecollection
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - patch
+      - update
+      # the following three privileges are necessary only when using endpoints
+      - create
+      - list
+      - watch
+      # delete and deletecollection are required only for 'patronictl remove'
+      - delete
+      - deletecollection
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  # The following privilege is only necessary for creation of headless service
+  # for postgresql-config endpoint, in order to prevent cleaning it up by the
+  # k8s master.
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - create
+      - get
+      - list
+      - patch
+      - update
+      - watch
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -28,19 +103,27 @@ metadata:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
   labels:
 {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+    cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
   serviceName: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  podManagementPolicy: "Parallel"
   replicas: {{ .Values.pod.replicas.server }}
   selector:
     matchLabels:
 {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+      cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
   template:
     metadata:
       labels:
 {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+        cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-admin-hash: {{ tuple "secret-admin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-replica-hash: {{ tuple "secret-replica.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-secrets-etc-hash: {{ tuple "secrets-etc.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
       serviceAccountName: {{ $serviceAccountName }}
 {{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
@@ -53,56 +136,193 @@ spec:
         - name: set-volume-perms
 {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          command:
+          command: ["/bin/sh", "-c"]
-            - "/bin/chown"
+          args:
-            - {{ .Values.pod.security_context.server.pod.runAsUser | quote }}
+            - set -xe;
-            - {{ .Values.storage.mount.path | quote }}
+              /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }};
+              /bin/chmod 700 {{ .Values.storage.mount.path }};
+              /bin/chmod 700 {{ .Values.storage.mount.path }}/*;
 {{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
             - name: postgresql-data
               mountPath: {{ .Values.storage.mount.path }}
-              subPath: {{ .Values.storage.mount.subpath }}
+        # This is for non-HA -> Patroni conversion and can be removed in the future
+        - name: patroni-conversion
+{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: PGDATA
+              value: "{{ .Values.storage.mount.path }}/pgdata"
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: PATRONI_KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: PATRONI_SUPERUSER_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.admin }}
+                  key: 'POSTGRES_USER'
+            - name: PATRONI_SUPERUSER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.admin }}
+                  key: 'POSTGRES_PASSWORD'
+            - name: PATRONI_REPLICATION_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.replica }}
+                  key: 'REPLICA_USER'
+            - name: PATRONI_REPLICATION_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.replica }}
+                  key: 'REPLICA_PASSWORD'
+            - name: PATRONI_RESTAPI_CONNECT_ADDRESS
+              value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_RESTAPI_LISTEN
+              value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS
+              value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_POSTGRESQL_LISTEN
+              value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_admin_PASSWORD
+              value: $(PATRONI_SUPERUSER_PASSWORD)
+            - name: PATRONI_admin_OPTIONS
+              value: 'createrole,createdb'
+          command:
+            - /tmp/patroni_conversion.sh
+{{ dict "envAll" $envAll "application" "server" "container" "patroni_conversion" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: patroni-conversion-tmp
+              mountPath: /var/run/postgresql
+            - name: postgresql-bin
+              mountPath: /tmp/patroni_conversion.sh
+              subPath: patroni_conversion.sh
+              readOnly: true
+            - name: postgresql-data
+              mountPath: {{ .Values.storage.mount.path }}
+            - name: postgresql-etc
+              mountPath: /tmp/patroni-templated.yaml
+              subPath: patroni.yaml
+              readOnly: true
       containers:
         - name: postgresql
 {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "server" "container" "postgresql" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           ports:
+            - containerPort: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              protocol: TCP
             - containerPort: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              protocol: TCP
           env:
-            - name: 'POSTGRES_PASSWORD'
+            - name: PGDATA
+              value: "{{ .Values.storage.mount.path }}/pgdata"
+            - name: PATRONI_KUBERNETES_POD_IP
               valueFrom:
-                secretKeyRef:
+                fieldRef:
-                  name: {{ .Values.secrets.postgresql.admin }}
+                  fieldPath: status.podIP
-                  key: 'POSTGRES_PASSWORD'
+            - name: PATRONI_KUBERNETES_NAMESPACE
-            - name: 'POSTGRES_USER'
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: PATRONI_SUPERUSER_USERNAME
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.secrets.postgresql.admin }}
                   key: 'POSTGRES_USER'
-            - name: 'PGDATA'
+            - name: PATRONI_SUPERUSER_PASSWORD
-              value: {{ .Values.storage.mount.path | quote }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.admin }}
+                  key: 'POSTGRES_PASSWORD'
+            - name: PATRONI_REPLICATION_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.replica }}
+                  key: 'REPLICA_USER'
+            - name: PATRONI_REPLICATION_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.postgresql.replica }}
+                  key: 'REPLICA_PASSWORD'
+            - name: PATRONI_RESTAPI_CONNECT_ADDRESS
+              value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_RESTAPI_LISTEN
+              value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS
+              value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_POSTGRESQL_LISTEN
+              value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: PATRONI_admin_PASSWORD
+              value: $(PATRONI_SUPERUSER_PASSWORD)
+            - name: PATRONI_admin_OPTIONS
+              value: 'createrole,createdb'
           command:
             - /tmp/start.sh
           livenessProbe:
             exec:
               command:
                 - /tmp/readiness.sh
-            initialDelaySeconds: 20
+            initialDelaySeconds: 30
             timeoutSeconds: 5
+            failureThreshold: 10
           readinessProbe:
             exec:
               command:
                 - /tmp/readiness.sh
-            initialDelaySeconds: 20
+            initialDelaySeconds: 30
             timeoutSeconds: 5
+            failureThreshold: 10
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
             - name: pg-run
               mountPath: /var/run/postgresql
+            - name: postgresql-bin
+              mountPath: /tmp/set_password.sh
+              subPath: set_password.sh
+              readOnly: true
             - name: postgresql-bin
               mountPath: /tmp/start.sh
               subPath: start.sh
@@ -111,19 +331,29 @@ spec:
               mountPath: /tmp/readiness.sh
               subPath: readiness.sh
               readOnly: true
+            - name: postgresql-etc
+              mountPath: /tmp/patroni-templated.yaml
+              subPath: patroni.yaml
+              readOnly: true
             - name: postgresql-data
               mountPath: {{ .Values.storage.mount.path }}
-              subPath: {{ .Values.storage.mount.subpath }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
         - name: pg-run
           emptyDir:
             medium: "Memory"
+        # This is for non-HA -> Patroni conversion and can be removed in the future
+        - name: patroni-conversion-tmp
+          emptyDir: {}
         - name: postgresql-bin
-          configMap:
+          secret:
-            name: postgresql-bin
+            secretName: postgresql-bin
             defaultMode: 0555
+        - name: postgresql-etc
+          secret:
+            secretName: postgresql-etc
+            defaultMode: 0444
 {{- if not .Values.storage.pvc.enabled }}
         - name: postgresql-data
           hostPath:

postgresql/values.yaml

@@ -1,4 +1,4 @@
-# Copyright 2017 The Openstack-Helm Authors.
+# Copyright 2019 The Openstack-Helm Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,6 +37,14 @@ pod:
         postgresql:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
+        patroni_conversion:
+          runAsUser: 999
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+      pod:
+        runAsUser: 999
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
   affinity:
     anti:
       type:
@@ -46,8 +54,7 @@ pod:
       weight:
         default: 10
   replicas:
-    #only 1 replica currently supported
+    server: 3
-    server: 1
     prometheus_postgresql_exporter: 1
   lifecycle:
     upgrades:
@@ -106,10 +113,10 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
-# using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/
+# using dockerhub patroni: https://hub.docker.com/r/openstackhelm/patroni/tags/
 images:
   tags:
-    postgresql: "docker.io/postgres:9.5"
+    postgresql: "docker.io/openstackhelm/patroni:latest-ubuntu_xenial"
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     image_repo_sync: docker.io/docker:17.07.0
     prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
@@ -131,8 +138,8 @@ storage:
   host:
     host_path: /data/openstack-helm/postgresql
   mount:
-    path: /var/lib/postgresql/data
+    path: /var/lib/postgresql
-    subpath: pgdata
+    subpath: .
 
 labels:
   server:
@@ -205,8 +212,137 @@ jobs:
 conf:
   debug: false
   postgresql:
-    max_connections: 100
     shared_buffers: 128MB
+    max_connections: 100
+  patroni: |
+    scope: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+    kubernetes:
+      labels:
+        application: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+        component: server
+      use_endpoints: true
+      ports:
+      - name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+        port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    bootstrap:
+      dcs:
+        ttl: 30
+        loop_wait: 10
+        retry_timeout: 10
+        maximum_lag_on_failover: 1048576
+        postgresql:
+          data_dir: '{{ .Values.storage.mount.path }}/pgdata'
+          pgpass: '{{ .Values.storage.mount.path }}/pgpass'
+          use_pg_rewind: true
+          parameters:
+            archive_mode: 'on'
+            archive_timeout: 1800s
+            autovacuum_analyze_scale_factor: 0.02
+            autovacuum_max_workers: 5
+            autovacuum_vacuum_scale_factor: 0.05
+            checkpoint_completion_target: 0.9
+            datestyle: 'iso, mdy'
+            default_text_search_config: 'pg_catalog.english'
+            external_pid_file: '/tmp/postgres.pid'
+            hot_standby: 'on'
+            lc_messages: 'en_US.utf8'
+            lc_monetary: 'en_US.utf8'
+            lc_numeric: 'en_US.utf8'
+            lc_time: 'en_US.utf8'
+            log_autovacuum_min_duration: 0
+            log_checkpoints: 'on'
+            log_connections: 'on'
+            log_disconnections: 'on'
+            log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m '
+            log_lock_waits: 'on'
+            log_min_duration_statement: 500
+            log_statement: none
+            log_temp_files: 0
+            log_timezone: 'UTC'
+            max_connections: {{ .Values.conf.postgresql.max_connections }}
+            max_replication_slots: 10
+            max_wal_senders: 10
+            max_worker_processes: 10
+            tcp_keepalives_idle: 900
+            tcp_keepalives_interval: 100
+            timezone: 'UTC'
+            track_commit_timestamp: 'on'
+            track_functions: all
+            wal_keep_segments: 100
+            wal_level: 'logical'
+            wal_log_hints: 'on'
+      initdb:
+      - auth-host: md5
+      - auth-local: trust
+      - encoding: UTF8
+      - locale: en_US.UTF-8
+      - data-checksums
+      pg_hba:
+      - host all all 127.0.0.1/32 trust
+      - host all all 0.0.0.0/0 md5
+      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5  # Fixes issue with Postgres 9.5
+      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
+      - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
+      - local all all trust
+    postgresql:
+      {{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data,
+           so let's just avoid it and use /var/lib/postgresql/pgdata instead.
+           Patroni moves this directory to a backup under the parent directory
+           (/var/lib/postgresql) under certain failure recovery scenarios, so
+           /var/lib/postgres itself must be exposed to the pod as a pvc mount.*/}}
+      data_dir: '{{ .Values.storage.mount.path }}/pgdata'
+      pgpass: '{{ .Values.storage.mount.path }}/pgpass'
+      callbacks:
+        on_role_change: /tmp/set_password.sh
+        on_start: /tmp/set_password.sh
+      use_pg_rewind: true
+      parameters:
+        archive_mode: 'on'
+        archive_timeout: 1800s
+        autovacuum_analyze_scale_factor: 0.02
+        autovacuum_max_workers: 5
+        autovacuum_vacuum_scale_factor: 0.05
+        checkpoint_completion_target: 0.9
+        datestyle: 'iso, mdy'
+        default_text_search_config: 'pg_catalog.english'
+        external_pid_file: '/tmp/postgres.pid'
+        hot_standby: 'on'
+        lc_messages: 'en_US.utf8'
+        lc_monetary: 'en_US.utf8'
+        lc_numeric: 'en_US.utf8'
+        lc_time: 'en_US.utf8'
+        log_autovacuum_min_duration: 0
+        log_checkpoints: 'on'
+        log_connections: 'on'
+        log_disconnections: 'on'
+        log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m '
+        log_lock_waits: 'on'
+        log_min_duration_statement: 500
+        log_statement: none
+        log_temp_files: 0
+        log_timezone: 'UTC'
+        max_connections: {{ .Values.conf.postgresql.max_connections }}
+        max_replication_slots: 10
+        max_wal_senders: 10
+        max_worker_processes: 10
+        tcp_keepalives_idle: 900
+        tcp_keepalives_interval: 100
+        timezone: 'UTC'
+        track_commit_timestamp: 'on'
+        track_functions: all
+        shared_buffers: {{ .Values.conf.postgresql.shared_buffers }}
+        wal_keep_segments: 100
+        wal_level: 'logical'
+        wal_log_hints: 'on'
+      pg_hba:
+      - host all all 127.0.0.1/32 trust
+      - host all all 0.0.0.0/0 md5
+      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5  # Fixes issue with Postgres 9.5
+      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
+      - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
+      - local all all trust
+    watchdog:
+      mode: off  # Allowed values: off, automatic, required
   backup:
     enabled: true
     base_path: /var/backup
@@ -216,6 +352,7 @@ conf:
 secrets:
   postgresql:
     admin: postgresql-admin
+    replica: postgresql-replication
     exporter: postgresql-exporter
 
 endpoints:
@@ -237,6 +374,9 @@ endpoints:
       admin:
         username: postgres
         password: password
+      replica:
+        username: standby
+        password: password
       exporter:
         username: psql_exporter
         password: psql_exp_pass
@@ -249,6 +389,16 @@ endpoints:
     port:
       postgresql:
         default: 5432
+  postgresql_restapi:
+    hosts:
+      default: postgresql-restapi
+    host_fqdn_override:
+      default: null
+    path: null
+    scheme: postgresql
+    port:
+      restapi:
+        default: 8008
   prometheus_postgresql_exporter:
     namespace: null
     hosts:
@@ -265,8 +415,10 @@ endpoints:
 
 manifests:
   configmap_bin: true
+  configmap_etc: true
   job_image_repo_sync: true
   secret_admin: true
+  secret_replica: true
   secret_etc: true
   service: true
   statefulset: true

tools/deployment/osh-infra-monitoring/130-postgresql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2017 The Openstack-Helm Authors.
+# Copyright 2019 The Openstack-Helm Authors.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -25,6 +25,8 @@ helm upgrade --install postgresql ./postgresql \
     --namespace=osh-infra \
     --set monitoring.prometheus.enabled=true \
     --set storage.pvc.size=1Gi \
+    --set storage.pvc.enabled=true \
+    --set pod.replicas.server=3 \
     ${OSH_INFRA_EXTRA_HELM_ARGS} \
     ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB}