diff --git a/postgresql/templates/bin/_patroni_conversion.sh.tpl b/postgresql/templates/bin/_patroni_conversion.sh.tpl new file mode 100644 index 000000000..318ed4d08 --- /dev/null +++ b/postgresql/templates/bin/_patroni_conversion.sh.tpl @@ -0,0 +1,121 @@ +#!/bin/bash + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This script creates the patroni replication user if it doesn't exist. +# This is only needed for brownfield upgrade scenarios, on top of sites that +# were greenfield-deployed with a pre-patroni version of postgres. +# +# For greenfield deployments, the patroni-enabled postgresql chart will +# create this user automatically. +# +# If any additional conversion steps are found to be needed, they can go here. + +set -e + +function patroni_started() { + HOST=$1 + PORT=$2 + STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT}; + echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3; + cat <&3 | tail -n1 | grep -o \"running\"") + + [[ x${STATUS} == "xrunning" ]] +} + +PGDATABASE=${PGDATABASE:-'postgres'} +PGHOST=${PGHOST:-'127.0.0.1'} +PGPORT={{- tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +PSQL="psql -h ${PGHOST} -p ${PGPORT} -d ${PGDATABASE}" + +PVC_MNT={{- .Values.storage.mount.path }} +FILE_MADE_BY_POSTGRES=${PVC_MNT}/pgdata/pg_xlog +FILE_MADE_BY_PATRONI=${PVC_MNT}/pgdata/patroni.dynamic.json + +TIMEOUT=0 + +# Only need to add the user once, on the first replica +if [ "x${POD_NAME}" != "xpostgresql-0" ]; then + echo "Nothing to do on ${POD_NAME}" + exit 0 +fi + +# Look for a file-based clue that we're migrating from vanilla pg to patroni. +# This is lighter-weight than checking in the database for the user, since +# we have to fire up the database at this point to do the check. +if [[ -e "${FILE_MADE_BY_POSTGRES}" && ! -e "${FILE_MADE_BY_PATRONI}" ]] +then + echo "We are upgrading to Patroni -- checking for replication user" + + # Fire up a temporary postgres + /docker-entrypoint.sh postgres & + while ! $PSQL -c "select 1;"; do + sleep 1 + if [[ $TIMEOUT -gt 120 ]]; then + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done + TIMEOUT=0 + + # Add the replication user if it doesn't exist + USER_COUNT=$(${PSQL} -qt -c \ + "SELECT COUNT(*) FROM pg_roles \ + WHERE rolname='${PATRONI_REPLICATION_USERNAME}'") + + if [ ${USER_COUNT} -eq 0 ]; then + echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:" + ${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \ + WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';" + echo "done." + else + echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do." + fi + + # Start Patroni to assimilate the postgres + sed "s/POD_IP_PATTERN/${PATRONI_KUBERNETES_POD_IP}/g" \ + /tmp/patroni-templated.yaml > /tmp/patroni.yaml + + READY_FLAG="i am the leader with the lock" + PATRONI_LOG=/tmp/patroni_conversion.log + /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni-templated.yaml &> ${PATRONI_LOG} & + + # Sleep until patroni is running + while ! grep -q "${READY_FLAG}" ${PATRONI_LOG}; do + sleep 5 + if [[ $TIMEOUT -gt 24 ]]; then + echo "A timeout occurred. Patroni logs:" + cat ${PATRONI_LOG} + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done + TIMEOUT=0 + + # Gracefully stop postgres and patroni + while pkill INT --uid postgres; do + sleep 5 + if [[ $TIMEOUT -gt 24 ]]; then + echo "A timeout occurred. Patroni logs:" + cat ${PATRONI_LOG} + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done +else + echo "Patroni is already in place: nothing to do." +fi diff --git a/postgresql/templates/bin/_readiness.sh.tpl b/postgresql/templates/bin/_readiness.sh.tpl index c8c6b269d..7c48fafad 100644 --- a/postgresql/templates/bin/_readiness.sh.tpl +++ b/postgresql/templates/bin/_readiness.sh.tpl @@ -1,7 +1,7 @@ #!/usr/bin/env bash {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,4 +18,8 @@ limitations under the License. set -ex -pg_isready -U ${POSTGRES_USER} +if [ -f /tmp/postgres-disable-liveness-probe ]; then + exit 0 +else + pg_isready -U ${PATRONI_SUPERUSER_USERNAME} +fi diff --git a/postgresql/templates/bin/_set_password.sh.tpl b/postgresql/templates/bin/_set_password.sh.tpl new file mode 100644 index 000000000..3a6a45069 --- /dev/null +++ b/postgresql/templates/bin/_set_password.sh.tpl @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +PGDATABASE=${PGDATABASE:-'postgres'} +PGHOST=${PGHOST:-'127.0.0.1'} +PGPORT={{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + +# These are passed in via the Patroni callback interface +action="$1" +role="$2" +cluster="$3" + +# Note: this script when rendered is stored in a secret and encrypted to disk. +PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }} +PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }} +PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }} +PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }} + +if [[ x${role} == "xmaster" ]]; then + echo "I have become the patroni master: updating superuser and replication passwords" + + # It can take a few seconds for a freshly promoted leader to become read/write. + sleep 10 + if [[ ! -z "$PATRONI_SUPERUSER_PASSWORD" && ! -z "$PATRONI_SUPERUSER_USERNAME" ]]; then + psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_SUPERUSER_USERNAME WITH PASSWORD '$PATRONI_SUPERUSER_PASSWORD';" + else + echo "WARNING: Did not set superuser password!!!" + fi + + if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then + psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';" + else + echo "WARNING: Did not set replication user password!!!" + fi + + echo "password update complete" +fi diff --git a/postgresql/templates/bin/_start.sh.tpl b/postgresql/templates/bin/_start.sh.tpl index e173caadc..2cd2edc3c 100644 --- a/postgresql/templates/bin/_start.sh.tpl +++ b/postgresql/templates/bin/_start.sh.tpl @@ -1,7 +1,7 @@ #!/usr/bin/env bash {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,25 +16,62 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# Disable echo mode while setting the password -# unless we are in debug mode -{{- if .Values.conf.debug }} -set -x -{{- end }} -set -e +set -ex -POSTGRES_DB=${POSTGRES_DB:-"postgres"} +function patroni_started() { + HOST=$1 + PORT=$2 + STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT}; + echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3; + cat <&3 | tail -n1 | grep -o \"running\"") -# Check if the Postgres data directory exists before attempting to -# set the password -if [[ -d "$PGDATA" && -s "$PGDATA/PG_VERSION" ]] -then - postgres --single -D "$PGDATA" "$POSTGRES_DB" < \ + /tmp/patroni.yaml + +FILE_MADE_BY_PATRONI=${PGDATA}/patroni.dynamic.json +if [[ ! $POD_NAME -eq "postgresql-0" ]]; then + + echo "I am not postgresql pod zero: disabling liveness probe temporarily" + # disable liveness probe as it may take some time for the pod to come online + touch /tmp/postgres-disable-liveness-probe + + # During normal upgrades, we just need to turn liveness probes off temporarily + # for the sake of password rotation - need to bounce all pods at once + # (overriding RollingUpdate) to avoid deadlock. This accounts for that. + sleep 60 + + # During initial bootstrapping, we need to sequence 0,1,2 + if [[ ! -e "${FILE_MADE_BY_PATRONI}" ]]; then + echo "patroni has not been initialized on this node" + # NOTE: this boolean forces a second check after a delay. This accounts for a + # scenario during initial vanilla postgres -> patroni conversion, where + # a temporary master is brought up, killed off, and then restarted. + # This can be safely removed in the future, once all clusters are converted. + WAITED_EXTRA="false" + + while [ ${WAITED_EXTRA} = "false" ]; do + while ! patroni_started "${SVC_FQDN}" "${SVC_PORT}"; do + echo "Waiting until a Leader is elected..." + sleep 5 + done + # See note above: this code can be removed once all clusters are Patroni. + if [ ${WAITED_EXTRA} = "false" ]; then + echo "Leader is up; sleeping to ensure it gets through restarts..." + sleep 10 + WAITED_EXTRA="true" + fi + done + fi + + rm -fv /tmp/postgres-disable-liveness-probe fi -set -x - -exec /docker-entrypoint.sh postgres -N {{ .Values.conf.postgresql.max_connections | quote }} -B {{ .Values.conf.postgresql.shared_buffers | quote }} +exec /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni.yaml diff --git a/postgresql/templates/configmap-bin.yaml b/postgresql/templates/configmap-bin.yaml index bae75be0f..f5c931ea5 100644 --- a/postgresql/templates/configmap-bin.yaml +++ b/postgresql/templates/configmap-bin.yaml @@ -19,24 +19,22 @@ limitations under the License. {{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} --- apiVersion: v1 -kind: ConfigMap +{{/* Note: this is a secret because credentials must be rendered into the password script. */}} +kind: Secret metadata: name: postgresql-bin +type: Opaque data: {{- if .Values.images.local_registry.active }} - image-repo-sync.sh: | -{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + image-repo-sync.sh: {{- include "helm-toolkit.scripts.image_repo_sync" . | b64enc }} {{- end }} - start.sh: | -{{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - readiness.sh: | -{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - db_test.sh: | -{{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start.sh: {{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + readiness.sh: {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + db_test.sh: {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- if .Values.conf.backup.enabled }} - backup_postgresql.sh: | -{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - restore_postgresql.sh: | -{{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_postgresql.sh: {{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + restore_postgresql.sh: {{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- end }} + set_password.sh: {{ tuple "bin/_set_password.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + patroni_conversion.sh: {{ tuple "bin/_patroni_conversion.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- end }} diff --git a/postgresql/templates/configmap-etc.yaml b/postgresql/templates/configmap-etc.yaml new file mode 100644 index 000000000..9dddf06a5 --- /dev/null +++ b/postgresql/templates/configmap-etc.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-etc +type: Opaque +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.patroni "key" "patroni.yaml" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml index aefb37774..1014c4f84 100644 --- a/postgresql/templates/cron-job-backup-postgres.yaml +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -96,10 +96,10 @@ spec: secret: secretName: postgresql-secrets defaultMode: 0600 - - configMap: + - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 365 - name: postgresql-bin - name: postgresql-bin {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} - name: postgresql-backup-dir persistentVolumeClaim: diff --git a/postgresql/templates/pod-test.yaml b/postgresql/templates/pod-test.yaml index d260f32a9..45ed8d436 100644 --- a/postgresql/templates/pod-test.yaml +++ b/postgresql/templates/pod-test.yaml @@ -70,8 +70,8 @@ spec: - name: pod-tmp emptyDir: {} - name: postgresql-bin - configMap: - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 0555 ... {{- end }} diff --git a/postgresql/templates/secret-replica.yaml b/postgresql/templates/secret-replica.yaml new file mode 100644 index 000000000..0c92b2008 --- /dev/null +++ b/postgresql/templates/secret-replica.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_replica }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.postgresql.replica }} +type: Opaque +data: + REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }} + REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }} +{{- end }} diff --git a/postgresql/templates/service.yaml b/postgresql/templates/service-postgres.yaml similarity index 82% rename from postgresql/templates/service.yaml rename to postgresql/templates/service-postgres.yaml index 7ad24b288..31d0195a9 100644 --- a/postgresql/templates/service.yaml +++ b/postgresql/templates/service-postgres.yaml @@ -1,5 +1,5 @@ {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,8 +23,6 @@ metadata: name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: db + - name: postgresql port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - selector: -{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} {{- end }} diff --git a/postgresql/templates/service-restapi.yaml b/postgresql/templates/service-restapi.yaml new file mode 100644 index 000000000..36dbc3f14 --- /dev/null +++ b/postgresql/templates/service-restapi.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "postgresql-restapi" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: restapi + port: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- end }} diff --git a/postgresql/templates/statefulset.yaml b/postgresql/templates/statefulset.yaml index 673ad16a7..3dfb8c85c 100644 --- a/postgresql/templates/statefulset.yaml +++ b/postgresql/templates/statefulset.yaml @@ -1,5 +1,5 @@ {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,6 +20,81 @@ limitations under the License. {{- $serviceAccountName := "postgresql" }} {{ tuple $envAll "postgresql" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch + # delete and deletecollection are required only for 'patronictl remove' + - delete + - deletecollection + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - patch + - update + # the following three privileges are necessary only when using endpoints + - create + - list + - watch + # delete and deletecollection are required only for 'patronictl remove' + - delete + - deletecollection + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + # The following privilege is only necessary for creation of headless service + # for postgresql-config endpoint, in order to prevent cleaning it up by the + # k8s master. + - apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - list + - patch + - update + - watch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -28,19 +103,27 @@ metadata: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} labels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: serviceName: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" replicas: {{ .Values.pod.replicas.server }} selector: matchLabels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} template: metadata: labels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-admin-hash: {{ tuple "secret-admin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-replica-hash: {{ tuple "secret-replica.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-secrets-etc-hash: {{ tuple "secrets-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} {{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} @@ -53,56 +136,193 @@ spec: - name: set-volume-perms {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - "/bin/chown" - - {{ .Values.pod.security_context.server.pod.runAsUser | quote }} - - {{ .Values.storage.mount.path | quote }} + command: ["/bin/sh", "-c"] + args: + - set -xe; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }}; + /bin/chmod 700 {{ .Values.storage.mount.path }}; + /bin/chmod 700 {{ .Values.storage.mount.path }}/*; {{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} - subPath: {{ .Values.storage.mount.subpath }} + # This is for non-HA -> Patroni conversion and can be removed in the future + - name: patroni-conversion +{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: PGDATA + value: "{{ .Values.storage.mount.path }}/pgdata" + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PATRONI_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_SUPERUSER_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_USER' + - name: PATRONI_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_PASSWORD' + - name: PATRONI_REPLICATION_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_USER' + - name: PATRONI_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_PASSWORD' + - name: PATRONI_RESTAPI_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_RESTAPI_LISTEN + value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_LISTEN + value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_admin_PASSWORD + value: $(PATRONI_SUPERUSER_PASSWORD) + - name: PATRONI_admin_OPTIONS + value: 'createrole,createdb' + command: + - /tmp/patroni_conversion.sh +{{ dict "envAll" $envAll "application" "server" "container" "patroni_conversion" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: patroni-conversion-tmp + mountPath: /var/run/postgresql + - name: postgresql-bin + mountPath: /tmp/patroni_conversion.sh + subPath: patroni_conversion.sh + readOnly: true + - name: postgresql-data + mountPath: {{ .Values.storage.mount.path }} + - name: postgresql-etc + mountPath: /tmp/patroni-templated.yaml + subPath: patroni.yaml + readOnly: true containers: - name: postgresql {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "server" "container" "postgresql" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} ports: + - containerPort: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP - containerPort: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP env: - - name: 'POSTGRES_PASSWORD' + - name: PGDATA + value: "{{ .Values.storage.mount.path }}/pgdata" + - name: PATRONI_KUBERNETES_POD_IP valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.admin }} - key: 'POSTGRES_PASSWORD' - - name: 'POSTGRES_USER' + fieldRef: + fieldPath: status.podIP + - name: PATRONI_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PATRONI_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_SUPERUSER_USERNAME valueFrom: secretKeyRef: name: {{ .Values.secrets.postgresql.admin }} key: 'POSTGRES_USER' - - name: 'PGDATA' - value: {{ .Values.storage.mount.path | quote }} + - name: PATRONI_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_PASSWORD' + - name: PATRONI_REPLICATION_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_USER' + - name: PATRONI_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_PASSWORD' + - name: PATRONI_RESTAPI_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_RESTAPI_LISTEN + value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_LISTEN + value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_admin_PASSWORD + value: $(PATRONI_SUPERUSER_PASSWORD) + - name: PATRONI_admin_OPTIONS + value: 'createrole,createdb' command: - /tmp/start.sh livenessProbe: exec: command: - /tmp/readiness.sh - initialDelaySeconds: 20 + initialDelaySeconds: 30 timeoutSeconds: 5 + failureThreshold: 10 readinessProbe: exec: command: - /tmp/readiness.sh - initialDelaySeconds: 20 + initialDelaySeconds: 30 timeoutSeconds: 5 + failureThreshold: 10 volumeMounts: - name: pod-tmp mountPath: /tmp - name: pg-run mountPath: /var/run/postgresql + - name: postgresql-bin + mountPath: /tmp/set_password.sh + subPath: set_password.sh + readOnly: true - name: postgresql-bin mountPath: /tmp/start.sh subPath: start.sh @@ -111,19 +331,29 @@ spec: mountPath: /tmp/readiness.sh subPath: readiness.sh readOnly: true + - name: postgresql-etc + mountPath: /tmp/patroni-templated.yaml + subPath: patroni.yaml + readOnly: true - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} - subPath: {{ .Values.storage.mount.subpath }} volumes: - name: pod-tmp emptyDir: {} - name: pg-run emptyDir: medium: "Memory" + # This is for non-HA -> Patroni conversion and can be removed in the future + - name: patroni-conversion-tmp + emptyDir: {} - name: postgresql-bin - configMap: - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 0555 + - name: postgresql-etc + secret: + secretName: postgresql-etc + defaultMode: 0444 {{- if not .Values.storage.pvc.enabled }} - name: postgresql-data hostPath: diff --git a/postgresql/values.yaml b/postgresql/values.yaml index 4b4485a0d..8dfa0ec77 100644 --- a/postgresql/values.yaml +++ b/postgresql/values.yaml @@ -1,4 +1,4 @@ -# Copyright 2017 The Openstack-Helm Authors. +# Copyright 2019 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +37,14 @@ pod: postgresql: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + patroni_conversion: + runAsUser: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + pod: + runAsUser: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true affinity: anti: type: @@ -46,8 +54,7 @@ pod: weight: default: 10 replicas: - #only 1 replica currently supported - server: 1 + server: 3 prometheus_postgresql_exporter: 1 lifecycle: upgrades: @@ -106,10 +113,10 @@ pod: memory: "1024Mi" cpu: "2000m" -# using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/ +# using dockerhub patroni: https://hub.docker.com/r/openstackhelm/patroni/tags/ images: tags: - postgresql: "docker.io/postgres:9.5" + postgresql: "docker.io/openstackhelm/patroni:latest-ubuntu_xenial" dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 image_repo_sync: docker.io/docker:17.07.0 prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6 @@ -131,8 +138,8 @@ storage: host: host_path: /data/openstack-helm/postgresql mount: - path: /var/lib/postgresql/data - subpath: pgdata + path: /var/lib/postgresql + subpath: . labels: server: @@ -205,8 +212,137 @@ jobs: conf: debug: false postgresql: - max_connections: 100 shared_buffers: 128MB + max_connections: 100 + patroni: | + scope: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + kubernetes: + labels: + application: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + component: server + use_endpoints: true + ports: + - name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + data_dir: '{{ .Values.storage.mount.path }}/pgdata' + pgpass: '{{ .Values.storage.mount.path }}/pgpass' + use_pg_rewind: true + parameters: + archive_mode: 'on' + archive_timeout: 1800s + autovacuum_analyze_scale_factor: 0.02 + autovacuum_max_workers: 5 + autovacuum_vacuum_scale_factor: 0.05 + checkpoint_completion_target: 0.9 + datestyle: 'iso, mdy' + default_text_search_config: 'pg_catalog.english' + external_pid_file: '/tmp/postgres.pid' + hot_standby: 'on' + lc_messages: 'en_US.utf8' + lc_monetary: 'en_US.utf8' + lc_numeric: 'en_US.utf8' + lc_time: 'en_US.utf8' + log_autovacuum_min_duration: 0 + log_checkpoints: 'on' + log_connections: 'on' + log_disconnections: 'on' + log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m ' + log_lock_waits: 'on' + log_min_duration_statement: 500 + log_statement: none + log_temp_files: 0 + log_timezone: 'UTC' + max_connections: {{ .Values.conf.postgresql.max_connections }} + max_replication_slots: 10 + max_wal_senders: 10 + max_worker_processes: 10 + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 + timezone: 'UTC' + track_commit_timestamp: 'on' + track_functions: all + wal_keep_segments: 100 + wal_level: 'logical' + wal_log_hints: 'on' + initdb: + - auth-host: md5 + - auth-local: trust + - encoding: UTF8 + - locale: en_US.UTF-8 + - data-checksums + pg_hba: + - host all all 127.0.0.1/32 trust + - host all all 0.0.0.0/0 md5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 + - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - local all all trust + postgresql: + {{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data, + so let's just avoid it and use /var/lib/postgresql/pgdata instead. + Patroni moves this directory to a backup under the parent directory + (/var/lib/postgresql) under certain failure recovery scenarios, so + /var/lib/postgres itself must be exposed to the pod as a pvc mount.*/}} + data_dir: '{{ .Values.storage.mount.path }}/pgdata' + pgpass: '{{ .Values.storage.mount.path }}/pgpass' + callbacks: + on_role_change: /tmp/set_password.sh + on_start: /tmp/set_password.sh + use_pg_rewind: true + parameters: + archive_mode: 'on' + archive_timeout: 1800s + autovacuum_analyze_scale_factor: 0.02 + autovacuum_max_workers: 5 + autovacuum_vacuum_scale_factor: 0.05 + checkpoint_completion_target: 0.9 + datestyle: 'iso, mdy' + default_text_search_config: 'pg_catalog.english' + external_pid_file: '/tmp/postgres.pid' + hot_standby: 'on' + lc_messages: 'en_US.utf8' + lc_monetary: 'en_US.utf8' + lc_numeric: 'en_US.utf8' + lc_time: 'en_US.utf8' + log_autovacuum_min_duration: 0 + log_checkpoints: 'on' + log_connections: 'on' + log_disconnections: 'on' + log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m ' + log_lock_waits: 'on' + log_min_duration_statement: 500 + log_statement: none + log_temp_files: 0 + log_timezone: 'UTC' + max_connections: {{ .Values.conf.postgresql.max_connections }} + max_replication_slots: 10 + max_wal_senders: 10 + max_worker_processes: 10 + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 + timezone: 'UTC' + track_commit_timestamp: 'on' + track_functions: all + shared_buffers: {{ .Values.conf.postgresql.shared_buffers }} + wal_keep_segments: 100 + wal_level: 'logical' + wal_log_hints: 'on' + pg_hba: + - host all all 127.0.0.1/32 trust + - host all all 0.0.0.0/0 md5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 + - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - local all all trust + watchdog: + mode: off # Allowed values: off, automatic, required backup: enabled: true base_path: /var/backup @@ -216,6 +352,7 @@ conf: secrets: postgresql: admin: postgresql-admin + replica: postgresql-replication exporter: postgresql-exporter endpoints: @@ -237,6 +374,9 @@ endpoints: admin: username: postgres password: password + replica: + username: standby + password: password exporter: username: psql_exporter password: psql_exp_pass @@ -249,6 +389,16 @@ endpoints: port: postgresql: default: 5432 + postgresql_restapi: + hosts: + default: postgresql-restapi + host_fqdn_override: + default: null + path: null + scheme: postgresql + port: + restapi: + default: 8008 prometheus_postgresql_exporter: namespace: null hosts: @@ -265,8 +415,10 @@ endpoints: manifests: configmap_bin: true + configmap_etc: true job_image_repo_sync: true secret_admin: true + secret_replica: true secret_etc: true service: true statefulset: true diff --git a/tools/deployment/osh-infra-monitoring/130-postgresql.sh b/tools/deployment/osh-infra-monitoring/130-postgresql.sh index d99351593..c4dd70b0e 100755 --- a/tools/deployment/osh-infra-monitoring/130-postgresql.sh +++ b/tools/deployment/osh-infra-monitoring/130-postgresql.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2017 The Openstack-Helm Authors. +# Copyright 2019 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -25,6 +25,8 @@ helm upgrade --install postgresql ./postgresql \ --namespace=osh-infra \ --set monitoring.prometheus.enabled=true \ --set storage.pvc.size=1Gi \ + --set storage.pvc.enabled=true \ + --set pod.replicas.server=3 \ ${OSH_INFRA_EXTRA_HELM_ARGS} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB}