diff --git a/etcd/Chart.yaml b/etcd/Chart.yaml index 7966fd452..7c7f7a871 100644 --- a/etcd/Chart.yaml +++ b/etcd/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v3.4.3 description: OpenStack-Helm etcd name: etcd -version: 0.1.7 +version: 0.1.8 home: https://coreos.com/etcd/ icon: https://raw.githubusercontent.com/CloudCoreo/etcd-cluster/master/images/icon.png sources: diff --git a/etcd/templates/bin/_etcd-healthcheck.sh.tpl b/etcd/templates/bin/_etcd-healthcheck.sh.tpl new file mode 100644 index 000000000..ef5442df5 --- /dev/null +++ b/etcd/templates/bin/_etcd-healthcheck.sh.tpl @@ -0,0 +1,24 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x + +export ETCDCTL_API=3 + +ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + +etcdctl endpoint health --endpoints=${POD_NAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT} diff --git a/etcd/templates/bin/_etcd.sh.tpl b/etcd/templates/bin/_etcd.sh.tpl index 18066fc8e..3ac97648d 100644 --- a/etcd/templates/bin/_etcd.sh.tpl +++ b/etcd/templates/bin/_etcd.sh.tpl @@ -16,6 +16,60 @@ limitations under the License. set -ex +active_members_present() { + res=1 + for endpoint in $(echo $ETCD_ENDPOINTS | tr ',' '\n'); do + if etcdctl endpoint health --endpoints=$endpoint >/dev/null 2>&1; then + res=$? + if [[ "$res" == 0 ]]; then + break + fi + fi + done + echo $res +} + +ETCD_REPLICAS={{ .Values.pod.replicas.etcd }} +PEER_PREFIX_NAME={{- printf "%s-%s" .Release.Name "etcd" }} +DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +ETCD_PEER_PORT=2380 +ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +ETCD_PROTOCOL={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +PEERS="${PEER_PREFIX_NAME}-0=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +ETCD_ENDPOINTS="${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +if [[ ${ETCD_REPLICAS} -gt 1 ]] ; then + for i in $(seq 1 $(( ETCD_REPLICAS - 1 ))); do + PEERS="$PEERS,${PEER_PREFIX_NAME}-${i}=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" + ETCD_ENDPOINTS="${ETCD_ENDPOINTS},${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" + done +fi +ADVERTISE_PEER_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +ADVERTISE_CLIENT_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT}" + +ETCD_INITIAL_CLUSTER_STATE=new + +if [[ -z "$(ls -A $ETCD_DATA_DIR)" ]]; then + echo "State directory $ETCD_DATA_DIR is empty." + if [[ $(active_members_present) -eq 0 ]]; then + ETCD_INITIAL_CLUSTER_STATE=existing + member_id=$(etcdctl --endpoints=${ETCD_ENDPOINTS} member list | grep -w ${ADVERTISE_CLIENT_URL} | awk -F "," '{ print $1 }') + if [[ -n "$member_id" ]]; then + echo "Current node is a member of cluster, member_id: ${member_id}" + echo "Rejoining..." + echo "Removing member from the cluster" + etcdctl member remove "$member_id" --endpoints=${ETCD_ENDPOINTS} + etcdctl member add ${ADVERTISE_CLIENT_URL} --peer-urls=${ADVERTISE_PEER_URL} --endpoints=${ETCD_ENDPOINTS} + fi + else + echo "Do not have active members. Starting initial cluster state." + fi +fi + exec etcd \ - --listen-client-urls http://0.0.0.0:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} \ - --advertise-client-urls {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix "/" }} + --name ${HOSTNAME} \ + --listen-peer-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_PEER_PORT} \ + --listen-client-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_CLIENT_PORT} \ + --advertise-client-urls ${ADVERTISE_CLIENT_URL} \ + --initial-advertise-peer-urls ${ADVERTISE_PEER_URL} \ + --initial-cluster ${PEERS} \ + --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} diff --git a/etcd/templates/configmap-bin.yaml b/etcd/templates/configmap-bin.yaml index c35af781b..905d34619 100644 --- a/etcd/templates/configmap-bin.yaml +++ b/etcd/templates/configmap-bin.yaml @@ -27,4 +27,6 @@ data: {{- end }} etcd.sh: | {{ tuple "bin/_etcd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + etcd-healthcheck.sh: | +{{ tuple "bin/_etcd-healthcheck.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/etcd/templates/service-discovery.yaml b/etcd/templates/service-discovery.yaml new file mode 100644 index 000000000..83a0808d9 --- /dev/null +++ b/etcd/templates/service-discovery.yaml @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: client + port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: client + - name: peer + port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: peer + publishNotReadyAddresses: true + clusterIP: None + selector: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/etcd/templates/service.yaml b/etcd/templates/service.yaml index 812c574d4..7c6dcf8a9 100644 --- a/etcd/templates/service.yaml +++ b/etcd/templates/service.yaml @@ -20,7 +20,14 @@ metadata: spec: sessionAffinity: ClientIP ports: - - port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: client + port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: client + - name: peer + port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: peer selector: {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{- end }} diff --git a/etcd/templates/deployment.yaml b/etcd/templates/statefulset.yaml similarity index 63% rename from etcd/templates/deployment.yaml rename to etcd/templates/statefulset.yaml index ed0bf0a2b..c6008167a 100644 --- a/etcd/templates/deployment.yaml +++ b/etcd/templates/statefulset.yaml @@ -10,7 +10,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.manifests.deployment }} +{{- define "etcdProbeTemplate" }} +exec: + command: + - /tmp/etcd-healthcheck.sh +{{- end }} + +{{- if .Values.manifests.statefulset }} {{- $envAll := . }} {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "etcd" }} @@ -19,7 +25,7 @@ {{ tuple $envAll "etcd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ $rcControllerName | quote }} annotations: @@ -27,11 +33,12 @@ metadata: labels: {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: + podManagementPolicy: "Parallel" + serviceName: "{{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}" replicas: {{ .Values.pod.replicas.etcd }} selector: matchLabels: {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} template: metadata: labels: @@ -41,8 +48,8 @@ spec: {{ dict "envAll" $envAll "podName" "etcd" "containerNames" (list "init" "etcd") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} spec: -{{ dict "envAll" $envAll "application" "etcd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $rcControllerName | quote }} +{{ dict "envAll" $envAll "application" "etcd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} affinity: {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} nodeSelector: @@ -53,13 +60,24 @@ spec: - name: etcd {{ tuple $envAll "etcd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 12 }} + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name command: - /tmp/etcd.sh ports: - containerPort: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - readinessProbe: - tcpSocket: - port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: client + protocol: TCP + - containerPort: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: peer + protocol: TCP volumeMounts: - name: pod-tmp mountPath: /tmp @@ -67,6 +85,12 @@ spec: mountPath: /tmp/etcd.sh subPath: etcd.sh readOnly: true + - name: etcd-data + mountPath: /var/lib/etcd + - name: etcd-bin + mountPath: /tmp/etcd-healthcheck.sh + subPath: etcd-healthcheck.sh + readOnly: true volumes: - name: pod-tmp emptyDir: {} @@ -74,4 +98,21 @@ spec: configMap: name: {{ $configMapBinName | quote }} defaultMode: 0555 + {{- if not .Values.volume.enabled }} + - name: etcd-data + emptyDir: {} + {{- end }} +{{- end }} +{{- if .Values.volume.enabled }} + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.size }} + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ .Values.volume.class_name }} + {{- end }} {{- end }} diff --git a/etcd/values.yaml b/etcd/values.yaml index 67dfd6bd7..fdfe8ffee 100644 --- a/etcd/values.yaml +++ b/etcd/values.yaml @@ -51,6 +51,10 @@ dependencies: jobs: null pod: + env: + etcd: + ETCD_DATA_DIR: /var/lib/etcd + ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster-1 security_context: etcd: pod: @@ -64,6 +68,21 @@ pod: etcd: init: runtime/default etcd: runtime/default + probes: + etcd: + etcd: + readiness: + enabled: True + params: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 + liveness: + enabled: True + params: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 affinity: anti: type: @@ -129,6 +148,7 @@ endpoints: name: etcd hosts: default: etcd + discovery: etcd-discovery host_fqdn_override: default: null path: @@ -138,11 +158,30 @@ endpoints: port: client: default: 2379 + etcd_discovery: + name: etcd-discovery + hosts: + default: etcd-discovery + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + client: + default: 2380 + +volume: + enabled: false + class_name: general + size: 5Gi manifests: configmap_bin: true - deployment: true + statefulset: true job_image_repo_sync: true secret_registry: true service: true + service_discovery: true ... diff --git a/releasenotes/notes/etcd.yaml b/releasenotes/notes/etcd.yaml index 085dafc6b..209a6d4cd 100644 --- a/releasenotes/notes/etcd.yaml +++ b/releasenotes/notes/etcd.yaml @@ -8,4 +8,5 @@ etcd: - 0.1.5 Added OCI registry authentication - 0.1.6 Update kubernetes registry to registry.k8s.io - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Switch etcd to staetefulset ...