diff --git a/fluentd/templates/deployment-fluentd.yaml b/fluentd/templates/deployment-fluentd.yaml index 7610ec0f5..167f7f927 100644 --- a/fluentd/templates/deployment-fluentd.yaml +++ b/fluentd/templates/deployment-fluentd.yaml @@ -19,8 +19,8 @@ limitations under the License. {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }} -{{- $kafkaBroker := tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} -{{- $kafkaBrokerPort := tuple "kafka" "public" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $kafkaBroker := tuple "kafka" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} {{- $kafkaBrokerURI := printf "%s:%s" $kafkaBroker $kafkaBrokerPort }} {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd" }} diff --git a/fluentd/values.yaml b/fluentd/values.yaml index 026a5f7c1..aab965778 100644 --- a/fluentd/values.yaml +++ b/fluentd/values.yaml @@ -366,6 +366,7 @@ endpoints: port: broker: default: 9092 + public: 80 prometheus_fluentd_exporter: namespace: null hosts: diff --git a/kafka/Chart.yaml b/kafka/Chart.yaml new file mode 100644 index 000000000..4e7056dd2 --- /dev/null +++ b/kafka/Chart.yaml @@ -0,0 +1,25 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Kafka +name: kafka +version: 0.1.0 +home: https://kafka.apache.org/ +sources: + - https://github.com/apache/kafka + - https://github.com/danielqsj/kafka_exporter + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/kafka/requirements.yaml b/kafka/requirements.yaml new file mode 100644 index 000000000..e69c985d8 --- /dev/null +++ b/kafka/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/kafka/templates/bin/_helm-test.sh.tpl b/kafka/templates/bin/_helm-test.sh.tpl new file mode 100644 index 000000000..0b0a48bc1 --- /dev/null +++ b/kafka/templates/bin/_helm-test.sh.tpl @@ -0,0 +1,118 @@ +#!/bin/bash +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +function create_topic () { + ./opt/kafka/bin/kafka-topics.sh \ + --create --topic $1 \ + --partitions $2 \ + --replication-factor $3 \ + --bootstrap-server $KAFKA_BROKERS +} + +function describe_topic () { + ./opt/kafka/bin/kafka-topics.sh \ + --describe --topic $1 \ + --bootstrap-server $KAFKA_BROKERS +} + +function produce_message () { + echo $2 | \ + ./opt/kafka/bin/kafka-console-producer.sh \ + --topic $1 \ + --broker-list $KAFKA_BROKERS +} + +function consume_messages () { + ./opt/kafka/bin/kafka-console-consumer.sh \ + --topic $1 \ + --timeout-ms 500 \ + --from-beginning \ + --bootstrap-server $KAFKA_BROKERS +} + +function delete_partition_messages () { + ./opt/kafka/bin/kafka-delete-records.sh \ + --offset-json-file $1 \ + --bootstrap-server $KAFKA_BROKERS +} + +function delete_topic () { + ./opt/kafka/bin/kafka-topics.sh \ + --delete --topic $1 \ + --bootstrap-server $KAFKA_BROKERS +} + +set -e + +TOPIC="kafka-test" +PARTITION_COUNT=3 +PARTITION_REPLICAS=2 + +echo "Creating topic $TOPIC" +create_topic $TOPIC $PARTITION_COUNT $PARTITION_REPLICAS +describe_topic $TOPIC + +echo "Producing 5 messages" +for i in {1..5}; do + MESSAGE="Message #$i" + produce_message $TOPIC "$MESSAGE" +done + +echo -e "\nConsuming messages (A \"TimeoutException\" is expected, else this would consume forever)" +consume_messages $TOPIC + +echo "Producing 5 more messages" +for i in {6..10}; do + MESSAGE="Message #$i" + produce_message $TOPIC "$MESSAGE" +done + +echo -e "\nCreating partition offset reset json file" +tee /tmp/partition_offsets.json << EOF +{ +"partitions": [ + { + "topic": "$TOPIC", + "partition": 0, + "offset": -1 + }, { + "topic": "$TOPIC", + "partition": 1, + "offset": -1 + }, { + "topic": "$TOPIC", + "partition": 2, + "offset": -1 + } +], +"version": 1 +} +EOF + +echo "Resetting $TOPIC partitions (deleting messages)" +delete_partition_messages /tmp/partition_offsets.json + +echo "Deleting topic $TOPIC" +delete_topic $TOPIC + +if [ $(describe_topic $TOPIC | wc -l) -eq 0 ]; then + echo "Topic $TOPIC was deleted successfully." + exit 0 +else + echo "Topic $TOPIC was not successfully deleted." + exit 1 +fi diff --git a/kafka/templates/bin/_kafka-probe.sh.tpl b/kafka/templates/bin/_kafka-probe.sh.tpl new file mode 100644 index 000000000..05bf2f0dc --- /dev/null +++ b/kafka/templates/bin/_kafka-probe.sh.tpl @@ -0,0 +1,21 @@ +#!/bin/sh + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +echo ruok | nc 127.0.0.1 ${KAFKA_PORT} diff --git a/kafka/templates/bin/_kafka.sh.tpl b/kafka/templates/bin/_kafka.sh.tpl new file mode 100644 index 000000000..ca3d1596a --- /dev/null +++ b/kafka/templates/bin/_kafka.sh.tpl @@ -0,0 +1,36 @@ +#!/bin/bash +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ if not (empty .Values.conf.kafka.server_settings) }} +{{ range $key, $value := .Values.conf.kafka.server_settings }} +{{ $varName := printf "%s%s" "KAFKA_" ($key | upper) }} +{{ $varValue := ternary ($value | quote) ($value | int) (kindIs "string" $value) }} +export {{ $varName }}={{ $varValue }} +{{ end }} +{{ end }} + +COMMAND="${@:-start}" + +function start() { + ./usr/bin/start-kafka.sh +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/kafka/templates/configmap-bin.yaml b/kafka/templates/configmap-bin.yaml new file mode 100644 index 000000000..12f994c39 --- /dev/null +++ b/kafka/templates/configmap-bin.yaml @@ -0,0 +1,33 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-bin +data: + kafka.sh: | +{{ tuple "bin/_kafka.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kafka-liveness.sh: | +{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kafka-readiness.sh: | +{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-test.sh: | +{{ tuple "bin/_helm-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end -}} \ No newline at end of file diff --git a/kafka/templates/helm_test_pod.yaml b/kafka/templates/helm_test_pod.yaml new file mode 100644 index 000000000..5d93bb188 --- /dev/null +++ b/kafka/templates/helm_test_pod.yaml @@ -0,0 +1,64 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_test }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "test" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "kafka" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "test" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: {{.Release.Name}}-helm-test +{{ tuple $envAll "helm_test" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.test | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "helm_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + command: + - "/tmp/helm-test.sh" + env: + - name: KAFKA_BROKERS + value: "{{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: kafka-bin + mountPath: /tmp/helm-test.sh + subPath: helm-test.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: kafka-bin + configMap: + name: kafka-bin + defaultMode: 0555 +{{- end }} diff --git a/kafka/templates/ingress-kafka.yaml b/kafka/templates/ingress-kafka.yaml new file mode 100644 index 000000000..3d12bed51 --- /dev/null +++ b/kafka/templates/ingress-kafka.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.kafka.ingress.public }} +{{- $ingressOpts := dict "envAll" . "backendService" "kafka" "backendServiceType" "kafka" "backendPort" "broker" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/kafka/templates/job-image-repo-sync.yaml b/kafka/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..2b90fb153 --- /dev/null +++ b/kafka/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kafka" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl b/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl new file mode 100644 index 000000000..802044fa6 --- /dev/null +++ b/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/sh + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +COMMAND="${@:-start}" + +function start () { + exec /bin/kafka_exporter \ + --kafka.server={{ tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/kafka/templates/monitoring/prometheus/configmap-bin.yaml b/kafka/templates/monitoring/prometheus/configmap-bin.yaml new file mode 100644 index 000000000..3f5215551 --- /dev/null +++ b/kafka/templates/monitoring/prometheus/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-exporter-bin +data: + kafka-exporter.sh: | +{{ tuple "bin/_kafka-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/kafka/templates/monitoring/prometheus/deployment.yaml b/kafka/templates/monitoring/prometheus/deployment.yaml new file mode 100644 index 000000000..858fa709e --- /dev/null +++ b/kafka/templates/monitoring/prometheus/deployment.yaml @@ -0,0 +1,88 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.deployment .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $kafkaExporterUserSecret := .Values.secrets.kafka_exporter.user }} + +{{- $serviceAccountName := "prometheus-kafka-exporter" }} +{{ tuple $envAll "kafka_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-kafka-exporter + labels: +{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.kafka_exporter }} + selector: + matchLabels: +{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "kafka_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: kafka-exporter +{{ tuple $envAll "kafka_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.kafka_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "kafka_exporter" "container" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/kafka-exporter.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/kafka-exporter.sh + - stop + # env: {} + ports: + - name: exporter + containerPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: kafka-exporter-bin + mountPath: /tmp/kafka-exporter.sh + subPath: kafka-exporter.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: kafka-exporter-bin + configMap: + name: kafka-exporter-bin + defaultMode: 0555 +{{- end }} diff --git a/kafka/templates/monitoring/prometheus/network-policy.yaml b/kafka/templates/monitoring/prometheus/network-policy.yaml new file mode 100644 index 000000000..5b693bb82 --- /dev/null +++ b/kafka/templates/monitoring/prometheus/network-policy.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.network_policy .Values.monitoring.prometheus.enabled -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-kafka-exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/kafka/templates/monitoring/prometheus/service.yaml b/kafka/templates/monitoring/prometheus/service.yaml new file mode 100644 index 000000000..39bfdeddb --- /dev/null +++ b/kafka/templates/monitoring/prometheus/service.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.service .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kafka_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kafka_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "kafka-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: exporter + port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/kafka/templates/network_policy.yaml b/kafka/templates/network_policy.yaml new file mode 100644 index 000000000..4806a7ac4 --- /dev/null +++ b/kafka/templates/network_policy.yaml @@ -0,0 +1,19 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kafka" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/kafka/templates/secret-ingress-tls.yaml b/kafka/templates/secret-ingress-tls.yaml new file mode 100644 index 000000000..5e532b0cc --- /dev/null +++ b/kafka/templates/secret-ingress-tls.yaml @@ -0,0 +1,19 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls -}} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "kafka" "backendService" "kafka" ) }} +{{- end }} diff --git a/kafka/templates/secret-kafka.yaml b/kafka/templates/secret-kafka.yaml new file mode 100644 index 000000000..673e4beae --- /dev/null +++ b/kafka/templates/secret-kafka.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_kafka }} +{{- $envAll := . }} +{{- $secretName := .Values.secrets.kafka.admin }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + KAFKA_ADMIN_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }} + KAFKA_ADMIN_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }} +{{- end }} diff --git a/kafka/templates/service-discovery.yaml b/kafka/templates/service-discovery.yaml new file mode 100644 index 000000000..aa6197e59 --- /dev/null +++ b/kafka/templates/service-discovery.yaml @@ -0,0 +1,34 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kafka" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + ports: + - name: broker + targetPort: broker + port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + clusterIP: None + selector: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/kafka/templates/service-ingress-kafka.yaml b/kafka/templates/service-ingress-kafka.yaml new file mode 100644 index 000000000..8590311ae --- /dev/null +++ b/kafka/templates/service-ingress-kafka.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.kafka.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kafka" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/kafka/templates/service.yaml b/kafka/templates/service.yaml new file mode 100644 index 000000000..6a53318c7 --- /dev/null +++ b/kafka/templates/service.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kafka" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + ports: + - name: broker + port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.kafka.node_port.enabled }} + nodePort: {{ .Values.network.kafka.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.kafka.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/kafka/templates/statefulset.yaml b/kafka/templates/statefulset.yaml new file mode 100644 index 000000000..50060966f --- /dev/null +++ b/kafka/templates/statefulset.yaml @@ -0,0 +1,178 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $mounts_kafka := .Values.pod.mounts.kafka.kafka }} +{{- $mounts_kafka_init := .Values.pod.mounts.kafka.init_container }} +{{- $kafkaUserSecret := .Values.secrets.kafka.admin }} +{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "kafka" }} +{{ tuple $envAll "kafka" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "kafka" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ .Values.pod.replicas.kafka }} + updateStrategy: + type: OnDelete + podManagementPolicy: Parallel + selector: + matchLabels: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "kafka" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka.timeout | default "30" }} + initContainers: +{{ tuple $envAll "kafka" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: kafka + command: + - "/tmp/kafka.sh" +{{ tuple $envAll "kafka" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.kafka | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "kafka" "container" "kafka" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + ports: + - name: broker + containerPort: {{ $kafkaBrokerPort }} + env: + - name: KAFKA_PORT + value: "{{ $kafkaBrokerPort }}" + - name: ZOOKEEPER_PORT + value: "{{ tuple "zookeeper" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: KAFKA_ZOOKEEPER_CONNECT + value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" + - name: KAFKA_LISTENERS + value: "PLAINTEXT://:{{$kafkaBrokerPort}}" + - name: KAFKA_CREATE_TOPICS + value: "{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.kafka.topics }}" + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 2 + successThreshold: 1 + exec: + command: + - /tmp/kafka-readiness.sh + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 2 + successThreshold: 1 + exec: + command: + - /tmp/kafka-liveness.sh + volumeMounts: + - name: kafka-bin + mountPath: /tmp/kafka.sh + subPath: kafka.sh + readOnly: true + - name: kafka-bin + mountPath: /tmp/kafka-liveness.sh + subPath: kafka-liveness.sh + readOnly: true + - name: kafka-bin + mountPath: /tmp/kafka-readiness.sh + subPath: kafka-readiness.sh + readOnly: true + - name: data + mountPath: {{ .Values.conf.kafka.config.data_directory }} +{{ if $mounts_kafka.volumeMounts }}{{ toYaml $mounts_kafka.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: kafka-bin + configMap: + name: kafka-bin + defaultMode: 0555 +{{ if $mounts_kafka.volumes }}{{ toYaml $mounts_kafka.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: {{ .Values.storage.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.requests.storage }} + storageClassName: {{ .Values.storage.storage_class }} +{{- end }} +{{- end }} diff --git a/kafka/values.yaml b/kafka/values.yaml new file mode 100644 index 000000000..d24ad1b54 --- /dev/null +++ b/kafka/values.yaml @@ -0,0 +1,300 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for kafka. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +images: + tags: + kafka: docker.io/wurstmeister/kafka:2.12-2.3.0 + kafka_exporter: docker.io/danielqsj/kafka-exporter:latest + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + helm_test: docker.io/wurstmeister/kafka:2.12-2.3.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + kafka: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + kafka: + pod: {} + container: + kafka: {} + kafka-init: {} + kafka_exporter: + pod: {} + container: + kafka_exporter: {} + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + kafka: + kafka: + init_container: null + replicas: + kafka: 3 + kafka_exporter: 1 + lifecycle: + upgrades: + statefulsets: + pod_replacement_strategy: RollingUpdate + termination_grace_period: + kafka: + timeout: 30 + kafka_exporter: + timeout: 30 + resources: + enabled: false + kafka: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + kafka_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + test: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + kafka: + name: kafka + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + default: kafka-broker + discovery: kafka-discovery + public: kafka + host_fqdn_override: + default: null + # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: 'http' + port: + broker: + default: 9092 + kafka-exporter: + default: 9141 + jmx-exporter: + default: 9404 + kafka_exporter: + namespace: null + hosts: + default: kafka-exporter + host_fqdn_override: + default: null + scheme: + default: 'http' + port: + exporter: + default: 9308 + zookeeper: + name: zookeeper + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + default: zookeeper-int + public: zookeeper + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + client: + default: 2181 + server: + default: 2888 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - kafka-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + kafka: + services: + - endpoint: internal + service: zookeeper-int + kafka_exporter: + services: + - endpoint: internal + service: kafka-broker + +monitoring: + prometheus: + enabled: true + kafka_exporter: + scrape: true + +network: + kafka: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-kafka + nginx.ingress.kubernetes.io/session-cookie-hash: sha1 + nginx.ingress.kubernetes.io/session-cookie-expires: "600" + nginx.ingress.kubernetes.io/session-cookie-max-age: "600" + node_port: + enabled: false + port: 31033 + +network_policy: + kafka: + ingress: + - {} + egress: + - {} + kafka_exporter: + ingress: + - {} + egress: + - {} + +secrets: + tls: + kafka: + kafka: + public: kafka-tls-public + kafka: + admin: kafka-admin-creds + kafka_exporter: + user: kafka-exporter-creds + +storage: + enabled: true + pvc: + name: kafka-pvc + access_mode: [ "ReadWriteOnce" ] + requests: + storage: 5Gi + storage_class: general + +manifests: + configmap_bin: true + configmap_etc: true + helm_test: true + ingress: true + job_image_repo_sync: true + monitoring: + prometheus: + configmap_bin: true + deployment: true + service: true + network_policy: false + network_policy: false + secret_ingress_tls: true + secret_kafka: true + secret_zookeeper: true + service_discovery: true + service_ingress: true + service: true + statefulset: true + +conf: + kafka: + config: + data_directory: /var/lib/kafka/data + server_settings: {} + # Optionally provide configuration overrides for + # Kafka's server.properties file ie: + # message_max_bytes: 5000000 + topics: [] + # List of topic strings formatted like: + # topic_name:number_of_partitions:replication_factor + # - "mytopic:1:1" diff --git a/tools/deployment/osh-infra-kafka/000-install-packages.sh b/tools/deployment/osh-infra-kafka/000-install-packages.sh new file mode 120000 index 000000000..d702c4899 --- /dev/null +++ b/tools/deployment/osh-infra-kafka/000-install-packages.sh @@ -0,0 +1 @@ +../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh b/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh new file mode 120000 index 000000000..257a39f7a --- /dev/null +++ b/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/010-ingress.sh b/tools/deployment/osh-infra-kafka/010-ingress.sh new file mode 120000 index 000000000..4c3d424df --- /dev/null +++ b/tools/deployment/osh-infra-kafka/010-ingress.sh @@ -0,0 +1 @@ +../osh-infra-logging/010-ingress.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/020-ceph.sh b/tools/deployment/osh-infra-kafka/020-ceph.sh new file mode 120000 index 000000000..1ab828eed --- /dev/null +++ b/tools/deployment/osh-infra-kafka/020-ceph.sh @@ -0,0 +1 @@ +../osh-infra-logging/020-ceph.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh new file mode 120000 index 000000000..10e71eedb --- /dev/null +++ b/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh @@ -0,0 +1 @@ +../osh-infra-logging/025-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh b/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh new file mode 120000 index 000000000..1ca42d153 --- /dev/null +++ b/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh @@ -0,0 +1 @@ +../osh-infra-logging/030-radosgw-osh-infra.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/040-zookeeper.sh b/tools/deployment/osh-infra-kafka/040-zookeeper.sh new file mode 120000 index 000000000..69bcd4139 --- /dev/null +++ b/tools/deployment/osh-infra-kafka/040-zookeeper.sh @@ -0,0 +1 @@ +../common/zookeeper.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/050-kafka.sh b/tools/deployment/osh-infra-kafka/050-kafka.sh new file mode 100755 index 000000000..2023b6ef6 --- /dev/null +++ b/tools/deployment/osh-infra-kafka/050-kafka.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make kafka + +#NOTE: Deploy command +helm upgrade --install kafka ./kafka \ + --namespace=osh-infra \ + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate deployment info +helm status kafka + +#NOTE: Test deployment +helm test kafka diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index ee31648b0..39f999167 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -137,6 +137,26 @@ - ./tools/deployment/osh-infra-logging/075-kibana.sh - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true +- job: + name: openstack-helm-infra-kafka + parent: openstack-helm-infra-functional + timeout: 7200 + pre-run: + - playbooks/osh-infra-upgrade-host.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node + vars: + gate_scripts: + - ./tools/deployment/osh-infra-kafka/000-install-packages.sh + - ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh + - ./tools/deployment/osh-infra-kafka/010-ingress.sh + - ./tools/deployment/osh-infra-kafka/020-ceph.sh + - ./tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh + - ./tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh + - ./tools/deployment/osh-infra-kafka/040-zookeeper.sh + - ./tools/deployment/osh-infra-kafka/050-kafka.sh + - job: name: openstack-helm-infra-aio-monitoring parent: openstack-helm-infra-functional diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 5d5d2a7e3..5444e9b30 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -23,6 +23,8 @@ - openstack-helm-infra-aio-monitoring - openstack-helm-infra-federated-monitoring: voting: false + - openstack-helm-infra-kafka: + voting: false - openstack-helm-infra-aio-network-policy: voting: false - openstack-helm-infra-openstack-support