From 39e1f7f9f38d6e8b704471acca4e30e912417f28 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Thu, 17 May 2018 00:55:55 -0500 Subject: [PATCH] KubeADM: Keystone Kubernetes Webhook This PS adds the ability to deploy the Keystone Kubernetes Webhook chart via kubeadm-aio Change-Id: I18b0477a775de942f940e9c0984559089dca1cdb Co-Authored-By: Tin Lam Co-Authored-By: Gage Hugo Signed-off-by: Pete Birley --- .zuul.yaml | 17 +++ kubernetes-keystone-webhook/values.yaml | 1 + playbooks/osh-infra-keystone-k8s-auth.yaml | 93 ++++++++++++++++ roles/build-images/tasks/kubeadm-aio.yaml | 4 +- .../defaults/main.yml | 1 + .../tasks/util-kubeadm-aio-run.yaml | 1 + tools/deployment/keystone-auth/check.sh | 48 +++++++++ tools/images/kubeadm-aio/Dockerfile | 2 +- tools/images/kubeadm-aio/assets/entrypoint.sh | 2 + .../tasks/helm-keystone-auth.yaml | 101 ++++++++++++++++++ .../deploy-kubeadm-master/tasks/main.yaml | 9 +- .../templates/webhook.kubeconfig.j2 | 16 +++ .../assets/opt/playbooks/vars.yaml | 1 + 13 files changed, 286 insertions(+), 10 deletions(-) create mode 100644 playbooks/osh-infra-keystone-k8s-auth.yaml create mode 100755 tools/deployment/keystone-auth/check.sh create mode 100644 tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-keystone-auth.yaml create mode 100644 tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/webhook.kubeconfig.j2 diff --git a/.zuul.yaml b/.zuul.yaml index 246696087..d44439200 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -41,6 +41,12 @@ - ^.*\.rst$ - ^doc/.*$ - ^releasenotes/.*$ + - openstack-helm-infra-kubernetes-keystone-auth: + voting: false + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ gate: jobs: - openstack-helm-infra-linter: @@ -245,3 +251,14 @@ name: openstack-helm-infra-five-fedora parent: openstack-helm-infra nodeset: openstack-helm-five-node-fedora + +- job: + name: openstack-helm-infra-kubernetes-keystone-auth + vars: + zuul_osh_relative_path: ../openstack-helm/ + kubernetes_keystone_auth: true + parent: openstack-helm-infra + nodeset: openstack-helm-single-node + run: playbooks/osh-infra-keystone-k8s-auth.yaml + required-projects: + - openstack/openstack-helm diff --git a/kubernetes-keystone-webhook/values.yaml b/kubernetes-keystone-webhook/values.yaml index 8d324938a..7af8c88f0 100644 --- a/kubernetes-keystone-webhook/values.yaml +++ b/kubernetes-keystone-webhook/values.yaml @@ -116,6 +116,7 @@ endpoints: key: null identity: name: keystone + namespace: null auth: admin: region_name: RegionOne diff --git a/playbooks/osh-infra-keystone-k8s-auth.yaml b/playbooks/osh-infra-keystone-k8s-auth.yaml new file mode 100644 index 000000000..95e28d9c4 --- /dev/null +++ b/playbooks/osh-infra-keystone-k8s-auth.yaml @@ -0,0 +1,93 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Setup OS and K8s Clients + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/020-setup-client.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Ingress + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/030-ingress.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy NFS + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/040-nfs-provisioner.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + OSH_INFRA_PATH: "../openstack-helm-infra/" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Mariadb + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/050-mariadb.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy RabbitMQ + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/060-rabbitmq.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Memcached + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/070-memcached.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Keystone + shell: | + set -xe; + cd "${OSH_PATH}" + ./tools/deployment/developer/nfs/080-keystone.sh + environment: + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Check Kubernetes Keystone Auth + shell: | + set -xe; + ./tools/deployment/keystone-auth/check.sh + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/roles/build-images/tasks/kubeadm-aio.yaml b/roles/build-images/tasks/kubeadm-aio.yaml index 99eb99dbe..537d87bc8 100644 --- a/roles/build-images/tasks/kubeadm-aio.yaml +++ b/roles/build-images/tasks/kubeadm-aio.yaml @@ -52,7 +52,7 @@ --build-arg KUBE_VERSION="{{ version.kubernetes }}" \ --build-arg CNI_VERSION="{{ version.cni }}" \ --build-arg HELM_VERSION="{{ version.helm }}" \ - --build-arg CHARTS="calico,flannel,tiller,kube-dns" \ + --build-arg CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook" \ --build-arg HTTP_PROXY="{{ proxy.http }}" \ --build-arg HTTPS_PROXY="{{ proxy.https }}" \ --build-arg NO_PROXY="{{ proxy.noproxy }}" \ @@ -78,7 +78,7 @@ --build-arg KUBE_VERSION="{{ version.kubernetes }}" \ --build-arg CNI_VERSION="{{ version.cni }}" \ --build-arg HELM_VERSION="{{ version.helm }}" \ - --build-arg CHARTS="calico,flannel,tiller,kube-dns" \ + --build-arg CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook" \ {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %} --build-arg UBUNTU_URL="http://{{ zuul_site_mirror_fqdn }}/ubuntu/" \ --build-arg ALLOW_UNAUTHENTICATED="true" \ diff --git a/roles/deploy-kubeadm-aio-common/defaults/main.yml b/roles/deploy-kubeadm-aio-common/defaults/main.yml index f7bb9a585..fd1cbf07b 100644 --- a/roles/deploy-kubeadm-aio-common/defaults/main.yml +++ b/roles/deploy-kubeadm-aio-common/defaults/main.yml @@ -17,6 +17,7 @@ kubernetes_cluster_pod_subnet: 192.168.0.0/16 kubernetes_cluster_domain: cluster.local kubernetes_network_default_device: null kubernetes_selfhosted: false +kubernetes_keystone_auth: false images: kubernetes: diff --git a/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml b/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml index 6f43cb5e6..8b1296ffc 100644 --- a/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml +++ b/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml @@ -51,6 +51,7 @@ CONTAINER_RUNTIME=docker KUBELET_NODE_LABELS="{{ kubeadm_kubelet_labels }}" KUBE_SELF_HOSTED="{{ kubernetes_selfhosted }}" + KUBE_KEYSTONE_AUTH="{{ kubernetes_keystone_auth }}" register: kubeadm_master_deploy rescue: - name: "getting logs for {{ kubeadm_aio_action }} action" diff --git a/tools/deployment/keystone-auth/check.sh b/tools/deployment/keystone-auth/check.sh new file mode 100755 index 000000000..1334964a4 --- /dev/null +++ b/tools/deployment/keystone-auth/check.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +export OS_CLOUD=openstack_helm +function keystone_token () { + openstack token issue -f value -c id +} +sudo cp -va $HOME/.kube/config /tmp/kubeconfig.yaml +sudo kubectl --kubeconfig /tmp/kubeconfig.yaml config unset users.kubernetes-admin + +# Test +if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods ; then + echo "Denied, as expected by policy" +else + exit 1 +fi +kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods -n openstack + +# create a demoUser +openstack user create --or-show --password demoPassword demoUser +unset OS_CLOUD +export OS_AUTH_URL="http://keystone.openstack.svc.cluster.local/v3" +export OS_IDENTITY_API_VERSION="3" +export OS_PASSWORD="demoPassword" +export OS_USERNAME="demoUser" + +# See this does fail as the policy does not allow for a non-admin user +TOKEN=$(openstack token issue -f value -c id) +if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods -n openstack ; then + echo "Denied, as expected by policy" +else + exit 1 +fi diff --git a/tools/images/kubeadm-aio/Dockerfile b/tools/images/kubeadm-aio/Dockerfile index f572ae313..aa17d375d 100644 --- a/tools/images/kubeadm-aio/Dockerfile +++ b/tools/images/kubeadm-aio/Dockerfile @@ -37,7 +37,7 @@ ENV CNI_VERSION ${CNI_VERSION} ARG HELM_VERSION="v2.8.2" ENV HELM_VERSION ${HELM_VERSION} -ARG CHARTS="calico,flannel,tiller,kube-dns" +ARG CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook" ENV CHARTS ${CHARTS} ARG HTTP_PROXY="" diff --git a/tools/images/kubeadm-aio/assets/entrypoint.sh b/tools/images/kubeadm-aio/assets/entrypoint.sh index 095d37251..5fbcbb073 100755 --- a/tools/images/kubeadm-aio/assets/entrypoint.sh +++ b/tools/images/kubeadm-aio/assets/entrypoint.sh @@ -52,6 +52,7 @@ fi : ${KUBE_API_BIND_ADDR:="${KUBE_BIND_ADDR}"} : ${KUBE_CERTS_DIR:="/etc/kubernetes/pki"} : ${KUBE_SELF_HOSTED:="false"} +: ${KUBE_KEYSTONE_AUTH:="false"} : ${KUBELET_NODE_LABELS:=""} PLAYBOOK_VARS="{ @@ -78,6 +79,7 @@ PLAYBOOK_VARS="{ \"imageRepository\": \"${KUBE_IMAGE_REPO}\", \"certificatesDir\": \"${KUBE_CERTS_DIR}\", \"selfHosted\": \"${KUBE_SELF_HOSTED}\", + \"keystoneAuth\": \"${KUBE_KEYSTONE_AUTH}\", \"api\": { \"bindPort\": ${KUBE_API_BIND_PORT} }, diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-keystone-auth.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-keystone-auth.yaml new file mode 100644 index 000000000..5e47bbebc --- /dev/null +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-keystone-auth.yaml @@ -0,0 +1,101 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: setting up bootstrap tiller + block: + - name: pull the helm tiller Image + become: true + become_user: root + docker_image: + pull: true + name: "{{ helm.tiller_image }}" + - name: deploying bootstrap tiller + become: true + become_user: root + docker_container: + name: "helm-tiller" + image: "{{ helm.tiller_image }}" + state: started + detach: true + recreate: yes + network_mode: host + volumes: + - /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf:ro + env: + KUBECONFIG=/etc/kubernetes/admin.conf + register: kubeadm_aio_tiller_container + ignore_errors: True + - name: wait for tiller to be ready + delegate_to: 127.0.0.1 + command: helm version --server + environment: + HELM_HOST: 'localhost:44134' + register: task_result + until: task_result.rc == 0 + retries: 120 + delay: 5 + +- name: kubeadm | get certs + block: + - name: kubeadm | get kubeapi cert + shell: cat /etc/kubernetes/pki/apiserver.crt | base64 -w0 + register: kubeadm_kubeapi_cert + - name: kubeadm | get kubeapi key + shell: cat /etc/kubernetes/pki/apiserver.key | base64 -w0 + register: kubeadm_kubeapi_key + +- name: kubeadm | keystone auth + delegate_to: 127.0.0.1 + block: + - name: kubeadm | keystone auth + command: "helm upgrade --install kubernetes-keystone-webhook /opt/charts/kubernetes-keystone-webhook --namespace=kube-system --set endpoints.identity.namespace=openstack --set endpoints.kubernetes.auth.api.tls.crt={{ kubeadm_kubeapi_cert.stdout }} --set endpoints.kubernetes.auth.api.tls.key={{ kubeadm_kubeapi_key.stdout }}" + environment: + HELM_HOST: 'localhost:44134' + - name: kubeadm | keystone auth + command: helm status kubernetes-keystone-webhook + environment: + HELM_HOST: 'localhost:44134' + register: kubeadm_helm_keystone_status + - name: kubeadm | keystone auth + debug: + msg: "{{ kubeadm_helm_keystone_status }}" + +- name: kubeadm | setup api server for keystone + block: + - name: kubeadm | copying webhook config to host + become: true + become_user: root + template: + src: webhook.kubeconfig.j2 + dest: /etc/kubernetes/pki/webhook.kubeconfig + mode: 0640 + - name: kubeadm | configuring api server + become: true + become_user: root + shell: | + # TODO(lamt): Clean up this way of restarting the kube-apiserver. Preferably, + # the setting is in place when the kube-apiserver comes up. Currently, the + # kube-apiserver does not start whenever the webhook fails. + cat /etc/kubernetes/manifests/kube-apiserver.yaml > /tmp/kube-apiserver.yaml + sed -i '/etcd-keyfile/a \ \ \ \ -\ --authentication-token-webhook-config-file=/etc/kubernetes/pki/webhook.kubeconfig\n \ \ \ \- --authorization-webhook-config-file=/etc/kubernetes/pki/webhook.kubeconfig' /tmp/kube-apiserver.yaml + sed -i -e 's/Node,RBAC/Node,Webhook,RBAC/g' /tmp/kube-apiserver.yaml + sed -i '/hostNetwork: true/a\ \ dnsPolicy: ClusterFirstWithHostNet' /tmp/kube-apiserver.yaml + mv /tmp/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml + +- name: "removing bootstrap tiller container" + become: true + become_user: root + docker_container: + name: "helm-tiller" + state: absent diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml index d06e2c70c..4ec063771 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml @@ -136,14 +136,9 @@ executable: /bin/bash - include_tasks: wait-for-kube-system-namespace.yaml -# - name: deploying kube-dns addon -# delegate_to: 127.0.0.1 -# block: -# - name: master | deploy | kube-dns -# command: kubeadm alpha phase addon kube-dns --config /mnt/rootfs/etc/kubernetes/kubeadm-conf.yaml -# - include_tasks: wait-for-kube-system-namespace.yaml - - include_tasks: helm-dns.yaml +- include_tasks: helm-keystone-auth.yaml + when: k8s.keystoneAuth - include_tasks: helm-deploy.yaml - name: uploading cluster config to api diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/webhook.kubeconfig.j2 b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/webhook.kubeconfig.j2 new file mode 100644 index 000000000..681c7db6d --- /dev/null +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/webhook.kubeconfig.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +clusters: + - cluster: + insecure-skip-tls-verify: true + server: https://k8sksauth-api.kube-system.svc.cluster.local:8443/webhook + name: webhook +contexts: + - context: + cluster: webhook + user: webhook + name: webhook +current-context: webhook +kind: Config +preferences: {} +users: + - name: webhook diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml index 2f69c1fa8..814150776 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml @@ -37,6 +37,7 @@ all: imageRepository: gcr.io/google_containers certificatesDir: /etc/kubernetes/pki selfHosted: false + keystoneAuth: false api: bindPort: 6443 #NOTE(portdirect): The following is a custom key, which resolves the