[Calico] Update TLS settings for Calico

PS provides possibility to use TLS in etcd (for Calico). The ansible scripts were updated as well. Change-Id: I522a78043a125660153aaa60f13d61ba8e325e75
2018-12-13 10:53:17 -08:00 · 2018-12-13 10:53:17 -08:00 · 0c5e2c4830
commit 0c5e2c4830
parent b3097f6a25
9 changed files with 162 additions and 2 deletions
--- a/calico/templates/daemonset-calico-etcd.yaml
+++ b/calico/templates/daemonset-calico-etcd.yaml
@ -84,6 +84,16 @@ spec:
          args:
          - --name=calico
          - --data-dir=/var/etcd/calico-data
+{{ if eq .Values.endpoints.etcd.scheme.default "https" }}
+          - --client-cert-auth=True
+          - --peer-client-cert-auth=True
+          - --trusted-ca-file=/etc/calico-certs/ca.crt
+          - --cert-file=/etc/calico-certs/server.crt
+          - --key-file=/etc/calico-certs/server.key
+          - --peer-trusted-ca-file=/etc/calico-certs/ca.crt
+          - --peer-cert-file=/etc/calico-certs/server.crt
+          - --peer-key-file=/etc/calico-certs/server.key
+{{ end }}
          - --advertise-client-urls={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
          - --listen-client-urls={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}://0.0.0.0:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
          - --listen-peer-urls={{ tuple "etcd" "internal" "peer" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}://0.0.0.0:{{ tuple "etcd" "internal" "peer" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
@ -91,8 +101,29 @@ spec:
          volumeMounts:
            - name: var-etcd
              mountPath: /var/etcd
+{{ if .Values.conf.etcd.credentials.ca }}
+            - name: calico-etcd-secrets
+              mountPath: /etc/calico-certs/ca.crt
+              subPath: etcd.ca
+              readOnly: true
+{{ end }}
+{{ if .Values.conf.etcd.credentials.certificate }}
+            - name: calico-etcd-secrets
+              mountPath: /etc/calico-certs/server.crt
+              subPath: etcd.crt
+              readOnly: true
+{{ end }}
+{{ if .Values.conf.etcd.credentials.key }}
+            - name: calico-etcd-secrets
+              mountPath: /etc/calico-certs/server.key
+              subPath: etcd.key
+              readOnly: true
+{{ end }}
      volumes:
        - name: var-etcd
          hostPath:
            path: /var/etcd
+        - name: calico-etcd-secrets
+          secret:
+            secretName: calico-etcd-secrets
 {{- end }}
--- a/calico/templates/daemonset-calico-node.yaml
+++ b/calico/templates/daemonset-calico-node.yaml
@ -228,6 +228,24 @@ spec:
              mountPath: /host/opt/cni/bin
            - name: cni-net-dir
              mountPath: /host/etc/cni/net.d
+{{ if .Values.conf.etcd.credentials.ca }}
+            - name: calico-etcd-secrets
+              mountPath: /calico-secrets/etcd-ca
+              subPath: etcd.ca
+              readOnly: true
+{{ end }}
+{{ if .Values.conf.etcd.credentials.certificate }}
+            - name: calico-etcd-secrets
+              mountPath: /calico-secrets/etcd-cert
+              subPath: etcd.crt
+              readOnly: true
+{{ end }}
+{{ if .Values.conf.etcd.credentials.key }}
+            - name: calico-etcd-secrets
+              mountPath: /calico-secrets/etcd-key
+              subPath: etcd.key
+              readOnly: true
+{{ end }}
      volumes:
        # Used by calico/node.
        - name: lib-modules
--- a/calico/templates/secret-etcd-certificates.yaml
+++ b/calico/templates/secret-etcd-certificates.yaml
@ -27,4 +27,7 @@ data:
  tls.ca: {{ .Values.endpoints.etcd.auth.client.tls.ca | default "" | b64enc }}
  tls.key: {{ .Values.endpoints.etcd.auth.client.tls.key | default "" | b64enc }}
  tls.crt: {{ .Values.endpoints.etcd.auth.client.tls.crt | default "" | b64enc }}
+  etcd.ca: {{ .Values.conf.etcd.credentials.ca | default "" | b64enc }}
+  etcd.crt: {{ .Values.conf.etcd.credentials.certificate | default "" | b64enc }}
+  etcd.key: {{ .Values.conf.etcd.credentials.key | default "" | b64enc }}
 {{- end }}
--- a/tools/images/kubeadm-aio/assets/entrypoint.sh
+++ b/tools/images/kubeadm-aio/assets/entrypoint.sh
@ -36,6 +36,7 @@ fi
 : ${CONTAINER_NAME:="null"}
 : ${CONTAINER_RUNTIME:="docker"}
 : ${CNI_ENABLED:="calico"}
+: ${CNI_HOST_IP:="10.96.232.136"}
 : ${NET_SUPPORT_LINUXBRIDGE:="true"}
 : ${PVC_SUPPORT_CEPH:="false"}
 : ${PVC_SUPPORT_NFS:="false"}
@ -66,7 +67,8 @@ PLAYBOOK_VARS="{
    \"home\": \"${USER_HOME}\"
  },
  \"cluster\": {
-    \"cni\": \"${CNI_ENABLED}\"
+    \"cni\": \"${CNI_ENABLED}\",
+    \"cni_host_ip\": \"${CNI_HOST_IP}\"
  },
  \"kubelet\": {
    \"container_runtime\": \"${CONTAINER_RUNTIME}\",
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml
@ -55,6 +55,47 @@
        executable: /bin/bash
      register: cni_default_device_mtu

+- name: Generate  self-signed certificates
+  when: cluster.cni == 'calico'
+  delegate_to: 127.0.0.1
+  block:
+    - name: Get cfssl
+      get_url:
+        url: "https://pkg.cfssl.org/R1.2/{{ item }}_linux-amd64"
+        dest: "/usr/local/bin/{{ item }}"
+        mode: 744
+      with_items:
+        - cfssl
+        - cfssljson
+    - name: Add ca-config
+      template:
+        src: ca-config.json.j2
+        dest: /etc/kubernetes/pki/calico/ca-config.json
+    - name: Copy CA
+      copy:
+        src: /etc/kubernetes/pki/{{ item }}
+        dest: /etc/kubernetes/pki/calico/{{ item }}
+      with_items:
+        - ca.crt
+        - ca.key
+    - name: Prepare certificates for Calico
+      shell: |
+        echo '{"CN":"server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=ca.crt -ca-key=ca.key -config=ca-config.json -profile=server -hostname="{{ cluster.cni_host_ip }},127.0.0.1,localhost" - | cfssljson -bare server
+        echo '{"CN":"client","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=ca.crt -ca-key=ca.key -config=ca-config.json -profile=client - | cfssljson -bare client
+      args:
+        chdir: /etc/kubernetes/pki/calico
+        executable: /bin/bash
+    - name: Add script file
+      template:
+        src: certs.py.j2
+        dest: /etc/kubernetes/pki/calico/certs.py
+
+    - name: Create yaml file
+      shell: python /etc/kubernetes/pki/calico/certs.py
+      args:
+        executable: /bin/bash
+        chdir: /etc/kubernetes/pki/calico
+
 - name: kubeadm | cni | calico
  when: cluster.cni == 'calico'
  delegate_to: 127.0.0.1
@ -64,7 +105,7 @@
      environment:
        KUBECONFIG: '/mnt/rootfs/etc/kubernetes/admin.conf'
    - name: kubeadm | cni | calico
-      command: helm install /opt/charts/calico --name calico --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --set conf.node.IP_AUTODETECTION_METHOD="can-reach={% if k8s.api.advertiseAddress is defined %}{{ k8s.api.advertiseAddress }}{% else %}{% if k8s.api.advertiseAddressDevice is defined %}{{ hostvars[inventory_hostname]['ansible_'+k8s.api.advertiseAddressDevice].ipv4.address }}{% else %}{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}{% endif %}{% endif %}" --set networking.mtu="{{ cni_default_device_mtu.stdout }}" --wait --timeout=600
+      command: helm install /opt/charts/calico --name calico --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --set conf.node.IP_AUTODETECTION_METHOD="can-reach={% if k8s.api.advertiseAddress is defined %}{{ k8s.api.advertiseAddress }}{% else %}{% if k8s.api.advertiseAddressDevice is defined %}{{ hostvars[inventory_hostname]['ansible_'+k8s.api.advertiseAddressDevice].ipv4.address }}{% else %}{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}{% endif %}{% endif %}" --set networking.mtu="{{ cni_default_device_mtu.stdout }}" --values="/etc/kubernetes/pki/calico/calico_certs.yaml" --wait --timeout=600
      environment:
        HELM_HOST: 'localhost:44134'
    - name: kubeadm | cni | calico
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml
@ -25,6 +25,7 @@
      with_items:
        - /etc/kubernetes
        - /etc/kubernetes/pki
+        - /etc/kubernetes/pki/calico
    - name: generating initial admin token
      delegate_to: 127.0.0.1
      command: /usr/bin/kubeadm token generate
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/ca-config.json.j2
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/ca-config.json.j2
@ -0,0 +1,35 @@
+{
+    "signing": {
+        "default": {
+            "expiry": "8760h"
+        },
+        "profiles": {
+            "server": {
+                "expiry": "8760h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "server auth",
+                    "client auth"
+                ]
+            },
+            "client": {
+                "expiry": "8760h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "client auth"
+                ]
+            },
+            "peer": {
+                "expiry": "8760h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "server auth",
+                    "client auth"
+                ]
+            }
+        }
+    }
+}
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/certs.py.j2
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/certs.py.j2
@ -0,0 +1,28 @@
+#!/usr/bin/python
+
+import yaml
+
+data = {'server': '',
+        'server-key': '',
+        'client': '',
+        'client-key': ''}
+
+for i in data.keys():
+    with open('{}.pem'.format(i)) as f:
+        data[i] = f.read()
+with open('ca.crt') as f:
+    data['ca'] = f.read()
+
+res = {'endpoints': {'etcd': {'auth': {'client': {'tls': ''}},
+                              'scheme': {'default': 'https'}}},
+       'conf': {'etcd': {'credentials': ''}}}
+
+res['endpoints']['etcd']['auth']['client']['tls'] = {'ca': data['ca'],
+                                                     'key': data['client-key'],
+                                                     'crt': data['client']}
+res['conf']['etcd']['credentials'] = {'ca': data['ca'],
+                                      'key': data['server-key'],
+                                      'certificate': data['server']}
+
+with open('calico_certs.yaml', 'w') as f:
+    yaml.dump(res, f, default_style='|')
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml
@ -24,6 +24,7 @@ all:
      - 8.8.4.4
    cluster:
      cni: calico
+      cni_host_ip: 10.96.232.136
    kubelet:
      container_runtime: docker
      net_support_linuxbridge: true