diff --git a/roles/deploy-env/README.md b/roles/deploy-env/README.md new file mode 100644 index 000000000..6dec5e634 --- /dev/null +++ b/roles/deploy-env/README.md @@ -0,0 +1,37 @@ +This role is used to deploy test environment which includes +- install necessary prerequisites including Helm +- deploy Containerd and a container runtime for Kubernetes +- deploy Kubernetes using Kubeadm with a single control plain node +- install Calico as a Kubernetes networking + +The role works both for singlenode and multinode inventories and +assumes the inventory has the node called `primary` and the group called `nodes`. + +See for example: + +```yaml +all: + children: + ungrouped: + hosts: + primary: + ansible_port: 22 + ansible_host: 10.10.10.10 + ansible_user: ubuntu + ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub + ansible_ssh_extra_args: -o StrictHostKeyChecking=no + nodes: + hosts: + node-1: + ansible_port: 22 + ansible_host: 10.10.10.11 + ansible_user: ubuntu + ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub + ansible_ssh_extra_args: -o StrictHostKeyChecking=no + node-2: + ansible_port: 22 + ansible_host: 10.10.10.12 + ansible_user: ubuntu + ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub + ansible_ssh_extra_args: -o StrictHostKeyChecking=no +``` diff --git a/roles/deploy-env/defaults/main.yaml b/roles/deploy-env/defaults/main.yaml new file mode 100644 index 000000000..07f340c5b --- /dev/null +++ b/roles/deploy-env/defaults/main.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kubectl: + user: zuul + group: zuul +... diff --git a/roles/deploy-env/files/calico_patch.yaml b/roles/deploy-env/files/calico_patch.yaml new file mode 100644 index 000000000..cdb38bb15 --- /dev/null +++ b/roles/deploy-env/files/calico_patch.yaml @@ -0,0 +1,23 @@ +--- +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9091" + spec: + containers: + - name: calico-node + env: + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "true" + - name: FELIX_PROMETHEUSMETRICSPORT + value: "9091" + - name: FELIX_IGNORELOOSERPF + value: "true" + # We assign IP on br-ex interface while testing the deployed Openstack cluster and + # we need Calico to skip this interface while discovering the + # network changes on the host to prevent announcing unnecessary networks. + - name: IP_AUTODETECTION_METHOD + value: "skip-interface=br-ex" +... diff --git a/roles/deploy-env/files/containerd_config.toml b/roles/deploy-env/files/containerd_config.toml new file mode 100644 index 000000000..cc6ab0bc4 --- /dev/null +++ b/roles/deploy-env/files/containerd_config.toml @@ -0,0 +1,11 @@ +version = 2 +disabled_plugins = [] +[plugins."io.containerd.grpc.v1.cri".registry] +config_path = "/etc/containerd/certs.d" + +{% for item in registry_namespaces %} +{% if item.auth is defined %} +[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth] +auth = "{{ item.auth }}" +{% endif %} +{% endfor %} \ No newline at end of file diff --git a/roles/deploy-env/files/daemon.json b/roles/deploy-env/files/daemon.json new file mode 100644 index 000000000..254799247 --- /dev/null +++ b/roles/deploy-env/files/daemon.json @@ -0,0 +1,9 @@ +{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2", + "live-restore": true +} diff --git a/roles/deploy-env/files/hosts.toml b/roles/deploy-env/files/hosts.toml new file mode 100644 index 000000000..e8c08eedb --- /dev/null +++ b/roles/deploy-env/files/hosts.toml @@ -0,0 +1,12 @@ +{% if item.skip_server is not defined or not item.skip_server %} +server = "{{ item.server | default('https://' + item.namespace) }}" +{% endif %} + +[host."{{ item.mirror }}"] +capabilities = ["pull", "resolve", "push"] +{% if item.ca is defined %} +ca = "{{ item.ca }}" +{% endif %} +{% if item.skip_verify is defined and item.skip_verify %} +skip_verify = true +{% endif %} diff --git a/roles/deploy-env/files/kubeadm_config.yaml b/roles/deploy-env/files/kubeadm_config.yaml new file mode 100644 index 000000000..25b1adcf2 --- /dev/null +++ b/roles/deploy-env/files/kubeadm_config.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +mode: ipvs +... +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +networking: + serviceSubnet: "10.96.0.0/16" + podSubnet: "10.244.0.0/24" # --pod-network-cidr + dnsDomain: "cluster.local" +... diff --git a/roles/deploy-env/files/resolv.conf b/roles/deploy-env/files/resolv.conf new file mode 100644 index 000000000..5f9818c77 --- /dev/null +++ b/roles/deploy-env/files/resolv.conf @@ -0,0 +1,4 @@ +nameserver 8.8.8.8 +nameserver 8.8.4.4 +search svc.cluster.local cluster.local +options ndots:5 timeout:1 attempts:1 diff --git a/roles/deploy-env/tasks/common_k8s.yaml b/roles/deploy-env/tasks/common_k8s.yaml new file mode 100644 index 000000000..ad222dfde --- /dev/null +++ b/roles/deploy-env/tasks/common_k8s.yaml @@ -0,0 +1,106 @@ +--- +- name: Load necessary modules + modprobe: + name: "{{ item }}" + state: present + with_items: + - overlay + - br_netfilter + +- name: Configure sysctl + sysctl: + name: "{{ item }}" + value: "1" + state: present + loop: + - net.ipv6.conf.default.disable_ipv6 + - net.ipv6.conf.all.disable_ipv6 + - net.ipv6.conf.lo.disable_ipv6 + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables + - net.ipv4.ip_forward + ignore_errors: true + +- name: Remove swapfile from /etc/fstab + mount: + name: "{{ item }}" + fstype: swap + state: absent + with_items: + - swap + - none + +- name: Disable swap + command: swapoff -a + when: ansible_swaptotal_mb > 0 + +- name: Ensure dependencies are installed + apt: + name: + - apt-transport-https + - ca-certificates + - gnupg2 + - ipvsadm + - jq + state: present + +- name: Add Kubernetes apt repository key + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + state: present + +- name: Add Kubernetes apt repository + apt_repository: + repo: deb https://apt.kubernetes.io/ kubernetes-xenial main + state: present + filename: kubernetes.list + +- name: Install Kubernetes binaries + apt: + state: present + update_cache: true + allow_downgrade: true + pkg: + - "kubelet={{ kube_version }}" + - "kubeadm={{ kube_version }}" + - "kubectl={{ kube_version }}" + +- name: Restart kubelet + service: + name: kubelet + daemon_reload: yes + state: restarted + +- name: Disable systemd-resolved + service: + name: systemd-resolved + enabled: false + state: stopped + +- name: Configure resolv.conf + copy: + src: files/resolv.conf + dest: "{{ item }}" + loop: + - /etc/resolv.conf + - /run/systemd/resolve/resolv.conf + +# We download Calico manifest on all nodes because we then want to download +# Calico images BEFORE deploying it +- name: Download Calico manifest + shell: | + curl -LSs https://docs.projectcalico.org/archive/{{ calico_version }}/manifests/calico.yaml -o /tmp/calico.yaml + sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + args: + executable: /bin/bash + +# Download images needed for calico before applying manifests, so that `kubectl wait` timeout +# for `k8s-app=kube-dns` isn't reached by slow download speeds +- name: Download Calico images + shell: | + export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock + export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock + awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {} + args: + executable: /bin/bash +... diff --git a/roles/deploy-env/tasks/containerd.yaml b/roles/deploy-env/tasks/containerd.yaml new file mode 100644 index 000000000..833b985c8 --- /dev/null +++ b/roles/deploy-env/tasks/containerd.yaml @@ -0,0 +1,148 @@ +--- +- name: Remove old docker packages + apt: + pkg: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + state: absent + +- name: Ensure dependencies are installed + apt: + name: + - apt-transport-https + - ca-certificates + - gnupg2 + state: present + +- name: Add Docker apt repository key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + keyring: /etc/apt/trusted.gpg.d/docker.gpg + state: present + +- name: Get dpkg arch + command: dpkg --print-architecture + register: dpkg_architecture + +- name: Add Docker apt repository + apt_repository: + repo: deb [arch="{{ dpkg_architecture.stdout }}" signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/ubuntu "{{ ansible_distribution_release }}" stable + state: present + filename: docker.list + +- name: Install docker packages + apt: + pkg: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: true + +- name: Install Crictl + shell: | + wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz + sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin + rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz + args: + executable: /bin/bash + +- name: Configure Docker daemon + copy: + src: files/daemon.json + dest: /etc/docker/daemon.json + +- name: Restart docker + service: + name: docker + daemon_reload: yes + state: restarted + +- name: Set mirror_fqdn fact + when: + - registry_mirror is not defined + - zuul_site_mirror_fqdn is defined + set_fact: + registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082" + +- name: Set regitstry namespaces + set_fact: + registry_namespaces: + - namespace: "_default" + mirror: "{{ registry_mirror }}" + skip_server: true + skip_verify: true + when: registry_mirror is defined + +- name: Buildset registry namespace + when: buildset_registry is defined + block: + - name: Buildset registry alias + include_tasks: + file: buildset_registry_alias.yaml + + - name: Write buildset registry TLS certificate + copy: + content: "{{ buildset_registry.cert }}" + dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt" + mode: 0644 + register: buildset_registry_tls_ca + + - name: Update CA certs + command: "update-ca-certificates" + when: buildset_registry_tls_ca is changed + + - name: Set buildset registry namespace + set_fact: + buildset_registry_namespace: + namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}' + mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}' + ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt" + auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}" + + - name: Init registry_namespaces if not defined + set_fact: + registry_namespaces: "[]" + when: not registry_namespaces is defined + + - name: Append buildset_registry to registry namespaces + when: + - buildset_registry_namespace is defined + - registry_namespaces is defined + set_fact: + registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}" + +- name: Configure containerd + template: + src: files/containerd_config.toml + dest: /etc/containerd/config.toml + +- name: Create containerd config directory hierarchy + file: + state: directory + path: /etc/containerd/certs.d + +- name: Create host namespace directory + file: + state: directory + path: "/etc/containerd/certs.d/{{ item.namespace }}" + loop: "{{ registry_namespaces }}" + +- name: Create hosts.toml file + template: + src: files/hosts.toml + dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml" + loop: "{{ registry_namespaces }}" + +- name: Restart containerd + service: + name: containerd + daemon_reload: yes + state: restarted +... diff --git a/roles/deploy-env/tasks/control-plane.yaml b/roles/deploy-env/tasks/control-plane.yaml new file mode 100644 index 000000000..1063aebfc --- /dev/null +++ b/roles/deploy-env/tasks/control-plane.yaml @@ -0,0 +1,70 @@ +--- +- name: Mount tmpfs to /var/lib/etcd + mount: + path: /var/lib/etcd + src: tmpfs + fstype: tmpfs + opts: size=1g + state: mounted + +- name: Prepare kubeadm config + copy: + src: files/kubeadm_config.yaml + dest: /tmp/kubeadm_config.yaml + +- name: Initialize the Kubernetes cluster using kubeadm + command: kubeadm init --config /tmp/kubeadm_config.yaml + +- name: "Setup kubeconfig for {{ kubectl.user }} user" + shell: | + mkdir -p /home/{{ kubectl.user }}/.kube + cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config + chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config + args: + executable: /bin/bash + +- name: Deploy Calico + become: false + command: kubectl apply -f /tmp/calico.yaml + +- name: Sleep before trying to check Calico pods + pause: + seconds: 20 + +- name: Wait for Calico pods ready + become: false + command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node + +- name: Prepare Calico patch + copy: + src: files/calico_patch.yaml + dest: /tmp/calico_patch.yaml + +- name: Patch Calico + become: false + command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml + +- name: Wait for Calico pods ready + become: false + command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node + +- name: Generate join command + command: kubeadm token create --print-join-command + register: join_command + +- name: Untaint Kubernetes control plane node + become: false + command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane- + +- name: Enable recursive queries for coredns + become: false + shell: | + PATCH=$(mktemp) + kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}" | sed 's/ready\\n/header \{\\n response set ra\\n \}\\n ready\\n/g' > "${PATCH}" + kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}" + kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4" + kubectl rollout restart -n kube-system deployment/coredns + rm -f "${PATCH}" + args: + executable: /bin/bash +... diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml new file mode 100644 index 000000000..99e7925cb --- /dev/null +++ b/roles/deploy-env/tasks/main.yaml @@ -0,0 +1,39 @@ +--- +- name: Include prerequisites tasks + include_tasks: + file: prerequisites.yaml + +- name: Include common tasks + include_tasks: + file: containerd.yaml + +- name: Include common tasks + include_tasks: + file: common_k8s.yaml + +- name: Include control-plane tasks + include_tasks: + file: control-plane.yaml + when: inventory_hostname == 'primary' + +- name: Join workload nodes to cluster + command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}" + when: inventory_hostname in (groups['nodes'] | default([])) + +- name: Wait for cluster is ready + become: false + block: + - name: Wait for Calico pods ready + command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node + + - name: Wait for Coredns pods ready + command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns + when: inventory_hostname == 'primary' + +- name: Add coredns to /etc/resolv.conf + lineinfile: + line: nameserver 10.96.0.10 + path: /etc/resolv.conf + state: present + insertbefore: "BOF" +... diff --git a/roles/deploy-env/tasks/prerequisites.yaml b/roles/deploy-env/tasks/prerequisites.yaml new file mode 100644 index 000000000..cd71a9a7a --- /dev/null +++ b/roles/deploy-env/tasks/prerequisites.yaml @@ -0,0 +1,61 @@ +--- +- name: Add Ceph apt repository key + apt_key: + url: https://download.ceph.com/keys/release.asc + state: present + +- name: Add Ceph apt repository + apt_repository: + repo: deb https://download.ceph.com/debian-reef/ "{{ ansible_distribution_release }}" main + state: present + filename: ceph.list + +- name: Install necessary packages + apt: + pkg: + - socat + - jq + - util-linux + - bridge-utils + - iptables + - conntrack + - libffi-dev + - ipvsadm + - make + - bc + - git-review + - notary + - ceph-common + - rbd-nbd + - nfs-common + - ethtool + - python3-dev + - ca-certificates + - git + - nmap + - curl + - uuid-runtime + - net-tools + - less + - telnet + - tcpdump + - vim + - lvm2 + +- name: Deploy Helm + when: inventory_hostname == 'primary' + block: + - name: Install Helm + shell: | + TMP_DIR=$(mktemp -d) + curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + mv "${TMP_DIR}"/helm /usr/local/bin/helm + rm -rf "${TMP_DIR}" + args: + executable: /bin/bash + + # This is to improve build time + - name: Remove stable Helm repo + command: helm repo remove stable + ignore_errors: true +...