Add deploy-env role

This role works both for singlenode and multinode
inventories. The role installs all necessary prerequisites
and deploys K8s with Containerd as a container runtime.
The idea is to use this role to deploy
all test singlenode/multinode environments for all test jobs.

This PR wraps into a role playbooks that
we are currently using for multinode compute-kit tests.

Change-Id: I41bbe80d806e614a155e6775c4505a4d81a086e8

roles/deploy-env/README.md

@@ -0,0 +1,37 @@
+This role is used to deploy test environment which includes
+- install necessary prerequisites including Helm
+- deploy Containerd and a container runtime for Kubernetes
+- deploy Kubernetes using Kubeadm with a single control plain node
+- install Calico as a Kubernetes networking
+
+The role works both for singlenode and multinode inventories and
+assumes the inventory has the node called `primary` and the group called `nodes`.
+
+See for example:
+
+```yaml
+all:
+  children:
+    ungrouped:
+      hosts:
+        primary:
+          ansible_port: 22
+          ansible_host: 10.10.10.10
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+    nodes:
+      hosts:
+        node-1:
+          ansible_port: 22
+          ansible_host: 10.10.10.11
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+        node-2:
+          ansible_port: 22
+          ansible_host: 10.10.10.12
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa.pub
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+```

roles/deploy-env/defaults/main.yaml

@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+kubectl:
+  user: zuul
+  group: zuul
+...

roles/deploy-env/files/calico_patch.yaml

@@ -0,0 +1,23 @@
+---
+spec:
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9091"
+    spec:
+      containers:
+        - name: calico-node
+          env:
+            - name: FELIX_PROMETHEUSMETRICSENABLED
+              value: "true"
+            - name: FELIX_PROMETHEUSMETRICSPORT
+              value: "9091"
+            - name: FELIX_IGNORELOOSERPF
+              value: "true"
+            # We assign IP on br-ex interface while testing the deployed Openstack cluster and
+            # we need Calico to skip this interface while discovering the
+            # network changes on the host to prevent announcing unnecessary networks.
+            - name: IP_AUTODETECTION_METHOD
+              value: "skip-interface=br-ex"
+...

roles/deploy-env/files/containerd_config.toml

@@ -0,0 +1,11 @@
+version = 2
+disabled_plugins = []
+[plugins."io.containerd.grpc.v1.cri".registry]
+config_path = "/etc/containerd/certs.d"
+
+{% for item in registry_namespaces %}
+{% if item.auth is defined %}
+[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth]
+auth = "{{ item.auth }}"
+{% endif %}
+{% endfor %}

roles/deploy-env/files/daemon.json

@@ -0,0 +1,9 @@
+{
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "100m"
+    },
+    "storage-driver": "overlay2",
+    "live-restore": true
+}

roles/deploy-env/files/hosts.toml

@@ -0,0 +1,12 @@
+{% if item.skip_server is not defined or not item.skip_server %}
+server = "{{ item.server | default('https://' + item.namespace) }}"
+{% endif %}
+
+[host."{{ item.mirror }}"]
+capabilities = ["pull", "resolve", "push"]
+{% if item.ca is defined %}
+ca = "{{ item.ca }}"
+{% endif %}
+{% if item.skip_verify is defined and item.skip_verify %}
+skip_verify = true
+{% endif %}

roles/deploy-env/files/kubeadm_config.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: ipvs
+...
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+networking:
+  serviceSubnet: "10.96.0.0/16"
+  podSubnet: "10.244.0.0/24"  # --pod-network-cidr
+  dnsDomain: "cluster.local"
+...

roles/deploy-env/files/resolv.conf

@@ -0,0 +1,4 @@
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+search svc.cluster.local cluster.local
+options ndots:5 timeout:1 attempts:1

roles/deploy-env/tasks/common_k8s.yaml

@@ -0,0 +1,106 @@
+---
+- name: Load necessary modules
+  modprobe:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - overlay
+    - br_netfilter
+
+- name: Configure sysctl
+  sysctl:
+    name: "{{ item }}"
+    value: "1"
+    state: present
+  loop:
+    - net.ipv6.conf.default.disable_ipv6
+    - net.ipv6.conf.all.disable_ipv6
+    - net.ipv6.conf.lo.disable_ipv6
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.ipv4.ip_forward
+  ignore_errors: true
+
+- name: Remove swapfile from /etc/fstab
+  mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  with_items:
+    - swap
+    - none
+
+- name: Disable swap
+  command: swapoff -a
+  when: ansible_swaptotal_mb > 0
+
+- name: Ensure dependencies are installed
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      - ipvsadm
+      - jq
+    state: present
+
+- name: Add Kubernetes apt repository key
+  apt_key:
+    url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
+    state: present
+
+- name: Add Kubernetes apt repository
+  apt_repository:
+    repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
+    state: present
+    filename: kubernetes.list
+
+- name: Install Kubernetes binaries
+  apt:
+    state: present
+    update_cache: true
+    allow_downgrade: true
+    pkg:
+      - "kubelet={{ kube_version }}"
+      - "kubeadm={{ kube_version }}"
+      - "kubectl={{ kube_version }}"
+
+- name: Restart kubelet
+  service:
+    name: kubelet
+    daemon_reload: yes
+    state: restarted
+
+- name: Disable systemd-resolved
+  service:
+    name: systemd-resolved
+    enabled: false
+    state: stopped
+
+- name: Configure resolv.conf
+  copy:
+    src: files/resolv.conf
+    dest: "{{ item }}"
+  loop:
+    - /etc/resolv.conf
+    - /run/systemd/resolve/resolv.conf
+
+# We download Calico manifest on all nodes because we then want to download
+# Calico images BEFORE deploying it
+- name: Download Calico manifest
+  shell: |
+    curl -LSs https://docs.projectcalico.org/archive/{{ calico_version }}/manifests/calico.yaml -o /tmp/calico.yaml
+    sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
+  args:
+    executable: /bin/bash
+
+# Download images needed for calico before applying manifests, so that `kubectl wait` timeout
+# for `k8s-app=kube-dns` isn't reached by slow download speeds
+- name: Download Calico images
+  shell: |
+    export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
+    export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
+    awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
+  args:
+    executable: /bin/bash
+...

roles/deploy-env/tasks/containerd.yaml

@@ -0,0 +1,148 @@
+---
+- name: Remove old docker packages
+  apt:
+    pkg:
+      - docker.io
+      - docker-doc
+      - docker-compose
+      - podman-docker
+      - containerd
+      - runc
+    state: absent
+
+- name: Ensure dependencies are installed
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+    state: present
+
+- name: Add Docker apt repository key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    keyring: /etc/apt/trusted.gpg.d/docker.gpg
+    state: present
+
+- name: Get dpkg arch
+  command: dpkg --print-architecture
+  register: dpkg_architecture
+
+- name: Add Docker apt repository
+  apt_repository:
+    repo: deb [arch="{{ dpkg_architecture.stdout }}" signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/ubuntu "{{ ansible_distribution_release }}" stable
+    state: present
+    filename: docker.list
+
+- name: Install docker packages
+  apt:
+    pkg:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+    update_cache: true
+
+- name: Install Crictl
+  shell: |
+    wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz
+    sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin
+    rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz
+  args:
+    executable: /bin/bash
+
+- name: Configure Docker daemon
+  copy:
+    src: files/daemon.json
+    dest: /etc/docker/daemon.json
+
+- name: Restart docker
+  service:
+    name: docker
+    daemon_reload: yes
+    state: restarted
+
+- name: Set mirror_fqdn fact
+  when:
+    - registry_mirror is not defined
+    - zuul_site_mirror_fqdn is defined
+  set_fact:
+    registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082"
+
+- name: Set regitstry namespaces
+  set_fact:
+    registry_namespaces:
+      - namespace: "_default"
+        mirror: "{{ registry_mirror }}"
+        skip_server: true
+        skip_verify: true
+  when: registry_mirror is defined
+
+- name: Buildset registry namespace
+  when: buildset_registry is defined
+  block:
+    - name: Buildset registry alias
+      include_tasks:
+        file: buildset_registry_alias.yaml
+
+    - name: Write buildset registry TLS certificate
+      copy:
+        content: "{{ buildset_registry.cert }}"
+        dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+        mode: 0644
+      register: buildset_registry_tls_ca
+
+    - name: Update CA certs
+      command: "update-ca-certificates"
+      when: buildset_registry_tls_ca is changed
+
+    - name: Set buildset registry namespace
+      set_fact:
+        buildset_registry_namespace:
+          namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+          mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+          ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+          auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}"
+
+    - name: Init registry_namespaces if not defined
+      set_fact:
+        registry_namespaces: "[]"
+      when: not registry_namespaces is defined
+
+    - name: Append buildset_registry to registry namespaces
+      when:
+        - buildset_registry_namespace is defined
+        - registry_namespaces is defined
+      set_fact:
+        registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}"
+
+- name: Configure containerd
+  template:
+    src: files/containerd_config.toml
+    dest: /etc/containerd/config.toml
+
+- name: Create containerd config directory hierarchy
+  file:
+    state: directory
+    path: /etc/containerd/certs.d
+
+- name: Create host namespace directory
+  file:
+    state: directory
+    path: "/etc/containerd/certs.d/{{ item.namespace }}"
+  loop: "{{ registry_namespaces }}"
+
+- name: Create hosts.toml file
+  template:
+    src: files/hosts.toml
+    dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml"
+  loop: "{{ registry_namespaces }}"
+
+- name: Restart containerd
+  service:
+    name: containerd
+    daemon_reload: yes
+    state: restarted
+...

roles/deploy-env/tasks/control-plane.yaml

@@ -0,0 +1,70 @@
+---
+- name: Mount tmpfs to /var/lib/etcd
+  mount:
+    path: /var/lib/etcd
+    src: tmpfs
+    fstype: tmpfs
+    opts: size=1g
+    state: mounted
+
+- name: Prepare kubeadm config
+  copy:
+    src: files/kubeadm_config.yaml
+    dest: /tmp/kubeadm_config.yaml
+
+- name: Initialize the Kubernetes cluster using kubeadm
+  command: kubeadm init --config /tmp/kubeadm_config.yaml
+
+- name: "Setup kubeconfig for {{ kubectl.user }} user"
+  shell: |
+    mkdir -p /home/{{ kubectl.user }}/.kube
+    cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config
+    chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config
+  args:
+    executable: /bin/bash
+
+- name: Deploy Calico
+  become: false
+  command: kubectl apply -f /tmp/calico.yaml
+
+- name: Sleep before trying to check Calico pods
+  pause:
+    seconds: 20
+
+- name: Wait for Calico pods ready
+  become: false
+  command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+
+- name: Prepare Calico patch
+  copy:
+    src: files/calico_patch.yaml
+    dest: /tmp/calico_patch.yaml
+
+- name: Patch Calico
+  become: false
+  command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml
+
+- name: Wait for Calico pods ready
+  become: false
+  command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+
+- name: Generate join command
+  command: kubeadm token create --print-join-command
+  register: join_command
+
+- name: Untaint Kubernetes control plane node
+  become: false
+  command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane-
+
+- name: Enable recursive queries for coredns
+  become: false
+  shell: |
+    PATCH=$(mktemp)
+    kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}"  | sed 's/ready\\n/header \{\\n        response set ra\\n    \}\\n    ready\\n/g' > "${PATCH}"
+    kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}"
+    kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4"
+    kubectl rollout restart -n kube-system deployment/coredns
+    rm -f "${PATCH}"
+  args:
+    executable: /bin/bash
+...

roles/deploy-env/tasks/main.yaml

@@ -0,0 +1,39 @@
+---
+- name: Include prerequisites tasks
+  include_tasks:
+    file: prerequisites.yaml
+
+- name: Include common tasks
+  include_tasks:
+    file: containerd.yaml
+
+- name: Include common tasks
+  include_tasks:
+    file: common_k8s.yaml
+
+- name: Include control-plane tasks
+  include_tasks:
+    file: control-plane.yaml
+  when: inventory_hostname == 'primary'
+
+- name: Join workload nodes to cluster
+  command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}"
+  when: inventory_hostname in (groups['nodes'] | default([]))
+
+- name: Wait for cluster is ready
+  become: false
+  block:
+    - name: Wait for Calico pods ready
+      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+
+    - name: Wait for Coredns pods ready
+      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
+  when: inventory_hostname == 'primary'
+
+- name: Add coredns to /etc/resolv.conf
+  lineinfile:
+    line: nameserver 10.96.0.10
+    path: /etc/resolv.conf
+    state: present
+    insertbefore: "BOF"
+...

roles/deploy-env/tasks/prerequisites.yaml

@@ -0,0 +1,61 @@
+---
+- name: Add Ceph apt repository key
+  apt_key:
+    url: https://download.ceph.com/keys/release.asc
+    state: present
+
+- name: Add Ceph apt repository
+  apt_repository:
+    repo: deb https://download.ceph.com/debian-reef/ "{{ ansible_distribution_release }}" main
+    state: present
+    filename: ceph.list
+
+- name: Install necessary packages
+  apt:
+    pkg:
+      - socat
+      - jq
+      - util-linux
+      - bridge-utils
+      - iptables
+      - conntrack
+      - libffi-dev
+      - ipvsadm
+      - make
+      - bc
+      - git-review
+      - notary
+      - ceph-common
+      - rbd-nbd
+      - nfs-common
+      - ethtool
+      - python3-dev
+      - ca-certificates
+      - git
+      - nmap
+      - curl
+      - uuid-runtime
+      - net-tools
+      - less
+      - telnet
+      - tcpdump
+      - vim
+      - lvm2
+
+- name: Deploy Helm
+  when: inventory_hostname == 'primary'
+  block:
+    - name: Install Helm
+      shell: |
+        TMP_DIR=$(mktemp -d)
+        curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}
+        mv "${TMP_DIR}"/helm /usr/local/bin/helm
+        rm -rf "${TMP_DIR}"
+      args:
+        executable: /bin/bash
+
+    # This is to improve build time
+    - name: Remove stable Helm repo
+      command: helm repo remove stable
+      ignore_errors: true
+...