Zun: Add zun-cni-daemon to compute node

Zun has a new component "zun-cni-daemon" which should be
deployed in every compute nodes. It is basically an implementation
of CNI (Container Network Interface) that performs the neutron
port binding.

If users is using the capsule (pod) API, the recommended deployment
option is using "cri" as capsule driver. This is basically to use
a CRI runtime (i.e. CRI plugin for containerd) for supporting
capsules (pods). A CRI runtime needs a CNI plugin which is what
the "zun-cni-daemon" provides.

The configuration is based on the Zun installation guide [1].
It consits of the following steps:
* Configure the containerd daemon in the host. The "zun-compute"
  container will use grpc to communicate with this service.
* Install the "zun-cni" binary at host. The containerd process
  will invoke this binary to call the CNI plugin.
* Run a "zun-cni-daemon" container. The "zun-cni" binary will
  communicate with this container via HTTP.

Relevant patches:
Blueprint: https://blueprints.launchpad.net/zun/+spec/add-support-cri-runtime
Install guide: https://review.opendev.org/#/c/707948/
Devstack plugin: https://review.opendev.org/#/c/705338/
Kolla image: https://review.opendev.org/#/c/708273/

[1] https://docs.openstack.org/zun/latest/install/index.html

Depends-On: https://review.opendev.org/#/c/721044/
Change-Id: I9c361a99b355af27907cf80f5c88d97191193495

ansible/group_vars/all.yml

@@ -115,6 +115,14 @@ docker_zun_options: -H tcp://{{ api_interface_address | put_address_in_context('
 docker_zun_config:
   cluster-store: etcd://{% for host in groups.get('etcd', []) %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['etcd_client_port'] }}{% if not loop.last %},{% endif %}{% endfor %}
 
+# Extra containerd options for Zun
+containerd_configure_for_zun: "no"
+
+# 42463 is the static group id of the zun user in the Zun image.
+# If users customize this value on building the Zun images,
+# they need to change this config accordingly.
+containerd_grpc_gid: 42463
+
 # Timeout after Docker sends SIGTERM before sending SIGKILL.
 docker_graceful_timeout: 60
 
@@ -474,6 +482,7 @@ zookeeper_quorum_port: "3888"
 
 zun_api_port: "9517"
 zun_wsproxy_port: "6784"
+zun_cni_daemon_port: "9036"
 
 vitrage_api_port: "8999"
 

ansible/inventory/all-in-one

@@ -683,6 +683,9 @@ zun
 [zun-compute:children]
 compute
 
+[zun-cni-daemon:children]
+compute
+
 # Skydive
 [skydive-analyzer:children]
 skydive

ansible/inventory/multinode

@@ -702,6 +702,9 @@ zun
 [zun-compute:children]
 compute
 
+[zun-cni-daemon:children]
+compute
+
 # Skydive
 [skydive-analyzer:children]
 skydive

ansible/roles/baremetal/tasks/bootstrap-servers.yml

@@ -4,3 +4,7 @@
 - include_tasks: install.yml
 
 - include_tasks: post-install.yml
+
+- include_tasks: configure-containerd-for-zun.yml
+  when: containerd_configure_for_zun|bool and
+        inventory_hostname in groups['zun-cni-daemon']

ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml

@@ -0,0 +1,50 @@
+---
+- name: Ensuring CNI config directory exist
+  file:
+    path: "{{ cni_config_dir }}"
+    state: "directory"
+    mode: "0770"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+  become: True
+
+- name: Copying CNI config file
+  template:
+    src: "10-zun-cni.conf.j2"
+    dest: "{{ cni_config_dir }}/10-zun-cni.conf"
+    mode: "0660"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+  become: True
+  register: cni_configured
+
+- name: Ensuring CNI bin directory exist
+  file:
+    path: "{{ cni_bin_dir }}"
+    state: "directory"
+    mode: "0770"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+  become: True
+
+- name: Copy zun-cni script
+  template:
+    src: "zun-cni.j2"
+    dest: "{{ cni_bin_dir }}/zun-cni"
+    mode: "0775"
+  become: True
+
+- name: Copying over containerd config
+  template:
+    src: "containerd_config.toml.j2"
+    dest: "/etc/containerd/config.toml"
+    mode: "0660"
+  become: true
+  register: containerd_configured
+
+- name: Restart containerd
+  service:
+    name: containerd
+    state: restarted
+  become: True
+  when: cni_configured.changed or containerd_configured.changed

ansible/roles/baremetal/templates/10-zun-cni.conf.j2

@@ -0,0 +1,5 @@
+{
+  "cniVersion": "0.3.1",
+  "name": "zun",
+  "type": "zun-cni"
+}

ansible/roles/baremetal/templates/containerd_config.toml.j2

@@ -0,0 +1,2 @@
+[grpc]
+  gid = {{ containerd_grpc_gid }}

ansible/roles/baremetal/templates/zun-cni.j2

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+env_list=""
+for line in $(env | grep "CNI_")
+do
+    key=$(echo "$line" | cut -d "=" -f 1)
+    value=$(echo "$line" | cut -d "=" -f 2-)
+    env_list="$env_list --env ${key}=\"${value}\""
+done
+
+cmd="docker exec -i $env_list zun_cni_daemon zun-cni <&0"
+eval "$cmd"

ansible/roles/baremetal/vars/main.yml

@@ -3,3 +3,6 @@ docker_config:
   log-opts:
     max-file: "{{ docker_log_max_file }}"
     max-size: "{{ docker_log_max_size }}"
+
+cni_config_dir: /etc/cni/net.d
+cni_bin_dir: /opt/cni/bin

ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2

@@ -25,7 +25,7 @@
     rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python
     rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python
     rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python
-    rewriterule25 programname ^(zun-api|zun-compute)$ openstack_python
+    rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python
     rewriterule26 programname ^(kuryr-server)$ openstack_python
     rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python
     rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python

ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2

@@ -128,7 +128,7 @@
   </rule>
   <rule>
     key     programname
-    pattern ^(zun-api|zun-compute)$
+    pattern ^(zun-api|zun-compute|zun-cni-daemon)$
     tag openstack_python
   </rule>
   <rule>

ansible/roles/zun/defaults/main.yml

@@ -46,6 +46,14 @@ zun_services:
     privileged: True
     volumes: "{{ zun_compute_default_volumes + zun_compute_extra_volumes }}"
     dimensions: "{{ zun_compute_dimensions }}"
+  zun-cni-daemon:
+    container_name: zun_cni_daemon
+    group: zun-cni-daemon
+    enabled: true
+    image: "{{ zun_cni_daemon_image_full }}"
+    privileged: True
+    volumes: "{{ zun_cni_daemon_default_volumes + zun_cni_daemon_extra_volumes }}"
+    dimensions: "{{ zun_cni_daemon_dimensions }}"
 
 ####################
 ## Database
@@ -73,9 +81,15 @@ zun_compute_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc
 zun_compute_tag: "{{ zun_tag }}"
 zun_compute_image_full: "{{ zun_compute_image }}:{{ zun_compute_tag }}"
 
+zun_cni_daemon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ zun_install_type }}-zun-cni-daemon"
+zun_cni_daemon_tag: "{{ zun_tag }}"
+zun_cni_daemon_image_full: "{{ zun_cni_daemon_image }}:{{ zun_cni_daemon_tag }}"
+
+
 zun_api_dimensions: "{{ default_container_dimensions }}"
 zun_wsproxy_dimensions: "{{ default_container_dimensions }}"
 zun_compute_dimensions: "{{ default_container_dimensions }}"
+zun_cni_daemon_dimensions: "{{ default_container_dimensions }}"
 
 zun_api_default_volumes:
   - "{{ node_config_directory }}/zun-api/:{{ container_config_directory }}/:ro"
@@ -101,11 +115,18 @@ zun_compute_default_volumes:
   - "/lib/modules:/lib/modules:ro"
   - "/dev:/dev"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
+zun_cni_daemon_default_volumes:
+  - "{{ node_config_directory }}/zun-cni-daemon/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/zun/zun:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/zun' if zun_dev_mode | bool else '' }}"
 
 zun_extra_volumes: "{{ default_extra_volumes }}"
 zun_api_extra_volumes: "{{ zun_extra_volumes }}"
 zun_wsproxy_extra_volumes: "{{ zun_extra_volumes }}"
 zun_compute_extra_volumes: "{{ zun_extra_volumes }}"
+zun_cni_daemon_extra_volumes: "{{ zun_extra_volumes }}"
 
 ####################
 ## OpenStack

ansible/roles/zun/handlers/main.yml

@@ -12,6 +12,8 @@
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
+  listen:
+    - zun-api container changed
   when:
     - kolla_action != "config"
 
@@ -28,6 +30,8 @@
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
+  listen:
+    - zun-wsproxy container changed
   when:
     - kolla_action != "config"
 
@@ -44,5 +48,37 @@
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
+  listen:
+    - zun-compute container changed
   when:
     - kolla_action != "config"
+
+- name: Restart zun-cni-daemon container
+  vars:
+    service_name: "zun-cni-daemon"
+    service: "{{ zun_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    privileged: "{{ service.privileged | default(False) }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+  listen:
+    - zun-cni-daemon container changed
+  when:
+    - kolla_action != "config"
+
+- name: Copy loopback binary from zun-cni-daemon container to host
+  vars:
+    service_name: "zun-cni-daemon"
+    service: "{{ zun_services[service_name] }}"
+  become: true
+  command: "docker cp {{ service.container_name }}:/opt/loopback /opt/cni/bin/"
+  # NOTE(yoctozepto): it would be cleaner to listen only on image change
+  # but there is no such mechanism (yet) and container change should be
+  # good enough (better than including config change triggers)
+  listen:
+    - zun-cni-daemon container changed

ansible/roles/zun/tasks/check-containers.yml

@@ -14,4 +14,7 @@
     - item.value.enabled | bool
   with_dict: "{{ zun_services }}"
   notify:
-    - "Restart {{ item.key }} container"
+    # NOTE(yoctozepto): Zun differs from other projects because we want
+    # to differentiate between config change and container property
+    # change
+    - "{{ item.key }} container changed"

ansible/roles/zun/tasks/deploy.yml

@@ -4,7 +4,8 @@
 
 - include_tasks: config.yml
   when: inventory_hostname in groups['zun-api'] or
-        inventory_hostname in groups['zun-compute']
+        inventory_hostname in groups['zun-compute'] or
+        inventory_hostname in groups['zun-cni-daemon']
 
 - include_tasks: clone.yml
   when: zun_dev_mode | bool

ansible/roles/zun/tasks/precheck.yml

@@ -11,6 +11,7 @@
     name:
       - zun_api
       - zun_wsproxy
+      - zun_cni_daemon
   register: container_facts
 
 - name: Checking free port for Zun API
@@ -35,6 +36,17 @@
     - container_facts['zun_wsproxy'] is not defined
     - inventory_hostname in groups['zun-wsproxy']
 
+- name: Checking free port for zun-cni-daemon
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ zun_cni_daemon_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['zun_cni_daemon'] is not defined
+    - inventory_hostname in groups['zun-cni-daemon']
+
 - name: Ensure kuryr enabled for zun
   fail:
     msg: "kuryr is required but not enabled"

ansible/roles/zun/templates/zun-cni-daemon.json.j2

@@ -0,0 +1,23 @@
+{
+    "command": "zun-cni-daemon --config-file /etc/zun/zun.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/zun.conf",
+            "dest": "/etc/zun/zun.conf",
+            "owner": "zun",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/zun",
+            "owner": "zun:kolla",
+            "recurse": true
+        },
+        {
+            "path": "/opt/cni/",
+            "owner": "zun:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/zun/templates/zun.conf.j2

@@ -11,6 +11,7 @@ transport_url = {{ rpc_transport_url }}
 
 state_path = /var/lib/zun
 container_driver = docker
+capsule_driver = cri
 
 [network]
 driver = kuryr
@@ -116,3 +117,6 @@ base_url = ws://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ zun
 api_url = tcp://{{ api_interface_address | put_address_in_context('url') }}:2375
 docker_remote_api_host = {{ api_interface_address }}
 docker_remote_api_port = 2375
+
+[cni_daemon]
+cni_daemon_port = {{ zun_cni_daemon_port }}

ansible/site.yml

@@ -1170,6 +1170,7 @@
     - zun-api
     - zun-wsproxy
     - zun-compute
+    - zun-cni-daemon
     - '&enable_zun_True'
   serial: '{{ kolla_serial|default("0") }}'
   roles:

doc/source/reference/compute/zun-guide.rst

@@ -21,6 +21,7 @@ following variables:
    enable_kuryr: "yes"
    enable_etcd: "yes"
    docker_configure_for_zun: "yes"
+   containerd_configure_for_zun: "yes"
 
 Docker reconfiguration requires reboostrapping before deploy.
 Make sure you understand the consequences of restarting Docker.

etc/kolla/globals.yml

@@ -69,6 +69,8 @@
 #docker_client_timeout: 120
 
 #docker_configure_for_zun: "no"
+#containerd_configure_for_zun: "no"
+#containerd_grpc_gid: 42463
 
 ###################
 # Messaging options

releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add a new container ``zun-cni-daemon`` for Zun service. This container
+    is a daemon service for implementing the CNI plugin for Zun.

tests/get_logs.sh

@@ -17,6 +17,7 @@ copy_logs() {
     if [[ -x "$(command -v journalctl)" ]]; then
         journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
         journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log
+        journalctl --no-pager -u containerd.service > ${LOG_DIR}/system_logs/containerd.log
     else
         cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log
     fi

tests/templates/globals-default.j2

@@ -65,6 +65,7 @@ enable_zun: "yes"
 enable_kuryr: "yes"
 enable_etcd: "yes"
 docker_configure_for_zun: "yes"
+containerd_configure_for_zun: "yes"
 enable_cinder: "yes"
 # lvm backup driver for cinder-backup does not exist
 enable_cinder_backup: "no"

tests/templates/inventory.j2

@@ -745,6 +745,9 @@ zun
 [zun-compute:children]
 compute
 
+[zun-cni-daemon:children]
+compute
+
 # Skydive
 [skydive-analyzer:children]
 skydive

tests/test-zun.sh

@@ -98,6 +98,48 @@ function test_zun_logged {
     done
     openstack volume delete zun_test_volume
     echo "SUCCESS: Zun Cinder volume attachment"
+
+    echo "TESTING: Zun capsule"
+    cat >/tmp/capsule.yaml <<EOF
+capsuleVersion: beta
+kind: capsule
+metadata:
+  name: capsule-test
+spec:
+  containers:
+  - image: alpine
+    command:
+    - sleep
+    - "1000"
+EOF
+    zun capsule-create -f /tmp/capsule.yaml
+    attempt=1
+    while [[ $(zun capsule-describe capsule-test | awk '/ status /{print $4}') != "Running" ]]; do
+        echo "Capsule not running yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 10 ]]; then
+            echo "Capsule failed to start"
+            zun capsule-describe capsule-test
+            return 1
+        fi
+        sleep 10
+    done
+    zun capsule-list
+    zun capsule-describe capsule-test
+    zun capsule-delete capsule-test
+
+    attempt=1
+    while zun capsule-describe capsule-test; do
+        echo "Capsule not deleted yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 10 ]]; then
+            echo "Zun failed to delete the capsule"
+            zun capsule-describe capsule-test
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Zun capsule"
 }
 
 function test_zun {