diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index e137b4ae03..aad8449000 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -115,6 +115,14 @@ docker_zun_options: -H tcp://{{ api_interface_address | put_address_in_context('
docker_zun_config:
cluster-store: etcd://{% for host in groups.get('etcd', []) %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['etcd_client_port'] }}{% if not loop.last %},{% endif %}{% endfor %}
+# Extra containerd options for Zun
+containerd_configure_for_zun: "no"
+
+# 42463 is the static group id of the zun user in the Zun image.
+# If users customize this value on building the Zun images,
+# they need to change this config accordingly.
+containerd_grpc_gid: 42463
+
# Timeout after Docker sends SIGTERM before sending SIGKILL.
docker_graceful_timeout: 60
@@ -474,6 +482,7 @@ zookeeper_quorum_port: "3888"
zun_api_port: "9517"
zun_wsproxy_port: "6784"
+zun_cni_daemon_port: "9036"
vitrage_api_port: "8999"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 8c539e21b0..58b3a41fd6 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -683,6 +683,9 @@ zun
[zun-compute:children]
compute
+[zun-cni-daemon:children]
+compute
+
# Skydive
[skydive-analyzer:children]
skydive
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 7afbb5775e..dc35e07185 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -702,6 +702,9 @@ zun
[zun-compute:children]
compute
+[zun-cni-daemon:children]
+compute
+
# Skydive
[skydive-analyzer:children]
skydive
diff --git a/ansible/roles/baremetal/tasks/bootstrap-servers.yml b/ansible/roles/baremetal/tasks/bootstrap-servers.yml
index efe2e3cdb5..820bdd3bbb 100644
--- a/ansible/roles/baremetal/tasks/bootstrap-servers.yml
+++ b/ansible/roles/baremetal/tasks/bootstrap-servers.yml
@@ -4,3 +4,7 @@
- include_tasks: install.yml
- include_tasks: post-install.yml
+
+- include_tasks: configure-containerd-for-zun.yml
+ when: containerd_configure_for_zun|bool and
+ inventory_hostname in groups['zun-cni-daemon']
diff --git a/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml b/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml
new file mode 100644
index 0000000000..b8a29d671f
--- /dev/null
+++ b/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml
@@ -0,0 +1,50 @@
+---
+- name: Ensuring CNI config directory exist
+ file:
+ path: "{{ cni_config_dir }}"
+ state: "directory"
+ mode: "0770"
+ owner: "{{ config_owner_user }}"
+ group: "{{ config_owner_group }}"
+ become: True
+
+- name: Copying CNI config file
+ template:
+ src: "10-zun-cni.conf.j2"
+ dest: "{{ cni_config_dir }}/10-zun-cni.conf"
+ mode: "0660"
+ owner: "{{ config_owner_user }}"
+ group: "{{ config_owner_group }}"
+ become: True
+ register: cni_configured
+
+- name: Ensuring CNI bin directory exist
+ file:
+ path: "{{ cni_bin_dir }}"
+ state: "directory"
+ mode: "0770"
+ owner: "{{ config_owner_user }}"
+ group: "{{ config_owner_group }}"
+ become: True
+
+- name: Copy zun-cni script
+ template:
+ src: "zun-cni.j2"
+ dest: "{{ cni_bin_dir }}/zun-cni"
+ mode: "0775"
+ become: True
+
+- name: Copying over containerd config
+ template:
+ src: "containerd_config.toml.j2"
+ dest: "/etc/containerd/config.toml"
+ mode: "0660"
+ become: true
+ register: containerd_configured
+
+- name: Restart containerd
+ service:
+ name: containerd
+ state: restarted
+ become: True
+ when: cni_configured.changed or containerd_configured.changed
diff --git a/ansible/roles/baremetal/templates/10-zun-cni.conf.j2 b/ansible/roles/baremetal/templates/10-zun-cni.conf.j2
new file mode 100644
index 0000000000..99a83987d3
--- /dev/null
+++ b/ansible/roles/baremetal/templates/10-zun-cni.conf.j2
@@ -0,0 +1,5 @@
+{
+ "cniVersion": "0.3.1",
+ "name": "zun",
+ "type": "zun-cni"
+}
diff --git a/ansible/roles/baremetal/templates/containerd_config.toml.j2 b/ansible/roles/baremetal/templates/containerd_config.toml.j2
new file mode 100644
index 0000000000..1fa5dd80fd
--- /dev/null
+++ b/ansible/roles/baremetal/templates/containerd_config.toml.j2
@@ -0,0 +1,2 @@
+[grpc]
+ gid = {{ containerd_grpc_gid }}
diff --git a/ansible/roles/baremetal/templates/zun-cni.j2 b/ansible/roles/baremetal/templates/zun-cni.j2
new file mode 100644
index 0000000000..cd86cb2178
--- /dev/null
+++ b/ansible/roles/baremetal/templates/zun-cni.j2
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+env_list=""
+for line in $(env | grep "CNI_")
+do
+ key=$(echo "$line" | cut -d "=" -f 1)
+ value=$(echo "$line" | cut -d "=" -f 2-)
+ env_list="$env_list --env ${key}=\"${value}\""
+done
+
+cmd="docker exec -i $env_list zun_cni_daemon zun-cni <&0"
+eval "$cmd"
diff --git a/ansible/roles/baremetal/vars/main.yml b/ansible/roles/baremetal/vars/main.yml
index 5752302538..cafc448bf5 100644
--- a/ansible/roles/baremetal/vars/main.yml
+++ b/ansible/roles/baremetal/vars/main.yml
@@ -3,3 +3,6 @@ docker_config:
log-opts:
max-file: "{{ docker_log_max_file }}"
max-size: "{{ docker_log_max_size }}"
+
+cni_config_dir: /etc/cni/net.d
+cni_bin_dir: /opt/cni/bin
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
index 8ae8e16ab6..d6b62b8257 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
@@ -25,7 +25,7 @@
rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python
rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python
rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python
- rewriterule25 programname ^(zun-api|zun-compute)$ openstack_python
+ rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python
rewriterule26 programname ^(kuryr-server)$ openstack_python
rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python
rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
index 2354b6a6a8..81a8979cdb 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
@@ -128,7 +128,7 @@
key programname
- pattern ^(zun-api|zun-compute)$
+ pattern ^(zun-api|zun-compute|zun-cni-daemon)$
tag openstack_python
diff --git a/ansible/roles/zun/defaults/main.yml b/ansible/roles/zun/defaults/main.yml
index 2aa8856313..aa3f6d8183 100644
--- a/ansible/roles/zun/defaults/main.yml
+++ b/ansible/roles/zun/defaults/main.yml
@@ -46,6 +46,14 @@ zun_services:
privileged: True
volumes: "{{ zun_compute_default_volumes + zun_compute_extra_volumes }}"
dimensions: "{{ zun_compute_dimensions }}"
+ zun-cni-daemon:
+ container_name: zun_cni_daemon
+ group: zun-cni-daemon
+ enabled: true
+ image: "{{ zun_cni_daemon_image_full }}"
+ privileged: True
+ volumes: "{{ zun_cni_daemon_default_volumes + zun_cni_daemon_extra_volumes }}"
+ dimensions: "{{ zun_cni_daemon_dimensions }}"
####################
## Database
@@ -73,9 +81,15 @@ zun_compute_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc
zun_compute_tag: "{{ zun_tag }}"
zun_compute_image_full: "{{ zun_compute_image }}:{{ zun_compute_tag }}"
+zun_cni_daemon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ zun_install_type }}-zun-cni-daemon"
+zun_cni_daemon_tag: "{{ zun_tag }}"
+zun_cni_daemon_image_full: "{{ zun_cni_daemon_image }}:{{ zun_cni_daemon_tag }}"
+
+
zun_api_dimensions: "{{ default_container_dimensions }}"
zun_wsproxy_dimensions: "{{ default_container_dimensions }}"
zun_compute_dimensions: "{{ default_container_dimensions }}"
+zun_cni_daemon_dimensions: "{{ default_container_dimensions }}"
zun_api_default_volumes:
- "{{ node_config_directory }}/zun-api/:{{ container_config_directory }}/:ro"
@@ -101,11 +115,18 @@ zun_compute_default_volumes:
- "/lib/modules:/lib/modules:ro"
- "/dev:/dev"
- "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
+zun_cni_daemon_default_volumes:
+ - "{{ node_config_directory }}/zun-cni-daemon/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+ - "kolla_logs:/var/log/kolla/"
+ - "{{ kolla_dev_repos_directory ~ '/zun/zun:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/zun' if zun_dev_mode | bool else '' }}"
zun_extra_volumes: "{{ default_extra_volumes }}"
zun_api_extra_volumes: "{{ zun_extra_volumes }}"
zun_wsproxy_extra_volumes: "{{ zun_extra_volumes }}"
zun_compute_extra_volumes: "{{ zun_extra_volumes }}"
+zun_cni_daemon_extra_volumes: "{{ zun_extra_volumes }}"
####################
## OpenStack
diff --git a/ansible/roles/zun/handlers/main.yml b/ansible/roles/zun/handlers/main.yml
index cbf03b5cfc..39a3ef0261 100644
--- a/ansible/roles/zun/handlers/main.yml
+++ b/ansible/roles/zun/handlers/main.yml
@@ -12,6 +12,8 @@
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
+ listen:
+ - zun-api container changed
when:
- kolla_action != "config"
@@ -28,6 +30,8 @@
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
+ listen:
+ - zun-wsproxy container changed
when:
- kolla_action != "config"
@@ -44,5 +48,37 @@
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
+ listen:
+ - zun-compute container changed
when:
- kolla_action != "config"
+
+- name: Restart zun-cni-daemon container
+ vars:
+ service_name: "zun-cni-daemon"
+ service: "{{ zun_services[service_name] }}"
+ become: true
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ privileged: "{{ service.privileged | default(False) }}"
+ volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+ dimensions: "{{ service.dimensions }}"
+ listen:
+ - zun-cni-daemon container changed
+ when:
+ - kolla_action != "config"
+
+- name: Copy loopback binary from zun-cni-daemon container to host
+ vars:
+ service_name: "zun-cni-daemon"
+ service: "{{ zun_services[service_name] }}"
+ become: true
+ command: "docker cp {{ service.container_name }}:/opt/loopback /opt/cni/bin/"
+ # NOTE(yoctozepto): it would be cleaner to listen only on image change
+ # but there is no such mechanism (yet) and container change should be
+ # good enough (better than including config change triggers)
+ listen:
+ - zun-cni-daemon container changed
diff --git a/ansible/roles/zun/tasks/check-containers.yml b/ansible/roles/zun/tasks/check-containers.yml
index dee55acbb5..5c28c2fc48 100644
--- a/ansible/roles/zun/tasks/check-containers.yml
+++ b/ansible/roles/zun/tasks/check-containers.yml
@@ -14,4 +14,7 @@
- item.value.enabled | bool
with_dict: "{{ zun_services }}"
notify:
- - "Restart {{ item.key }} container"
+ # NOTE(yoctozepto): Zun differs from other projects because we want
+ # to differentiate between config change and container property
+ # change
+ - "{{ item.key }} container changed"
diff --git a/ansible/roles/zun/tasks/deploy.yml b/ansible/roles/zun/tasks/deploy.yml
index c2966d9cca..e55ac2ddc4 100644
--- a/ansible/roles/zun/tasks/deploy.yml
+++ b/ansible/roles/zun/tasks/deploy.yml
@@ -4,7 +4,8 @@
- include_tasks: config.yml
when: inventory_hostname in groups['zun-api'] or
- inventory_hostname in groups['zun-compute']
+ inventory_hostname in groups['zun-compute'] or
+ inventory_hostname in groups['zun-cni-daemon']
- include_tasks: clone.yml
when: zun_dev_mode | bool
diff --git a/ansible/roles/zun/tasks/precheck.yml b/ansible/roles/zun/tasks/precheck.yml
index a23dc95886..38585515b4 100644
--- a/ansible/roles/zun/tasks/precheck.yml
+++ b/ansible/roles/zun/tasks/precheck.yml
@@ -11,6 +11,7 @@
name:
- zun_api
- zun_wsproxy
+ - zun_cni_daemon
register: container_facts
- name: Checking free port for Zun API
@@ -35,6 +36,17 @@
- container_facts['zun_wsproxy'] is not defined
- inventory_hostname in groups['zun-wsproxy']
+- name: Checking free port for zun-cni-daemon
+ wait_for:
+ host: "{{ api_interface_address }}"
+ port: "{{ zun_cni_daemon_port }}"
+ connect_timeout: 1
+ timeout: 1
+ state: stopped
+ when:
+ - container_facts['zun_cni_daemon'] is not defined
+ - inventory_hostname in groups['zun-cni-daemon']
+
- name: Ensure kuryr enabled for zun
fail:
msg: "kuryr is required but not enabled"
diff --git a/ansible/roles/zun/templates/zun-cni-daemon.json.j2 b/ansible/roles/zun/templates/zun-cni-daemon.json.j2
new file mode 100644
index 0000000000..504fc48f90
--- /dev/null
+++ b/ansible/roles/zun/templates/zun-cni-daemon.json.j2
@@ -0,0 +1,23 @@
+{
+ "command": "zun-cni-daemon --config-file /etc/zun/zun.conf",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/zun.conf",
+ "dest": "/etc/zun/zun.conf",
+ "owner": "zun",
+ "perm": "0600"
+ }
+ ],
+ "permissions": [
+ {
+ "path": "/var/log/kolla/zun",
+ "owner": "zun:kolla",
+ "recurse": true
+ },
+ {
+ "path": "/opt/cni/",
+ "owner": "zun:kolla",
+ "recurse": true
+ }
+ ]
+}
diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2
index e6cc023a38..89e82ccd73 100644
--- a/ansible/roles/zun/templates/zun.conf.j2
+++ b/ansible/roles/zun/templates/zun.conf.j2
@@ -11,6 +11,7 @@ transport_url = {{ rpc_transport_url }}
state_path = /var/lib/zun
container_driver = docker
+capsule_driver = cri
[network]
driver = kuryr
@@ -116,3 +117,6 @@ base_url = ws://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ zun
api_url = tcp://{{ api_interface_address | put_address_in_context('url') }}:2375
docker_remote_api_host = {{ api_interface_address }}
docker_remote_api_port = 2375
+
+[cni_daemon]
+cni_daemon_port = {{ zun_cni_daemon_port }}
diff --git a/ansible/site.yml b/ansible/site.yml
index 1e425f7a47..6d3fb8584d 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -1170,6 +1170,7 @@
- zun-api
- zun-wsproxy
- zun-compute
+ - zun-cni-daemon
- '&enable_zun_True'
serial: '{{ kolla_serial|default("0") }}'
roles:
diff --git a/doc/source/reference/compute/zun-guide.rst b/doc/source/reference/compute/zun-guide.rst
index 17e9b34c9e..5f263bbc38 100644
--- a/doc/source/reference/compute/zun-guide.rst
+++ b/doc/source/reference/compute/zun-guide.rst
@@ -21,6 +21,7 @@ following variables:
enable_kuryr: "yes"
enable_etcd: "yes"
docker_configure_for_zun: "yes"
+ containerd_configure_for_zun: "yes"
Docker reconfiguration requires reboostrapping before deploy.
Make sure you understand the consequences of restarting Docker.
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index f6b0901a1d..e79c03cce4 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -69,6 +69,8 @@
#docker_client_timeout: 120
#docker_configure_for_zun: "no"
+#containerd_configure_for_zun: "no"
+#containerd_grpc_gid: 42463
###################
# Messaging options
diff --git a/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml b/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml
new file mode 100644
index 0000000000..98e0c52091
--- /dev/null
+++ b/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml
@@ -0,0 +1,5 @@
+---
+features:
+ - |
+ Add a new container ``zun-cni-daemon`` for Zun service. This container
+ is a daemon service for implementing the CNI plugin for Zun.
diff --git a/tests/get_logs.sh b/tests/get_logs.sh
index 9b619dc1a7..6ba8605921 100644
--- a/tests/get_logs.sh
+++ b/tests/get_logs.sh
@@ -17,6 +17,7 @@ copy_logs() {
if [[ -x "$(command -v journalctl)" ]]; then
journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log
+ journalctl --no-pager -u containerd.service > ${LOG_DIR}/system_logs/containerd.log
else
cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log
fi
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index 2dc8b86e6e..e44b46809d 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -65,6 +65,7 @@ enable_zun: "yes"
enable_kuryr: "yes"
enable_etcd: "yes"
docker_configure_for_zun: "yes"
+containerd_configure_for_zun: "yes"
enable_cinder: "yes"
# lvm backup driver for cinder-backup does not exist
enable_cinder_backup: "no"
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 2aca86b7a3..229fc7bc35 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -745,6 +745,9 @@ zun
[zun-compute:children]
compute
+[zun-cni-daemon:children]
+compute
+
# Skydive
[skydive-analyzer:children]
skydive
diff --git a/tests/test-zun.sh b/tests/test-zun.sh
index 155f667a49..909acb6afd 100755
--- a/tests/test-zun.sh
+++ b/tests/test-zun.sh
@@ -98,6 +98,48 @@ function test_zun_logged {
done
openstack volume delete zun_test_volume
echo "SUCCESS: Zun Cinder volume attachment"
+
+ echo "TESTING: Zun capsule"
+ cat >/tmp/capsule.yaml <