diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index e137b4ae03..aad8449000 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -115,6 +115,14 @@ docker_zun_options: -H tcp://{{ api_interface_address | put_address_in_context(' docker_zun_config: cluster-store: etcd://{% for host in groups.get('etcd', []) %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['etcd_client_port'] }}{% if not loop.last %},{% endif %}{% endfor %} +# Extra containerd options for Zun +containerd_configure_for_zun: "no" + +# 42463 is the static group id of the zun user in the Zun image. +# If users customize this value on building the Zun images, +# they need to change this config accordingly. +containerd_grpc_gid: 42463 + # Timeout after Docker sends SIGTERM before sending SIGKILL. docker_graceful_timeout: 60 @@ -474,6 +482,7 @@ zookeeper_quorum_port: "3888" zun_api_port: "9517" zun_wsproxy_port: "6784" +zun_cni_daemon_port: "9036" vitrage_api_port: "8999" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 8c539e21b0..58b3a41fd6 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -683,6 +683,9 @@ zun [zun-compute:children] compute +[zun-cni-daemon:children] +compute + # Skydive [skydive-analyzer:children] skydive diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 7afbb5775e..dc35e07185 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -702,6 +702,9 @@ zun [zun-compute:children] compute +[zun-cni-daemon:children] +compute + # Skydive [skydive-analyzer:children] skydive diff --git a/ansible/roles/baremetal/tasks/bootstrap-servers.yml b/ansible/roles/baremetal/tasks/bootstrap-servers.yml index efe2e3cdb5..820bdd3bbb 100644 --- a/ansible/roles/baremetal/tasks/bootstrap-servers.yml +++ b/ansible/roles/baremetal/tasks/bootstrap-servers.yml @@ -4,3 +4,7 @@ - include_tasks: install.yml - include_tasks: post-install.yml + +- include_tasks: configure-containerd-for-zun.yml + when: containerd_configure_for_zun|bool and + inventory_hostname in groups['zun-cni-daemon'] diff --git a/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml b/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml new file mode 100644 index 0000000000..b8a29d671f --- /dev/null +++ b/ansible/roles/baremetal/tasks/configure-containerd-for-zun.yml @@ -0,0 +1,50 @@ +--- +- name: Ensuring CNI config directory exist + file: + path: "{{ cni_config_dir }}" + state: "directory" + mode: "0770" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + become: True + +- name: Copying CNI config file + template: + src: "10-zun-cni.conf.j2" + dest: "{{ cni_config_dir }}/10-zun-cni.conf" + mode: "0660" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + become: True + register: cni_configured + +- name: Ensuring CNI bin directory exist + file: + path: "{{ cni_bin_dir }}" + state: "directory" + mode: "0770" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + become: True + +- name: Copy zun-cni script + template: + src: "zun-cni.j2" + dest: "{{ cni_bin_dir }}/zun-cni" + mode: "0775" + become: True + +- name: Copying over containerd config + template: + src: "containerd_config.toml.j2" + dest: "/etc/containerd/config.toml" + mode: "0660" + become: true + register: containerd_configured + +- name: Restart containerd + service: + name: containerd + state: restarted + become: True + when: cni_configured.changed or containerd_configured.changed diff --git a/ansible/roles/baremetal/templates/10-zun-cni.conf.j2 b/ansible/roles/baremetal/templates/10-zun-cni.conf.j2 new file mode 100644 index 0000000000..99a83987d3 --- /dev/null +++ b/ansible/roles/baremetal/templates/10-zun-cni.conf.j2 @@ -0,0 +1,5 @@ +{ + "cniVersion": "0.3.1", + "name": "zun", + "type": "zun-cni" +} diff --git a/ansible/roles/baremetal/templates/containerd_config.toml.j2 b/ansible/roles/baremetal/templates/containerd_config.toml.j2 new file mode 100644 index 0000000000..1fa5dd80fd --- /dev/null +++ b/ansible/roles/baremetal/templates/containerd_config.toml.j2 @@ -0,0 +1,2 @@ +[grpc] + gid = {{ containerd_grpc_gid }} diff --git a/ansible/roles/baremetal/templates/zun-cni.j2 b/ansible/roles/baremetal/templates/zun-cni.j2 new file mode 100644 index 0000000000..cd86cb2178 --- /dev/null +++ b/ansible/roles/baremetal/templates/zun-cni.j2 @@ -0,0 +1,12 @@ +#!/bin/bash + +env_list="" +for line in $(env | grep "CNI_") +do + key=$(echo "$line" | cut -d "=" -f 1) + value=$(echo "$line" | cut -d "=" -f 2-) + env_list="$env_list --env ${key}=\"${value}\"" +done + +cmd="docker exec -i $env_list zun_cni_daemon zun-cni <&0" +eval "$cmd" diff --git a/ansible/roles/baremetal/vars/main.yml b/ansible/roles/baremetal/vars/main.yml index 5752302538..cafc448bf5 100644 --- a/ansible/roles/baremetal/vars/main.yml +++ b/ansible/roles/baremetal/vars/main.yml @@ -3,3 +3,6 @@ docker_config: log-opts: max-file: "{{ docker_log_max_file }}" max-size: "{{ docker_log_max_size }}" + +cni_config_dir: /etc/cni/net.d +cni_bin_dir: /opt/cni/bin diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 index 8ae8e16ab6..d6b62b8257 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 @@ -25,7 +25,7 @@ rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python - rewriterule25 programname ^(zun-api|zun-compute)$ openstack_python + rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python rewriterule26 programname ^(kuryr-server)$ openstack_python rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 index 2354b6a6a8..81a8979cdb 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 @@ -128,7 +128,7 @@ key programname - pattern ^(zun-api|zun-compute)$ + pattern ^(zun-api|zun-compute|zun-cni-daemon)$ tag openstack_python diff --git a/ansible/roles/zun/defaults/main.yml b/ansible/roles/zun/defaults/main.yml index 2aa8856313..aa3f6d8183 100644 --- a/ansible/roles/zun/defaults/main.yml +++ b/ansible/roles/zun/defaults/main.yml @@ -46,6 +46,14 @@ zun_services: privileged: True volumes: "{{ zun_compute_default_volumes + zun_compute_extra_volumes }}" dimensions: "{{ zun_compute_dimensions }}" + zun-cni-daemon: + container_name: zun_cni_daemon + group: zun-cni-daemon + enabled: true + image: "{{ zun_cni_daemon_image_full }}" + privileged: True + volumes: "{{ zun_cni_daemon_default_volumes + zun_cni_daemon_extra_volumes }}" + dimensions: "{{ zun_cni_daemon_dimensions }}" #################### ## Database @@ -73,9 +81,15 @@ zun_compute_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc zun_compute_tag: "{{ zun_tag }}" zun_compute_image_full: "{{ zun_compute_image }}:{{ zun_compute_tag }}" +zun_cni_daemon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ zun_install_type }}-zun-cni-daemon" +zun_cni_daemon_tag: "{{ zun_tag }}" +zun_cni_daemon_image_full: "{{ zun_cni_daemon_image }}:{{ zun_cni_daemon_tag }}" + + zun_api_dimensions: "{{ default_container_dimensions }}" zun_wsproxy_dimensions: "{{ default_container_dimensions }}" zun_compute_dimensions: "{{ default_container_dimensions }}" +zun_cni_daemon_dimensions: "{{ default_container_dimensions }}" zun_api_default_volumes: - "{{ node_config_directory }}/zun-api/:{{ container_config_directory }}/:ro" @@ -101,11 +115,18 @@ zun_compute_default_volumes: - "/lib/modules:/lib/modules:ro" - "/dev:/dev" - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}" +zun_cni_daemon_default_volumes: + - "{{ node_config_directory }}/zun-cni-daemon/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" + - "kolla_logs:/var/log/kolla/" + - "{{ kolla_dev_repos_directory ~ '/zun/zun:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/zun' if zun_dev_mode | bool else '' }}" zun_extra_volumes: "{{ default_extra_volumes }}" zun_api_extra_volumes: "{{ zun_extra_volumes }}" zun_wsproxy_extra_volumes: "{{ zun_extra_volumes }}" zun_compute_extra_volumes: "{{ zun_extra_volumes }}" +zun_cni_daemon_extra_volumes: "{{ zun_extra_volumes }}" #################### ## OpenStack diff --git a/ansible/roles/zun/handlers/main.yml b/ansible/roles/zun/handlers/main.yml index cbf03b5cfc..39a3ef0261 100644 --- a/ansible/roles/zun/handlers/main.yml +++ b/ansible/roles/zun/handlers/main.yml @@ -12,6 +12,8 @@ privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" dimensions: "{{ service.dimensions }}" + listen: + - zun-api container changed when: - kolla_action != "config" @@ -28,6 +30,8 @@ privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" dimensions: "{{ service.dimensions }}" + listen: + - zun-wsproxy container changed when: - kolla_action != "config" @@ -44,5 +48,37 @@ privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" dimensions: "{{ service.dimensions }}" + listen: + - zun-compute container changed when: - kolla_action != "config" + +- name: Restart zun-cni-daemon container + vars: + service_name: "zun-cni-daemon" + service: "{{ zun_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + privileged: "{{ service.privileged | default(False) }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + listen: + - zun-cni-daemon container changed + when: + - kolla_action != "config" + +- name: Copy loopback binary from zun-cni-daemon container to host + vars: + service_name: "zun-cni-daemon" + service: "{{ zun_services[service_name] }}" + become: true + command: "docker cp {{ service.container_name }}:/opt/loopback /opt/cni/bin/" + # NOTE(yoctozepto): it would be cleaner to listen only on image change + # but there is no such mechanism (yet) and container change should be + # good enough (better than including config change triggers) + listen: + - zun-cni-daemon container changed diff --git a/ansible/roles/zun/tasks/check-containers.yml b/ansible/roles/zun/tasks/check-containers.yml index dee55acbb5..5c28c2fc48 100644 --- a/ansible/roles/zun/tasks/check-containers.yml +++ b/ansible/roles/zun/tasks/check-containers.yml @@ -14,4 +14,7 @@ - item.value.enabled | bool with_dict: "{{ zun_services }}" notify: - - "Restart {{ item.key }} container" + # NOTE(yoctozepto): Zun differs from other projects because we want + # to differentiate between config change and container property + # change + - "{{ item.key }} container changed" diff --git a/ansible/roles/zun/tasks/deploy.yml b/ansible/roles/zun/tasks/deploy.yml index c2966d9cca..e55ac2ddc4 100644 --- a/ansible/roles/zun/tasks/deploy.yml +++ b/ansible/roles/zun/tasks/deploy.yml @@ -4,7 +4,8 @@ - include_tasks: config.yml when: inventory_hostname in groups['zun-api'] or - inventory_hostname in groups['zun-compute'] + inventory_hostname in groups['zun-compute'] or + inventory_hostname in groups['zun-cni-daemon'] - include_tasks: clone.yml when: zun_dev_mode | bool diff --git a/ansible/roles/zun/tasks/precheck.yml b/ansible/roles/zun/tasks/precheck.yml index a23dc95886..38585515b4 100644 --- a/ansible/roles/zun/tasks/precheck.yml +++ b/ansible/roles/zun/tasks/precheck.yml @@ -11,6 +11,7 @@ name: - zun_api - zun_wsproxy + - zun_cni_daemon register: container_facts - name: Checking free port for Zun API @@ -35,6 +36,17 @@ - container_facts['zun_wsproxy'] is not defined - inventory_hostname in groups['zun-wsproxy'] +- name: Checking free port for zun-cni-daemon + wait_for: + host: "{{ api_interface_address }}" + port: "{{ zun_cni_daemon_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['zun_cni_daemon'] is not defined + - inventory_hostname in groups['zun-cni-daemon'] + - name: Ensure kuryr enabled for zun fail: msg: "kuryr is required but not enabled" diff --git a/ansible/roles/zun/templates/zun-cni-daemon.json.j2 b/ansible/roles/zun/templates/zun-cni-daemon.json.j2 new file mode 100644 index 0000000000..504fc48f90 --- /dev/null +++ b/ansible/roles/zun/templates/zun-cni-daemon.json.j2 @@ -0,0 +1,23 @@ +{ + "command": "zun-cni-daemon --config-file /etc/zun/zun.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/zun.conf", + "dest": "/etc/zun/zun.conf", + "owner": "zun", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/zun", + "owner": "zun:kolla", + "recurse": true + }, + { + "path": "/opt/cni/", + "owner": "zun:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2 index e6cc023a38..89e82ccd73 100644 --- a/ansible/roles/zun/templates/zun.conf.j2 +++ b/ansible/roles/zun/templates/zun.conf.j2 @@ -11,6 +11,7 @@ transport_url = {{ rpc_transport_url }} state_path = /var/lib/zun container_driver = docker +capsule_driver = cri [network] driver = kuryr @@ -116,3 +117,6 @@ base_url = ws://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ zun api_url = tcp://{{ api_interface_address | put_address_in_context('url') }}:2375 docker_remote_api_host = {{ api_interface_address }} docker_remote_api_port = 2375 + +[cni_daemon] +cni_daemon_port = {{ zun_cni_daemon_port }} diff --git a/ansible/site.yml b/ansible/site.yml index 1e425f7a47..6d3fb8584d 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -1170,6 +1170,7 @@ - zun-api - zun-wsproxy - zun-compute + - zun-cni-daemon - '&enable_zun_True' serial: '{{ kolla_serial|default("0") }}' roles: diff --git a/doc/source/reference/compute/zun-guide.rst b/doc/source/reference/compute/zun-guide.rst index 17e9b34c9e..5f263bbc38 100644 --- a/doc/source/reference/compute/zun-guide.rst +++ b/doc/source/reference/compute/zun-guide.rst @@ -21,6 +21,7 @@ following variables: enable_kuryr: "yes" enable_etcd: "yes" docker_configure_for_zun: "yes" + containerd_configure_for_zun: "yes" Docker reconfiguration requires reboostrapping before deploy. Make sure you understand the consequences of restarting Docker. diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index f6b0901a1d..e79c03cce4 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -69,6 +69,8 @@ #docker_client_timeout: 120 #docker_configure_for_zun: "no" +#containerd_configure_for_zun: "no" +#containerd_grpc_gid: 42463 ################### # Messaging options diff --git a/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml b/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml new file mode 100644 index 0000000000..98e0c52091 --- /dev/null +++ b/releasenotes/notes/add-zun-cni-daemon-container-453cdb31052af6b7.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add a new container ``zun-cni-daemon`` for Zun service. This container + is a daemon service for implementing the CNI plugin for Zun. diff --git a/tests/get_logs.sh b/tests/get_logs.sh index 9b619dc1a7..6ba8605921 100644 --- a/tests/get_logs.sh +++ b/tests/get_logs.sh @@ -17,6 +17,7 @@ copy_logs() { if [[ -x "$(command -v journalctl)" ]]; then journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log + journalctl --no-pager -u containerd.service > ${LOG_DIR}/system_logs/containerd.log else cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log fi diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 2dc8b86e6e..e44b46809d 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -65,6 +65,7 @@ enable_zun: "yes" enable_kuryr: "yes" enable_etcd: "yes" docker_configure_for_zun: "yes" +containerd_configure_for_zun: "yes" enable_cinder: "yes" # lvm backup driver for cinder-backup does not exist enable_cinder_backup: "no" diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 2aca86b7a3..229fc7bc35 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -745,6 +745,9 @@ zun [zun-compute:children] compute +[zun-cni-daemon:children] +compute + # Skydive [skydive-analyzer:children] skydive diff --git a/tests/test-zun.sh b/tests/test-zun.sh index 155f667a49..909acb6afd 100755 --- a/tests/test-zun.sh +++ b/tests/test-zun.sh @@ -98,6 +98,48 @@ function test_zun_logged { done openstack volume delete zun_test_volume echo "SUCCESS: Zun Cinder volume attachment" + + echo "TESTING: Zun capsule" + cat >/tmp/capsule.yaml <