Ironic: Support both plain PXE and iPXE

Depends-On: https://review.opendev.org/c/openstack/kolla/+/832163
Change-Id: Ia2dba1854e925041ae23c731273b810bb2d5ec30

ansible/roles/ironic/defaults/main.yml

@@ -59,6 +59,9 @@ ironic_services:
     group: ironic-pxe
     enabled: true
     image: "{{ ironic_pxe_image_full }}"
+    environment:
+      TFTPBOOT_PATH: /var/lib/ironic/tftpboot
+      HTTPBOOT_PATH: /var/lib/ironic/httpboot
     volumes: "{{ ironic_pxe_default_volumes + ironic_pxe_extra_volumes }}"
     dimensions: "{{ ironic_pxe_dimensions }}"
   ironic-ipxe:
@@ -195,21 +198,18 @@ ironic_conductor_default_volumes:
   - "/run:/run:shared"
   - "kolla_logs:/var/log/kolla"
   - "ironic:/var/lib/ironic"
-  - "ironic_pxe:/tftpboot/"
-  - "ironic_ipxe:/httpboot/"
   - "{{ kolla_dev_repos_directory ~ '/ironic/ironic:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ironic' if ironic_dev_mode | bool else '' }}"
 ironic_pxe_default_volumes:
   - "{{ node_config_directory }}/ironic-pxe/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "ironic_pxe:/tftpboot/"
+  - "ironic:/var/lib/ironic"
   - "kolla_logs:/var/log/kolla"
 ironic_ipxe_default_volumes:
   - "{{ node_config_directory }}/ironic-ipxe/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "ironic:/var/lib/ironic:ro"
-  - "ironic_ipxe:/httpboot/"
+  - "ironic:/var/lib/ironic"
   - "kolla_logs:/var/log/kolla"
 ironic_inspector_default_volumes:
   - "{{ node_config_directory }}/ironic-inspector/:{{ container_config_directory }}/:ro"
@@ -260,6 +260,7 @@ ironic_cleaning_network:
 ironic_console_serial_speed: "115200n8"
 ironic_ipxe_url: "http://{{ api_interface_address | put_address_in_context('url') }}:{{ ironic_ipxe_port }}"
 ironic_enable_rolling_upgrade: "yes"
+ironic_upgrade_skip_wait_check: false
 ironic_inspector_kernel_cmdline_extras: []
 ironic_inspector_pxe_filter: "{% if enable_neutron | bool %}dnsmasq{% else %}noop{% endif %}"
 

ansible/roles/ironic/handlers/main.yml

@@ -79,6 +79,7 @@
     common_options: "{{ docker_common_options }}"
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
+    environment: "{{ service.environment }}"
     volumes: "{{ service.volumes }}"
     dimensions: "{{ service.dimensions }}"
   when:

ansible/roles/ironic/tasks/bootstrap_service.yml

@@ -54,6 +54,8 @@
     common_options: "{{ docker_common_options }}"
     detach: False
     environment:
+      TFTPBOOT_PATH: /var/lib/ironic/tftpboot
+      HTTPBOOT_PATH: /var/lib/ironic/httpboot
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
     image: "{{ ironic_pxe.image }}"

ansible/roles/ironic/tasks/check-containers.yml

@@ -8,6 +8,7 @@
     image: "{{ item.value.image }}"
     privileged: "{{ item.value.privileged|default(False) }}"
     cap_add: "{{ item.value.cap_add|default([]) }}"
+    environment: "{{ item.value.environment | default(omit) }}"
     volumes: "{{ item.value.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ item.value.dimensions }}"
     healthcheck: "{{ item.value.healthcheck | default(omit) }}"

ansible/roles/ironic/tasks/upgrade.yml

@@ -1,6 +1,45 @@
 ---
+- name: Wait for Ironic nodes not to wait
+  become: true
+  command: >
+    docker exec kolla_toolbox openstack
+    --os-interface {{ openstack_interface }}
+    --os-auth-url {{ openstack_auth.auth_url }}
+    --os-username {{ openstack_auth.username }}
+    --os-password {{ openstack_auth.password }}
+    --os-identity-api-version 3
+    --os-user-domain-name {{ openstack_auth.user_domain_name }}
+    --os-system-scope {{ openstack_auth.system_scope }}
+    --os-region-name {{ openstack_region_name }}
+    {% if openstack_cacert != '' %}--os-cacert {{ openstack_cacert }}{% endif %}
+    baremetal node list --format json --column "Provisioning State"
+  register: ironic_nodes
+  changed_when: false
+  retries: 10
+  delay: 30
+  until:
+    - ironic_nodes is success
+    - (ironic_nodes.stdout |
+       from_json |
+       map(attribute='Provisioning State') |
+       select('search', '\\bwait\\b') |
+       length) == 0
+  run_once: true
+  when: not ironic_upgrade_skip_wait_check | bool
+
 - include_tasks: rolling_upgrade.yml
   when: ironic_enable_rolling_upgrade | bool
 
 - include_tasks: legacy_upgrade.yml
   when: not ironic_enable_rolling_upgrade | bool
+
+# TODO(yoctozepto): Remove this task in Zed.
+- name: Remove old Ironic volumes
+  become: true
+  kolla_docker:
+    action: "remove_volume"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item }}"
+  with_items:
+    - ironic_pxe
+    - ironic_ipxe

ansible/roles/ironic/templates/ironic-conductor.json.j2

@@ -24,16 +24,6 @@
             "path": "/var/lib/ironic",
             "owner": "ironic:ironic",
             "recurse": true
-        },
-        {
-            "path": "/tftpboot",
-            "owner": "ironic:ironic",
-            "recurse": true
-        },
-        {
-            "path": "/httpboot",
-            "owner": "ironic:ironic",
-            "recurse": true
         }
     ]
 }

ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2

@@ -17,7 +17,7 @@ dhcp-option=3,{{ ironic_dnsmasq_default_gateway }}
 {% endif %}
 dhcp-option=option:tftp-server,{{ api_interface_address }}
 dhcp-option=option:server-ip-address,{{ api_interface_address }}
-dhcp-option=210,/tftpboot/
+dhcp-option=210,/var/lib/ironic/tftpboot/
 {% if enable_ironic_ipxe | bool %}
 dhcp-match=ipxe,175
 dhcp-match=set:efi,option:client-arch,7

ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2

@@ -7,8 +7,8 @@ TraceEnable off
     ErrorLog "/var/log/kolla/ironic/ironic-ipxe-error.log"
     LogFormat "%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
     CustomLog "/var/log/kolla/ironic/ironic-ipxe-access.log" logformat
-    DocumentRoot "/httpboot"
-    <Directory /httpboot>
+    DocumentRoot "/var/lib/ironic/httpboot"
+    <Directory /var/lib/ironic/httpboot>
         Options FollowSymLinks
         AllowOverride None
         Require all granted

ansible/roles/ironic/templates/ironic-ipxe.json.j2

@@ -6,19 +6,19 @@
 {% if groups['ironic-inspector'] | length > 0 %}
         {
             "source": "{{ container_config_directory }}/ironic-agent.kernel",
-            "dest": "/httpboot/ironic-agent.kernel",
+            "dest": "/var/lib/ironic/httpboot/ironic-agent.kernel",
             "owner": "root",
             "perm": "0644"
         },
         {
             "source": "{{ container_config_directory }}/ironic-agent.initramfs",
-            "dest": "/httpboot/ironic-agent.initramfs",
+            "dest": "/var/lib/ironic/httpboot/ironic-agent.initramfs",
             "owner": "root",
             "perm": "0644"
         },
         {
             "source": "{{ container_config_directory }}/inspector.ipxe",
-            "dest": "/httpboot/inspector.ipxe",
+            "dest": "/var/lib/ironic/httpboot/inspector.ipxe",
             "owner": "root",
             "perm": "0644"
         },

ansible/roles/ironic/templates/ironic-pxe.json.j2

@@ -1,20 +1,20 @@
-{% set pxe_dir = ('/tftpboot/grub' if kolla_base_distro in ['ubuntu', 'debian'] else '/tftpboot/EFI/centos') if enable_ironic_pxe_uefi | bool else '/tftpboot/pxelinux.cfg' %}
+{% set pxe_dir = ('/var/lib/ironic/tftpboot/grub' if kolla_base_distro in ['ubuntu', 'debian'] else '/var/lib/ironic/tftpboot/EFI/centos') if enable_ironic_pxe_uefi | bool else '/var/lib/ironic/tftpboot/pxelinux.cfg' %}
 {% set pxe_cfg = 'grub.cfg' if enable_ironic_pxe_uefi | bool else 'default' %}
 
 {
-    "command": "/usr/sbin/in.tftpd --verbose --foreground --user root --address 0.0.0.0:69 --map-file /map-file /tftpboot",
+    "command": "/usr/sbin/in.tftpd --verbose --foreground --user root --address 0.0.0.0:69 --map-file /map-file /var/lib/ironic/tftpboot",
     "config_files": [
 {% if not enable_ironic_ipxe | bool and groups['ironic-inspector'] | length > 0 %}
 {% if not enable_ironic_pxe_uefi | bool %}
         {
             "source": "{{ container_config_directory }}/ironic-agent.kernel",
-            "dest": "/tftpboot/ironic-agent.kernel",
+            "dest": "/var/lib/ironic/tftpboot/ironic-agent.kernel",
             "owner": "root",
             "perm": "0644"
         },
         {
             "source": "{{ container_config_directory }}/ironic-agent.initramfs",
-            "dest": "/tftpboot/ironic-agent.initramfs",
+            "dest": "/var/lib/ironic/tftpboot/ironic-agent.initramfs",
             "owner": "root",
             "perm": "0644"
         },

ansible/roles/ironic/templates/ironic.conf.j2

@@ -176,18 +176,12 @@ deploy_logs_collect = always
 
 [pxe]
 kernel_append_params = nofb nomodeset vga=normal console=tty0 console=ttyS0,{{ ironic_console_serial_speed }}
-{% if enable_ironic_ipxe | bool %}
-{# NOTE(mgoddard): iPXE uses the TFTP image cache (tftp_master_path, default
-   /tftpboot/master_images), in which images get hard linked to the http_root
-   directory (/httpboot). These must be on the same device, but /httpboot and
-   /tftpboot live in separate Docker volumes. Override the default paths for
-   iPXE to place them both in /httpboot. This prevents mixing PXE and iPXE. #}
-tftp_root = /httpboot
-tftp_master_path = /httpboot/master_images
+tftp_root = /var/lib/ironic/tftpboot
+tftp_master_path = /var/lib/ironic/master_images
 tftp_server = {{ api_interface_address }}
-{% endif %}
 
 [deploy]
+http_root = /var/lib/ironic/httpboot
 http_url = {{ ironic_ipxe_url }}
 
 [oslo_middleware]

ansible/roles/ironic/templates/ironic_pxe_uefi.default.j2

@@ -3,5 +3,5 @@ set timeout=5
 set hidden_timeout_quiet=false
 
 menuentry "master"  {
-    configfile /tftpboot/$net_default_mac.conf
+    configfile /var/lib/ironic/tftpboot/$net_default_mac.conf
 }

doc/source/reference/bare-metal/ironic-guide.rst

@@ -36,9 +36,9 @@ Inspector inspection network:
    ironic_dnsmasq_default_gateway: 192.168.5.1
 
 In the same file, specify the PXE bootloader file for Ironic Inspector. The
-file is relative to the ``/tftpboot`` directory. The default is ``pxelinux.0``,
-and should be correct for x86 systems. Other platforms may require a different
-value, for example aarch64 on Debian requires
+file is relative to the ``/var/lib/ironic/tftpboot`` directory. The default is
+``pxelinux.0``, and should be correct for x86 systems. Other platforms may
+require a differentvalue, for example aarch64 on Debian requires
 ``debian-installer/arm64/bootnetaa64.efi``.
 
 .. code-block:: yaml
@@ -78,28 +78,22 @@ The port used for the iPXE webserver is controlled via ``ironic_ipxe_port`` in
 Revert to plain PXE (not recommended)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Starting with Yoga, Ironic has changed the default PXE from plain PXE to iPXE.
-Kolla Ansible follows this upstream decision but allows users to revert to
-plain PXE. Please note Kolla Ansible does not support plain PXE and iPXE at the
-same time - the user must choose one.
-
-If you have to revert to plain iPXE, set:
+Kolla Ansible follows this upstream decision by choosing iPXE as the default
+for Ironic Inspector but allows users to revert to the previous default of
+plain PXE by setting the following in
+``/etc/kolla/globals.yml``:
 
 .. code-block:: yaml
 
    enable_ironic_ipxe: "no"
 
-And also remove ``ipxe`` from the ``enabled_boot_interfaces`` in
-``/etc/kolla/config/ironic.conf``, leaving only ``pxe`` (and possibly other
-alternatives) around:
+To revert Ironic to previous default as well, set ``pxe`` as
+``default_boot_interface`` in ``/etc/kolla/config/ironic.conf``:
 
 .. code-block:: yaml
 
    [DEFAULT]
-   enabled_boot_interfaces = pxe
-
-When iPXE booting is enabled, the ``ironic_ipxe`` container is used to serve
-the iPXE boot images as described below. Regardless of that setting, the
-same container is used to support the ``direct`` deploy interface.
+   default_boot_interface = pxe
 
 Attach ironic to external keystone (optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

etc/kolla/globals.yml

@@ -601,7 +601,7 @@
 # The following value must be set when enabling ironic, the value format is
 # "192.168.0.10,192.168.0.100,255.255.255.0" the last being an optional netmask.
 #ironic_dnsmasq_dhcp_range:
-# PXE bootloader file for Ironic Inspector, relative to /tftpboot.
+# PXE bootloader file for Ironic Inspector, relative to /var/lib/ironic/tftpboot.
 #ironic_dnsmasq_boot_file: "pxelinux.0"
 
 # Configure ironic upgrade option, due to currently kolla support

releasenotes/notes/ironic-default-ipxe-1e9bb7a3f4994d84.yaml

@@ -4,6 +4,5 @@ upgrade:
     Starting with Yoga, Ironic has changed the default PXE from plain PXE to
     iPXE.
     Kolla Ansible follows this upstream decision but allows users to revert to
-    plain PXE. Please note Kolla Ansible does not support plain PXE and iPXE at
-    the same time - the user must choose one. For details, please refer to
+    the previous default of plain PXE. For details, please refer to
     Kolla Ansible's documentation.

releasenotes/notes/ironic-pxe-and-ipxe-57357e928ed9ea48.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    Support for both PXE and iPXE enabled in Ironic at the same time.
+upgrade:
+  - |
+    Ironic volumes related to PXE (TFTP) and iPXE & direct deploy (HTTP)
+    are refactored to share a common parent path at ``/var/lib/ironic``.
+    This is done to support both PXE and iPXE at the same time.
+    Operators doing advanced customisations might need to review the
+    relevant defaults section.
+  - |
+    Upgrades of Ironic will now wait for nodes in ``wait`` states to change
+    their state. This is to improve the user experience by avoiding
+    breaking processes being waited on. This can be disabled by setting
+    ``ironic_upgrade_skip_wait_check`` to ``yes``.