From 0a1ccc2612240ff15c255eeafc67f56835278adf Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Wed, 18 Apr 2018 17:46:38 +0100
Subject: [PATCH] Add support for enabling ipxe boot with ironic
When enable_ironic_ipxe is set in /etc/kolla/globals.yml,
the following happens:
- a new docker container, ironic_ipxe, is created. This contains
an apache webserver used to serve up the boot images
- ironic is configured to use ipxe
Change-Id: I08fca1864a00afb768494406c49e968920c83ae7
Implements: blueprint ironic-ipxe
---
ansible/group_vars/all.yml | 2 +
ansible/inventory/all-in-one | 3 +
ansible/inventory/multinode | 3 +
ansible/roles/ironic/defaults/main.yml | 14 ++++-
ansible/roles/ironic/handlers/main.yml | 22 +++++++
ansible/roles/ironic/tasks/config.yml | 63 ++++++++++++++++++-
ansible/roles/ironic/tasks/deploy.yml | 3 +-
ansible/roles/ironic/tasks/precheck.yml | 16 ++++-
.../roles/ironic/templates/inspector.ipxe.j2 | 10 +++
.../ironic/templates/ironic-conductor.json.j2 | 5 ++
.../ironic/templates/ironic-dnsmasq.conf.j2 | 13 +++-
.../templates/ironic-ipxe-httpd.conf.j2 | 16 +++++
.../ironic/templates/ironic-ipxe.json.j2 | 33 ++++++++++
.../roles/ironic/templates/ironic-pxe.json.j2 | 2 +-
ansible/roles/ironic/templates/ironic.conf.j2 | 14 +++++
doc/source/reference/ironic-guide.rst | 33 ++++++++++
...ort-ironic-ipxe-boot-2ea7f598748403bd.yaml | 5 ++
tests/templates/inventory.j2 | 3 +
18 files changed, 254 insertions(+), 6 deletions(-)
create mode 100644 ansible/roles/ironic/templates/inspector.ipxe.j2
create mode 100644 ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2
create mode 100644 ansible/roles/ironic/templates/ironic-ipxe.json.j2
create mode 100644 releasenotes/notes/support-ironic-ipxe-boot-2ea7f598748403bd.yaml
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 7ecf845374..4167a4a06a 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -210,6 +210,7 @@ influxdb_http_port: "8086"
ironic_api_port: "6385"
ironic_inspector_port: "5050"
+ironic_ipxe_port: "8089"
iscsi_port: "3260"
@@ -454,6 +455,7 @@ enable_horizon_zun: "{{ enable_zun | bool }}"
enable_hyperv: "no"
enable_influxdb: "{{ enable_monasca | bool }}"
enable_ironic: "no"
+enable_ironic_ipxe: "no"
enable_ironic_pxe_uefi: "no"
enable_iscsid: "{{ (enable_cinder | bool and enable_cinder_backend_iscsi | bool) or enable_ironic | bool }}"
enable_karbor: "no"
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index b0fdd5ec05..f81cd70eed 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -459,6 +459,9 @@ ironic
[ironic-pxe:children]
ironic
+[ironic-ipxe:children]
+ironic
+
# Magnum
[magnum-api:children]
magnum
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 0755114ba9..91ee102165 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -468,6 +468,9 @@ ironic
[ironic-pxe:children]
ironic
+[ironic-ipxe:children]
+ironic
+
# Magnum
[magnum-api:children]
magnum
diff --git a/ansible/roles/ironic/defaults/main.yml b/ansible/roles/ironic/defaults/main.yml
index d2c48b57df..3b91bcce3a 100644
--- a/ansible/roles/ironic/defaults/main.yml
+++ b/ansible/roles/ironic/defaults/main.yml
@@ -27,6 +27,7 @@ ironic_services:
- "kolla_logs:/var/log/kolla"
- "ironic:/var/lib/ironic"
- "ironic_pxe:/tftpboot/"
+ - "ironic_ipxe:/httpboot/"
ironic-inspector:
container_name: ironic_inspector
group: ironic-inspector
@@ -47,6 +48,16 @@ ironic_services:
- "/etc/localtime:/etc/localtime:ro"
- "ironic_pxe:/tftpboot/"
- "kolla_logs:/var/log/kolla"
+ ironic-ipxe:
+ container_name: ironic_ipxe
+ group: ironic-ipxe
+ enabled: "{{ enable_ironic_ipxe | bool }}"
+ image: "{{ ironic_pxe_image_full }}"
+ volumes:
+ - "{{ node_config_directory }}/ironic-ipxe/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "ironic_ipxe:/httpboot/"
+ - "kolla_logs:/var/log/kolla"
ironic-dnsmasq:
container_name: ironic_dnsmasq
group: ironic-inspector
@@ -125,6 +136,7 @@ openstack_ironic_inspector_auth: "{{ openstack_auth }}"
ironic_dnsmasq_interface: "{{ api_interface }}"
ironic_dnsmasq_dhcp_range:
-ironic_dnsmasq_boot_file: "pxelinux.0"
+ironic_dnsmasq_boot_file: "{% if enable_ironic_ipxe | bool %}undionly.kpxe{% else %}pxelinux.0{% endif %}"
ironic_cleaning_network:
ironic_console_serial_speed: "115200n8"
+ironic_ipxe_url: http://{{ api_interface_address }}:{{ ironic_ipxe_port }}
diff --git a/ansible/roles/ironic/handlers/main.yml b/ansible/roles/ironic/handlers/main.yml
index 2c179a3d37..49ee9132e5 100644
--- a/ansible/roles/ironic/handlers/main.yml
+++ b/ansible/roles/ironic/handlers/main.yml
@@ -91,6 +91,28 @@
or ironic_kernel.changed | bool
or ironic_pxe_container.changed | bool
+- name: Restart ironic-ipxe container
+ vars:
+ service_name: "ironic-ipxe"
+ service: "{{ ironic_services[service_name] }}"
+ config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ ironic_ipxe_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ when:
+ - kolla_action != "config"
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ - config_json.changed | bool
+ or ironic_ipxe_inspector_boot_script.changed | bool
+ or ironic_ipxe_apache_confs.changed | bool
+ or ironic_kernel_ipxe.changed | bool
+ or ironic_ipxe_container.changed | bool
+
- name: Restart ironic-dnsmasq container
vars:
service_name: "ironic-dnsmasq"
diff --git a/ansible/roles/ironic/tasks/config.yml b/ansible/roles/ironic/tasks/config.yml
index 689efdfb99..99f284e23f 100644
--- a/ansible/roles/ironic/tasks/config.yml
+++ b/ansible/roles/ironic/tasks/config.yml
@@ -123,6 +123,7 @@
- inventory_hostname in groups[service.group]
- service.enabled | bool
- not enable_ironic_pxe_uefi | bool
+ - not enable_ironic_ipxe | bool
notify:
- Restart ironic-pxe container
@@ -146,7 +147,7 @@
notify:
- Restart ironic-pxe container
-- name: Copying ironic-agent kernel and initramfs
+- name: Copying ironic-agent kernel and initramfs (PXE)
vars:
service: "{{ ironic_services['ironic-pxe'] }}"
copy:
@@ -165,9 +166,69 @@
- inventory_hostname in groups[service.group]
- service.enabled | bool
- not enable_ironic_pxe_uefi | bool
+ - not enable_ironic_ipxe | bool
notify:
- Restart ironic-pxe container
+- name: Copying ironic-agent kernel and initramfs (iPXE)
+ vars:
+ service: "{{ ironic_services['ironic-ipxe'] }}"
+ copy:
+ src: "{{ node_custom_config }}/ironic/{{ item }}"
+ dest: "{{ node_config_directory }}/ironic-ipxe/{{ item }}"
+ mode: "0660"
+ become: true
+ register: ironic_kernel_ipxe
+ with_items:
+ - "ironic-agent.kernel"
+ - "ironic-agent.initramfs"
+ when:
+ # Only required when Ironic inspector is in use.
+ - groups['ironic-inspector'] | length > 0
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ notify:
+ - Restart ironic-ipxe container
+
+- name: Copying inspector.ipxe
+ vars:
+ service: "{{ ironic_services['ironic-ipxe'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/ironic-ipxe/inspector.ipxe"
+ mode: "0660"
+ become: true
+ register: ironic_ipxe_inspector_boot_script
+ with_first_found:
+ - "{{ node_custom_config }}/ironic/{{ inventory_hostname }}/inspector.ipxe"
+ - "{{ node_custom_config }}/ironic/inspector.ipxe"
+ - "inspector.ipxe.j2"
+ when:
+ # Only required when Ironic inspector is in use.
+ - groups['ironic-inspector'] | length > 0
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ notify:
+ - Restart ironic-ipxe container
+
+- name: Copying iPXE apache config
+ vars:
+ service: "{{ ironic_services['ironic-ipxe'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/ironic-ipxe/httpd.conf"
+ mode: "0660"
+ become: true
+ register: ironic_ipxe_apache_confs
+ with_first_found:
+ - "{{ node_custom_config }}/ironic/ironic-ipxe-httpd.conf"
+ - "ironic-ipxe-httpd.conf.j2"
+ when:
+ - service.enabled | bool
+ - inventory_hostname in groups[service.group]
+ notify:
+ - Restart ironic-ipxe container
+
- name: Copying over existing policy file
vars:
services_require_policy_json:
diff --git a/ansible/roles/ironic/tasks/deploy.yml b/ansible/roles/ironic/tasks/deploy.yml
index dbbd9b8bba..d599fbe518 100644
--- a/ansible/roles/ironic/tasks/deploy.yml
+++ b/ansible/roles/ironic/tasks/deploy.yml
@@ -8,7 +8,8 @@
when: inventory_hostname in groups['ironic-api'] or
inventory_hostname in groups['ironic-conductor'] or
inventory_hostname in groups['ironic-inspector'] or
- inventory_hostname in groups['ironic-pxe']
+ inventory_hostname in groups['ironic-pxe'] or
+ inventory_hostname in groups['ironic-ipxe']
- include: bootstrap.yml
when: inventory_hostname in groups['ironic-api'] or
diff --git a/ansible/roles/ironic/tasks/precheck.yml b/ansible/roles/ironic/tasks/precheck.yml
index 52f6b50225..a3d03bbb65 100644
--- a/ansible/roles/ironic/tasks/precheck.yml
+++ b/ansible/roles/ironic/tasks/precheck.yml
@@ -4,6 +4,7 @@
name:
- ironic_api
- ironic_inspector
+ - ironic_ipxe
register: container_facts
- name: Checking free port for Ironic API
@@ -28,6 +29,18 @@
- container_facts['ironic_inspector'] is not defined
- inventory_hostname in groups['ironic-inspector']
+- name: Checking free port for Ironic iPXE
+ wait_for:
+ host: "{{ api_interface_address }}"
+ port: "{{ ironic_ipxe_port }}"
+ connect_timeout: 1
+ timeout: 1
+ state: stopped
+ when:
+ - enable_ironic_ipxe | bool
+ - container_facts['ironic_ipxe'] is not defined
+ - inventory_hostname in groups['ironic-ipxe']
+
- name: Checking ironic-agent files exist for Ironic Inspector
local_action: stat path="{{ node_custom_config }}/ironic/{{ item }}"
run_once: True
@@ -36,7 +49,8 @@
when:
# Only required when Ironic inspector is in use.
- groups['ironic-inspector'] | length > 0
- - inventory_hostname in groups['ironic-pxe']
+ - (not enable_ironic_ipxe | bool and inventory_hostname in groups['ironic-pxe']) or
+ (enable_ironic_ipxe | bool and inventory_hostname in groups['ironic-ipxe'])
- not enable_ironic_pxe_uefi | bool
with_items:
- "ironic-agent.kernel"
diff --git a/ansible/roles/ironic/templates/inspector.ipxe.j2 b/ansible/roles/ironic/templates/inspector.ipxe.j2
new file mode 100644
index 0000000000..25bfc6e64b
--- /dev/null
+++ b/ansible/roles/ironic/templates/inspector.ipxe.j2
@@ -0,0 +1,10 @@
+#!ipxe
+
+:retry_dhcp
+dhcp || goto retry_dhcp
+
+:retry_boot
+imgfree
+kernel --timeout 30000 {{ ironic_ipxe_url }}/ironic-agent.kernel ipa-inspection-callback-url=http://{{ kolla_internal_vip_address }}:{{ ironic_inspector_port }}/v1/continue systemd.journald.forward_to_console=yes BOOTIF=${mac} initrd=agent.ramdisk || goto retry_boot
+initrd --timeout 30000 {{ ironic_ipxe_url }}/ironic-agent.initramfs || goto retry_boot
+boot
diff --git a/ansible/roles/ironic/templates/ironic-conductor.json.j2 b/ansible/roles/ironic/templates/ironic-conductor.json.j2
index 94dfe227a5..19b96fb819 100644
--- a/ansible/roles/ironic/templates/ironic-conductor.json.j2
+++ b/ansible/roles/ironic/templates/ironic-conductor.json.j2
@@ -29,6 +29,11 @@
"path": "/tftpboot",
"owner": "ironic:ironic",
"recurse": true
+ },
+ {
+ "path": "/httpboot",
+ "owner": "ironic:ironic",
+ "recurse": true
}
]
}
diff --git a/ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2 b/ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2
index 48e5cf436e..1e21eb8b13 100644
--- a/ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2
+++ b/ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2
@@ -5,5 +5,16 @@ dhcp-option=option:tftp-server,{{ api_interface_address }}
dhcp-option=option:server-ip-address,{{ api_interface_address }}
bind-interfaces
dhcp-sequential-ip
-dhcp-option=option:bootfile-name,{{ ironic_dnsmasq_boot_file }}
dhcp-option=210,/tftpboot/
+{% if enable_ironic_ipxe | bool %}
+dhcp-match=ipxe,175
+dhcp-match=set:efi,option:client-arch,7
+dhcp-match=set:efi,option:client-arch,9
+# Client is already running iPXE; move to next stage of chainloading
+dhcp-option=tag:ipxe,option:bootfile-name,{{ ironic_ipxe_url }}/inspector.ipxe
+# Client is PXE booting over EFI without iPXE ROM,
+# send EFI version of iPXE chainloader
+dhcp-option=tag:efi,tag:!ipxe,option:bootfile-name,ipxe.efi
+{% endif %}
+dhcp-option=option:bootfile-name,{{ ironic_dnsmasq_boot_file }}
+
diff --git a/ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2 b/ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2
new file mode 100644
index 0000000000..8109e21e3e
--- /dev/null
+++ b/ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2
@@ -0,0 +1,16 @@
+Listen {{ api_interface_address }}:{{ ironic_ipxe_port }}
+
+TraceEnable off
+
+<VirtualHost *:{{ ironic_ipxe_port }}>
+ LogLevel warn
+ ErrorLog "/var/log/kolla/ironic/ironic-ipxe-error.log"
+ LogFormat "%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
+ CustomLog "/var/log/kolla/ironic/ironic-ipxe-access.log" logformat
+ DocumentRoot "/httpboot"
+ <Directory /httpboot>
+ Options FollowSymLinks
+ AllowOverride None
+ Require all granted
+ </Directory>
+</VirtualHost>
diff --git a/ansible/roles/ironic/templates/ironic-ipxe.json.j2 b/ansible/roles/ironic/templates/ironic-ipxe.json.j2
new file mode 100644
index 0000000000..1387171523
--- /dev/null
+++ b/ansible/roles/ironic/templates/ironic-ipxe.json.j2
@@ -0,0 +1,33 @@
+{% set apache_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
+{% set apache_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+{
+ "command": "{{ apache_cmd }} -DFOREGROUND",
+ "config_files": [
+{% if groups['ironic-inspector'] | length > 0 %}
+ {
+ "source": "{{ container_config_directory }}/ironic-agent.kernel",
+ "dest": "/httpboot/ironic-agent.kernel",
+ "owner": "root",
+ "perm": "0644"
+ },
+ {
+ "source": "{{ container_config_directory }}/ironic-agent.initramfs",
+ "dest": "/httpboot/ironic-agent.initramfs",
+ "owner": "root",
+ "perm": "0644"
+ },
+ {
+ "source": "{{ container_config_directory }}/inspector.ipxe",
+ "dest": "/httpboot/inspector.ipxe",
+ "owner": "root",
+ "perm": "0644"
+ },
+{% endif %}
+ {
+ "source": "{{ container_config_directory }}/httpd.conf",
+ "dest": "/etc/{{ apache_conf_dir }}/httpboot.conf",
+ "owner": "root",
+ "perm": "0644"
+ }
+ ]
+}
diff --git a/ansible/roles/ironic/templates/ironic-pxe.json.j2 b/ansible/roles/ironic/templates/ironic-pxe.json.j2
index a7bd604bfa..96e0979e24 100644
--- a/ansible/roles/ironic/templates/ironic-pxe.json.j2
+++ b/ansible/roles/ironic/templates/ironic-pxe.json.j2
@@ -4,7 +4,7 @@
{
"command": "/usr/sbin/in.tftpd --verbose --foreground --user root --address 0.0.0.0:69 --map-file /map-file /tftpboot",
"config_files": [
-{% if groups['ironic-inspector'] | length > 0 %}
+{% if not enable_ironic_ipxe | bool and groups['ironic-inspector'] | length > 0 %}
{% if not enable_ironic_pxe_uefi | bool %}
{
"source": "{{ container_config_directory }}/ironic-agent.kernel",
diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2
index 0a9744e671..d468af7f80 100644
--- a/ansible/roles/ironic/templates/ironic.conf.j2
+++ b/ansible/roles/ironic/templates/ironic.conf.j2
@@ -97,6 +97,20 @@ deploy_logs_collect = always
[pxe]
pxe_append_params = nofb nomodeset vga=normal console=tty0 console=ttyS0,{{ ironic_console_serial_speed }}
+{% if enable_ironic_ipxe | bool %}
+ipxe_enabled = True
+pxe_bootfile_name = undionly.kpxe
+uefi_pxe_bootfile_name = ipxe.efi
+pxe_config_template = $pybasedir/drivers/modules/ipxe_config.template
+uefi_pxe_config_template = $pybasedir/drivers/modules/ipxe_config.template
+tftp_root = /httpboot
+tftp_master_path = /httpboot/master_images
+{% endif %}
+
+{% if enable_ironic_ipxe | bool %}
+[deploy]
+http_url = {{ ironic_ipxe_url }}
+{% endif %}
[oslo_middleware]
enable_proxy_headers_parsing = True
diff --git a/doc/source/reference/ironic-guide.rst b/doc/source/reference/ironic-guide.rst
index 8cee33b746..4eae760b08 100644
--- a/doc/source/reference/ironic-guide.rst
+++ b/doc/source/reference/ironic-guide.rst
@@ -57,6 +57,39 @@ be used:
.. end
+Enable iPXE booting (optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can optionally enable booting via iPXE by setting ``enable_ironic_ipxe`` to
+true in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+ enable_ironic_ipxe: "yes"
+
+.. end
+
+This will enable deployment of a docker container, called ironic_ipxe, running
+the web server which iPXE uses to obtain it's boot images.
+
+The port used for the iPXE webserver is controlled via ``ironic_ipxe_port`` in
+``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+ ironic_ipxe_port: "8089"
+
+.. end
+
+The following changes will occur if iPXE booting is enabled:
+
+- Ironic will be configured with the ``ipxe_enabled`` configuration option set
+ to true
+- The inspection ramdisk and kernel will be loaded via iPXE
+- The DHCP servers will be configured to chainload iPXE from an existing PXE
+ environment. You may also boot directly to iPXE by some other means e.g by
+ burning it to the option rom of your ethernet card.
+
Deployment
~~~~~~~~~~
Run the deploy as usual:
diff --git a/releasenotes/notes/support-ironic-ipxe-boot-2ea7f598748403bd.yaml b/releasenotes/notes/support-ironic-ipxe-boot-2ea7f598748403bd.yaml
new file mode 100644
index 0000000000..b2f5e50f02
--- /dev/null
+++ b/releasenotes/notes/support-ironic-ipxe-boot-2ea7f598748403bd.yaml
@@ -0,0 +1,5 @@
+---
+features:
+ - |
+ Adds support for booting bare metal nodes with Ironic using iPXE.
+ This is enabled via the ``enable_ironic_ipxe`` flag.
diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2
index 806705f5a1..545750f9d4 100644
--- a/tests/templates/inventory.j2
+++ b/tests/templates/inventory.j2
@@ -438,6 +438,9 @@ ironic
[ironic-pxe:children]
ironic
+[ironic-ipxe:children]
+ironic
+
# Magnum
[magnum-api:children]
magnum