diff --git a/ansible/group_vars/all/idrac b/ansible/group_vars/all/idrac new file mode 100644 index 000000000..4df40736a --- /dev/null +++ b/ansible/group_vars/all/idrac @@ -0,0 +1,43 @@ +--- +############################################################################### +# iDRAC configuration. + +# Default username for iDRACs. +idrac_default_username: root + +# Default password for iDRACs. +idrac_default_password: calvin + +# Default IP address for iDRACs. +idrac_default_ip: 192.168.0.120 + +# Dict mapping host description (as found in switch interface configuration) to +# IP address of the iDRAC for that host. +idrac_network_ips: + +# Gateway IP address for iDRAC network. +idrac_network_gateway: + +# IP netmask for iDRAC network. +idrac_network_netmask: + +# VLAN for iDRAC network. +idrac_network_vlan: + +# ID of VLAN to use for bootstrapping iDRACs. +idrac_bootstrap_vlan: + +# Name of network namespace on controller to use for bootstrapping iDRACs. +idrac_bootstrap_net_namespace: idrac-bootstrap + +# Base network interface on controller to use for bootstrapping iDRACs. +idrac_bootstrap_controller_interface: + +# VLAN network interface on controller to create for bootstrapping iDRACs. +idrac_bootstrap_controller_vlan_interface: "{{ idrac_bootstrap_controller_interface }}.{{ idrac_bootstrap_vlan }}" + +# IP address of controller to use for bootstrapping iDRACs. +idrac_bootstrap_controller_ip: 192.168.0.1 + +# Name of an Ansible group containing switches forming the iDRAC network. +idrac_bootstrap_switch_group: diff --git a/ansible/idrac-bootstrap-one.yml b/ansible/idrac-bootstrap-one.yml new file mode 100644 index 000000000..617121470 --- /dev/null +++ b/ansible/idrac-bootstrap-one.yml @@ -0,0 +1,190 @@ +--- +# This is a tasks file used by the idrac-bootstrap.yml playbook to support +# bootstrapping the network configuration of a single iDRAC. +# We use the following procedure to configure the iDRAC: +# 1. Check whether the required IP is already reachable. Skip remaining tasks +# if so. +# 2. Configure the switch interface to which the iDRAC is attached as an +# access port on the bootstrap VLAN. +# 3. Clear the ARP cache on the controller in the bootstrap network namespace. +# 4. Check whether the iDRAC default IP address is reachable. +# 5. Enable IPMI on the iDRAC. +# 6. Configure networking for the iDRAC. +# 7. Configure the switch interface to which the iDRAC is attached as an +# access port on the iDRAC management VLAN. + +- name: Check whether we can ping the iDRAC's configured IP address + command: "ping -c 1 {{ idrac_network_ip }}" + run_once: True + # We use this convoluted method to allow a delegate_to with a variable host. + # See http://www.elmund.io/configuration%20management/2015/07/23/ansible-delegate_to-and-variables/. + with_items: + - "{{ idrac_bootstrap_controller }}" + loop_control: + loop_var: delegate_host + delegate_to: "{{ delegate_host }}" + register: ping_result + changed_when: False + failed_when: False + +- name: Set a fact about whether the iDRAC requires bootstrapping + set_fact: + idrac_bootstrap_required: "{{ ping_result.results[0].rc != 0 }}" + run_once: True + +- name: Display the result of the ping + debug: + msg: > + The iDRAC on switch port with description {{ idrac_port_description }} + and configured IP address {{ idrac_network_ip }} was + {{ 'un' if idrac_bootstrap_required else '' }}reachable. The iDRAC will + {{ '' if idrac_bootstrap_required else 'not ' }}be bootstrapped. + run_once: True + +# The tasks in this block are only executed when the bootstrap is required. +- block: + - name: Ensure DellOS6 switch interface is a member of the bootstrap VLAN + dellos6_config: + provider: "{{ dell_switch_provider }}" + lines: + - "switchport access vlan {{ idrac_bootstrap_vlan }}" + parents: + - "interface {{ switch_interface_name }}" + delegate_to: localhost + when: "{{ switch_type == 'dellos6' }}" + + # The tasks in this block are delegated to the controller. + - block: + - name: Ensure the iDRAC default IP address is removed from the controller's ARP cache + command: > + ip netns exec {{ idrac_bootstrap_net_namespace }} + arp -d {{ idrac_default_ip }} + become: True + with_items: + - "{{ idrac_bootstrap_controller }}" + loop_control: + loop_var: delegate_host + delegate_to: "{{ delegate_host }}" + register: arp_result + failed_when: + - "{{ arp_result | failed }}" + - "{{ 'No ARP entry for ' ~ idrac_default_ip not in arp_result.stdout }}" + + # Ansible's until keyword seems to not work nicely with failed_when, causing + # the task to fail even though we have specified failed_when: False. + - name: Check whether we can ping the iDRAC's default IP address + shell: | + max_attempts=3 + interval=5 + for attempt in $(seq $max_attempts); do + ip netns exec {{ idrac_bootstrap_net_namespace }} \ + ping -c 1 {{ idrac_default_ip }} + ping_rc=$? + if [[ $ping_rc -eq 0 ]] || [[ $attempt -eq $max_attempts ]]; then + break + fi + sleep $interval + done + exit $ping_rc + become: True + with_items: + - "{{ idrac_bootstrap_controller }}" + loop_control: + loop_var: delegate_host + delegate_to: "{{ delegate_host }}" + register: ping_result + changed_when: False + failed_when: False + + - name: Initialise a fact about whether iDRAC bootstrap failed + set_fact: + idrac_bootstrap_failure: {} + + - name: Set a fact about whether the iDRAC default IP was reachable + set_fact: + idrac_bootstrap_failure: "{{ ping_result.results[0] }}" + when: "{{ ping_result.results[0].rc != 0 }}" + + - name: Ensure IPMI is enabled on the iDRAC + command: > + ip netns exec {{ idrac_bootstrap_net_namespace }} + /opt/dell/srvadmin/bin/idracadm7 + -r {{ idrac_default_ip }} -u {{ idrac_default_username }} -p {{ idrac_default_password }} + set iDRAC.IPMILan.Enable 1 + become: True + with_items: + - "{{ idrac_bootstrap_controller }}" + loop_control: + loop_var: delegate_host + delegate_to: "{{ delegate_host }}" + when: "{{ not idrac_bootstrap_failure }}" + register: racadm_ipmi_enable + failed_when: False + + - name: Set a fact about whether enabling IPMI on the iDRAC failed + set_fact: + idrac_bootstrap_failure: "{{ racadm_ipmi_enable.results[0] }}" + when: + - "{{ not idrac_bootstrap_failure }}" + - "{{ racadm_ipmi_enable.results[0].rc != 0 }}" + + - name: Ensure the iDRAC IP address is configured + command: > + ip netns exec {{ idrac_bootstrap_net_namespace }} + /opt/dell/srvadmin/bin/idracadm7 + -r {{ idrac_default_ip }} -u {{ idrac_default_username }} -p {{ idrac_default_password }} + setniccfg -s {{ idrac_network_ip }} {{ idrac_network_netmask }} {{ idrac_network_gateway }} + become: True + with_items: + - "{{ idrac_bootstrap_controller }}" + loop_control: + loop_var: delegate_host + delegate_to: "{{ delegate_host }}" + when: + - "{{ not idrac_bootstrap_failure }}" + register: racadm_setniccfg + failed_when: False + + - name: Set a fact about whether setting network configuration on the iDRAC failed + set_fact: + idrac_bootstrap_failure: "{{ racadm_setniccfg.results[0] }}" + when: + - "{{ not idrac_bootstrap_failure }}" + - "{{ racadm_setniccfg.results[0].rc != 0 }}" + + - name: Append the iDRAC to the successful list on success + set_fact: + idrac_bootstrap_success: > + {{ idrac_bootstrap_success + [idrac_port_description] }} + when: "{{ not idrac_bootstrap_failure }}" + + - name: Append the iDRAC to the failed list on failure + set_fact: + idrac_bootstrap_failed: > + {{ idrac_bootstrap_failed + + [{"port description": idrac_port_description, + "failure": idrac_bootstrap_failure}] }} + when: "{{ idrac_bootstrap_failure }}" + run_once: True + + # Ensure we reconfigure the switch interface. + always: + - name: Ensure DellOS6 switch iDRAC interface is a member of the management VLAN + dellos6_config: + provider: "{{ dell_switch_provider }}" + lines: + - "switchport access vlan {{ idrac_network_vlan }}" + parents: + - "interface {{ switch_interface_name }}" + delegate_to: localhost + when: + - "{{ switch_type == 'dellos6' }}" + + when: "{{ idrac_bootstrap_required }}" + +- name: Append the iDRAC to the unchanged list when unchanged + set_fact: + idrac_bootstrap_unchanged: > + {{ idrac_bootstrap_unchanged + [idrac_port_description] }} + run_once: True + when: "{{ not idrac_bootstrap_required }}" diff --git a/ansible/idrac-bootstrap.yml b/ansible/idrac-bootstrap.yml new file mode 100644 index 000000000..0fa06b400 --- /dev/null +++ b/ansible/idrac-bootstrap.yml @@ -0,0 +1,219 @@ +--- +# This playbook is intended to bootstrap network configuration of the iDRACs on +# a set of Dell servers. By default, iDRACs have a known static IP address +# configured. Since all iDRACs have the same default IP address, we need a way +# to isolate a single iDRAC while we set its network configuration. We do this +# using a temporary VLAN accessible from one of the controllers. +# +# We use the following procedure: +# 1. Create a VLAN interface on the controller node with IP in the iDRAC +# default subnet. +# 2. Create the temporary bootstrap VLAN on the switch, accessible by the +# controller and trunked to all switches within the network. +# 3. For each iDRAC switch port in turn, flip to the temporary VLAN and +# configure the iDRAC's IP address, before returning the port to the iDRAC +# management VLAN. +# 4. Remove the temporary bootstrap VLAN from the switch. +# 5. Remove the VLAN interface on the controller node. + +- name: Ensure the iDRAC switches are supported + hosts: "{{ idrac_bootstrap_switch_group }}" + gather_facts: no + vars: + supported_switch_types: + - dellos6 + tasks: + - name: Ensure switch type is supported + fail: + msg: > + The iDRAC bootstrap process currently only supports DellOS6 switches. + when: "{{ switch_type not in supported_switch_types }}" + +# 1. Create a VLAN interface on the controller node with IP in the iDRAC +# default subnet. +- name: Ensure the controller bootstrap network is configured + hosts: controllers[0] + tasks: + # Install Dell server admin tools. + - block: + - name: Ensure wget is installed + yum: + name: wget + state: installed + + - name: Ensure Dell srvadmin repository is installed + shell: "wget -q -O - http://linux.dell.com/repo/hardware/latest/bootstrap.cgi | bash" + + - name: Ensure Dell srvadmin-idrac7 package is installed + yum: + name: srvadmin-idrac7 + state: installed + + # Configure access to the temporary network on a controller. + - block: + # Clear any previous state. + - name: Ensure iDRAC bootstrap network namespace is deleted from the controller + command: "ip netns delete {{ idrac_bootstrap_net_namespace }}" + args: + removes: "/var/run/netns/{{ idrac_bootstrap_net_namespace }}" + + - name: Ensure iDRAC bootstrap network namespace exists on controller + command: "ip netns add {{ idrac_bootstrap_net_namespace }}" + + - name: Ensure bootstrap VLAN interface exists on the controller + command: "ip link add link {{ idrac_bootstrap_controller_interface }} name {{ idrac_bootstrap_controller_vlan_interface }} type vlan id {{ idrac_bootstrap_vlan }}" + + - name: Ensure bootstrap VLAN interface is in network namespace + command: "ip link set {{ idrac_bootstrap_controller_vlan_interface }} netns {{ idrac_bootstrap_net_namespace }}" + + - name: Ensure the bootstrap VLAN interface is active + command: "ip netns exec {{ idrac_bootstrap_net_namespace }} ip link set {{ idrac_bootstrap_controller_vlan_interface }} up" + + - name: Ensure the bootstrap VLAN interface IP address is configured + command: "ip netns exec {{ idrac_bootstrap_net_namespace }} ip address add {{ idrac_bootstrap_controller_ip }}/24 dev {{ idrac_bootstrap_controller_vlan_interface }}" + rescue: + - name: Rescue | Ensure the bootstrap network namespace is removed from the controller + command: "ip netns delete {{ idrac_bootstrap_net_namespace }}" + + - name: Rescue | Fail playbook execution on error + fail: + msg: > + Failed to configure access to temporary iDRAC bootstrap + network on controller. + become: True + +# 2. Create the temporary bootstrap VLAN on the switch, accessible by the +# controller and trunked to all switches within the network. +- name: Ensure the bootstrap VLAN is configured on switches + hosts: "{{ idrac_bootstrap_switch_group }}" + gather_facts: no + vars: + switch_interface_config_bootstrap_trunk: + config: + - "switchport trunk allowed vlan add {{ idrac_bootstrap_vlan }}" + # Initialise the switch interface configuration. + switch_interface_config_bootstrap: {} + pre_tasks: + - name: Update facts about switch trunk interfaces + set_fact: + switch_interface_config_bootstrap: > + {{ switch_interface_config_bootstrap | combine({item.key: switch_interface_config_bootstrap_trunk}) }} + with_dict: "{{ switch_interface_config }}" + when: > + {{ item.value.description == groups['controllers'][0] or + item.value.description | replace('-trunk', '') in groups[idrac_bootstrap_switch_group] }} + + roles: + # Configure bootstrap VLAN on the switch and add controller and trunk + # interfaces to it. + - role: dell-switch + dell_switch_delegate_to: localhost + dell_switch_type: "{{ switch_type }}" + dell_switch_provider: "{{ switch_dellos_provider }}" + dell_switch_config: + - "vlan {{ idrac_bootstrap_vlan }}" + dell_switch_interface_config: "{{ switch_interface_config_bootstrap }}" + when: "{{ switch_interface_config_bootstrap != {} }}" + +# 3. For each iDRAC switch port in turn, flip to the temporary VLAN and +# configure the iDRAC's IP address, before returning the port to the iDRAC +# management VLAN. +- name: Ensure iDRACs are bootstrapped + hosts: "{{ idrac_bootstrap_switch_group }}" + gather_facts: no + # This is a separate play so that we can apply the serial keyword. + serial: 1 + tasks: + - name: Initialise facts containing successful, unchanged and failed iDRACs + set_fact: + idrac_bootstrap_success: [] + idrac_bootstrap_unchanged: [] + idrac_bootstrap_failed: [] + + # Iterate over each switch port with an iDRAC attached in turn. + - name: Ensure iDRACs are (sequentially) bootstrapped + include: idrac-bootstrap-one.yml + vars: + dell_switch_delegate_to: localhost + dell_switch_type: "{{ switch_type }}" + dell_switch_provider: "{{ switch_dellos_provider }}" + switch_interface_name: "{{ item.key }}" + idrac_port_description: "{{ item.value.description }}" + idrac_network_ip: "{{ idrac_network_ips[idrac_port_description] }}" + idrac_bootstrap_controller: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + with_dict: "{{ switch_interface_config }}" + when: "{{ item.value.description in idrac_network_ips }}" + +# 4. Remove the temporary bootstrap VLAN from the switch. +- name: Ensure the bootstrap VLAN is removed from switches + hosts: "{{ idrac_bootstrap_switch_group }}" + gather_facts: no + vars: + switch_interface_config_bootstrap_trunk: + config: + - "switchport trunk allowed vlan remove {{ idrac_bootstrap_vlan }}" + # Initialise the switch interface configuration. + switch_interface_config_bootstrap: {} + pre_tasks: + - name: Update facts about switch trunk interfaces + set_fact: + switch_interface_config_bootstrap: > + {{ switch_interface_config_bootstrap | combine({item.key: switch_interface_config_bootstrap_trunk}) }} + with_dict: "{{ switch_interface_config }}" + when: > + {{ item.value.description == groups['controllers'][0] or + item.value.description | replace('-trunk', '') in groups[idrac_bootstrap_switch_group] }} + roles: + # Remove bootstrap VLAN from the switch and remove controller and trunk + # interfaces from it. + - role: dell-switch + dell_switch_delegate_to: localhost + dell_switch_type: "{{ switch_type }}" + dell_switch_provider: "{{ switch_dellos_provider }}" + dell_switch_config: + - "no vlan {{ idrac_bootstrap_vlan }}" + dell_switch_interface_config: "{{ switch_interface_config_bootstrap }}" + when: "{{ switch_interface_config_bootstrap != {} }}" + +# 5. Remove the VLAN interface on the controller node. +- name: Ensure the controller bootstrap network is cleaned up + hosts: controllers[0] + tasks: + # This should also delete the network interface within the namespace. + - name: Ensure the bootstrap network namespace is removed from the controller + command: "ip netns delete {{ idrac_bootstrap_net_namespace }}" + become: True + +- name: Display the results of the iDRAC bootstrap procedure + hosts: "{{ idrac_bootstrap_switch_group }}" + gather_facts: no + tasks: + - name: Display a list of failed iDRACs + set_fact: + idrac_bootstrap_failed_port_descriptions: "{{ idrac_bootstrap_failed | map(attribute='port description') | list }}" + when: "{{ idrac_bootstrap_failed | length > 0 }}" + + - name: Display a list of successfully bootstrapped iDRACs + debug: + var: idrac_bootstrap_success + + - name: Display a list of iDRACs that did not require bootstrapping + debug: + var: idrac_bootstrap_unchanged + + - name: Display a list of failed iDRACs + debug: + var: idrac_bootstrap_failed_port_descriptions + when: "{{ idrac_bootstrap_failed | length > 0 }}" + + - name: Display a list of failed iDRACs with debug output for the failed tasks + debug: + var: idrac_bootstrap_failed + when: "{{ idrac_bootstrap_failed | length > 0 }}" + + - name: Fail if there were any iDRAC bootstrapping failures + fail: + msg: > + One or more iDRACs failed to bootstrap, see the list above for + details. + when: "{{ idrac_bootstrap_failed | length > 0 }}" diff --git a/etc/kayobe/idrac.yml b/etc/kayobe/idrac.yml new file mode 100644 index 000000000..246f7076c --- /dev/null +++ b/etc/kayobe/idrac.yml @@ -0,0 +1,43 @@ +--- +############################################################################### +# iDRAC configuration. + +# Default username for iDRACs. +#idrac_default_username: + +# Default password for iDRACs. +#idrac_default_password: + +# Default IP address for iDRACs. +#idrac_default_ip: + +# Dict mapping host description (as found in switch interface configuration) to +# IP address of the iDRAC for that host. +#idrac_network_ips: + +# Gateway IP address for iDRAC network. +#idrac_network_gateway: + +# IP netmask for iDRAC network. +#idrac_network_netmask: + +# VLAN for iDRAC network. +#idrac_network_vlan: + +# ID of VLAN to use for bootstrapping iDRACs. +#idrac_bootstrap_vlan: + +# Name of network namespace on controller to use for bootstrapping iDRACs. +#idrac_bootstrap_net_namespace: + +# Base network interface on controller to use for bootstrapping iDRACs. +#idrac_bootstrap_controller_interface: + +# VLAN network interface on controller to create for bootstrapping iDRACs. +#idrac_bootstrap_controller_vlan_interface: + +# IP address of controller to use for bootstrapping iDRACs. +#idrac_bootstrap_controller_ip: + +# Name of an Ansible group containing switches forming the iDRAC network. +#idrac_bootstrap_switch_group: