[CI] Use Tenks in Ironic job

Installs Tenks [1] and uses it to create virtual machines to pose
as bare metal compute nodes.
The nodes are registered in Ironic, and used to provision
instances.

[1] https://docs.openstack.org/tenks/latest/

Depends-On: https://review.opendev.org/c/openstack/tenks/+/830182
Depends-On: https://review.opendev.org/c/openstack/tenks/+/830675
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/831055
Change-Id: Idfb8fbb50dc7442225967b2a2ec38ae7114f3c11
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>

tests/deploy-tenks.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+function deploy_tenks_logged {
+    . /etc/kolla/admin-openrc.sh
+
+    echo "Creating IPA images for Ironic"
+    ~/openstackclient-venv/bin/openstack image create --disk-format aki --container-format aki --private \
+        --file /etc/kolla/config/ironic/ironic-agent.kernel ipa.vmlinuz
+    ~/openstackclient-venv/bin/openstack image create --disk-format ari --container-format ari --private \
+        --file /etc/kolla/config/ironic/ironic-agent.initramfs ipa.initramfs
+
+    # Install a trivial script for ovs-vsctl that talks to containerised Open
+    # vSwitch.
+    sudo tee /usr/bin/ovs-vsctl >/dev/null <<EOF
+#!/usr/bin/env bash
+
+# Script installed onto the host to fool tenks into using the containerised
+# Open vSwitch rather than installing its own.
+
+sudo docker exec openvswitch_vswitchd ovs-vsctl "\$@"
+EOF
+    sudo chmod 755 /usr/bin/ovs-vsctl
+
+    # Install the Tenks venv.
+    python3 -m venv ${TENKS_VENV_PATH}
+    ${TENKS_VENV_PATH}/bin/pip install -U pip
+    ${TENKS_VENV_PATH}/bin/pip install ${TENKS_SRC_PATH}
+
+    local attempt
+    attempt=1
+    while ! ${TENKS_VENV_PATH}/bin/ansible-galaxy install \
+            --role-file="${TENKS_SRC_PATH}/requirements.yml" \
+            --roles-path="${TENKS_SRC_PATH}/ansible/roles/"; do
+        echo "ansible-galaxy install failed, attempt $attempt"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 10 ]]; then
+            echo "enough retrying, farewell"
+            return 1
+        fi
+        sleep 10
+    done
+
+    # Run Tenks.
+    ${TENKS_VENV_PATH}/bin/ansible-playbook \
+        -vvv \
+        --inventory "${TENKS_SRC_PATH}/ansible/inventory" \
+        --extra-vars=@"$HOME/tenks.yml" \
+        "${TENKS_SRC_PATH}/ansible/deploy.yml"
+}
+
+function deploy_tenks {
+    echo "Configuring virtual bare metal via Tenks"
+    deploy_tenks_logged > /tmp/logs/ansible/deploy-tenks 2>&1
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Deploying tenks failed. See ansible/deploy-tenks for details"
+    else
+        echo "Successfully deployed tenks. See ansible/deploy-tenks for details"
+    fi
+    return $result
+}
+
+deploy_tenks

tests/run.yml

@@ -74,7 +74,7 @@
       when:
         # NOTE(yoctozepto): build container images if there is any tested
         # change that impacts them.
-        - item.project.short_name not in ["ansible-collection-kolla", "kayobe", "kolla-ansible"]
+        - item.project.short_name not in ["ansible-collection-kolla", "kayobe", "kolla-ansible", "tenks"]
       with_items: "{{ zuul['items'] }}"
 
     # NOTE(yoctozepto): required to template template_overrides.j2 for Zuul
@@ -178,7 +178,9 @@
         - src: "tests/templates/ironic-overrides.j2"
           dest: /etc/kolla/config/ironic.conf
           when: "{{ scenario == 'ironic' }}"
-
+        - src: "tests/templates/tenks-deploy-config.yml.j2"
+          dest: "{{ ansible_env.HOME }}/tenks.yml"
+          when: "{{ scenario == 'ironic' }}"
       when: item.when | default(true)
 
     - block:
@@ -449,13 +451,23 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "scenario_nfv"
 
-        - name: Run test-ironic.sh script
+        - block:
-          script:
+            - name: Run deploy-tenks.sh script
-            cmd: test-ironic.sh
+              script:
-            executable: /bin/bash
+                cmd: deploy-tenks.sh
-            chdir: "{{ kolla_ansible_src_dir }}"
+                executable: /bin/bash
-          environment:
+                chdir: "{{ kolla_ansible_src_dir }}"
-            TLS_ENABLED: "{{ tls_enabled }}"
+              environment:
+                TENKS_VENV_PATH: "{{ ansible_env.HOME }}/tenks-venv"
+                TENKS_SRC_PATH: "{{ ansible_env.HOME }}/src/opendev.org/openstack/tenks"
+
+            - name: Run test-ironic.sh script
+              script:
+                cmd: test-ironic.sh
+                executable: /bin/bash
+                chdir: "{{ kolla_ansible_src_dir }}"
+              environment:
+                TLS_ENABLED: "{{ tls_enabled }}"
           when: scenario == "ironic"
 
         - name: Run test-magnum.sh script

tests/templates/ironic-overrides.j2

@@ -1,19 +1,3 @@
-[DEFAULT]
-# Enable all fake hardware types and interfaces.
-enabled_hardware_types = fake-hardware
-enabled_boot_interfaces = fake
-enabled_console_interfaces = ipmitool-socat,no-console
-enabled_deploy_interfaces = fake
-enabled_inspect_interfaces = inspector,no-inspect
-enabled_management_interfaces = fake
-enabled_network_interfaces = noop,flat,neutron
-default_network_interface = neutron
-enabled_power_interfaces = fake
-enabled_raid_interfaces = agent,no-raid
-default_raid_interface = no-raid
-enabled_rescue_interfaces = fake
-enabled_vendor_interfaces = no-vendor
-
 [neutron]
 cleaning_network = public1
 provisioning_network = public1

tests/templates/tenks-deploy-config.yml.j2

@@ -0,0 +1,55 @@
+---
+# This file holds the config given to Tenks when running `deploy-tenks.sh`.
+
+node_types:
+  type0:
+    memory_mb: 1024
+    vcpus: 1
+    volumes:
+      # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
+      # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
+      - capacity: 4GiB
+    physical_networks:
+      - physnet1
+    console_log_enabled: true
+    # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g:
+    # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38
+    cpu_mode:
+specs:
+  - type: type0
+    count: 1
+    ironic_config:
+      resource_class: test-rc
+      network_interface: flat
+
+nova_flavors:
+  - resource_class: test-rc
+    node_type: type0
+
+physnet_mappings:
+  physnet1: {{ neutron_external_bridge_name }}
+
+deploy_kernel: ipa.vmlinuz
+deploy_ramdisk: ipa.initramfs
+
+default_boot_mode: "bios"
+
+# Use the libvirt daemon deployed by Kolla Ansible in the nova_libvirt
+# container. Tenks will install libvirt client packages.
+libvirt_host_install_daemon: false
+
+# Nested virtualisation is not working well in CI currently. Force the use of
+# QEMU.
+libvirt_vm_engine: "qemu"
+
+# QEMU may not be installed on the host, so set the path and avoid
+# autodetection.
+libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}"
+
+# Specify a log path in the kolla_logs Docker volume. It is accessible on the
+# host at the same path.
+libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks"
+
+# Console logs are owned by the ID of the Nova user in the nova_libvirt
+# container.
+libvirt_vm_log_owner: 42436

tests/test-ironic.sh

@@ -7,134 +7,48 @@ set -o pipefail
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
-# Adapted from the function of the same name in the ironic devstack plugin.
-function wait_for_placement_resources {
-    # After nodes have been enrolled, we need to wait for both ironic and
-    # nova's periodic tasks to populate the resource tracker with available
-    # nodes and resources. Wait up to 2 minutes for a given resource before
-    # timing out.
-    local expected_count=1
-    local resource_class="RC0"
-
-    curl --fail -L -o jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
-    chmod +x jq
-
-    # TODO(mgoddard): switch to Placement OSC plugin, once it exists
-    local token
-    token=$(openstack token issue -f value -c id)
-    local endpoint
-    endpoint=$(openstack endpoint list --service placement --interface public -f value -c URL)
-    if [[ -z $endpoint ]]; then
-        echo "Cannot find Placement API endpoint"
-        return 1
-    fi
-
-    local i
-    local count
-    echo "Waiting 2 minutes for Nova resource tracker to pick up $expected_count nodes"
-    for i in $(seq 1 120); do
-        # Fetch provider UUIDs from Placement
-        local providers
-        args=(
-            --fail
-            -sH "X-Auth-Token: $token"
-        )
-        if [[ "$TLS_ENABLED" = "True" ]]; then
-            args+=(--cacert $OS_CACERT)
-        fi
-        providers=$(curl "${args[@]}" $endpoint/resource_providers \
-            | ./jq -r '.resource_providers[].uuid')
-
-        local p
-        # Total count of the resource class, has to be equal to nodes count
-        count=0
-        for p in $providers; do
-            local amount
-            # A resource class inventory record looks something like
-            # {"max_unit": 1, "min_unit": 1, "step_size": 1, "reserved": 0, "total": 1, "allocation_ratio": 1}
-            # Subtract reserved from total (defaulting both to 0)
-            amount=$(curl "${args[@]}" $endpoint/resource_providers/$p/inventories \
-                | ./jq ".inventories.CUSTOM_$resource_class as \$cls
-                    | (\$cls.total // 0) - (\$cls.reserved // 0)")
-            if [ $amount -gt 0 ]; then
-                count=$(( count + $amount ))
-            fi
-        done
-
-        if [ $count -ge $expected_count ]; then
-            return 0
-        fi
-        sleep 1
-    done
-
-    echo "Timed out waiting for Nova to track $expected_count nodes"
-    return 1
-}
-
-function create_resources {
-    # Create a bare metal node and port.
-    openstack baremetal node create \
-        --name node-0 \
-        --driver fake-hardware \
-        --network-interface noop \
-        --property cpu_arch=x86_64 \
-        --resource-class rc0
-    node_uuid=$(openstack baremetal node show node-0 -f value -c uuid)
-    openstack baremetal port create \
-        00:11:22:33:44:55 \
-        --node $node_uuid
-    openstack baremetal node power off node-0
-    openstack baremetal node manage node-0 --wait
-    openstack baremetal node provide node-0 --wait
-
-    # Create a bare metal flavor in nova.
-    openstack flavor create \
-        baremetal \
-        --vcpus 1 \
-        --ram 1024 \
-        --disk 10 \
-        --property resources:CUSTOM_RC0=1 \
-        --property resources:VCPU=0 \
-        --property resources:MEMORY_MB=0 \
-        --property resources:DISK_GB=0 \
-        --public
-}
-
 function test_ironic_logged {
     # Assumes init-runonce has been executed.
     . /etc/kolla/admin-openrc.sh
     . ~/openstackclient-venv/bin/activate
 
+    echo "Enabling DHCP on the external (\"public\") subnet"
+    openstack subnet set --dhcp public1-subnet
+
     # Smoke test ironic API.
-    local baremetal_driver_list
+    openstack baremetal driver list
-    baremetal_driver_list=$(openstack baremetal driver list)
     openstack baremetal node list
     openstack baremetal port list
     # Ironic Inspector API
     openstack baremetal introspection rule list
 
-    # Sanity check.
+    openstack baremetal node show tk0
-    if ! echo "$baremetal_driver_list" | grep fake-hardware; then
+    openstack baremetal node power off tk0
-        echo "No active conductors with fake-hardware driver"
+    openstack baremetal node show tk0
-        exit 1
+    openstack baremetal node manage tk0
-    fi
+    openstack baremetal node show tk0
-
+    openstack baremetal node provide tk0
-    create_resources
+    openstack baremetal node show tk0
-    wait_for_placement_resources
+    openstack baremetal node validate tk0
 
     echo "TESTING: Server creation"
-    openstack server create --wait --image cirros --flavor baremetal --key-name mykey --network demo-net kolla_boot_test
+    openstack server create --image cirros --flavor test-rc --key-name mykey --network public1 kolla_bm_boot_test
-    openstack --debug server list
+    local attempt
-    # If the status is not ACTIVE, print info and exit 1
+    attempt=1
-    if [[ $(openstack server show kolla_boot_test -f value -c status) != "ACTIVE" ]]; then
+    while [[ $(openstack server show kolla_bm_boot_test -f value -c status) != "ACTIVE" ]]; do
-        echo "FAILED: Instance is not active"
+        echo "Server not yet active, check $attempt - retrying"
-        openstack --debug server show kolla_boot_test
+        attempt=$((attempt+1))
-        return 1
+        if [[ $attempt -eq 16 ]]; then
-    fi
+            echo "FAILED: Server did not become active after $attempt checks"
+            openstack server show kolla_bm_boot_test
+            return 1
+        fi
+        sleep 60
+    done
     echo "SUCCESS: Server creation"
 
     echo "TESTING: Server deletion"
-    openstack server delete --wait kolla_boot_test
+    openstack server delete --wait kolla_bm_boot_test
     echo "SUCCESS: Server deletion"
 }
 

zuul.d/base.yaml

@@ -83,8 +83,13 @@
     voting: false
     files:
       - ^ansible/roles/(ironic|neutron|nova|nova-cell)/
-      - ^tests/test-ironic.sh
+      - ^tests/deploy-tenks\.sh$
-      - ^tests/test-dashboard.sh
+      - ^tests/templates/ironic-overrides\.j2$
+      - ^tests/templates/tenks-deploy-config\.yml\.j2$
+      - ^tests/test-dashboard\.sh$
+      - ^tests/test-ironic\.sh$
+    required-projects:
+      - openstack/tenks
     vars:
       scenario: ironic