From a242c96970c7264a2c7e1ec4b7e6a5a6d5b37671 Mon Sep 17 00:00:00 2001
From: Tony Breeds <tony@bakeyournoodle.com>
Date: Thu, 15 May 2025 15:05:36 -0500
Subject: [PATCH] Add new openstack/devstack based functional testing

A few differences from the nodepool based testsing:
 * Don't set ELEMENTS_PATH in default_diskimage_environment

    Setting the ELEMENTS_PATH isn't helpful as it generates a number of
    warnings like:

    Element </home/zuul/src/opendev.org/openstack/diskimage-builder/diskimage_builder/elements/block-device-gpt>
    overrides </home/zuul/dib/lib/python3.12/site-packages/diskimage_builder/elements/block-device-gpt>

    We install DIB from the source via the ensure-dib role so the elements
    in site-packages will be installed from source anyway.
 * Do not use opendev opensuse mirror

    We removed this in Ib5d15ce800ff0620187345e1cfec0b7b5d65bee5 which is
    causing the build to fail in nodepool anyway.   It's possible this is
    masking another deeper error but let's find out.
 * The gentoo build works in nodepool, but not under devstack.  The gentoo
   image being built runs out of space.

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
Change-Id: If989c1b04963348fd16121c1c698b75e0e94b22a
---
 .zuul.d/jobs.yaml                            | 285 +++++++++++++++++++
 .zuul.d/project.yaml                         |  30 ++
 playbooks/dib-devstack/build-boot-run.yaml   | 105 +++++++
 playbooks/dib-devstack/files/cpu-level.sh    |  34 +++
 playbooks/dib-devstack/pre.yaml              |  18 ++
 playbooks/dib-nodepool/node-information.yaml |   2 +-
 tools/functional-test-check.sh               | 127 +++++++++
 7 files changed, 600 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/dib-devstack/build-boot-run.yaml
 create mode 100644 playbooks/dib-devstack/files/cpu-level.sh
 create mode 100644 playbooks/dib-devstack/pre.yaml
 create mode 100755 tools/functional-test-check.sh

diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml
index 559a32c3e..4aedd0533 100644
--- a/.zuul.d/jobs.yaml
+++ b/.zuul.d/jobs.yaml
@@ -73,6 +73,62 @@
 # Build and boot functional tests
 #
 
+- job:
+    name: dib-devstack-functional-src-base
+    abstract: true
+    description: |
+      Base job for DIB functional tests, which build and boot an image
+      under devstack, using source versions of dependencies such as
+      devstack, dib (of course) and glean.
+    parent: devstack
+    pre-run: playbooks/dib-devstack/pre.yaml
+    run: playbooks/dib-devstack/build-boot-run.yaml
+    required-projects:
+      - name: opendev.org/openstack/devstack
+      - name: opendev.org/openstack/diskimage-builder
+      - name: opendev.org/opendev/glean
+      - name: opendev.org/openstack/requirements
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+    vars:
+      default_elements:
+        - vm
+        - simple-init
+        - growroot
+        - devuser
+        - openssh-server
+        - journal-to-console
+        - openstack-ci-mirrors
+      build_diskimage_image_name: "{{ diskimage.base_element }}"
+      build_diskimage_formats:
+        - raw
+      build_diskimage_image_root: "{{ ansible_user_dir }}/dib_tmp/dib-images"
+      build_diskimage_elements: >-
+        {{ ( [diskimage.base_element]
+            + default_elements
+            + diskimage.additional_elements|default([]) ) | flatten }}
+      default_diskimage_environment:
+        DIB_CHECKSUM: '1'
+        DIB_DEBUG_TRACE: 1
+        DIB_INSTALLTYPE_simple_init: 'repo'
+        DIB_REPOLOCATION_glean: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/opendev/glean'].src_dir }}"
+        DIB_REPOREF_glean: "{{ zuul.projects['opendev.org/opendev/glean'].checkout }}"
+        DIB_RELEASE: "{{ diskimage.release }}"
+        DIB_SHOW_IMAGE_USAGE: '1'
+        ROOT_LABEL: 'gate-rootfs'
+        TMPDIR: "{{ ansible_user_dir }}/dib_tmp"
+      build_diskimage_environment: "{{ default_diskimage_environment|combine(diskimage['env-vars']|default({})) }}"
+      # NOTE(tonyb):  For the time being we need to disable this as the cli-server always uses the
+      #               'devstack-admin' cloud and therefore the private network ends in the wrong
+      #               tennant and causes failures
+      devstack_services:
+        openstack-cli-server: false
+      devstack_localrc:
+        SWIFT_MAX_FILE_SIZE: 5368709122 # 5G + 2 bytes (swift default)
+        SWIFT_LOOPBACK_DISK_SIZE: "6G"  # Alittle more than ^^
+
 - job:
     name: dib-nodepool-functional-src-base
     description: |
@@ -102,6 +158,17 @@
         base_element: almalinux-container
         release: '8'
 
+- job:
+    name: dib-devstack-functional-openstack-almalinux-8-containerfile-src
+    description: |
+      Test building and booting a AlmaLinux image from a
+      containerfile with devstack and OpenStack (from git)
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: almalinux-container
+        release: '8'
+
 - job:
     name: dib-nodepool-functional-openstack-almalinux-9-containerfile-src
     description: |
@@ -115,6 +182,18 @@
         base_element: almalinux-container
         release: '9'
 
+- job:
+    name: dib-devstack-functional-openstack-almalinux-9-containerfile-src
+    description: |
+      Test building and booting a AlmaLinux image from a
+      containerfile with devstack and OpenStack (using released
+      libraries)
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: almalinux-container
+        release: '9'
+
 - job:
     name: dib-nodepool-functional-openstack-centos-8-stream-src
     description: |
@@ -128,6 +207,17 @@
         # TODO: set when CentOS 8 Stream repos mirrored.
         # mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
 
+- job:
+    name: dib-devstack-functional-openstack-centos-8-stream-src
+    description: |
+      Test building and booting a Centos 8 Stream image with devstack and
+      OpenStack.
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: centos-minimal
+        release: '8-stream'
+
 - job:
     name: dib-nodepool-functional-openstack-centos-9-stream-src
     description: |
@@ -141,6 +231,17 @@
         # TODO: set when CentOS 9 Stream repos mirrored.
         # mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
 
+- job:
+    name: dib-devstack-functional-openstack-centos-9-stream-src
+    description: |
+      Test building and booting a Centos 9 Stream image with devsatck and
+      OpenStack.
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: centos-minimal
+        release: '9-stream'
+
 - job:
     name: dib-nodepool-functional-openstack-fedora-37-containerfile-src
     description: |
@@ -154,6 +255,18 @@
         base_element : fedora-container
         release: '37'
 
+- job:
+    name: dib-devstack-functional-openstack-fedora-37-containerfile-src
+    description: |
+      Test building and booting an Fedora image from a
+      containerfile with devstack and OpenStack (using released
+      libraries)
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element : fedora-container
+        release: '37'
+
 - job:
     name: dib-nodepool-functional-openstack-rockylinux-8-containerfile-src
     description: |
@@ -167,6 +280,18 @@
         base_element: rocky-container
         release: '8'
 
+- job:
+    name: dib-devstack-functional-openstack-rockylinux-8-containerfile-src
+    description: |
+      Test building and booting a Rocky Linux image from a
+      containerfile with devstack and OpenStack (using released
+      libraries)
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: rocky-container
+        release: '8'
+
 - job:
     name: dib-nodepool-functional-openstack-rockylinux-9-containerfile-src
     description: |
@@ -180,6 +305,18 @@
         base_element: rocky-container
         release: '9'
 
+- job:
+    name: dib-devstack-functional-openstack-rockylinux-9-containerfile-src
+    description: |
+      Test building and booting a Rocky Linux image from a
+      containerfile with devstack and OpenStack (using released
+      libraries)
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: rocky-container
+        release: '9'
+
 - job:
     name: dib-nodepool-functional-openstack-ubuntu-xenial-src
     description: |
@@ -196,6 +333,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-ubuntu-xenial-src
+    description: |
+      Test building and booting an Ubuntu xenial image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: ubuntu-minimal
+        release: 'xenial'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/ubuntu"
+
 - job:
     name: dib-nodepool-functional-openstack-ubuntu-bionic-src
     description: |
@@ -212,6 +365,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-ubuntu-bionic-src
+    description: |
+      Test building and booting an Ubuntu bionic image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: ubuntu-minimal
+        release: 'bionic'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/ubuntu"
+
 - job:
     name: dib-nodepool-functional-openstack-ubuntu-focal-src
     description: |
@@ -228,6 +397,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-ubuntu-focal-src
+    description: |
+      Test building and booting an Ubuntu focal image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: ubuntu-minimal
+        release: 'focal'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/ubuntu"
+
 - job:
     name: dib-nodepool-functional-openstack-ubuntu-jammy-src
     description: |
@@ -244,6 +429,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-ubuntu-jammy-src
+    description: |
+      Test building and booting an Ubuntu jammy image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: ubuntu-minimal
+        release: 'jammy'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/ubuntu"
+
 - job:
     name: dib-nodepool-functional-openstack-ubuntu-noble-src
     description: |
@@ -260,6 +461,21 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-ubuntu-noble-src
+    description: |
+      Test building and booting Ubuntu Noble with devstack.
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: ubuntu-minimal
+        release: 'noble'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/ubuntu"
+
 - job:
     name: dib-nodepool-functional-openstack-gentoo-src
     description: |
@@ -270,6 +486,16 @@
         base_element: gentoo
         release: ''
 
+- job:
+    name: dib-devstack-functional-openstack-gentoo-src
+    description: |
+      Test building and booting a Gentoo image with devstack and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: gentoo
+        release: ''
+
 - job:
     name: dib-nodepool-functional-openstack-debian-stretch-src
     description: |
@@ -286,6 +512,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-debian-stretch-src
+    description: |
+      Test building and booting a Debian stretch image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: debian-minimal
+        release: 'stretch'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/debian"
+
 - job:
     name: dib-nodepool-functional-openstack-debian-bullseye-src
     description: |
@@ -302,6 +544,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-debian-bullseye-src
+    description: |
+      Test building and booting a Debian bullseye image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: debian-minimal
+        release: 'bullseye'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/debian"
+
 - job:
     name: dib-nodepool-functional-openstack-debian-buster-src
     description: |
@@ -318,6 +576,22 @@
           DIB_DISABLE_APT_CLEANUP: '1'
           DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
 
+- job:
+    name: dib-devstack-functional-openstack-debian-buster-src
+    description: |
+      Test building and booting a Debian buster image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: debian-minimal
+        release: 'buster'
+        env-vars:
+          DIB_APT_LOCAL_CACHE: '0'
+          DIB_DISABLE_APT_CLEANUP: '1'
+          DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+          DIB_DISTRIBUTION_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/debian"
+
 - job:
     name: dib-nodepool-functional-openstack-opensuse-15-src
     description: |
@@ -329,3 +603,14 @@
         base_element: opensuse-minimal
         release: '15.2'
         mirror: "http://{{ zuul_site_mirror_fqdn }}/opensuse"
+
+- job:
+    name: dib-devstack-functional-openstack-opensuse-15-src
+    description: |
+      Test building and booting an openSUSE 15.x image with devstack
+      and OpenStack
+    parent: dib-devstack-functional-src-base
+    vars:
+      diskimage:
+        base_element: opensuse-minimal
+        release: '15.2'
diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml
index 532722124..e52552175 100644
--- a/.zuul.d/project.yaml
+++ b/.zuul.d/project.yaml
@@ -41,30 +41,52 @@
         - nodepool-build-image-siblings
         - dib-nodepool-functional-openstack-almalinux-8-containerfile-src:
             voting: false
+        - dib-devstack-functional-openstack-almalinux-8-containerfile-src:
+            voting: false
         - dib-nodepool-functional-openstack-almalinux-9-containerfile-src:
             voting: false
+        - dib-devstack-functional-openstack-almalinux-9-containerfile-src:
+            voting: false
         - dib-nodepool-functional-openstack-centos-9-stream-src
+        - dib-devstack-functional-openstack-centos-9-stream-src
         # NOTE(rpittau) Fedora 37 is EOL, moving to non-voting for the time being
         # until we move to Fedora 39 or more recent.
         - dib-nodepool-functional-openstack-fedora-37-containerfile-src:
             voting: false
+        - dib-devstack-functional-openstack-fedora-37-containerfile-src:
+            voting: false
         - dib-nodepool-functional-openstack-rockylinux-8-containerfile-src:
             voting: false
+        - dib-devstack-functional-openstack-rockylinux-8-containerfile-src:
+            voting: false
         - dib-nodepool-functional-openstack-rockylinux-9-containerfile-src:
             voting: false
+        - dib-devstack-functional-openstack-rockylinux-9-containerfile-src:
+            voting: false
         - dib-nodepool-functional-openstack-ubuntu-xenial-src
+        - dib-devstack-functional-openstack-ubuntu-xenial-src
         - dib-nodepool-functional-openstack-ubuntu-bionic-src
+        - dib-devstack-functional-openstack-ubuntu-bionic-src
         - dib-nodepool-functional-openstack-ubuntu-focal-src
+        - dib-devstack-functional-openstack-ubuntu-focal-src
         - dib-nodepool-functional-openstack-ubuntu-jammy-src
+        - dib-devstack-functional-openstack-ubuntu-jammy-src
         - dib-nodepool-functional-openstack-ubuntu-noble-src
+        - dib-devstack-functional-openstack-ubuntu-noble-src
         # NOTE(ianw): 2020-04-22 these have been out of the gate for a
         # while; promote to voting after some stability.
         - dib-nodepool-functional-openstack-opensuse-15-src:
             voting: false
+        - dib-devstack-functional-openstack-opensuse-15-src:
+            voting: false
         - dib-nodepool-functional-openstack-gentoo-src:
             voting: false
+        - dib-devstack-functional-openstack-gentoo-src:
+            voting: false
         - dib-nodepool-functional-openstack-debian-bullseye-src:
             voting: false
+        - dib-devstack-functional-openstack-debian-bullseye-src:
+            voting: false
         - ironic-python-agent-check-image-dib-centos9
         # IPA job that builds AND validates an image via deployment
         - ipa-tempest-bios-ipmi-direct-src:
@@ -109,11 +131,17 @@
         - opendev-buildset-registry
         - nodepool-build-image-siblings
         - dib-nodepool-functional-openstack-centos-9-stream-src
+        - dib-devstack-functional-openstack-centos-9-stream-src
         - dib-nodepool-functional-openstack-ubuntu-xenial-src
+        - dib-devstack-functional-openstack-ubuntu-xenial-src
         - dib-nodepool-functional-openstack-ubuntu-bionic-src
+        - dib-devstack-functional-openstack-ubuntu-bionic-src
         - dib-nodepool-functional-openstack-ubuntu-focal-src
+        - dib-devstack-functional-openstack-ubuntu-focal-src
         - dib-nodepool-functional-openstack-ubuntu-jammy-src
+        - dib-devstack-functional-openstack-ubuntu-jammy-src
         - dib-nodepool-functional-openstack-ubuntu-noble-src
+        - dib-devstack-functional-openstack-ubuntu-noble-src
         - dib-functests
         - ironic-python-agent-check-image-dib-centos9
     experimental:
@@ -121,4 +149,6 @@
         - opendev-buildset-registry
         - nodepool-build-image-siblings
         - dib-nodepool-functional-openstack-debian-stretch-src
+        - dib-devstack-functional-openstack-debian-stretch-src
         - dib-nodepool-functional-openstack-debian-buster-src
+        - dib-devstack-functional-openstack-debian-buster-src
diff --git a/playbooks/dib-devstack/build-boot-run.yaml b/playbooks/dib-devstack/build-boot-run.yaml
new file mode 100644
index 000000000..93239d872
--- /dev/null
+++ b/playbooks/dib-devstack/build-boot-run.yaml
@@ -0,0 +1,105 @@
+- hosts: all
+  gather_facts: false
+  vars:
+    dib_src_path: "{{ ansible_user_dir }}/{{ zuul['projects']['opendev.org/openstack/diskimage-builder']['src_dir'] }}"
+    DIB_KEY_NAME: "root"
+    DIB_SSH_KEY: "{{ ansible_user_dir }}/.ssh/id_dib"
+    DIB_SSH_PUBKEY: "{{ ansible_user_dir }}/.ssh/id_dib.pub"
+  roles:
+    - name: ensure-dib
+    - name: build-diskimage
+  tasks:
+  - name: Add image to devstack as test-image
+    command: |-
+      openstack --os-cloud devstack image create \
+        --disk-format '{{ build_diskimage_formats[0] }}' --container-format bare \
+        --file '{{ build_diskimage_image_root }}/{{ diskimage.base_element }}.{{ build_diskimage_formats[0] }}' \
+        test-image
+
+  - name: Create DIB flavor
+    args:
+      executable: /bin/bash
+    shell: |
+      openstack --os-cloud devstack-admin flavor create --ram=1024 --disk=5 --vcpus=1 --id=128 dib1024
+
+  - name: Create security groups
+    args:
+      executable: /bin/bash
+    shell: |
+      openstack --os-cloud devstack security group rule create --ingress --protocol tcp --dst-port 1:65535 --remote-ip 0.0.0.0/0 default
+      openstack --os-cloud devstack security group rule create --ingress --protocol udp --dst-port 1:65535 --remote-ip 0.0.0.0/0 default
+
+  - name: Create nodepool SSH keypair
+    args:
+      executable: /bin/bash
+    shell: |
+      ssh-keygen -f {{ DIB_SSH_KEY }} -P ""
+      openstack --os-cloud=devstack keypair create --public-key="{{ DIB_SSH_PUBKEY }}" "{{ DIB_KEY_NAME }}"
+
+  - name: Create a floating IP
+    register: floating_ip
+    command: openstack --os-cloud devstack floating ip create public -f json
+
+  - name: Boot test-image in devstack as test-server
+    command: |-
+      openstack --os-cloud devstack server create \
+        --flavor dib1024 \
+        --image test-image \
+        --use-config-drive \
+        --key-name "{{ DIB_KEY_NAME }}" \
+        --network private \
+        test-server
+
+  # Give it, up to, another 5 minutes to boot or fail
+  - name: Wait for server to boot or fail
+    command: openstack --os-cloud devstack server show -f json test-server
+    register: server_status
+    until: server_status.stdout|from_json|json_query("status") == "ACTIVE"
+    retries: 100
+    delay: 3
+
+  - name: Convert test-server details from JSON
+    set_fact:
+      server_status: "{{ server_status.stdout|from_json }}"
+      floating_ip: "{{ floating_ip.stdout|from_json }}"
+
+  - name: Extract IP addresses from server details
+    set_fact:
+      access_ipv4: "{{ server_status|json_query('addresses.private')| ansible.utils.ipv4 | join(' ') }}"
+      # TODO: Uncomment and add IPv6, hostname, routing and ssh-keyscan.
+      # access_ipv6: "{{ server_status|json_query('addresses.private')| ansible.utils.ipv6 | join(' ') }}"
+
+  - name: Add floating IP to test-server
+    command: openstack --os-cloud devstack server add floating ip --fixed-ip {{ access_ipv4 }} test-server {{ floating_ip.floating_ip_address }}
+
+  - name: Remove any test-server entries from /etc/hosts
+    become: true
+    lineinfile:
+      regex: ".*test-server.*"
+      path: /etc/hosts
+      state: absent
+
+  - name: Add entries for test-server entries to /etc/hosts
+    become: true
+    lineinfile:
+      path: /etc/hosts
+      line: "{{ floating_ip.floating_ip_address }}\t\ttest-server"
+
+  - name: Check for ssh connectivity with log collection on failure
+    block:
+      # Wait a total of 20 mins, this is fairly high as we're probably running in "usermode emulation" QEMU+TCG
+      - name: Wait for SSH to come online
+        command: ssh-keyscan -4 test-server
+        register: keyscan
+        until: keyscan.rc == 0
+        retries: 120
+        delay: 10
+    rescue:
+      - name: Print console logs
+        command: openstack --os-cloud devstack console log show test-server
+      - name: Force failure now that we're in the rescue block
+        fail:
+          msg: SSH connectivity failed
+
+  - name: Run functional tests
+    command: "{{ dib_src_path }}/tools/functional-test-check.sh"
diff --git a/playbooks/dib-devstack/files/cpu-level.sh b/playbooks/dib-devstack/files/cpu-level.sh
new file mode 100644
index 000000000..a6c4c0cf9
--- /dev/null
+++ b/playbooks/dib-devstack/files/cpu-level.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+function report_cpu_flags() {
+    local flag
+
+    for flag; do
+        ## Note, it's important to keep a trailing space
+        case " ${flags} " in
+            *" ${flag} "*)
+                echo "  ${flag} Found"
+                ;;
+            *)
+                echo "  ${flag} NOT Found"
+                ;;
+        esac
+    done
+}
+
+data=$(< /proc/cpuinfo)
+flags=""
+flags=$(grep "^flags[[:space:]]*:" <<< "${data}" | head -n 1)
+flags="${flags#*:}"
+flags="${flags## }"
+
+echo "${flags}"
+
+echo "x86_64-v1"
+report_cpu_flags lm cmov cx8 fpu fxsr mmx syscall sse2
+echo "x86_64-v2"
+report_cpu_flags cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3
+echo "x86_64-v3"
+report_cpu_flags avx avx2 bmi1 bmi2 f16c fma abm movbe xsave
+echo "x86_64-v4"
+report_cpu_flags avx512f avx512bw avx512cd avx512dq avx512vl
diff --git a/playbooks/dib-devstack/pre.yaml b/playbooks/dib-devstack/pre.yaml
new file mode 100644
index 000000000..a4a99cc8b
--- /dev/null
+++ b/playbooks/dib-devstack/pre.yaml
@@ -0,0 +1,18 @@
+- hosts: all
+  roles:
+    - name: run-devstack
+  tasks:
+  - name: Copy the cpu-level script into place
+    copy:
+      src: cpu-level.sh
+      dest: /tmp/cpu-level.sh
+      mode:
+  - name: Collect CPU flags for node
+    command:  bash /tmp/cpu-level.sh
+    # NOTE(tonyb):  This is interesting as QEMU gains the ability to emuluate
+    # newer machine types for example 7.2 is needed for haswell.
+  - name: Collect QEMU version
+    ignore_errors: true
+    shell: |-
+      qemu-kvm --version
+      qemu-system-$(arch) --version
diff --git a/playbooks/dib-nodepool/node-information.yaml b/playbooks/dib-nodepool/node-information.yaml
index 8bbbf9ac5..b750bd856 100644
--- a/playbooks/dib-nodepool/node-information.yaml
+++ b/playbooks/dib-nodepool/node-information.yaml
@@ -13,4 +13,4 @@
     ignore_errors: true
     shell: |-
       qemu-kvm --version
-      qemu-system-$(arch) --version
\ No newline at end of file
+      qemu-system-$(arch) --version
diff --git a/tools/functional-test-check.sh b/tools/functional-test-check.sh
new file mode 100755
index 000000000..c70108371
--- /dev/null
+++ b/tools/functional-test-check.sh
@@ -0,0 +1,127 @@
+#!/bin/bash -ex
+
+LOGDIR=/home/zuul/zuul-output/logs
+
+# Set to indiciate an error return
+RETURN=0
+FAILURE_REASON=""
+
+cat > /tmp/ssh_wrapper <<EOF
+#!/bin/bash -ex
+sudo -H -u zuul ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/id_dib root@\$@
+
+EOF
+sudo chmod 0755 /tmp/ssh_wrapper
+
+function sshintonode {
+    node=$1
+
+    /tmp/ssh_wrapper $node ls /
+
+    # Check that the root partition grew on boot; it should be a 5GiB
+    # partition minus some space for the boot partition.  However
+    # emperical evidence suggests there is some modulo maths going on,
+    # (possibly with alignment?) that means we can vary up to even
+    # 64MiB.  Thus we choose an expected value that gives us enough
+    # slop to avoid false matches, but still indicates we resized up.
+    root_size=$(/tmp/ssh_wrapper $node -- lsblk -rbno SIZE /dev/vda1)
+    expected_root_size=$(( 5000000000 ))
+    if [[ $root_size -lt $expected_root_size ]]; then
+        echo "*** Root device does not appear to have grown: $root_size"
+        FAILURE_REASON="Root partition of $name does not appear to have grown: $root_size < $expected_root_size"
+        RETURN=1
+    fi
+
+    # Check we saw metadata deployed to the config-drive
+    /tmp/ssh_wrapper $node \
+        "dd status=none if=/dev/sr0 | tr -cd '[:print:]' | grep -q test-server.novalocal"
+    if [[ $? -ne 0 ]]; then
+        echo "*** Failed to find metadata in config-drive"
+        FAILURE_REASON="Failed to find meta-data in config-drive for $node"
+        RETURN=1
+    fi
+
+    # Debugging that we're seeing the right node
+    /tmp/ssh_wrapper $node -- "cat /etc/os-release; blkid"
+
+    # This ensures DIB setup the bootloader kernel arguments correctly
+    # by looking for the default console setup it does.  In the past
+    # we have seen issues with bootloader installation for all sorts
+    # of reasons (our errors and upstream changes); but generally what
+    # has happened is that case grub has silently fallen-back to
+    # "guessing" from the running build-system what kernel flags to
+    # use.
+    #
+    # DIB images should be booting from a root device labeled
+    # "cloudimg-rootfs".  In the gate, where you're *using* a
+    # DIB-image to build a DIB-image, it's /proc/cmdline contains
+    # "root=LABEL=cloudimg-rootfs" and a misconfigured grub can
+    # actually guess the correct root device.  However, when this
+    # builds an image in production on a totally different host you
+    # get a non-booting, wrong image.  Ergo, although this is mostly
+    # what we're interested in validating, this is not a reliable
+    # thing to test directly.
+    #
+    # So below, we probe for something else; the console setup that
+    # DIB will put in.  If this is missing, it's an indication that
+    # the bootloader is not setting up the kernel arguments correctly.
+    kernel_cmd_line=$(/tmp/ssh_wrapper $node -- cat /proc/cmdline)
+    echo "Kernel command line: ${kernel_cmd_line}"
+    if [[ ! $kernel_cmd_line =~ 'console=tty0 console=ttyS0,115200' ]]; then
+       echo "*** Failed to find correct kernel boot flags"
+       FAILURE_REASON="Failed to find correct kernel boot flags $node"
+       RETURN=1
+    fi
+
+    # Ensure glean services have loaded
+    # The output is like:
+    # ---
+    #  glean-early.service loaded active exited    Early glean execution
+    #  glean@ens3.service  loaded active exited    Glean for interface ens3 with NetworkManager
+    #  glean@lo.service    loaded active exited    Glean for interface lo with NetworkManager
+    # ---
+    # So if we see anything other than 'loaded active exited' we have a problem.
+    glean_status=$(/tmp/ssh_wrapper $node -- "systemctl | egrep 'glean[@|-]' | { grep -v 'loaded active exited' || true; }")
+    if [[ ${glean_status} != '' ]]; then
+        echo "*** Glean not loaded correctly"
+        echo "*** saw: ${glean_status}"
+        FAILURE_REASON="Failed to start glean correctly"
+        RETURN=1
+    fi
+}
+
+function checknm {
+    node=$1
+    state='ready'
+    nm_output=$(/tmp/ssh_wrapper $node -- nmcli c)
+
+    # virtio device is eth0 on older, ens3 on newer
+    if [[ ! ${nm_output} =~ (eth0|ens3) ]]; then
+        echo "*** Failed to find interface in NetworkManager connections"
+        /tmp/ssh_wrapper $node -- nmcli c
+        /tmp/ssh_wrapper $node -- nmcli device
+        FAILURE_REASON="Failed to find interface in NetworkManager connections"
+        RETURN=1
+    fi
+}
+
+(
+# check ssh for root user
+sshintonode test-server
+# networkmanager check
+# TODO(jeblair): This should not run in all cases; in fact, most of
+# this checking should move into the dib repo
+#checknm test-image
+# userdata check
+) |& tee "$LOGDIR/functional-checks.log"
+
+set -o errexit
+
+openstack --os-cloud devstack image show test-image
+openstack --os-cloud devstack server list
+openstack --os-cloud devstack server show test-server
+
+if [[ -n "${FAILURE_REASON}" ]]; then
+    echo "${FAILURE_REASON}"
+fi
+exit $RETURN