CI: separate image builds into a non-voting job

Disk and container image builds tend to be fairly unreliable.
With 3 voting seed jobs all building images, this can introduce
instability into the CI jobs.

This change adds a non-voting kayobe-seed-images-centos8s job, which
does the following:

* Builds IPA images
* Builds an overcloud host image
* Builds a base container image

Similar Rocky and Ubuntu jobs are added to the experimental pipeline,
and may be run by commenting 'check experimental' in gerrit.

The existing kayobe-seed-* jobs no longer build images.

Change-Id: Idecda342f3ab86733e8d59061458d44af834dbb0

playbooks/kayobe-seed-base/overrides.yml.j2

@@ -32,3 +32,11 @@ pip_trusted_hosts:
 # are using for SSH to be removed. Use a dummy interface.
 aio_bridge_ports:
   - dummy1
+
+# Build seed deployment images (IPA) with extra-hardware element
+ipa_build_images: {{ build_images }}
+ipa_build_dib_elements_extra:
+  - "extra-hardware"
+
+# Build overcloud host image.
+overcloud_dib_build_host_images: {{ build_images }}

playbooks/kayobe-seed-base/pre.yml

@@ -25,6 +25,7 @@
         src: globals.yml.j2
         dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
 
+    - block:
         - name: Ensure bifrost overrides directory exists
           file:
             path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
@@ -35,6 +36,7 @@
           template:
             src: bifrost-overrides.yml.j2
             dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/dib.yml"
+      when: not build_images
 
     - name: Ensure kayobe is installed
       shell:

playbooks/kayobe-seed-base/run.yml

@@ -10,6 +10,7 @@
         cmd: "{{ kayobe_src_dir }}/dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy"
         executable: /bin/bash
 
+    - block:
         # NOTE(mgoddard): The stack user cannot read the kolla source code, which
         # lives in the zuul user's home directory. Copy it to a readable location.
         - name: Create a readable copy of kolla source code
@@ -27,4 +28,4 @@
               source {{ kayobe_src_dir }}/dev/environment-setup.sh &&
               kayobe seed container image build ^base &> {{ logs_dir }}/ansible/container-image-build
             executable: /bin/bash
-      when: ansible_os_family == 'RedHat'
+      when: build_images

zuul.d/jobs.yaml

@@ -160,6 +160,8 @@
     pre-run: playbooks/kayobe-seed-base/pre.yml
     run: playbooks/kayobe-seed-base/run.yml
     timeout: 5400
+    vars:
+      build_images: false
 
 - job:
     name: kayobe-seed-centos8s
@@ -176,6 +178,32 @@
     parent: kayobe-seed-base
     nodeset: kayobe-ubuntu-focal
 
+- job:
+    name: kayobe-seed-images-base
+    parent: kayobe-seed-base
+    description: |
+      Base job for testing seed image builds.
+
+      Configures the primary VM as a seed.
+    voting: false
+    vars:
+      build_images: true
+
+- job:
+    name: kayobe-seed-images-centos8s
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-centos8s
+
+- job:
+    name: kayobe-seed-images-rocky8
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-rocky8
+
+- job:
+    name: kayobe-seed-images-ubuntu-focal
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-ubuntu-focal
+
 - job:
     name: kayobe-overcloud-host-configure-base
     parent: kayobe-overcloud-base

zuul.d/project.yaml

@@ -22,6 +22,7 @@
         - kayobe-seed-centos8s
         - kayobe-seed-rocky8
         - kayobe-seed-ubuntu-focal
+        - kayobe-seed-images-centos8s
         - kayobe-seed-upgrade-centos8s
         - kayobe-seed-upgrade-ubuntu-focal
         - kayobe-seed-vm-centos8s
@@ -57,3 +58,8 @@
         - kayobe-infra-vm-centos8s
         - kayobe-infra-vm-rocky8
         - kayobe-infra-vm-ubuntu-focal
+
+    experimental:
+      jobs:
+        - kayobe-seed-images-rocky8
+        - kayobe-seed-images-ubuntu-focal