CI: separate image builds into a non-voting job

Disk and container image builds tend to be fairly unreliable.
With 3 voting seed jobs all building images, this can introduce
instability into the CI jobs.

This change adds a non-voting kayobe-seed-images-centos8s job, which
does the following:

* Builds IPA images
* Builds an overcloud host image
* Builds a base container image

Similar Rocky and Ubuntu jobs are added to the experimental pipeline,
and may be run by commenting 'check experimental' in gerrit.

The existing kayobe-seed-* jobs no longer build images.

Change-Id: Idecda342f3ab86733e8d59061458d44af834dbb0

playbooks/kayobe-seed-base/overrides.yml.j2

@@ -32,3 +32,11 @@ pip_trusted_hosts:
 # are using for SSH to be removed. Use a dummy interface.
 aio_bridge_ports:
   - dummy1
+
+# Build seed deployment images (IPA) with extra-hardware element
+ipa_build_images: {{ build_images }}
+ipa_build_dib_elements_extra:
+  - "extra-hardware"
+
+# Build overcloud host image.
+overcloud_dib_build_host_images: {{ build_images }}

playbooks/kayobe-seed-base/pre.yml

@@ -25,16 +25,18 @@
         src: globals.yml.j2
         dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
 
-    - name: Ensure bifrost overrides directory exists
-      file:
-        path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
-        state: "directory"
+    - block:
+        - name: Ensure bifrost overrides directory exists
+          file:
+            path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+            state: "directory"
 
-    # NOTE(mgoddard): Use dib.yml, since it takes precedence over bifrost.yml.
-    - name: Ensure bifrost overrides file exists
-      template:
-        src: bifrost-overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/dib.yml"
+        # NOTE(mgoddard): Use dib.yml, since it takes precedence over bifrost.yml.
+        - name: Ensure bifrost overrides file exists
+          template:
+            src: bifrost-overrides.yml.j2
+            dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/dib.yml"
+      when: not build_images
 
     - name: Ensure kayobe is installed
       shell:

playbooks/kayobe-seed-base/run.yml

@@ -10,21 +10,22 @@
         cmd: "{{ kayobe_src_dir }}/dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy"
         executable: /bin/bash
 
-    # NOTE(mgoddard): The stack user cannot read the kolla source code, which
-    # lives in the zuul user's home directory. Copy it to a readable location.
-    - name: Create a readable copy of kolla source code
-      become: true
-      copy:
-        src: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
-        dest: /tmp
-        owner: stack
-        group: stack
-        remote_src: true
+    - block:
+        # NOTE(mgoddard): The stack user cannot read the kolla source code, which
+        # lives in the zuul user's home directory. Copy it to a readable location.
+        - name: Create a readable copy of kolla source code
+          become: true
+          copy:
+            src: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
+            dest: /tmp
+            owner: stack
+            group: stack
+            remote_src: true
 
-    - name: Ensure base container image is built
-      shell:
-        cmd: >
-          source {{ kayobe_src_dir }}/dev/environment-setup.sh &&
-          kayobe seed container image build ^base &> {{ logs_dir }}/ansible/container-image-build
-        executable: /bin/bash
-      when: ansible_os_family == 'RedHat'
+        - name: Ensure base container image is built
+          shell:
+            cmd: >
+              source {{ kayobe_src_dir }}/dev/environment-setup.sh &&
+              kayobe seed container image build ^base &> {{ logs_dir }}/ansible/container-image-build
+            executable: /bin/bash
+      when: build_images

zuul.d/jobs.yaml

@@ -160,6 +160,8 @@
     pre-run: playbooks/kayobe-seed-base/pre.yml
     run: playbooks/kayobe-seed-base/run.yml
     timeout: 5400
+    vars:
+      build_images: false
 
 - job:
     name: kayobe-seed-centos8s
@@ -176,6 +178,32 @@
     parent: kayobe-seed-base
     nodeset: kayobe-ubuntu-focal
 
+- job:
+    name: kayobe-seed-images-base
+    parent: kayobe-seed-base
+    description: |
+      Base job for testing seed image builds.
+
+      Configures the primary VM as a seed.
+    voting: false
+    vars:
+      build_images: true
+
+- job:
+    name: kayobe-seed-images-centos8s
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-centos8s
+
+- job:
+    name: kayobe-seed-images-rocky8
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-rocky8
+
+- job:
+    name: kayobe-seed-images-ubuntu-focal
+    parent: kayobe-seed-images-base
+    nodeset: kayobe-ubuntu-focal
+
 - job:
     name: kayobe-overcloud-host-configure-base
     parent: kayobe-overcloud-base

zuul.d/project.yaml

@@ -22,6 +22,7 @@
         - kayobe-seed-centos8s
         - kayobe-seed-rocky8
         - kayobe-seed-ubuntu-focal
+        - kayobe-seed-images-centos8s
         - kayobe-seed-upgrade-centos8s
         - kayobe-seed-upgrade-ubuntu-focal
         - kayobe-seed-vm-centos8s
@@ -57,3 +58,8 @@
         - kayobe-infra-vm-centos8s
         - kayobe-infra-vm-rocky8
         - kayobe-infra-vm-ubuntu-focal
+
+    experimental:
+      jobs:
+        - kayobe-seed-images-rocky8
+        - kayobe-seed-images-ubuntu-focal