Native multibackend-matrix Zuul v3 job

Port the legacy legacy-tempest-dsvm-multibackend-matrix job to the
native Zuul v3 syntax, and rename it following the guidelines
(cinder-multibackend-matrix-migration).

This job tests the migration between two different backends
specified through the volume.backend_names configuration
key in tempest.conf.

Now the job leverages the existing zuul code, namely the
run-tempest role, which is called multiple times with all
the possible combinations of the 3 tested backends
(lvm, ceph, nfs) where the source and the destination differ.

The final JUnitXML output summarizes the test results
for each of the tested combinations.

Change-Id: I34e7e48ee63c4c269f82ae178a7118ed402cad6d

.zuul.yaml

@@ -98,7 +98,7 @@
             irrelevant-files: *gate-irrelevant-files
     experimental:
       jobs:
-        - legacy-tempest-dsvm-multibackend-matrix:
+        - cinder-multibackend-matrix-migration:
             irrelevant-files: *gate-irrelevant-files
         - cinder-grenade-mn-sub-volschbak:
             irrelevant-files: *gate-irrelevant-files
@@ -310,3 +310,31 @@
         GLANCE_SHOW_DIRECT_URL: True
         GLANCE_SHOW_MULTIPLE_LOCATIONS: True
         CINDER_ALLOWED_DIRECT_URL_SCHEMES: cinder
+
+- job:
+    name: cinder-multibackend-matrix-migration
+    parent: devstack-tempest
+    description: |
+      Run migration tests between several combinations of backends
+      (LVM, Ceph, NFS)
+      Former names for this job were:
+        * legacy-tempest-dsvm-multibackend-matrix
+    timeout: 10800
+    required-projects:
+      - opendev.org/openstack/devstack-plugin-ceph
+      - opendev.org/openstack/devstack-plugin-nfs
+    run: playbooks/cinder-multibackend-matrix.yaml
+    host-vars:
+      controller:
+        devstack_plugins:
+          devstack-plugin-ceph: https://opendev.org/openstack/devstack-plugin-ceph
+          devstack-plugin-nfs: https://opendev.org/openstack/devstack-plugin-nfs
+    vars:
+      devstack_localrc:
+        CINDER_ENABLED_BACKENDS: lvm:lvm,nfs:nfs,ceph:ceph
+        ENABLE_NFS_CINDER: true
+      devstack_local_conf:
+        test-config:
+          $TEMPEST_CONFIG:
+            volume:
+              build_timeout: 900

playbooks/cinder-multibackend-matrix.yaml

@@ -0,0 +1,35 @@
+# Playbook originally inspired by
+# https://opendev.org/openstack/tempest/src/tag/23.0.0/playbooks/devstack-tempest.yaml
+
+# Changes that run through devstack-tempest are likely to have an impact on
+# the devstack part of the job, so we keep devstack in the main play to
+# avoid zuul retrying on legitimate failures.
+- hosts: all
+  roles:
+    - orchestrate-devstack
+
+# We run tests only on one node, regardless how many nodes are in the system
+- hosts: tempest
+  vars:
+    migration_backends:
+      - lvm
+      - ceph
+      - nfs
+    migration_test_results: []
+    migration_tempest_conf: "/opt/stack/tempest/etc/tempest.conf"
+  tasks:
+    - include_role:
+        name: setup-tempest-run-dir
+    - include_role:
+        name: setup-tempest-data-dir
+    - include_role:
+        name: acl-devstack-files
+    - include_role:
+        name: configure-run-migration-tests
+      vars:
+        migration_source_backend: "{{ item[0] }}"
+        migration_destination_backend: "{{ item[1] }}"
+      loop: "{{ migration_backends|product(migration_backends)|list }}"
+      when: item[0] != item[1]
+    - include_role:
+        name: save-cinder-migration-results

roles/configure-run-migration-tests/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+migration_source_backend: lvm
+migration_destination_backend: lvm
+migration_test_regex: "(.*test_volume_retype_with_migration.*|.*test_volume_migrate_attached.*)"
+migration_test_results: []
+tempest_run_result: {}

roles/configure-run-migration-tests/tasks/main.yaml

@@ -0,0 +1,33 @@
+---
+- name: Reconfigure tempest.conf
+  ini_file:
+    path: "{{ migration_tempest_conf }}"
+    section: volume
+    option: backend_names
+    value: "{{ migration_source_backend }},{{ migration_destination_backend }}"
+  become: true
+  become_user: tempest
+
+- set_fact:
+    tempest_run_result: {}
+
+- name: Run migration ({{ migration_source_backend }} -> {{ migration_destination_backend }})
+  include_role:
+    name: run-tempest
+    apply:
+      # ignore the errors for this run, otherwise the other migration tests
+      # won't be executed
+      ignore_errors: yes
+  vars:
+    tempest_test_regex: "{{ migration_test_regex }}"
+    tox_envlist: all
+
+- set_fact:
+    _migration_result_item:
+        source: "{{ migration_source_backend }}"
+        destination: "{{ migration_destination_backend }}"
+        result: "{{ tempest_run_result.get('rc', 1) }}"
+
+- name: Update the migration test results
+  set_fact:
+    migration_test_results: "{{ migration_test_results + [ _migration_result_item ] }}"

roles/save-cinder-migration-results/defaults/main.yaml

@@ -0,0 +1,3 @@
+---
+devstack_base_dir: /opt/stack
+tempest_work_dir: "{{ devstack_base_dir }}/tempest"

roles/save-cinder-migration-results/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+- block:
+    - template:
+        src: migration_results_reporter.py.j2
+        dest: "{{ tempest_work_dir }}/migration_results_reporter.py"
+
+    - name: Generate the results using stestr
+      shell: |
+        stestr run --no-discover --test-path . migration_results_reporter
+      args:
+        chdir: "{{ tempest_work_dir }}"
+
+  become: true
+  become_user: tempest

roles/save-cinder-migration-results/templates/migration_results_reporter.py.j2

@@ -0,0 +1,10 @@
+import unittest
+
+
+class CinderMigrationsMatrixTest(unittest.TestCase):
+
+{% for test_case in migration_test_results %}
+    def test__{{ test_case.source }}_to_{{ test_case.destination }}(self):
+        self.assertEqual({{ test_case.result }}, 0)
+
+{% endfor %}

tools/hooks/README

@@ -1,4 +0,0 @@
-These are hooks to be used by the OpenStack infra test system. These scripts
-may be called by certain jobs at important times to do extra testing, setup,
-etc. They are really only relevant within the scope of the OpenStack infra
-system and are not expected to be useful to anyone else.

tools/hooks/run_multi_backend_matrix.sh

@@ -1,94 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2016, Hitachi, Erlon Cruz <erlon.cruz@fit-tecnologia.org.br>
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -x
-export TEMPEST_USER=${TEMPEST_USER:-tempest}
-chmod +w $BASE/new/tempest
-cd $BASE/new/tempest
-source $BASE/new/devstack/functions
-source $BASE/new/devstack/functions-common
-source $WORKSPACE/devstack-gate/functions.sh
-source $BASE/new/cinder/tools/hooks/utils.sh
-export TEMPEST_CONFIG=$BASE/new/tempest/etc/tempest.conf
-
-# Disable bash verbose so we have a cleaner output. Also, exit on error must
-# be disable as we will run several tests that can return error.
-set +x +e
-
-function configure_tempest_backends {
-    be1=$1
-    be2=$2
-    echo "Configuring tempest conf in ${TEMPEST_CONFIG}"
-    iniset -sudo $TEMPEST_CONFIG 'volume' 'backend_names' ${be1},${be2}
-
-}
-
-BACKENDS='lvm ceph nfs'
-RGEX="(.*test_volume_retype_with_migration.*|.*test_volume_migrate_attached.*)"
-final_result=0
-final_message='Migrations tests finished SUCCESSFULLY!'
-declare -A TEST_RESULTS
-start_time=`date +%s`
-for be1 in ${BACKENDS}; do
-    for be2 in ${BACKENDS}; do
-        if [ ${be1} != ${be2} ]; then
-            configure_tempest_backends ${be1} ${be2}
-            echo "============================================================"
-            echo "Testing multibackend features: ${be1} vs ${be2}"
-            echo "============================================================"
-            run_tempest "${be1} vs ${be2}" ${RGEX}
-            result=$?
-            # If any of the test fail, we keep running but return failure as
-            # the final result
-            if [ ${result} -ne 0 ]; then
-                TEST_RESULTS[${be1},${be2}]="FAILURE"
-                final_message='Migrations tests FAILED!'
-                final_result=1
-            else
-                TEST_RESULTS[${be1},${be2}]="SUCCESS"
-            fi
-        fi
-    done
-done
-end_time=`date +%s`
-elapsed=$(expr $(expr ${end_time} - ${start_time}) / 60)
-
-# Print the results
-num_rows=$(echo $BACKENDS | wc -w)
-fmt=" %15s"
-echo "============================================================"
-echo " ${final_message} In ${elapsed} minutes."
-echo "============================================================"
-
-printf "$fmt" ''
-for be1 in ${BACKENDS}; do
-    printf "$fmt" ${be1}
-done
-echo
-for be1 in ${BACKENDS}; do
-    printf "$fmt" ${be1}
-    for be2 in ${BACKENDS}; do
-        if [ ${be1} == ${be2} ]; then
-            printf "$fmt" '---'
-        else
-            printf "$fmt" ${TEST_RESULTS[${be1},${be2}]}
-        fi
-    done
-    echo
-done
-
-exit ${final_result}

tools/hooks/utils.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-function run_tempest {
-    local message=$1
-    local tempest_regex=$2
-    sudo -H -u ${TEMPEST_USER}  tox -eall -- $tempest_regex \
-    --concurrency=${TEMPEST_CONCURRENCY}
-    exitcode=$?
-    return ${exitcode}
-}