Add Puppet Litmus and remove Puppet 5 testing

This patch adds the needed bits to replace Beaker
with Puppet Litmus.

Since Puppet Litmus only supports Puppet 6 we
need to remove Puppet 5 testing. The probability
that we break Puppet 5 support (we have promised
to keep it in Victoria) this late in the cycle is
very low hence we should try to get this through.

When this is merged we can move all modules over
to use Litmus and then cleanup all Beaker things.

Change-Id: I73f18345ed3e6481f86ad7af7d68f66b19d2250e

Puppetfile_unit

@@ -18,3 +18,8 @@ mod 'yumrepo_core',
 mod 'mount_core',
   :git => 'https://github.com/puppetlabs/puppetlabs-mount_core',
   :ref => '1.0.4'
+
+# NOTE(tobias-urdin): Needed by p-o-i.
+mod 'selinux_core',
+  :git => 'https://github.com/puppetlabs/puppetlabs-selinux_core',
+  :ref => '1.0.4'

manifests/repos.pp

@@ -1,10 +1,10 @@
 class openstack_integration::repos {
 
-  # To make beaker tests work.
+  # To make litmus tests work.
-  if $::ceph_version != '' {
+  if defined('$::ceph_version') and $::ceph_version != '' {
     $ceph_version_real = $::ceph_version
   } else {
-    $ceph_version_real = 'mimic'
+    $ceph_version_real = 'nautilus'
   }
   case $::osfamily {
     'Debian': {
@@ -30,7 +30,7 @@ class openstack_integration::repos {
       }
       # Ceph is both packaged on UCA and official download.ceph.com packages
       # which we mirror. We want to use the official packages or our mirror.
-      if $::nodepool_mirror_host != '' {
+      if defined('$::nodepool_mirror_host') and $::nodepool_mirror_host != '' {
         $ceph_version_cap = capitalize($ceph_version_real)
         apt::pin { 'ceph':
           priority   => 1001,
@@ -47,10 +47,15 @@ class openstack_integration::repos {
       $ceph_mirror = pick($::ceph_mirror_host, "http://download.ceph.com/debian-${ceph_version_real}/")
     }
     'RedHat': {
+      if defined('$::centos_mirror_host') and $::centos_mirror_host != '' {
+        $centos_mirror = $::centos_mirror_host
+      } else {
+        $centos_mirror = 'http://mirror.centos.org'
+      }
       class { 'openstack_extras::repo::redhat::redhat':
         manage_rdo        => false,
         manage_epel       => false,
-        centos_mirror_url => $::centos_mirror_host,
+        centos_mirror_url => $centos_mirror,
         repo_source_hash  => {
           'delorean.repo'      => "https://trunk.rdoproject.org/centos${::os['release']['major']}-master/puppet-passed-ci/delorean.repo",
           'delorean-deps.repo' => "https://trunk.rdoproject.org/centos${::os['release']['major']}-master/delorean-deps.repo"
@@ -60,9 +65,13 @@ class openstack_integration::repos {
       # NOTE(tobias-urdin): Mimic was never released by Storage SIG to official mirros.
       $ceph_mirror_fallback = $ceph_version_real ? {
         'mimic' => "https://trunk.rdoproject.org/centos${::os['release']['major']}/deps/storage/storage${::os['release']['major']}-ceph-mimic/x86_64/",
-        default => "${::centos_mirror_host}/centos/${::os['release']['major']}/storage/x86_64/ceph-${ceph_version_real}/"
+        default => "${centos_mirror}/centos/${::os['release']['major']}/storage/x86_64/ceph-${ceph_version_real}/"
+      }
+      if defined('$::ceph_mirror_host') and $::ceph_mirror_host != '' {
+        $ceph_mirror = pick($::ceph_mirror_host, $ceph_mirror_fallback)
+      } else {
+        $ceph_mirror = $ceph_mirror_fallback
       }
-      $ceph_mirror = pick($::ceph_mirror_host, $ceph_mirror_fallback)
       # On CentOS, deploy Ceph using SIG repository and get rid of EPEL.
       # https://wiki.centos.org/SpecialInterestGroup/Storage/
       if $::operatingsystem == 'CentOS' {

playbooks/prepare-node-litmus.yaml

@@ -0,0 +1,24 @@
+- hosts: all
+  tasks:
+    - shell:
+        cmd: |
+          set -e
+          set -x
+
+          if [ -d /home/zuul/src/opendev.org/$ZUUL_PROJECT ]; then
+              cp -dR /home/zuul/src/opendev.org/$ZUUL_PROJECT/. .
+          else
+              git clone https://opendev.org/$ZUUL_PROJECT .
+          fi
+          if [ -d /home/zuul/src/opendev.org/openstack/puppet-openstack-integration ]; then
+              [ ! -d puppet-openstack-integration ] && mkdir puppet-openstack-integration
+              cp -dR /home/zuul/src/opendev.org/openstack/puppet-openstack-integration/. puppet-openstack-integration
+          else
+              git clone https://opendev.org/openstack/puppet-openstack-integration puppet-openstack-integration
+          fi
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - include_role:
+        name: bindep

playbooks/run-litmus-tests.yaml

@@ -0,0 +1,34 @@
+- hosts: all
+  tasks:
+    - name: Run Litmus
+      shell:
+        cmd: |
+          set -e
+          function trap_exit_sig() {
+            # NOTE(tobias-urdin): Litmus does not execute spec_clean rake task after a run.
+            $GEM_BIN_DIR/bundle exec rake spec_clean
+
+            {{ ansible_user_dir }}/workspace/puppet-openstack-integration/copy_logs.sh
+          }
+          trap trap_exit_sig EXIT
+          export CEPH_VERSION={{ ceph }}
+          if [ "{{ puppet_gem_version }}" != "latest" ]; then
+            export PUPPET_GEM_VERSION='~> {{ puppet_gem_version }}'
+          fi
+          mkdir .bundled_gems
+          export GEM_HOME=`pwd`/.bundled_gems
+          export GEM_BIN_DIR=$GEM_HOME/bin
+          ruby <<EOF
+          cmd = 'gem install bundler --no-rdoc --no-ri --verbose --no-user-install --bindir=${GEM_BIN_DIR}'
+          if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3.0') then
+              cmd += ' -v 1.17.3'
+          end
+          system(cmd)
+          EOF
+          $GEM_BIN_DIR/bundle install --without system_tests --retry 3
+          export RSPEC_DEBUG=true
+          $GEM_BIN_DIR/bundle exec rake litmus:acceptance:localhost
+        chdir: '{{ ansible_user_dir }}/workspace'
+        executable: /bin/bash
+      become: yes
+      environment: '{{ zuul | zuul_legacy_vars }}'

zuul.d/base.yaml

@@ -186,7 +186,7 @@
     voting: false
     vars:
       nodepool_type: centos8
-      puppet_gem_version: 5.5
+      puppet_gem_version: 6.18
 
 - job:
     name: puppet-openstack-beaker-ubuntu-bionic

zuul.d/beaker.yaml

@@ -30,7 +30,7 @@
       - ^.gitignore$
     vars:
       nodepool_type: centos8
-      puppet_gem_version: 5.5.0
+      puppet_gem_version: 6.18.0
 
 - job:
     name: puppet-openstack-libraries-puppet-beaker-rspec-ubuntu-bionic

zuul.d/layout.yaml

@@ -8,12 +8,10 @@
     check:
       jobs:
         - puppet-openstack-lint-ubuntu-bionic
-        - puppet-openstack-syntax-5-ubuntu-bionic
         - puppet-openstack-syntax-6-ubuntu-bionic
     gate:
       jobs:
         - puppet-openstack-lint-ubuntu-bionic
-        - puppet-openstack-syntax-5-ubuntu-bionic
         - puppet-openstack-syntax-6-ubuntu-bionic
 
 - project:
@@ -27,10 +25,8 @@
     check:
       jobs:
         - puppet-openstack-libraries-puppet-lint-ubuntu-bionic
-        - puppet-openstack-libraries-puppet-syntax-5-centos-8
         - puppet-openstack-libraries-puppet-syntax-6-centos-8
         - puppet-openstack-libraries-puppet-syntax-6-ubuntu-bionic
-        - puppet-openstack-libraries-puppet-unit-5.5-centos-8
         - puppet-openstack-libraries-puppet-unit-6.14-centos-8
         - puppet-openstack-libraries-puppet-unit-6.14-ubuntu-bionic
         - puppet-openstack-libraries-puppet-unit-latest-ubuntu-bionic
@@ -39,10 +35,8 @@
     gate:
       jobs:
         - puppet-openstack-libraries-puppet-lint-ubuntu-bionic
-        - puppet-openstack-libraries-puppet-syntax-5-centos-8
         - puppet-openstack-libraries-puppet-syntax-6-centos-8
         - puppet-openstack-libraries-puppet-syntax-6-ubuntu-bionic
-        - puppet-openstack-libraries-puppet-unit-5.5-centos-8
         - puppet-openstack-libraries-puppet-unit-6.14-centos-8
         - puppet-openstack-libraries-puppet-unit-6.14-ubuntu-bionic
         - puppet-openstack-libraries-puppet-beaker-rspec-centos-8

zuul.d/libraries.yaml

@@ -1,13 +1,6 @@
 ---
 # This file contains the jobs for the puppet openstack libraries
 #
-- job:
-    name: puppet-openstack-libraries-puppet-unit-5.5-centos-8
-    parent: puppet-openstack-libraries-puppet-unit-base
-    nodeset: centos-8
-    vars:
-      puppet_gem_version: 5.5.0
-
 - job:
     name: puppet-openstack-libraries-puppet-unit-6.14-centos-8
     parent: puppet-openstack-libraries-puppet-unit-base

zuul.d/linters.yaml

@@ -10,12 +10,6 @@
       - ^requirements.txt$
       - ^test-requirements.txt$
 
-- job:
-    name: puppet-openstack-syntax-5-ubuntu-bionic
-    parent: puppet-openstack-syntax-base
-    vars:
-      puppet: 5
-
 - job:
     name: puppet-openstack-syntax-6-ubuntu-bionic
     parent: puppet-openstack-syntax-base
@@ -52,13 +46,6 @@
     vars:
       puppet: 6
 
-- job:
-    name: puppet-openstack-libraries-puppet-syntax-5-centos-8
-    parent: puppet-openstack-libraries-puppet-syntax
-    nodeset: centos-8
-    vars:
-      puppet: 5
-
 - job:
     name: puppet-openstack-libraries-puppet-syntax-6-centos-8
     parent: puppet-openstack-libraries-puppet-syntax

zuul.d/litmus.yaml

@@ -0,0 +1,51 @@
+---
+- job:
+    name: puppet-openstack-litmus-run-base
+    parent: puppet-openstack-integration-base
+    abstract: true
+    pre-run: playbooks/prepare-node-litmus.yaml
+    post-run: playbooks/upload-logs.yaml
+    run: playbooks/run-litmus-tests.yaml
+    timeout: 5400
+    irrelevant-files:
+      - ^.*\.md$
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^spec/unit/.*$
+      - ^spec/classes/.*$
+      - ^spec/defines/.*$
+      - ^requirements.txt$
+      - ^test-requirements.txt$
+      - ^metadata.json$
+      - ^LICENSE$
+      - ^.gitignore$
+      - ^setup.cfg$
+      - ^setup.py$
+
+- job:
+    name: puppet-openstack-litmus-centos-8
+    parent: puppet-openstack-litmus-run-base
+    nodeset: centos-8
+    vars:
+      puppet_gem_version: 6.16
+      ceph: nautilus
+
+- job:
+    name: puppet-openstack-litmus-ubuntu-bionic
+    parent: puppet-openstack-litmus-run-base
+    nodeset: ubuntu-bionic
+    vars:
+      puppet_gem_version: latest
+      ceph: nautilus
+
+- project-template:
+    name: puppet-openstack-litmus-jobs
+    check:
+      jobs:
+        - puppet-openstack-litmus-centos-8
+        - puppet-openstack-litmus-ubuntu-bionic
+    gate:
+      jobs:
+        - puppet-openstack-litmus-centos-8
+        - puppet-openstack-litmus-ubuntu-bionic

zuul.d/unit.yaml

@@ -1,13 +1,6 @@
 ---
 # This file contains the puppet unit job configurations
 #
-- job:
-    name: puppet-openstack-unit-5.5-centos-8
-    parent: puppet-openstack-unit-base
-    nodeset: centos-8
-    vars:
-      puppet_gem_version: 5.5.0
-
 - job:
     name: puppet-openstack-unit-6.14-centos-8
     parent: puppet-openstack-unit-base
@@ -37,12 +30,10 @@
     name: puppet-openstack-module-unit-jobs
     check:
       jobs:
-        - puppet-openstack-unit-5.5-centos-8
         - puppet-openstack-unit-6.14-centos-8
         - puppet-openstack-unit-6.14-ubuntu-bionic
         - puppet-openstack-unit-latest-ubuntu-bionic
     gate:
       jobs:
-        - puppet-openstack-unit-5.5-centos-8
         - puppet-openstack-unit-6.14-centos-8
         - puppet-openstack-unit-6.14-ubuntu-bionic