[Testing Infra] Apply here the improvements done to tripleo-validations

This patch brings out the improvements done from tripleo-validations to validations-common regarding the molecule testing infrastructure with the UBI8 images. This patch integrates molecule fix after release of cryptography3.4. See https://review.opendev.org/c/openstack/tripleo-validations/+/774747 Related-Bug: #1915101 Co-Authored-By: David Vallee Delisle <dvd@redhat.com> Change-Id: I2e5ebebdd48e4013aa87868a22807e91feb3ffab Signed-off-by: Gael Chamoulaud (Strider) <gchamoul@redhat.com>
2020-11-26 09:06:36 +01:00 · 2020-11-26 09:06:36 +01:00 · ab26d8263a
commit ab26d8263a
parent 5795fa8f43
25 changed files with 118 additions and 57 deletions
--- a/ansible-test-env.rc
+++ b/ansible-test-env.rc
@ -1,6 +1,5 @@
 export VALIDATIONS_COMMON_WORKPATH="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 export ANSIBLE_STDOUT_CALLBACK=debug
-export ANSIBLE_CALLBACK_PLUGINS="${VALIDATIONS_COMMON_WORKPATH}/validations_common/callback_plugins"
 export ANSIBLE_LIBRARY="${VALIDATIONS_COMMON_WORKPATH}/validations_common/library"

 # TODO(gchamoul): uncomment once we host lookup plugins
--- a/bindep.txt
+++ b/bindep.txt
@ -24,6 +24,10 @@ libxml2-devel     [platform:rpm]
 libxslt-devel     [platform:rpm]
 libxslt1-dev      [platform:dpkg]
 libxslt-dev       [platform:apk]
+
+# RH Mechanisms
+python-rhsm-certificates [platform: redhat]
+
 podman            [platform:rpm]
 python-devel      [platform:rpm !platform:rhel-8 !platform:centos-8]
 python3-devel     [platform:rpm !platform:rhel-7 !platform:centos-7]
--- a/molecule-requirements.txt
+++ b/molecule-requirements.txt
@ -1,11 +1,11 @@
 # this is required for the molecule jobs
 ansible>=2.8,!=2.8.9,!=2.9.12,<2.10.0
 ansi2html
+jinja2>=2.8.0,<3
+mock
+molecule>=3.0,<3.1
 pytest
 pytest-cov
 pytest-html
 pytest-xdist
-mock
-molecule>=3.0,<3.1
 selinux  # MIT
-jinja2>=2.8.0,<3
--- a/playbooks/molecule/pre.yml
+++ b/playbooks/molecule/pre.yml
@ -1,6 +1,10 @@
 ---
 - hosts: all
  pre_tasks:
+    - name: Set project path fact
+      set_fact:
+        validations_common_project_path: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/validations-common'].src_dir }}"
+
    - name: Ensure output dirs
      file:
        path: "{{ ansible_user_dir }}/zuul-output/logs"
@ -18,18 +22,25 @@
      pip:
        name: "bindep"
        virtualenv: "{{ ansible_user_dir }}/test-python"
+        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
        virtualenv_site_packages: true

    - name: Run bindep
      shell: |-
        . {{ ansible_user_dir }}/test-python/bin/activate
-        {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/scripts/bindep-install
+        {{ validations_common_project_path }}/scripts/bindep-install
      become: true
      changed_when: false

+    - name: Ensure a recent version of pip is installed in virtualenv
+      pip:
+        name: "pip>=19.1.1"
+        virtualenv: "{{ ansible_user_dir }}/test-python"
+        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
+
    - name: Setup test-python
      pip:
-        requirements: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/molecule-requirements.txt"
+        requirements: "{{ validations_common_project_path }}/molecule-requirements.txt"
        virtualenv: "{{ ansible_user_dir }}/test-python"
        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
        virtualenv_site_packages: true
@ -76,6 +87,18 @@
              * hard nproc 4096
            dest: /etc/security/limits.d/containers.conf

+        - name: Install epel-release
+          package:
+            name: epel-release
+            state: present
+
+        - name: Be sure to enable epel repo
+          become: true
+          command: >-
+            dnf config-manager --set-enabled epel
+          args:
+            warn: false
+
        - name: Copy repos directory to opt workspace
          copy:
            src: /etc/yum.repos.d
--- a/playbooks/molecule/run-local.yml
+++ b/playbooks/molecule/run-local.yml
@ -4,8 +4,9 @@
    - name: set basic zuul fact
      set_fact:
        zuul:
-          project:
-            src_dir: "{{ tripleo_src }}"
+          projects:
+            "opendev.org/openstack/validations-common":
+              src_dir: "{{ tripleo_src }}"
        ansible_connection: ssh

 - import_playbook: pre.yml
--- a/playbooks/molecule/run.yml
+++ b/playbooks/molecule/run.yml
@ -2,18 +2,38 @@
 - hosts: all
  environment:
    ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/zuul-output/logs/ansible-execution.log"
+  pre_tasks:
+
+    - name: Set project path fact
+      set_fact:
+        validations_common_project_path: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/validations-common'].src_dir }}"
+
+    - name: Set roles path fact
+      set_fact:
+        validations_common_roles_paths:
+          - "{{ validations_common_project_path }}/validations_common/roles"
+          - "/usr/share/ansible/roles"
+
+    - name: Set library path fact
+      set_fact:
+        validations_common_library_paths:
+          - "{{ validations_common_project_path }}/validations_common/library"
+          - "/usr/share/ansible/library"
+
  tasks:
    - name: Run role test job
      shell: |-
        . {{ ansible_user_dir }}/test-python/bin/activate
-        . {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/ansible-test-env.rc
-        pytest --color=no \
+        . {{ validations_common_project_path }}/ansible-test-env.rc
+        pytest --color=yes \
               --html={{ ansible_user_dir }}/zuul-output/logs/reports.html \
               --self-contained-html \
               --ansible-args='{{ tripleo_job_ansible_args | default("") }}' \
-               {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/test_molecule.py
+               {{ validations_common_project_path }}/tests/test_molecule.py
      args:
        chdir:
-          "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/validations_common/roles/{{ validations_common_role_name }}"
+          "{{ validations_common_project_path }}/validations_common/roles/{{ validations_common_role_name }}"
        executable: /bin/bash
-      changed_when: false
+      environment:
+        ANSIBLE_ROLES_PATH: "{{ validations_common_roles_paths | join(':') }}"
+        ANSIBLE_LIBRARY: "{{ validations_common_library_paths | join(':') }}"
--- a/scripts/run-local-test
+++ b/scripts/run-local-test
@ -15,16 +15,25 @@
 # under the License.


-## Shell Opts ----------------------------------------------------------------
-
-set -o pipefail
-set -xeuo
+## Functions -----------------------------------------------------------------
+function usage {
+    echo "Usage: ROLE_NAME=ROLE_NAME ${0##*/} or ${0##*/} ROLE_NAME"
+}

 ## Vars ----------------------------------------------------------------------

 export PROJECT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))/../"
+if [ "${ROLE_NAME}x" = "x" -a "${1}x" = "x" ]; then
+    usage;
+    exit 2
+fi
 export ROLE_NAME="${ROLE_NAME:-$1}"
-export TRIPLEO_JOB_ANSIBLE_ARGS=${TRIPLEO_JOB_ANSIBLE_ARGS:-"-v"}
+export TRIPLEO_JOB_ANSIBLE_ARGS=${TRIPLEO_JOB_ANSIBLE_ARGS:-""}
+
+## Shell Opts ----------------------------------------------------------------
+
+set -o pipefail
+set -xeuo

 ## Main ----------------------------------------------------------------------

@ -47,8 +56,11 @@ sudo touch /etc/ci/mirror_info.sh
 # Create a virtual env
 "${PYTHON_EXEC}" -m virtualenv --system-site-packages "${HOME}/test-python"

+# Activate a virtual env
+PS1="[\u@\h \W]\$" source "${HOME}/test-python/bin/activate"
+
 # Run bindep
-"${HOME}/test-python/bin/pip" install pip setuptools bindep --upgrade
+"${HOME}/test-python/bin/pip" install "pip>=19.1.1" setuptools bindep --upgrade
 "${PROJECT_DIR}/scripts/bindep-install"

 # Install local requirements
@ -61,7 +73,6 @@ fi
                              -r "${PROJECT_DIR}/molecule-requirements.txt"

 # Run local test
-PS1="[\u@\h \W]\$" source "${HOME}/test-python/bin/activate"
 source "${PROJECT_DIR}/ansible-test-env.rc"
 export ANSIBLE_ROLES_PATH="${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles"
 ansible-playbook -i "${PROJECT_DIR}/tests/hosts.ini" \
@ -71,4 +82,5 @@ ansible-playbook -i "${PROJECT_DIR}/tests/hosts.ini" \
                 -e "ansible_user=${USER}" \
                 -e "ansible_user_dir=${HOME}" \
                 "${PROJECT_DIR}/tests/prepare-test-host.yml" \
-                 "${PROJECT_DIR}/playbooks/molecule/run-local.yml"
+                 "${PROJECT_DIR}/playbooks/molecule/run-local.yml" \
+                 -v
--- a/tests/prepare-test-host.yml
+++ b/tests/prepare-test-host.yml
@ -44,11 +44,9 @@
        mode: "0700"

    - name: Create ssh key pair
-      user:
-        name: "{{ ansible_user }}"
-        generate_ssh_key: true
-        ssh_key_bits: 2048
-        ssh_key_file: "{{ ansible_user_dir }}/.ssh/id_rsa"
+      openssh_keypair:
+        path: "{{ ansible_user_dir }}/.ssh/id_rsa"
+        size: 2048

    - name: Slurp pub key
      slurp:
--- a/validations_common/roles/advanced_format_512e_support/molecule/default/molecule.yml
+++ b/validations_common/roles/advanced_format_512e_support/molecule/default/molecule.yml
@ -11,7 +11,7 @@ platforms:
    registry:
      url: registry.access.redhat.com
    dockerfile: Dockerfile
-    pkg_extras: python*setuptools python*-PyYAML
+    pkg_extras: python*setuptools python*-pyyaml
    volumes:
      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/check_cpu/molecule/default/molecule.yml
+++ b/validations_common/roles/check_cpu/molecule/default/molecule.yml
@ -34,6 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/check_disk_space/molecule/default/molecule.yml
+++ b/validations_common/roles/check_disk_space/molecule/default/molecule.yml
@ -34,6 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/check_latest_packages_version/molecule/default/molecule.yml
+++ b/validations_common/roles/check_latest_packages_version/molecule/default/molecule.yml
@ -11,7 +11,7 @@ platforms:
    registry:
      url: registry.access.redhat.com
    dockerfile: Dockerfile
-    pkg_extras: python*setuptools python*-PyYAML
+    pkg_extras: python*setuptools python*-pyyaml
    volumes:
      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/check_ram/molecule/default/molecule.yml
+++ b/validations_common/roles/check_ram/molecule/default/molecule.yml
@ -34,6 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/check_selinux_mode/molecule/default/molecule.yml
+++ b/validations_common/roles/check_selinux_mode/molecule/default/molecule.yml
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/dns/molecule/default/molecule.yml
+++ b/validations_common/roles/dns/molecule/default/molecule.yml
@ -34,6 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/haproxy/molecule/default/molecule.yml
+++ b/validations_common/roles/haproxy/molecule/default/molecule.yml
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/ntp/molecule/default/molecule.yml
+++ b/validations_common/roles/ntp/molecule/default/molecule.yml
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/service_status/molecule/default/Dockerfile.j2
+++ b/validations_common/roles/service_status/molecule/default/Dockerfile.j2
--- a/validations_common/roles/service_status/molecule/default/molecule.yml
+++ b/validations_common/roles/service_status/molecule/default/molecule.yml
@ -12,7 +12,7 @@ platforms:
      url: registry.access.redhat.com
    dockerfile: Dockerfile
    command: /sbin/init
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
+    pkg_extras: python*-setuptools python*-pyyaml
    volumes:
      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@ -35,7 +35,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/service_status/molecule/docker/Dockerfile.j2
+++ b/validations_common/roles/service_status/molecule/docker/Dockerfile.j2
--- a/validations_common/roles/service_status/molecule/docker/molecule.yml
+++ b/validations_common/roles/service_status/molecule/docker/molecule.yml
@ -11,7 +11,7 @@ platforms:
    registry:
      url: registry.access.redhat.com
    dockerfile: Dockerfile
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML python*-libselinux
+    pkg_extras: python*-setuptools python*-pyyaml python*-libselinux
    command: /sbin/init
    capabilities:
      - SYS_ADMIN
@ -36,7 +36,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/service_status/molecule/podman/Dockerfile.j2
+++ b/validations_common/roles/service_status/molecule/podman/Dockerfile.j2
--- a/validations_common/roles/service_status/molecule/podman/molecule.yml
+++ b/validations_common/roles/service_status/molecule/podman/molecule.yml
@ -11,7 +11,7 @@ platforms:
    registry:
      url: registry.access.redhat.com
    dockerfile: Dockerfile
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
+    pkg_extras: python*-setuptools python*-pyyaml
    volumes:
      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@ -34,7 +34,7 @@ provisioner:
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence:
--- a/validations_common/roles/service_status/molecule/systemd/Dockerfile.j2
+++ b/validations_common/roles/service_status/molecule/systemd/Dockerfile.j2
--- a/validations_common/roles/service_status/molecule/systemd/molecule.yml
+++ b/validations_common/roles/service_status/molecule/systemd/molecule.yml
@ -5,13 +5,18 @@ driver:
 log: true

 platforms:
-  - name: centos7
-    hostname: centos7
-    image: centos:7
-    pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools python*-pyyaml
+    volumes:
      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    easy_install:
-      - pip
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    privileged: true
    command: /sbin/init
    environment: &env
      http_proxy: "{{ lookup('env', 'http_proxy') }}"
@ -19,23 +24,18 @@ platforms:
    ulimits: &ulimit
      - host

-  - name: centos8
-    hostname: centos8
-    image: centos:8
-    command: /sbin/init
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
-    volumes:
-      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    environment:
-      <<: *env
-    ulimits: *ulimit
-
 provisioner:
  name: ansible
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
  log: true
  env:
    ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"

 scenario:
  test_sequence: