Merge "Migrate neutron-functional job to zuul v3 syntax"

.zuul.yaml

@@ -102,24 +102,46 @@
 
 - job:
     name: neutron-functional
-    parent: legacy-dsvm-base
+    parent: devstack-minimal
-    run: playbooks/legacy/neutron-functional/run.yaml
+    description: Run neutron functional tests
-    post-run: playbooks/legacy/neutron-functional/post.yaml
     timeout: 7800
     required-projects:
-      - openstack-infra/devstack-gate
+      - git.openstack.org/openstack-dev/devstack
       - openstack/neutron
+    roles:
+      - zuul: openstack-dev/devstack
+    pre-run: playbooks/configure_functional_job.yaml
+    run: playbooks/run_functional_job.yaml
+    post-run: playbooks/post_functional_job.yaml
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
       - ^neutron/locale/.*$
       - ^releasenotes/.*$
+    vars:
+      devstack_services:
+        # Ignore any default set by devstack. Emit a "disable_all_services".
+        base: false
+      devstack_localrc:
+        USE_PYTHON3: true
+        INSTALL_TESTONLY_PACKAGES: true
+        DATABASE_PASSWORD: stackdb
+      tox_envlist: dsvm-functional
+      tox_install_siblings: false
+      zuul_copy_output:
+        # We need to copy directory with logs to have it in job artifacts also,
+        # /opt/stack/logs is default logs directory defined in neutron's
+        # tox.ini file
+        '{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs
 
 - job:
     name: neutron-functional-python27
     parent: neutron-functional
-    run: playbooks/legacy/neutron-functional-python27/run.yaml
+    description: Run neutron functional tests with Python 2.7
-    post-run: playbooks/legacy/neutron-functional-python27/post.yaml
+    vars:
+      devstack_localrc:
+        USE_PYTHON3: false
+      tox_envlist: dsvm-functional-python27
 
 - job:
     name: neutron-fullstack

neutron/tests/contrib/gate_hook.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-VENV=${1:-"dsvm-functional"}
+VENV=${1:-"dsvm-fullstack"}
 FLAVOR=${2:-"all"}
 
 GATE_DEST=$BASE/new
@@ -55,7 +55,7 @@ function load_rc_for_rally {
 
 
 case $VENV in
-"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack")
+"dsvm-fullstack")
     # The following need to be set before sourcing
     # configure_for_func_testing.
     GATE_STACK_USER=stack

playbooks/configure_functional_job.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - setup_logdir
+    - configure_functional_tests

playbooks/legacy/neutron-functional-python27/post.yaml

@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/neutron-functional-python27/run.yaml

@@ -1,54 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function gate_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27
-          }
-          export -f gate_hook
-
-          function post_test_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/neutron-functional/post.yaml

@@ -1,96 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    # TODO(slaweq): this should be moved to separate role and used in zuulv3
-    # jobs definitions also
-    - name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log
-      become: yes
-      shell:
-        cmd: |
-          /bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log
-
-    - name: Set journal.log file permissions
-      become: yes
-      file:
-        path: '{{ ansible_user_dir }}/workspace/logs/journal.log'
-        owner: '{{ ansible_user }}'
-        group: '{{ ansible_user }}'
-        mode: 0644
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/neutron-functional/run.yaml

@@ -1,55 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export USE_PYTHON3=true
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function gate_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional
-          }
-          export -f gate_hook
-
-          function post_test_hook {
-              bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/post_functional_job.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  roles:
+    - fetch_journal_log
+    - fetch-tox-output
+    - fetch-subunit-output

playbooks/run_functional_job.yaml

@@ -0,0 +1,10 @@
+- hosts: all
+  roles:
+    - run-devstack
+    # Run bindep and test-setup after devstack so that they won't interfere
+    - role: bindep
+      bindep_profile: test
+      bindep_dir: "{{ zuul_work_dir }}"
+    - test-setup
+    - ensure-tox
+    - tox

roles/configure_functional_tests/README.rst

@@ -0,0 +1,18 @@
+Configure host to run on it Neutron functional/fullstack tests
+
+**Role Variables**
+
+.. zuul:rolevar:: tests_venv
+   :default: {{ tox_envlist }}
+
+.. zuul:rolevar:: base_dir
+   :default: {{ ansible_user_dir }}/src/git.openstack.org
+
+.. zuul:rolevar:: gate_dest_dir
+   :default: {{ base_dir }}/openstack
+
+.. zuul:rolevar:: devstack_dir
+   :default: {{ base_dir }}/openstack-dev/devstack
+
+.. zuul:rolevar:: neutron_dir
+   :default: {{ gate_dest_dir }}/neutron

roles/configure_functional_tests/defaults/main.yaml

@@ -0,0 +1,5 @@
+tests_venv: "{{ tox_envlist }}"
+base_dir: "{{ ansible_user_dir }}/src/git.openstack.org"
+gate_dest_dir: "{{ base_dir }}/openstack"
+devstack_dir: "{{ base_dir }}/openstack-dev/devstack"
+neutron_dir: "{{ gate_dest_dir }}/neutron"

roles/configure_functional_tests/tasks/main.yaml

@@ -0,0 +1,19 @@
+- shell:
+    cmd: |
+        set -e
+        set -x
+        GATE_STACK_USER={{ ansible_user }}
+        IS_GATE=True
+
+        BASE_DIR={{ base_dir }}
+        GATE_DEST={{ gate_dest_dir }}
+        NEUTRON_PATH={{ neutron_dir }}
+        DEVSTACK_PATH={{ devstack_dir }}
+        VENV={{ tests_venv }}
+
+        source $DEVSTACK_PATH/functions
+        source $NEUTRON_PATH/devstack/lib/ovs
+        source $NEUTRON_PATH/tools/configure_for_func_testing.sh
+
+        configure_host_for_func_testing
+    executable: /bin/bash

roles/fetch_journal_log/README.rst

@@ -0,0 +1,16 @@
+Collect journal log from test run
+
+By default, this stores journal log into log file and store it in
+"journal_log_file_path"
+
+**Role Variables**
+
+.. zuul:rolevar:: journal_log_path
+   :default: {{ ansible_user_dir }}/workspace/logs
+
+   Path where journal log file will be stored on job's node.
+
+.. zuul:rolevar:: journal_log_file_name
+   :default: {{ journal_log_path }}/journal.log
+
+   Name of journal log file.

roles/fetch_journal_log/defaults/main.yaml

@@ -0,0 +1,2 @@
+journal_log_path: "{{ ansible_user_dir }}/logs"
+journal_log_file_name: "{{ journal_log_path }}/journal.log"

roles/fetch_journal_log/tasks/main.yaml

@@ -0,0 +1,23 @@
+- name: Ensure {{ journal_log_path }} exists
+  become: yes
+  msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission"
+  file:
+    path: "{{ journal_log_path }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0775
+
+- name: Store journal logs in {{ journal_log_file_name }}
+  become: yes
+  shell:
+    cmd: |
+      /bin/journalctl -a > {{ journal_log_file_name }}
+
+- name: Set journal.log file permissions
+  become: yes
+  file:
+    path: '{{ journal_log_file_name }}'
+    owner: '{{ ansible_user }}'
+    group: '{{ ansible_user }}'
+    mode: 0644

roles/setup_logdir/README.rst

@@ -0,0 +1,8 @@
+Configure logs dir to be accessible for ``stack`` user.
+
+**Role Variables**
+
+.. zuul:rolevar:: logdir
+   :default: /opt/stack/logs
+
+   Name of the directory where logs will be stored.

roles/setup_logdir/defaults/main.yaml

@@ -0,0 +1 @@
+logdir: /opt/stack/logs

roles/setup_logdir/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Ensure logdir exists
+  become: yes
+  msg: "Ensure logdir {{ logdir }} exists and have correct permission"
+  file:
+    path: "{{ logdir }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0775