diff --git a/.zuul.yaml b/.zuul.yaml index a2e9396d083..3e7c6355c70 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -102,24 +102,46 @@ - job: name: neutron-functional - parent: legacy-dsvm-base - run: playbooks/legacy/neutron-functional/run.yaml - post-run: playbooks/legacy/neutron-functional/post.yaml + parent: devstack-minimal + description: Run neutron functional tests timeout: 7800 required-projects: - - openstack-infra/devstack-gate + - git.openstack.org/openstack-dev/devstack - openstack/neutron + roles: + - zuul: openstack-dev/devstack + pre-run: playbooks/configure_functional_job.yaml + run: playbooks/run_functional_job.yaml + post-run: playbooks/post_functional_job.yaml irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - ^neutron/locale/.*$ - ^releasenotes/.*$ + vars: + devstack_services: + # Ignore any default set by devstack. Emit a "disable_all_services". + base: false + devstack_localrc: + USE_PYTHON3: true + INSTALL_TESTONLY_PACKAGES: true + DATABASE_PASSWORD: stackdb + tox_envlist: dsvm-functional + tox_install_siblings: false + zuul_copy_output: + # We need to copy directory with logs to have it in job artifacts also, + # /opt/stack/logs is default logs directory defined in neutron's + # tox.ini file + '{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs - job: name: neutron-functional-python27 parent: neutron-functional - run: playbooks/legacy/neutron-functional-python27/run.yaml - post-run: playbooks/legacy/neutron-functional-python27/post.yaml + description: Run neutron functional tests with Python 2.7 + vars: + devstack_localrc: + USE_PYTHON3: false + tox_envlist: dsvm-functional-python27 - job: name: neutron-fullstack diff --git a/neutron/tests/contrib/gate_hook.sh b/neutron/tests/contrib/gate_hook.sh index 6ec291d8392..86673734e00 100644 --- a/neutron/tests/contrib/gate_hook.sh +++ b/neutron/tests/contrib/gate_hook.sh @@ -2,7 +2,7 @@ set -ex -VENV=${1:-"dsvm-functional"} +VENV=${1:-"dsvm-fullstack"} FLAVOR=${2:-"all"} GATE_DEST=$BASE/new @@ -55,7 +55,7 @@ function load_rc_for_rally { case $VENV in -"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack") +"dsvm-fullstack") # The following need to be set before sourcing # configure_for_func_testing. GATE_STACK_USER=stack diff --git a/playbooks/configure_functional_job.yaml b/playbooks/configure_functional_job.yaml new file mode 100644 index 00000000000..01dccba3866 --- /dev/null +++ b/playbooks/configure_functional_job.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - setup_logdir + - configure_functional_tests diff --git a/playbooks/legacy/neutron-functional-python27/post.yaml b/playbooks/legacy/neutron-functional-python27/post.yaml deleted file mode 100644 index dac875340aa..00000000000 --- a/playbooks/legacy/neutron-functional-python27/post.yaml +++ /dev/null @@ -1,80 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/neutron-functional-python27/run.yaml b/playbooks/legacy/neutron-functional-python27/run.yaml deleted file mode 100644 index 538a66a8909..00000000000 --- a/playbooks/legacy/neutron-functional-python27/run.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - - function gate_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27 - } - export -f gate_hook - - function post_test_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27 - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/legacy/neutron-functional/post.yaml b/playbooks/legacy/neutron-functional/post.yaml deleted file mode 100644 index 3b3df752f8f..00000000000 --- a/playbooks/legacy/neutron-functional/post.yaml +++ /dev/null @@ -1,96 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - # TODO(slaweq): this should be moved to separate role and used in zuulv3 - # jobs definitions also - - name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log - become: yes - shell: - cmd: | - /bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log - - - name: Set journal.log file permissions - become: yes - file: - path: '{{ ansible_user_dir }}/workspace/logs/journal.log' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' - mode: 0644 - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/neutron-functional/run.yaml b/playbooks/legacy/neutron-functional/run.yaml deleted file mode 100644 index 397178ca01d..00000000000 --- a/playbooks/legacy/neutron-functional/run.yaml +++ /dev/null @@ -1,55 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export USE_PYTHON3=true - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - - function gate_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional - } - export -f gate_hook - - function post_test_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/post_functional_job.yaml b/playbooks/post_functional_job.yaml new file mode 100644 index 00000000000..2537f1cbd9a --- /dev/null +++ b/playbooks/post_functional_job.yaml @@ -0,0 +1,5 @@ +- hosts: all + roles: + - fetch_journal_log + - fetch-tox-output + - fetch-subunit-output diff --git a/playbooks/run_functional_job.yaml b/playbooks/run_functional_job.yaml new file mode 100644 index 00000000000..e85c2eee965 --- /dev/null +++ b/playbooks/run_functional_job.yaml @@ -0,0 +1,10 @@ +- hosts: all + roles: + - run-devstack + # Run bindep and test-setup after devstack so that they won't interfere + - role: bindep + bindep_profile: test + bindep_dir: "{{ zuul_work_dir }}" + - test-setup + - ensure-tox + - tox diff --git a/roles/configure_functional_tests/README.rst b/roles/configure_functional_tests/README.rst new file mode 100644 index 00000000000..0fb2f575289 --- /dev/null +++ b/roles/configure_functional_tests/README.rst @@ -0,0 +1,18 @@ +Configure host to run on it Neutron functional/fullstack tests + +**Role Variables** + +.. zuul:rolevar:: tests_venv + :default: {{ tox_envlist }} + +.. zuul:rolevar:: base_dir + :default: {{ ansible_user_dir }}/src/git.openstack.org + +.. zuul:rolevar:: gate_dest_dir + :default: {{ base_dir }}/openstack + +.. zuul:rolevar:: devstack_dir + :default: {{ base_dir }}/openstack-dev/devstack + +.. zuul:rolevar:: neutron_dir + :default: {{ gate_dest_dir }}/neutron diff --git a/roles/configure_functional_tests/defaults/main.yaml b/roles/configure_functional_tests/defaults/main.yaml new file mode 100644 index 00000000000..e849e80429b --- /dev/null +++ b/roles/configure_functional_tests/defaults/main.yaml @@ -0,0 +1,5 @@ +tests_venv: "{{ tox_envlist }}" +base_dir: "{{ ansible_user_dir }}/src/git.openstack.org" +gate_dest_dir: "{{ base_dir }}/openstack" +devstack_dir: "{{ base_dir }}/openstack-dev/devstack" +neutron_dir: "{{ gate_dest_dir }}/neutron" diff --git a/roles/configure_functional_tests/tasks/main.yaml b/roles/configure_functional_tests/tasks/main.yaml new file mode 100644 index 00000000000..8e4f791de4c --- /dev/null +++ b/roles/configure_functional_tests/tasks/main.yaml @@ -0,0 +1,19 @@ +- shell: + cmd: | + set -e + set -x + GATE_STACK_USER={{ ansible_user }} + IS_GATE=True + + BASE_DIR={{ base_dir }} + GATE_DEST={{ gate_dest_dir }} + NEUTRON_PATH={{ neutron_dir }} + DEVSTACK_PATH={{ devstack_dir }} + VENV={{ tests_venv }} + + source $DEVSTACK_PATH/functions + source $NEUTRON_PATH/devstack/lib/ovs + source $NEUTRON_PATH/tools/configure_for_func_testing.sh + + configure_host_for_func_testing + executable: /bin/bash diff --git a/roles/fetch_journal_log/README.rst b/roles/fetch_journal_log/README.rst new file mode 100644 index 00000000000..71276c3f6a4 --- /dev/null +++ b/roles/fetch_journal_log/README.rst @@ -0,0 +1,16 @@ +Collect journal log from test run + +By default, this stores journal log into log file and store it in +"journal_log_file_path" + +**Role Variables** + +.. zuul:rolevar:: journal_log_path + :default: {{ ansible_user_dir }}/workspace/logs + + Path where journal log file will be stored on job's node. + +.. zuul:rolevar:: journal_log_file_name + :default: {{ journal_log_path }}/journal.log + + Name of journal log file. diff --git a/roles/fetch_journal_log/defaults/main.yaml b/roles/fetch_journal_log/defaults/main.yaml new file mode 100644 index 00000000000..dfc540ba9d1 --- /dev/null +++ b/roles/fetch_journal_log/defaults/main.yaml @@ -0,0 +1,2 @@ +journal_log_path: "{{ ansible_user_dir }}/logs" +journal_log_file_name: "{{ journal_log_path }}/journal.log" diff --git a/roles/fetch_journal_log/tasks/main.yaml b/roles/fetch_journal_log/tasks/main.yaml new file mode 100644 index 00000000000..c67ecd075f1 --- /dev/null +++ b/roles/fetch_journal_log/tasks/main.yaml @@ -0,0 +1,23 @@ +- name: Ensure {{ journal_log_path }} exists + become: yes + msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission" + file: + path: "{{ journal_log_path }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0775 + +- name: Store journal logs in {{ journal_log_file_name }} + become: yes + shell: + cmd: | + /bin/journalctl -a > {{ journal_log_file_name }} + +- name: Set journal.log file permissions + become: yes + file: + path: '{{ journal_log_file_name }}' + owner: '{{ ansible_user }}' + group: '{{ ansible_user }}' + mode: 0644 diff --git a/roles/setup_logdir/README.rst b/roles/setup_logdir/README.rst new file mode 100644 index 00000000000..159cc8c7ec9 --- /dev/null +++ b/roles/setup_logdir/README.rst @@ -0,0 +1,8 @@ +Configure logs dir to be accessible for ``stack`` user. + +**Role Variables** + +.. zuul:rolevar:: logdir + :default: /opt/stack/logs + + Name of the directory where logs will be stored. diff --git a/roles/setup_logdir/defaults/main.yaml b/roles/setup_logdir/defaults/main.yaml new file mode 100644 index 00000000000..f056848826e --- /dev/null +++ b/roles/setup_logdir/defaults/main.yaml @@ -0,0 +1 @@ +logdir: /opt/stack/logs diff --git a/roles/setup_logdir/tasks/main.yaml b/roles/setup_logdir/tasks/main.yaml new file mode 100644 index 00000000000..1476ae2a5b5 --- /dev/null +++ b/roles/setup_logdir/tasks/main.yaml @@ -0,0 +1,9 @@ +- name: Ensure logdir exists + become: yes + msg: "Ensure logdir {{ logdir }} exists and have correct permission" + file: + path: "{{ logdir }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0775