diff --git a/.zuul.yaml b/.zuul.yaml index 48c26b13..56cc071b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,68 +1,348 @@ -- job: - name: grenade-postgresql - parent: legacy-dsvm-base - run: playbooks/legacy/grenade-postgresql/run.yaml - post-run: playbooks/legacy/grenade-postgresql/post.yaml - timeout: 10800 - required-projects: - - openstack/grenade - - openstack/devstack-gate - irrelevant-files: &grenade-irrelevant-files - - ^.*\.rst$ - - ^doc/.*$ - -- job: - name: grenade-dsvm-neutron-forward - # Old name: grenade-dsvm-neutron-forward - parent: legacy-dsvm-base - # This is py3 version for ussuri onwards rest all branch needs to be py2 - # version which is present in openstack-zuul-jobs. - # We need to take care of this branch variant and python version while - # migrating these jobs to zuulv3. - branches: ^(?!(stable/(ocata|pike|queens|rocky|stein|train))).*$ - run: playbooks/legacy/grenade-dsvm-neutron-forward/run.yaml - post-run: playbooks/legacy/grenade-dsvm-neutron-forward/post.yaml - timeout: 10800 - required-projects: - - openstack/grenade - - openstack/devstack-gate - - openstack/neutron - -- job: - name: grenade-py3 - parent: legacy-dsvm-base - run: playbooks/legacy/grenade-py3/run.yaml - post-run: playbooks/legacy/grenade-py3/post.yaml - timeout: 10800 - required-projects: - - openstack/grenade - - openstack/devstack-gate - irrelevant-files: *grenade-irrelevant-files - - project: templates: - publish-openstack-docs-pti check: jobs: + - grenade - grenade-heat: voting: false - irrelevant-files: *grenade-irrelevant-files - - grenade-py3 - - neutron-grenade-multinode: - irrelevant-files: *grenade-irrelevant-files + - grenade-multinode - ironic-grenade-dsvm: voting: false - irrelevant-files: *grenade-irrelevant-files + irrelevant-files: &legacy-irrelevant-files + - ^.*\.rst$ + - ^doc/.*$ gate: jobs: - - grenade-py3 - - neutron-grenade-multinode: - irrelevant-files: *grenade-irrelevant-files + - grenade + - grenade-multinode experimental: jobs: - grenade-postgresql - - grenade-dsvm-neutron-forward: - irrelevant-files: *grenade-irrelevant-files - trove-grenade: - irrelevant-files: *grenade-irrelevant-files + irrelevant-files: *legacy-irrelevant-files +- job: + name: grenade-base + parent: multinode + abstract: true + description: | + Base grenade job + + This is the base grenade job for both single and multi node grenade + tests. It does not define any service to be used, so that this can + be used to build an upgrade test with a minimal number of services. + required-projects: + - opendev.org/openstack/devstack + - opendev.org/openstack/requirements + - opendev.org/openstack/tempest + roles: + - zuul: opendev.org/openstack/openstack-zuul-jobs + - zuul: opendev.org/openstack/devstack + - zuul: opendev.org/openstack/tempest + vars: + devstack_bases: + shared: /opt/stack + old: /opt/stack/old + new: /opt/stack/new + logs_old: /opt/stack/logs/old + logs_new: /opt/stack/logs/new + grenade_devstack_localrc: &grenade_base_devstack_localrc + old: + DEST: "{{ devstack_bases.old }}" + LOGFILE: "{{ devstack_bases.old }}/logs/devstacklog.txt" + new: + DEST: "{{ devstack_bases.new }}" + LOGFILE: "{{ devstack_bases.new }}/logs/devstacklog.txt" + shared: + USE_PYTHON3: true + # Common settings for OpenStack CI + DATABASE_PASSWORD: secretdatabase + # FIXME: temporarily define MYSQL_PASSWORD too, + # because some code in devstack and in grenade + # still requires it to be defined. + MYSQL_PASSWORD: secretdatabase + RABBIT_PASSWORD: secretrabbit + ADMIN_PASSWORD: secretadmin + SERVICE_PASSWORD: secretservice + NETWORK_GATEWAY: 10.1.0.1 + FIXED_RANGE: 10.1.0.0/20 + IPV4_ADDRS_SAFE_TO_USE: 10.1.0.0/20 + FLOATING_RANGE: 172.24.5.0/24 + PUBLIC_NETWORK_GATEWAY: 172.24.5.1 + FIXED_NETWORK_SIZE: 4096 + LOG_COLOR: false + VERBOSE: true + VERBOSE_NO_TIMESTAMP: true + NOVNC_FROM_PACKAGE: true + ERROR_ON_CLONE: true + # Gate jobs can't deal with nested virt. Disable it. + LIBVIRT_TYPE: qemu + grenade_localrc: + SAVE_DIR: "{{ devstack_bases.shared }}/save" + # from tempest playbooks + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + compute: + min_compute_nodes: "{{ groups['compute'] | default(['controller']) | length }}" + test_results_stage_name: test_results + zuul_copy_output: + '{{ devstack_bases.logs_old }}': logs + '{{ devstack_bases.logs_new }}': logs + '{{ grenade_save_dir }}': logs + '{{ grenade_log_dir }}/grenade.sh.log': logs + '{{ grenade_log_dir }}/grenade.sh.log.summary': logs + '{{ grenade_conf_dir }}/grenade_localrc': logs_txt + '{{ grenade_conf_dir }}/grenade_pluginrc': logs_txt + '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs + '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs + '{{ stage_dir }}/stackviz': logs + '{{ stage_dir }}/apache': logs + '{{ stage_dir }}/apache_config': logs + '{{ stage_dir }}/etc': logs + '/var/log/rabbitmq': logs + '/var/log/postgresql': logs + '/var/log/mysql': logs + '/var/log/libvirt': logs + '/etc/sudoers': logs + '/etc/sudoers.d': logs + '{{ stage_dir }}/iptables.txt': logs + '{{ stage_dir }}/df.txt': logs + '{{ stage_dir }}/pip2-freeze.txt': logs + '{{ stage_dir }}/pip3-freeze.txt': logs + '{{ stage_dir }}/dpkg-l.txt': logs + '{{ stage_dir }}/rpm-qa.txt': logs + '{{ stage_dir }}/core': logs + '{{ stage_dir }}/listen53.txt': logs + '{{ stage_dir }}/deprecations.log': logs + '/var/log/ceph': logs + '/var/log/openvswitch': logs + '/var/log/glusterfs': logs + '/etc/glusterfs/glusterd.vol': logs + '/etc/resolv.conf': logs + '/var/log/unbound.log': logs + extensions_to_txt: + auto: True + conf: True + log: True + localrc: True + stackenv: True + summary: True + group-vars: + subnode: + grenade_devstack_localrc: + <<: *grenade_base_devstack_localrc + pre-run: playbooks/pre.yaml + run: playbooks/grenade.yaml + post-run: playbooks/post.yaml + irrelevant-files: &grenade-irrelevant-files + # Documentation related + - ^.*\.rst$ + - ^api-ref/.*$ + - ^doc/.*$ + - ^releasenotes/.*$ + # Translations + - ^.*/locale/.*po$ + +- job: + name: grenade + parent: grenade-base + description: | + Default grenade job + + Usually grenade jobs should inherit from this job. + nodeset: openstack-single-node-bionic + required-projects: + # NOTE(andreaf) The devstack and grenade roles take care of setting up + # repos on the right branch for this job. + - opendev.org/openstack/cinder + - opendev.org/openstack/glance + - opendev.org/openstack/keystone + - opendev.org/openstack/neutron + - opendev.org/openstack/nova + - opendev.org/openstack/placement + - opendev.org/openstack/swift + timeout: 10800 + vars: + grenade_devstack_localrc: + shared: + # Common OpenStack services settings + SWIFT_REPLICAS: 1 + SWIFT_START_ALL_SERVICES: false + SWIFT_HASH: 1234123412341234 + CINDER_PERIODIC_INTERVAL: 10 + DEBUG_LIBVIRT_COREDUMPS: True + # Multinode specific settings + SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + HOST_IP: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + PUBLIC_BRIDGE_MTU: "{{ external_bridge_mtu }}" + CELLSV2_SETUP: "singleconductor" + # Node specific settings + DATABASE_TYPE: mysql + devstack_local_conf: + post-config: + $NEUTRON_CONF: + DEFAULT: + global_physnet_mtu: "{{ external_bridge_mtu }}" + grenade_localrc: + # enable the execution of the base smoke tests by default + BASE_RUN_SMOKE: True + devstack_services: + # Core services enabled for this branch. + # This list replaces the test-matrix. + # Shared services + dstat: true + etcd3: true + mysql: true + peakmem_tracker: true + rabbit: true + # TODO: tls-proxy is not tested by legacy jobs; + # enable this after testing. + tls-proxy: false + # Keystone services + key: true + # Glance services + g-api: true + g-reg: true + # Nova services + n-api: true + n-api-meta: true + n-cauth: true + n-cond: true + n-cpu: true + n-novnc: true + n-obj: true + n-sch: true + placement-api: true + # Neutron services + # We need to keep using the neutron-legacy based services for + # now until all issues with the new lib/neutron code are solved + q-agt: true + q-dhcp: true + q-l3: true + q-meta: true + q-metering: true + q-svc: true + # neutron-api: true + # neutron-agent: true + # neutron-dhcp: true + # neutron-l3: true + # neutron-metadata-agent: true + # neutron-metering: true + # Swift services + s-account: true + s-container: true + s-object: true + s-proxy: true + # Cinder services + c-api: true + c-bak: true + c-sch: true + c-vol: true + cinder: true + # Tempest was enabled by default even before the native Zuul v3 job + tempest: true + # Services we don't need. + # This section is not really needed, it's for readability. + horizon: false + # Test matrix emits ceilometer but ceilometer is not installed in the + # integrated gate, so specifying the services has not effect. + # ceilometer-*: false + group-vars: + subnode: + devstack_services: + # Core services enabled for this branch. + # This list replaces the test-matrix. + # Shared services + dstat: true + peakmem_tracker: true + # TODO: tls-proxy is not tested by legacy jobs; + # enable this after testing. + tls-proxy: false + # Nova services + n-cpu: true + placement-client: true + # Neutron services + neutron-agent: true + # Cinder services + c-bak: true + c-vol: true + # Services we don't run at all on subnode. + # This section is not really needed, it's for readability. + # keystone: false + # s-*: false + horizon: false + tempest: false + # Test matrix emits ceilometer but ceilomenter is not installed in the + # integrated gate, so specifying the services has not effect. + # ceilometer-*: false + grenade_devstack_localrc: + shared: + # Multinode specific settings + HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" + SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + PUBLIC_BRIDGE_MTU: "{{ external_bridge_mtu }}" + CELLSV2_SETUP: "singleconductor" + # Subnode specific settings + DATABASE_TYPE: mysql + GLANCE_HOSTPORT: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}:9292" + Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + RABBIT_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + DATABASE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + +- job: + name: grenade-py3 + parent: grenade + description: | + Grenade job based on Python 3 + vars: + grenade_devstack_localrc: + shared: + USE_PYTHON3: true + +- job: + name: grenade-forward + parent: grenade + description: | + Forward test. This is mostly relevant for stable branches, + because running it in master will execute an upgrade + to the same commit. + Former names for this job were: + * grenade-dsvm-neutron-forward + * legacy-grenade-dsvm-neutron-forward + vars: + # When a stable branch stable/X is created from master, + # this value should be updated to stable/X. + grenade_from_branch: master + +- job: + name: grenade-dsvm-neutron-forward + branches: ^(?!(stable/(ocata|pike|queens|rocky|stein|train))).*$ + parent: grenade-forward + +- job: + name: grenade-postgresql + description: | + Default grenade job which uses PostgreSQL + parent: grenade + vars: + grenade_devstack_localrc: + shared: + DATABASE_TYPE: postgresql + devstack_services: + mysql: false + postgresql: true + group-vars: + subnode: + devstack_services: + grenade_devstack_localrc: + shared: + DATABASE_TYPE: postgresql + +- job: + name: grenade-multinode + description: | + Basic multinode grenade job + parent: grenade + nodeset: openstack-two-node-bionic diff --git a/PLUGINS.rst b/PLUGINS.rst index 3b065a47..16f29a4f 100644 --- a/PLUGINS.rst +++ b/PLUGINS.rst @@ -222,11 +222,9 @@ The plugin is enabled by adding:: To ``pluginrc`` in the ``GRENADE_DIR``. An additional rc file was required due to sequencing of when plugin functions become available. -Note: For using this with ``openstack-infra/devstack-gate`` you would set this -by setting the GRENADE_PLUGINRC environment variable. For example to set it for -Ironic:: - - export GRENADE_PLUGINRC="enable_grenade_plugin ironic https://opendev.org/openstack/ironic" +Note: when running a job based on the ``grenade-base`` job, +for each devstack plugin defined using the ``devstack_plugins``, +the corresponding grenade plugin is enabled automatically. Changing Devstack Localrc diff --git a/grenade.sh b/grenade.sh index 5b82bd3d..6f36f93b 100755 --- a/grenade.sh +++ b/grenade.sh @@ -290,13 +290,15 @@ if [[ "$RUN_BASE" == "True" ]]; then echo_summary "Running base smoke test" cd $BASE_RELEASE_DIR/tempest tox -esmoke -- --concurrency=$TEMPEST_CONCURRENCY - # once we are done, copy our created artifacts to the target - if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then - rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/ - if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then - rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/ - elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then - rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/ + if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then + # once we are done, copy our created artifacts to the target + if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then + rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/ + if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then + rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/ + elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then + rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/ + fi fi fi fi diff --git a/playbooks/grenade.yaml b/playbooks/grenade.yaml new file mode 100644 index 00000000..494673a8 --- /dev/null +++ b/playbooks/grenade.yaml @@ -0,0 +1,36 @@ +- hosts: all + strategy: linear + roles: + - role: orchestrate-devstack + devstack_base_dir: "{{ devstack_bases.old }}" + devstack_data_base_dir: "{{ devstack_bases.shared }}" + +- hosts: controller + roles: + - configure-grenade-branches + - role: write-grenade-conf + base_dir: "{{ devstack_bases.shared }}" + grenade_base_dir: "{{ devstack_bases.new }}" + grenade_plugins: "{{ devstack_plugins|default({}) }}" + # run-grenade is configured to run also tempest smoke tests + # on the old node by default. + - role: run-grenade + base_dir: "{{ devstack_bases.shared }}" + grenade_base_dir: "{{ devstack_bases.new }}" + +# Copied (with minimal changes) from tempest.git:playbooks/devstack-tempest.yaml +# We run tests only on one node, regardless how many nodes are in the system, +# after grenade completes. +- hosts: tempest + environment: + # This enviroment variable is used by the optional tempest-gabbi + # job provided by the gabbi-tempest plugin. It can be safely ignored + # if that plugin is not being used. + GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}" + vars: + devstack_base_dir: "{{ devstack_bases.new }}" + roles: + - setup-tempest-run-dir + - setup-tempest-data-dir + - acl-devstack-files + - run-tempest diff --git a/playbooks/legacy/grenade-dsvm-neutron-forward/post.yaml b/playbooks/legacy/grenade-dsvm-neutron-forward/post.yaml deleted file mode 100644 index e07f5510..00000000 --- a/playbooks/legacy/grenade-dsvm-neutron-forward/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/grenade-dsvm-neutron-forward/run.yaml b/playbooks/legacy/grenade-dsvm-neutron-forward/run.yaml deleted file mode 100644 index 2306d784..00000000 --- a/playbooks/legacy/grenade-dsvm-neutron-forward/run.yaml +++ /dev/null @@ -1,43 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-grenade-dsvm-neutron-forward from old job gate-grenade-dsvm-neutron-forward-ubuntu-xenial - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - https://opendev.org \ - openstack/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PROJECTS="openstack/grenade $PROJECTS" - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=1 - export DEVSTACK_GATE_GRENADE=forward - export DEVSTACK_GATE_NEUTRON=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/legacy/grenade-postgresql/post.yaml b/playbooks/legacy/grenade-postgresql/post.yaml deleted file mode 100644 index e07f5510..00000000 --- a/playbooks/legacy/grenade-postgresql/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/grenade-postgresql/run.yaml b/playbooks/legacy/grenade-postgresql/run.yaml deleted file mode 100644 index 55c833e7..00000000 --- a/playbooks/legacy/grenade-postgresql/run.yaml +++ /dev/null @@ -1,44 +0,0 @@ -- hosts: all - name: Copy of the neutron-grenade job but with postgresql for the database. - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - https://opendev.org \ - openstack/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PROJECTS="openstack/grenade $PROJECTS" - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=1 - export DEVSTACK_GATE_GRENADE=pullup - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_POSTGRES=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/legacy/grenade-py3/post.yaml b/playbooks/legacy/grenade-py3/post.yaml deleted file mode 100644 index e07f5510..00000000 --- a/playbooks/legacy/grenade-py3/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/grenade-py3/run.yaml b/playbooks/legacy/grenade-py3/run.yaml deleted file mode 100644 index 2e54bdf4..00000000 --- a/playbooks/legacy/grenade-py3/run.yaml +++ /dev/null @@ -1,44 +0,0 @@ -- hosts: all - name: Copy of the neutron-grenade job using python3. - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - https://opendev.org \ - openstack/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PROJECTS="openstack/grenade $PROJECTS" - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=1 - export DEVSTACK_GATE_GRENADE=pullup - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_USE_PYTHON3=True - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/post.yaml b/playbooks/post.yaml new file mode 100644 index 00000000..3d016994 --- /dev/null +++ b/playbooks/post.yaml @@ -0,0 +1,35 @@ +- hosts: all + become: True + vars: + grenade_conf_dir: "{{ devstack_bases.new }}/grenade" + grenade_save_dir: "{{ devstack_bases.shared }}/grenade_save" + grenade_log_dir: "{{ devstack_bases.shared }}/logs" + roles: + - role: prepare-grenade-logs + grenade_config_dir: "{{ grenade_conf_dir }}" + grenade_devstack_shared_dir: "{{ devstack_bases.shared }}" + grenade_devstack_data_log_dirs: + 'old': "{{ devstack_bases.logs_old }}" + 'new': "{{ devstack_bases.logs_new }}" + grenade_save_data_dir: "{{ grenade_save_dir }}" + - role: export-devstack-journal + devstack_base_dir: "{{ devstack_bases.old }}" + - apache-logs-conf + - devstack-project-conf + # capture-system-logs should be the last role before stage-output + - capture-system-logs + - role: stage-output + # NOTE(andreaf) We need fetch-devstack-log-dir only as long as the base job + # starts pulling logs for us from {{ ansible_user_dir }}/logs. + # Meanwhile we already store things in ansible_user_dir and use + # fetch-devstack-log-dir setting devstack_base_dir + - role: fetch-devstack-log-dir + devstack_base_dir: "{{ ansible_user_dir }}" + +# Copied from tempest.git:playbooks/post-tempest.yaml +- hosts: tempest + become: true + roles: + - role: fetch-subunit-output + zuul_work_dir: '{{ devstack_bases.new }}/tempest' + - role: process-stackviz diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml new file mode 100644 index 00000000..f8d0a3d8 --- /dev/null +++ b/playbooks/pre.yaml @@ -0,0 +1,51 @@ +- hosts: all + pre_tasks: + - name: Gather minimum local MTU + set_fact: + local_mtu: > + {% set mtus = [] -%} + {% for interface in ansible_interfaces -%} + {% set _ = mtus.append(hostvars[inventory_hostname]['ansible_' + interface]['mtu']|int) -%} + {% endfor -%} + {{- mtus|min -}} + - name: Calculate external_bridge_mtu + # 50 bytes is overhead for vxlan (which is greater than GRE + # allowing us to use either overlay option with this MTU. + # TODO(andreaf) This should work, but it may have to be reconcilied with + # the MTU setting used by the multinode setup roles in multinode pre.yaml + set_fact: + external_bridge_mtu: "{{ local_mtu | int - 50 }}" + roles: + - configure-swap + - setup-stack-user + - setup-tempest-user + - configure-grenade-branches + - role: setup-devstack-source-dirs + devstack_base_dir: "{{ devstack_bases.old }}" + devstack_sources_branch: "{{ grenade_from_branch }}" + # NOTE(andreaf) Grenade pullup jobs migrate from a stable branch to + # the current patch(es). Zuul prepares all repos checked out to the right + # refspec to include all the right patches, so we should re-use that, + # and not specify any "devstack_sources_branch", unless a grenade_to_branch + # is set. + - role: setup-devstack-source-dirs + devstack_base_dir: "{{ devstack_bases.new }}" + devstack_sources_branch: "{{ grenade_to_branch | default(omit)}}" + # Create two separate log dirs + - role: setup-devstack-log-dir + devstack_base_dir: "{{ devstack_bases.old }}" + - role: setup-devstack-log-dir + devstack_base_dir: "{{ devstack_bases.new }}" + # Setup the logging (only once, as it does not depend on the branch) + - role: start-fresh-logging + devstack_base_dir: "{{ devstack_bases.old }}" + # Write devstack configuration for old and new (almost the same) + - role: write-devstack-local-conf + devstack_base_dir: "{{ devstack_bases.old }}" + devstack_localrc: "{{ grenade_devstack_localrc.shared|combine(grenade_devstack_localrc.old) }}" + - role: write-devstack-local-conf + devstack_base_dir: "{{ devstack_bases.new }}" + devstack_localrc: "{{ grenade_devstack_localrc.shared|combine(grenade_devstack_localrc.new) }}" + # Setup the cache location (used by the initial installation) + - role: setup-devstack-cache + devstack_base_dir: "{{ devstack_bases.old }}" diff --git a/roles/configure-grenade-branches/README.rst b/roles/configure-grenade-branches/README.rst new file mode 100644 index 00000000..06c7f28b --- /dev/null +++ b/roles/configure-grenade-branches/README.rst @@ -0,0 +1,15 @@ +Set the value of grenade_from_branch and grenade_to_branch +when not specified by the user. +The default values must be updated when grenade is branched. + +**Role Variables** + +.. zuul:rolevar:: grenade_from_branch + :default: + + The base branch for the upgrade. + +.. zuul:rolevar:: grenade_to_branch + :default: + + The target branch for the upgrade. diff --git a/roles/configure-grenade-branches/defaults/main.yaml b/roles/configure-grenade-branches/defaults/main.yaml new file mode 100644 index 00000000..574207bf --- /dev/null +++ b/roles/configure-grenade-branches/defaults/main.yaml @@ -0,0 +1,3 @@ +--- +grenade_from_branch: stable/train +grenade_to_branch: master diff --git a/roles/configure-grenade-branches/tasks/main.yaml b/roles/configure-grenade-branches/tasks/main.yaml new file mode 100644 index 00000000..06d56b4d --- /dev/null +++ b/roles/configure-grenade-branches/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: Set the grenade start and destination branches + set_fact: + grenade_from_branch: "{{ grenade_from_branch }}" + grenade_to_branch: "{{ grenade_to_branch }}" diff --git a/roles/prepare-grenade-logs/README.rst b/roles/prepare-grenade-logs/README.rst new file mode 100644 index 00000000..172b6a4a --- /dev/null +++ b/roles/prepare-grenade-logs/README.rst @@ -0,0 +1,36 @@ +Move the logs and other data files which are relevant +for the starting and destination branches, which are +generated by devstack, grenade and tempest. +This role is needed because stage-output does not support +duplicated file names, even if they come from different +directories. + +**Role Variables** + +.. zuul:rolevar:: grenade_devstack_shared_dir + + The grenade base shared directory. + +.. zuul:rolevar:: grenade_config_dir + + The grenade base directory of the deployment. + +.. zuul:rolevar:: grenade_save_data_dir + + The final name of the directory which should store + the content of the save/ directory generated by grenade. + +.. zuul:rolevar:: grenade_devstack_data_log_dirs + :type: dict + + A dictionary with two keys ('old' and 'new'). Each + value contains the directory used to store the configuration + and data files from the corresponding devstack settings. + +.. zuul:rolevar:: grenade_devstack_saved_files + :type: list + + A list of files (base names only, no directories) to be saved + from the devstack directories. + The list is added to the default set of saved files + (defined as grenade_devstack_saved_files_default). diff --git a/roles/prepare-grenade-logs/defaults/main.yaml b/roles/prepare-grenade-logs/defaults/main.yaml new file mode 100644 index 00000000..0df54671 --- /dev/null +++ b/roles/prepare-grenade-logs/defaults/main.yaml @@ -0,0 +1,13 @@ +--- +grenade_devstack_saved_files: [] +grenade_devstack_saved_files_default: + - devstack/local.conf + - devstack/localrc + - devstack/.localrc.auto + - devstack/.stackenv + - logs/devstacklog.txt + - logs/devstacklog.txt.summary + - logs/dstat-csv.log + - tempest/etc/tempest.conf + - tempest/etc/accounts.yaml + - tempest/tempest.log diff --git a/roles/prepare-grenade-logs/tasks/main.yaml b/roles/prepare-grenade-logs/tasks/main.yaml new file mode 100644 index 00000000..71535047 --- /dev/null +++ b/roles/prepare-grenade-logs/tasks/main.yaml @@ -0,0 +1,31 @@ +- name: Rename the grenade custom config files + copy: + remote_src: yes + src: "{{ grenade_config_dir }}/{{ item }}" + dest: "{{ grenade_config_dir }}/grenade_{{ item }}" + loop: + - localrc + - pluginrc + ignore_errors: True + +- name: Create a link with a better name to the grenade save directory + file: + src: "{{ grenade_devstack_shared_dir }}/save" + dest: "{{ grenade_save_data_dir }}" + state: link + ignore_errors: True + +- name: Create the save directories + file: + path: "{{ grenade_devstack_data_log_dirs[item] }}" + state: directory + loop: "{{ ['old', 'new'] }}" + +- name: Copy the files to the expected save location + copy: + remote_src: yes + src: "{{ grenade_devstack_shared_dir }}/{{ item.0 }}/{{ item.1 }}" + dest: "{{ grenade_devstack_data_log_dirs[item.0] }}/{{ item.1|basename }}" + loop: "{{ ['old', 'new']|product(grenade_devstack_saved_files_default + grenade_devstack_saved_files)|list }}" + ignore_errors: True + no_log: True diff --git a/roles/run-grenade/README.rst b/roles/run-grenade/README.rst new file mode 100644 index 00000000..dbb965ce --- /dev/null +++ b/roles/run-grenade/README.rst @@ -0,0 +1,8 @@ +Run grenade + +**Role Variables** + +.. zuul:rolevar:: grenade_base_dir + :default: /opt/stack + + The grenade base directory. diff --git a/roles/run-grenade/defaults/main.yaml b/roles/run-grenade/defaults/main.yaml new file mode 100644 index 00000000..254cc10e --- /dev/null +++ b/roles/run-grenade/defaults/main.yaml @@ -0,0 +1 @@ +grenade_base_dir: /opt/stack diff --git a/roles/run-grenade/tasks/main.yaml b/roles/run-grenade/tasks/main.yaml new file mode 100644 index 00000000..639a76ed --- /dev/null +++ b/roles/run-grenade/tasks/main.yaml @@ -0,0 +1,19 @@ +- name: Run grenade + shell: + cmd: | + ./grenade.sh 2>&1 + rc=$? + echo "*** FINISHED ***" + exit $rc + environment: + # While originally grenade.sh used to take care of driving + # the entire upgrade progress, including calling stack.sh, + # after the Zuul v3 switch we have a proven method for + # calling DevStack: the roles defined inside its repository. + # Avoid the duplication and let grenade only execute + # the upgrade part. + GRENADE_USE_EXTERNAL_DEVSTACK: True + args: + chdir: "{{grenade_base_dir}}/grenade" + become: true + become_user: stack diff --git a/roles/write-grenade-conf/README.rst b/roles/write-grenade-conf/README.rst new file mode 100644 index 00000000..2ae63fe8 --- /dev/null +++ b/roles/write-grenade-conf/README.rst @@ -0,0 +1,45 @@ +Write the configuration files for use by grenade + +**Role Variables** + +.. zuul:rolevar:: base_dir + :default: /opt/stack + + The sources base directory. + +.. zuul:rolevar:: grenade_base_dir + :default: /opt/stack + + The grenade base directory. + +.. zuul:rolevar:: grenade_localrc_path + :default: {{ grenade_base_dir }}/grenade/localrc + + The path of the localrc file used by grenade. + +.. zuul:rolevar:: grenade_pluginrc_path + :default: {{ grenade_base_dir }}/grenade/pluginrc + + The path of the pluginrc file used by grenade. + +.. zuul:rolevar:: grenade_localrc + :type: dict + + A dictionary of variables that should be written into + the localrc file used by grenade. + +.. zuul:rolevar:: grenade_plugins + :type: list + + A list of grenade plugins that should be deployed. + +.. zuul:rolevar:: grenade_tempest_concurrency + :default: 2 + + The concurrency level for the tempest tests executed + by grenade. + +.. zuul:rolevar:: grenade_test_timeout + :default: 1200 + + The timeout (in seconds) for each test executed by grenade. diff --git a/roles/write-grenade-conf/defaults/main.yaml b/roles/write-grenade-conf/defaults/main.yaml new file mode 100644 index 00000000..36f68fa3 --- /dev/null +++ b/roles/write-grenade-conf/defaults/main.yaml @@ -0,0 +1,6 @@ +grenade_base_dir: /opt/stack +grenade_localrc_path: "{{ grenade_base_dir }}/grenade/localrc" +grenade_pluginrc_path: "{{ grenade_base_dir }}/grenade/pluginrc" +grenade_plugins: {} +grenade_tempest_concurrency: 2 +grenade_test_timeout: 1200 diff --git a/roles/write-grenade-conf/tasks/main.yaml b/roles/write-grenade-conf/tasks/main.yaml new file mode 100644 index 00000000..ebaad2be --- /dev/null +++ b/roles/write-grenade-conf/tasks/main.yaml @@ -0,0 +1,13 @@ +- name: Write the localrc file used by grenade + become: true + become_user: stack + template: + src: localrc.j2 + dest: "{{ grenade_localrc_path }}" + +- name: Write the pluginrc file used by grenade + become: true + become_user: stack + template: + src: pluginrc.j2 + dest: "{{ grenade_pluginrc_path }}" diff --git a/roles/write-grenade-conf/templates/localrc.j2 b/roles/write-grenade-conf/templates/localrc.j2 new file mode 100644 index 00000000..0fd6687e --- /dev/null +++ b/roles/write-grenade-conf/templates/localrc.j2 @@ -0,0 +1,18 @@ +BASE_RELEASE=old +BASE_RELEASE_DIR={{ base_dir }}/$BASE_RELEASE +BASE_DEVSTACK_DIR=$BASE_RELEASE_DIR/devstack +BASE_DEVSTACK_BRANCH={{ grenade_from_branch }} +TARGET_RELEASE=new +TARGET_RELEASE_DIR={{ base_dir }}/$TARGET_RELEASE +TARGET_DEVSTACK_DIR=$TARGET_RELEASE_DIR/devstack +TARGET_DEVSTACK_BRANCH={{ grenade_to_branch }} +TARGET_RUN_SMOKE=False +SAVE_DIR=$BASE_RELEASE_DIR/save +TEMPEST_CONCURRENCY={{ grenade_tempest_concurrency }} +export OS_TEST_TIMEOUT={{ grenade_test_timeout }} +VERBOSE=False +PLUGIN_DIR=$TARGET_RELEASE_DIR +# Additional variables +{% for key, value in grenade_localrc.items() %} +{{ key }}={{ value }} +{% endfor %} diff --git a/roles/write-grenade-conf/templates/pluginrc.j2 b/roles/write-grenade-conf/templates/pluginrc.j2 new file mode 100644 index 00000000..a5d031b4 --- /dev/null +++ b/roles/write-grenade-conf/templates/pluginrc.j2 @@ -0,0 +1,3 @@ +{% for key, value in grenade_plugins.items() %} +enable_grenade_plugin {{ key }} {{ value }} +{% endfor %}