Functional test for running validations from file
Functional tests for a change Ifc6c28003c4c2c5f3dd6198e650f9713a02dc82d In order to use the 'validation_dir' and 'ansible_dir' parameters in the YAML file when executing the 'file' command, the original parameters 'validation_dir' and 'ansible_dir' are changed to 'validation_dir_arg' and 'ansible_dir_arg' when using as a argument in the 'run' command. Related-to: rhbz#2122209 Signed-off-by: Veronika Fisarova <vfisarov@redhat.com> Change-Id: I56d974814987ef7a1cc105d4efe3123dd97f6b3e
This commit is contained in:
parent
dec9846f74
commit
d00778f1cd
@ -23,3 +23,5 @@ cli_command: "validation"
|
||||
run_validation: true
|
||||
execute_full_vf_catalogue: "{{ job.execute_full_vf_catalogue|default(false)|bool }}"
|
||||
vf_catalogue_overrides: "{{ ansible_user_dir }}/catalog_vars_override.yaml"
|
||||
files_to_run_dest: "{{ ansible_user_dir }}"
|
||||
files_test_failure: false
|
||||
|
@ -7,7 +7,7 @@
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} run --validation {{ validation_catalogue.full_list | join(",") }}
|
||||
{{ validation_dir }} {{ ansible_dir }}
|
||||
{{ validation_dir_arg }} {{ ansible_dir_arg }}
|
||||
--inventory {{ inventory }}
|
||||
--output-log validation_catalogue_execution.log
|
||||
{{ validation_catalogue.extra_args }}
|
||||
|
118
roles/validations/tasks/file.yaml
Normal file
118
roles/validations/tasks/file.yaml
Normal file
@ -0,0 +1,118 @@
|
||||
---
|
||||
- name: Run validations from the YAML file
|
||||
vars:
|
||||
file_wrong_path: foo/bar.yaml
|
||||
junitxml_path: /home/stack/logs
|
||||
junitxml_wrong_path: /foo/bar
|
||||
wrong_path_string: must be properly formatted
|
||||
wrong_directory_string: No such file or directory
|
||||
junitxml_missing_string: junitxml output disabled
|
||||
validation_failed_string: have failed
|
||||
validation_not_found_string: Following validations were not found
|
||||
no_validation_run_string: No validation has been run
|
||||
expected_result: 1
|
||||
block:
|
||||
- debug:
|
||||
msg: "{{ item }}"
|
||||
loop: "{{ files }}"
|
||||
|
||||
- name: Execute the file command
|
||||
block:
|
||||
- name: Passed validation test
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[0] }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when: "{{ run_results.rc }} == 1"
|
||||
|
||||
- name: Failed validation test
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[3] }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when: run_results.rc != expected_result
|
||||
|
||||
- name: Run with no executed validation
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[1] }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when:
|
||||
- run_results.rc != expected_result
|
||||
|
||||
- name: Run with non-existent validation
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[2] }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when:
|
||||
- validation_not_found_string not in run_results.stdout
|
||||
- validation_not_found_string not in run_results.stderr
|
||||
|
||||
- name: Execute the file command with wrong path
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ file_wrong_path }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when:
|
||||
- wrong_path_string not in run_results.stdout
|
||||
- wrong_path_string not in run_results.stderr
|
||||
|
||||
- name: Execute the file command with junitxml output
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_path }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when:
|
||||
- wrong_directory_string in run_results.stdout
|
||||
- junitxml_missing_string not in run_results.stdout
|
||||
- wrong_directory_string in run_results.stderr
|
||||
- junitxml_missing_string not in run_results.stderr
|
||||
- "{{ run_results.rc }} == 1"
|
||||
|
||||
- name: Execute the file command with wrong Junitxml path
|
||||
ignore_errors: true
|
||||
register: run_results
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} file {{ files[0] }} --junitxml {{ junitxml_wrong_path }}
|
||||
executable: /bin/bash
|
||||
|
||||
- set_fact:
|
||||
files_test_failure: true
|
||||
when:
|
||||
- wrong_directory_string in run_results.stdout
|
||||
- junitxml_missing_string not in run_results.stdout
|
||||
- wrong_directory_string in run_results.stderr
|
||||
- junitxml_missing_string not in run_results.stderr
|
||||
- "{{ run_results.rc }} == 1"
|
@ -1,19 +1,19 @@
|
||||
---
|
||||
- name: List Validations - all - to file
|
||||
shell:
|
||||
cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1"
|
||||
cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f json > {{ val_working_dir }}/list.log 2>&1"
|
||||
executable: /bin/bash
|
||||
when: val_format == "json"
|
||||
|
||||
- name: List Validations - all - to stdout - {{ val_format }}
|
||||
shell:
|
||||
cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}"
|
||||
cmd: "{{ validation_command }} list {{ validation_dir_arg }} -f {{ val_format }}"
|
||||
executable: /bin/bash
|
||||
|
||||
# Metadata dependent list output
|
||||
- name: List Validations - group - to stdout - {{ val_format }}
|
||||
shell:
|
||||
cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}"
|
||||
cmd: "{{ validation_command }} list {{ validation_dir_arg }} --group {{ val_group }} -f {{ val_format }}"
|
||||
executable: /bin/bash
|
||||
loop: "{{ validation_metadata.group }}"
|
||||
loop_control:
|
||||
@ -21,7 +21,7 @@
|
||||
|
||||
- name: " List Validations - category - to stdout - {{ val_format }} "
|
||||
shell:
|
||||
cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}"
|
||||
cmd: "{{ validation_command }} list {{ validation_dir_arg }} --category {{ val_category }} -f {{ val_format }}"
|
||||
executable: /bin/bash
|
||||
loop: "{{ validation_metadata.category }}"
|
||||
loop_control:
|
||||
@ -29,7 +29,7 @@
|
||||
|
||||
- name: "List Validations - product - to stdout - {{ val_format }}"
|
||||
shell:
|
||||
cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}"
|
||||
cmd: "{{ validation_command }} list {{ validation_dir_arg }} --product {{ val_product }} -f {{ val_format }}"
|
||||
executable: /bin/bash
|
||||
loop: "{{ validation_metadata.product }}"
|
||||
loop_control:
|
||||
|
@ -39,12 +39,22 @@
|
||||
|
||||
- name: Set Validation directory if virtualenv exists
|
||||
set_fact:
|
||||
validation_dir: "--validation-dir {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks"
|
||||
validation_dir: "{{ zuul_work_virtualenv }}/share/ansible/validation-playbooks"
|
||||
when: is_virtualenv.stat.exists
|
||||
|
||||
- name: Set Ansible base directory if virtualenv exists
|
||||
- name: Set Validation directory argument if virtualenv exists
|
||||
set_fact:
|
||||
ansible_dir: "--ansible-base-dir {{ zuul_work_virtualenv }}/share/ansible/"
|
||||
validation_dir_arg: "--validation-dir {{ validation_dir }}"
|
||||
when: is_virtualenv.stat.exists
|
||||
|
||||
- name: Set Ansible base directory path if virtualenv exists
|
||||
set_fact:
|
||||
ansible_dir: "{{ zuul_work_virtualenv }}/share/ansible/"
|
||||
when: is_virtualenv.stat.exists
|
||||
|
||||
- name: Set Ansible base directory argument if virtualenv exists
|
||||
set_fact:
|
||||
ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}"
|
||||
when: is_virtualenv.stat.exists
|
||||
|
||||
- name: Set commmand without virtualenv
|
||||
@ -56,7 +66,22 @@
|
||||
|
||||
- name: Set validation dir without virtualenv
|
||||
set_fact:
|
||||
validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
|
||||
validation_dir: "/usr/share/ansible/validation-playbooks"
|
||||
when: not is_virtualenv.stat.exists
|
||||
|
||||
- name: Set validation dir argument without virtualenv
|
||||
set_fact:
|
||||
validation_dir_arg: "--validation-dir {{ validation_dir }}"
|
||||
when: not is_virtualenv.stat.exists
|
||||
|
||||
- name: Set Ansible base directory path withnout virtualenv
|
||||
set_fact:
|
||||
ansible_dir: "/usr/share/ansible/"
|
||||
when: not is_virtualenv.stat.exists
|
||||
|
||||
- name: Set Ansible base directory argument withnout virtualenv exists
|
||||
set_fact:
|
||||
ansible_dir_arg: "--ansible-base-dir {{ ansible_dir }}"
|
||||
when: not is_virtualenv.stat.exists
|
||||
|
||||
- name: Set a valid inventory
|
||||
@ -167,3 +192,39 @@
|
||||
- name: Execute entire validations catalogue
|
||||
include_tasks: execute_full_catalogue.yaml
|
||||
when: execute_full_vf_catalogue
|
||||
|
||||
- debug:
|
||||
msg: "{{ item }}"
|
||||
loop: "{{ test_arguments_run_from_file }}"
|
||||
|
||||
- name: Check if the File command is present
|
||||
register: subcommand_list
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} --help
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Execute the file command tests
|
||||
block:
|
||||
- name: Copy files to run
|
||||
template:
|
||||
src: './templates/file-template.j2'
|
||||
dest: rendered_file_{{ ansible_loop.index }}.yaml
|
||||
mode: "0644"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: "{{ ansible_user }}"
|
||||
loop: "{{ test_arguments_run_from_file }}"
|
||||
loop_control:
|
||||
extended: true
|
||||
register: rendered_files
|
||||
|
||||
- name: Run validations from the File
|
||||
include_tasks: file.yaml
|
||||
vars:
|
||||
files: "{{ rendered_files.results | map(attribute='dest') | list }}"
|
||||
when: "'Include and exclude validations by' in subcommand_list.stdout"
|
||||
|
||||
- name: Fail if something went wrong
|
||||
fail:
|
||||
msg: "One or more file runs have failed, check the log results for more information."
|
||||
when: files_test_failure | default(False) | bool and "'file' in subcommand_list.stdout"
|
||||
|
@ -14,7 +14,7 @@
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} run --validation {{ name.key }}
|
||||
{{ validation_dir }} {{ ansible_dir }}
|
||||
{{ validation_dir_arg }} {{ ansible_dir_arg }}
|
||||
--inventory {{ inventory }}
|
||||
--output-log validation_{{ name.key }}_positive.log
|
||||
{{ execution_extra_args | default(name.value.extra_args) }}
|
||||
|
@ -8,7 +8,7 @@
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} run --validation {{ name.key }}
|
||||
{{ validation_dir }} {{ ansible_dir }}
|
||||
{{ validation_dir_arg }} {{ ansible_dir_arg }}
|
||||
--inventory {{ inventory }}
|
||||
--output-log validation_{{ name.key }}_extra_vars_file.log
|
||||
--extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml
|
||||
|
@ -2,7 +2,7 @@
|
||||
- name: Show Validation - correct id
|
||||
register: show_output
|
||||
shell:
|
||||
cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
|
||||
cmd: "{{ validation_command }} show {{ validation_dir_arg }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
|
||||
executable: /bin/bash
|
||||
|
||||
# Simulating a typo in the validation name
|
||||
@ -11,7 +11,7 @@
|
||||
- name: Show validations - incorrect id
|
||||
register: show_output_incorrect
|
||||
shell:
|
||||
cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
|
||||
cmd: "{{ validation_command }} show {{ validation_dir_arg }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
|
||||
executable: /bin/bash
|
||||
ignore_errors: true
|
||||
- fail:
|
||||
@ -22,7 +22,7 @@
|
||||
shell:
|
||||
cmd: >-
|
||||
{{ validation_command }} show parameter
|
||||
--validation {{ name.key }} {{ validation_dir }}
|
||||
--validation {{ name.key }} {{ validation_dir_arg }}
|
||||
--download {{ name.key }}_params.{{ format_type }}
|
||||
--format-output {{ format_type }}
|
||||
executable: /bin/bash
|
||||
|
34
roles/validations/templates/file-template.j2
Normal file
34
roles/validations/templates/file-template.j2
Normal file
@ -0,0 +1,34 @@
|
||||
---
|
||||
include_validation: {{ item.validation }}
|
||||
include_group: {{ item.validation_group }}
|
||||
include_category: {{ item.validation_category }}
|
||||
include_product: {{ item.validation_product }}
|
||||
exclude_validation: {{ item.exclude_validation }}
|
||||
exclude_group: {{ item.exclude_validation_group }}
|
||||
exclude_category: {{ item.exclude_validation_category }}
|
||||
exclude_product: {{ item.exclude_validation_product }}
|
||||
config: {{ validation_config | default('') }}
|
||||
inventory: {{ inventory }}
|
||||
validation-log-dir: {{ vf_log_dir }}
|
||||
output-log: {{ vf_log_dir }}/run-from-file.log
|
||||
# Checks if the following variables are defined, if the variables
|
||||
# don't have the value, based on the vars/main.yaml file, they are skipped.
|
||||
{% if item.extra_env_vars | default('') %}
|
||||
extra-env-vars: {{ item.extra_env_vars }}
|
||||
{% endif %}
|
||||
{% if item.extra_vars | default('') %}
|
||||
extra-vars: {{ item.extra_vars }}
|
||||
{% endif %}
|
||||
{% if item.limit_hosts | default('') %}
|
||||
limit: {{ item.limit_hosts.limit | default('') }}
|
||||
{% endif %}
|
||||
# Checks if the zuul virtualenv exists, if not default path is used instead.
|
||||
{% if is_virtualenv.stat.exists %}
|
||||
validation-dir: {{ zuul_work_virtualenv }}/share/ansible/validation-playbooks
|
||||
ansible-base-dir: {{ zuul_work_virtualenv }}/share/ansible
|
||||
python-interpreter: {{ zuul_work_virtualenv }}/bin/python3
|
||||
{% else %}
|
||||
validation-dir: /usr/share/ansible/validation-playbooks
|
||||
ansible-base-dir: /usr/share/ansible
|
||||
python-interpreter: /usr/bin/python3
|
||||
{% endif %}
|
@ -186,3 +186,82 @@ validation_catalogue:
|
||||
# - tls-everywhere-prep
|
||||
# - undercloud-debug
|
||||
# - undercloud-service-status
|
||||
#
|
||||
# List of dictionaries for testing 4 different versions of file-to-run the CLI file command
|
||||
# Each dictionary consists of different options for inclusion and exclusion
|
||||
# validations/groups/categories/products
|
||||
test_arguments_run_from_file:
|
||||
# 1st valid file
|
||||
# expected rc is 0
|
||||
- validation:
|
||||
- check-cpu
|
||||
validation_group: []
|
||||
validation_category: []
|
||||
validation_product: []
|
||||
exclude_validation:
|
||||
exclude_validation_group: []
|
||||
exclude_validation_category: []
|
||||
exclude_validation_product:
|
||||
- tripleo
|
||||
validation-dir:
|
||||
extra_vars:
|
||||
minimal_cpu_count: 2
|
||||
# 2nd valid file with 1 non-existent validation
|
||||
# networking group should be run, expected rc is 1 due to the failing
|
||||
# validations
|
||||
- validation:
|
||||
- check-cpu
|
||||
- i-dont-exist
|
||||
validation_group: []
|
||||
validation_category:
|
||||
- compute
|
||||
- networking
|
||||
validation_product: []
|
||||
exclude_validation:
|
||||
- fips-enabled
|
||||
exclude_validation_group:
|
||||
- prep
|
||||
exclude_validation_category:
|
||||
- compute
|
||||
exclude_validation_product:
|
||||
- rabbitmq
|
||||
- tripleo
|
||||
limit_hosts:
|
||||
limit: [undercloud-0]
|
||||
# 3rd valid file testing the Invalid operation: no validation to run
|
||||
# expected rc is 1
|
||||
- validation:
|
||||
- i-dont-exist
|
||||
validation_group: []
|
||||
validation_category: []
|
||||
validation_product: []
|
||||
exclude_validation:
|
||||
- fips-enabled
|
||||
exclude_validation_group:
|
||||
- prep
|
||||
exclude_validation_category:
|
||||
- compute
|
||||
exclude_validation_product:
|
||||
- rabbitmq
|
||||
- tripleo
|
||||
# 4th valid file, testing the proper inclusion and exclusion
|
||||
# only networking group should run (except the dns and the ntp validations)
|
||||
# expected rc is 1 due to the failed check-cpu validation
|
||||
- validation:
|
||||
- check-cpu
|
||||
validation_group:
|
||||
validation_category:
|
||||
- networking
|
||||
validation_product:
|
||||
- tripleo
|
||||
exclude_validation:
|
||||
- fips-enabled
|
||||
- dns
|
||||
- ntp
|
||||
exclude_validation_group:
|
||||
exclude_validation_category:
|
||||
exclude_validation_product:
|
||||
- rabbitmq
|
||||
- tripleo
|
||||
extra_vars:
|
||||
minimal_cpu_count: 8000
|
||||
|
Loading…
Reference in New Issue
Block a user