diff --git a/roles/fetch_validations/tasks/main.yaml b/roles/fetch_validations/tasks/main.yaml index 58348e6..d7cc94b 100644 --- a/roles/fetch_validations/tasks/main.yaml +++ b/roles/fetch_validations/tasks/main.yaml @@ -35,7 +35,7 @@ - name: Find validations data find: paths: "{{ output_dir }}" - patterns: "*.json,*.log" + patterns: "*.json,*.log,*.yaml" register: validation_json - name: Collect Validation logs diff --git a/roles/validations/tasks/list.yaml b/roles/validations/tasks/list.yaml index 1245c33..fe6b236 100644 --- a/roles/validations/tasks/list.yaml +++ b/roles/validations/tasks/list.yaml @@ -1,6 +1,36 @@ --- -- name: List Validation - register: list_output +- name: List Validations - all - to file shell: cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1" executable: /bin/bash + when: val_format == "json" + +- name: List Validations - all - to stdout - {{ val_format }} + shell: + cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}" + executable: /bin/bash + +# Metadata dependent list output +- name: List Validations - group - to stdout - {{ val_format }} + shell: + cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}" + executable: /bin/bash + loop: "{{ validation_metadata.group }}" + loop_control: + loop_var: val_group + +- name: " List Validations - category - to stdout - {{ val_format }} " + shell: + cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}" + executable: /bin/bash + loop: "{{ validation_metadata.category }}" + loop_control: + loop_var: val_category + +- name: "List Validations - product - to stdout - {{ val_format }}" + shell: + cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}" + executable: /bin/bash + loop: "{{ validation_metadata.product }}" + loop_control: + loop_var: val_product diff --git a/roles/validations/tasks/list_validation_history.yaml b/roles/validations/tasks/list_validation_history.yaml new file mode 100644 index 0000000..3668329 --- /dev/null +++ b/roles/validations/tasks/list_validation_history.yaml @@ -0,0 +1,52 @@ +--- +# The subcommand used is 'show history' but it is implemented +# as a subclass of Lister and it formats the results as such. +# Both tests use regex to keep only lines starting with UUID[1]. +# As every validation run has UUID assigned and the 'value' output format +# places it's octal form in the first column, it is possible to use it to +# match only records about validation runs, and remove the rest. +# [1]https://datatracker.ietf.org/doc/html/rfc4122#section-4.1 +- name: List all history + register: list_all_history_output + shell: + cmd: >- + {{ validation_command }} {{ history_command }} -f value 2>&1 + | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}" + | tee {{ val_working_dir }}/full_validation_history.log + executable: /bin/bash + +- name: List truncated history + register: list_truncated_history_output + shell: + cmd: >- + {{ validation_command }} {{ history_command }} --limit 1 -f value 2>&1 + | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}" + | tee {{ val_working_dir }}/truncated_validation_history.log + executable: /bin/bash + +- name: Verify history output + block: + # To ensure that we are getting the right number of validation runs + # we are querying the relevant item of the 'validations_list', for the number of 'extra_args' entries. + # As all validations defined in the 'validations_list' have 'extra_args' defined for both normal, + # and false positive run, we can use the number of 'extra_args' keys as an indication + # of the validations runs that were supposed to occur. + # Please note that this assertion will not hold, if the format of the default_vars changes substantially. + - name: Verify full history output + fail: + msg: > + The history output length {{ list_all_history_output.stdout_lines | length }} + doesn't match the number of expected validations runs {{ expected_history_length }}. + when: (list_all_history_output.stdout_lines | length) != (expected_history_length | int) + vars: + expected_history_length: "{{ validations_list[validation_component] | string | regex_findall('extra_args') | length }}" + + - name: Verify truncated history output + fail: + msg: > + The number of history items displayed is {{ list_truncated_history_output.stdout_lines | length }} + but it should be 1. + when: (list_truncated_history_output.stdout_lines | length) != 1 + when: + - run_validation|default(false)|bool + - validation_component | length > 0 diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml index 766d857..1803b37 100644 --- a/roles/validations/tasks/main.yaml +++ b/roles/validations/tasks/main.yaml @@ -43,7 +43,8 @@ validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks" when: not is_virtualenv.stat.exists -- include_tasks: run.yaml +- name: Run validations + include_tasks: run.yaml vars: name: "{{ item }}" when: @@ -51,7 +52,16 @@ - validation_component | length > 0 with_dict: "{{ validations_list[validation_component] }}" -- include_tasks: show.yaml +- name: List validations + include_tasks: list.yaml + vars: + val_format: "{{ tested_format }}" + loop: "{{ validation_list_formats }}" + loop_control: + loop_var: tested_format + +- name: Show validation run results + include_tasks: show_results.yaml vars: name: "{{ item }}" when: @@ -59,4 +69,26 @@ - validation_component | length > 0 with_dict: "{{ validations_list[validation_component] }}" -- include_tasks: list.yaml +- name: Show validation + include_tasks: show_validation_info.yaml + vars: + name: "{{ item }}" + when: + - run_validation|default(false)|bool + - validation_component | length > 0 + with_dict: "{{ validations_list[validation_component] }}" + +- name: List history + include_tasks: list_validation_history.yaml + vars: + history_command: "{{'show history' if validation_command == 'openstack tripleo validator' else 'history list'}}" + +- name: Run validations with extra vars file + include_tasks: run_extra_vars_file.yaml + vars: + name: "{{ item }}" + extra_vars_uuid: "{{ 'extra vars for tests' | to_uuid }}" + when: + - run_validation|default(false)|bool + - validation_component | length > 0 + with_dict: "{{ validations_list[validation_component] }}" diff --git a/roles/validations/tasks/run_extra_vars_file.yaml b/roles/validations/tasks/run_extra_vars_file.yaml new file mode 100644 index 0000000..41291fb --- /dev/null +++ b/roles/validations/tasks/run_extra_vars_file.yaml @@ -0,0 +1,41 @@ +--- +- name: Create extra vars file + shell: + cmd: "echo -e 'minimal_cpu_count: 2\nminimal_ram_gb: 2\n' > {{ extra_vars_uuid }}extra_vars.yaml" + executable: /bin/bash + +- name: Run validations with extra vars file + shell: + cmd: >- + {{ validation_command }} run --validation {{ name.key }} + {{ validation_dir }} {{ ansible_dir }} + --inventory {{ inventory }} + --output-log validation_{{ name.key }}_extra_vars_file.log + --extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml + {{ name.value.extra_env_args }} + executable: /bin/bash + +- name: Get Run results + block: + - name: Get run results + register: result + shell: + cmd: "cat validation_{{ name.key }}_extra_vars_file.log" + executable: /bin/bash + + - name: Get json data + set_fact: + jsondata: "{{ result.stdout | from_json }}" + + - name: Get Validations Status + set_fact: + status: "{{ jsondata | json_query(jsonres) }}" + vars: + jsonres: 'results[*].Status' + + - fail: + msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}" + when: validation_status != "PASSED" + loop: "{{ status }}" + loop_control: + loop_var: validation_status diff --git a/roles/validations/tasks/show.yaml b/roles/validations/tasks/show.yaml deleted file mode 100644 index 37a4565..0000000 --- a/roles/validations/tasks/show.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Show Validation - register: show_output - shell: - cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1" - executable: /bin/bash diff --git a/roles/validations/tasks/show_results.yaml b/roles/validations/tasks/show_results.yaml new file mode 100644 index 0000000..c7052c9 --- /dev/null +++ b/roles/validations/tasks/show_results.yaml @@ -0,0 +1,21 @@ +--- +- name: Get run UUID + block: + - name: Get run results file + register: result + shell: + cmd: "cat validation_{{ name.key }}_positive.log" + executable: /bin/bash + - name: Get uuid from log + set_fact: + validation_run_uuids: "{{ result.stdout | from_json | json_query(uuid_selector) }}" + vars: + uuid_selector: 'results[*].UUID' + +- name: Show Validation run results + shell: + cmd: "{{ validation_command }} history get {{ run_uuid }} --full > {{ val_working_dir }}/show_run.log 2>&1" + executable: /bin/bash + loop: "{{ validation_run_uuids }}" + loop_control: + loop_var: run_uuid diff --git a/roles/validations/tasks/show_validation_info.yaml b/roles/validations/tasks/show_validation_info.yaml new file mode 100644 index 0000000..09bfa49 --- /dev/null +++ b/roles/validations/tasks/show_validation_info.yaml @@ -0,0 +1,33 @@ +--- +- name: Show Validation - correct id + register: show_output + shell: + cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1" + executable: /bin/bash + +# Simulating a typo in the validation name +- name: Show Validation - incorrect id + block: + - name: Show validations - incorrect id + register: show_output_incorrect + shell: + cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log" + executable: /bin/bash + ignore_errors: true + - fail: + msg: "The expected error message was not displayed." + when: "'Validation chuck-cpu not found' not in show_output_incorrect.stdout" + +- name: Show Validation parameters + shell: + cmd: >- + {{ validation_command }} show parameter + --validation {{ name.key }} {{ validation_dir }} + --download {{ name.key }}_params.{{ format_type }} + --format-output {{ format_type }} + executable: /bin/bash + loop: + - json + - yaml + loop_control: + loop_var: format_type diff --git a/roles/validations/vars/main.yaml b/roles/validations/vars/main.yaml index 9ec9bc4..132639c 100644 --- a/roles/validations/vars/main.yaml +++ b/roles/validations/vars/main.yaml @@ -39,3 +39,20 @@ validations_group: - compute network: - network + +validation_metadata: + group: + - no-op + - prep + category: + - storage + - container + product: + - tripleo + +validation_list_formats: + - csv + - json + - table + - value + - yaml