Validation Framework functional test expansion

Issues present in the existing tests of the validation framework
runtime and CLI behavior were resolved. New tests were added to ensure
correct function of all of the most important commands, subcommands
and combinations of parameters.

* conflicting 'run_validations' variable was removed
* show command tests were enabled
* all test blocks are descriptively named
* tests for listing installed validations in various formats
* tests for running validations with parameters from a file
* tests for history display, both with and without limit
* tests for detailed validation information display
* tests for validation parameter display and download

Validation log fetch was modified to retrieve more data.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: Ic206e39f792b31d7f2adff89e4d38a2e8a656733

roles/fetch_validations/tasks/main.yaml

@@ -35,7 +35,7 @@
 - name: Find validations data
   find:
     paths: "{{ output_dir }}"
-    patterns: "*.json,*.log"
+    patterns: "*.json,*.log,*.yaml"
   register: validation_json
 
 - name: Collect Validation logs

roles/validations/tasks/list.yaml

@@ -1,6 +1,36 @@
 ---
-- name: List Validation
-  register: list_output
+- name: List Validations - all - to file
   shell:
     cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1"
     executable: /bin/bash
+  when: val_format == "json"
+
+- name: List Validations - all - to stdout - {{ val_format }}
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}"
+    executable: /bin/bash
+
+# Metadata dependent list output
+- name: List Validations - group - to stdout - {{ val_format }}
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.group }}"
+  loop_control:
+    loop_var: val_group
+
+- name: " List Validations - category - to stdout - {{ val_format }} "
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.category }}"
+  loop_control:
+    loop_var: val_category
+
+- name: "List Validations - product - to stdout - {{ val_format }}"
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.product }}"
+  loop_control:
+    loop_var: val_product

roles/validations/tasks/list_validation_history.yaml

@@ -0,0 +1,52 @@
+---
+# The subcommand used is 'show history' but it is implemented
+# as a subclass of Lister and it formats the results as such.
+# Both tests use regex to keep only lines starting with UUID[1].
+# As every validation run has UUID assigned and the 'value' output format
+# places it's octal form in the first column, it is possible to use it to
+# match only records about validation runs, and remove the rest.
+# [1]https://datatracker.ietf.org/doc/html/rfc4122#section-4.1
+- name: List all history
+  register: list_all_history_output
+  shell:
+    cmd: >-
+     {{ validation_command }} {{ history_command }} -f value 2>&1
+     | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
+     | tee {{ val_working_dir }}/full_validation_history.log
+    executable: /bin/bash
+
+- name: List truncated history
+  register: list_truncated_history_output
+  shell:
+    cmd: >-
+     {{ validation_command }} {{ history_command }} --limit 1 -f value 2>&1
+     | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
+     | tee {{ val_working_dir }}/truncated_validation_history.log
+    executable: /bin/bash
+
+- name: Verify history output
+  block:
+    # To ensure that we are getting the right number of validation runs
+    # we are querying the relevant item of the 'validations_list', for the number of 'extra_args' entries.
+    # As all validations defined in the 'validations_list' have 'extra_args' defined for both normal,
+    # and false positive run, we can use the number of 'extra_args' keys as an indication
+    # of the validations runs that were supposed to occur.
+    # Please note that this assertion will not hold, if the format of the default_vars changes substantially.
+    - name: Verify full history output
+      fail:
+        msg: >
+         The history output length {{ list_all_history_output.stdout_lines | length }}
+         doesn't match the number of expected validations runs {{ expected_history_length }}.
+      when: (list_all_history_output.stdout_lines | length) != (expected_history_length | int)
+      vars:
+        expected_history_length: "{{ validations_list[validation_component] | string | regex_findall('extra_args') | length }}"
+
+    - name: Verify truncated history output
+      fail:
+        msg: >
+         The number of history items displayed is {{ list_truncated_history_output.stdout_lines | length }}
+         but it should be 1.
+      when: (list_truncated_history_output.stdout_lines | length) != 1
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0

roles/validations/tasks/main.yaml

@@ -43,7 +43,8 @@
     validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
   when: not is_virtualenv.stat.exists
 
-- include_tasks: run.yaml
+- name: Run validations
+  include_tasks: run.yaml
   vars:
     name: "{{ item }}"
   when:
@@ -51,7 +52,16 @@
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
-- include_tasks: show.yaml
+- name: List validations
+  include_tasks: list.yaml
+  vars:
+    val_format: "{{ tested_format }}"
+  loop: "{{ validation_list_formats }}"
+  loop_control:
+    loop_var: tested_format
+
+- name: Show validation run results
+  include_tasks: show_results.yaml
   vars:
     name: "{{ item }}"
   when:
@@ -59,4 +69,26 @@
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
-- include_tasks: list.yaml
+- name: Show validation
+  include_tasks: show_validation_info.yaml
+  vars:
+    name: "{{ item }}"
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"
+
+- name: List history
+  include_tasks: list_validation_history.yaml
+  vars:
+    history_command: "{{'show history' if validation_command == 'openstack tripleo validator' else 'history list'}}"
+
+- name: Run validations with extra vars file
+  include_tasks: run_extra_vars_file.yaml
+  vars:
+    name: "{{ item }}"
+    extra_vars_uuid: "{{ 'extra vars for tests' | to_uuid }}"
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"

roles/validations/tasks/run_extra_vars_file.yaml

@@ -0,0 +1,41 @@
+---
+- name: Create extra vars file
+  shell:
+    cmd: "echo -e 'minimal_cpu_count: 2\nminimal_ram_gb: 2\n' > {{ extra_vars_uuid }}extra_vars.yaml"
+    executable: /bin/bash
+
+- name: Run validations with extra vars file
+  shell:
+    cmd: >-
+      {{ validation_command }} run --validation {{ name.key }}
+      {{ validation_dir }} {{ ansible_dir }}
+      --inventory {{ inventory }}
+      --output-log validation_{{ name.key }}_extra_vars_file.log
+      --extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml
+      {{ name.value.extra_env_args }}
+    executable: /bin/bash
+
+- name: Get Run results
+  block:
+    - name: Get run results
+      register: result
+      shell:
+        cmd: "cat validation_{{ name.key }}_extra_vars_file.log"
+        executable: /bin/bash
+
+    - name: Get json data
+      set_fact:
+        jsondata: "{{ result.stdout | from_json }}"
+
+    - name: Get Validations Status
+      set_fact:
+        status: "{{ jsondata | json_query(jsonres) }}"
+      vars:
+        jsonres: 'results[*].Status'
+
+    - fail:
+        msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}"
+      when: validation_status != "PASSED"
+      loop: "{{ status }}"
+      loop_control:
+        loop_var: validation_status

roles/validations/tasks/show.yaml

@@ -1,6 +0,0 @@
----
-- name: Show Validation
-  register: show_output
-  shell:
-    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
-    executable: /bin/bash

roles/validations/tasks/show_results.yaml

@@ -0,0 +1,21 @@
+---
+- name: Get run UUID
+  block:
+    - name: Get run results file
+      register: result
+      shell:
+        cmd: "cat validation_{{ name.key }}_positive.log"
+        executable: /bin/bash
+    - name: Get uuid from log
+      set_fact:
+        validation_run_uuids: "{{ result.stdout | from_json | json_query(uuid_selector) }}"
+      vars:
+        uuid_selector: 'results[*].UUID'
+
+- name: Show Validation run results
+  shell:
+    cmd: "{{ validation_command }} history get {{ run_uuid }} --full > {{ val_working_dir }}/show_run.log 2>&1"
+    executable: /bin/bash
+  loop: "{{ validation_run_uuids }}"
+  loop_control:
+    loop_var: run_uuid

roles/validations/tasks/show_validation_info.yaml

@@ -0,0 +1,33 @@
+---
+- name: Show Validation - correct id
+  register: show_output
+  shell:
+    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
+    executable: /bin/bash
+
+# Simulating a typo in the validation name
+- name: Show Validation - incorrect id
+  block:
+    - name: Show validations - incorrect id
+      register: show_output_incorrect
+      shell:
+        cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
+        executable: /bin/bash
+      ignore_errors: true
+    - fail:
+        msg: "The expected error message was not displayed."
+      when: "'Validation chuck-cpu not found' not in show_output_incorrect.stdout"
+
+- name: Show Validation parameters
+  shell:
+    cmd: >-
+     {{ validation_command }} show parameter
+     --validation {{ name.key }} {{ validation_dir }}
+     --download {{ name.key }}_params.{{ format_type }}
+     --format-output {{ format_type }}
+    executable: /bin/bash
+  loop:
+    - json
+    - yaml
+  loop_control:
+    loop_var: format_type

roles/validations/vars/main.yaml

@@ -39,3 +39,20 @@ validations_group:
     - compute
   network:
     - network
+
+validation_metadata:
+  group:
+    - no-op
+    - prep
+  category:
+    - storage
+    - container
+  product:
+    - tripleo
+
+validation_list_formats:
+  - csv
+  - json
+  - table
+  - value
+  - yaml