From 7c2026616b0b440bd41ad14150508dad8c21f527 Mon Sep 17 00:00:00 2001
From: Jiri Podivin <jpodivin@redhat.com>
Date: Tue, 3 Aug 2021 09:38:28 +0200
Subject: [PATCH] Validation Framework functional test expansion

Issues present in the existing tests of the validation framework
runtime and CLI behavior were resolved. New tests were added to ensure
correct function of all of the most important commands, subcommands
and combinations of parameters.

* conflicting 'run_validations' variable was removed
* show command tests were enabled
* all test blocks are descriptively named
* tests for listing installed validations in various formats
* tests for running validations with parameters from a file
* tests for history display, both with and without limit
* tests for detailed validation information display
* tests for validation parameter display and download

Validation log fetch was modified to retrieve more data.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: Ic206e39f792b31d7f2adff89e4d38a2e8a656733
---
 roles/fetch_validations/tasks/main.yaml       |  2 +-
 roles/validations/tasks/list.yaml             | 34 +++++++++++-
 .../tasks/list_validation_history.yaml        | 52 +++++++++++++++++++
 roles/validations/tasks/main.yaml             | 38 ++++++++++++--
 .../tasks/run_extra_vars_file.yaml            | 41 +++++++++++++++
 roles/validations/tasks/show.yaml             |  6 ---
 roles/validations/tasks/show_results.yaml     | 21 ++++++++
 .../tasks/show_validation_info.yaml           | 33 ++++++++++++
 roles/validations/vars/main.yaml              | 17 ++++++
 9 files changed, 232 insertions(+), 12 deletions(-)
 create mode 100644 roles/validations/tasks/list_validation_history.yaml
 create mode 100644 roles/validations/tasks/run_extra_vars_file.yaml
 delete mode 100644 roles/validations/tasks/show.yaml
 create mode 100644 roles/validations/tasks/show_results.yaml
 create mode 100644 roles/validations/tasks/show_validation_info.yaml

diff --git a/roles/fetch_validations/tasks/main.yaml b/roles/fetch_validations/tasks/main.yaml
index 58348e6..d7cc94b 100644
--- a/roles/fetch_validations/tasks/main.yaml
+++ b/roles/fetch_validations/tasks/main.yaml
@@ -35,7 +35,7 @@
 - name: Find validations data
   find:
     paths: "{{ output_dir }}"
-    patterns: "*.json,*.log"
+    patterns: "*.json,*.log,*.yaml"
   register: validation_json
 
 - name: Collect Validation logs
diff --git a/roles/validations/tasks/list.yaml b/roles/validations/tasks/list.yaml
index 1245c33..fe6b236 100644
--- a/roles/validations/tasks/list.yaml
+++ b/roles/validations/tasks/list.yaml
@@ -1,6 +1,36 @@
 ---
-- name: List Validation
-  register: list_output
+- name: List Validations - all - to file
   shell:
     cmd: "{{ validation_command }} list {{ validation_dir }} -f json > {{ val_working_dir }}/list.log 2>&1"
     executable: /bin/bash
+  when: val_format == "json"
+
+- name: List Validations - all - to stdout - {{ val_format }}
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} -f {{ val_format }}"
+    executable: /bin/bash
+
+# Metadata dependent list output
+- name: List Validations - group - to stdout - {{ val_format }}
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --group {{ val_group }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.group }}"
+  loop_control:
+    loop_var: val_group
+
+- name: " List Validations - category - to stdout - {{ val_format }} "
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --category {{ val_category }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.category }}"
+  loop_control:
+    loop_var: val_category
+
+- name: "List Validations - product - to stdout - {{ val_format }}"
+  shell:
+    cmd: "{{ validation_command }} list {{ validation_dir }} --product {{ val_product }} -f {{ val_format }}"
+    executable: /bin/bash
+  loop: "{{ validation_metadata.product }}"
+  loop_control:
+    loop_var: val_product
diff --git a/roles/validations/tasks/list_validation_history.yaml b/roles/validations/tasks/list_validation_history.yaml
new file mode 100644
index 0000000..3668329
--- /dev/null
+++ b/roles/validations/tasks/list_validation_history.yaml
@@ -0,0 +1,52 @@
+---
+# The subcommand used is 'show history' but it is implemented
+# as a subclass of Lister and it formats the results as such.
+# Both tests use regex to keep only lines starting with UUID[1].
+# As every validation run has UUID assigned and the 'value' output format
+# places it's octal form in the first column, it is possible to use it to
+# match only records about validation runs, and remove the rest.
+# [1]https://datatracker.ietf.org/doc/html/rfc4122#section-4.1
+- name: List all history
+  register: list_all_history_output
+  shell:
+    cmd: >-
+     {{ validation_command }} {{ history_command }} -f value 2>&1
+     | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
+     | tee {{ val_working_dir }}/full_validation_history.log
+    executable: /bin/bash
+
+- name: List truncated history
+  register: list_truncated_history_output
+  shell:
+    cmd: >-
+     {{ validation_command }} {{ history_command }} --limit 1 -f value 2>&1
+     | grep "^[[:alnum:]]\{8\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{4\}-[[:alnum:]]\{12\}"
+     | tee {{ val_working_dir }}/truncated_validation_history.log
+    executable: /bin/bash
+
+- name: Verify history output
+  block:
+    # To ensure that we are getting the right number of validation runs
+    # we are querying the relevant item of the 'validations_list', for the number of 'extra_args' entries.
+    # As all validations defined in the 'validations_list' have 'extra_args' defined for both normal,
+    # and false positive run, we can use the number of 'extra_args' keys as an indication
+    # of the validations runs that were supposed to occur.
+    # Please note that this assertion will not hold, if the format of the default_vars changes substantially.
+    - name: Verify full history output
+      fail:
+        msg: >
+         The history output length {{ list_all_history_output.stdout_lines | length }}
+         doesn't match the number of expected validations runs {{ expected_history_length }}.
+      when: (list_all_history_output.stdout_lines | length) != (expected_history_length | int)
+      vars:
+        expected_history_length: "{{ validations_list[validation_component] | string | regex_findall('extra_args') | length }}"
+
+    - name: Verify truncated history output
+      fail:
+        msg: >
+         The number of history items displayed is {{ list_truncated_history_output.stdout_lines | length }}
+         but it should be 1.
+      when: (list_truncated_history_output.stdout_lines | length) != 1
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
diff --git a/roles/validations/tasks/main.yaml b/roles/validations/tasks/main.yaml
index 766d857..1803b37 100644
--- a/roles/validations/tasks/main.yaml
+++ b/roles/validations/tasks/main.yaml
@@ -43,7 +43,8 @@
     validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
   when: not is_virtualenv.stat.exists
 
-- include_tasks: run.yaml
+- name: Run validations
+  include_tasks: run.yaml
   vars:
     name: "{{ item }}"
   when:
@@ -51,7 +52,16 @@
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
-- include_tasks: show.yaml
+- name: List validations
+  include_tasks: list.yaml
+  vars:
+    val_format: "{{ tested_format }}"
+  loop: "{{ validation_list_formats }}"
+  loop_control:
+    loop_var: tested_format
+
+- name: Show validation run results
+  include_tasks: show_results.yaml
   vars:
     name: "{{ item }}"
   when:
@@ -59,4 +69,26 @@
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
-- include_tasks: list.yaml
+- name: Show validation
+  include_tasks: show_validation_info.yaml
+  vars:
+    name: "{{ item }}"
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"
+
+- name: List history
+  include_tasks: list_validation_history.yaml
+  vars:
+    history_command: "{{'show history' if validation_command == 'openstack tripleo validator' else 'history list'}}"
+
+- name: Run validations with extra vars file
+  include_tasks: run_extra_vars_file.yaml
+  vars:
+    name: "{{ item }}"
+    extra_vars_uuid: "{{ 'extra vars for tests' | to_uuid }}"
+  when:
+    - run_validation|default(false)|bool
+    - validation_component | length > 0
+  with_dict: "{{ validations_list[validation_component] }}"
diff --git a/roles/validations/tasks/run_extra_vars_file.yaml b/roles/validations/tasks/run_extra_vars_file.yaml
new file mode 100644
index 0000000..41291fb
--- /dev/null
+++ b/roles/validations/tasks/run_extra_vars_file.yaml
@@ -0,0 +1,41 @@
+---
+- name: Create extra vars file
+  shell:
+    cmd: "echo -e 'minimal_cpu_count: 2\nminimal_ram_gb: 2\n' > {{ extra_vars_uuid }}extra_vars.yaml"
+    executable: /bin/bash
+
+- name: Run validations with extra vars file
+  shell:
+    cmd: >-
+      {{ validation_command }} run --validation {{ name.key }}
+      {{ validation_dir }} {{ ansible_dir }}
+      --inventory {{ inventory }}
+      --output-log validation_{{ name.key }}_extra_vars_file.log
+      --extra-vars-file {{ extra_vars_uuid }}extra_vars.yaml
+      {{ name.value.extra_env_args }}
+    executable: /bin/bash
+
+- name: Get Run results
+  block:
+    - name: Get run results
+      register: result
+      shell:
+        cmd: "cat validation_{{ name.key }}_extra_vars_file.log"
+        executable: /bin/bash
+
+    - name: Get json data
+      set_fact:
+        jsondata: "{{ result.stdout | from_json }}"
+
+    - name: Get Validations Status
+      set_fact:
+        status: "{{ jsondata | json_query(jsonres) }}"
+      vars:
+        jsonres: 'results[*].Status'
+
+    - fail:
+        msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}"
+      when: validation_status != "PASSED"
+      loop: "{{ status }}"
+      loop_control:
+        loop_var: validation_status
diff --git a/roles/validations/tasks/show.yaml b/roles/validations/tasks/show.yaml
deleted file mode 100644
index 37a4565..0000000
--- a/roles/validations/tasks/show.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-- name: Show Validation
-  register: show_output
-  shell:
-    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
-    executable: /bin/bash
diff --git a/roles/validations/tasks/show_results.yaml b/roles/validations/tasks/show_results.yaml
new file mode 100644
index 0000000..c7052c9
--- /dev/null
+++ b/roles/validations/tasks/show_results.yaml
@@ -0,0 +1,21 @@
+---
+- name: Get run UUID
+  block:
+    - name: Get run results file
+      register: result
+      shell:
+        cmd: "cat validation_{{ name.key }}_positive.log"
+        executable: /bin/bash
+    - name: Get uuid from log
+      set_fact:
+        validation_run_uuids: "{{ result.stdout | from_json | json_query(uuid_selector) }}"
+      vars:
+        uuid_selector: 'results[*].UUID'
+
+- name: Show Validation run results
+  shell:
+    cmd: "{{ validation_command }} history get {{ run_uuid }} --full > {{ val_working_dir }}/show_run.log 2>&1"
+    executable: /bin/bash
+  loop: "{{ validation_run_uuids }}"
+  loop_control:
+    loop_var: run_uuid
diff --git a/roles/validations/tasks/show_validation_info.yaml b/roles/validations/tasks/show_validation_info.yaml
new file mode 100644
index 0000000..09bfa49
--- /dev/null
+++ b/roles/validations/tasks/show_validation_info.yaml
@@ -0,0 +1,33 @@
+---
+- name: Show Validation - correct id
+  register: show_output
+  shell:
+    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
+    executable: /bin/bash
+
+# Simulating a typo in the validation name
+- name: Show Validation - incorrect id
+  block:
+    - name: Show validations - incorrect id
+      register: show_output_incorrect
+      shell:
+        cmd: "{{ validation_command }} show {{ validation_dir }} chuck-cpu -f json 2>&1 | tee {{val_working_dir}}/show_typo.log"
+        executable: /bin/bash
+      ignore_errors: true
+    - fail:
+        msg: "The expected error message was not displayed."
+      when: "'Validation chuck-cpu not found' not in show_output_incorrect.stdout"
+
+- name: Show Validation parameters
+  shell:
+    cmd: >-
+     {{ validation_command }} show parameter
+     --validation {{ name.key }} {{ validation_dir }}
+     --download {{ name.key }}_params.{{ format_type }}
+     --format-output {{ format_type }}
+    executable: /bin/bash
+  loop:
+    - json
+    - yaml
+  loop_control:
+    loop_var: format_type
diff --git a/roles/validations/vars/main.yaml b/roles/validations/vars/main.yaml
index 9ec9bc4..132639c 100644
--- a/roles/validations/vars/main.yaml
+++ b/roles/validations/vars/main.yaml
@@ -39,3 +39,20 @@ validations_group:
     - compute
   network:
     - network
+
+validation_metadata:
+  group:
+    - no-op
+    - prep
+  category:
+    - storage
+    - container
+  product:
+    - tripleo
+
+validation_list_formats:
+  - csv
+  - json
+  - table
+  - value
+  - yaml