Test validations for false positives

By running validations with arguments intentionally chosen
to make the validation requirements impossible to fulfill,
we can check how the framework handles failures, and if
the tested validations do not produce false positives.

The functional test now, in addition to the existing
set of checks, allow for optional 'negative_results' testing.

Variables entries for all tested validations are checked
for the presence of 'negative_results' dictionary key.
If it is found, the 'extra_args' key contents within are used
to run the validation in question, after it has passed the test
for a successful validation run.

The 'negative results' run is expected to fail, failure is supressed,
and result recorded. Passing result, conversely, is interpreted as an error,
because validation passing shouldn't be possible with the arguments supplied.

Show validations test was patched to proper form of the command.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: If7e7b25b9c876184bd04f3f0ddefdee20172a5b4

roles/validations/tasks/main.yaml

@@ -43,16 +43,20 @@
     validation_dir: "--validation-dir /usr/share/ansible/validation-playbooks"
   when: not is_virtualenv.stat.exists
 
-- include: run.yaml name="{{ item }}"
+- include_tasks: run.yaml
+  vars:
+    name: "{{ item }}"
   when:
     - run_validation|default(false)|bool
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
 
-- include: list.yaml
-
-- include: show.yaml name="{{ item }}"
+- include_tasks: show.yaml
+  vars:
+    name: "{{ item }}"
   when:
     - run_validation|default(false)|bool
     - validation_component | length > 0
   with_dict: "{{ validations_list[validation_component] }}"
+
+- include_tasks: list.yaml

roles/validations/tasks/run.yaml

@@ -1,22 +1,21 @@
 ---
-- name: Run validations
-  register: run_validation
+- name: Run validations - positive - these are supposed to pass
   shell:
     cmd: >-
       {{ validation_command }} run --validation {{ name.key }}
       {{ validation_dir }} {{ ansible_dir }}
       --inventory {{ inventory }}
-      --output-log validation_{{ name.key }}.log
+      --output-log validation_{{ name.key }}_positive.log
       {{ name.value.extra_args }}
       {{ name.value.extra_env_args }}
     executable: /bin/bash
 
-- name: Get Run results
+- name: Get Run results - positive - these are supposed to pass
   block:
     - name: Get run results
       register: result
       shell:
-        cmd: "cat validation_{{ name.key }}.log"
+        cmd: "cat validation_{{ name.key }}_positive.log"
         executable: /bin/bash
 
     - name: Get json data
@@ -30,8 +29,49 @@
         jsonres: 'results[*].Status'
 
     - fail:
-        msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}"
+        msg: "Validation failed with {{ validation_status }}: some of the validations have failed. {{ status }}"
       when: validation_status != "PASSED"
       loop: "{{ status }}"
       loop_control:
         loop_var: validation_status
+
+- name: Run validations - negative - these are not supposed to pass
+  shell:
+    cmd: >-
+      {{ validation_command }} run --validation {{ name.key }}
+      {{ validation_dir }} {{ ansible_dir }}
+      --inventory {{ inventory }}
+      --output-log validation_{{ name.key }}_negative.log
+      {{ name.value.negative_results.extra_args }}
+      {{ name.value.extra_env_args }}
+    executable: /bin/bash
+  when: "'negative_results' in name.value "
+  ignore_errors: true
+
+- name: Get Run results - negative - these are not supposed to pass
+  # This task should fail with return code != 0
+  # The validation is supplied with parameters that make it impossible to pass.
+  block:
+    - name: Get run results
+      register: result
+      shell:
+        cmd: "cat validation_{{ name.key }}_negative.log"
+        executable: /bin/bash
+
+    - name: Get json data
+      set_fact:
+        jsondata: "{{ result.stdout | from_json }}"
+
+    - name: Get Validations Status
+      set_fact:
+        status: "{{ jsondata | json_query(jsonres) }}"
+      vars:
+        jsonres: 'results[*].Status'
+
+    - fail:
+        msg: "Validation passed with {{ validation_status }} when it shouldn't have: some of the validations have passed. {{ status }}"
+      when: validation_status == "PASSED"
+      loop: "{{ status }}"
+      loop_control:
+        loop_var: validation_status
+  when: "'negative_results' in name.value"

roles/validations/tasks/show.yaml

@@ -2,5 +2,5 @@
 - name: Show Validation
   register: show_output
   shell:
-    cmd: "{{ validation_command }} show {{ validation_dir }} --validation {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
+    cmd: "{{ validation_command }} show {{ validation_dir }} {{ name.key }} -f json > {{ val_working_dir }}/show.log 2>&1"
     executable: /bin/bash

roles/validations/tasks/validations.yaml

@@ -7,7 +7,7 @@
 - name: set fact for Validation action
   set_fact: v_action="{{ action }}"
 
-- name: Get Run results
+- name: Get Run results - Positive - these are supposed to pass
   block:
     - name: Get run results
       register: result
@@ -26,9 +26,39 @@
         jsonres: 'results[*].Status'
 
     - fail:
-        msg: "Validation failed with {{ validation_status }}: some of the validations has failed. {{ status }}"
+        msg: "Validation failed with {{ validation_status }}: some of the validations have failed. {{ status }}"
       when: validation_status != "PASSED"
       loop: "{{ status }}"
       loop_control:
         loop_var: validation_status
-  when: v_action == "run"
+  when: v_action == 'run'
+
+- name: Get Run results - Negative - these are supposed to fail
+  # This task should fail with return code != 0
+  # The validation is supplied with parameters that make it impossible to pass.
+  block:
+    - name: Get run results
+      register: result
+      shell:
+        cmd: "cat {{ val_output }}"
+        executable: /bin/bash
+
+    - name: Get json data
+      set_fact:
+        jsondata: "{{ result.stdout | from_json }}"
+
+    - name: Get Validations Status
+      set_fact:
+        status: "{{ jsondata | json_query(jsonres) }}"
+      vars:
+        jsonres: 'results[*].Status'
+
+    - fail:
+        msg: "Validation passed with {{ validation_status }} when it shouldn't have: some of the validations have passed. {{ status }}"
+      when: validation_status == "PASSED"
+      loop: "{{ status }}"
+      loop_control:
+        loop_var: validation_status
+  when:
+    - v_action == 'run'
+    - "'negative_results' in name.value"

roles/validations/vars/main.yaml

@@ -12,12 +12,16 @@ validations_list:
         extra_env_args: "{{ extra_env_vars | default('') }}"
   validation:
     - check-cpu:
+        negative_results:
+          extra_args: "--extra-vars minimal_cpu_count=20000"
         extra_args: "--extra-vars minimal_cpu_count=2"
         extra_env_args: "--extra-env-vars ANSIBLE_STDOUT_CALLBACK=default {{ extra_env_vars | default('') }}"
     - 512e:
         extra_args: ""
         extra_env_args: ""
     - check-ram:
+        negative_results:
+          extra_args: "--extra-vars minimal_ram_gb=2000000"
         extra_args: "--extra-vars minimal_ram_gb=2"
         extra_env_args: "--extra-env-vars ANSIBLE_STDOUT_CALLBACK=default {{ extra_env_vars | default('') }}"
   tripleo: