Update the embedded-ansible-setup process to be configurable

This change allows the embedded ansible process to be configurable by
the end user.
  * Python requirements and ansible roles will all now be user
    configurable.
  * Setup is now a local only playbook. This playbook replaces the bash
    commands we were rerunning when the `bootstrap-embedded-ansible.sh`
    script was executed.
  * Embedded ansible version is now 2.7.5 as default.
  * Deprecation warnings have been resolved.
  * Tests impacted by this change have been updated.

Change-Id: I4303c44e249cda31457a4f05a681e298d225a8b7
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

bootstrap-embedded-ansible/README.rst

@@ -23,3 +23,21 @@ With the script sourced, the ansible environment will create a virtual environme
 `${HOME}/ansible_venv` if it does not already exist.
 
 To leave the embedded ansible environment run the function `deactivate`.
+
+
+Options
+^^^^^^^
+
+All options are passed in using environment variables.
+
+ANSIBLE_VERSION:
+  Allows for the Ansible XXX to be overridden. When set the full ansible version is required.
+
+ANSIBLE_EMBED_HOME:
+  Allows for the Ansible XXX to be overridden. When set the full path is required.
+
+ANSIBLE_ROLE_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the role requirements file is required.
+
+ANSIBLE_PYTHON_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the python requirements file is required.

bootstrap-embedded-ansible/ansible-requirements.yml

@@ -0,0 +1,17 @@
+---
+- name: config_template
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-config_template
+  version: d97279293d09ab3cae6e20119662029d6b64b3b7
+- name: plugins
+  scm: git
+  src: https://git.openstack.org/openstack/openstack-ansible-plugins
+  version: 7fd6dd21123f162091631d87bf1db8fb7700fcda
+- name: systemd_service
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-role-systemd_service
+  version: 86ad639f4171b0c01bba030ebc3fd96ec020aa45
+- name: systemd_mount
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-role-systemd_mount
+  version: master

bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh

@@ -13,11 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Check if embedded ansible is already activated. If it is active, deactivate it.
+alias deactivate &> /dev/null && deactivate
+
 export OPTS=()
-export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.5.5.0}"
+export CLONE_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+OPTS+=('CLONE_DIR')
+
+export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.5.0}"
+OPTS+=('ANSIBLE_VERSION')
+
 export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
 OPTS+=('ANSIBLE_EMBED_HOME')
 
+export ANSIBLE_ROLE_REQUIREMENTS="${ANSIBLE_ROLE_REQUIREMENTS:-$CLONE_DIR/ansible-requirements.yml}"
+OPTS+=('ANSIBLE_ROLE_REQUIREMENTS')
+
+export ANSIBLE_PYTHON_REQUIREMENTS="${ANSIBLE_PYTHON_REQUIREMENTS:-${CLONE_DIR}/python-requirements.txt}"
+OPTS+=('ANSIBLE_PYTHON_REQUIREMENTS')
+
 source /etc/os-release
 export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
 
@@ -42,55 +56,15 @@ if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
     virtualenv "${ANSIBLE_EMBED_HOME}"
   fi
   eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade --force pip"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible==${ANSIBLE_VERSION} --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade jmespath --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade hvac --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade netaddr --isolated"
   echo "Ansible can be found here: ${ANSIBLE_EMBED_HOME}/bin"
 fi
 
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" ]]; then
+# Run ansible setup
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
+eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible=='${ANSIBLE_VERSION}' --isolated"
-  git clone https://git.openstack.org/openstack/ansible-config_template "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-galaxy install --force --role-file='${ANSIBLE_ROLE_REQUIREMENTS}' --roles-path='${ANSIBLE_EMBED_HOME}/repositories/roles'"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-playbook -i 'localhost,' '${CLONE_DIR}/embedded-ansible-setup.yml' -e 'ansible_venv_path=${ANSIBLE_EMBED_HOME}' -e 'ansible_python_requirement_file=${ANSIBLE_PYTHON_REQUIREMENTS}'"
-    git checkout a5c9d97e18683f0fdf9769d94ba174c72e2d093c  # HEAD of master from 20-06-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/openstack-ansible-plugins "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
-    git checkout 761338d09c4cfb356c53fbd0d28a0e55a4776da0  # HEAD of master from 29-11-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/ansible-role-systemd_service "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
-    git checkout 02f5ff1c0e073af53bed2141a045e608162970ea  # HEAD of master from 20-06-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/ansible-role-systemd_mount "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
-    git checkout 0cca0b06e20a4e3d2b6b4ca19172717b6b37b68a  # HEAD of master from 20-06-18
-  popd
-fi
 
 if [[ -f "/etc/openstack_deploy/openstack_inventory.json" ]]; then
-  if [[ ! -f "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" ]]; then
-    mkdir -p "${ANSIBLE_EMBED_HOME}/inventory"
-    cat > "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" <<EOF
-#!/usr/bin/env bash
-cat /etc/openstack_deploy/openstack_inventory.json
-EOF
-    chmod +x "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh"
-  fi
-
   export USER_VARS="$(for i in $(ls -1 /etc/openstack_deploy/user_*secret*.yml); do echo -n "-e@$i "; done)"
   OPTS+=('USER_VARS')
   echo "env USER_VARS set"
@@ -114,11 +88,11 @@ export ANSIBLE_ROLES_PATH="${ANSIBLE_EMBED_HOME}/repositories/roles"
 OPTS+=('ANSIBLE_ROLES_PATH')
 echo "env ANSIBLE_ROLES_PATH set"
 
-export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template/action"
+export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/config_template/action"
 OPTS+=('ANSIBLE_ACTION_PLUGINS')
 echo "env ANSIBLE_ACTION_PLUGINS set"
 
-export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins/connection/"
+export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/plugins/connection"
 OPTS+=('ANSIBLE_CONNECTION_PLUGINS')
 echo "env ANSIBLE_CONNECTION_PLUGINS set"
 
@@ -131,8 +105,8 @@ OPTS+=('ANSIBLE_SSH_PIPELINING')
 echo "env ANSIBLE_SSH_PIPELINING set"
 
 export ANSIBLE_PIPELINING="${ANSIBLE_SSH_PIPELINING}"
-OPTS+=('ANSIBLE_SSH_PIPELINING')
+OPTS+=('ANSIBLE_PIPELINING')
-echo "env ANSIBLE_SSH_PIPELINING set"
+echo "env ANSIBLE_PIPELINING set"
 
 export ANSIBLE_SSH_RETRIES="${ANSIBLE_SSH_RETRIES:-5}"
 OPTS+=('ANSIBLE_SSH_RETRIES')

bootstrap-embedded-ansible/embedded-ansible-setup.yml

@@ -0,0 +1,34 @@
+---
+- name: Setup embedded ansible
+  hosts: localhost
+  connection: local
+  user: root
+  gather_facts: no
+  vars:
+    default_ansible_venv_path: "{{ lookup('env','HOME') + '/ansible_venv' }}"
+  tasks:
+    - name: Ensure pip packages are installed
+      pip:
+        requirements: "{{ ansible_python_requirement_file | default('python-requirements.txt') }}"
+        virtualenv: "{{ ansible_venv_path | default(default_ansible_venv_path) }}"
+        extra_args: '--isolated'
+
+    - name: Check for OSA inventory
+      stat:
+        path: /etc/openstack_deploy/openstack_inventory.json
+      register: osa_inventory
+
+    - name: osa block
+      block:
+        - name: Create inventory shim directory
+          file:
+            path: "{{ default_ansible_venv_path }}/inventory"
+            state: "directory"
+
+        - name: Copy embedded ansible inventory shim
+          copy:
+            dest: "{{ default_ansible_venv_path }}/inventory/openstack_inventory.sh"
+            src: osa-inventory.sh
+            mode: '0755'
+      when:
+        - osa_inventory.stat.exists | bool

bootstrap-embedded-ansible/osa-inventory.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+cat /etc/openstack_deploy/openstack_inventory.json

bootstrap-embedded-ansible/python-requirements.txt

@@ -0,0 +1,3 @@
+jmespath
+hvac
+netaddr

elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml

@@ -72,7 +72,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml

@@ -85,7 +85,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml

@@ -16,7 +16,6 @@
 - name: Run the systemd service role
   include_role:
     name: systemd_service
-    private: true
   vars:
     systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
     systemd_service_restart_changed: false

elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml

@@ -85,7 +85,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml

@@ -91,7 +91,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml

@@ -91,7 +91,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_logstash/tasks/main.yml

@@ -125,7 +125,6 @@
     - name: Run the systemd mount role
       include_role:
         name: systemd_mount
-        private: true
       vars:
         systemd_mounts:
           - what: "tmpfs"

elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml

@@ -308,7 +308,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml

@@ -76,7 +76,6 @@
 - name: Run the beat setup role
   include_role:
     name: elastic_beat_setup
-    private: true
   when:
     - (groups['kibana'] | length) > 0
   vars:

elk_metrics_6x/tests/functional.yml

@@ -26,6 +26,9 @@
     ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
     ANSIBLE_HOST_KEY_CHECKING: "False"
     ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log"
+    ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action"
+    ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection"
+    ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
 
   vars:
     inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml"
@@ -114,10 +117,7 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests"
       when:
@@ -129,10 +129,7 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests"
       when:
@@ -150,10 +147,7 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-deployment.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/elk_metrics_6x"
 
@@ -162,9 +156,6 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml showElasticCluster.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-show-cluster.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/elk_metrics_6x"

osquery/roles/fleet/tasks/fleetService.yml

@@ -16,7 +16,6 @@
 - name: Run the systemd service role
   include_role:
     name: systemd_service
-    private: true
   vars:
     systemd_service_restart_changed: false
     systemd_services:

osquery/roles/fleet/tasks/main.yml

@@ -28,7 +28,6 @@
 - name: Run the systemd service role
   include_role:
     name: redis
-    private: true
 
 # install kolide fleet server
 - include_tasks: fleetServerInstall.yml

osquery/tests/functional.yml

@@ -26,6 +26,9 @@
     ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
     ANSIBLE_HOST_KEY_CHECKING: "False"
     ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test.log"
+    ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action"
+    ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection"
+    ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
 
   vars:
     inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml"
@@ -114,10 +117,7 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/osquery/tests"
       when:
@@ -129,10 +129,7 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/osquery/tests"
       when:
@@ -150,9 +147,6 @@
       become_user: root
       command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml"
       environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
         ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-deployment.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
       args:
         chdir: "src/{{ current_test_repo }}/osquery"