Update the embedded-ansible-setup process to be configurable

This change allows the embedded ansible process to be configurable by the end user. * Python requirements and ansible roles will all now be user configurable. * Setup is now a local only playbook. This playbook replaces the bash commands we were rerunning when the `bootstrap-embedded-ansible.sh` script was executed. * Embedded ansible version is now 2.7.5 as default. * Deprecation warnings have been resolved. * Tests impacted by this change have been updated. Change-Id: I4303c44e249cda31457a4f05a681e298d225a8b7 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
2019-01-03 17:03:28 -06:00 · 2019-01-03 17:03:28 -06:00 · 7491b6df8e
commit 7491b6df8e
parent b1232aead5
19 changed files with 103 additions and 81 deletions
--- a/bootstrap-embedded-ansible/README.rst
+++ b/bootstrap-embedded-ansible/README.rst
@ -23,3 +23,21 @@ With the script sourced, the ansible environment will create a virtual environme
 `${HOME}/ansible_venv` if it does not already exist.

 To leave the embedded ansible environment run the function `deactivate`.
+
+
+Options
+^^^^^^^
+
+All options are passed in using environment variables.
+
+ANSIBLE_VERSION:
+  Allows for the Ansible XXX to be overridden. When set the full ansible version is required.
+
+ANSIBLE_EMBED_HOME:
+  Allows for the Ansible XXX to be overridden. When set the full path is required.
+
+ANSIBLE_ROLE_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the role requirements file is required.
+
+ANSIBLE_PYTHON_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the python requirements file is required.
--- a/bootstrap-embedded-ansible/ansible-requirements.yml
+++ b/bootstrap-embedded-ansible/ansible-requirements.yml
@ -0,0 +1,17 @@
+---
+- name: config_template
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-config_template
+  version: d97279293d09ab3cae6e20119662029d6b64b3b7
+- name: plugins
+  scm: git
+  src: https://git.openstack.org/openstack/openstack-ansible-plugins
+  version: 7fd6dd21123f162091631d87bf1db8fb7700fcda
+- name: systemd_service
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-role-systemd_service
+  version: 86ad639f4171b0c01bba030ebc3fd96ec020aa45
+- name: systemd_mount
+  scm: git
+  src: https://git.openstack.org/openstack/ansible-role-systemd_mount
+  version: master
--- a/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh
+++ b/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh
@ -13,11 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# Check if embedded ansible is already activated. If it is active, deactivate it.
+alias deactivate &> /dev/null && deactivate
+
 export OPTS=()
-export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.5.5.0}"
+export CLONE_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+OPTS+=('CLONE_DIR')
+
+export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.5.0}"
+OPTS+=('ANSIBLE_VERSION')
+
 export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
 OPTS+=('ANSIBLE_EMBED_HOME')

+export ANSIBLE_ROLE_REQUIREMENTS="${ANSIBLE_ROLE_REQUIREMENTS:-$CLONE_DIR/ansible-requirements.yml}"
+OPTS+=('ANSIBLE_ROLE_REQUIREMENTS')
+
+export ANSIBLE_PYTHON_REQUIREMENTS="${ANSIBLE_PYTHON_REQUIREMENTS:-${CLONE_DIR}/python-requirements.txt}"
+OPTS+=('ANSIBLE_PYTHON_REQUIREMENTS')
+
 source /etc/os-release
 export ID="$(echo ${ID} | awk -F'-' '{print $1}')"

@ -42,55 +56,15 @@ if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
    virtualenv "${ANSIBLE_EMBED_HOME}"
  fi
  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade --force pip"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible==${ANSIBLE_VERSION} --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade jmespath --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade hvac --isolated"
-  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade netaddr --isolated"
  echo "Ansible can be found here: ${ANSIBLE_EMBED_HOME}/bin"
 fi

-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/ansible-config_template "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template"
-    git checkout a5c9d97e18683f0fdf9769d94ba174c72e2d093c  # HEAD of master from 20-06-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/openstack-ansible-plugins "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins"
-    git checkout 761338d09c4cfb356c53fbd0d28a0e55a4776da0  # HEAD of master from 29-11-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/ansible-role-systemd_service "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service"
-    git checkout 02f5ff1c0e073af53bed2141a045e608162970ea  # HEAD of master from 20-06-18
-  popd
-fi
-
-if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" ]]; then
-  mkdir -p "${ANSIBLE_EMBED_HOME}/repositories"
-  git clone https://git.openstack.org/openstack/ansible-role-systemd_mount "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
-  pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount"
-    git checkout 0cca0b06e20a4e3d2b6b4ca19172717b6b37b68a  # HEAD of master from 20-06-18
-  popd
-fi
+# Run ansible setup
+eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible=='${ANSIBLE_VERSION}' --isolated"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-galaxy install --force --role-file='${ANSIBLE_ROLE_REQUIREMENTS}' --roles-path='${ANSIBLE_EMBED_HOME}/repositories/roles'"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-playbook -i 'localhost,' '${CLONE_DIR}/embedded-ansible-setup.yml' -e 'ansible_venv_path=${ANSIBLE_EMBED_HOME}' -e 'ansible_python_requirement_file=${ANSIBLE_PYTHON_REQUIREMENTS}'"

 if [[ -f "/etc/openstack_deploy/openstack_inventory.json" ]]; then
-  if [[ ! -f "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" ]]; then
-    mkdir -p "${ANSIBLE_EMBED_HOME}/inventory"
-    cat > "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" <<EOF
-#!/usr/bin/env bash
-cat /etc/openstack_deploy/openstack_inventory.json
-EOF
-    chmod +x "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh"
-  fi
-
  export USER_VARS="$(for i in $(ls -1 /etc/openstack_deploy/user_*secret*.yml); do echo -n "-e@$i "; done)"
  OPTS+=('USER_VARS')
  echo "env USER_VARS set"
@ -114,11 +88,11 @@ export ANSIBLE_ROLES_PATH="${ANSIBLE_EMBED_HOME}/repositories/roles"
 OPTS+=('ANSIBLE_ROLES_PATH')
 echo "env ANSIBLE_ROLES_PATH set"

-export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template/action"
+export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/config_template/action"
 OPTS+=('ANSIBLE_ACTION_PLUGINS')
 echo "env ANSIBLE_ACTION_PLUGINS set"

-export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins/connection/"
+export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/plugins/connection"
 OPTS+=('ANSIBLE_CONNECTION_PLUGINS')
 echo "env ANSIBLE_CONNECTION_PLUGINS set"

@ -131,8 +105,8 @@ OPTS+=('ANSIBLE_SSH_PIPELINING')
 echo "env ANSIBLE_SSH_PIPELINING set"

 export ANSIBLE_PIPELINING="${ANSIBLE_SSH_PIPELINING}"
-OPTS+=('ANSIBLE_SSH_PIPELINING')
-echo "env ANSIBLE_SSH_PIPELINING set"
+OPTS+=('ANSIBLE_PIPELINING')
+echo "env ANSIBLE_PIPELINING set"

 export ANSIBLE_SSH_RETRIES="${ANSIBLE_SSH_RETRIES:-5}"
 OPTS+=('ANSIBLE_SSH_RETRIES')
--- a/bootstrap-embedded-ansible/embedded-ansible-setup.yml
+++ b/bootstrap-embedded-ansible/embedded-ansible-setup.yml
@ -0,0 +1,34 @@
+---
+- name: Setup embedded ansible
+  hosts: localhost
+  connection: local
+  user: root
+  gather_facts: no
+  vars:
+    default_ansible_venv_path: "{{ lookup('env','HOME') + '/ansible_venv' }}"
+  tasks:
+    - name: Ensure pip packages are installed
+      pip:
+        requirements: "{{ ansible_python_requirement_file | default('python-requirements.txt') }}"
+        virtualenv: "{{ ansible_venv_path | default(default_ansible_venv_path) }}"
+        extra_args: '--isolated'
+
+    - name: Check for OSA inventory
+      stat:
+        path: /etc/openstack_deploy/openstack_inventory.json
+      register: osa_inventory
+
+    - name: osa block
+      block:
+        - name: Create inventory shim directory
+          file:
+            path: "{{ default_ansible_venv_path }}/inventory"
+            state: "directory"
+
+        - name: Copy embedded ansible inventory shim
+          copy:
+            dest: "{{ default_ansible_venv_path }}/inventory/openstack_inventory.sh"
+            src: osa-inventory.sh
+            mode: '0755'
+      when:
+        - osa_inventory.stat.exists | bool
--- a/bootstrap-embedded-ansible/osa-inventory.sh
+++ b/bootstrap-embedded-ansible/osa-inventory.sh
@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+cat /etc/openstack_deploy/openstack_inventory.json
--- a/bootstrap-embedded-ansible/python-requirements.txt
+++ b/bootstrap-embedded-ansible/python-requirements.txt
@ -0,0 +1,3 @@
+jmespath
+hvac
+netaddr
--- a/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml
@ -72,7 +72,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml
@ -85,7 +85,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml
+++ b/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml
@ -16,7 +16,6 @@
 - name: Run the systemd service role
  include_role:
    name: systemd_service
-    private: true
  vars:
    systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
    systemd_service_restart_changed: false
--- a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml
@ -85,7 +85,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml
@ -91,7 +91,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml
@ -91,7 +91,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml
@ -125,7 +125,6 @@
    - name: Run the systemd mount role
      include_role:
        name: systemd_mount
-        private: true
      vars:
        systemd_mounts:
          - what: "tmpfs"
--- a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml
@ -308,7 +308,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml
+++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml
@ -76,7 +76,6 @@
 - name: Run the beat setup role
  include_role:
    name: elastic_beat_setup
-    private: true
  when:
    - (groups['kibana'] | length) > 0
  vars:
--- a/elk_metrics_6x/tests/functional.yml
+++ b/elk_metrics_6x/tests/functional.yml
@ -26,6 +26,9 @@
    ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
    ANSIBLE_HOST_KEY_CHECKING: "False"
    ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log"
+    ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action"
+    ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection"
+    ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"

  vars:
    inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml"
@ -114,10 +117,7 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests"
      when:
@ -129,10 +129,7 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests"
      when:
@ -150,10 +147,7 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-deployment.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/elk_metrics_6x"

@ -162,9 +156,6 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml showElasticCluster.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-show-cluster.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/elk_metrics_6x"
--- a/osquery/roles/fleet/tasks/fleetService.yml
+++ b/osquery/roles/fleet/tasks/fleetService.yml
@ -16,7 +16,6 @@
 - name: Run the systemd service role
  include_role:
    name: systemd_service
-    private: true
  vars:
    systemd_service_restart_changed: false
    systemd_services:
--- a/osquery/roles/fleet/tasks/main.yml
+++ b/osquery/roles/fleet/tasks/main.yml
@ -28,7 +28,6 @@
 - name: Run the systemd service role
  include_role:
    name: redis
-    private: true

 # install kolide fleet server
 - include_tasks: fleetServerInstall.yml
--- a/osquery/tests/functional.yml
+++ b/osquery/tests/functional.yml
@ -26,6 +26,9 @@
    ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
    ANSIBLE_HOST_KEY_CHECKING: "False"
    ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test.log"
+    ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action"
+    ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection"
+    ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"

  vars:
    inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml"
@ -114,10 +117,7 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/osquery/tests"
      when:
@ -129,10 +129,7 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/osquery/tests"
      when:
@ -150,9 +147,6 @@
      become_user: root
      command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml"
      environment:
-        ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action"
-        ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection"
        ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-deployment.log"
-        ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles"
      args:
        chdir: "src/{{ current_test_repo }}/osquery"