From 7491b6df8efc72ef7e85df9e335091cb6cab1e1f Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 3 Jan 2019 17:03:28 -0600 Subject: [PATCH] Update the embedded-ansible-setup process to be configurable This change allows the embedded ansible process to be configurable by the end user. * Python requirements and ansible roles will all now be user configurable. * Setup is now a local only playbook. This playbook replaces the bash commands we were rerunning when the `bootstrap-embedded-ansible.sh` script was executed. * Embedded ansible version is now 2.7.5 as default. * Deprecation warnings have been resolved. * Tests impacted by this change have been updated. Change-Id: I4303c44e249cda31457a4f05a681e298d225a8b7 Signed-off-by: Kevin Carter --- bootstrap-embedded-ansible/README.rst | 18 +++++ .../ansible-requirements.yml | 17 +++++ .../bootstrap-embedded-ansible.sh | 72 ++++++------------- .../embedded-ansible-setup.yml | 34 +++++++++ bootstrap-embedded-ansible/osa-inventory.sh | 2 + .../python-requirements.txt | 3 + .../roles/elastic_apm_server/tasks/main.yml | 1 - .../roles/elastic_auditbeat/tasks/main.yml | 1 - .../elastic_curator/tasks/curator_systemd.yml | 1 - .../roles/elastic_filebeat/tasks/main.yml | 1 - .../roles/elastic_heartbeat/tasks/main.yml | 1 - .../roles/elastic_journalbeat/tasks/main.yml | 1 - .../roles/elastic_logstash/tasks/main.yml | 1 - .../roles/elastic_metricbeat/tasks/main.yml | 1 - .../roles/elastic_packetbeat/tasks/main.yml | 1 - elk_metrics_6x/tests/functional.yml | 15 +--- osquery/roles/fleet/tasks/fleetService.yml | 1 - osquery/roles/fleet/tasks/main.yml | 1 - osquery/tests/functional.yml | 12 +--- 19 files changed, 103 insertions(+), 81 deletions(-) create mode 100644 bootstrap-embedded-ansible/ansible-requirements.yml create mode 100644 bootstrap-embedded-ansible/embedded-ansible-setup.yml create mode 100644 bootstrap-embedded-ansible/osa-inventory.sh create mode 100644 bootstrap-embedded-ansible/python-requirements.txt diff --git a/bootstrap-embedded-ansible/README.rst b/bootstrap-embedded-ansible/README.rst index 2e88c271..fc4b3d69 100644 --- a/bootstrap-embedded-ansible/README.rst +++ b/bootstrap-embedded-ansible/README.rst @@ -23,3 +23,21 @@ With the script sourced, the ansible environment will create a virtual environme `${HOME}/ansible_venv` if it does not already exist. To leave the embedded ansible environment run the function `deactivate`. + + +Options +^^^^^^^ + +All options are passed in using environment variables. + +ANSIBLE_VERSION: + Allows for the Ansible XXX to be overridden. When set the full ansible version is required. + +ANSIBLE_EMBED_HOME: + Allows for the Ansible XXX to be overridden. When set the full path is required. + +ANSIBLE_ROLE_REQUIREMENTS: + Allows for the Ansible XXX to be overridden. When set the full path to the role requirements file is required. + +ANSIBLE_PYTHON_REQUIREMENTS: + Allows for the Ansible XXX to be overridden. When set the full path to the python requirements file is required. diff --git a/bootstrap-embedded-ansible/ansible-requirements.yml b/bootstrap-embedded-ansible/ansible-requirements.yml new file mode 100644 index 00000000..2f558498 --- /dev/null +++ b/bootstrap-embedded-ansible/ansible-requirements.yml @@ -0,0 +1,17 @@ +--- +- name: config_template + scm: git + src: https://git.openstack.org/openstack/ansible-config_template + version: d97279293d09ab3cae6e20119662029d6b64b3b7 +- name: plugins + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-plugins + version: 7fd6dd21123f162091631d87bf1db8fb7700fcda +- name: systemd_service + scm: git + src: https://git.openstack.org/openstack/ansible-role-systemd_service + version: 86ad639f4171b0c01bba030ebc3fd96ec020aa45 +- name: systemd_mount + scm: git + src: https://git.openstack.org/openstack/ansible-role-systemd_mount + version: master diff --git a/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh b/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh index 37f2cee2..841f1138 100755 --- a/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh +++ b/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh @@ -13,11 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Check if embedded ansible is already activated. If it is active, deactivate it. +alias deactivate &> /dev/null && deactivate + export OPTS=() -export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.5.5.0}" +export CLONE_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +OPTS+=('CLONE_DIR') + +export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.5.0}" +OPTS+=('ANSIBLE_VERSION') + export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv" OPTS+=('ANSIBLE_EMBED_HOME') +export ANSIBLE_ROLE_REQUIREMENTS="${ANSIBLE_ROLE_REQUIREMENTS:-$CLONE_DIR/ansible-requirements.yml}" +OPTS+=('ANSIBLE_ROLE_REQUIREMENTS') + +export ANSIBLE_PYTHON_REQUIREMENTS="${ANSIBLE_PYTHON_REQUIREMENTS:-${CLONE_DIR}/python-requirements.txt}" +OPTS+=('ANSIBLE_PYTHON_REQUIREMENTS') + source /etc/os-release export ID="$(echo ${ID} | awk -F'-' '{print $1}')" @@ -42,55 +56,15 @@ if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then virtualenv "${ANSIBLE_EMBED_HOME}" fi eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade --force pip" - eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible==${ANSIBLE_VERSION} --isolated" - eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade jmespath --isolated" - eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade hvac --isolated" - eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade netaddr --isolated" echo "Ansible can be found here: ${ANSIBLE_EMBED_HOME}/bin" fi -if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" ]]; then - mkdir -p "${ANSIBLE_EMBED_HOME}/repositories" - git clone https://git.openstack.org/openstack/ansible-config_template "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" - pushd "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" - git checkout a5c9d97e18683f0fdf9769d94ba174c72e2d093c # HEAD of master from 20-06-18 - popd -fi - -if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" ]]; then - mkdir -p "${ANSIBLE_EMBED_HOME}/repositories" - git clone https://git.openstack.org/openstack/openstack-ansible-plugins "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" - pushd "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" - git checkout 761338d09c4cfb356c53fbd0d28a0e55a4776da0 # HEAD of master from 29-11-18 - popd -fi - -if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" ]]; then - mkdir -p "${ANSIBLE_EMBED_HOME}/repositories" - git clone https://git.openstack.org/openstack/ansible-role-systemd_service "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" - pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_service" - git checkout 02f5ff1c0e073af53bed2141a045e608162970ea # HEAD of master from 20-06-18 - popd -fi - -if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" ]]; then - mkdir -p "${ANSIBLE_EMBED_HOME}/repositories" - git clone https://git.openstack.org/openstack/ansible-role-systemd_mount "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" - pushd "${ANSIBLE_EMBED_HOME}/repositories/roles/systemd_mount" - git checkout 0cca0b06e20a4e3d2b6b4ca19172717b6b37b68a # HEAD of master from 20-06-18 - popd -fi +# Run ansible setup +eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible=='${ANSIBLE_VERSION}' --isolated" +eval "${ANSIBLE_EMBED_HOME}/bin/ansible-galaxy install --force --role-file='${ANSIBLE_ROLE_REQUIREMENTS}' --roles-path='${ANSIBLE_EMBED_HOME}/repositories/roles'" +eval "${ANSIBLE_EMBED_HOME}/bin/ansible-playbook -i 'localhost,' '${CLONE_DIR}/embedded-ansible-setup.yml' -e 'ansible_venv_path=${ANSIBLE_EMBED_HOME}' -e 'ansible_python_requirement_file=${ANSIBLE_PYTHON_REQUIREMENTS}'" if [[ -f "/etc/openstack_deploy/openstack_inventory.json" ]]; then - if [[ ! -f "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" ]]; then - mkdir -p "${ANSIBLE_EMBED_HOME}/inventory" - cat > "${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh" < 0 vars: diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml index aed5986f..2e6cf744 100644 --- a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml @@ -85,7 +85,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml b/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml index a46854e3..bf5752f5 100644 --- a/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml +++ b/elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml @@ -16,7 +16,6 @@ - name: Run the systemd service role include_role: name: systemd_service - private: true vars: systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" systemd_service_restart_changed: false diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml index 7fb6daff..5fd827e3 100644 --- a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml @@ -85,7 +85,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml index c1e9f19a..f6f7ddf8 100644 --- a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml @@ -91,7 +91,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml index 0d453815..e6fee44f 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml @@ -91,7 +91,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml index e852b126..773a2a96 100644 --- a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml @@ -125,7 +125,6 @@ - name: Run the systemd mount role include_role: name: systemd_mount - private: true vars: systemd_mounts: - what: "tmpfs" diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml index 07319c7d..6ca09fb6 100644 --- a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml @@ -308,7 +308,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml index 89acee01..2b5337a0 100644 --- a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml @@ -76,7 +76,6 @@ - name: Run the beat setup role include_role: name: elastic_beat_setup - private: true when: - (groups['kibana'] | length) > 0 vars: diff --git a/elk_metrics_6x/tests/functional.yml b/elk_metrics_6x/tests/functional.yml index ea7bec4f..9b971cdf 100644 --- a/elk_metrics_6x/tests/functional.yml +++ b/elk_metrics_6x/tests/functional.yml @@ -26,6 +26,9 @@ ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}" ANSIBLE_HOST_KEY_CHECKING: "False" ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action" + ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection" + ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" vars: inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml" @@ -114,10 +117,7 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests" when: @@ -129,10 +129,7 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests" when: @@ -150,10 +147,7 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-deployment.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/elk_metrics_6x" @@ -162,9 +156,6 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml showElasticCluster.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-show-cluster.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/elk_metrics_6x" diff --git a/osquery/roles/fleet/tasks/fleetService.yml b/osquery/roles/fleet/tasks/fleetService.yml index 2eba403c..12c20ec2 100644 --- a/osquery/roles/fleet/tasks/fleetService.yml +++ b/osquery/roles/fleet/tasks/fleetService.yml @@ -16,7 +16,6 @@ - name: Run the systemd service role include_role: name: systemd_service - private: true vars: systemd_service_restart_changed: false systemd_services: diff --git a/osquery/roles/fleet/tasks/main.yml b/osquery/roles/fleet/tasks/main.yml index d40ac2a2..ad638ef7 100644 --- a/osquery/roles/fleet/tasks/main.yml +++ b/osquery/roles/fleet/tasks/main.yml @@ -28,7 +28,6 @@ - name: Run the systemd service role include_role: name: redis - private: true # install kolide fleet server - include_tasks: fleetServerInstall.yml diff --git a/osquery/tests/functional.yml b/osquery/tests/functional.yml index 6cf6fcb3..75ed57c7 100644 --- a/osquery/tests/functional.yml +++ b/osquery/tests/functional.yml @@ -26,6 +26,9 @@ ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}" ANSIBLE_HOST_KEY_CHECKING: "False" ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test.log" + ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action" + ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection" + ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" vars: inventory_file: "inventory/test-{{ (contianer_inventory | bool) | ternary('container', 'metal') }}-inventory.yml" @@ -114,10 +117,7 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/osquery/tests" when: @@ -129,10 +129,7 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-container-setup.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/osquery/tests" when: @@ -150,9 +147,6 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml" environment: - ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/ansible-config_template/action" - ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/openstack-ansible-plugins/connection" ANSIBLE_LOG_PATH: "/tmp/osquery-logs/ansible-osquery-test-deployment.log" - ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" args: chdir: "src/{{ current_test_repo }}/osquery"