install-ansible: overhaul install ansible requirements

Change I4789fe99651597b073e35066ec3be312e18659b8 made me realise that
with the extant code, nothing will update the /usr/ansible-env
environment when we bump the versions.

The installation of the Ansible, openstacksdk and ARA packages as part
of the "install-ansible" role was done this way to facilitate being
able to install all three of these from their main/master/devel
branches for the "-devel" job, which is our basic canary for upstream
things that might affect us.  Because of the way the pip: role works
with "state: latest" and mixing on-disk paths with pypi package names,
this became a bit of a complex swizzling operation.

Some thing have changed since then; particularly us now using a
separate venv and upstream Ansible's change to use "collections"; so
pulling in a bug-fix for Ansible is not as simple as just cloning
github.com/ansible/ansible at a particular tag any more.  This means
we should reconsider how we're specifying the packages here.

This simplifies things to list the required packages in a
requirements.txt file, which we install into the venv root.  The nice
thing about this is that creating requirements.txt with the template:
role is idempotent, so we can essentially monitor the file for changes
and only (re-)run the pip install into /usr/ansible-env when we change
versions (forcing upgrades so we get the versions we want, and fixing
the original issue mentioned above).

Change-Id: I3696740112fa691d1700040b557f53f6721393e7

playbooks/bootstrap-bridge.yaml

@@ -16,26 +16,10 @@
   name: "Bridge: bootstrap the bastion host"
   become: true
   tasks:
-    # Note for production use we expect to take the defaults; unit
+
-    # test jobs override this to test with latest upstream ansible.
-    # For example, if there is a fix on the ansible stable branch we
-    # need that is unreleased, you could do the following:
-    #
-    #  install_ansible_name: '{{ bridge_ansible_name | default("git+https://github.com/ansible/ansible.git@stable-2.7") }}'
-    #  install_ansible_version: '{{ bridge_ansible_version | default(None) }}'
     - name: Install ansible
       include_role:
         name: install-ansible
-      vars:
-        install_ansible_name: '{{ bridge_ansible_name | default("ansible") }}'
-        install_ansible_version: '{{ bridge_ansible_version | default("<8") }}'
-        install_ansible_openstacksdk_name: '{{ bridge_openstacksdk_name | default("openstacksdk") }}'
-        install_ansible_openstacksdk_version: '{{ bridge_openstacksdk_verison | default("latest") }}'
-        # NOTE(ianw): At 2018-12, ARA is only enabled during gate
-        # testing jobs as we decide if or how to store data on
-        # production bridge.o.o
-        install_ansible_ara_name: '{{ bridge_ara_name | default("ara[server]") }}'
-        install_ansible_ara_version: '{{ bridge_ara_version | default("latest") }}'
 
     # This is the key that bridge uses to log into remote hosts.
     #

playbooks/roles/install-ansible/README.rst

@@ -1,62 +1,31 @@
 Install and configure Ansible on a host via pip
 
+This will install ansible into a virtualenv at ``/usr/ansible-venv``
+
 **Role Variables**
 
-.. zuul:rolevar:: install_ansible_name
+.. zuul:rolevar:: install_ansible_requirements
-   :default: ansible
+   :default: [ansible, openstacksdk]
 
-   The name of the ansible package to install.  To install from
+   The packages to install into the virtualenv.  A list in Python
-   alternative sources, this can be a URL for a remote package;
+   ``requirements.txt`` format.
-   e.g. to install from upstream devel branch
-   ``git+https://github.com/ansible/ansible.git@devel``
 
-.. zuul:rolevar:: install_ansible_version
+.. zuul:rolevar:: install_ansible_collections
-   :default: latest
+   :default: undefined
 
-   The version of the library from
+   A list of Ansible collections to install.  In the format
-   :zuul:rolevar:`install-ansible.install_ansible_name`.  Set this to
-   empty (YAML ``null``) if specifying versions via URL in
-   :zuul:rolevar:`install-ansible.install_ansible_name`.  The special
-   value "latest" will ensure ``state: latest`` is set for the
-   package and thus the latest version is always installed.
 
-.. zuul:rolevar:: install_ansible_openstacksdk_name
+   ..
-   :default: openstacksdk
+      - namespace:
-
+        name:
-   The name of the openstacksdk package to install.  To install from
+        repo:
-   alternative sources, this can be a URL for a remote package;
-   e.g. to install from a gerrit change
-   ``git+https://opendev.org/openstack/openstacksdk@refs/changes/12/3456/1#egg=openstacksdk``
-
-.. zuul:rolevar:: install_ansible_openstacksdk_version
-   :default: latest
-
-   The version of the library from
-   :zuul:rolevar:`install-ansible.install_ansible_openstacksdk_name`.  Set
-   this to empty (YAML ``null``) if specifying versions via
-   :zuul:rolevar:`install-ansible.install_ansible_openstacksdk_name`.  The
-   special value "latest" will ensure ``state: latest`` is set for the
-   package and thus the latest version is always installed.
 
 .. zuul:rolevar:: install_ansible_ara_enable
    :default: false
 
    Whether or not to install the ARA Records Ansible callback plugin
-
+   into Ansible.  If using the default
-.. zuul:rolevar:: install_ansible_ara_name
+   ``install_ansible_requirements`` will install the ARA package too.
-   :default: ara[server]
-
-   The name of the ARA package to install.  To install from
-   alternative sources, this can be a URL for a remote package.
-
-.. zuul:rolevar:: install_ansible_ara_version
-   :default: latest
-
-   Version of ARA to install.  Set this to empty (YAML ``null``) if
-   specifying versions via URL in
-   :zuul:rolevar:`install-ansible.install_ansible_ara_name`.  The
-   special value "latest" will ensure ``state: latest`` is set for the
-   package and hence the latest version is always installed.
 
 .. zuul:rolevar:: install_ansible_ara_config
 

playbooks/roles/install-ansible/tasks/install_ara.yaml

@@ -1,33 +0,0 @@
-# If ansible_install_ansible_ara_version is not defined it should be "latest"
-- name: Set ara default version to latest
-  set_fact:
-    install_ansible_ara_version: latest
-  when: install_ansible_ara_version is not defined
-
-# If a version is not explicitly set we want to make sure to
-# completely omit the version argument to pip, as it will be coming
-# from the long-form install_ansible_ara_name variable.  Additionally,
-# if the version is the special value "latest", then we also want to
-# omit any version number, but also set the package state to "latest".
-- name: Set ARA version for installation
-  set_fact:
-    _install_ansible_ara_version: '{{ install_ansible_ara_version }}'
-  when: install_ansible_ara_version not in ('', 'latest')
-
-- name: Set ARA package state for installation
-  set_fact:
-    _install_ansible_ara_state: latest
-  when: install_ansible_ara_version == 'latest'
-
-- name: Install ARA
-  pip:
-    name: '{{ install_ansible_ara_name | default("ara[server]") }}'
-    version: '{{ _install_ansible_ara_version | default(omit) }}'
-    state: '{{ _install_ansible_ara_state | default(omit) }}'
-    virtualenv: '/usr/ansible-venv'
-
-# For configuring the callback plugins location in ansible.cfg
-- name: Get ARA's location for callback plugins
-  command: /usr/ansible-venv/bin/python3 -m ara.setup.callback_plugins
-  register: install_ansible_ara_callback_plugins
-  changed_when: false

playbooks/roles/install-ansible/tasks/main.yaml

@@ -1,65 +1,25 @@
-- name: Install python-venv package
+# The -devel job in particular already defines
-  package:
+# install_ansbile_requirements in the job definition to pick
-    name:
+# main/devel branch repos checked out from Zuul
-      - python3-venv
+- name: Set default ansible install requirements
-    state: present
+  when: install_ansible_requirements is not defined
+  block:
+  - name: Set defaults
+    set_fact:
+      _install_ansible_requirements:
+        - 'ansible<8'
+        - 'openstacksdk'
 
-- name: Create venv
+  - name: Add ARA to defaults if enabled
-  include_role:
+    when: install_ansible_ara_enable
-    name: create-venv
+    set_fact:
-  vars:
+      _install_ansible_requirements: '{{ _install_ansible_requirements + ["ara[server]"] }}'
-    create_venv_path: '/usr/ansible-venv'
 
-# If install_ansible_version is not defined (note; not *empty*) it
+  - name: Set variable
-# should be "latest"
+    # NOTE(ianw) the block when: statement is calcuated for each task
-- name: Set ansible default version to latest
+    # -- keep this last!
-  set_fact:
+    set_fact:
-    install_ansible_version: latest
+      install_ansible_requirements: '{{ _install_ansible_requirements }}'
-  when: install_ansible_version is not defined
-
-# If a version is not explicitly set we want to make sure to
-# completely omit the version argument to pip:, as it will be coming
-# from the long-form install_ansible_name variable.  Additionally, if
-# the version is the special value "latest", then we also want to omit
-# any version number, but also set the package state to "latest".
-- name: Set ansible version for installation
-  set_fact:
-    _install_ansible_version: '{{ install_ansible_version }}'
-  when: install_ansible_version not in ('', 'latest')
-
-- name: Set ansible package state for installation
-  set_fact:
-    _install_ansible_state: latest
-  when: install_ansible_version == 'latest'
-
-# From Ansible 2.10 >= most of the fun stuff is in collections.  Clone
-# our required collections here.  Note this is only for our testing of
-# the devel branch; if we're using a release we use the Ansible
-# distribution package which bundles all this.
-#
-# TODO(ianw): we should add these to zuul and link the speculative
-# copies into ansible, then we could test changes in the collections!
-- name: Check if running devel branch
-  set_fact:
-    _install_ansible_from_devel: true
-  when: '"github.com/ansible/ansible" in install_ansible_name'
-
-- name: Install Ansible collections
-  include_tasks: install_ansible_collection.yaml
-  when: _install_ansible_from_devel is defined
-  loop:
-    - namespace: ansible
-      name: netcommon
-      repo: ansible-collections/ansible.netcommon
-    - namespace: ansible
-      name: posix
-      repo: ansible-collections/ansible.posix
-    - namespace: community
-      name: general
-      repo: ansible-collections/community.general
-    - namespace: community
-      name: crypto
-      repo: ansible-collections/community.crypto
 
 # NOTE(ianw) 2022-10-26 : ARM64 generally needs this because upstream
 # projects don't always ship arm64 wheels.  But x86 may need it when
@@ -73,14 +33,45 @@
       - build-essential
       - python3-dev
 
-- name: Install ansible
+- name: Install python-venv package
-  pip:
+  package:
-    name: '{{ install_ansible_name | default("ansible") }}'
+    name:
-    version: '{{ _install_ansible_version | default(omit) }}'
+      - python3-venv
-    state: '{{ _install_ansible_state | default(omit) }}'
+    state: present
-    virtualenv: '/usr/ansible-venv'
 
-- name: Symlink to local
+- name: Create venv
+  include_role:
+    name: create-venv
+  vars:
+    create_venv_path: '/usr/ansible-venv'
+
+- name: Write out requirements file
+  template:
+    src: requirements.txt.j2
+    dest: '/usr/ansible-venv/requirements.txt'
+    owner: root
+    group: root
+    mode: 0644
+  register: _requirements_updated
+
+- name: Install packages
+  when: _requirements_updated.changed
+  pip:
+    requirements: '/usr/ansible-venv/requirements.txt'
+    virtualenv: '/usr/ansible-venv'
+    # If the requirements.txt has changed, force things to upgrade
+    extra_args: '--upgrade'
+
+# From Ansible 2.10 >= most of the fun stuff is in collections.  Clone
+# our required collections here.  Note this is only for our testing of
+# the devel branch; if we're using a release we use the Ansible
+# distribution package which bundles all this.
+- name: Install Ansible collections
+  include_tasks: install_ansible_collection.yaml
+  when: install_ansible_collections is defined
+  loop: '{{ install_ansible_collections }}'
+
+- name: Symlink Ansible globally
   file:
     src: '{{ item.src }}'
     dest: '{{ item.dest }}'
@@ -105,9 +96,13 @@
   debug:
     msg: '{{ _ansible_version_check.stdout }}'
 
-- name: Set up the ARA callback
+# This registered variable is templated into ansible.cfg below
-  include_tasks: install_ara.yaml
+# to setup the callback plugins for ARA
+- name: Get ARA's location for callback plugins
   when: install_ansible_ara_enable
+  command: /usr/ansible-venv/bin/python3 -m ara.setup.callback_plugins
+  register: install_ansible_ara_callback_plugins
+  changed_when: false
 
 # For use by k8s_raw ansible module
 # - name: Install openshift client
@@ -115,30 +110,6 @@
 #     name: 'openshift'
 # TODO(corvus): re-add this once kubernetes 9.0.0 is released
 
-# Same version/state default swizzling as described above for
-# openstacksdk
-- name: Set openstacksdk default version to latest
-  set_fact:
-    install_ansible_openstacksdk_version: latest
-  when: install_ansible_openstacksdk_version is not defined
-
-- name: Set openstacksdk version for installation
-  set_fact:
-    _install_ansible_openstacksdk_version: '{{ install_ansible_openstacksdk_version }}'
-  when: install_ansible_openstacksdk_version not in ('', 'latest')
-
-- name: Set openstacksdk package state for installation
-  set_fact:
-    _install_openstacksdk_state: latest
-  when: install_ansible_openstacksdk_version == 'latest'
-
-- name: Install openstacksdk
-  pip:
-    name: '{{ install_ansible_openstacksdk_name | default("openstacksdk") }}'
-    version: '{{ _install_ansible_openstacksdk_version | default(omit) }}'
-    state: '{{ _install_openstacksdk_state | default(omit) }}'
-    virtualenv: '/usr/ansible-venv'
-
 - name: Ensure /etc/ansible and /etc/ansible/hosts
   file:
     state: directory

playbooks/roles/install-ansible/templates/requirements.txt.j2

@@ -0,0 +1,3 @@
+{% for r in install_ansible_requirements %}
+{{ r }}
+{% endfor %}

zuul.d/system-config-run.yaml

@@ -10,7 +10,6 @@
     run: playbooks/zuul/run-base.yaml
     post-run: playbooks/zuul/run-base-post.yaml
     vars:
-      install_ansible_ara_enable: true
       zuul_copy_output: "{{ copy_output | combine(host_copy_output | default({})) }}"
       stage_dir: "{{ ansible_user_dir }}/zuul-output"
       copy_output:
@@ -24,6 +23,7 @@
         '/etc/iptables/rules.v6': logs_txt
     host-vars:
       bridge99.opendev.org:
+        install_ansible_ara_enable: true
         host_copy_output:
           '{{ zuul.project.src_dir }}/junit.xml': logs
           '{{ zuul.project.src_dir }}/test-results.html': logs
@@ -121,12 +121,27 @@
         override-checkout: main
       - name: github.com/ansible-community/ara
     vars:
-      bridge_ansible_name: '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
+      install_ansible_requirements:
-      bridge_ansible_version: null
+        # Zuul checkout of Ansible devel
-      bridge_openstacksdk_name: '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
+        - '{{ ansible_user_dir}}/src/github.com/ansible/ansible'
-      bridge_openstacksdk_version: null
+        - '{{ ansible_user_dir }}/src/opendev.org/openstack/openstacksdk'
-      bridge_ara_name: '{{ ansible_user_dir}}/src/github.com/ansible-community/ara[server]'
+        - '{{ ansible_user_dir}}/src/github.com/ansible-community/ara[server]'
-      bridge_ara_version: null
+      # These are required because we are not install the pypi
+      # "ansible" bundle here, but the upstream devel branch
+      install_ansible_collections:
+        - namespace: ansible
+          name: netcommon
+          repo: ansible-collections/ansible.netcommon
+        - namespace: ansible
+          name: posix
+          repo: ansible-collections/ansible.posix
+        - namespace: community
+          name: general
+          repo: ansible-collections/community.general
+        - namespace: community
+          name: crypto
+          repo: ansible-collections/community.crypto
+
 
 # Although we don't have an arm64 based bridge; Zuul can't currently
 # allocate a mixed x86/arm64 situation across clouds.  Thus it helps