Make Kayobe code compatible with Python 3

Co-Authored-By: Mark Goddard <mark@stackhpc.com>

Change-Id: I2a7a82d7f576739c5516a0072f953712ffa5c233
Story: 2004959
Task: 29392

ansible/disable-selinux.yml

@@ -6,3 +6,4 @@
   roles:
     - role: disable-selinux
       disable_selinux_reboot_timeout: "{{ 600 if ansible_virtualization_role == 'host' else 300 }}"
+      when: ansible_os_family == 'RedHat'

ansible/group_vars/all/overcloud

@@ -8,7 +8,7 @@ overcloud_group_default: controllers
 
 # List of names of Ansible groups for overcloud hosts.
 overcloud_groups: >
-  {{ (overcloud_group_hosts_map.keys() +
+  {{ (list(overcloud_group_hosts_map) +
       [overcloud_group_default]) | reject('equalto', 'ignore') | unique | sort | list }}
 
 # Dict mapping overcloud Ansible group names to lists of hosts in the group.

ansible/kayobe-ansible-user.yml

@@ -42,7 +42,7 @@
     ansible_user: "{{ bootstrap_user }}"
     # We can't assume that a virtualenv exists at this point, so use the system
     # python interpreter.
-    ansible_python_interpreter: /usr/bin/python
+    ansible_python_interpreter: /usr/libexec/platform-python
   roles:
     - role: singleplatform-eng.users
       users:
@@ -69,7 +69,7 @@
   vars:
     # We can't assume that a virtualenv exists at this point, so use the system
     # python interpreter.
-    ansible_python_interpreter: /usr/bin/python
+    ansible_python_interpreter: /usr/libexec/platform-python
   tasks:
     - name: Verify that a command can be executed
       command: hostname

ansible/kayobe-target-venv.yml

@@ -20,9 +20,9 @@
         - name: Gather facts
           setup:
 
-        - name: Ensure the python-virtualenv package is installed
+        - name: Ensure the Python virtualenv package is installed
           package:
-            name: python-virtualenv
+            name: python{{ ansible_python.version.major }}-virtualenv
             state: present
           become: True
 
@@ -52,6 +52,9 @@
             name: pip
             virtualenv: "{{ virtualenv }}"
             virtualenv_site_packages: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int == 7
 
         - name: Ensure kayobe virtualenv has the latest version of pip installed
           pip:
@@ -61,6 +64,18 @@
             # Site packages are required for using the yum and selinux python
             # modules, which are not available via PyPI.
             virtualenv_site_packages: True
+            virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
+
+        # NOTE(mgoddard): SELinux python bindings available on PyPI only work
+        # with Python 3 on CentOS 8.
+        - name: Ensure kayobe virtualenv has SELinux bindings installed
+          pip:
+            name: selinux
+            state: latest
+            virtualenv: "{{ virtualenv }}"
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int >= 8
       vars:
         # Use the system python interpreter since the virtualenv might not
         # exist.
@@ -68,9 +83,13 @@
       when: virtualenv is defined
 
     - block:
-        - name: Ensure the python-setuptools package is installed
+        - name: Ensure Python setuptools and pip packages are installed
+          vars:
+            packages:
+              - python{{ ansible_python.version.major }}-setuptools
+              - "{% if ansible_distribution_major_version | int >= 8 %}python3-pip{% endif %}"
           package:
-            name: python-setuptools
+            name: "{{ packages | select }}"
             state: present
           become: True
 
@@ -78,4 +97,7 @@
           easy_install:
             name: pip
           become: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int == 7
       when: virtualenv is not defined

ansible/kolla-target-venv.yml

@@ -19,9 +19,13 @@
     - kolla-target-venv
   tasks:
     - block:
-        - name: Ensure the python-virtualenv package is installed
+        - name: Gather facts
+          setup:
+          when: ansible_python is not defined
+
+        - name: Ensure the Python virtualenv package is installed
           package:
-            name: python-virtualenv
+            name: python{{ ansible_python.version.major }}-virtualenv
             state: present
           become: True
 
@@ -33,6 +37,7 @@
             # Site packages are required for using the yum and selinux python
             # modules, which are not available via PyPI.
             virtualenv_site_packages: True
+            virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
           become: True
 
         - name: Ensure kolla-ansible virtualenv has docker SDK for python installed
@@ -43,6 +48,18 @@
             extra_args: "{% if kolla_upper_constraints_file %}-c {{ kolla_upper_constraints_file }}{% endif %}"
           become: True
 
+        # NOTE(mgoddard): SELinux python bindings available on PyPI only work
+        # with Python 3 on CentOS 8.
+        - name: Ensure kolla-ansible virtualenv has SELinux bindings installed
+          pip:
+            name: selinux
+            state: latest
+            virtualenv: "{{ kolla_ansible_target_venv }}"
+          become: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int >= 8
+
         - name: Ensure kolla-ansible virtualenv has correct ownership
           file:
             path: "{{ kolla_ansible_target_venv }}"

ansible/network.yml

@@ -96,7 +96,8 @@
         # interfaces with an explicit MTU set will be taken account of. If no
         # interface has an explicit MTU set, then the corresponding veth will
         # not either.
-        mtu: "{{ [veth_bridge_mtu_map.get(interface), item | net_mtu] | max }}"
+        mtu_list: "{{ [veth_bridge_mtu_map.get(interface), item | net_mtu] | reject('none') | list }}"
+        mtu: "{{ mtu_list | max if mtu_list | length > 0 else None }}"
 
     - name: Update a fact containing veth interfaces
       set_fact:

ansible/roles/console-allocation/library/console_allocation.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #

ansible/roles/disable-selinux/tasks/main.yml

@@ -1,8 +1,7 @@
 ---
 - name: Ensure required packages are installed
   package:
-    name:
+    name: "{% if ansible_distribution_major_version | int == 7 %}libselinux-python{% else %}python3-libselinux{% endif %}"
-      - libselinux-python
     state: present
   become: True
 

ansible/roles/ip-allocation/library/ip_allocation.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #

ansible/roles/ironic-inspector-rules/library/os_ironic_inspector_rule.py

@@ -1,3 +1,5 @@
+#!/usr/bin/python3
+
 # Copyright (c) 2017 StackHPC Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -12,8 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-#!/usr/bin/python
-
 from ansible.module_utils.basic import *
 from ansible.module_utils.openstack import *
 

ansible/roles/kolla-ansible/library/kolla_passwords.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #
@@ -127,7 +127,9 @@ def kolla_passwords(module):
         # Merge in overrides.
         if module.params['overrides']:
             with tempfile.NamedTemporaryFile(delete=False) as f:
-                yaml.dump(module.params['overrides'], f)
+                # NOTE(mgoddard): Temporary files are opened in binary mode, so
+                # specify an encoding.
+                yaml.dump(module.params['overrides'], f, encoding='utf-8')
                 overrides_path = f.name
             try:
                 kolla_mergepwd(module, overrides_path, temp_file_path, temp_file_path)

ansible/roles/kolla-ansible/tasks/install.yml

@@ -80,6 +80,7 @@
     state: latest
     extra_args: "{% if kolla_upper_constraints_file %}-c {{ kolla_upper_constraints_file }}{% endif %}"
     virtualenv: "{{ kolla_ansible_venv }}"
+    virtualenv_python: "{{ kolla_ansible_venv_python }}"
 
 # This is a workaround for the lack of a python package for libselinux-python
 # on PyPI. Without using --system-site-packages to create the virtualenv, it
@@ -94,6 +95,7 @@
     state: link
   when:
     - ansible_os_family == 'RedHat'
+    - ansible_distribution_major_version | int == 7
     - ansible_selinux != False
     - ansible_selinux.status != 'disabled'
     - kolla_ansible_venv_python_major_version | int == 2

ansible/roles/kolla-ansible/templates/requirements.txt.j2

@@ -8,6 +8,9 @@ kolla-ansible=={{ kolla_openstack_release }}
 # Limit the version of ansible used by kolla-ansible to avoid new releases from
 # breaking tested code. Changes to this limit should be tested.
 ansible>=2.6,<2.9
+{% if ansible_os_family == 'RedHat' and ansible_distribution_major_version | int >= 8 %}
+selinux
+{% endif %}
 {% if kolla_ansible_venv_extra_requirements is defined %}
 {% for item in kolla_ansible_venv_extra_requirements %}
 {{ item }}

ansible/roles/kolla/tasks/install.yml

@@ -14,9 +14,9 @@
       - gcc
       - libffi-devel
       - openssl-devel
-      - python-devel
+      - python{{ ansible_python.version.major }}-devel
-      - python-pip
+      - python{{ ansible_python.version.major }}-pip
-      - python-virtualenv
+      - python{{ ansible_python.version.major }}-virtualenv
     state: present
   become: True
 
@@ -50,6 +50,7 @@
     name: "{{ item.name }}"
     state: latest
     virtualenv: "{{ kolla_venv }}"
+    virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
   with_items:
     - { name: pip }
 

ansible/roles/swift-rings/files/swift-ring-builder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Script to build a Swift ring from a declarative YAML configuration. This has

dev/functions

@@ -85,12 +85,41 @@ function config_init {
 
 # Installation
 
+function is_dnf {
+    if [[ -e /etc/centos-release ]]; then
+        which dnf >/dev/null 2>&1
+    else
+        return 1
+    fi
+}
+
+function is_yum {
+    if [[ -e /etc/centos-release ]]; then
+        which yum >/dev/null 2>&1
+    else
+        return 1
+    fi
+}
+
+function python_version {
+    # Echo python major version.
+    if is_dnf; then
+        echo 3
+    elif is_yum; then
+        echo 2
+    else
+        echo 3
+    fi
+}
+
 function install_dependencies {
     echo "Installing package dependencies for kayobe"
-    if [[ -e /etc/centos-release ]]; then
+    if is_dnf; then
-        sudo yum -y install gcc git vim python-virtualenv libffi-devel
+        sudo dnf -y install gcc git vim python3-pyyaml python3-virtualenv libffi-devel
+    elif is_yum; then
+        sudo yum -y install gcc git vim python2-virtualenv libffi-devel
     else
-        sudo apt install -y python-dev python-virtualenv gcc git libffi-dev
+        sudo apt install -y python-dev python3-virtualenv gcc git libffi-dev
     fi
 }
 
@@ -106,7 +135,7 @@ function install_venv {
     fi
     if [[ ! -f "${venv_path}/bin/activate" ]]; then
         echo "Creating virtual environment in ${venv_path}"
-        virtualenv "${venv_path}"
+        virtualenv -p python$(python_version) "${venv_path}"
         # NOTE: Virtualenv's activate and deactivate scripts reference an
         # unbound variable.
         set +u
@@ -132,7 +161,7 @@ function install_kayobe_dev_venv {
 
 function upgrade_kayobe_venv {
     echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}"
-    virtualenv "${KAYOBE_VENV_PATH}"
+    virtualenv -p python$(python_version) "${KAYOBE_VENV_PATH}"
     # NOTE: Virtualenv's activate and deactivate scripts reference an
     # unbound variable.
     set +u