From 15109ccb547e603c0aca2a73e43f4db523fa1bcd Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Thu, 7 Feb 2019 17:41:30 +0000
Subject: [PATCH] Make Kayobe code compatible with Python 3

Co-Authored-By: Mark Goddard <mark@stackhpc.com>

Change-Id: I2a7a82d7f576739c5516a0072f953712ffa5c233
Story: 2004959
Task: 29392
---
 ansible/disable-selinux.yml                   |  1 +
 ansible/group_vars/all/overcloud              |  2 +-
 ansible/kayobe-ansible-user.yml               |  4 +-
 ansible/kayobe-target-venv.yml                | 30 ++++++++++++--
 ansible/kolla-target-venv.yml                 | 21 +++++++++-
 ansible/network.yml                           |  3 +-
 .../library/console_allocation.py             |  2 +-
 ansible/roles/disable-selinux/tasks/main.yml  |  3 +-
 .../ip-allocation/library/ip_allocation.py    |  2 +-
 .../library/os_ironic_inspector_rule.py       |  4 +-
 .../kolla-ansible/library/kolla_passwords.py  |  6 ++-
 ansible/roles/kolla-ansible/tasks/install.yml |  2 +
 .../templates/requirements.txt.j2             |  3 ++
 ansible/roles/kolla/tasks/install.yml         |  7 ++--
 .../swift-rings/files/swift-ring-builder.py   |  2 +-
 dev/functions                                 | 39 ++++++++++++++++---
 16 files changed, 104 insertions(+), 27 deletions(-)

diff --git a/ansible/disable-selinux.yml b/ansible/disable-selinux.yml
index 0124879f4..760743435 100644
--- a/ansible/disable-selinux.yml
+++ b/ansible/disable-selinux.yml
@@ -6,3 +6,4 @@
   roles:
     - role: disable-selinux
       disable_selinux_reboot_timeout: "{{ 600 if ansible_virtualization_role == 'host' else 300 }}"
+      when: ansible_os_family == 'RedHat'
diff --git a/ansible/group_vars/all/overcloud b/ansible/group_vars/all/overcloud
index 90ed6a2b8..e01e1ed49 100644
--- a/ansible/group_vars/all/overcloud
+++ b/ansible/group_vars/all/overcloud
@@ -8,7 +8,7 @@ overcloud_group_default: controllers
 
 # List of names of Ansible groups for overcloud hosts.
 overcloud_groups: >
-  {{ (overcloud_group_hosts_map.keys() +
+  {{ (list(overcloud_group_hosts_map) +
       [overcloud_group_default]) | reject('equalto', 'ignore') | unique | sort | list }}
 
 # Dict mapping overcloud Ansible group names to lists of hosts in the group.
diff --git a/ansible/kayobe-ansible-user.yml b/ansible/kayobe-ansible-user.yml
index 956d7f750..75ab76306 100644
--- a/ansible/kayobe-ansible-user.yml
+++ b/ansible/kayobe-ansible-user.yml
@@ -42,7 +42,7 @@
     ansible_user: "{{ bootstrap_user }}"
     # We can't assume that a virtualenv exists at this point, so use the system
     # python interpreter.
-    ansible_python_interpreter: /usr/bin/python
+    ansible_python_interpreter: /usr/libexec/platform-python
   roles:
     - role: singleplatform-eng.users
       users:
@@ -69,7 +69,7 @@
   vars:
     # We can't assume that a virtualenv exists at this point, so use the system
     # python interpreter.
-    ansible_python_interpreter: /usr/bin/python
+    ansible_python_interpreter: /usr/libexec/platform-python
   tasks:
     - name: Verify that a command can be executed
       command: hostname
diff --git a/ansible/kayobe-target-venv.yml b/ansible/kayobe-target-venv.yml
index dbbc536e7..b58ed6343 100644
--- a/ansible/kayobe-target-venv.yml
+++ b/ansible/kayobe-target-venv.yml
@@ -20,9 +20,9 @@
         - name: Gather facts
           setup:
 
-        - name: Ensure the python-virtualenv package is installed
+        - name: Ensure the Python virtualenv package is installed
           package:
-            name: python-virtualenv
+            name: python{{ ansible_python.version.major }}-virtualenv
             state: present
           become: True
 
@@ -52,6 +52,9 @@
             name: pip
             virtualenv: "{{ virtualenv }}"
             virtualenv_site_packages: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int == 7
 
         - name: Ensure kayobe virtualenv has the latest version of pip installed
           pip:
@@ -61,6 +64,18 @@
             # Site packages are required for using the yum and selinux python
             # modules, which are not available via PyPI.
             virtualenv_site_packages: True
+            virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
+
+        # NOTE(mgoddard): SELinux python bindings available on PyPI only work
+        # with Python 3 on CentOS 8.
+        - name: Ensure kayobe virtualenv has SELinux bindings installed
+          pip:
+            name: selinux
+            state: latest
+            virtualenv: "{{ virtualenv }}"
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int >= 8
       vars:
         # Use the system python interpreter since the virtualenv might not
         # exist.
@@ -68,9 +83,13 @@
       when: virtualenv is defined
 
     - block:
-        - name: Ensure the python-setuptools package is installed
+        - name: Ensure Python setuptools and pip packages are installed
+          vars:
+            packages:
+              - python{{ ansible_python.version.major }}-setuptools
+              - "{% if ansible_distribution_major_version | int >= 8 %}python3-pip{% endif %}"
           package:
-            name: python-setuptools
+            name: "{{ packages | select }}"
             state: present
           become: True
 
@@ -78,4 +97,7 @@
           easy_install:
             name: pip
           become: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int == 7
       when: virtualenv is not defined
diff --git a/ansible/kolla-target-venv.yml b/ansible/kolla-target-venv.yml
index e4f58753c..e5d36742b 100644
--- a/ansible/kolla-target-venv.yml
+++ b/ansible/kolla-target-venv.yml
@@ -19,9 +19,13 @@
     - kolla-target-venv
   tasks:
     - block:
-        - name: Ensure the python-virtualenv package is installed
+        - name: Gather facts
+          setup:
+          when: ansible_python is not defined
+
+        - name: Ensure the Python virtualenv package is installed
           package:
-            name: python-virtualenv
+            name: python{{ ansible_python.version.major }}-virtualenv
             state: present
           become: True
 
@@ -33,6 +37,7 @@
             # Site packages are required for using the yum and selinux python
             # modules, which are not available via PyPI.
             virtualenv_site_packages: True
+            virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
           become: True
 
         - name: Ensure kolla-ansible virtualenv has docker SDK for python installed
@@ -43,6 +48,18 @@
             extra_args: "{% if kolla_upper_constraints_file %}-c {{ kolla_upper_constraints_file }}{% endif %}"
           become: True
 
+        # NOTE(mgoddard): SELinux python bindings available on PyPI only work
+        # with Python 3 on CentOS 8.
+        - name: Ensure kolla-ansible virtualenv has SELinux bindings installed
+          pip:
+            name: selinux
+            state: latest
+            virtualenv: "{{ kolla_ansible_target_venv }}"
+          become: True
+          when:
+            - ansible_os_family == 'RedHat'
+            - ansible_distribution_major_version | int >= 8
+
         - name: Ensure kolla-ansible virtualenv has correct ownership
           file:
             path: "{{ kolla_ansible_target_venv }}"
diff --git a/ansible/network.yml b/ansible/network.yml
index 2de5070b3..2db2ffe10 100644
--- a/ansible/network.yml
+++ b/ansible/network.yml
@@ -96,7 +96,8 @@
         # interfaces with an explicit MTU set will be taken account of. If no
         # interface has an explicit MTU set, then the corresponding veth will
         # not either.
-        mtu: "{{ [veth_bridge_mtu_map.get(interface), item | net_mtu] | max }}"
+        mtu_list: "{{ [veth_bridge_mtu_map.get(interface), item | net_mtu] | reject('none') | list }}"
+        mtu: "{{ mtu_list | max if mtu_list | length > 0 else None }}"
 
     - name: Update a fact containing veth interfaces
       set_fact:
diff --git a/ansible/roles/console-allocation/library/console_allocation.py b/ansible/roles/console-allocation/library/console_allocation.py
index 2797a85a3..ee652d0ee 100644
--- a/ansible/roles/console-allocation/library/console_allocation.py
+++ b/ansible/roles/console-allocation/library/console_allocation.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #
diff --git a/ansible/roles/disable-selinux/tasks/main.yml b/ansible/roles/disable-selinux/tasks/main.yml
index f9b06a9db..44b1540e9 100644
--- a/ansible/roles/disable-selinux/tasks/main.yml
+++ b/ansible/roles/disable-selinux/tasks/main.yml
@@ -1,8 +1,7 @@
 ---
 - name: Ensure required packages are installed
   package:
-    name:
-      - libselinux-python
+    name: "{% if ansible_distribution_major_version | int == 7 %}libselinux-python{% else %}python3-libselinux{% endif %}"
     state: present
   become: True
 
diff --git a/ansible/roles/ip-allocation/library/ip_allocation.py b/ansible/roles/ip-allocation/library/ip_allocation.py
index 9f30bad06..1a135518d 100644
--- a/ansible/roles/ip-allocation/library/ip_allocation.py
+++ b/ansible/roles/ip-allocation/library/ip_allocation.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #
diff --git a/ansible/roles/ironic-inspector-rules/library/os_ironic_inspector_rule.py b/ansible/roles/ironic-inspector-rules/library/os_ironic_inspector_rule.py
index c97f625f2..cb58ad600 100644
--- a/ansible/roles/ironic-inspector-rules/library/os_ironic_inspector_rule.py
+++ b/ansible/roles/ironic-inspector-rules/library/os_ironic_inspector_rule.py
@@ -1,3 +1,5 @@
+#!/usr/bin/python3
+
 # Copyright (c) 2017 StackHPC Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -12,8 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-#!/usr/bin/python
-
 from ansible.module_utils.basic import *
 from ansible.module_utils.openstack import *
 
diff --git a/ansible/roles/kolla-ansible/library/kolla_passwords.py b/ansible/roles/kolla-ansible/library/kolla_passwords.py
index be461a92b..d7bc6314c 100644
--- a/ansible/roles/kolla-ansible/library/kolla_passwords.py
+++ b/ansible/roles/kolla-ansible/library/kolla_passwords.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Copyright (c) 2017 StackHPC Ltd.
 #
@@ -127,7 +127,9 @@ def kolla_passwords(module):
         # Merge in overrides.
         if module.params['overrides']:
             with tempfile.NamedTemporaryFile(delete=False) as f:
-                yaml.dump(module.params['overrides'], f)
+                # NOTE(mgoddard): Temporary files are opened in binary mode, so
+                # specify an encoding.
+                yaml.dump(module.params['overrides'], f, encoding='utf-8')
                 overrides_path = f.name
             try:
                 kolla_mergepwd(module, overrides_path, temp_file_path, temp_file_path)
diff --git a/ansible/roles/kolla-ansible/tasks/install.yml b/ansible/roles/kolla-ansible/tasks/install.yml
index 0a3724dc2..24def5d02 100644
--- a/ansible/roles/kolla-ansible/tasks/install.yml
+++ b/ansible/roles/kolla-ansible/tasks/install.yml
@@ -80,6 +80,7 @@
     state: latest
     extra_args: "{% if kolla_upper_constraints_file %}-c {{ kolla_upper_constraints_file }}{% endif %}"
     virtualenv: "{{ kolla_ansible_venv }}"
+    virtualenv_python: "{{ kolla_ansible_venv_python }}"
 
 # This is a workaround for the lack of a python package for libselinux-python
 # on PyPI. Without using --system-site-packages to create the virtualenv, it
@@ -94,6 +95,7 @@
     state: link
   when:
     - ansible_os_family == 'RedHat'
+    - ansible_distribution_major_version | int == 7
     - ansible_selinux != False
     - ansible_selinux.status != 'disabled'
     - kolla_ansible_venv_python_major_version | int == 2
diff --git a/ansible/roles/kolla-ansible/templates/requirements.txt.j2 b/ansible/roles/kolla-ansible/templates/requirements.txt.j2
index dd71d6676..ff323513f 100644
--- a/ansible/roles/kolla-ansible/templates/requirements.txt.j2
+++ b/ansible/roles/kolla-ansible/templates/requirements.txt.j2
@@ -8,6 +8,9 @@ kolla-ansible=={{ kolla_openstack_release }}
 # Limit the version of ansible used by kolla-ansible to avoid new releases from
 # breaking tested code. Changes to this limit should be tested.
 ansible>=2.6,<2.9
+{% if ansible_os_family == 'RedHat' and ansible_distribution_major_version | int >= 8 %}
+selinux
+{% endif %}
 {% if kolla_ansible_venv_extra_requirements is defined %}
 {% for item in kolla_ansible_venv_extra_requirements %}
 {{ item }}
diff --git a/ansible/roles/kolla/tasks/install.yml b/ansible/roles/kolla/tasks/install.yml
index 69ac843c8..b27bbb4bd 100644
--- a/ansible/roles/kolla/tasks/install.yml
+++ b/ansible/roles/kolla/tasks/install.yml
@@ -14,9 +14,9 @@
       - gcc
       - libffi-devel
       - openssl-devel
-      - python-devel
-      - python-pip
-      - python-virtualenv
+      - python{{ ansible_python.version.major }}-devel
+      - python{{ ansible_python.version.major }}-pip
+      - python{{ ansible_python.version.major }}-virtualenv
     state: present
   become: True
 
@@ -50,6 +50,7 @@
     name: "{{ item.name }}"
     state: latest
     virtualenv: "{{ kolla_venv }}"
+    virtualenv_python: "python{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
   with_items:
     - { name: pip }
 
diff --git a/ansible/roles/swift-rings/files/swift-ring-builder.py b/ansible/roles/swift-rings/files/swift-ring-builder.py
index bf2ae655a..bf6d8eb1a 100644
--- a/ansible/roles/swift-rings/files/swift-ring-builder.py
+++ b/ansible/roles/swift-rings/files/swift-ring-builder.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Script to build a Swift ring from a declarative YAML configuration. This has
diff --git a/dev/functions b/dev/functions
index 496a694ac..b9b7f154f 100644
--- a/dev/functions
+++ b/dev/functions
@@ -85,12 +85,41 @@ function config_init {
 
 # Installation
 
+function is_dnf {
+    if [[ -e /etc/centos-release ]]; then
+        which dnf >/dev/null 2>&1
+    else
+        return 1
+    fi
+}
+
+function is_yum {
+    if [[ -e /etc/centos-release ]]; then
+        which yum >/dev/null 2>&1
+    else
+        return 1
+    fi
+}
+
+function python_version {
+    # Echo python major version.
+    if is_dnf; then
+        echo 3
+    elif is_yum; then
+        echo 2
+    else
+        echo 3
+    fi
+}
+
 function install_dependencies {
     echo "Installing package dependencies for kayobe"
-    if [[ -e /etc/centos-release ]]; then
-        sudo yum -y install gcc git vim python-virtualenv libffi-devel
+    if is_dnf; then
+        sudo dnf -y install gcc git vim python3-pyyaml python3-virtualenv libffi-devel
+    elif is_yum; then
+        sudo yum -y install gcc git vim python2-virtualenv libffi-devel
     else
-        sudo apt install -y python-dev python-virtualenv gcc git libffi-dev
+        sudo apt install -y python-dev python3-virtualenv gcc git libffi-dev
     fi
 }
 
@@ -106,7 +135,7 @@ function install_venv {
     fi
     if [[ ! -f "${venv_path}/bin/activate" ]]; then
         echo "Creating virtual environment in ${venv_path}"
-        virtualenv "${venv_path}"
+        virtualenv -p python$(python_version) "${venv_path}"
         # NOTE: Virtualenv's activate and deactivate scripts reference an
         # unbound variable.
         set +u
@@ -132,7 +161,7 @@ function install_kayobe_dev_venv {
 
 function upgrade_kayobe_venv {
     echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}"
-    virtualenv "${KAYOBE_VENV_PATH}"
+    virtualenv -p python$(python_version) "${KAYOBE_VENV_PATH}"
     # NOTE: Virtualenv's activate and deactivate scripts reference an
     # unbound variable.
     set +u