Use python3 as default python command

After Python 2 is getting unsupported, new distros
like CentOS 8 and RHEL8 have stopped providing 'python'
package forcing user to decide which alternative to
use by installing 'python2' or 'python3.x' package
and then setting python alternative.

This change is intended to make using python3 command as
much as possible and use it as default 'python' alternative
where needed.

The final goals motivating this change are:
 - stop using python2 as much as possible
 - help adding support for CentOS 8 and RHEL8

Change-Id: I1e90db987c0bfa6206c211e066be03ea8738ad3f

inc/python

@@ -450,6 +450,31 @@ function python3_enabled {
     fi
 }
 
+# Provide requested python version and sets PYTHON variable
+function install_python {
+    # NOTE: install_python function should finally just do what install_python3
+    # does as soon Python 2 support has been dropped
+    if python3_enabled; then
+        install_python3
+        export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null ||
+                        which python3 2>/dev/null)
+        if [[ "${DISTRO}" =~ (rhel8) ]]; then
+            # Use Python 3 as default python command so that we have only one
+            # python alternative to use on the system for either python and
+            # python3
+            sudo alternatives --set python "${PYTHON}"
+        else
+            # Install anyway Python 2 for legacy scripts that still requires
+            # python instead of python3 command
+            install_package python
+        fi
+    else
+        echo "WARNING - Python 2 support has been deprecated in favor of Python 3"
+        install_package python
+        export PYTHON=$(which python 2>/dev/null)
+    fi
+}
+
 # Install python3 packages
 function install_python3 {
     if is_ubuntu; then

stack.sh

@@ -415,11 +415,8 @@ fi
 
 # Ensure python is installed
 # --------------------------
-install_python3
+install_python
 
-if ! python3_enabled; then
-    is_package_installed python || install_package python
-fi
 
 # Configure Logging
 # -----------------
@@ -497,14 +494,14 @@ if [[ -n "$LOGFILE" ]]; then
             _of_args="$_of_args --no-timestamp"
         fi
         # Set fd 1 and 2 to write the log file
-        exec 1> >( $TOP_DIR/tools/outfilter.py $_of_args -o "${LOGFILE}" ) 2>&1
+        exec 1> >( $PYTHON $TOP_DIR/tools/outfilter.py $_of_args -o "${LOGFILE}" ) 2>&1
         # Set fd 6 to summary log file
-        exec 6> >( $TOP_DIR/tools/outfilter.py -o "${SUMFILE}" )
+        exec 6> >( $PYTHON $TOP_DIR/tools/outfilter.py -o "${SUMFILE}" )
     else
         # Set fd 1 and 2 to primary logfile
-        exec 1> >( $TOP_DIR/tools/outfilter.py -o "${LOGFILE}" ) 2>&1
+        exec 1> >( $PYTHON $TOP_DIR/tools/outfilter.py -o "${LOGFILE}" ) 2>&1
         # Set fd 6 to summary logfile and stdout
-        exec 6> >( $TOP_DIR/tools/outfilter.py -v -o "${SUMFILE}" >&3 )
+        exec 6> >( $PYTHON $TOP_DIR/tools/outfilter.py -v -o "${SUMFILE}" >&3 )
     fi
 
     echo_summary "stack.sh log $LOGFILE"
@@ -521,7 +518,7 @@ else
         exec 1>/dev/null 2>&1
     fi
     # Always send summary fd to original stdout
-    exec 6> >( $TOP_DIR/tools/outfilter.py -v >&3 )
+    exec 6> >( $PYTHON $TOP_DIR/tools/outfilter.py -v >&3 )
 fi
 
 # Basic test for ``$DEST`` path permissions (fatal on error unless skipped)
@@ -557,9 +554,9 @@ function exit_trap {
             generate-subunit $DEVSTACK_START_TIME $SECONDS 'fail' >> ${SUBUNIT_OUTPUT}
         fi
         if [[ -z $LOGDIR ]]; then
-            $TOP_DIR/tools/worlddump.py
+            ${PYTHON} $TOP_DIR/tools/worlddump.py
         else
-            $TOP_DIR/tools/worlddump.py -d $LOGDIR
+            ${PYTHON} $TOP_DIR/tools/worlddump.py -d $LOGDIR
         fi
     else
         # If we error before we've installed os-testr, this will fail.

tests/test_worlddump.sh

@@ -8,7 +8,7 @@ source $TOP/tests/unittest.sh
 
 OUT_DIR=$(mktemp -d)
 
-$TOP/tools/worlddump.py -d $OUT_DIR
+${PYTHON} $TOP/tools/worlddump.py -d $OUT_DIR
 
 if [[ $? -ne 0 ]]; then
     fail "worlddump failed"

tools/generate-devstack-plugins-list.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
 # Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
 #

tools/install_prereqs.sh

@@ -81,12 +81,6 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then
     fi
 fi
 
-if python3_enabled; then
-    install_python3
-    export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null || which python3 2>/dev/null)
-else
-    export PYTHON=$(which python 2>/dev/null)
-fi
 
 # Mark end of run
 # ---------------

tools/outfilter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-#
+#!/usr/bin/env python3
+
 # Copyright 2014 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may

tools/update_clouds_yaml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain

tools/worlddump.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Hewlett-Packard Development Company, L.P.
 #
@@ -23,8 +23,8 @@ import argparse
 import datetime
 from distutils import spawn
 import fnmatch
+import io
 import os
-import os.path
 import shutil
 import subprocess
 import sys
@@ -109,9 +109,10 @@ def _bridge_list():
 # This method gets max version searching 'OpenFlow versions 0x1:0x'.
 # And return a version value converted to an integer type.
 def _get_ofp_version():
-    process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
+    process = subprocess.Popen(['ovs-ofctl', '--version'],
+                               stdout=subprocess.PIPE)
     stdout, _ = process.communicate()
-    find_str = 'OpenFlow versions 0x1:0x'
+    find_str = b'OpenFlow versions 0x1:0x'
     offset = stdout.find(find_str)
     return int(stdout[offset + len(find_str):-1]) - 1
 
@@ -206,7 +207,7 @@ def process_list():
 
 def compute_consoles():
     _header("Compute consoles")
-    for root, dirnames, filenames in os.walk('/opt/stack'):
+    for root, _, filenames in os.walk('/opt/stack'):
         for filename in fnmatch.filter(filenames, 'console.log'):
             fullpath = os.path.join(root, filename)
             _dump_cmd("sudo cat %s" % fullpath)
@@ -234,12 +235,22 @@ def var_core():
         # tools out there that can do that sort of thing though.
         _dump_cmd("ls -ltrah /var/core")
 
+
+def disable_stdio_buffering():
+    # re-open STDOUT as binary, then wrap it in a
+    # TextIOWrapper, and write through everything.
+    binary_stdout = io.open(sys.stdout.fileno(), 'wb', 0)
+    sys.stdout = io.TextIOWrapper(binary_stdout, write_through=True)
+
+
 def main():
     opts = get_options()
     fname = filename(opts.dir, opts.name)
     print("World dumping... see %s for details" % fname)
-    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-    with open(fname, 'w') as f:
+
+    disable_stdio_buffering()
+
+    with io.open(fname, 'w') as f:
         os.dup2(f.fileno(), sys.stdout.fileno())
         disk_space()
         process_list()