Enable optional Python 3 support

Add USE_PYTHON3 and PYTHON3_VERSION variables to allow services to use
python 3 if they indicate support in their python package metadata.

Tested in Heat here -> I837c2fba682ab430d50e9f43913f2fed20325a7a.
Project config change to add a dedicated job to Heat is here -> I0837e62d6ccc66397a5e409f0961edd4be31f467

Change-Id: I079e18b58b214bf8362945c253d6d894ca8b1a6b

inc/python

@@ -28,10 +28,13 @@ declare -A PROJECT_VENV
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
-    which pip || which pip-python
+    local version="$1"
+    # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
+    # under any circumstances.
+    which pip${version} || which pip${version}-python
 
     if [ $? -ne 0 ]; then
-        die $LINENO "Unable to find pip; cannot continue"
+        die $LINENO "Unable to find pip${version}; cannot continue"
     fi
 }
 
@@ -66,6 +69,13 @@ function pip_install_gr {
     pip_install $clean_name
 }
 
+# Determine the python versions supported by a package
+function get_python_versions_for_package {
+    local name=$1
+    cd $name && python setup.py --classifiers \
+        | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
+}
+
 # Wrapper for ``pip install`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
 # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
@@ -104,8 +114,22 @@ function pip_install {
             local sudo_pip="env"
         else
             local cmd_pip
-            cmd_pip=$(get_pip_command)
+            cmd_pip=$(get_pip_command $PYTHON2_VERSION)
             local sudo_pip="sudo -H"
+            if python3_enabled; then
+                # Look at the package classifiers to find the python
+                # versions supported, and if we find the version of
+                # python3 we've been told to use, use that instead of the
+                # default pip
+                local package_dir=${!#}
+                local python_versions
+                if [[ -d "$package_dir" ]]; then
+                    python_versions=$(get_python_versions_for_package $package_dir)
+                    if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
+                        cmd_pip=$(get_pip_command $PYTHON3_VERSION)
+                    fi
+                fi
+            fi
         fi
     fi
 
@@ -113,6 +137,8 @@ function pip_install {
     # Always apply constraints
     cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
 
+    # FIXME(dhellmann): Need to force multiple versions of pip for
+    # packages like setuptools?
     local pip_version
     pip_version=$(python -c "import pip; \
                         print(pip.__version__.strip('.')[0])")
@@ -276,6 +302,21 @@ function setup_package {
     fi
 }
 
+# Report whether python 3 should be used
+function python3_enabled {
+    if [[ $USE_PYTHON3 == "True" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Install python3 packages
+function install_python3 {
+    if is_ubuntu; then
+        apt_get install python3.4 python3.4-dev
+    fi
+}
 
 # Restore xtrace
 $INC_PY_TRACE

lib/stack

@@ -19,6 +19,7 @@
 function stack_install_service {
     local service=$1
     if type install_${service} >/dev/null 2>&1; then
+        # FIXME(dhellmann): Needs to be python3-aware at some point.
         if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then
             rm -rf ${PROJECT_VENV[$service]}
             source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]} ${ADDITIONAL_VENV_PACKAGES//,/ }

stackrc

@@ -118,6 +118,17 @@ if [[ -r $RC_DIR/.localrc.password ]]; then
     source $RC_DIR/.localrc.password
 fi
 
+# Control whether Python 3 should be used.
+export USE_PYTHON3=${USE_PYTHON3:-False}
+
+# When Python 3 is supported by an application, adding the specific
+# version of Python 3 to this variable will install the app using that
+# version of the interpreter instead of 2.7.
+export PYTHON3_VERSION=${PYTHON3_VERSION:-3.4}
+
+# Just to be more explicit on the Python 2 version to use.
+export PYTHON2_VERSION=${PYTHON2_VERSION:-2.7}
+
 # allow local overrides of env variables, including repo config
 if [[ -f $RC_DIR/localrc ]]; then
     # Old-style user-supplied config

tools/install_pip.sh

@@ -8,6 +8,7 @@
 
 # Assumptions:
 # - update pip to $INSTALL_PIP_VERSION
+# - if USE_PYTHON3=True, PYTHON3_VERSION refers to a version already installed
 
 set -o errexit
 set -o xtrace
@@ -31,6 +32,8 @@ GetDistro
 echo "Distro: $DISTRO"
 
 function get_versions {
+    # FIXME(dhellmann): Deal with multiple python versions here? This
+    # is just used for reporting, so maybe not?
     PIP=$(which pip 2>/dev/null || which pip-python 2>/dev/null || true)
     if [[ -n $PIP ]]; then
         PIP_VERSION=$($PIP --version | awk '{ print $2}')
@@ -75,6 +78,9 @@ function install_get_pip {
         touch $LOCAL_PIP.downloaded
     fi
     sudo -H -E python $LOCAL_PIP
+    if python3_enabled; then
+        sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP
+    fi
 }
 
 
@@ -114,6 +120,7 @@ get_versions
 # python in f23 depends on the python-pip package
 if ! { is_fedora && [[ $DISTRO == "f23" ]]; }; then
     uninstall_package python-pip
+    uninstall_package python3-pip
 fi
 
 install_get_pip
@@ -122,6 +129,7 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then
     configure_pypi_alternative_url
 fi
 
+set -x
 pip_install -U setuptools
 
 get_versions

tools/install_prereqs.sh

@@ -81,6 +81,9 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then
     fi
 fi
 
+if python3_enabled; then
+    install_python3
+fi
 
 # Mark end of run
 # ---------------