From 94129c7d02902e0f000c09c8245be341df1c5965 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 9 Jan 2017 21:24:24 +0000 Subject: [PATCH] allow config to manage python3 use explicitly Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which packages are installed using Python 3. Move the list of whitelisted packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES, except swift which is not enabled by default for now. Add enable_python3_package and disable_python3_package functions to make editing the variables from local.conf easier. Add python3_enabled_for and python3_disabled_for functions to check the settings against packages being installed by pip. Update pip_install to check if python3 is disabled for a service, then see if it is explicitly enabled, and only then fall back to looking at the classifiers in the packaging metadata. Update pip_install messages to give more detail about why the choice between python 2 and 3 is being made for a given package. Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f Signed-off-by: Doug Hellmann --- inc/python | 119 ++++++++++++++++++++++++++++++++++++++++--- stackrc | 12 ++++- tests/test_python.sh | 30 +++++++++++ 3 files changed, 153 insertions(+), 8 deletions(-) create mode 100755 tests/test_python.sh diff --git a/inc/python b/inc/python index 5a9a9ed588..1c581ba22b 100644 --- a/inc/python +++ b/inc/python @@ -97,6 +97,111 @@ function check_python3_support_for_package_remote { echo $classifier } +# python3_enabled_for() checks if the service(s) specified as arguments are +# enabled by the user in ``ENABLED_PYTHON3_PACKAGES``. +# +# Multiple services specified as arguments are ``OR``'ed together; the test +# is a short-circuit boolean, i.e it returns on the first match. +# +# Uses global ``ENABLED_PYTHON3_PACKAGES`` +# python3_enabled_for dir [dir ...] +function python3_enabled_for { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + + local enabled=1 + local dirs=$@ + local dir + for dir in ${dirs}; do + [[ ,${ENABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0 + done + + $xtrace + return $enabled +} + +# python3_disabled_for() checks if the service(s) specified as arguments are +# disabled by the user in ``DISABLED_PYTHON3_PACKAGES``. +# +# Multiple services specified as arguments are ``OR``'ed together; the test +# is a short-circuit boolean, i.e it returns on the first match. +# +# Uses global ``DISABLED_PYTHON3_PACKAGES`` +# python3_disabled_for dir [dir ...] +function python3_disabled_for { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + + local enabled=1 + local dirs=$@ + local dir + for dir in ${dirs}; do + [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0 + done + + $xtrace + return $enabled +} + +# enable_python3_package() adds the repositories passed as argument to the +# ``ENABLED_PYTHON3_PACKAGES`` list, if they are not already present. +# +# For example: +# enable_python3_package nova +# +# Uses global ``ENABLED_PYTHON3_PACKAGES`` +# enable_python3_package dir [dir ...] +function enable_python3_package { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + + local tmpsvcs="${ENABLED_PYTHON3_PACKAGES}" + local python3 + for dir in $@; do + if [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]]; then + warn $LINENO "Attempt to enable_python3_package ${dir} when it has been disabled" + continue + fi + if ! python3_enabled_for $dir; then + tmpsvcs+=",$dir" + fi + done + ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$tmpsvcs") + + $xtrace +} + +# disable_python3_package() prepares the services passed as argument to be +# removed from the ``ENABLED_PYTHON3_PACKAGES`` list, if they are present. +# +# For example: +# disable_python3_package swift +# +# Uses globals ``ENABLED_PYTHON3_PACKAGES`` and ``DISABLED_PYTHON3_PACKAGES`` +# disable_python3_package dir [dir ...] +function disable_python3_package { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + + local disabled_svcs="${DISABLED_PYTHON3_PACKAGES}" + local enabled_svcs=",${ENABLED_PYTHON3_PACKAGES}," + local dir + for dir in $@; do + disabled_svcs+=",$dir" + if python3_enabled_for $dir; then + enabled_svcs=${enabled_svcs//,$dir,/,} + fi + done + DISABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$disabled_svcs") + ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$enabled_svcs") + + $xtrace +} + # Wrapper for ``pip install`` to set cache and proxy environment variables # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``, # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``, @@ -149,16 +254,16 @@ function pip_install { # support for python3 in progress, but don't claim support # in their classifier echo "Check python version for : $package_dir" - if [[ ${package_dir##*/} == "nova" || ${package_dir##*/} == "glance" || \ - ${package_dir##*/} == "cinder" || ${package_dir##*/} == "swift" || \ - ${package_dir##*/} == "uwsgi" ]]; then - echo "Using $PYTHON3_VERSION version to install $package_dir" + if python3_disabled_for ${package_dir##*/}; then + echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES" + elif python3_enabled_for ${package_dir##*/}; then + echo "Explicitly using $PYTHON3_VERSION version to install $package_dir based on ENABLED_PYTHON3_PACKAGES" sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" cmd_pip=$(get_pip_command $PYTHON3_VERSION) elif [[ -d "$package_dir" ]]; then python_versions=$(get_python_versions_for_package $package_dir) if [[ $python_versions =~ $PYTHON3_VERSION ]]; then - echo "Using $PYTHON3_VERSION version to install $package_dir" + echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on classifiers" sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" cmd_pip=$(get_pip_command $PYTHON3_VERSION) else @@ -167,7 +272,7 @@ function pip_install { # a warning. python3_classifier=$(check_python3_support_for_package_local $package_dir) if [[ ! -z "$python3_classifier" ]]; then - echo "Using $PYTHON3_VERSION version to install $package_dir" + echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on local package settings" sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" cmd_pip=$(get_pip_command $PYTHON3_VERSION) fi @@ -177,7 +282,7 @@ function pip_install { package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*') python3_classifier=$(check_python3_support_for_package_remote $package) if [[ ! -z "$python3_classifier" ]]; then - echo "Using $PYTHON3_VERSION version to install $package" + echo "Automatically using $PYTHON3_VERSION version to install $package based on remote package settings" sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" cmd_pip=$(get_pip_command $PYTHON3_VERSION) fi diff --git a/stackrc b/stackrc index 19f5b53372..ae7177214f 100644 --- a/stackrc +++ b/stackrc @@ -102,9 +102,19 @@ if [[ -r $RC_DIR/.localrc.password ]]; then source $RC_DIR/.localrc.password fi -# Control whether Python 3 should be used. +# Control whether Python 3 should be used at all. export USE_PYTHON3=$(trueorfalse False USE_PYTHON3) +# Control whether Python 3 is enabled for specific services by the +# base name of the directory from which they are installed. See +# enable_python3_package to edit this variable and use_python3_for to +# test membership. +export ENABLED_PYTHON3_PACKAGES="nova,glance,cinder,uwsgi" + +# Explicitly list services not to run under Python 3. See +# disable_python3_package to edit this variable. +export DISABLED_PYTHON3_PACKAGES="" + # When Python 3 is supported by an application, adding the specific # version of Python 3 to this variable will install the app using that # version of the interpreter instead of 2.7. diff --git a/tests/test_python.sh b/tests/test_python.sh new file mode 100755 index 0000000000..8652798778 --- /dev/null +++ b/tests/test_python.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Tests for DevStack INI functions + +TOP=$(cd $(dirname "$0")/.. && pwd) + +source $TOP/functions-common +source $TOP/inc/python + +source $TOP/tests/unittest.sh + +echo "Testing Python 3 functions" + +# Initialize variables manipulated by functions under test. +export ENABLED_PYTHON3_PACKAGES="" +export DISABLED_PYTHON3_PACKAGES="" + +assert_false "should not be enabled yet" python3_enabled_for testpackage1 + +enable_python3_package testpackage1 +assert_equal "$ENABLED_PYTHON3_PACKAGES" "testpackage1" "unexpected result" +assert_true "should be enabled" python3_enabled_for testpackage1 + +assert_false "should not be disabled yet" python3_disabled_for testpackage2 + +disable_python3_package testpackage2 +assert_equal "$DISABLED_PYTHON3_PACKAGES" "testpackage2" "unexpected result" +assert_true "should be disabled" python3_disabled_for testpackage2 + +report_results