#!/bin/bash # # **inc/python** - Python-related functions # # Support for pip/setuptools interfaces and virtual environments # # External functions used: # - GetOSVersion # - is_fedora # - is_suse # - safe_chown # Save trace setting INC_PY_TRACE=$(set +o | grep xtrace) set +o xtrace # Global Config Variables # PROJECT_VENV contains the name of the virtual enviromnet for each # project. A null value installs to the system Python directories. declare -A PROJECT_VENV # Python Functions # ================ # Get the path to the pip command. # get_pip_command function get_pip_command { which pip || which pip-python if [ $? -ne 0 ]; then die $LINENO "Unable to find pip; cannot continue" fi } # Get the path to the direcotry where python executables are installed. # get_python_exec_prefix function get_python_exec_prefix { local xtrace=$(set +o | grep xtrace) set +o xtrace if [[ -z "$os_PACKAGE" ]]; then GetOSVersion fi $xtrace if is_fedora || is_suse; then echo "/usr/bin" else echo "/usr/local/bin" fi } # Wrapper for ``pip install`` that only installs versions of libraries # from the global-requirements specification. # # Uses globals ``REQUIREMENTS_DIR`` # # pip_install_gr packagename function pip_install_gr { local name=$1 local clean_name=$(get_from_global_requirements $name) pip_install $clean_name } # Wrapper for ``pip install`` to set cache and proxy environment variables # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``, # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``, # ``USE_CONSTRAINTS`` # pip_install package [package ...] function pip_install { local xtrace=$(set +o | grep xtrace) set +o xtrace local upgrade="" local offline=${OFFLINE:-False} if [[ "$offline" == "True" || -z "$@" ]]; then $xtrace return fi PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE) if [[ "$PIP_UPGRADE" = "True" ]] ; then upgrade="--upgrade" fi if [[ -z "$os_PACKAGE" ]]; then GetOSVersion fi if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then # TRACK_DEPENDS=True installation creates a circular dependency when # we attempt to install virtualenv into a virualenv, so we must global # that installation. source $DEST/.venv/bin/activate local cmd_pip=$DEST/.venv/bin/pip local sudo_pip="env" else if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip local sudo_pip="env" else local cmd_pip=$(get_pip_command) local sudo_pip="sudo -H" fi fi cmd_pip="$cmd_pip install" # Handle a constraints file, if needed. if [[ "$USE_CONSTRAINTS" == "True" ]]; then cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt" fi local pip_version=$(python -c "import pip; \ print(pip.__version__.strip('.')[0])") if (( pip_version<6 )); then die $LINENO "Currently installed pip version ${pip_version} does not" \ "meet minimum requirements (>=6)." fi $xtrace $sudo_pip \ http_proxy="${http_proxy:-}" \ https_proxy="${https_proxy:-}" \ no_proxy="${no_proxy:-}" \ PIP_FIND_LINKS=$PIP_FIND_LINKS \ $cmd_pip $upgrade \ $@ # Also install test requirements local test_req="$@/test-requirements.txt" if [[ -e "$test_req" ]]; then echo "Installing test-requirements for $test_req" $sudo_pip \ http_proxy=${http_proxy:-} \ https_proxy=${https_proxy:-} \ no_proxy=${no_proxy:-} \ PIP_FIND_LINKS=$PIP_FIND_LINKS \ $cmd_pip $upgrade \ -r $test_req fi } # get version of a package from global requirements file # get_from_global_requirements function get_from_global_requirements { local package=$1 local required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1) if [[ $required_pkg == "" ]]; then die $LINENO "Can't find package $package in requirements" fi echo $required_pkg } # should we use this library from their git repo, or should we let it # get pulled in via pip dependencies. function use_library_from_git { local name=$1 local enabled=1 [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0 return $enabled } # setup a library by name. If we are trying to use the library from # git, we'll do a git based install, otherwise we'll punt and the # library should be installed by a requirements pull from another # project. function setup_lib { local name=$1 local dir=${GITDIR[$name]} setup_install $dir } # setup a library by name in editiable mode. If we are trying to use # the library from git, we'll do a git based install, otherwise we'll # punt and the library should be installed by a requirements pull from # another project. # # use this for non namespaced libraries function setup_dev_lib { local name=$1 local dir=${GITDIR[$name]} setup_develop $dir } # this should be used if you want to install globally, all libraries should # use this, especially *oslo* ones function setup_install { local project_dir=$1 setup_package_with_req_sync $project_dir } # this should be used for projects which run services, like all services function setup_develop { local project_dir=$1 setup_package_with_req_sync $project_dir -e } # determine if a project as specified by directory is in # projects.txt. This will not be an exact match because we throw away # the namespacing when we clone, but it should be good enough in all # practical ways. function is_in_projects_txt { local project_dir=$1 local project_name=$(basename $project_dir) grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt } # ``pip install -e`` the package, which processes the dependencies # using pip before running `setup.py develop` # # Updates the dependencies in project_dir from the # openstack/requirements global list before installing anything. # # Uses globals ``TRACK_DEPENDS``, ``REQUIREMENTS_DIR``, ``UNDO_REQUIREMENTS`` # setup_develop directory function setup_package_with_req_sync { local project_dir=$1 local flags=$2 # Don't update repo if local changes exist # Don't use buggy "git diff --quiet" # ``errexit`` requires us to trap the exit code when the repo is changed local update_requirements=$(cd $project_dir && git diff --exit-code >/dev/null || echo "changed") if [[ $update_requirements != "changed" && "$USE_CONSTRAINTS" == "False" ]]; then if is_in_projects_txt $project_dir; then (cd $REQUIREMENTS_DIR; \ ./.venv/bin/python update.py $project_dir) else # soft update projects not found in requirements project.txt echo "$project_dir not a constrained repository, soft enforcing requirements" (cd $REQUIREMENTS_DIR; \ ./.venv/bin/python update.py -s $project_dir) fi fi if [ -n "$REQUIREMENTS_DIR" ]; then # Constrain this package to this project directory from here on out. local name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg) $REQUIREMENTS_DIR/.venv/bin/edit-constraints \ $REQUIREMENTS_DIR/upper-constraints.txt -- $name \ "$flags file://$project_dir#egg=$name" fi setup_package $project_dir $flags # We've just gone and possibly modified the user's source tree in an # automated way, which is considered bad form if it's a development # tree because we've screwed up their next git checkin. So undo it. # # However... there are some circumstances, like running in the gate # where we really really want the overridden version to stick. So provide # a variable that tells us whether or not we should UNDO the requirements # changes (this will be set to False in the OpenStack ci gate) if [ $UNDO_REQUIREMENTS = "True" ]; then if [[ $update_requirements != "changed" ]]; then (cd $project_dir && git reset --hard) fi fi } # ``pip install -e`` the package, which processes the dependencies # using pip before running `setup.py develop` # Uses globals ``STACK_USER`` # setup_develop_no_requirements_update directory function setup_package { local project_dir=$1 local flags=$2 pip_install $flags $project_dir # ensure that further actions can do things like setup.py sdist if [[ "$flags" == "-e" ]]; then safe_chown -R $STACK_USER $1/*.egg-info fi } # Restore xtrace $INC_PY_TRACE # Local variables: # mode: shell-script # End: