![Clark Boylan](/assets/img/avatar_default.png)
There are two problems with dbcounter installation on Jammy. The first is straightforward. We have to use `py_modules` instead of `modules` to specify the source file. I don't know how this works on other distros but the docs [0] seem to clearly indicate py_modules does this. The second issue is quite an issue and requires story time. When pip/setuptools insteall editable installs (as is done for many of the openstack projects) it creates an easy-install.pth file that tells the python interpreter to add the source dirs of those repos to the python path. Normally these paths are appended to your sys.path. Pip's isolated build env relies on the assumption that these paths are appeneded to the path when it santizes sys.path to create the isolated environemnt. However, when SETUPTOOLS_SYS_PATH_TECHNIQUE is set to rewrite the paths are not appended and are inserted in the middle. This breaks pip's isolated build env which broke dbcounter installations. We fix this by not setting SETUPTOOLS_SYS_PATH_TECHNIQUE to rewrite. Upstream indicates the reason we set this half a decade ago has since been fixed properly. The reason Jammy and nothing else breaks is that python3.10 is the first python version to use pip's isolated build envs by default. I've locally fiddled with a patch to pip [1] to try and fix this behavior even when rewrite is set. I don't plan to push this upstream but it helps to illustrate where the problem lies. If someone else would like to upstream this feel free. Finally this change makes the jammy platform job voting again and adds it to the gate to ensure we don't regress again. [0] https://docs.python.org/3/distutils/sourcedist.html#specifying-the-files-to-distribute [1] https://paste.opendev.org/show/bqVAuhgMtVtfYupZK5J6/ Change-Id: I237f5663b0f8b060f6df130de04e17e2b1695f8a
492 lines
15 KiB
Bash
492 lines
15 KiB
Bash
#!/bin/bash
|
|
#
|
|
# **inc/python** - Python-related functions
|
|
#
|
|
# Support for pip/setuptools interfaces and virtual environments
|
|
#
|
|
# External functions used:
|
|
# - GetOSVersion
|
|
# - is_fedora
|
|
# - is_suse
|
|
# - safe_chown
|
|
|
|
# Save trace setting
|
|
INC_PY_TRACE=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
|
|
# Global Config Variables
|
|
|
|
# PROJECT_VENV contains the name of the virtual environment for each
|
|
# project. A null value installs to the system Python directories.
|
|
declare -A -g PROJECT_VENV
|
|
|
|
# Utility Functions
|
|
# =================
|
|
|
|
# Joins bash array of extras with commas as expected by other functions
|
|
function join_extras {
|
|
local IFS=","
|
|
echo "$*"
|
|
}
|
|
|
|
# Python Functions
|
|
# ================
|
|
|
|
# Get the path to the pip command.
|
|
# get_pip_command
|
|
function get_pip_command {
|
|
local version="$1"
|
|
if [ -z "$version" ]; then
|
|
die $LINENO "pip python version is not set."
|
|
fi
|
|
|
|
# NOTE(dhellmann): I don't know if we actually get a pip3.4-python
|
|
# under any circumstances.
|
|
which pip${version} || which pip${version}-python
|
|
|
|
if [ $? -ne 0 ]; then
|
|
die $LINENO "Unable to find pip${version}; cannot continue"
|
|
fi
|
|
}
|
|
|
|
# Get the path to the directory where python executables are installed.
|
|
# get_python_exec_prefix
|
|
function get_python_exec_prefix {
|
|
local xtrace
|
|
xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
if [[ -z "$os_PACKAGE" ]]; then
|
|
GetOSVersion
|
|
fi
|
|
$xtrace
|
|
|
|
local PYTHON_PATH=/usr/local/bin
|
|
is_suse && PYTHON_PATH=/usr/bin
|
|
echo $PYTHON_PATH
|
|
}
|
|
|
|
# Wrapper for ``pip install`` that only installs versions of libraries
|
|
# from the global-requirements specification.
|
|
#
|
|
# Uses globals ``REQUIREMENTS_DIR``
|
|
#
|
|
# pip_install_gr packagename
|
|
function pip_install_gr {
|
|
local name=$1
|
|
local clean_name
|
|
clean_name=$(get_from_global_requirements $name)
|
|
pip_install $clean_name
|
|
}
|
|
|
|
# Wrapper for ``pip install`` that only installs versions of libraries
|
|
# from the global-requirements specification with extras.
|
|
#
|
|
# Uses globals ``REQUIREMENTS_DIR``
|
|
#
|
|
# pip_install_gr_extras packagename extra1,extra2,...
|
|
function pip_install_gr_extras {
|
|
local name=$1
|
|
local extras=$2
|
|
local version_constraints
|
|
version_constraints=$(get_version_constraints_from_global_requirements $name)
|
|
pip_install $name[$extras]$version_constraints
|
|
}
|
|
|
|
# enable_python3_package() -- no-op for backwards compatibility
|
|
#
|
|
# enable_python3_package dir [dir ...]
|
|
function enable_python3_package {
|
|
local xtrace
|
|
xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
echo "It is no longer necessary to call enable_python3_package()."
|
|
|
|
$xtrace
|
|
}
|
|
|
|
# disable_python3_package() -- no-op for backwards compatibility
|
|
#
|
|
# disable_python3_package dir [dir ...]
|
|
function disable_python3_package {
|
|
local xtrace
|
|
xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
echo "It is no longer possible to call disable_python3_package()."
|
|
|
|
$xtrace
|
|
}
|
|
|
|
# Wrapper for ``pip install`` to set cache and proxy environment variables
|
|
# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
|
|
# ``PIP_UPGRADE``, ``*_proxy``,
|
|
# Usage:
|
|
# pip_install pip_arguments
|
|
function pip_install {
|
|
local xtrace result
|
|
xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
local upgrade=""
|
|
local offline=${OFFLINE:-False}
|
|
if [[ "$offline" == "True" || -z "$@" ]]; then
|
|
$xtrace
|
|
return
|
|
fi
|
|
|
|
time_start "pip_install"
|
|
|
|
PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
|
|
if [[ "$PIP_UPGRADE" = "True" ]] ; then
|
|
upgrade="--upgrade"
|
|
fi
|
|
|
|
if [[ -z "$os_PACKAGE" ]]; then
|
|
GetOSVersion
|
|
fi
|
|
|
|
# Try to extract the path of the package we are installing into
|
|
# package_dir. We need this to check for test-requirements.txt,
|
|
# at least.
|
|
#
|
|
# ${!#} expands to the last positional argument to this function.
|
|
# With "extras" syntax included, our arguments might be something
|
|
# like:
|
|
# -e /path/to/fooproject[extra]
|
|
# Thus this magic line grabs just the path without extras
|
|
#
|
|
# Note that this makes no sense if this is a pypi (rather than
|
|
# local path) install; ergo you must check this path exists before
|
|
# use. Also, if we had multiple or mixed installs, we would also
|
|
# likely break. But for historical reasons, it's basically only
|
|
# the other wrapper functions in here calling this to install
|
|
# local packages, and they do so with single call per install. So
|
|
# this works (for now...)
|
|
local package_dir=${!#%\[*\]}
|
|
|
|
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
|
|
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
|
|
local sudo_pip="env"
|
|
else
|
|
local cmd_pip="python$PYTHON3_VERSION -m pip"
|
|
# See
|
|
# https://github.com/pypa/setuptools/issues/2232
|
|
# http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
|
|
# this makes setuptools >=50 use the platform distutils.
|
|
# We only want to do this on global pip installs, not if
|
|
# installing in a virtualenv
|
|
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
|
|
echo "Using python $PYTHON3_VERSION to install $package_dir"
|
|
fi
|
|
|
|
cmd_pip="$cmd_pip install"
|
|
# Always apply constraints
|
|
cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
|
|
|
|
$xtrace
|
|
|
|
# adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
|
|
# the same behaviour of setuptools before version 25.0.0.
|
|
# related issue: https://github.com/pypa/pip/issues/3874
|
|
$sudo_pip \
|
|
http_proxy="${http_proxy:-}" \
|
|
https_proxy="${https_proxy:-}" \
|
|
no_proxy="${no_proxy:-}" \
|
|
PIP_FIND_LINKS=$PIP_FIND_LINKS \
|
|
$cmd_pip $upgrade \
|
|
$@
|
|
result=$?
|
|
|
|
time_stop "pip_install"
|
|
return $result
|
|
}
|
|
|
|
function pip_uninstall {
|
|
# Skip uninstall if offline
|
|
[[ "${OFFLINE}" = "True" ]] && return
|
|
|
|
local name=$1
|
|
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
|
|
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
|
|
local sudo_pip="env"
|
|
else
|
|
local cmd_pip="python$PYTHON3_VERSION -m pip"
|
|
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
|
|
fi
|
|
# don't error if we can't uninstall, it might not be there
|
|
$sudo_pip $cmd_pip uninstall -y $name || /bin/true
|
|
}
|
|
|
|
# get version of a package from global requirements file
|
|
# get_from_global_requirements <package>
|
|
function get_from_global_requirements {
|
|
local package=$1
|
|
local required_pkg
|
|
required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
|
|
if [[ $required_pkg == "" ]]; then
|
|
die $LINENO "Can't find package $package in requirements"
|
|
fi
|
|
echo $required_pkg
|
|
}
|
|
|
|
# get only version constraints of a package from global requirements file
|
|
# get_version_constraints_from_global_requirements <package>
|
|
function get_version_constraints_from_global_requirements {
|
|
local package=$1
|
|
local required_pkg_version_constraint
|
|
# drop the package name from output (\K)
|
|
required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
|
|
if [[ $required_pkg_version_constraint == "" ]]; then
|
|
die $LINENO "Can't find package $package in requirements"
|
|
fi
|
|
echo $required_pkg_version_constraint
|
|
}
|
|
|
|
# should we use this library from their git repo, or should we let it
|
|
# get pulled in via pip dependencies.
|
|
function use_library_from_git {
|
|
local name=$1
|
|
local enabled=1
|
|
[[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
|
|
return $enabled
|
|
}
|
|
|
|
# determine if a package was installed from git
|
|
function lib_installed_from_git {
|
|
local name=$1
|
|
local safe_name
|
|
safe_name=$(python -c "from pkg_resources import safe_name; \
|
|
print(safe_name('${name}'))")
|
|
# Note "pip freeze" doesn't always work here, because it tries to
|
|
# be smart about finding the remote of the git repo the package
|
|
# was installed from. This doesn't work with zuul which clones
|
|
# repos with no remote.
|
|
#
|
|
# The best option seems to be to use "pip list" which will tell
|
|
# you the path an editable install was installed from; for example
|
|
# in response to something like
|
|
# pip install -e 'git+https://opendev.org/openstack/bashate#egg=bashate'
|
|
# pip list --format columns shows
|
|
# bashate 0.5.2.dev19 /tmp/env/src/bashate
|
|
# Thus we check the third column to see if we're installed from
|
|
# some local place.
|
|
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
|
|
}
|
|
|
|
# setup a library by name. If we are trying to use the library from
|
|
# git, we'll do a git based install, otherwise we'll punt and the
|
|
# library should be installed by a requirements pull from another
|
|
# project.
|
|
function setup_lib {
|
|
local name=$1
|
|
local dir=${GITDIR[$name]}
|
|
setup_install $dir
|
|
}
|
|
|
|
# setup a library by name in editable mode. If we are trying to use
|
|
# the library from git, we'll do a git based install, otherwise we'll
|
|
# punt and the library should be installed by a requirements pull from
|
|
# another project.
|
|
#
|
|
# use this for non namespaced libraries
|
|
#
|
|
# setup_dev_lib [-bindep] <name> [<extras>]
|
|
function setup_dev_lib {
|
|
local bindep
|
|
if [[ $1 == -bindep* ]]; then
|
|
bindep="${1}"
|
|
shift
|
|
fi
|
|
local name=$1
|
|
local dir=${GITDIR[$name]}
|
|
local extras=$2
|
|
setup_develop $bindep $dir $extras
|
|
}
|
|
|
|
# this should be used if you want to install globally, all libraries should
|
|
# use this, especially *oslo* ones
|
|
#
|
|
# setup_install project_dir [extras]
|
|
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
|
|
# extras: comma-separated list of optional dependencies to install
|
|
# (e.g., ldap,memcache).
|
|
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
|
|
# bindep: Set "-bindep" as first argument to install bindep.txt packages
|
|
# The command is like "pip install <project_dir>[<extras>]"
|
|
function setup_install {
|
|
local bindep
|
|
if [[ $1 == -bindep* ]]; then
|
|
bindep="${1}"
|
|
shift
|
|
fi
|
|
local project_dir=$1
|
|
local extras=$2
|
|
_setup_package_with_constraints_edit $bindep $project_dir "" $extras
|
|
}
|
|
|
|
# this should be used for projects which run services, like all services
|
|
#
|
|
# setup_develop project_dir [extras]
|
|
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
|
|
# extras: comma-separated list of optional dependencies to install
|
|
# (e.g., ldap,memcache).
|
|
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
|
|
# The command is like "pip install -e <project_dir>[<extras>]"
|
|
function setup_develop {
|
|
local bindep
|
|
if [[ $1 == -bindep* ]]; then
|
|
bindep="${1}"
|
|
shift
|
|
fi
|
|
local project_dir=$1
|
|
local extras=$2
|
|
_setup_package_with_constraints_edit $bindep $project_dir -e $extras
|
|
}
|
|
|
|
# ``pip install -e`` the package, which processes the dependencies
|
|
# using pip before running `setup.py develop`
|
|
#
|
|
# Updates the constraints from REQUIREMENTS_DIR to reflect the
|
|
# future installed state of this package. This ensures when we
|
|
# install this package we get the from source version.
|
|
#
|
|
# Uses globals ``REQUIREMENTS_DIR``
|
|
# _setup_package_with_constraints_edit project_dir flags [extras]
|
|
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
|
|
# flags: pip CLI options/flags
|
|
# extras: comma-separated list of optional dependencies to install
|
|
# (e.g., ldap,memcache).
|
|
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
|
|
# The command is like "pip install <flags> <project_dir>[<extras>]"
|
|
function _setup_package_with_constraints_edit {
|
|
local bindep
|
|
if [[ $1 == -bindep* ]]; then
|
|
bindep="${1}"
|
|
shift
|
|
fi
|
|
local project_dir=$1
|
|
local flags=$2
|
|
local extras=$3
|
|
|
|
# Normalize the directory name to avoid
|
|
# "installation from path or url cannot be constrained to a version"
|
|
# error.
|
|
# REVISIT(yamamoto): Remove this when fixed in pip.
|
|
# https://github.com/pypa/pip/pull/3582
|
|
project_dir=$(cd $project_dir && pwd)
|
|
|
|
if [ -n "$REQUIREMENTS_DIR" ]; then
|
|
# Remove this package from constraints before we install it.
|
|
# That way, later installs won't "downgrade" the install from
|
|
# source we are about to do.
|
|
local name
|
|
name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
|
|
$REQUIREMENTS_DIR/.venv/bin/edit-constraints \
|
|
$REQUIREMENTS_DIR/upper-constraints.txt -- $name
|
|
fi
|
|
|
|
setup_package $bindep $project_dir "$flags" $extras
|
|
|
|
# If this project is in LIBS_FROM_GIT, verify it was actually installed
|
|
# correctly. This helps catch errors caused by constraints mismatches.
|
|
if use_library_from_git "$project_dir"; then
|
|
if ! lib_installed_from_git "$project_dir"; then
|
|
die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ``pip install -e`` the package, which processes the dependencies
|
|
# using pip before running `setup.py develop`. The command is like
|
|
# "pip install <flags> <project_dir>[<extras>]"
|
|
#
|
|
# Uses globals ``STACK_USER``
|
|
#
|
|
# Usage:
|
|
# setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
|
|
#
|
|
# -bindep : Use bindep to install dependencies; select extra profiles
|
|
# as comma separated arguments after "="
|
|
# project_dir : directory of project repo (e.g., /opt/stack/keystone)
|
|
# flags : pip CLI options/flags
|
|
# extras : comma-separated list of optional dependencies to install
|
|
# (e.g., ldap,memcache).
|
|
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
|
|
function setup_package {
|
|
local bindep=0
|
|
local bindep_flag=""
|
|
local bindep_profiles=""
|
|
if [[ $1 == -bindep* ]]; then
|
|
bindep=1
|
|
IFS="=" read bindep_flag bindep_profiles <<< ${1}
|
|
shift
|
|
fi
|
|
local project_dir=$1
|
|
local flags=$2
|
|
local extras=$3
|
|
|
|
# if the flags variable exists, and it doesn't look like a flag,
|
|
# assume it's actually the extras list.
|
|
if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
|
|
extras=$flags
|
|
flags=""
|
|
fi
|
|
|
|
if [[ ! -z "$extras" ]]; then
|
|
extras="[$extras]"
|
|
fi
|
|
|
|
# install any bindep packages
|
|
if [[ $bindep == 1 ]]; then
|
|
install_bindep $project_dir/bindep.txt $bindep_profiles
|
|
fi
|
|
|
|
pip_install $flags "$project_dir$extras"
|
|
# ensure that further actions can do things like setup.py sdist
|
|
if [[ "$flags" == "-e" ]]; then
|
|
safe_chown -R $STACK_USER $1/*.egg-info
|
|
fi
|
|
}
|
|
|
|
# Report whether python 3 should be used
|
|
# TODO(frickler): drop this once all legacy uses are removed
|
|
function python3_enabled {
|
|
return 0
|
|
}
|
|
|
|
# Provide requested python version and sets PYTHON variable
|
|
function install_python {
|
|
install_python3
|
|
export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
|
|
}
|
|
|
|
# Install python3 packages
|
|
function install_python3 {
|
|
if is_ubuntu; then
|
|
apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
|
|
elif is_suse; then
|
|
install_package python3-devel python3-dbm
|
|
elif is_fedora; then
|
|
if [ "$os_VENDOR" = "Fedora" ]; then
|
|
install_package python${PYTHON3_VERSION//.}
|
|
else
|
|
install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function install_devstack_tools {
|
|
# intentionally old to ensure devstack-gate has control
|
|
local dstools_version=${DSTOOLS_VERSION:-0.1.2}
|
|
install_python3
|
|
sudo pip3 install -U devstack-tools==${dstools_version}
|
|
}
|
|
|
|
# Restore xtrace
|
|
$INC_PY_TRACE
|
|
|
|
# Local variables:
|
|
# mode: shell-script
|
|
# End:
|