Build wheel cache for venvs

Building a bunch of virtual envs later is going to be tedious if we do not
pre-cache certain annoying-to-build packages.

* tools/build_wheels.sh: pre-build some wheels for annoying package installs
* list distro package dependencies in files/*/venv
* list packages to pre-build as wheels in files/venv-requirements.txt
* install database Python modules when setting up the database

Change-Id: Idff1ea69a5ca12ba56098e664dbf6924fe6a2e47
This commit is contained in:
Dean Troyer 2015-02-16 13:58:35 -06:00
parent 4a9b154f5e
commit b1d8e8e274
13 changed files with 138 additions and 3 deletions

View File

@ -119,6 +119,10 @@ if [[ -n "$SCREEN_LOGDIR" ]] && [[ -d "$SCREEN_LOGDIR" ]]; then
sudo rm -rf $SCREEN_LOGDIR
fi
# Clean up venvs
DIRS_TO_CLEAN="$WHEELHOUSE"
rm -rf $DIRS_TO_CLEAN
# Clean up files
FILES_TO_CLEAN=".localrc.auto docs/files docs/html shocco/ stack-screenrc test*.conf* test.ini*"

7
files/debs/devlibs Normal file
View File

@ -0,0 +1,7 @@
libffi-dev # pyOpenSSL
libmysqlclient-dev # MySQL-python
libpq-dev # psycopg2
libssl-dev # pyOpenSSL
libxml2-dev # lxml
libxslt1-dev # lxml
python-dev # pyOpenSSL

View File

@ -1 +0,0 @@
python-psycopg2

6
files/rpms-suse/devlibs Normal file
View File

@ -0,0 +1,6 @@
libffi-devel # pyOpenSSL
libopenssl-devel # pyOpenSSL
libxml2-devel # lxml
libxslt1-dev # lxml
postgresql-devel # psycopg2
python-devel # pyOpenSSL

View File

@ -1 +0,0 @@
python-psycopg2

8
files/rpms/devlibs Normal file
View File

@ -0,0 +1,8 @@
libffi-devel # pyOpenSSL
libxml2-devel # lxml
libxslt1-devel # lxml
mariadb-devel # MySQL-python f20,f21,rhel7
mysql-devel # MySQL-python rhel6
openssl-devel # pyOpenSSL
postgresql-devel # psycopg2
python-devel # pyOpenSSL

View File

@ -1 +0,0 @@
python-psycopg2

View File

@ -0,0 +1,10 @@
lxml
MySQL-python
netifaces
numpy
posix-ipc
psycopg2
pycrypto
pyOpenSSL
PyYAML
xattr

View File

@ -151,6 +151,9 @@ EOF
else
exit_distro_not_supported "mysql installation"
fi
# Install Python client module
pip_install MySQL-python
}
function database_connection_url_mysql {

View File

@ -100,6 +100,9 @@ EOF
else
exit_distro_not_supported "postgresql installation"
fi
# Install Python client module
pip_install MySQL-python psycopg2
}
function database_connection_url_postgresql {

View File

@ -671,6 +671,21 @@ fi
source $TOP_DIR/tools/fixup_stuff.sh
# Virtual Environment
# -------------------
# Temporary hack for testing
# This belongs in d-g functions.sh setup_host() or devstack-vm-gate.sh
if [[ -d /var/cache/pip ]]; then
sudo chown -R $STACK_USER:$STACK_USER /var/cache/pip
fi
# Pre-build some problematic wheels
if [[ ! -d ${WHEELHOUSE:-} ]]; then
source tools/build_wheels.sh
fi
# Extras Pre-install
# ------------------

View File

@ -112,6 +112,11 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then
source $RC_DIR/.localrc.auto
fi
# Configure wheel cache location
export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse}
export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE}
export PIP_FIND_LINKS=${PIP_FIND_LINKS:-file://$WHEELHOUSE}
# This can be used to turn database query logging on and off
# (currently only implemented for MySQL backend)
DATABASE_QUERY_LOGGING=$(trueorfalse True DATABASE_QUERY_LOGGING)

77
tools/build_wheels.sh Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env bash
#
# **tools/build_wheels.sh** - Build a cache of Python wheels
#
# build_wheels.sh [package [...]]
#
# System package prerequisites listed in files/*/devlibs will be installed
#
# Builds wheels for all virtual env requirements listed in
# ``venv-requirements.txt`` plus any supplied on the command line.
#
# Assumes ``tools/install_pip.sh`` has been run and a suitable pip/setuptools is available.
# If TOP_DIR is set we're being sourced rather than running stand-alone
# or in a sub-shell
if [[ -z "$TOP_DIR" ]]; then
set -o errexit
set -o nounset
# Keep track of the devstack directory
TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
FILES=$TOP_DIR/files
# Import common functions
source $TOP_DIR/functions
GetDistro
source $TOP_DIR/stackrc
trap err_trap ERR
fi
# Get additional packages to build
MORE_PACKAGES="$@"
# Set a fall-back default, assume that since this script is being called
# package builds are to happen even if WHEELHOUSE is not configured
export WHEELHOUSE=${WHEELHOUSE:-.wheelhouse}
# Exit on any errors so that errors don't compound
function err_trap {
local r=$?
set +o xtrace
rm -rf $TMP_VENV_PATH
exit $r
}
# Get system prereqs
install_package $(get_packages devlibs)
# Get a modern ``virtualenv``
pip_install virtualenv
# Prepare the workspace
TMP_VENV_PATH=$(mktemp -d tmp-venv-XXXX)
virtualenv $TMP_VENV_PATH
# Install modern pip and wheel
$TMP_VENV_PATH/bin/pip install -U pip wheel
# VENV_PACKAGES is a list of packages we want to pre-install
VENV_PACKAGE_FILE=$FILES/venv-requirements.txt
if [[ -r $VENV_PACKAGE_FILE ]]; then
VENV_PACKAGES=$(grep -v '^#' $VENV_PACKAGE_FILE)
fi
for pkg in ${VENV_PACKAGES,/ } ${MORE_PACKAGES}; do
$TMP_VENV_PATH/bin/pip wheel $pkg
done
# Clean up wheel workspace
rm -rf $TMP_VENV_PATH