From b1d8e8e274f0c220ae19ee3ea1a3b9a533459297 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Mon, 16 Feb 2015 13:58:35 -0600 Subject: [PATCH] Build wheel cache for venvs Building a bunch of virtual envs later is going to be tedious if we do not pre-cache certain annoying-to-build packages. * tools/build_wheels.sh: pre-build some wheels for annoying package installs * list distro package dependencies in files/*/venv * list packages to pre-build as wheels in files/venv-requirements.txt * install database Python modules when setting up the database Change-Id: Idff1ea69a5ca12ba56098e664dbf6924fe6a2e47 --- clean.sh | 4 ++ files/debs/devlibs | 7 ++++ files/debs/postgresql | 1 - files/rpms-suse/devlibs | 6 +++ files/rpms-suse/postgresql | 1 - files/rpms/devlibs | 8 ++++ files/rpms/postgresql | 1 - files/venv-requirements.txt | 10 +++++ lib/databases/mysql | 3 ++ lib/databases/postgresql | 3 ++ stack.sh | 15 ++++++++ stackrc | 5 +++ tools/build_wheels.sh | 77 +++++++++++++++++++++++++++++++++++++ 13 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 files/debs/devlibs delete mode 100644 files/debs/postgresql create mode 100644 files/rpms-suse/devlibs delete mode 100644 files/rpms-suse/postgresql create mode 100644 files/rpms/devlibs delete mode 100644 files/rpms/postgresql create mode 100644 files/venv-requirements.txt create mode 100755 tools/build_wheels.sh diff --git a/clean.sh b/clean.sh index 50d414c1d9..fea1230c7b 100755 --- a/clean.sh +++ b/clean.sh @@ -119,6 +119,10 @@ if [[ -n "$SCREEN_LOGDIR" ]] && [[ -d "$SCREEN_LOGDIR" ]]; then sudo rm -rf $SCREEN_LOGDIR fi +# Clean up venvs +DIRS_TO_CLEAN="$WHEELHOUSE" +rm -rf $DIRS_TO_CLEAN + # Clean up files FILES_TO_CLEAN=".localrc.auto docs/files docs/html shocco/ stack-screenrc test*.conf* test.ini*" diff --git a/files/debs/devlibs b/files/debs/devlibs new file mode 100644 index 0000000000..0446ceb6b6 --- /dev/null +++ b/files/debs/devlibs @@ -0,0 +1,7 @@ +libffi-dev # pyOpenSSL +libmysqlclient-dev # MySQL-python +libpq-dev # psycopg2 +libssl-dev # pyOpenSSL +libxml2-dev # lxml +libxslt1-dev # lxml +python-dev # pyOpenSSL diff --git a/files/debs/postgresql b/files/debs/postgresql deleted file mode 100644 index bf19d397cb..0000000000 --- a/files/debs/postgresql +++ /dev/null @@ -1 +0,0 @@ -python-psycopg2 diff --git a/files/rpms-suse/devlibs b/files/rpms-suse/devlibs new file mode 100644 index 0000000000..11722adf3c --- /dev/null +++ b/files/rpms-suse/devlibs @@ -0,0 +1,6 @@ +libffi-devel # pyOpenSSL +libopenssl-devel # pyOpenSSL +libxml2-devel # lxml +libxslt1-dev # lxml +postgresql-devel # psycopg2 +python-devel # pyOpenSSL diff --git a/files/rpms-suse/postgresql b/files/rpms-suse/postgresql deleted file mode 100644 index bf19d397cb..0000000000 --- a/files/rpms-suse/postgresql +++ /dev/null @@ -1 +0,0 @@ -python-psycopg2 diff --git a/files/rpms/devlibs b/files/rpms/devlibs new file mode 100644 index 0000000000..6010c9fd37 --- /dev/null +++ b/files/rpms/devlibs @@ -0,0 +1,8 @@ +libffi-devel # pyOpenSSL +libxml2-devel # lxml +libxslt1-devel # lxml +mariadb-devel # MySQL-python f20,f21,rhel7 +mysql-devel # MySQL-python rhel6 +openssl-devel # pyOpenSSL +postgresql-devel # psycopg2 +python-devel # pyOpenSSL diff --git a/files/rpms/postgresql b/files/rpms/postgresql deleted file mode 100644 index bf19d397cb..0000000000 --- a/files/rpms/postgresql +++ /dev/null @@ -1 +0,0 @@ -python-psycopg2 diff --git a/files/venv-requirements.txt b/files/venv-requirements.txt new file mode 100644 index 0000000000..3c50061e96 --- /dev/null +++ b/files/venv-requirements.txt @@ -0,0 +1,10 @@ +lxml +MySQL-python +netifaces +numpy +posix-ipc +psycopg2 +pycrypto +pyOpenSSL +PyYAML +xattr diff --git a/lib/databases/mysql b/lib/databases/mysql index c8ceec2455..70073c4c6f 100644 --- a/lib/databases/mysql +++ b/lib/databases/mysql @@ -151,6 +151,9 @@ EOF else exit_distro_not_supported "mysql installation" fi + + # Install Python client module + pip_install MySQL-python } function database_connection_url_mysql { diff --git a/lib/databases/postgresql b/lib/databases/postgresql index 317e0ebf53..dfda9adbaf 100644 --- a/lib/databases/postgresql +++ b/lib/databases/postgresql @@ -100,6 +100,9 @@ EOF else exit_distro_not_supported "postgresql installation" fi + + # Install Python client module + pip_install MySQL-python psycopg2 } function database_connection_url_postgresql { diff --git a/stack.sh b/stack.sh index 753135bc81..49288a7438 100755 --- a/stack.sh +++ b/stack.sh @@ -671,6 +671,21 @@ fi source $TOP_DIR/tools/fixup_stuff.sh +# Virtual Environment +# ------------------- + +# Temporary hack for testing +# This belongs in d-g functions.sh setup_host() or devstack-vm-gate.sh +if [[ -d /var/cache/pip ]]; then + sudo chown -R $STACK_USER:$STACK_USER /var/cache/pip +fi + +# Pre-build some problematic wheels +if [[ ! -d ${WHEELHOUSE:-} ]]; then + source tools/build_wheels.sh +fi + + # Extras Pre-install # ------------------ diff --git a/stackrc b/stackrc index 386c5d53e3..ee601a9b45 100644 --- a/stackrc +++ b/stackrc @@ -112,6 +112,11 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then source $RC_DIR/.localrc.auto fi +# Configure wheel cache location +export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse} +export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE} +export PIP_FIND_LINKS=${PIP_FIND_LINKS:-file://$WHEELHOUSE} + # This can be used to turn database query logging on and off # (currently only implemented for MySQL backend) DATABASE_QUERY_LOGGING=$(trueorfalse True DATABASE_QUERY_LOGGING) diff --git a/tools/build_wheels.sh b/tools/build_wheels.sh new file mode 100755 index 0000000000..c87b6dafe6 --- /dev/null +++ b/tools/build_wheels.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# **tools/build_wheels.sh** - Build a cache of Python wheels +# +# build_wheels.sh [package [...]] +# +# System package prerequisites listed in files/*/devlibs will be installed +# +# Builds wheels for all virtual env requirements listed in +# ``venv-requirements.txt`` plus any supplied on the command line. +# +# Assumes ``tools/install_pip.sh`` has been run and a suitable pip/setuptools is available. + +# If TOP_DIR is set we're being sourced rather than running stand-alone +# or in a sub-shell +if [[ -z "$TOP_DIR" ]]; then + + set -o errexit + set -o nounset + + # Keep track of the devstack directory + TOP_DIR=$(cd $(dirname "$0")/.. && pwd) + FILES=$TOP_DIR/files + + # Import common functions + source $TOP_DIR/functions + + GetDistro + + source $TOP_DIR/stackrc + + trap err_trap ERR + +fi + +# Get additional packages to build +MORE_PACKAGES="$@" + +# Set a fall-back default, assume that since this script is being called +# package builds are to happen even if WHEELHOUSE is not configured +export WHEELHOUSE=${WHEELHOUSE:-.wheelhouse} + +# Exit on any errors so that errors don't compound +function err_trap { + local r=$? + set +o xtrace + + rm -rf $TMP_VENV_PATH + + exit $r +} + +# Get system prereqs +install_package $(get_packages devlibs) + +# Get a modern ``virtualenv`` +pip_install virtualenv + +# Prepare the workspace +TMP_VENV_PATH=$(mktemp -d tmp-venv-XXXX) +virtualenv $TMP_VENV_PATH + +# Install modern pip and wheel +$TMP_VENV_PATH/bin/pip install -U pip wheel + +# VENV_PACKAGES is a list of packages we want to pre-install +VENV_PACKAGE_FILE=$FILES/venv-requirements.txt +if [[ -r $VENV_PACKAGE_FILE ]]; then + VENV_PACKAGES=$(grep -v '^#' $VENV_PACKAGE_FILE) +fi + +for pkg in ${VENV_PACKAGES,/ } ${MORE_PACKAGES}; do + $TMP_VENV_PATH/bin/pip wheel $pkg +done + +# Clean up wheel workspace +rm -rf $TMP_VENV_PATH