Build wheel cache for venvs

Building a bunch of virtual envs later is going to be tedious if we do not
pre-cache certain annoying-to-build packages.

* tools/build_wheels.sh: pre-build some wheels for annoying package installs
* list distro package dependencies in files/*/venv
* list packages to pre-build as wheels in files/venv-requirements.txt
* install database Python modules when setting up the database

Change-Id: Idff1ea69a5ca12ba56098e664dbf6924fe6a2e47

clean.sh

@@ -119,6 +119,10 @@ if [[ -n "$SCREEN_LOGDIR" ]] && [[ -d "$SCREEN_LOGDIR" ]]; then
     sudo rm -rf $SCREEN_LOGDIR
 fi
 
+# Clean up venvs
+DIRS_TO_CLEAN="$WHEELHOUSE"
+rm -rf $DIRS_TO_CLEAN
+
 # Clean up files
 
 FILES_TO_CLEAN=".localrc.auto docs/files docs/html shocco/ stack-screenrc test*.conf* test.ini*"

files/debs/devlibs

@@ -0,0 +1,7 @@
+libffi-dev  # pyOpenSSL
+libmysqlclient-dev  # MySQL-python
+libpq-dev  # psycopg2
+libssl-dev  # pyOpenSSL
+libxml2-dev  # lxml
+libxslt1-dev  # lxml
+python-dev  # pyOpenSSL

files/debs/postgresql

@@ -1 +0,0 @@
-python-psycopg2

files/rpms-suse/devlibs

@@ -0,0 +1,6 @@
+libffi-devel  # pyOpenSSL
+libopenssl-devel  # pyOpenSSL
+libxml2-devel  # lxml
+libxslt1-dev  # lxml
+postgresql-devel  # psycopg2
+python-devel  # pyOpenSSL

files/rpms-suse/postgresql

@@ -1 +0,0 @@
-python-psycopg2

files/rpms/devlibs

@@ -0,0 +1,8 @@
+libffi-devel  # pyOpenSSL
+libxml2-devel  # lxml
+libxslt1-devel  # lxml
+mariadb-devel  # MySQL-python  f20,f21,rhel7
+mysql-devel  # MySQL-python  rhel6
+openssl-devel  # pyOpenSSL
+postgresql-devel  # psycopg2
+python-devel  # pyOpenSSL

files/rpms/postgresql

@@ -1 +0,0 @@
-python-psycopg2

files/venv-requirements.txt

@@ -0,0 +1,10 @@
+lxml
+MySQL-python
+netifaces
+numpy
+posix-ipc
+psycopg2
+pycrypto
+pyOpenSSL
+PyYAML
+xattr

lib/databases/mysql

@@ -151,6 +151,9 @@ EOF
     else
         exit_distro_not_supported "mysql installation"
     fi
+
+    # Install Python client module
+    pip_install MySQL-python
 }
 
 function database_connection_url_mysql {

lib/databases/postgresql

@@ -100,6 +100,9 @@ EOF
     else
         exit_distro_not_supported "postgresql installation"
     fi
+
+    # Install Python client module
+    pip_install MySQL-python psycopg2
 }
 
 function database_connection_url_postgresql {

stack.sh

@@ -671,6 +671,21 @@ fi
 source $TOP_DIR/tools/fixup_stuff.sh
 
 
+# Virtual Environment
+# -------------------
+
+# Temporary hack for testing
+# This belongs in d-g functions.sh setup_host() or devstack-vm-gate.sh
+if [[ -d /var/cache/pip ]]; then
+    sudo chown -R $STACK_USER:$STACK_USER /var/cache/pip
+fi
+
+# Pre-build some problematic wheels
+if [[ ! -d ${WHEELHOUSE:-} ]]; then
+    source tools/build_wheels.sh
+fi
+
+
 # Extras Pre-install
 # ------------------
 

stackrc

@@ -112,6 +112,11 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then
     source $RC_DIR/.localrc.auto
 fi
 
+# Configure wheel cache location
+export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse}
+export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE}
+export PIP_FIND_LINKS=${PIP_FIND_LINKS:-file://$WHEELHOUSE}
+
 # This can be used to turn database query logging on and off
 # (currently only implemented for MySQL backend)
 DATABASE_QUERY_LOGGING=$(trueorfalse True DATABASE_QUERY_LOGGING)

tools/build_wheels.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# **tools/build_wheels.sh** - Build a cache of Python wheels
+#
+# build_wheels.sh [package [...]]
+#
+# System package prerequisites listed in files/*/devlibs will be installed
+#
+# Builds wheels for all virtual env requirements listed in
+# ``venv-requirements.txt`` plus any supplied on the command line.
+#
+# Assumes ``tools/install_pip.sh`` has been run and a suitable pip/setuptools is available.
+
+# If TOP_DIR is set we're being sourced rather than running stand-alone
+# or in a sub-shell
+if [[ -z "$TOP_DIR" ]]; then
+
+    set -o errexit
+    set -o nounset
+
+    # Keep track of the devstack directory
+    TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
+    FILES=$TOP_DIR/files
+
+    # Import common functions
+    source $TOP_DIR/functions
+
+    GetDistro
+
+    source $TOP_DIR/stackrc
+
+    trap err_trap ERR
+
+fi
+
+# Get additional packages to build
+MORE_PACKAGES="$@"
+
+# Set a fall-back default, assume that since this script is being called
+# package builds are to happen even if WHEELHOUSE is not configured
+export WHEELHOUSE=${WHEELHOUSE:-.wheelhouse}
+
+# Exit on any errors so that errors don't compound
+function err_trap {
+    local r=$?
+    set +o xtrace
+
+    rm -rf $TMP_VENV_PATH
+
+    exit $r
+}
+
+# Get system prereqs
+install_package $(get_packages devlibs)
+
+# Get a modern ``virtualenv``
+pip_install virtualenv
+
+# Prepare the workspace
+TMP_VENV_PATH=$(mktemp -d tmp-venv-XXXX)
+virtualenv $TMP_VENV_PATH
+
+# Install modern pip and wheel
+$TMP_VENV_PATH/bin/pip install -U pip wheel
+
+# VENV_PACKAGES is a list of packages we want to pre-install
+VENV_PACKAGE_FILE=$FILES/venv-requirements.txt
+if [[ -r $VENV_PACKAGE_FILE ]]; then
+    VENV_PACKAGES=$(grep -v '^#' $VENV_PACKAGE_FILE)
+fi
+
+for pkg in ${VENV_PACKAGES,/ } ${MORE_PACKAGES}; do
+    $TMP_VENV_PATH/bin/pip wheel $pkg
+done
+
+# Clean up wheel workspace
+rm -rf $TMP_VENV_PATH