Add global venv enable/disable knob

Adds USE_VENV to globally enable/disable use of virtual environments.

ADDITIONAL_VENV_PACKAGES is used to manually add packages that do not
appear in requirements.txt or test-requirements.txt to be installed
into each venv.  Database Python bindings are handled this way when
a dataabse service is enabled.

Change-Id: I9cf298b936fd10c95e2ce5f51aab0d49d4b7f37f

doc/source/configuration.rst

@@ -170,6 +170,30 @@ Libraries from Git
 
       LIBS_FROM_GIT=python-keystoneclient,oslo.config
 
+Virtual Environments
+--------------------
+
+  | *Default: ``USE_VENV=False``*
+  |   Enable the use of Python virtual environments by setting ``USE_VENV``
+      to ``True``.  This will enable the creation of venvs for each project
+      that is defined in the ``PROJECT_VENV`` array.
+
+  | *Default: ``PROJECT_VENV['<project>']='<project-dir>.venv'*
+  |   Each entry in the ``PROJECT_VENV`` array contains the directory name
+      of a venv to be used for the project.  The array index is the project
+      name.  Multiple projects can use the same venv if desired.
+
+  ::
+
+    PROJECT_VENV["glance"]=${GLANCE_DIR}.venv
+
+  | *Default: ``ADDITIONAL_VENV_PACKAGES=""``*
+  |   A comma-separated list of additional packages to be installed into each
+      venv.  Often projects will not have certain packages listed in its
+      ``requirements.txt`` file because they are 'optional' requirements,
+      i.e. only needed for certain configurations.  By default, the enabled
+      databases will have their Python bindings added when they are enabled.
+
 Enable Logging
 --------------
 

doc/source/index.rst

@@ -210,6 +210,8 @@ Tools
 -----
 
 * `tools/build\_docs.sh <tools/build_docs.sh.html>`__
+* `tools/build\_venv.sh <tools/build_venv.sh.html>`__
+* `tools/build\_wheels.sh <tools/build_wheels.sh.html>`__
 * `tools/create-stack-user.sh <tools/create-stack-user.sh.html>`__
 * `tools/create\_userrc.sh <tools/create_userrc.sh.html>`__
 * `tools/fixup\_stuff.sh <tools/fixup_stuff.sh.html>`__

lib/database

@@ -109,6 +109,11 @@ function install_database {
     install_database_$DATABASE_TYPE
 }
 
+# Install the database Python packages
+function install_database_python {
+    install_database_python_$DATABASE_TYPE
+}
+
 # Configure and start the database
 function configure_database {
     configure_database_$DATABASE_TYPE

lib/databases/mysql

@@ -151,9 +151,12 @@ EOF
     else
         exit_distro_not_supported "mysql installation"
     fi
+}
 
+function install_database_python_mysql {
     # Install Python client module
     pip_install MySQL-python
+    ADDITIONAL_VENV_PACKAGES+=",MySQL-python"
 }
 
 function database_connection_url_mysql {

lib/databases/postgresql

@@ -100,9 +100,12 @@ EOF
     else
         exit_distro_not_supported "postgresql installation"
     fi
+}
 
+function install_database_python_postgresql {
     # Install Python client module
     pip_install psycopg2
+    ADDITIONAL_VENV_PACKAGES+=",psycopg2"
 }
 
 function database_connection_url_postgresql {

lib/stack

@@ -16,13 +16,17 @@
 function stack_install_service {
     local service=$1
     if type install_${service} >/dev/null 2>&1; then
-        if [[ -n ${PROJECT_VENV[$service]:-} ]]; then
+        if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then
             rm -rf ${PROJECT_VENV[$service]}
-            source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]}
+            source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]} ${ADDITIONAL_VENV_PACKAGES//,/ }
             export PIP_VIRTUAL_ENV=${PROJECT_VENV[$service]:-}
+
+            # Install other OpenStack prereqs that might come from source repos
+            install_oslo
+            install_keystonemiddleware
         fi
         install_${service}
-        if [[ -n ${PROJECT_VENV[$service]:-} ]]; then
+        if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then
             unset PIP_VIRTUAL_ENV
         fi
     fi

stack.sh

@@ -702,6 +702,7 @@ install_rpc_backend
 
 if is_service_enabled $DATABASE_BACKENDS; then
     install_database
+    install_database_python
 fi
 
 if is_service_enabled neutron; then

stackrc

@@ -104,6 +104,16 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then
     source $RC_DIR/.localrc.auto
 fi
 
+# Enable use of Python virtual environments.  Individual project use of
+# venvs are controlled by the PROJECT_VENV array; every project with
+# an entry in the array will be installed into the named venv.
+# By default this will put each project into its own venv.
+USE_VENV=$(trueorfalse False USE_VENV)
+
+# Add packages that need to be installed into a venv but are not in any
+# requirmenets files here, in a comma-separated list
+ADDITIONAL_VENV_PACKAGES=${ADITIONAL_VENV_PACKAGES:-""}
+
 # Configure wheel cache location
 export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse}
 export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE}