Add tools/install_pip.sh

Install a known working recent version of pip that handles installation
dependencies more correctly than before.  Extract to a separate script
so it can be used apart from stack.sh.

* Install distro setuptools if it not already present
* Install pip from source tarball as get-pip.py proved to be unreliable
* Remove python-distribute and python-pip from all prereq files,
  move python-setuptools to 'general'
* Remove the earlier unfubar_setuptppls() call that attenpted to fix this
* Only update requirements.txt when no changes in repo

Tested on Precise, F18 and CentOS6.
* Fedora and RHEL allow pip to install packages ON TOP OF RPM-installed
  packages.  THIS IS BROKEN.  And is one reason we have to be so picky
  about order and so forth.

Change-Id: Ibb4b42119dc2e51577c77bbbbffb110863e5324d

files/apts/general

@@ -1,6 +1,6 @@
 bridge-utils
 pylint
-python-pip
+python-setuptools
 screen
 unzip
 wget

files/apts/keystone

@@ -1,4 +1,3 @@
-python-setuptools
 python-dev
 python-lxml
 python-pastescript

files/apts/ryu

@@ -1,4 +1,3 @@
-python-setuptools
 python-gevent
 python-gflags
 python-netifaces

files/apts/swift

@@ -10,7 +10,6 @@ python-greenlet
 python-netifaces
 python-nose
 python-pastedeploy
-python-setuptools
 python-simplejson
 python-webob
 python-xattr

files/rpms-suse/general

@@ -6,8 +6,8 @@ iputils
 openssh
 openssl
 psmisc
+python-setuptools # instead of python-distribute; dist:sle11sp2
 python-cmd2 # dist:opensuse-12.3
-python-pip
 python-pylint
 python-unittest2
 python-virtualenv

files/rpms-suse/keystone

@@ -7,8 +7,6 @@ python-Routes
 python-SQLAlchemy
 python-WebOb
 python-devel
-python-distribute
-python-setuptools # instead of python-distribute; dist:sle11sp2
 python-greenlet
 python-lxml
 python-mysql

files/rpms-suse/ryu

@@ -1,5 +1,3 @@
-python-distribute
-python-setuptools # instead of python-distribute; dist:sle11sp2
 python-Sphinx
 python-gevent
 python-netifaces

files/rpms-suse/swift

@@ -6,8 +6,6 @@ python-WebOb
 python-configobj
 python-coverage
 python-devel
-python-distribute
-python-setuptools # instead of python-distribute; dist:sle11sp2
 python-eventlet
 python-greenlet
 python-netifaces

files/rpms/general

@@ -11,7 +11,7 @@ libxml2-devel # dist:rhel6 [2]
 libxslt-devel # dist:rhel6 [2]
 psmisc
 pylint
-python-pip
+python-setuptools
 python-prettytable # dist:rhel6 [1]
 python-unittest2
 python-virtualenv

files/rpms/keystone

@@ -4,7 +4,6 @@ python-paste        #dist:f16,f17,f18,f19
 python-paste-deploy #dist:f16,f17,f18,f19
 python-paste-script #dist:f16,f17,f18,f19
 python-routes
-python-setuptools   #dist:f16,f17,f18,f19
 python-sqlalchemy
 python-sqlite2
 python-webob

files/rpms/ryu

@@ -1,5 +1,4 @@
 python-gevent
 python-gflags
 python-netifaces
-python-setuptools #dist:f16,f17,f18,f19
 python-sphinx

files/rpms/swift

@@ -10,7 +10,6 @@ python-greenlet
 python-netifaces
 python-nose
 python-paste-deploy # dist:f16,f17,f18,f19
-python-setuptools   # dist:f16,f17,f18,f19
 python-simplejson
 python-webob
 pyxattr

functions

@@ -1140,8 +1140,11 @@ function setup_develop() {
 
     echo "cd $REQUIREMENTS_DIR; $SUDO_CMD python update.py $project_dir"
 
-    (cd $REQUIREMENTS_DIR; \
+    # Don't update repo if local changes exist
-        $SUDO_CMD python update.py $project_dir)
+    if (cd $project_dir && git diff --quiet); then
+        (cd $REQUIREMENTS_DIR; \
+            $SUDO_CMD python update.py $project_dir)
+    fi
 
     pip_install -e $project_dir
     # ensure that further actions can do things like setup.py sdist

stack.sh

@@ -578,18 +578,8 @@ set -o xtrace
 echo_summary "Installing package prerequisites"
 source $TOP_DIR/tools/install_prereqs.sh
 
-install_rpc_backend
+# Configure an appropriate python environment
-
+$TOP_DIR/tools/install_pip.sh
-if is_service_enabled $DATABASE_BACKENDS; then
-    install_database
-fi
-
-if is_service_enabled neutron; then
-    install_neutron_agent_packages
-fi
-
-# Unbreak the giant mess that is the current state of setuptools
-unfubar_setuptools
 
 # System-specific preconfigure
 # ============================
@@ -642,6 +632,16 @@ if [[ is_fedora && $DISTRO =~ (rhel6) ]]; then
     sudo ln -sf /usr/bin/nosetests1.1 /usr/local/bin/nosetests
 fi
 
+install_rpc_backend
+
+if is_service_enabled $DATABASE_BACKENDS; then
+    install_database
+fi
+
+if is_service_enabled neutron; then
+    install_neutron_agent_packages
+fi
+
 TRACK_DEPENDS=${TRACK_DEPENDS:-False}
 
 # Install python packages into a virtualenv so that we can track them

tools/install_pip.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+# **install_pip.sh**
+
+# install_pip.sh [--pip-version <version>] [--use-get-pip] [--setuptools] [--force]
+#
+# Update pip and friends to a known common version
+
+# Assumptions:
+# - currently we try to leave the system setuptools alone, install
+#   the system package if it is not already present
+# - update pip to $INSTALL_PIP_VERSION
+
+# Keep track of the current directory
+TOOLS_DIR=$(cd $(dirname "$0") && pwd)
+TOP_DIR=`cd $TOOLS_DIR/..; pwd`
+
+# Change dir to top of devstack
+cd $TOP_DIR
+
+# Import common functions
+source $TOP_DIR/functions
+
+FILES=$TOP_DIR/files
+
+# Handle arguments
+
+INSTALL_PIP_VERSION=${INSTALL_PIP_VERSION:-"1.4"}
+while [[ -n "$1" ]]; do
+    case $1 in
+        --force)
+            FORCE=1
+            ;;
+        --pip-version)
+            INSTALL_PIP_VERSION="$2"
+            shift
+            ;;
+        --setuptools)
+            SETUPTOOLS=1
+            ;;
+        --use-get-pip)
+            USE_GET_PIP=1;
+            ;;
+    esac
+    shift
+done
+
+SETUPTOOLS_EZ_SETUP_URL=https://bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py
+PIP_GET_PIP_URL=https://raw.github.com/pypa/pip/master/contrib/get-pip.py
+PIP_TAR_URL=https://pypi.python.org/packages/source/p/pip/pip-$INSTALL_PIP_VERSION.tar.gz
+
+GetDistro
+echo "Distro: $DISTRO"
+
+function get_versions() {
+    PIP=$(which pip 2>/dev/null || which pip-python 2>/dev/null)
+    if [[ -n $PIP ]]; then
+        DISTRIBUTE_VERSION=$($PIP freeze | grep 'distribute==')
+        SETUPTOOLS_VERSION=$($PIP freeze | grep 'setuptools==')
+        PIP_VERSION=$($PIP --version | awk '{ print $2}')
+        echo "pip: $PIP_VERSION  setuptools: $SETUPTOOLS_VERSION  distribute: $DISTRIBUTE_VERSION"
+    fi
+}
+
+function setuptools_ez_setup() {
+    if [[ ! -r $FILES/ez_setup.py ]]; then
+        (cd $FILES; \
+         curl -OR $SETUPTOOLS_EZ_SETUP_URL; \
+        )
+    fi
+    sudo python $FILES/ez_setup.py
+}
+
+function install_get_pip() {
+    if [[ ! -r $FILES/get-pip.py ]]; then
+        (cd $FILES; \
+            curl $PIP_GET_PIP_URL; \
+        )
+    fi
+    sudo python $FILES/get-pip.py
+}
+
+function install_pip_tarball() {
+    curl -O $PIP_TAR_URL
+    tar xvfz pip-$INSTALL_PIP_VERSION.tar.gz
+    cd pip-$INSTALL_PIP_VERSION
+    sudo python setup.py install
+}
+
+# Show starting versions
+get_versions
+
+# Do setuptools
+if [[ -n "$SETUPTOOLS" ]]; then
+    # We want it from source
+    uninstall_package python-setuptools
+    setuptools_ez_setup
+else
+    # See about installing the distro setuptools
+    if ! python -c "import setuptools"; then
+        install_package python-setuptools
+    fi
+fi
+
+# Do pip
+if [[ -z $PIP || "$PIP_VERSION" != "$INSTALL_PIP_VERSION" || -n $FORCE ]]; then
+
+    # Eradicate any and all system packages
+    uninstall_package python-pip
+
+    if [[ -n "$USE_GET_PIP" ]]; then
+        install_get_pip
+    else
+        install_pip_tarball
+    fi
+
+    get_versions
+fi