Clean up test environment and remove unused imports.

First round of adding more complete unit test coverage.

Change-Id: Ic1979c499ca6fcb784892a95954a3527539c4e53

.mailmap

@@ -0,0 +1,2 @@
+<josh.kearney@pistoncloud.com> <josh@jk0.org>
+<matt.joyce@cloudscaling.com> <matt@nycresistor.com>

README.rst

@@ -98,7 +98,6 @@ Additional command-line options and their associated environment variables
 are listed here::
 
    --debug             # turns on some debugging of the API conversation
-                         (via httplib2)
    --verbose | -v      # Increase verbosity of output. Can be repeated.
    --quiet | -q        # suppress output except warnings and errors
    --help | -h         # show a help message and exit

openstackclient/common/clientmanager.py

@@ -20,7 +20,6 @@
 
 import logging
 
-from openstackclient.common import exceptions as exc
 from openstackclient.compute import client as compute_client
 from openstackclient.identity import client as identity_client
 from openstackclient.image import client as image_client

openstackclient/common/openstackkeyring.py

@@ -21,7 +21,6 @@ Keyring backend for Openstack, to store encrypted password in a file.
 
 from Crypto.Cipher import AES
 
-import crypt
 import keyring
 import os
 

openstackclient/common/utils.py

@@ -23,8 +23,6 @@ import os
 import sys
 import uuid
 
-import prettytable
-
 from openstackclient.common import exceptions
 
 

openstackclient/compute/client.py

@@ -17,7 +17,6 @@
 
 import logging
 
-from openstackclient.common import exceptions as exc
 from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)

openstackclient/compute/v2/server.py

@@ -21,7 +21,6 @@ Server action implementations
 
 import logging
 import os
-import sys
 import time
 
 from cliff import command

openstackclient/identity/client.py

@@ -17,7 +17,6 @@
 
 import logging
 
-from openstackclient.common import exceptions as exc
 from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)

openstackclient/openstack/common/setup.py

@@ -78,10 +78,6 @@ def parse_requirements(requirements_files=['requirements.txt',
         # -f lines are for index locations, and don't get used here
         elif re.match(r'\s*-f\s+', line):
             pass
-        # argparse is part of the standard library starting with 2.7
-        # adding it to the requirements list screws distro installs
-        elif line == 'argparse' and sys.version_info >= (2, 7):
-            pass
         else:
             requirements.append(line)
 

run_tests.sh

@@ -1,49 +1,152 @@
 #!/bin/bash
 
+set -eu
+
 function usage {
   echo "Usage: $0 [OPTION]..."
-  echo "Run python-openstackclient's test suite(s)"
+  echo "Run python-openstackclient test suite"
   echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -s, --no-site-packages   Isolate the virtualenv from the global Python environment"
+  echo "  -x, --stop               Stop running tests after the first error or failure."
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
   echo "  -p, --pep8               Just run pep8"
+  echo "  -P, --no-pep8            Don't run pep8"
+  echo "  -c, --coverage           Generate coverage report"
   echo "  -h, --help               Print this usage message"
+  echo "  --hide-elapsed           Don't print the elapsed time for each test along with slow test list"
   echo ""
-  echo "This script is deprecated and currently retained for compatibility."
-  echo 'You can run the full test suite for multiple environments by running "tox".'
-  echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only'
-  echo 'the pep8 tests with "tox -e pep8".'
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
   exit
 }
 
-command -v tox > /dev/null 2>&1
-if [ $? -ne 0 ]; then
-  echo 'This script requires "tox" to run.'
-  echo 'You can install it with "pip install tox".'
-  exit 1; 
-fi
-
-just_pep8=0
-
 function process_option {
   case "$1" in
     -h|--help) usage;;
-    -p|--pep8) let just_pep8=1;;
+    -V|--virtual-env) always_venv=1; never_venv=0;;
+    -N|--no-virtual-env) always_venv=0; never_venv=1;;
+    -s|--no-site-packages) no_site_packages=1;;
+    -f|--force) force=1;;
+    -p|--pep8) just_pep8=1;;
+    -P|--no-pep8) no_pep8=1;;
+    -c|--coverage) coverage=1;;
+    -*) noseopts="$noseopts $1";;
+    *) noseargs="$noseargs $1"
   esac
 }
 
+venv=.venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+no_site_packages=0
+installvenvopts=
+noseargs=
+noseopts=
+wrapper=""
+just_pep8=0
+no_pep8=0
+coverage=0
+
 for arg in "$@"; do
   process_option $arg
 done
 
+# If enabled, tell nose to collect coverage data
+if [ $coverage -eq 1 ]; then
+    noseopts="$noseopts --with-coverage --cover-package=openstackclient"
+fi
+
+if [ $no_site_packages -eq 1 ]; then
+  installvenvopts="--no-site-packages"
+fi
+
+function run_tests {
+  # Just run the test suites in current environment
+  ${wrapper} $NOSETESTS
+  # If we get some short import error right away, print the error log directly
+  RESULT=$?
+  return $RESULT
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  srcfiles="openstackclient tests"
+  # Just run PEP8 in current environment
+  #
+  # NOTE(sirp): W602 (deprecated 3-arg raise) is being ignored for the
+  # following reasons:
+  #
+  #  1. It's needed to preserve traceback information when re-raising
+  #     exceptions; this is needed b/c Eventlet will clear exceptions when
+  #     switching contexts.
+  #
+  #  2. There doesn't appear to be an alternative, "pep8-tool" compatible way of doing this
+  #     in Python 2 (in Python 3 `with_traceback` could be used).
+  #
+  #  3. Can find no corroborating evidence that this is deprecated in Python 2
+  #     other than what the PEP8 tool claims. It is deprecated in Python 3, so,
+  #     perhaps the mistake was thinking that the deprecation applied to Python 2
+  #     as well.
+  pep8_opts="--ignore=E202,W602 --repeat"
+  ${wrapper} pep8 ${pep8_opts} ${srcfiles}
+}
+
+NOSETESTS="nosetests $noseopts $noseargs"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py $installvenvopts
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py $installvenvopts
+        wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+# Delete old coverage data from previous runs
+if [ $coverage -eq 1 ]; then
+    ${wrapper} coverage erase
+fi
+
 if [ $just_pep8 -eq 1 ]; then
-  tox -e pep8
-  exit
+    run_pep8
+    exit
 fi
 
-tox -e py27 $toxargs 2>&1 | tee run_tests.err.log  || exit
-if [ ${PIPESTATUS[0]} -ne 0 ]; then
-  exit ${PIPESTATUS[0]}
+run_tests
+
+# NOTE(sirp): we only want to run pep8 when we're running the full-test suite,
+# not when we're running tests individually. To handle this, we need to
+# distinguish between options (noseopts), which begin with a '-', and
+# arguments (noseargs).
+if [ -z "$noseargs" ]; then
+  if [ $no_pep8 -eq 0 ]; then
+    run_pep8
+  fi
 fi
 
-if [ -z "$toxargs" ]; then
-  tox -e pep8
+if [ $coverage -eq 1 ]; then
+    echo "Generating coverage report in covhtml/"
+    ${wrapper} coverage html -d covhtml -i
 fi

tests/utils.py

@@ -2,7 +2,6 @@
 
 import time
 
-import mox
 import testtools
 
 
@@ -10,12 +9,9 @@ class TestCase(testtools.TestCase):
 
     def setUp(self):
         super(TestCase, self).setUp()
-        self.mox = mox.Mox()
         self._original_time = time.time
         time.time = lambda: 1234
 
     def tearDown(self):
         time.time = self._original_time
-        self.mox.UnsetStubs()
-        self.mox.VerifyAll()
         super(TestCase, self).tearDown()

tools/install_venv.py

@@ -0,0 +1,244 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Copyright 2010 OpenStack, LLC
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Installation script for python-openstackclient's development virtualenv
+"""
+
+import optparse
+import os
+import subprocess
+import sys
+import platform
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+VENV = os.path.join(ROOT, '.venv')
+PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
+TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires')
+PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
+
+
+def die(message, *args):
+    print >> sys.stderr, message % args
+    sys.exit(1)
+
+
+def check_python_version():
+    if sys.version_info < (2, 6):
+        die("Need Python Version >= 2.6")
+
+
+def run_command_with_code(cmd, redirect_output=True, check_exit_code=True):
+    """
+    Runs a command in an out-of-process shell, returning the
+    output of that command.  Working directory is ROOT.
+    """
+    if redirect_output:
+        stdout = subprocess.PIPE
+    else:
+        stdout = None
+
+    proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout)
+    output = proc.communicate()[0]
+    if check_exit_code and proc.returncode != 0:
+        die('Command "%s" failed.\n%s', ' '.join(cmd), output)
+    return (output, proc.returncode)
+
+
+def run_command(cmd, redirect_output=True, check_exit_code=True):
+    return run_command_with_code(cmd, redirect_output, check_exit_code)[0]
+
+
+class Distro(object):
+
+    def check_cmd(self, cmd):
+        return bool(run_command(['which', cmd], check_exit_code=False).strip())
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if self.check_cmd('easy_install'):
+            print 'Installing virtualenv via easy_install...',
+            if run_command(['easy_install', 'virtualenv']):
+                print 'Succeeded'
+                return
+            else:
+                print 'Failed'
+
+        die('ERROR: virtualenv not found.\n\nDevelopment'
+            ' requires virtualenv, please install it using your'
+            ' favorite package management tool')
+
+    def post_process(self):
+        """Any distribution-specific post-processing gets done here.
+
+        In particular, this is useful for applying patches to code inside
+        the venv."""
+        pass
+
+
+class Debian(Distro):
+    """This covers all Debian-based distributions."""
+
+    def check_pkg(self, pkg):
+        return run_command_with_code(['dpkg', '-l', pkg],
+                                     check_exit_code=False)[1] == 0
+
+    def apt_install(self, pkg, **kwargs):
+        run_command(['sudo', 'apt-get', 'install', '-y', pkg], **kwargs)
+
+    def apply_patch(self, originalfile, patchfile):
+        run_command(['patch', originalfile, patchfile])
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if not self.check_pkg('python-virtualenv'):
+            self.apt_install('python-virtualenv', check_exit_code=False)
+
+        super(Debian, self).install_virtualenv()
+
+
+class Fedora(Distro):
+    """This covers all Fedora-based distributions.
+
+    Includes: Fedora, RHEL, CentOS, Scientific Linux"""
+
+    def check_pkg(self, pkg):
+        return run_command_with_code(['rpm', '-q', pkg],
+                                     check_exit_code=False)[1] == 0
+
+    def yum_install(self, pkg, **kwargs):
+        run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
+
+    def apply_patch(self, originalfile, patchfile):
+        run_command(['patch', originalfile, patchfile])
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if not self.check_pkg('python-virtualenv'):
+            self.yum_install('python-virtualenv', check_exit_code=False)
+
+        super(Fedora, self).install_virtualenv()
+
+
+def get_distro():
+    if os.path.exists('/etc/fedora-release') or \
+       os.path.exists('/etc/redhat-release'):
+        return Fedora()
+    elif os.path.exists('/etc/debian_version'):
+        return Debian()
+    else:
+        return Distro()
+
+
+def check_dependencies():
+    get_distro().install_virtualenv()
+
+
+def create_virtualenv(venv=VENV, no_site_packages=True):
+    """Creates the virtual environment and installs PIP only into the
+    virtual environment
+    """
+    print 'Creating venv...',
+    if no_site_packages:
+        run_command(['virtualenv', '-q', '--no-site-packages', VENV])
+    else:
+        run_command(['virtualenv', '-q', VENV])
+    print 'done.'
+    print 'Installing pip in virtualenv...',
+    if not run_command(['tools/with_venv.sh', 'easy_install',
+                        'pip>1.0']).strip():
+        die("Failed to install pip.")
+    print 'done.'
+
+
+def pip_install(*args):
+    run_command(['tools/with_venv.sh',
+                 'pip', 'install', '--upgrade'] + list(args),
+                redirect_output=False)
+
+
+def install_dependencies(venv=VENV):
+    print 'Installing dependencies with pip (this can take a while)...'
+
+    # First things first, make sure our venv has the latest pip and distribute.
+    pip_install('pip')
+    pip_install('distribute')
+
+    pip_install('-r', PIP_REQUIRES)
+    pip_install('-r', TEST_REQUIRES)
+
+    # Tell the virtual env how to "import openstackclient"
+    pthfile = os.path.join(venv, "lib", PY_VERSION, "site-packages",
+                        "openstackclient.pth")
+    f = open(pthfile, 'w')
+    f.write("%s\n" % ROOT)
+
+
+def post_process():
+    get_distro().post_process()
+
+
+def print_help():
+    help = """
+    python-openstackclient development environment setup is complete.
+
+    python-openstackclient development uses virtualenv to track and manage Python
+    dependencies while in development and testing.
+
+    To activate the python-openstackclient virtualenv for the extent of your current
+    shell session you can run:
+
+    $ source .venv/bin/activate
+
+    Or, if you prefer, you can run commands in the virtualenv on a case by case
+    basis by running:
+
+    $ tools/with_venv.sh <your command>
+
+    Also, make test will automatically use the virtualenv.
+    """
+    print help
+
+
+def parse_args():
+    """Parse command-line arguments"""
+    parser = optparse.OptionParser()
+    parser.add_option("-n", "--no-site-packages", dest="no_site_packages",
+        default=False, action="store_true",
+        help="Do not inherit packages from global Python install")
+    return parser.parse_args()
+
+
+def main(argv):
+    (options, args) = parse_args()
+    check_python_version()
+    check_dependencies()
+    create_virtualenv(no_site_packages=options.no_site_packages)
+    install_dependencies()
+    post_process()
+    print_help()
+
+if __name__ == '__main__':
+    main(sys.argv)

tools/pip-requires

@@ -1,11 +1,6 @@
-argparse
 cliff
-cmd2
-httplib2
 keyring
-prettytable
-pyparsing>=1.5.6,<2.0
 pycrypto
+python-glanceclient>=0.5.1
 python-keystoneclient>=0.2,<1.0
 python-novaclient>=2
-simplejson

tools/test-requires

@@ -1,14 +1,12 @@
 distribute>=0.6.24
 
-coverage
 fixtures
 mock
-mox
 nose
 nose-exclude
-nosehtmloutput
 nosexcover
+nosehtmloutput
 openstack.nose_plugin
-pep8==0.6.1
+pep8==1.1
 sphinx>=1.1.2
 testtools>=0.9.22

tools/with_venv.sh

@@ -1,10 +1,4 @@
 #!/bin/bash
-
-command -v tox > /dev/null 2>&1
-if [ $? -ne 0 ]; then
-  echo 'This script requires "tox" to run.'
-  echo 'You can install it with "pip install tox".'
-  exit 1; 
-fi
-
-tox -evenv -- $@
+TOOLS=`dirname $0`
+VENV=$TOOLS/../.venv
+source $VENV/bin/activate && $@