From e8b33600385ecd6812964c746daa86cf36770de2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 5 Jul 2013 23:03:37 -0400 Subject: [PATCH] Sync install_venv_common from oslo Change-Id: Iadcfa6d00b963a19ce15300b2293036cf0e8204b --- openstack-common.conf | 3 +- tools/install_venv.py | 185 +++++++++--------------------- tools/install_venv_common.py | 214 +++++++++++++++++++++++++++++++++++ 3 files changed, 269 insertions(+), 133 deletions(-) create mode 100644 tools/install_venv_common.py diff --git a/openstack-common.conf b/openstack-common.conf index aa99f1b1bd..2ed20f17bb 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,9 +1,10 @@ [DEFAULT] +module=config module=eventlet_backdoor +module=install_venv_common module=notifier module=rpc module=service module=threadgroup -module=config base=openstack_dashboard diff --git a/tools/install_venv.py b/tools/install_venv.py index 6e824e78a7..0011a8be14 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -1,156 +1,77 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2012 United States Government as represented by the +# Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # -# Copyright 2012 OpenStack, LLC +# Copyright 2010 OpenStack Foundation +# Copyright 2013 IBM Corp. +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # -# Copyright 2012 Nebula, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Installation script for the OpenStack Dashboard development virtualenv. -""" +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import ConfigParser import os -import subprocess import sys - -ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.venv') -WITH_VENV = os.path.join(ROOT, 'tools', 'with_venv.sh') -PIP_REQUIRES = os.path.join(ROOT, 'requirements.txt') -TEST_REQUIRES = os.path.join(ROOT, 'test-requirements.txt') +import install_venv_common as install_venv # flake8: noqa -def die(message, *args): - print >> sys.stderr, message % args - sys.exit(1) +def print_help(project, venv, root): + help = """ + %(project)s development environment setup is complete. + %(project)s development uses virtualenv to track and manage Python + dependencies while in development and testing. -def run_command(cmd, redirect_output=True, check_exit_code=True, cwd=ROOT, - die_message=None): + To activate the %(project)s virtualenv for the extent of your current + shell session you can run: + + $ source %(venv)s/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by + case basis by running: + + $ %(root)s/tools/with_venv.sh """ - Runs a command in an out-of-process shell, returning the - output of that command. Working directory is ROOT. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=cwd, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - if die_message is None: - die('Command "%s" failed.\n%s', ' '.join(cmd), output) - else: - die(die_message) - return output + print help % dict(project=project, venv=venv, root=root) -HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], - check_exit_code=False).strip()) -HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], - check_exit_code=False).strip()) +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if os.environ.get('tools_path'): + root = os.environ['tools_path'] + venv = os.path.join(root, '.venv') + if os.environ.get('venv'): + venv = os.environ['venv'] -def check_dependencies(): - """Make sure virtualenv is in the path.""" + pip_requires = os.path.join(root, 'requirements.txt') + test_requires = os.path.join(root, 'test-requirements.txt') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + setup_cfg = ConfigParser.ConfigParser() + setup_cfg.read('setup.cfg') + project = setup_cfg.get('metadata', 'name') - print 'Checking dependencies...' - if not HAS_VIRTUALENV: - print 'Virtual environment not found.' - # Try installing it via easy_install... - if HAS_EASY_INSTALL: - print 'Installing virtualenv via easy_install...', - run_command(['easy_install', 'virtualenv'], - die_message='easy_install failed to install virtualenv' - '\ndevelopment requires virtualenv, please' - ' install it using your favorite tool') - if not run_command(['which', 'virtualenv']): - die('ERROR: virtualenv not found in path.\n\ndevelopment ' - ' requires virtualenv, please install it using your' - ' favorite package management tool and ensure' - ' virtualenv is in your path') - print 'virtualenv installation done.' - else: - die('easy_install not found.\n\nInstall easy_install' - ' (python-setuptools in ubuntu) or virtualenv by hand,' - ' then rerun.') - print 'dependency check done.' - - -def create_virtualenv(venv=VENV): - """Creates the virtual environment and installs PIP only into the - virtual environment - """ - print 'Creating venv...', - run_command(['virtualenv', '-q', '--no-site-packages', VENV]) - print 'done.' - print 'Installing pip in virtualenv...', - if not run_command([WITH_VENV, 'easy_install', 'pip']).strip(): - die("Failed to install pip.") - print 'done.' - print 'Installing distribute in virtualenv...' - pip_install('distribute>=0.6.24') - print 'done.' - - -def pip_install(*args): - args = [WITH_VENV, 'pip', 'install', '--upgrade'] + list(args) - run_command(args, redirect_output=False) - - -def install_dependencies(venv=VENV): - print "Installing dependencies..." - print "(This may take several minutes, don't panic)" - pip_install('-r', TEST_REQUIRES) - pip_install('-r', PIP_REQUIRES) - - # Tell the virtual env how to "import dashboard" - py = 'python%d.%d' % (sys.version_info[0], sys.version_info[1]) - pthfile = os.path.join(venv, "lib", py, "site-packages", "dashboard.pth") - f = open(pthfile, 'w') - f.write("%s\n" % ROOT) - - -def install_horizon(): - print 'Installing horizon module in development mode...' - run_command([WITH_VENV, 'python', 'setup.py', 'develop'], cwd=ROOT) - - -def print_summary(): - summary = """ -Horizon development environment setup is complete. - -To activate the virtualenv for the extent of your current shell session you -can run: - -$ source .venv/bin/activate -""" - print summary - - -def main(): - check_dependencies() - create_virtualenv() - install_dependencies() - install_horizon() - print_summary() + install = install_venv.InstallVenv( + root, venv, pip_requires, test_requires, py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + install.post_process() + print_help(project, venv, root) if __name__ == '__main__': - main() + main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 0000000000..0999e2c29e --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,214 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + +Synced in from openstack-common +""" + +from __future__ import print_function + +import optparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, requirements, + test_requirements, py_version, + project): + self.root = root + self.venv = venv + self.requirements = requirements + self.test_requirements = test_requirements + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print(message % args, file=sys.stderr) + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + else: + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print('Creating venv...', end=' ') + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print('done.') + else: + print("venv already exists...") + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print('Installing dependencies with pip (this can take a while)...') + + # First things first, make sure our venv has the latest pip and + # setuptools and pbr + self.pip_install('pip>=1.4') + self.pip_install('setuptools') + self.pip_install('pbr') + + self.pip_install('-r', self.requirements) + self.pip_install('-r', self.test_requirements) + + def post_process(self): + self.get_distro().post_process() + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:])[0] + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print('Installing virtualenv via easy_install...', end=' ') + if self.run_command(['easy_install', 'virtualenv']): + print('Succeeded') + return + else: + print('Failed') + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def apply_patch(self, originalfile, patchfile): + self.run_command(['patch', '-N', originalfile, patchfile], + check_exit_code=False) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.die("Please install 'python-virtualenv'.") + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/eventlet/eventlet/issue/89 + RHEL: https://bugzilla.redhat.com/958868 + """ + + if os.path.exists('contrib/redhat-eventlet.patch'): + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.die("Please install 'patch'.") + + # Apply the eventlet patch + self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, + 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch')