From e1edde38edb697f56e111f55e5992e7bf22f6284 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 2 Mar 2018 15:05:14 +0000 Subject: [PATCH] Automatically set LIBS_FROM_GIT based on required projects If a project shows up in zuul's required-projects list, add it to LIBS_FROM_GIT automatically. This way, when a user specifies that a job requires a zuul-project, it gets used in testing, but otherwise, it doesn't (pypi is used instead). Also add information about what happens behind the scenes for both LIBS_FROM_GIT and plugin dependencies. This moves the check performed in check_libs_from_git to a helper function which is installed for most kinds of installations. This means that if someone sets LIBS_FROM_GIT to "foobar", devstack won't error anymore, as nothing is going to try to install foobar, therefore the check won't run on that. However, as we move to automated generation of the local config, that error is not likely to happen. This check was originally added due to an error in the upper-constraints file (where a constraint name did not match a package name). This location of the check would still catch that type of error. Change-Id: Ifcf3ad008cf42d3d4762cfb3b6c31c93cfeb40db --- inc/python | 23 ++---- roles/write-devstack-local-conf/README.rst | 12 +++ .../library/devstack_local_conf.py | 29 ++++++-- .../write-devstack-local-conf/library/test.py | 74 +++++++++++++++++-- .../write-devstack-local-conf/tasks/main.yaml | 1 + stack.sh | 5 -- 6 files changed, 110 insertions(+), 34 deletions(-) diff --git a/inc/python b/inc/python index e074ea498f..37b4617777 100644 --- a/inc/python +++ b/inc/python @@ -435,22 +435,6 @@ function lib_installed_from_git { [[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]] } -# check that everything that's in LIBS_FROM_GIT was actually installed -# correctly, this helps double check issues with library fat fingering. -function check_libs_from_git { - local lib="" - local not_installed="" - for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do - if ! lib_installed_from_git "$lib"; then - not_installed+=" $lib" - fi - done - # if anything is not installed, say what it is. - if [[ -n "$not_installed" ]]; then - die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed" - fi -} - # setup a library by name. If we are trying to use the library from # git, we'll do a git based install, otherwise we'll punt and the # library should be installed by a requirements pull from another @@ -561,6 +545,13 @@ function _setup_package_with_constraints_edit { setup_package $project_dir "$flags" $extras + # If this project is in LIBS_FROM_GIT, verify it was actually installed + # correctly. This helps catch errors caused by constraints mismatches. + if use_library_from_git "$project_dir"; then + if ! lib_installed_from_git "$project_dir"; then + die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir" + fi + fi } # ``pip install -e`` the package, which processes the dependencies diff --git a/roles/write-devstack-local-conf/README.rst b/roles/write-devstack-local-conf/README.rst index 73f9f0d6fd..bfce9c98cd 100644 --- a/roles/write-devstack-local-conf/README.rst +++ b/roles/write-devstack-local-conf/README.rst @@ -20,6 +20,14 @@ Write the local.conf file for use by devstack bash shell variables, and will be ordered so that variables used by later entries appear first. + As a special case, the variable ``LIBS_FROM_GIT`` will be + constructed automatically from the projects which appear in the + ``required-projects`` list defined by the job. To instruct + devstack to install a library from source rather than pypi, simply + add that library to the job's ``required-projects`` list. To + override the automatically-generated value, set ``LIBS_FROM_GIT`` + in ``devstack_localrc`` to the desired value. + .. zuul:rolevar:: devstack_local_conf :type: dict @@ -75,3 +83,7 @@ Write the local.conf file for use by devstack A dictionary mapping a plugin name to a git repo location. If the location is a non-empty string, then an ``enable_plugin`` line will be emmitted for the plugin name. + + If a plugin declares a dependency on another plugin (via + ``plugin_requires`` in the plugin's settings file), this role will + automatically emit ``enable_plugin`` lines in the correct order. diff --git a/roles/write-devstack-local-conf/library/devstack_local_conf.py b/roles/write-devstack-local-conf/library/devstack_local_conf.py index 746f54f921..9728fef37b 100644 --- a/roles/write-devstack-local-conf/library/devstack_local_conf.py +++ b/roles/write-devstack-local-conf/library/devstack_local_conf.py @@ -207,17 +207,17 @@ class PluginGraph(DependencyGraph): class LocalConf(object): def __init__(self, localrc, localconf, base_services, services, plugins, - base_dir): + base_dir, projects): self.localrc = [] self.meta_sections = {} self.plugin_deps = {} self.base_dir = base_dir + self.projects = projects if plugins: self.handle_plugins(plugins) if services or base_services: self.handle_services(base_services, services or {}) - if localrc: - self.handle_localrc(localrc) + self.handle_localrc(localrc) if localconf: self.handle_localconf(localconf) @@ -241,9 +241,22 @@ class LocalConf(object): self.localrc.append('enable_service {}'.format(k)) def handle_localrc(self, localrc): - vg = VarGraph(localrc) - for k, v in vg.getVars(): - self.localrc.append('{}={}'.format(k, v)) + lfg = False + if localrc: + vg = VarGraph(localrc) + for k, v in vg.getVars(): + self.localrc.append('{}={}'.format(k, v)) + if k == 'LIBS_FROM_GIT': + lfg = True + + if not lfg and self.projects: + required_projects = [] + for project_name, project_info in self.projects.items(): + if project_info.get('required'): + required_projects.append(project_info['short_name']) + if required_projects: + self.localrc.append('LIBS_FROM_GIT={}'.format( + ','.join(required_projects))) def handle_localconf(self, localconf): for phase, phase_data in localconf.items(): @@ -277,6 +290,7 @@ def main(): local_conf=dict(type='dict'), base_dir=dict(type='path'), path=dict(type='str'), + projects=dict(type='dict'), ) ) @@ -286,7 +300,8 @@ def main(): p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) module.exit_json() diff --git a/roles/write-devstack-local-conf/library/test.py b/roles/write-devstack-local-conf/library/test.py index 843ca6e9fd..7ccb68f08d 100644 --- a/roles/write-devstack-local-conf/library/test.py +++ b/roles/write-devstack-local-conf/library/test.py @@ -56,7 +56,8 @@ class TestDevstackLocalConf(unittest.TestCase): p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) plugins = [] @@ -66,6 +67,7 @@ class TestDevstackLocalConf(unittest.TestCase): plugins.append(line.split()[1]) self.assertEqual(['bar', 'baz', 'foo'], plugins) + def test_plugin_deps(self): "Test that plugins with dependencies work" os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack')) @@ -101,20 +103,80 @@ class TestDevstackLocalConf(unittest.TestCase): plugins=plugins, base_dir=self.tmpdir, path=os.path.join(self.tmpdir, 'test.local.conf')) + + def test_libs_from_git(self): + "Test that LIBS_FROM_GIT is auto-generated" + projects = { + 'git.openstack.org/openstack/nova': { + 'required': True, + 'short_name': 'nova', + }, + 'git.openstack.org/openstack/oslo.messaging': { + 'required': True, + 'short_name': 'oslo.messaging', + }, + 'git.openstack.org/openstack/devstack-plugin': { + 'required': False, + 'short_name': 'devstack-plugin', + }, + } + p = dict(base_services=[], + base_dir='./test', + path=os.path.join(self.tmpdir, 'test.local.conf'), + projects=projects) lc = LocalConf(p.get('localrc'), p.get('local_conf'), p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) - plugins = [] + lfg = None with open(p['path']) as f: for line in f: - if line.startswith('enable_plugin'): - plugins.append(line.split()[1]) - self.assertEqual(['foo', 'bar'], plugins) + if line.startswith('LIBS_FROM_GIT'): + lfg = line.strip().split('=')[1] + self.assertEqual('nova,oslo.messaging', lfg) + + def test_overridelibs_from_git(self): + "Test that LIBS_FROM_GIT can be overridden" + localrc = {'LIBS_FROM_GIT': 'oslo.db'} + projects = { + 'git.openstack.org/openstack/nova': { + 'required': True, + 'short_name': 'nova', + }, + 'git.openstack.org/openstack/oslo.messaging': { + 'required': True, + 'short_name': 'oslo.messaging', + }, + 'git.openstack.org/openstack/devstack-plugin': { + 'required': False, + 'short_name': 'devstack-plugin', + }, + } + p = dict(localrc=localrc, + base_services=[], + base_dir='./test', + path=os.path.join(self.tmpdir, 'test.local.conf'), + projects=projects) + lc = LocalConf(p.get('localrc'), + p.get('local_conf'), + p.get('base_services'), + p.get('services'), + p.get('plugins'), + p.get('base_dir'), + p.get('projects')) + lc.write(p['path']) + + lfg = None + with open(p['path']) as f: + for line in f: + if line.startswith('LIBS_FROM_GIT'): + lfg = line.strip().split('=')[1] + self.assertEqual('oslo.db', lfg) def test_plugin_circular_deps(self): "Test that plugins with circular dependencies fail" diff --git a/roles/write-devstack-local-conf/tasks/main.yaml b/roles/write-devstack-local-conf/tasks/main.yaml index 2a9f8985fc..a294cae608 100644 --- a/roles/write-devstack-local-conf/tasks/main.yaml +++ b/roles/write-devstack-local-conf/tasks/main.yaml @@ -9,3 +9,4 @@ localrc: "{{ devstack_localrc|default(omit) }}" local_conf: "{{ devstack_local_conf|default(omit) }}" base_dir: "{{ devstack_base_dir|default(omit) }}" + projects: "{{ zuul.projects }}" diff --git a/stack.sh b/stack.sh index 9b496c0e20..5643b4db1b 100755 --- a/stack.sh +++ b/stack.sh @@ -1402,11 +1402,6 @@ fi # Check the status of running services service_check -# ensure that all the libraries we think we installed from git, -# actually were. -check_libs_from_git - - # Configure nova cellsv2 # ----------------------