Merge "Automatically set LIBS_FROM_GIT based on required projects"

inc/python

@@ -429,22 +429,6 @@ function lib_installed_from_git {
     [[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
 }
 
-# check that everything that's in LIBS_FROM_GIT was actually installed
-# correctly, this helps double check issues with library fat fingering.
-function check_libs_from_git {
-    local lib=""
-    local not_installed=""
-    for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
-        if ! lib_installed_from_git "$lib"; then
-            not_installed+=" $lib"
-        fi
-    done
-    # if anything is not installed, say what it is.
-    if [[ -n "$not_installed" ]]; then
-        die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
-    fi
-}
-
 # setup a library by name. If we are trying to use the library from
 # git, we'll do a git based install, otherwise we'll punt and the
 # library should be installed by a requirements pull from another
@@ -555,6 +539,13 @@ function _setup_package_with_constraints_edit {
 
     setup_package $project_dir "$flags" $extras
 
+    # If this project is in LIBS_FROM_GIT, verify it was actually installed
+    # correctly.  This helps catch errors caused by constraints mismatches.
+    if use_library_from_git "$project_dir"; then
+        if ! lib_installed_from_git "$project_dir"; then
+            die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
+        fi
+    fi
 }
 
 # ``pip install -e`` the package, which processes the dependencies

roles/write-devstack-local-conf/README.rst

@@ -20,6 +20,14 @@ Write the local.conf file for use by devstack
    bash shell variables, and will be ordered so that variables used by
    later entries appear first.
 
+   As a special case, the variable ``LIBS_FROM_GIT`` will be
+   constructed automatically from the projects which appear in the
+   ``required-projects`` list defined by the job.  To instruct
+   devstack to install a library from source rather than pypi, simply
+   add that library to the job's ``required-projects`` list.  To
+   override the automatically-generated value, set ``LIBS_FROM_GIT``
+   in ``devstack_localrc`` to the desired value.
+
 .. zuul:rolevar:: devstack_local_conf
    :type: dict
 
@@ -75,3 +83,7 @@ Write the local.conf file for use by devstack
    A dictionary mapping a plugin name to a git repo location.  If the
    location is a non-empty string, then an ``enable_plugin`` line will
    be emmitted for the plugin name.
+
+   If a plugin declares a dependency on another plugin (via
+   ``plugin_requires`` in the plugin's settings file), this role will
+   automatically emit ``enable_plugin`` lines in the correct order.

roles/write-devstack-local-conf/library/devstack_local_conf.py

@@ -207,17 +207,17 @@ class PluginGraph(DependencyGraph):
 class LocalConf(object):
 
     def __init__(self, localrc, localconf, base_services, services, plugins,
-                 base_dir):
+                 base_dir, projects):
         self.localrc = []
         self.meta_sections = {}
         self.plugin_deps = {}
         self.base_dir = base_dir
+        self.projects = projects
         if plugins:
             self.handle_plugins(plugins)
         if services or base_services:
             self.handle_services(base_services, services or {})
-        if localrc:
+        self.handle_localrc(localrc)
-            self.handle_localrc(localrc)
         if localconf:
             self.handle_localconf(localconf)
 
@@ -241,9 +241,22 @@ class LocalConf(object):
                 self.localrc.append('enable_service {}'.format(k))
 
     def handle_localrc(self, localrc):
-        vg = VarGraph(localrc)
+        lfg = False
-        for k, v in vg.getVars():
+        if localrc:
-            self.localrc.append('{}={}'.format(k, v))
+            vg = VarGraph(localrc)
+            for k, v in vg.getVars():
+                self.localrc.append('{}={}'.format(k, v))
+                if k == 'LIBS_FROM_GIT':
+                    lfg = True
+
+        if not lfg and self.projects:
+            required_projects = []
+            for project_name, project_info in self.projects.items():
+                if project_info.get('required'):
+                    required_projects.append(project_info['short_name'])
+            if required_projects:
+                self.localrc.append('LIBS_FROM_GIT={}'.format(
+                    ','.join(required_projects)))
 
     def handle_localconf(self, localconf):
         for phase, phase_data in localconf.items():
@@ -277,6 +290,7 @@ def main():
             local_conf=dict(type='dict'),
             base_dir=dict(type='path'),
             path=dict(type='str'),
+            projects=dict(type='dict'),
         )
     )
 
@@ -286,7 +300,8 @@ def main():
                    p.get('base_services'),
                    p.get('services'),
                    p.get('plugins'),
-                   p.get('base_dir'))
+                   p.get('base_dir'),
+                   p.get('projects'))
     lc.write(p['path'])
 
     module.exit_json()

roles/write-devstack-local-conf/library/test.py

@@ -56,7 +56,8 @@ class TestDevstackLocalConf(unittest.TestCase):
                        p.get('base_services'),
                        p.get('services'),
                        p.get('plugins'),
-                       p.get('base_dir'))
+                       p.get('base_dir'),
+                       p.get('projects'))
         lc.write(p['path'])
 
         plugins = []
@@ -66,6 +67,7 @@ class TestDevstackLocalConf(unittest.TestCase):
                     plugins.append(line.split()[1])
         self.assertEqual(['bar', 'baz', 'foo'], plugins)
 
+
     def test_plugin_deps(self):
         "Test that plugins with dependencies work"
         os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
@@ -101,20 +103,80 @@ class TestDevstackLocalConf(unittest.TestCase):
                  plugins=plugins,
                  base_dir=self.tmpdir,
                  path=os.path.join(self.tmpdir, 'test.local.conf'))
+
+    def test_libs_from_git(self):
+        "Test that LIBS_FROM_GIT is auto-generated"
+        projects = {
+            'git.openstack.org/openstack/nova': {
+                'required': True,
+                'short_name': 'nova',
+            },
+            'git.openstack.org/openstack/oslo.messaging': {
+                'required': True,
+                'short_name': 'oslo.messaging',
+            },
+            'git.openstack.org/openstack/devstack-plugin': {
+                'required': False,
+                'short_name': 'devstack-plugin',
+            },
+        }
+        p = dict(base_services=[],
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'),
+                 projects=projects)
         lc = LocalConf(p.get('localrc'),
                        p.get('local_conf'),
                        p.get('base_services'),
                        p.get('services'),
                        p.get('plugins'),
-                       p.get('base_dir'))
+                       p.get('base_dir'),
+                       p.get('projects'))
         lc.write(p['path'])
 
-        plugins = []
+        lfg = None
         with open(p['path']) as f:
             for line in f:
-                if line.startswith('enable_plugin'):
+                if line.startswith('LIBS_FROM_GIT'):
-                    plugins.append(line.split()[1])
+                    lfg = line.strip().split('=')[1]
-        self.assertEqual(['foo', 'bar'], plugins)
+        self.assertEqual('nova,oslo.messaging', lfg)
+
+    def test_overridelibs_from_git(self):
+        "Test that LIBS_FROM_GIT can be overridden"
+        localrc = {'LIBS_FROM_GIT': 'oslo.db'}
+        projects = {
+            'git.openstack.org/openstack/nova': {
+                'required': True,
+                'short_name': 'nova',
+            },
+            'git.openstack.org/openstack/oslo.messaging': {
+                'required': True,
+                'short_name': 'oslo.messaging',
+            },
+            'git.openstack.org/openstack/devstack-plugin': {
+                'required': False,
+                'short_name': 'devstack-plugin',
+            },
+        }
+        p = dict(localrc=localrc,
+                 base_services=[],
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'),
+                 projects=projects)
+        lc = LocalConf(p.get('localrc'),
+                       p.get('local_conf'),
+                       p.get('base_services'),
+                       p.get('services'),
+                       p.get('plugins'),
+                       p.get('base_dir'),
+                       p.get('projects'))
+        lc.write(p['path'])
+
+        lfg = None
+        with open(p['path']) as f:
+            for line in f:
+                if line.startswith('LIBS_FROM_GIT'):
+                    lfg = line.strip().split('=')[1]
+        self.assertEqual('oslo.db', lfg)
 
     def test_plugin_circular_deps(self):
         "Test that plugins with circular dependencies fail"

roles/write-devstack-local-conf/tasks/main.yaml

@@ -9,3 +9,4 @@
     localrc: "{{ devstack_localrc|default(omit) }}"
     local_conf: "{{ devstack_local_conf|default(omit) }}"
     base_dir: "{{ devstack_base_dir|default(omit) }}"
+    projects: "{{ zuul.projects }}"

stack.sh

@@ -1398,11 +1398,6 @@ fi
 # Check the status of running services
 service_check
 
-# ensure that all the libraries we think we installed from git,
-# actually were.
-check_libs_from_git
-
-
 # Configure nova cellsv2
 # ----------------------