Set the base set of services

Use the test-matrix role from devstack-gate to define a base set of
services to be enabled for the controller and compute nodes.

Extend the local conf module to handle the base set of services.

Since the test-matrix defines services for primary and subnode nodes, we
need a multinode job to test that this works. Add a new host group
called subnode that includes the non-controller hosts. Add a new job
that runs devstack on a two nodes environment.

Using service from the test matrix enables swift in the gate, so we need
to set SWIFT_HASH for devstack to work.

Depends-on: Ie36ba0cd7cfcd450b75000a76a64d856f2a83eba
Depends-on: Id9ad3be4be25e699f77d6b5a252f046ce8234f45
Change-Id: I379abf482c89122533324e64fefbff3d5a618a89

.zuul.yaml

@@ -23,6 +23,9 @@
         nodes:
           - controller
           - compute1
+      - name: subnode
+        nodes:
+          - compute1
 
 - job:
     name: devstack
@@ -39,9 +42,11 @@
       - openstack/requirements
       - openstack/swift
     roles:
+      - zuul: openstack-infra/devstack-gate
       - zuul: openstack-infra/openstack-zuul-jobs
     timeout: 7200
     vars:
+      test_matrix_configs: ['neutron', 'tlsproxy']
       devstack_localrc:
         DATABASE_PASSWORD: secretdatabase
         RABBIT_PASSWORD: secretrabbit
@@ -57,6 +62,7 @@
         FLOATING_HOST_MASK: 23
         SWIFT_REPLICAS: 1
         SWIFT_START_ALL_SERVICES: false
+        SWIFT_HASH: 1234123412341234
         LOGFILE: /opt/stack/logs/devstacklog.txt
         LOG_COLOR: false
         VERBOSE: true
@@ -75,9 +81,19 @@
     run: playbooks/devstack.yaml
     post-run: playbooks/post.yaml
 
+- job:
+    name: devstack-multinode
+    parent: devstack
+    description: Base devstack multinode job
+    nodeset: openstack-two-node
+    # NOTE(andreaf) The multinode job is useful to see the setup of different
+    # services on different nodes, however the subnode configuration is not
+    # ready yet. Until then this job should stay non-voting.
+    voting: false
 
 - project:
     name: openstack-dev/devstack
     check:
       jobs:
         - devstack
+        - devstack-multinode

playbooks/pre.yaml

@@ -1,3 +1,13 @@
+- hosts: controller
+  roles:
+    - role: test-matrix
+      test_matrix_role: primary 
+
+- hosts: subnode
+  roles:
+    - role: test-matrix
+      test_matrix_role: subnode
+
 - hosts: all
   roles:
     - configure-swap
@@ -8,7 +18,7 @@
     - setup-devstack-cache
     - start-fresh-logging
     - write-devstack-local-conf
-  # TODO(jeblair): remove when configure-mirrors is fixed  
+  # TODO(jeblair): remove when configure-mirrors is fixed
   tasks:
     - name: Hack mirror_info
       shell:

roles/write-devstack-local-conf/README.rst

@@ -47,6 +47,14 @@ Write the local.conf file for use by devstack
             This is a dictionary of key-value pairs which comprise
             this section of the INI file.
 
+.. zuul:rolevar:: devstack_base_services
+   :type: list
+   :default: {{ base_services | default(omit) }}
+
+   A list of base services which are enabled. Services can be added or removed
+   from this list via the ``devstack_services`` variable. This is ignored if
+   ``base`` is set to ``False`` in ``devstack_services``.
+
 .. zuul:rolevar:: devstack_services
    :type: dict
 
@@ -54,11 +62,12 @@ Write the local.conf file for use by devstack
    boolean value is ``false``, a ``disable_service`` line will be
    emitted for the service name.  If it is ``true``, then
    ``enable_service`` will be emitted. All other values are ignored.
+
    The special key ``base`` can be used to enable or disable the base set of
    services enabled by default. If ``base`` is found, it will processed before
    all other keys. If its value is ``False`` a ``disable_all_services`` will be
-   emitted; if its value is ``True`` nothing will be emitted since base
+   emitted; if its value is ``True`` services from ``devstack_base_services``
-   services are enabled by default.
+   will be emitted via ``ENABLED_SERVICES``.
 
 .. zuul:rolevar:: devstack_plugins
    :type: dict

roles/write-devstack-local-conf/defaults/main.yaml

@@ -1,2 +1,3 @@
 devstack_base_dir: /opt/stack
 devstack_local_conf_path: "{{ devstack_base_dir }}/devstack/local.conf"
+devstack_base_services: "{{ enabled_services | default(omit) }}"

roles/write-devstack-local-conf/library/devstack_local_conf.py

@@ -106,13 +106,13 @@ class VarGraph(object):
 
 class LocalConf(object):
 
-    def __init__(self, localrc, localconf, services, plugins):
+    def __init__(self, localrc, localconf, base_services, services, plugins):
         self.localrc = []
         self.meta_sections = {}
         if plugins:
             self.handle_plugins(plugins)
-        if services:
+        if services or base_services:
-            self.handle_services(services)
+            self.handle_services(base_services, services or {})
         if localrc:
             self.handle_localrc(localrc)
         if localconf:
@@ -123,9 +123,12 @@ class LocalConf(object):
             if v:
                 self.localrc.append('enable_plugin {} {}'.format(k, v))
 
-    def handle_services(self, services):
+    def handle_services(self, base_services, services):
-        base_services = services.pop('base', True)
+        enable_base_services = services.pop('base', True)
-        if not base_services:
+        if enable_base_services and base_services:
+            self.localrc.append('ENABLED_SERVICES={}'.format(
+                ",".join(base_services)))
+        else:
             self.localrc.append('disable_all_services')
         for k, v in services.items():
             if v is False:
@@ -164,6 +167,7 @@ def main():
     module = AnsibleModule(
         argument_spec=dict(
             plugins=dict(type='dict'),
+            base_services=dict(type='list'),
             services=dict(type='dict'),
             localrc=dict(type='dict'),
             local_conf=dict(type='dict'),
@@ -174,6 +178,7 @@ def main():
     p = module.params
     lc = LocalConf(p.get('localrc'),
                    p.get('local_conf'),
+                   p.get('base_services'),
                    p.get('services'),
                    p.get('plugins'))
     lc.write(p['path'])

roles/write-devstack-local-conf/tasks/main.yaml

@@ -4,6 +4,7 @@
   devstack_local_conf:
     path: "{{ devstack_local_conf_path }}"
     plugins: "{{ devstack_plugins|default(omit) }}"
+    base_services: "{{ devstack_base_services|default(omit) }}"
     services: "{{ devstack_services|default(omit) }}"
     localrc: "{{ devstack_localrc|default(omit) }}"
     local_conf: "{{ devstack_local_conf|default(omit) }}"