Distributed Keystone for Distributed Cloud

This feature adds the ability to run Keystone in each Subcloud and
implements a Distributed Service Catalog such that the Central Region
Keystone ONLY contains the Identity endpoint for each subcloud.

The DC Manager and DC Orchestration framework then does a 2 stage lookup
to first procure a token from the subcloud and then use that for further
communication with that subcloud.

This delivery adds the following:
- New DC Orch Identity Proxy SM service
- Keystone manifest changes to run init_keystone in Subcloud to spawn a
  local Keystone instance
- Modify System Controller Identity endpoints 5000 to 25000, i.e binding
  to DC-Orch-API-Proxy
- DC Manager and DC Orch Changes to do a 2-stage lookup on subclouds
  (Distributed Service Catalog)
- Cherry pick Endpoint Filter Group patches into Openstack client
- Add Resource Sync for Keystone Users, Projects and Roles and reporting
  to DC Manager
- Add Auditing for Keystone Users, Projects and Roles on Central Region
  and Subclouds
- Lab Setup changes to configure Tenant users and projects against the
  Keystone DC Proxy (port 25000) so that these may be synced to
subclouds.

Story: 2002842
Task: 22785

Change-Id: I2db7610532d1835246b29bedf2cb719669f11935
Signed-off-by: Andy Ning <andy.ning@windriver.com>
Signed-off-by: Jack Ding <jack.ding@windriver.com>

controllerconfig/controllerconfig/controllerconfig/configassistant.py

@@ -3488,12 +3488,6 @@ class ConfigAssistant():
     def apply_bootstrap_manifest(self):
         filename = None
         try:
-            if (self.system_dc_role ==
-                    sysinv_constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER):
-                filename = os.path.join(constants.HIERADATA_WORKDIR,
-                                        'systemcontroller.yaml')
-                utils.create_system_controller_config(filename)
-
             utils.apply_manifest(self.controller_address_0,
                                  sysinv_constants.CONTROLLER,
                                  'bootstrap',

puppet-manifests/src/manifests/controller.pp

@@ -106,11 +106,12 @@ include ::openstack::ironic::api
 
 include ::platform::dcmanager
 include ::platform::dcmanager::manager 
-include ::platform::dcmanager::api
 
 include ::platform::dcorch
 include ::platform::dcorch::engine
 include ::platform::dcorch::api_proxy
+include ::platform::dcmanager::api
+
 include ::platform::dcorch::snmp
 
 include ::platform::sm

puppet-manifests/src/modules/openstack/manifests/keystone.pp

@@ -21,7 +21,15 @@ class openstack::keystone (
 
   include ::platform::params
 
-  if !$::platform::params::region_config {
+  # In the case of a classical Multi-Region deployment, apply the Keystone
+  # controller configuration for Primary Region ONLY
+  # (i.e. on which region_config is False), since Keystone is a Shared service
+  #
+  # In the case of a Distributed Cloud deployment, apply the Keystone
+  # controller configuration for each SubCloud, since Keystone is also
+  # a localized service.
+  if (!$::platform::params::region_config or 
+      $::platform::params::distributed_cloud_role == 'subcloud')  {
     include ::platform::amqp::params
     include ::platform::network::mgmt::params
     include ::platform::drbd::cgcs::params
@@ -167,8 +175,17 @@ class openstack::keystone::bootstrap(
   $eng_workers = $::platform::params::eng_workers
   $bind_host = '0.0.0.0'
   
+  # In the case of a classical Multi-Region deployment, apply the Keystone
+  # controller configuration for Primary Region ONLY
+  # (i.e. on which region_config is False), since Keystone is a Shared service
+  #
+  # In the case of a Distributed Cloud deployment, apply the Keystone
+  # controller configuration for each SubCloud, since Keystone is also
+  # a localized service.
   if ($::platform::params::init_keystone and
-      !$::platform::params::region_config) {
+      (!$::platform::params::region_config or 
+       $::platform::params::distributed_cloud_role == 'subcloud'))  {
+
     include ::keystone::db::postgresql
 
     Class[$name] -> Class['::openstack::client']

puppet-manifests/src/modules/openstack/templates/openrc.admin.erb

@@ -12,9 +12,7 @@ export OS_USER_DOMAIN_NAME=<%= @admin_user_domain %>
 export OS_PROJECT_DOMAIN_NAME=<%= @admin_project_domain %>
 export OS_IDENTITY_API_VERSION=<%= @identity_api_version %>
 export OS_REGION_NAME=<%= @identity_region %>
-<%- if @keystone_identity_region != @identity_region -%>
 export OS_KEYSTONE_REGION_NAME=<%= @keystone_identity_region %>
-<%- end -%>
 export OS_INTERFACE=internal
 
 if [ ! -z "${OS_PASSWORD}" ]; then

puppet-manifests/src/modules/openstack/templates/openrc.ldap.erb

@@ -9,6 +9,4 @@ export OS_USER_DOMAIN_NAME=<%= @admin_user_domain %>
 export OS_PROJECT_DOMAIN_NAME=<%= @admin_project_domain %>
 export OS_IDENTITY_API_VERSION=<%= @identity_api_version %>
 export OS_REGION_NAME=<%= @identity_region %>
-<%- if @keystone_identity_region != @identity_region -%>
 export OS_KEYSTONE_REGION_NAME=<%= @keystone_identity_region %>
-<%- end -%>

puppet-manifests/src/modules/platform/manifests/dcorch.pp

@@ -13,6 +13,7 @@ class platform::dcorch::params (
   $cinder_api_proxy_port = 28776,
   $cinder_enable_ports   = false,
   $patch_api_proxy_port = 25491,
+  $identity_api_proxy_port = 25000,
 ) {
   include ::platform::params
 
@@ -73,6 +74,10 @@ class platform::dcorch::firewall
       service_name => 'dcorch-patch-api-proxy',
       ports => $patch_api_proxy_port,
     }
+    platform::firewall::rule { 'dcorch-identity-api-proxy':
+      service_name => 'dcorch-identity-api-proxy',
+      ports => $identity_api_proxy_port,
+    }
   }
 }
 
@@ -108,6 +113,11 @@ class platform::dcorch::haproxy
       public_port => $patch_api_proxy_port,
       private_port => $patch_api_proxy_port,
     }
+    platform::haproxy::proxy { 'dcorch-identity-api-proxy':
+      server_name => 's-dcorch-identity-api-proxy',
+      public_port => $identity_api_proxy_port,
+      private_port => $identity_api_proxy_port,
+    }
   }
 }
 

puppet-manifests/src/modules/platform/manifests/sm.pp

@@ -379,12 +379,22 @@ class platform::sm
   }
 
   if $region_config {
+      # In a default Multi-Region configuration, Keystone is running as a
+      # shared service in the Primary Region so need to deprovision that
+      # service in all non-Primary Regions. 
+      # However in the case of Distributed Cloud Multi-Region configuration,
+      # each Subcloud is running its own Keystone
+      if $::platform::params::distributed_cloud_role =='subcloud' {
+        $configure_keystone = true
+      } else {
         exec { 'Deprovision OpenStack - Keystone (service-group-member)':
           command => "sm-deprovision service-group-member cloud-services keystone",
         } ->
         exec { 'Deprovision OpenStack - Keystone (service)':
           command => "sm-deprovision service keystone",
         }
+        $configure_keystone = false
+      }
 
       if $glance_region_name != $region_2_name {
         $configure_glance = false
@@ -413,10 +423,14 @@ class platform::sm
          }
       }
   } else {
+      $configure_keystone = true
+      $configure_glance = true
+  }
+
+  if $configure_keystone {
     exec { 'Configure OpenStack - Keystone':
         command => "sm-configure service_instance keystone keystone \"config=/etc/keystone/keystone.conf,user=root,os_username=${os_username},os_project_name=${os_project_name},os_user_domain_name=${os_user_domain_name},os_project_domain_name=${os_project_domain_name},os_auth_url=${os_auth_url}, \"",
     }
-      $configure_glance = true
   }
 
   if $configure_glance {
@@ -1174,6 +1188,12 @@ class platform::sm
     exec { 'Provision DCOrch-Snmp in SM (service dcorch-snmp)':
        command => "sm-provision service dcorch-snmp",
     } ->
+    exec { 'Provision DCOrch-Identity-Api-Proxy (service-group-member dcorch-identity-api-proxy)':
+       command => "sm-provision service-group-member distributed-cloud-services dcorch-identity-api-proxy",
+    } ->
+    exec { 'Provision DCOrch-Identity-Api-Proxy in SM (service dcorch-identity-api-proxy)':
+       command => "sm-provision service dcorch-identity-api-proxy",
+    } ->
     exec { 'Provision DCOrch-Sysinv-Api-Proxy (service-group-member dcorch-sysinv-api-proxy)':
        command => "sm-provision service-group-member distributed-cloud-services dcorch-sysinv-api-proxy",
     } ->
@@ -1210,6 +1230,9 @@ class platform::sm
     exec { 'Configure OpenStack - DCOrch-Snmp':
       command => "sm-configure service_instance dcorch-snmp dcorch-snmp \"\"",
     } ->
+    exec { 'Configure OpenStack - DCOrch-identity-api-proxy':
+      command => "sm-configure service_instance dcorch-identity-api-proxy dcorch-identity-api-proxy \"\"",
+    } ->
     exec { 'Configure OpenStack - DCOrch-sysinv-api-proxy':
       command => "sm-configure service_instance dcorch-sysinv-api-proxy dcorch-sysinv-api-proxy \"\"",
     } ->

puppet-modules-wrs/puppet-dcorch/src/dcorch/manifests/init.pp

@@ -55,6 +55,8 @@ class dcorch (
   $network_remote_port         = 9696,
   $patching_bind_port          = 25491,
   $patching_remote_port        = 5491,
+  $identity_bind_port          = 25000,
+  $identity_remote_port        = 5000,
 ) {
 
   include dcorch::params
@@ -146,6 +148,11 @@ class dcorch (
     'patching/bind_port' : value => $patching_bind_port;
     'patching/remote_host' : value => '0.0.0.0';
     'patching/remote_port' : value => $patching_remote_port;
+
+    'identity/bind_host' : value => $proxy_bind_host;
+    'identity/bind_port' : value => $identity_bind_port;
+    'identity/remote_host' : value => $proxy_remote_host;
+    'identity/remote_port' : value => $identity_remote_port;
   }
 
   dcorch_api_paste_ini {

puppet-modules-wrs/puppet-dcorch/src/dcorch/manifests/keystone/auth.pp

@@ -27,18 +27,22 @@ class dcorch::keystone::auth (
   $public_url                    = 'http://127.0.0.1:8118/v1.0',
   $admin_url                     = 'http://127.0.0.1:8118/v1.0',
   $internal_url                  = 'http://127.0.0.1:8118/v1.0',
+  
   $neutron_proxy_internal_url    = 'http://127.0.0.1:29696',
   $nova_proxy_internal_url       = 'http://127.0.0.1:28774/v2.1',
   $sysinv_proxy_internal_url     = 'http://127.0.0.1:26385/v1',
   $cinder_proxy_internal_url_v2  = 'http://127.0.0.1:28776/v2/%(tenant_id)s',
   $cinder_proxy_internal_url_v3  = 'http://127.0.0.1:28776/v3/%(tenant_id)s',
   $patching_proxy_internal_url   = 'http://127.0.0.1:25491',
+  $identity_proxy_internal_url   = 'http://127.0.0.1:25000/v3',
+
   $neutron_proxy_public_url      = 'http://127.0.0.1:29696',
   $nova_proxy_public_url         = 'http://127.0.0.1:28774/v2.1',
   $sysinv_proxy_public_url       = 'http://127.0.0.1:26385/v1',
   $cinder_proxy_public_url_v2    = 'http://127.0.0.1:28776/v2/%(tenant_id)s',
   $cinder_proxy_public_url_v3    = 'http://127.0.0.1:28776/v3/%(tenant_id)s',
   $patching_proxy_public_url     = 'http://127.0.0.1:25491',
+  $identity_proxy_public_url     = 'http://127.0.0.1:25000/v3',
 ) {
   if $::platform::params::distributed_cloud_role =='systemcontroller' {
     keystone::resource::service_identity { 'dcorch':
@@ -115,5 +119,14 @@ class dcorch::keystone::auth (
       admin_url    =>  $patching_proxy_internal_url,
       internal_url =>  $patching_proxy_internal_url
     }
+    keystone_endpoint { "${region}/keystone::identity" :
+      ensure       =>  "present",
+      name         =>  "keystone",
+      type         =>  "identity",
+      region       =>  $region,
+      public_url   =>  $identity_proxy_public_url,
+      admin_url    =>  $identity_proxy_internal_url,
+      internal_url =>  $identity_proxy_internal_url
+    }
   }
 }

puppet-modules-wrs/puppet-patching/src/patching/manifests/api.pp

@@ -17,6 +17,7 @@ class patching::api (
   $keystone_auth_uri          = false,
   $keystone_auth_version      = false,
   $keystone_identity_uri      = false,
+  $keystone_region_name       = 'RegionOne',
   $auth_type                  = 'password',
   $service_port               = '5000',
   $package_ensure             = 'latest',
@@ -56,6 +57,7 @@ class patching::api (
       'keystone_authtoken/username':         value => $keystone_user;
       'keystone_authtoken/user_domain_name': value => $keystone_user_domain;
       'keystone_authtoken/project_domain_name': value => $keystone_project_domain;
+      'keystone_authtoken/region_name':      value => $keystone_region_name;
       'keystone_authtoken/password':         value => $keystone_password, secret => true;
     }
 

sysinv/sysinv/sysinv/sysinv/puppet/cinder.py

@@ -366,8 +366,10 @@ class CinderPuppet(openstack.OpenstackBasePuppet):
         config_ksuser = True
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
         service_config = None
+        # If we are in Region config and Cinder is a shared service
+        # then don't configure an account for Cinder
         if self._region_config():
-            if self.get_region_name() == self._keystone_region_name():
+            if self.SERVICE_TYPE in self._get_shared_services():
                 service_config = self._get_service_config(self.SERVICE_NAME)
                 config_ksuser = False
             else:

sysinv/sysinv/sysinv/sysinv/puppet/dcorch.py

@@ -31,6 +31,8 @@ class DCOrchPuppet(openstack.OpenstackBasePuppet):
     CINDER_SERVICE_PORT = 28776
     PATCHING_SERVICE_PORT = 25491
     PATCHING_SERVICE_PATH = ''
+    IDENTITY_SERVICE_PORT = 25000
+    IDENTITY_SERVICE_PATH = 'v3'
 
     def get_static_config(self):
         dbuser = self._get_database_username(self.SERVICE_NAME)
@@ -84,6 +86,9 @@ class DCOrchPuppet(openstack.OpenstackBasePuppet):
             'dcorch::keystone::auth::patching_proxy_internal_url':
                 self.get_proxy_internal_url(self.PATCHING_SERVICE_PORT,
                                             self.PATCHING_SERVICE_PATH),
+            'dcorch::keystone::auth::identity_proxy_internal_url':
+                self.get_proxy_internal_url(self.IDENTITY_SERVICE_PORT,
+                                            self.IDENTITY_SERVICE_PATH),
             'dcorch::keystone::auth::neutron_proxy_public_url':
                 self.get_proxy_public_url(self.NETWORKING_SERVICE_PORT,
                                           self.NETWORKING_SERVICE_PATH),
@@ -102,6 +107,9 @@ class DCOrchPuppet(openstack.OpenstackBasePuppet):
             'dcorch::keystone::auth::patching_proxy_public_url':
                 self.get_proxy_public_url(self.PATCHING_SERVICE_PORT,
                                           self.PATCHING_SERVICE_PATH),
+            'dcorch::keystone::auth::identity_proxy_public_url':
+                self.get_proxy_internal_url(self.IDENTITY_SERVICE_PORT,
+                                            self.IDENTITY_SERVICE_PATH),
             'dcorch::keystone::auth::region': self.get_region_name(),
             'dcorch::keystone::auth::auth_name': ksuser,
             'dcorch::keystone::auth::service_name': self.SERVICE_NAME,

sysinv/sysinv/sysinv/sysinv/puppet/keystone.py

@@ -31,6 +31,14 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
 
     DEFAULT_DOMAIN_NAME = 'Default'
 
+    def _region_config(self):
+        # A wrapper over the Base region_config check.
+        if (self._distributed_cloud_role() ==
+                constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD):
+            return False
+        else:
+            return super(KeystonePuppet, self)._region_config()
+
     def get_static_config(self):
         dbuser = self._get_database_username(self.SERVICE_NAME)
         admin_username = self.get_admin_user_name()
@@ -81,7 +89,7 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
             'keystone::endpoint::public_url': self.get_public_url(),
             'keystone::endpoint::internal_url': self.get_internal_url(),
             'keystone::endpoint::admin_url': self.get_admin_url(),
-            'keystone::endpoint::region': self._endpoint_region_name(),
+            'keystone::endpoint::region': self._region_name(),
 
             'keystone::roles::admin::admin': admin_username,
 
@@ -94,9 +102,9 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
             'openstack::client::params::identity_region': self._region_name(),
             'openstack::client::params::identity_auth_url': self.get_auth_url(),
             'openstack::client::params::keystone_identity_region':
-                self.get_region_name(),
+                self._identity_specific_region_name(),
-            'openstack::client::params::auth_region': self.get_region_name(),
+            'openstack::client::params::auth_region':
-
+                self._identity_specific_region_name(),
             'openstack::keystone::params::api_version': self.SERVICE_PATH,
             'openstack::keystone::params::identity_uri':
                 self.get_identity_uri(),
@@ -107,7 +115,8 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
             # The region in which the identity server can be found
             # and it could be different than the region where the
             # system resides
-            'openstack::keystone::params::region_name': self.get_region_name(),
+            'openstack::keystone::params::region_name':
+                self._identity_specific_region_name(),
             'openstack::keystone::params::service_create':
                 self._to_create_services(),
 
@@ -125,11 +134,12 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
         admin_password = self._get_keyring_password(self.ADMIN_SERVICE,
                                                     self.ADMIN_USER)
         db_connection = self._format_database_connection(self.SERVICE_NAME)
-        return {
+        config = {
             'keystone::admin_password': admin_password,
             'keystone::roles::admin::password': admin_password,
             'keystone::database_connection': db_connection,
         }
+        return config
 
     def _get_service_parameter_config(self):
         service_parameters = self._get_service_parameter_configs(
@@ -233,10 +243,14 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
                 pass
         return password_rule
 
-    def _endpoint_region_name(self):
+    def _identity_specific_region_name(self):
-        if (self._distributed_cloud_role() ==
+        """
-                constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER):
+        Returns the Identity Region name based on the System mode:
-            return constants.SYSTEM_CONTROLLER_REGION
+            If Multi-Region then Keystone is shared: return Primary Region
+            Else: Local Region
+        """
+        if (self._region_config()):
+            return self.get_region_name()
         else:
             return self._region_name()
 

sysinv/sysinv/sysinv/sysinv/puppet/openstack.py

@@ -139,7 +139,7 @@ class OpenstackBasePuppet(base.BasePuppet):
         return self._operator.keystone.get_identity_uri()
 
     def _keystone_region_name(self):
-        return self._operator.keystone.get_region_name()
+        return self._operator.keystone._identity_specific_region_name()
 
     def _get_service_region_name(self, service):
         if self._region_config():

sysinv/sysinv/sysinv/sysinv/puppet/patching.py

@@ -49,6 +49,8 @@ class PatchingPuppet(openstack.OpenstackBasePuppet):
             'patching::api::keystone_tenant': self._get_service_tenant_name(),
             'patching::api::keystone_auth_uri': patch_keystone_auth_uri,
             'patching::api::keystone_identity_uri': patch_keystone_identity_uri,
+            'patching::api::keystone_region_name':
+                self._get_service_region_name(self.SERVICE_NAME),
 
             'patching::api::keystone_user_domain':
                 self._get_service_user_domain_name(),