Configure dcmanager user for endpoint_cache

The following changes are made, in order to remove the
dependencies to 'admin' user and use 'dcmanager' user to
authenticate with services in the subclouds:
. Add endpoint_cache section to both dcorch and dcmanager
. Configure dcmanager user in the endpoint_cache section
. Sync the dcmanager user to subclouds
. Use 'dcmanager' user for subcloud authentication by default,
  and it falls back to 'admin' user if the authentication fails
 (This is required during software upgrade).

Depends-On: https://review.opendev.org/#/c/735994/
Partial-Bug: 1883758

Change-Id: Ibc78abc86a7a825f83f2cac9fd54e4183e7ccd80
Signed-off-by: Tao Liu <tao.liu@windriver.com>

distributedcloud/dccommon/consts.py

@@ -40,3 +40,4 @@ USER_HEADER = {'User-Header': USER_HEADER_VALUE}
 ADMIN_USER_NAME = "admin"
 ADMIN_PROJECT_NAME = "admin"
 SYSINV_USER_NAME = "sysinv"
+DCMANAGER_USER_NAME = "dcmanager"

distributedcloud/dccommon/endpoint_cache.py

@@ -23,6 +23,7 @@
 import collections
 import threading
 
+from keystoneauth1 import exceptions as keystone_exceptions
 from keystoneauth1 import loading
 from keystoneauth1 import session
 
@@ -33,6 +34,8 @@ from oslo_log import log as logging
 
 from dccommon import consts
 
+CONF = cfg.CONF
+
 LOG = logging.getLogger(__name__)
 
 
@@ -51,29 +54,21 @@ class EndpointCache(object):
         if auth_url:
             self.external_auth_url = auth_url
         else:
-            self.external_auth_url = cfg.CONF.cache.auth_uri
+            self.external_auth_url = CONF.endpoint_cache.auth_uri
 
         self._initialize_keystone_client(region_name, auth_url)
 
         self._update_endpoints()
 
     def _initialize_keystone_client(self, region_name=None, auth_url=None):
-        with EndpointCache.plugin_lock:
+        self.admin_session = EndpointCache.get_admin_session(
-            if EndpointCache.plugin_loader is None:
+            self.external_auth_url,
-                EndpointCache.plugin_loader = loading.get_plugin_loader(
+            CONF.endpoint_cache.username,
-                    cfg.CONF.keystone_authtoken.auth_type)
+            CONF.endpoint_cache.user_domain_name,
+            CONF.endpoint_cache.password,
+            CONF.endpoint_cache.project_name,
+            CONF.endpoint_cache.project_domain_name)
 
-        auth = EndpointCache.plugin_loader.load_from_options(
-            auth_url=self.external_auth_url,
-            username=cfg.CONF.cache.admin_username,
-            user_domain_name=cfg.CONF.cache.admin_user_domain_name,
-            password=cfg.CONF.cache.admin_password,
-            project_name=cfg.CONF.cache.admin_tenant,
-            project_domain_name=cfg.CONF.cache.admin_project_domain_name,
-        )
-        self.admin_session = session.Session(
-            auth=auth, additional_headers=consts.USER_HEADER,
-            timeout=cfg.CONF.keystone_authtoken.http_connect_timeout)
         self.keystone_client = keystone_client.Client(
             session=self.admin_session,
             region_name=consts.CLOUD_0)
@@ -98,24 +93,75 @@ class EndpointCache(object):
                 LOG.error("Cannot find identity auth_url for %s", region_name)
                 raise
 
-            # We assume that the Admin user names and passwords are the same
+            # We assume that the dcmanager user names and passwords are the
-            # on this subcloud since this is an audited resource
+            # same on this subcloud since this is an audited resource
-            sc_auth = EndpointCache.plugin_loader.load_from_options(
+            self.admin_session = EndpointCache.get_admin_session(
-                auth_url=sc_auth_url,
+                sc_auth_url,
-                username=cfg.CONF.cache.admin_username,
+                CONF.endpoint_cache.username,
-                user_domain_name=cfg.CONF.cache.admin_user_domain_name,
+                CONF.endpoint_cache.user_domain_name,
-                password=cfg.CONF.cache.admin_password,
+                CONF.endpoint_cache.password,
-                project_name=cfg.CONF.cache.admin_tenant,
+                CONF.endpoint_cache.project_name,
-                project_domain_name=cfg.CONF.cache.admin_project_domain_name,
+                CONF.endpoint_cache.project_domain_name)
-            )
+            # check if the current session is valid and get an admin session
-            self.admin_session = session.Session(
+            # if necessary
-                auth=sc_auth, additional_headers=consts.USER_HEADER,
+            self.admin_session = EndpointCache.get_admin_backup_session(
-                timeout=cfg.CONF.keystone_authtoken.http_connect_timeout)
+                self.admin_session, CONF.endpoint_cache.username, sc_auth_url)
+
             self.keystone_client = keystone_client.Client(
                 session=self.admin_session,
                 region_name=region_name)
             self.external_auth_url = sc_auth_url
 
+    @classmethod
+    def get_admin_session(cls, auth_url, user_name, user_domain_name,
+                          user_password, user_project, user_project_domain,
+                          timeout=None):
+        with EndpointCache.plugin_lock:
+            if EndpointCache.plugin_loader is None:
+                EndpointCache.plugin_loader = loading.get_plugin_loader(
+                    CONF.endpoint_cache.auth_plugin)
+
+        user_auth = EndpointCache.plugin_loader.load_from_options(
+            auth_url=auth_url,
+            username=user_name,
+            user_domain_name=user_domain_name,
+            password=user_password,
+            project_name=user_project,
+            project_domain_name=user_project_domain,
+        )
+        timeout = (CONF.endpoint_cache.http_connect_timeout if timeout is None
+                   else timeout)
+        return session.Session(
+            auth=user_auth, additional_headers=consts.USER_HEADER,
+            timeout=timeout)
+
+    @classmethod
+    def get_admin_backup_session(cls, admin_session, user_name, auth_url):
+        """Validate a session and open an admin session if it fails.
+
+        This method is require to handle an upgrade to stx 4.0 and it
+        can be removed in stx 5.0.
+
+        """
+
+        try:
+            admin_session.get_auth_headers()
+        except keystone_exceptions.Unauthorized:
+            # this will only happen briefly during an upgrade to stx 4.0
+            # just until the dcorch has synced the dcmanager user to each
+            # subcloud
+            LOG.info("Failed to authenticate user:%s, use %s user instead"
+                     % (user_name,
+                        CONF.cache.admin_username))
+            admin_session = EndpointCache.get_admin_session(
+                auth_url,
+                CONF.cache.admin_username,
+                CONF.cache.admin_user_domain_name,
+                CONF.cache.admin_password,
+                CONF.cache.admin_tenant,
+                CONF.cache.admin_project_domain_name)
+        return admin_session
+
     @staticmethod
     def _is_central_cloud(region_id):
         central_cloud_regions = [consts.CLOUD_0, consts.VIRTUAL_MASTER_CLOUD]

distributedcloud/dccommon/tests/unit/test_endpoint_cache.py

@@ -48,7 +48,7 @@ class EndpointCacheTest(base.DCCommonTestCase):
         auth_uri_opts = [
             cfg.StrOpt('auth_uri',
                        default="fake_auth_uri")]
-        cfg.CONF.register_opts(auth_uri_opts, 'cache')
+        cfg.CONF.register_opts(auth_uri_opts, 'endpoint_cache')
 
     @patch.object(endpoint_cache.EndpointCache, '_initialize_keystone_client')
     @patch.object(endpoint_cache.EndpointCache, '_get_endpoint_from_keystone')

distributedcloud/dcmanager/common/config.py

@@ -82,7 +82,7 @@ pecan_opts = [
 ]
 
 
-# OpenStack credentials used for Endpoint Cache
+# OpenStack admin user credentials used for Endpoint Cache
 cache_opts = [
     cfg.StrOpt('auth_uri',
                help='Keystone authorization url'),
@@ -107,6 +107,29 @@ cache_opts = [
                     ' auto_refresh_endpoint set to True')
 ]
 
+# OpenStack credentials used for Endpoint Cache
+endpoint_cache_opts = [
+    cfg.StrOpt('auth_uri',
+               help='Keystone authorization url'),
+    cfg.StrOpt('auth_plugin',
+               help='Name of the plugin to load'),
+    cfg.StrOpt('username',
+               help='Username of account'),
+    cfg.StrOpt('password',
+               help='Password of account'),
+    cfg.StrOpt('project_name',
+               help='Project name of account'),
+    cfg.StrOpt('user_domain_name',
+               default='Default',
+               help='User domain name of account'),
+    cfg.StrOpt('project_domain_name',
+               default='Default',
+               help='Project domain name of account'),
+    cfg.IntOpt('http_connect_timeout',
+               help='Request timeout value for communicating with Identity'
+                    ' API server.'),
+]
+
 scheduler_opts = [
     cfg.BoolOpt('periodic_enable',
                 default=True,
@@ -138,11 +161,15 @@ pecan_group = cfg.OptGroup(name='pecan',
                            title='Pecan options')
 
 cache_opt_group = cfg.OptGroup(name='cache',
-                               title='OpenStack Credentials')
+                               title='OpenStack Admin Credentials')
+
+endpoint_cache_opt_group = cfg.OptGroup(name='endpoint_cache',
+                                        title='OpenStack Credentials')
 
 
 def list_opts():
     yield cache_opt_group.name, cache_opts
+    yield endpoint_cache_opt_group.name, endpoint_cache_opts
     yield scheduler_opt_group.name, scheduler_opts
     yield pecan_group.name, pecan_opts
     yield None, global_opts

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -76,7 +76,8 @@ USERS_TO_REPLICATE = [
     'vim',
     'mtce',
     'fm',
-    'barbican']
+    'barbican',
+    'dcmanager']
 
 SERVICES_USER = 'services'
 
@@ -289,8 +290,8 @@ class SubcloudManager(manager.Manager):
             self._create_addn_hosts_dc(context)
 
             # Query system controller keystone admin user/project IDs,
-            # services project id and sysinv user id and store in payload so
+            # services project id, sysinv and dcmanager user id and store in
-            # they get copied to the override file
+            # payload so they get copied to the override file
             admin_user = m_ks_client.get_user_by_name(
                 dccommon_consts.ADMIN_USER_NAME)
             admin_project = m_ks_client.get_project_by_name(
@@ -298,6 +299,8 @@ class SubcloudManager(manager.Manager):
             services_project = m_ks_client.get_project_by_name(SERVICES_USER)
             sysinv_user = m_ks_client.get_user_by_name(
                 dccommon_consts.SYSINV_USER_NAME)
+            dcmanager_user = m_ks_client.get_user_by_name(
+                dccommon_consts.DCMANAGER_USER_NAME)
             payload['system_controller_keystone_admin_user_id'] = \
                 admin_user.id
             payload['system_controller_keystone_admin_project_id'] = \
@@ -306,6 +309,8 @@ class SubcloudManager(manager.Manager):
                 services_project.id
             payload['system_controller_keystone_sysinv_user_id'] = \
                 sysinv_user.id
+            payload['system_controller_keystone_dcmanager_user_id'] = \
+                dcmanager_user.id
 
             # Add the admin and service user passwords to the payload so they
             # get copied to the override file

distributedcloud/dcorch/common/config.py

@@ -133,7 +133,7 @@ cinder_quotas = [
                     'for backups per project.')
 ]
 
-# OpenStack credentials used for Endpoint Cache
+# OpenStack admin user credentials used for Endpoint Cache
 cache_opts = [
     cfg.StrOpt('auth_uri',
                help='Keystone authorization url'),
@@ -158,6 +158,29 @@ cache_opts = [
                     ' auto_refresh_endpoint set to True')
 ]
 
+# OpenStack credentials used for Endpoint Cache
+endpoint_cache_opts = [
+    cfg.StrOpt('auth_uri',
+               help='Keystone authorization url'),
+    cfg.StrOpt('auth_plugin',
+               help='Name of the plugin to load'),
+    cfg.StrOpt('username',
+               help='Username of account'),
+    cfg.StrOpt('password',
+               help='Password of account'),
+    cfg.StrOpt('project_name',
+               help='Project name of account'),
+    cfg.StrOpt('user_domain_name',
+               default='Default',
+               help='User domain name of account'),
+    cfg.StrOpt('project_domain_name',
+               default='Default',
+               help='Project domain name of account'),
+    cfg.IntOpt('http_connect_timeout',
+               help='Request timeout value for communicating with Identity'
+                    ' API server.'),
+]
+
 scheduler_opts = [
     cfg.BoolOpt('periodic_enable',
                 default=True,
@@ -195,7 +218,10 @@ pecan_group = cfg.OptGroup(name='pecan',
                            title='Pecan options')
 
 cache_opt_group = cfg.OptGroup(name='cache',
-                               title='OpenStack Credentials')
+                               title='OpenStack Admin Credentials')
+
+endpoint_cache_opt_group = cfg.OptGroup(name='endpoint_cache',
+                                        title='OpenStack Credentials')
 
 openstack_cache_opt_group = cfg.OptGroup(name='openstack_cache',
                                          title='Containerized OpenStack'
@@ -210,6 +236,7 @@ def list_opts():
     yield default_quota_group.name, neutron_quotas
     yield default_quota_group.name, cinder_quotas
     yield cache_opt_group.name, cache_opts
+    yield endpoint_cache_opt_group.name, endpoint_cache_opts
     yield openstack_cache_opt_group.name, cache_opts
     yield scheduler_opt_group.name, scheduler_opts
     yield pecan_group.name, pecan_opts

distributedcloud/dcorch/engine/sync_services/identity.py

@@ -74,7 +74,7 @@ class IdentitySyncThread(SyncThread):
         # resources
         self.filtered_audit_resources = {
             consts.RESOURCE_TYPE_IDENTITY_USERS:
-                ['dcdbsync', 'dcorch', 'dcmanager', 'heat_admin', 'smapi',
+                ['dcdbsync', 'dcorch', 'heat_admin', 'smapi',
                  'fm', 'cinder' + self.subcloud_engine.subcloud.region_name],
             consts.RESOURCE_TYPE_IDENTITY_ROLES:
                 ['heat_stack_owner', 'heat_stack_user', 'ResellerAdmin'],
@@ -157,10 +157,10 @@ class IdentitySyncThread(SyncThread):
                  extra=self.log_extra)
 
     def _initial_sync_users(self, m_users, sc_users):
-        # Particularly sync users with same name but different ID.  admin and
+        # Particularly sync users with same name but different ID.  admin,
-        # sysinv users are special cases as the id's will match (as this is
+        # sysinv, and dcmanager users are special cases as the id's will match
-        # forced during the subcloud deploy) but the details will not so we
+        # (as this is forced during the subcloud deploy) but the details will
-        # still need to sync them here.
+        # not so we still need to sync them here.
         m_client = self.m_dbs_client.identity_manager
         sc_client = self.sc_dbs_client.identity_manager
 
@@ -171,7 +171,8 @@ class IdentitySyncThread(SyncThread):
                         (m_user.id != sc_user.id or
                          sc_user.local_user.name in
                          [dccommon_consts.ADMIN_USER_NAME,
-                          dccommon_consts.SYSINV_USER_NAME])):
+                          dccommon_consts.SYSINV_USER_NAME,
+                          dccommon_consts.DCMANAGER_USER_NAME])):
                     user_records = m_client.user_detail(m_user.id)
                     if not user_records:
                         LOG.error("No data retrieved from master cloud for"

distributedcloud/dcorch/engine/sync_thread.py

@@ -22,6 +22,7 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 
 from dccommon import consts as dccommon_consts
+from dccommon.endpoint_cache import EndpointCache
 from dcdbsync.dbsyncclient import client as dbsyncclient
 from dcmanager.common import consts as dcmanager_consts
 from dcmanager.rpc import client as dcmanager_rpc_client
@@ -33,8 +34,6 @@ from dcorch.objects import orchrequest
 from dcorch.objects import resource
 from dcorch.objects import subcloud_resource
 
-from keystoneauth1 import loading
-from keystoneauth1 import session
 from keystoneclient import client as keystoneclient
 
 
@@ -125,29 +124,31 @@ class SyncThread(object):
     def initialize(self):
         # base implementation of initializing the master client.
         # The specific SyncThread subclasses may extend this.
-        loader = loading.get_plugin_loader(
-            cfg.CONF.keystone_authtoken.auth_type)
 
-        config = None
         if self.endpoint_type in consts.ENDPOINT_TYPES_LIST:
-            config = cfg.CONF.cache
+            config = cfg.CONF.endpoint_cache
+            self.admin_session = EndpointCache.get_admin_session(
+                config.auth_uri,
+                config.username,
+                config.user_domain_name,
+                config.password,
+                config.project_name,
+                config.project_domain_name,
+                timeout=60)
         elif self.endpoint_type in dccommon_consts.ENDPOINT_TYPES_LIST_OS:
             config = cfg.CONF.openstack_cache
+            self.admin_session = EndpointCache.get_admin_session(
+                config.auth_uri,
+                config.admin_username,
+                config.admin_user_domain_name,
+                config.admin_password,
+                config.admin_tenant,
+                config.admin_project_domain_name,
+                timeout=60)
         else:
             raise exceptions.EndpointNotSupported(
                 endpoint=self.endpoint_type)
 
-        auth = loader.load_from_options(
-            auth_url=config.auth_uri,
-            username=config.admin_username,
-            password=config.admin_password,
-            project_name=config.admin_tenant,
-            project_domain_name=config.admin_project_domain_name,
-            user_domain_name=config.admin_user_domain_name)
-        self.admin_session = session.Session(
-            auth=auth, timeout=60,
-            additional_headers=dccommon_consts.USER_HEADER)
-
         # keystone client
         self.ks_client = keystoneclient.Client(
             session=self.admin_session,
@@ -180,26 +181,31 @@ class SyncThread(object):
                          extra=self.log_extra)
                 return
 
-            loader = loading.get_plugin_loader(
-                cfg.CONF.keystone_authtoken.auth_type)
-
             config = None
             if self.endpoint_type in consts.ENDPOINT_TYPES_LIST:
-                config = cfg.CONF.cache
+                config = cfg.CONF.endpoint_cache
+                self.sc_admin_session = EndpointCache.get_admin_session(
+                    sc_auth_url,
+                    config.username,
+                    config.user_domain_name,
+                    config.password,
+                    config.project_name,
+                    config.project_domain_name,
+                    timeout=60)
             elif self.endpoint_type in dccommon_consts.ENDPOINT_TYPES_LIST_OS:
                 config = cfg.CONF.openstack_cache
+                self.sc_admin_session = EndpointCache.get_admin_session(
+                    sc_auth_url,
+                    config.admin_username,
+                    config.admin_user_domain_name,
+                    config.admin_password,
+                    config.admin_tenant,
+                    config.admin_project_domain_name,
+                    timeout=60)
 
-            sc_auth = loader.load_from_options(
+            if config is cfg.CONF.endpoint_cache:
-                auth_url=sc_auth_url,
+                self.sc_admin_session = EndpointCache.get_admin_backup_session(
-                username=config.admin_username,
+                    self.sc_admin_session, config.username, sc_auth_url)
-                password=config.admin_password,
-                project_name=config.admin_tenant,
-                project_domain_name=config.admin_project_domain_name,
-                user_domain_name=config.admin_user_domain_name)
-
-            self.sc_admin_session = session.Session(
-                auth=sc_auth, timeout=60,
-                additional_headers=dccommon_consts.USER_HEADER)
 
     def initial_sync(self):
         # Return True to indicate initial sync success