starlingx/distcloud
Code Issues Proposed changes

Merge "Support identity groups in DC"

Browse Source
This commit is contained in:
Zuul
2021-09-24 20:38:57 +00:00
committed by Gerrit Code Review
parent f4799c1cd2 8ed5018d8b
commit 1ce3dbc8a5
12 changed files with 831 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
distributedcloud/dcdbsync/api/controllers/v1/identity/identity.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -131,3 +131,99 @@ class UsersController(object):
except Exception as e:
LOG.exception(e)
pecan.abort(500, _('Unable to update user'))
class GroupsController(object):
VERSION_ALIASES = {
'Stein': '1.0',
}
def __init__(self):
super(GroupsController, self).__init__()
# to do the version compatibility for future purpose
def _determine_version_cap(self, target):
version_cap = 1.0
return version_cap
@expose(generic=True, template='json')
def index(self):
# Route the request to specific methods with parameters
pass
@index.when(method='GET', template='json')
def get(self, group_ref=None):
"""Get a list of groups."""
context = restcomm.extract_context_from_environ()
try:
if group_ref is None:
return db_api.group_get_all(context)
else:
group = db_api.group_get(context, group_ref)
return group
except exceptions.GroupNotFound as e:
pecan.abort(404, _("Group not found: %s") % e)
except Exception as e:
LOG.exception(e)
pecan.abort(500, _('Unable to get group'))
@index.when(method='POST', template='json')
def post(self):
"""Create a new group."""
context = restcomm.extract_context_from_environ()
# Convert JSON string in request to Python dict
try:
payload = json.loads(request.body)
except ValueError:
pecan.abort(400, _('Request body decoding error'))
if not payload:
pecan.abort(400, _('Body required'))
group_name = payload.get('group').get('name')
if not group_name:
pecan.abort(400, _('Group name required'))
try:
# Insert the group into DB tables
group_ref = db_api.group_create(context, payload)
response.status = 201
return (group_ref)
except Exception as e:
LOG.exception(e)
pecan.abort(500, _('Unable to create group'))
@index.when(method='PUT', template='json')
def put(self, group_ref=None):
"""Update a existing group."""
context = restcomm.extract_context_from_environ()
if group_ref is None:
pecan.abort(400, _('Group ID required'))
# Convert JSON string in request to Python dict
try:
payload = json.loads(request.body)
except ValueError:
pecan.abort(400, _('Request body decoding error'))
if not payload:
pecan.abort(400, _('Body required'))
try:
# Update the group in DB tables
return db_api.group_update(context, group_ref, payload)
except exceptions.GroupNotFound as e:
pecan.abort(404, _("Group not found: %s") % e)
except Exception as e:
LOG.exception(e)
pecan.abort(500, _('Unable to update group'))

3
distributedcloud/dcdbsync/api/controllers/v1/identity/root.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -42,6 +42,7 @@ class IdentityController(object):
res_controllers = dict()
res_controllers["users"] = identity.UsersController
res_controllers["groups"] = identity.GroupsController
res_controllers["projects"] = project.ProjectsController
res_controllers["roles"] = role.RolesController
res_controllers["token-revocation-events"] = \

6
distributedcloud/dcdbsync/common/exceptions.py
View File

@@ -14,7 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -79,6 +79,10 @@ class UserNotFound(NotFound):
message = _("User with id %(user_id)s doesn't exist.")
class GroupNotFound(NotFound):
message = _("Group with id %(group_id)s doesn't exist.")
class ProjectNotFound(NotFound):
message = _("Project with id %(project_id)s doesn't exist.")

28
distributedcloud/dcdbsync/db/identity/api.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -69,6 +69,32 @@ def user_update(context, user_ref, payload):
return IMPL.user_update(context, user_ref, payload)
###################
# group db methods
###################
def group_get_all(context):
"""Retrieve all groups."""
return IMPL.group_get_all(context)
def group_get(context, group_id):
"""Retrieve details of a group."""
return IMPL.group_get(context, group_id)
def group_create(context, payload):
"""Create a group."""
return IMPL.group_create(context, payload)
def group_update(context, group_ref, payload):
"""Update a group"""
return IMPL.group_update(context, group_ref, payload)
###################
# project db methods

125
distributedcloud/dcdbsync/db/identity/sqlalchemy/api.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2019-2020 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -230,8 +230,9 @@ def user_get_all(context):
user_passwords = {'password': [password for password in passwords
if password['local_user_id'] ==
local_user['id']]}
user_consolidated = dict({'local_user': local_user}.items() +
user.items() + user_passwords.items())
user_consolidated = dict(list({'local_user': local_user}.items()) +
list(user.items()) +
list(user_passwords.items()))
result.append(user_consolidated)
return result
@@ -328,15 +329,131 @@ def user_update(context, user_id, payload):
updated_local_users[0]['id']
insert(conn, table, password)
# Need to update the actor_id in assignment and system_assignment
# tables if the user id is updated
# along with the user_id in user_group_membership tables if the
# user id is updated
if user_id != new_user_id:
assignment = {'actor_id': new_user_id}
user_group_membership = {'user_id': new_user_id}
update(conn, 'assignment', 'actor_id', user_id, assignment)
update(conn, 'system_assignment', 'actor_id', user_id, assignment)
update(conn, 'user_group_membership', 'user_id', user_id, user_group_membership)
return user_get(context, new_user_id)
###################
# identity groups
###################
@require_context
def group_get_all(context):
result = []
with get_read_connection() as conn:
# groups table
groups = query(conn, 'group')
# user_group_membership table
user_group_memberships = query(conn, 'user_group_membership')
for group in groups:
local_user_id_list = [membership['user_id'] for membership
in user_group_memberships if
membership['group_id'] == group['id']]
local_user_id_list.sort()
local_user_ids = {'local_user_ids': local_user_id_list}
group_consolidated = dict(list({'group': group}.items()) +
list(local_user_ids.items()))
result.append(group_consolidated)
return result
@require_context
def group_get(context, group_id):
result = {}
with get_read_connection() as conn:
local_user_id_list = []
# group table
group = query(conn, 'group', 'id', group_id)
if not group:
raise exception.GroupNotFound(group_id=group_id)
result['group'] = group[0]
# user_group_membership table
user_group_memberships = query(conn, 'user_group_membership', 'group_id', group_id)
for user_group_membership in user_group_memberships:
local_user = query(conn, 'local_user', 'user_id', user_group_membership.get('user_id'))
if not local_user:
raise exception.UserNotFound(user_id=user_group_membership.get('user_id'))
local_user_id_list.append(local_user[0]['user_id'])
result['local_user_ids'] = local_user_id_list
return result
@require_admin_context
def group_create(context, payload):
group = payload['group']
local_user_ids = payload['local_user_ids']
with get_write_connection() as conn:
insert(conn, 'group', group)
for local_user_id in local_user_ids:
user_group_membership = {'user_id': local_user_id, 'group_id': group['id']}
insert(conn, 'user_group_membership', user_group_membership)
return group_get(context, payload['group']['id'])
@require_admin_context
def group_update(context, group_id, payload):
with get_write_connection() as conn:
new_group_id = group_id
if 'group' in payload and 'local_user_ids' in payload:
group = payload['group']
new_group_id = group.get('id')
# local_user_id_list is a sorted list of user IDs that
# belong to this group
local_user_id_list = payload['local_user_ids']
user_group_memberships = query(conn, 'user_group_membership',
'group_id', group_id)
existing_user_list = [user_group_membership['user_id'] for user_group_membership
in user_group_memberships]
existing_user_list.sort()
deleted = False
if (group_id != new_group_id) or (local_user_id_list != existing_user_list):
# Foreign key constraint exists on 'group_id' of user_group_membership table
# and 'id' of group table. So delete user group membership records before
# updating group if groups IDs are different
# Alternatively, if there is a discrepency in the user group memberships,
# delete and re-create them
delete(conn, 'user_group_membership', 'group_id', group_id)
deleted = True
# Update group table
update(conn, 'group', 'id', group_id, group)
if deleted:
for local_user_id in local_user_id_list:
item = {'user_id': local_user_id, 'group_id': new_group_id}
insert(conn, 'user_group_membership', item)
# Need to update the actor_id in assignment and system_assignment
# tables if the group id is updated
if group_id != new_group_id:
assignment = {'actor_id': new_group_id}
update(conn, 'assignment', 'actor_id', group_id, assignment)
update(conn, 'system_assignment', 'actor_id', group_id, assignment)
return group_get(context, new_group_id)
###################
# identity projects

8
distributedcloud/dcdbsync/dbsyncclient/v1/client.py
View File

@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -23,7 +23,8 @@ import keystoneauth1.identity.generic as auth_plugin
from keystoneauth1 import session as ks_session
from dcdbsync.dbsyncclient import httpclient
from dcdbsync.dbsyncclient.v1.identity import identity_manager as im
from dcdbsync.dbsyncclient.v1.identity import identity_group_manager as igm
from dcdbsync.dbsyncclient.v1.identity import identity_user_manager as ium
from dcdbsync.dbsyncclient.v1.identity import project_manager as pm
from dcdbsync.dbsyncclient.v1.identity import role_manager as rm
from dcdbsync.dbsyncclient.v1.identity \
@@ -97,7 +98,8 @@ class Client(object):
)
# Create all managers
self.identity_manager = im.identity_manager(self.http_client)
self.identity_user_manager = ium.identity_user_manager(self.http_client)
self.identity_group_manager = igm.identity_group_manager(self.http_client)
self.project_manager = pm.project_manager(self.http_client)
self.role_manager = rm.role_manager(self.http_client)
self.revoke_event_manager = trem.revoke_event_manager(self.http_client)

133
distributedcloud/dcdbsync/dbsyncclient/v1/identity/identity_group_manager.py Normal file
View File

@@ -0,0 +1,133 @@
# Copyright (c) 2017 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from dcdbsync.dbsyncclient import base
from dcdbsync.dbsyncclient.base import get_json
from dcdbsync.dbsyncclient import exceptions
class Group(base.Resource):
resource_name = 'group'
def __init__(self, manager, id, domain_id, name,
description, local_user_ids, extra={}):
self.manager = manager
self.id = id
self.domain_id = domain_id
self.name = name
self.description = description
self.local_user_ids = local_user_ids
self.extra = extra
def info(self):
resource_info = dict()
resource_info.update({self.resource_name:
{'name': self.name,
'id': self.id,
'domain_id': self.domain_id}})
return resource_info
class identity_group_manager(base.ResourceManager):
resource_class = Group
def group_create(self, url, data):
resp = self.http_client.post(url, data)
# Unauthorized request
if resp.status_code == 401:
raise exceptions.Unauthorized('Unauthorized request.')
if resp.status_code != 201:
self._raise_api_exception(resp)
# Converted into python dict
json_object = get_json(resp)
return json_object
def group_list(self, url):
resp = self.http_client.get(url)
# Unauthorized
if resp.status_code == 401:
raise exceptions.Unauthorized('Unauthorized request')
if resp.status_code != 200:
self._raise_api_exception(resp)
# Converted into python dict
json_objects = get_json(resp)
groups = []
for json_object in json_objects:
group = Group(
self,
id=json_object['group']['id'],
domain_id=json_object['group']['domain_id'],
name=json_object['group']['name'],
extra=json_object['group']['extra'],
description=json_object['group']['description'],
local_user_ids=json_object['local_user_ids'])
groups.append(group)
return groups
def _group_detail(self, url):
resp = self.http_client.get(url)
# Unauthorized request
if resp.status_code == 401:
raise exceptions.Unauthorized('Unauthorized request.')
if resp.status_code != 200:
self._raise_api_exception(resp)
# Return group details in original json format,
# ie, without convert it into python dict
return resp.content
def _group_update(self, url, data):
resp = self.http_client.put(url, data)
# Unauthorized request
if resp.status_code == 401:
raise exceptions.Unauthorized('Unauthorized request.')
if resp.status_code != 200:
self._raise_api_exception(resp)
# Converted into python dict
json_object = get_json(resp)
return json_object
def add_group(self, data):
url = '/identity/groups/'
return self.group_create(url, data)
def list_groups(self):
url = '/identity/groups/'
return self.group_list(url)
def group_detail(self, group_ref):
url = '/identity/groups/%s' % group_ref
return self._group_detail(url)
def update_group(self, group_ref, data):
url = '/identity/groups/%s' % group_ref
return self._group_update(url, data)

2
distributedcloud/dcdbsync/dbsyncclient/v1/identity/identity_manager.py → distributedcloud/dcdbsync/dbsyncclient/v1/identity/identity_user_manager.py
View File

@@ -84,7 +84,7 @@ class User(base.Resource):
return resource_info
class identity_manager(base.ResourceManager):
class identity_user_manager(base.ResourceManager):
resource_class = User
def user_create(self, url, data):

28
distributedcloud/dcorch/api/proxy/apps/controller.py
View File

@@ -789,16 +789,20 @@ class IdentityAPIController(APIController):
def _generate_assignment_rid(self, url, environ):
resource_id = None
# for role assignment or revocation, the URL is of format:
# /v3/projects/{project_id}/users/{user_id}/roles/{role_id}
# /v3/projects/{project_id}/users/{user_id}/roles/{role_id} or
# /v3/projects/{project_id}/groups/{group_id}/roles/{role_id}
# We need to extract all ID parameters from the URL
role_id = proxy_utils.get_routing_match_value(environ, 'role_id')
proj_id = proxy_utils.get_routing_match_value(environ, 'project_id')
user_id = proxy_utils.get_routing_match_value(environ, 'user_id')
if 'user_id' in proxy_utils.get_routing_match_arguments(environ):
actor_id = proxy_utils.get_routing_match_value(environ, 'user_id')
else:
actor_id = proxy_utils.get_routing_match_value(environ, 'group_id')
if (not role_id or not proj_id or not user_id):
if (not role_id or not proj_id or not actor_id):
LOG.error("Malformed Role Assignment or Revocation URL: %s", url)
else:
resource_id = "{}_{}_{}".format(proj_id, user_id, role_id)
resource_id = "{}_{}_{}".format(proj_id, actor_id, role_id)
return resource_id
def _retrieve_token_revoke_event_rid(self, url, environ):
@@ -826,7 +830,7 @@ class IdentityAPIController(APIController):
resource_type = self._get_resource_type_from_environ(environ)
# if this is a Role Assignment or Revocation request then
# we need to extract Project ID, User ID and Role ID from the
# we need to extract Project ID, User ID/Group ID and Role ID from the
# URL, and not just the Role ID
if (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_PROJECT_ROLE_ASSIGNMENTS):
@@ -852,6 +856,20 @@ class IdentityAPIController(APIController):
if operation_type == consts.OPERATION_TYPE_POST:
operation_type = consts.OPERATION_TYPE_PATCH
resource_type = consts.RESOURCE_TYPE_IDENTITY_USERS
elif (resource_type == consts.RESOURCE_TYPE_IDENTITY_GROUPS
and operation_type != consts.OPERATION_TYPE_POST):
if("users" in request_header):
# Requests for adding a user (PUT) and removing a user (DELETE)
# should be converted to a PUT request
# The url in this case looks like /groups/{group_id}/users/{user_id}
# We need to extract the group_id and assign that to resource_id
index = request_header.find("/users")
resource_id = self.get_resource_id_from_link(request_header[0:index])
resource_info = {'group':
{'id': resource_id}}
operation_type = consts.OPERATION_TYPE_PUT
else:
resource_id = self.get_resource_id_from_link(request_header)
else:
if operation_type == consts.OPERATION_TYPE_POST:
# Retrieve the ID from the response

4
distributedcloud/dcorch/api/proxy/common/constants.py
View File

@@ -287,6 +287,7 @@ IDENTITY_PROJECTS_PATH = [
IDENTITY_PROJECTS_ROLE_PATH = [
'/v3/projects/{project_id}/users/{user_id}/roles/{role_id}',
'/v3/projects/{project_id}/groups/{group_id}/roles/{role_id}',
]
IDENTITY_TOKEN_REVOKE_EVENTS_PATH = [
@@ -296,6 +297,7 @@ IDENTITY_TOKEN_REVOKE_EVENTS_PATH = [
IDENTITY_PATH_MAP = {
consts.RESOURCE_TYPE_IDENTITY_USERS: IDENTITY_USERS_PATH,
consts.RESOURCE_TYPE_IDENTITY_USERS_PASSWORD: IDENTITY_USERS_PW_PATH,
consts.RESOURCE_TYPE_IDENTITY_GROUPS: IDENTITY_USER_GROUPS_PATH,
consts.RESOURCE_TYPE_IDENTITY_ROLES: IDENTITY_ROLES_PATH,
consts.RESOURCE_TYPE_IDENTITY_PROJECTS: IDENTITY_PROJECTS_PATH,
consts.RESOURCE_TYPE_IDENTITY_PROJECT_ROLE_ASSIGNMENTS:
@@ -346,6 +348,8 @@ ROUTE_METHOD_MAP = {
consts.ENDPOINT_TYPE_IDENTITY: {
consts.RESOURCE_TYPE_IDENTITY_USERS:
['POST', 'PATCH', 'DELETE'],
consts.RESOURCE_TYPE_IDENTITY_GROUPS:
['POST', 'PUT', 'PATCH', 'DELETE'],
consts.RESOURCE_TYPE_IDENTITY_USERS_PASSWORD:
['POST'],
consts.RESOURCE_TYPE_IDENTITY_ROLES:

1
distributedcloud/dcorch/common/consts.py
View File

@@ -106,6 +106,7 @@ RESOURCE_TYPE_QOS_POLICY = "qos"
# Identity Resources
RESOURCE_TYPE_IDENTITY_USERS = "users"
RESOURCE_TYPE_IDENTITY_GROUPS = "groups"
RESOURCE_TYPE_IDENTITY_USERS_PASSWORD = "users_password"
RESOURCE_TYPE_IDENTITY_ROLES = "roles"
RESOURCE_TYPE_IDENTITY_PROJECTS = "projects"

463
distributedcloud/dcorch/engine/sync_services/identity.py
View File

@@ -1,4 +1,4 @@
# Copyright 2018-2020 Wind River
# Copyright 2018-2021 Wind River
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -46,6 +46,8 @@ class IdentitySyncThread(SyncThread):
self.sync_handler_map = {
consts.RESOURCE_TYPE_IDENTITY_USERS:
self.sync_identity_resource,
consts.RESOURCE_TYPE_IDENTITY_GROUPS:
self.sync_identity_resource,
consts.RESOURCE_TYPE_IDENTITY_USERS_PASSWORD:
self.sync_identity_resource,
consts.RESOURCE_TYPE_IDENTITY_ROLES:
@@ -63,6 +65,7 @@ class IdentitySyncThread(SyncThread):
# that users are replicated prior to assignment data (roles/projects)
self.audit_resources = [
consts.RESOURCE_TYPE_IDENTITY_USERS,
consts.RESOURCE_TYPE_IDENTITY_GROUPS,
consts.RESOURCE_TYPE_IDENTITY_PROJECTS,
consts.RESOURCE_TYPE_IDENTITY_ROLES,
consts.RESOURCE_TYPE_IDENTITY_PROJECT_ROLE_ASSIGNMENTS,
@@ -79,6 +82,8 @@ class IdentitySyncThread(SyncThread):
consts.RESOURCE_TYPE_IDENTITY_ROLES:
['heat_stack_owner', 'heat_stack_user', 'ResellerAdmin'],
consts.RESOURCE_TYPE_IDENTITY_PROJECTS:
[],
consts.RESOURCE_TYPE_IDENTITY_GROUPS:
[]
}
@@ -109,8 +114,8 @@ class IdentitySyncThread(SyncThread):
# sysinv, and dcmanager users are special cases as the id's will match
# (as this is forced during the subcloud deploy) but the details will
# not so we still need to sync them here.
m_client = self.get_dbs_client(self.master_region_name).identity_manager
sc_client = self.get_dbs_client(self.region_name).identity_manager
m_client = self.get_dbs_client(self.master_region_name).identity_user_manager
sc_client = self.get_dbs_client(self.region_name).identity_user_manager
for m_user in m_users:
for sc_user in sc_users:
@@ -141,7 +146,7 @@ class IdentitySyncThread(SyncThread):
sdk.OpenStackDriver.delete_region_clients(self.region_name)
# Retry with a new token
sc_client = self.get_dbs_client(
self.region_name).identity_manager
self.region_name).identity_user_manager
user_ref = sc_client.update_user(sc_user.id,
user_records)
if not user_ref:
@@ -149,6 +154,45 @@ class IdentitySyncThread(SyncThread):
" in subcloud.".format(sc_user.id))
raise exceptions.SyncRequestFailed
def _initial_sync_groups(self, m_groups, sc_groups):
# Particularly sync groups with same name but different ID.
m_client = self.get_dbs_client(self.master_region_name).identity_group_manager
sc_client = self.get_dbs_client(self.region_name).identity_group_manager
for m_group in m_groups:
for sc_group in sc_groups:
if (m_group.name == sc_group.name and
m_group.domain_id == sc_group.domain_id and
m_group.id != sc_group.id):
group_records = m_client.group_detail(m_group.id)
if not group_records:
LOG.error("No data retrieved from master cloud for"
" group {} to update its equivalent in"
" subcloud.".format(m_group.id))
raise exceptions.SyncRequestFailed
# update the group by pushing down the DB records to
# subcloud
try:
group_ref = sc_client.update_group(sc_group.id,
group_records)
# Retry once if unauthorized
except dbsync_exceptions.Unauthorized as e:
LOG.info("Update group {} request failed for {}: {}."
.format(sc_group.id,
self.region_name, str(e)))
# Clear the cache so that the old token will not be validated
sdk.OpenStackDriver.delete_region_clients(self.region_name)
sc_client = self.get_dbs_client(
self.region_name).identity_group_manager
group_ref = sc_client.update_group(sc_group.id,
group_records)
if not group_ref:
LOG.error("No group data returned when updating"
" group {} in subcloud.".
format(sc_group.id))
raise exceptions.SyncRequestFailed
def _initial_sync_projects(self, m_projects, sc_projects):
# Particularly sync projects with same name but different ID.
m_client = self.get_dbs_client(self.master_region_name).project_manager
@@ -232,10 +276,10 @@ class IdentitySyncThread(SyncThread):
# before dcorch starts to audit resources. Later on when dcorch audits
# and sync them over(including their IDs) to the subcloud, running
# services at the subcloud with tokens issued before their ID are
# changed will get user/project not found error since their IDs are
# changed will get user/group/project not found error since their IDs are
# changed. This will continue until their tokens expire in up to
# 1 hour. Before that these services basically stop working.
# By an initial synchronization on existing users/projects,
# By an initial synchronization on existing users/groups/projects,
# synchronously followed by a fernet keys synchronization, existing
# tokens at subcloud are revoked and services are forced to
# re-authenticate to get new tokens. This significantly decreases
@@ -261,6 +305,25 @@ class IdentitySyncThread(SyncThread):
self._initial_sync_users(m_users, sc_users)
# get groups from master cloud
m_groups = self.get_master_resources(
consts.RESOURCE_TYPE_IDENTITY_GROUPS)
if not m_groups:
LOG.info("No groups returned from {}".
format(dccommon_consts.VIRTUAL_MASTER_CLOUD))
# get groups from the subcloud
sc_groups = self.get_subcloud_resources(
consts.RESOURCE_TYPE_IDENTITY_GROUPS)
if not sc_groups:
LOG.info("No groups returned from subcloud {}".
format(self.region_name))
if m_groups and sc_groups:
self._initial_sync_groups(m_groups, sc_groups)
# get projects from master cloud
m_projects = self.get_master_resources(
consts.RESOURCE_TYPE_IDENTITY_PROJECTS)
@@ -368,7 +431,7 @@ class IdentitySyncThread(SyncThread):
# format
try:
user_records = self.get_dbs_client(self.master_region_name).\
identity_manager.user_detail(user_id)
identity_user_manager.user_detail(user_id)
except dbsync_exceptions.Unauthorized:
raise dbsync_exceptions.UnauthorizedMaster
if not user_records:
@@ -379,7 +442,7 @@ class IdentitySyncThread(SyncThread):
# Create the user on subcloud by pushing the DB records to subcloud
user_ref = self.get_dbs_client(
self.region_name).identity_manager.add_user(
self.region_name).identity_user_manager.add_user(
user_records)
if not user_ref:
LOG.error("No user data returned when creating user {} in"
@@ -421,7 +484,7 @@ class IdentitySyncThread(SyncThread):
# format
try:
user_records = self.get_dbs_client(self.master_region_name).\
identity_manager.user_detail(user_id)
identity_user_manager.user_detail(user_id)
except dbsync_exceptions.Unauthorized:
raise dbsync_exceptions.UnauthorizedMaster
if not user_records:
@@ -432,7 +495,7 @@ class IdentitySyncThread(SyncThread):
# Update the corresponding user on subcloud by pushing the DB records
# to subcloud
user_ref = self.get_dbs_client(self.region_name).identity_manager.\
user_ref = self.get_dbs_client(self.region_name).identity_user_manager.\
update_user(sc_user_id, user_records)
if not user_ref:
LOG.error("No user data returned when updating user {} in"
@@ -531,6 +594,180 @@ class IdentitySyncThread(SyncThread):
extra=self.log_extra)
user_subcloud_rsrc.delete()
def post_groups(self, request, rsrc):
# Create this group on this subcloud
# The DB level resource creation process is, retrieve the resource
# records from master cloud by its ID, send the records in its original
# JSON format by REST call to the DB synchronization service on this
# subcloud, which then inserts the resource records into DB tables.
group_id = request.orch_job.source_resource_id
if not group_id:
LOG.error("Received group create request without required "
"'source_resource_id' field", extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Retrieve DB records of the group just created. The records is in JSON
# format
try:
group_records = self.get_dbs_client(self.master_region_name).\
identity_group_manager.group_detail(group_id)
except dbsync_exceptions.Unauthorized:
raise dbsync_exceptions.UnauthorizedMaster
if not group_records:
LOG.error("No data retrieved from master cloud for group {} to"
" create its equivalent in subcloud.".format(group_id),
extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Create the group on subcloud by pushing the DB records to subcloud
group_ref = self.get_dbs_client(
self.region_name).identity_group_manager.add_group(
group_records)
if not group_ref:
LOG.error("No group data returned when creating group {} in"
" subcloud.".format(group_id), extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Persist the subcloud resource.
group_ref_id = group_ref.get('group').get('id')
subcloud_rsrc_id = self.persist_db_subcloud_resource(rsrc.id,
group_ref_id)
groupname = group_ref.get('group').get('name')
LOG.info("Created Keystone group {}:{} [{}]"
.format(rsrc.id, subcloud_rsrc_id, groupname),
extra=self.log_extra)
def put_groups(self, request, rsrc):
# Update this group on this subcloud
# The DB level resource update process is, retrieve the resource
# records from master cloud by its ID, send the records in its original
# JSON format by REST call to the DB synchronization service on this
# subcloud, which then updates the resource records in its DB tables.
group_id = request.orch_job.source_resource_id
if not group_id:
LOG.error("Received group update request without required "
"source resource id", extra=self.log_extra)
raise exceptions.SyncRequestFailed
group_dict = jsonutils.loads(request.orch_job.resource_info)
if 'group' in group_dict:
group_dict = group_dict['group']
sc_group_id = group_dict.pop('id', None)
if not sc_group_id:
LOG.error("Received group update request without required "
"subcloud resource id", extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Retrieve DB records of the group. The records is in JSON
# format
try:
group_records = self.get_dbs_client(self.master_region_name).\
identity_group_manager.group_detail(group_id)
except dbsync_exceptions.Unauthorized:
raise dbsync_exceptions.UnauthorizedMaster
if not group_records:
LOG.error("No data retrieved from master cloud for group {} to"
" update its equivalent in subcloud.".format(group_id),
extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Update the corresponding group on subcloud by pushing the DB records
# to subcloud
group_ref = self.get_dbs_client(self.region_name).identity_group_manager.\
update_group(sc_group_id, group_records)
if not group_ref:
LOG.error("No group data returned when updating group {} in"
" subcloud.".format(sc_group_id), extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Persist the subcloud resource.
group_ref_id = group_ref.get('group').get('id')
subcloud_rsrc_id = self.persist_db_subcloud_resource(rsrc.id,
group_ref_id)
groupname = group_ref.get('group').get('name')
LOG.info("Updated Keystone group {}:{} [{}]"
.format(rsrc.id, subcloud_rsrc_id, groupname),
extra=self.log_extra)
def patch_groups(self, request, rsrc):
# Update group reference on this subcloud
group_update_dict = jsonutils.loads(request.orch_job.resource_info)
if not group_update_dict.keys():
LOG.error("Received group update request "
"without any update fields", extra=self.log_extra)
raise exceptions.SyncRequestFailed
group_update_dict = group_update_dict['group']
group_subcloud_rsrc = self.get_db_subcloud_resource(rsrc.id)
if not group_subcloud_rsrc:
LOG.error("Unable to update group reference {}:{}, "
"cannot find equivalent Keystone group in subcloud."
.format(rsrc, group_update_dict),
extra=self.log_extra)
return
# instead of stowing the entire group reference or
# retrieving it, we build an opaque wrapper for the
# v3 Group Manager, containing the ID field which is
# needed to update this group reference
GroupReferenceWrapper = namedtuple('GroupReferenceWrapper',
'id')
group_id = group_subcloud_rsrc.subcloud_resource_id
original_group_ref = GroupReferenceWrapper(id=group_id)
sc_ks_client = self.get_ks_client(self.region_name)
# Update the group in the subcloud
group_ref = sc_ks_client.groups.update(
original_group_ref,
name=group_update_dict.pop('name', None),
domain=group_update_dict.pop('domain', None),
description=group_update_dict.pop('description', None))
if group_ref.id == group_id:
LOG.info("Updated Keystone group: {}:{}"
.format(rsrc.id, group_ref.id), extra=self.log_extra)
else:
LOG.error("Unable to update Keystone group {}:{} for subcloud"
.format(rsrc.id, group_id), extra=self.log_extra)
def delete_groups(self, request, rsrc):
# Delete group reference on this subcloud
group_subcloud_rsrc = self.get_db_subcloud_resource(rsrc.id)
if not group_subcloud_rsrc:
LOG.error("Unable to delete group reference {}, "
"cannot find equivalent Keystone group in subcloud."
.format(rsrc), extra=self.log_extra)
return
# instead of stowing the entire group reference or
# retrieving it, we build an opaque wrapper for the
# v3 User Manager, containing the ID field which is
# needed to delete this group reference
GroupReferenceWrapper = namedtuple('GroupReferenceWrapper',
'id')
group_id = group_subcloud_rsrc.subcloud_resource_id
original_group_ref = GroupReferenceWrapper(id=group_id)
# Delete the group in the subcloud
try:
sc_ks_client = self.get_ks_client(self.region_name)
sc_ks_client.groups.delete(original_group_ref)
except keystone_exceptions.NotFound:
LOG.info("Delete group: group {} not found in {}, "
"considered as deleted.".
format(original_group_ref.id,
self.region_name),
extra=self.log_extra)
# Master Resource can be deleted only when all subcloud resources
# are deleted along with corresponding orch_job and orch_requests.
LOG.info("Keystone group {}:{} [{}] deleted"
.format(rsrc.id, group_subcloud_rsrc.id,
group_subcloud_rsrc.subcloud_resource_id),
extra=self.log_extra)
group_subcloud_rsrc.delete()
def post_projects(self, request, rsrc):
# Create this project on this subcloud
# The DB level resource creation process is, retrieve the resource
@@ -881,7 +1118,7 @@ class IdentitySyncThread(SyncThread):
role_subcloud_rsrc.delete()
def post_project_role_assignments(self, request, rsrc):
# Assign this role to user on project on this subcloud
# Assign this role to user/group on project on this subcloud
# Project role assignments creation is still using keystone APIs since
# the APIs can be used to sync them.
resource_tags = rsrc.master_id.split('_')
@@ -892,7 +1129,8 @@ class IdentitySyncThread(SyncThread):
raise exceptions.SyncRequestFailed
project_id = resource_tags[0]
user_id = resource_tags[1]
# actor_id can be either user_id or group_id
actor_id = resource_tags[1]
role_id = resource_tags[2]
# Ensure that we have already synced the project, user and role
@@ -928,31 +1166,50 @@ class IdentitySyncThread(SyncThread):
sc_user = None
sc_user_list = self._get_all_users(sc_ks_client)
for user in sc_user_list:
if user.id == user_id:
if user.id == actor_id:
sc_user = user
break
if not sc_user:
LOG.error("Unable to assign role to user on project reference {}:"
"{}, cannot find equivalent Keystone User in subcloud."
.format(rsrc, user_id),
sc_group = None
sc_group_list = self._get_all_groups(sc_ks_client)
for group in sc_group_list:
if group.id == actor_id:
sc_group = group
break
if not sc_user and not sc_group:
LOG.error("Unable to assign role to user/group on project reference {}:"
"{}, cannot find equivalent Keystone User/Group in subcloud."
.format(rsrc, actor_id),
extra=self.log_extra)
raise exceptions.SyncRequestFailed
# Create role assignment
sc_ks_client.roles.grant(
sc_role,
user=sc_user,
project=sc_proj)
role_ref = sc_ks_client.role_assignments.list(
user=sc_user,
project=sc_proj,
role=sc_role)
if sc_user:
sc_ks_client.roles.grant(
sc_role,
user=sc_user,
project=sc_proj)
role_ref = sc_ks_client.role_assignments.list(
user=sc_user,
project=sc_proj,
role=sc_role)
elif sc_group:
sc_ks_client.roles.grant(
sc_role,
group=sc_group,
project=sc_proj)
role_ref = sc_ks_client.role_assignments.list(
group=sc_group,
project=sc_proj,
role=sc_role)
if role_ref:
LOG.info("Added Keystone role assignment: {}:{}"
.format(rsrc.id, role_ref), extra=self.log_extra)
# Persist the subcloud resource.
sc_rid = sc_proj.id + '_' + sc_user.id + '_' + sc_role.id
if sc_user:
sc_rid = sc_proj.id + '_' + sc_user.id + '_' + sc_role.id
elif sc_group:
sc_rid = sc_proj.id + '_' + sc_group.id + '_' + sc_role.id
subcloud_rsrc_id = self.persist_db_subcloud_resource(rsrc.id,
sc_rid)
LOG.info("Created Keystone role assignment {}:{} [{}]"
@@ -986,33 +1243,55 @@ class IdentitySyncThread(SyncThread):
resource_tags = subcloud_rid.split('_')
if len(resource_tags) < 3:
LOG.error("Malformed subcloud resource tag {}, expected to be in "
"format: ProjectID_UserID_RoleID."
"format: ProjectID_UserID_RoleID or ProjectID_GroupID_RoleID."
.format(assignment_subcloud_rsrc), extra=self.log_extra)
assignment_subcloud_rsrc.delete()
return
project_id = resource_tags[0]
user_id = resource_tags[1]
actor_id = resource_tags[1]
role_id = resource_tags[2]
# Revoke role assignment
actor = None
try:
sc_ks_client = self.get_ks_client(self.region_name)
sc_ks_client.roles.revoke(
role_id,
user=user_id,
user=actor_id,
project=project_id)
actor = 'user'
except keystone_exceptions.NotFound:
LOG.info("Revoke role assignment: (role {}, user {}, project {})"
" not found in {}, considered as deleted.".
format(role_id, user_id, project_id,
format(role_id, actor_id, project_id,
self.region_name),
extra=self.log_extra)
try:
sc_ks_client = self.get_ks_client(self.region_name)
sc_ks_client.roles.revoke(
role_id,
group=actor_id,
project=project_id)
actor = 'group'
except keystone_exceptions.NotFound:
LOG.info("Revoke role assignment: (role {}, group {}, project {})"
" not found in {}, considered as deleted.".
format(role_id, actor_id, project_id,
self.region_name),
extra=self.log_extra)
role_ref = sc_ks_client.role_assignments.list(
user=user_id,
project=project_id,
role=role_id)
role_ref = None
if actor == 'user':
role_ref = sc_ks_client.role_assignments.list(
user=actor_id,
project=project_id,
role=role_id)
elif actor == 'group':
role_ref = sc_ks_client.role_assignments.list(
group=actor_id,
project=project_id,
role=role_id)
if not role_ref:
LOG.info("Deleted Keystone role assignment: {}:{}"
@@ -1182,7 +1461,9 @@ class IdentitySyncThread(SyncThread):
# ---- Override common audit functions ----
def _get_resource_audit_handler(self, resource_type, client):
if resource_type == consts.RESOURCE_TYPE_IDENTITY_USERS:
return self._get_users_resource(client.identity_manager)
return self._get_users_resource(client.identity_user_manager)
if resource_type == consts.RESOURCE_TYPE_IDENTITY_GROUPS:
return self._get_groups_resource(client.identity_group_manager)
elif resource_type == consts.RESOURCE_TYPE_IDENTITY_ROLES:
return self._get_roles_resource(client.role_manager)
elif resource_type == consts.RESOURCE_TYPE_IDENTITY_PROJECTS:
@@ -1211,6 +1492,14 @@ class IdentitySyncThread(SyncThread):
users = users + domain_users
return users
def _get_all_groups(self, client):
domains = client.domains.list()
groups = []
for domain in domains:
domain_groups = client.groups.list(domain=domain)
groups = groups + domain_groups
return groups
def _get_users_resource(self, client):
try:
services = []
@@ -1251,6 +1540,33 @@ class IdentitySyncThread(SyncThread):
# None will force skip of audit
return None
def _get_groups_resource(self, client):
try:
# get groups from DB API
if hasattr(client, 'list_groups'):
groups = client.list_groups()
# get groups from keystone API
else:
groups = client.groups.list()
# Filter out admin or services projects
filtered_list = self.filtered_audit_resources[
consts.RESOURCE_TYPE_IDENTITY_GROUPS]
filtered_groups = [group for group in groups if
all(group.name != filtered for
filtered in filtered_list)]
return filtered_groups
except (keystone_exceptions.connection.ConnectTimeout,
keystone_exceptions.ConnectFailure,
dbsync_exceptions.ConnectTimeout,
dbsync_exceptions.ConnectFailure) as e:
LOG.info("Group Audit: subcloud {} is not reachable [{}]"
.format(self.region_name,
str(e)), extra=self.log_extra)
# None will force skip of audit
return None
def _get_roles_resource(self, client):
try:
# get roles from DB API
@@ -1316,22 +1632,28 @@ class IdentitySyncThread(SyncThread):
roles = self._get_roles_resource(client)
projects = self._get_projects_resource(client)
users = self._get_users_resource(client)
groups = self._get_groups_resource(client)
for assignment in assignments:
if 'project' not in assignment.scope:
# this is a domain scoped role, we don't care
# about syncing or auditing them for now
continue
role_id = assignment.role['id']
user_id = assignment.user['id']
actor_id = assignment.user['id'] if hasattr(assignment, 'user') else assignment.group['id']
project_id = assignment.scope['project']['id']
assignment_dict = {}
for user in users:
if user.id == user_id:
assignment_dict['user'] = user
if user.id == actor_id:
assignment_dict['actor'] = user
break
else:
continue
for group in groups:
if group.id == actor_id:
assignment_dict['actor'] = group
break
else:
continue
for role in roles:
if role.id == role_id:
@@ -1350,7 +1672,7 @@ class IdentitySyncThread(SyncThread):
# The id of a Role Assigment is:
# projectID_userID_roleID
assignment_dict['id'] = "{}_{}_{}".format(
project_id, user_id, role_id)
project_id, actor_id, role_id)
# Build an opaque object wrapper for this RoleAssignment
refactored_assignment = namedtuple(
@@ -1411,15 +1733,15 @@ class IdentitySyncThread(SyncThread):
# None will force skip of audit
return None
def _same_identity_resource(self, m, sc):
def _same_identity_user_resource(self, m, sc):
LOG.debug("master={}, subcloud={}".format(m, sc),
extra=self.log_extra)
# For user the comparison is DB records by DB records.
# The user DB records are from multiple tables, including user,
# local_user, and password tables. If any of them are not matched,
# it is considered not a same identity resource.
# Note that the user id is compared, since user id is to be synced
# to subcloud too.
# it is considered as a different identity resource.
# Note that the user id is compared, since user id has to be synced
# to the subcloud too.
same_user = (m.id == sc.id and
m.domain_id == sc.domain_id and
m.default_project_id == sc.default_project_id and
@@ -1451,7 +1773,31 @@ class IdentitySyncThread(SyncThread):
result = True
return result
def _has_same_identity_ids(self, m, sc):
def _same_identity_group_resource(self, m, sc):
LOG.debug("master={}, subcloud={}".format(m, sc),
extra=self.log_extra)
# For group the comparison is DB records by DB records.
# The group DB records are from two tables - group and
# user_group_membership tables. If any of them are not matched,
# it is considered as different identity resource.
# Note that the group id is compared, since group id has to be synced
# to the subcloud too.
same_group = (m.id == sc.id and
m.domain_id == sc.domain_id and
m.description == sc.description and
m.name == sc.name and
m.extra == sc.extra)
if not same_group:
return False
same_local_user_ids = (m.local_user_ids ==
sc.local_user_ids)
if not same_local_user_ids:
return False
return True
def _has_same_identity_user_ids(self, m, sc):
# If (user name + domain name) or use id is the same,
# the resources are considered to be the same resource.
# Any difference in other attributes will trigger an update (PUT)
@@ -1459,6 +1805,14 @@ class IdentitySyncThread(SyncThread):
return ((m.local_user.name == sc.local_user.name and
m.domain_id == sc.domain_id) or m.id == sc.id)
def _has_same_identity_group_ids(self, m, sc):
# If (group name + domain name) or group id is the same,
# then the resources are considered to be the same.
# Any difference in other attributes will trigger an update (PUT)
# to that resource in subcloud.
return ((m.name == sc.name and m.domain_id == sc.domain_id)
or m.id == sc.id)
def _same_project_resource(self, m, sc):
LOG.debug("master={}, subcloud={}".format(m, sc),
extra=self.log_extra)
@@ -1510,7 +1864,7 @@ class IdentitySyncThread(SyncThread):
LOG.debug("same_assignment master={}, subcloud={}".format(m, sc),
extra=self.log_extra)
# For an assignment to be the same, all 3 of its role, project and
# user information must match up.
# actor (user/group) information must match up.
# Compare by names here is fine, since this comparison gets called
# only if the mapped subcloud assignment is found by id in subcloud
# resources just retrieved. In another word, the ids are guaranteed
@@ -1518,8 +1872,8 @@ class IdentitySyncThread(SyncThread):
# audit_find_missing(). same_resource() in audit_find_missing() is
# actually redundant for assignment but it's the generic algorithm
# for all types of resources.
return((m.user.name == sc.user.name and
m.user.domain_id == sc.user.domain_id) and
return((m.actor.name == sc.actor.name and
m.actor.domain_id == sc.actor.domain_id) and
(m.role.name == sc.role.name and
m.role.domain_id == sc.role.domain_id) and
(m.project.name == sc.project.name and
@@ -1558,7 +1912,7 @@ class IdentitySyncThread(SyncThread):
def get_master_resources(self, resource_type):
# Retrieve master resources from DB or through Keystone.
# users, projects, roles, and token revocation events use
# users, groups, projects, roles, and token revocation events use
# dbsync client, other resources use keystone client.
if self.is_resource_handled_by_dbs_client(resource_type):
try:
@@ -1642,7 +1996,10 @@ class IdentitySyncThread(SyncThread):
def same_resource(self, resource_type, m_resource, sc_resource):
if (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_USERS):
return self._same_identity_resource(m_resource, sc_resource)
return self._same_identity_user_resource(m_resource, sc_resource)
elif (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_GROUPS):
return self._same_identity_group_resource(m_resource, sc_resource)
elif (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_PROJECTS):
return self._same_project_resource(m_resource, sc_resource)
@@ -1662,7 +2019,10 @@ class IdentitySyncThread(SyncThread):
def has_same_ids(self, resource_type, m_resource, sc_resource):
if (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_USERS):
return self._has_same_identity_ids(m_resource, sc_resource)
return self._has_same_identity_user_ids(m_resource, sc_resource)
elif (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_GROUPS):
return self._has_same_identity_group_ids(m_resource, sc_resource)
elif (resource_type ==
consts.RESOURCE_TYPE_IDENTITY_PROJECTS):
return self._has_same_project_ids(m_resource, sc_resource)
@@ -1790,6 +2150,7 @@ class IdentitySyncThread(SyncThread):
def is_resource_handled_by_dbs_client(resource_type):
if resource_type in [
consts.RESOURCE_TYPE_IDENTITY_USERS,
consts.RESOURCE_TYPE_IDENTITY_GROUPS,
consts.RESOURCE_TYPE_IDENTITY_PROJECTS,
consts.RESOURCE_TYPE_IDENTITY_ROLES,
consts.RESOURCE_TYPE_IDENTITY_TOKEN_REVOKE_EVENTS,