starlingx/distcloud
Code Issues Proposed changes
Files
4027eee3906531addcf568b86daf02b777204a31
distcloud/distributedcloud/dccommon/drivers/openstack/dcmanager_v1.py
twang4 4027eee390 System peer and subcloud peer group monitoring and handling 
System peer monitoring and handling:
It does peer health check by querying peer group list.
After failure is detected and the number of heartbeat
failure reaches the threshold, alarm will be raised.
After connection is back online, alarm will be cleared,
and will perform an audit to the peer groups that are associated
to the system peer.

Subcloud peer group audit and handling:
If the remote peer group's migration_status is in the 'migrating'
state, unmanage the subclouds of the local peer group.
If the remote peer group's migration_status is in the 'complete'
 state, compare the subclouds on both ends.
If the remote end is in a 'managed+online' state,
set the local subclouds with the same region_name to
'unmanaged+secondary.'
If the remote peer group's migration_status is in the 'none' state,
set the migration_status of the local peer group to 'none' as well.

Batch rehome update:
When Subcloud Peer Group based batch rehoming is performed,
need to check if the associated System Peer is alive or not:
If yes, and the remote subcloud with same region_name is
being managed in peer site, need to unmanage it before
rehoming it to local site.
If not, this means the peer site is not available for the
subcloud management any more, rehoming can be performed
directly.
If the subcloud peer group priority is 0, will try to clear alarm
of FM_ALARM_ID_DC_SUBCLOUD_PEER_GROUP_NOT_MANAGED
after batch rehoming is complete.

Security enhancement:
Base64 encode/decode admin_password when save/load rehome_data
from DB.

Test Plan:
1. PASS - Add a system peer on DC0, unplug the OAM network
          between DC0 and DC1. Alarm raised.
2. PASS - Reconnect the OAM network between DC0 and DC1,
          previous alarm will be cleared.
3. PASS - Add a subcloud peer group on DC0.
          Add two subclouds on DC0 under the subcloud peer group.
          which should be in managed, online and complete states.
          Add a system peer on DC0 pointing to DC1,
          Add a peer-group-association associating the peer-group
          and the system-peer above on DC0.
          create another association in DC1, for system DC0 and the
          peer group synced to DC1.
          Shutdown DC0, perform the migrate operation on
          subcloud-peer-group from DC1, after all subclouds have
          been migrated and entered online managed states.
          Power on DC0, check alarm like "Subcloud peer group
          xxx is managed by remote system (peer_uuid=xxx) with
          lower priority."  has been raised.
          Check the subcloud state on DC0, should be in 'secondary' state,
          Perform the migrate operation on subcloud-peer-group from DC0.
          After all subclouds have been migrated and entered online
          managed states. Alarm is cleared.
          Check subclouds on DC1, subcloud state should be in secondary state.
4.PASS - Migrate a subcloud peer group with 0 priority,
          Before migrate, check alarm
          FM_ALARM_ID_DC_SUBCLOUD_PEER_GROUP_NOT_MANAGED
          exists.
          After migration done, check alarm cleared.
5. PASS - When the remote peer group's migration_status is in the
          'migrating' state (if DC0 comes back online while rehoming
          is still in progress); DC0's subcloud will be auto set to 'unmanaged',
          after DC1's migration is complete and all subcloud in 'managed+online',
          DC0's subcloud will auto set to 'secondary'.

Story: 2010852
Task: 48483
Task: 48509
Task: 48819

Change-Id: Ic97e7c4a7628445522adfba4b0b2e0cc945cbe22
Signed-off-by: Wang Tao <tao.wang@windriver.com>
2023-11-06 10:12:31 +08:00

330 lines
13 KiB
Python
Raw Blame History

# Copyright (c) 2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log
import requests
from requests_toolbelt import MultipartEncoder
from dccommon import consts
from dccommon.drivers import base
from dccommon import exceptions
LOG = log.getLogger(__name__)
DCMANAGER_CLIENT_REST_DEFAULT_TIMEOUT = 600
class DcmanagerClient(base.DriverBase):
"""Dcmanager V1 driver."""
def __init__(self, region, session,
timeout=DCMANAGER_CLIENT_REST_DEFAULT_TIMEOUT,
endpoint_type=consts.KS_ENDPOINT_PUBLIC,
endpoint=None):
if endpoint is None:
endpoint = session.get_endpoint(
service_type='dcmanager',
region_name=region,
interface=endpoint_type)
self.endpoint = endpoint
self.token = session.get_token()
self.timeout = timeout
def get_subcloud(self, subcloud_ref, is_region_name=False):
"""Get subcloud."""
if subcloud_ref is None:
raise ValueError("subcloud_ref is required.")
url = f"{self.endpoint}/subclouds/{subcloud_ref}/detail"
headers = {"X-Auth-Token": self.token}
if is_region_name:
headers["User-Agent"] = consts.DCMANAGER_V1_HTTP_AGENT
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud not found' in response.text:
raise exceptions.SubcloudNotFound(subcloud_ref=subcloud_ref)
message = "Get Subcloud: subcloud_ref %s failed with RC: %d" % \
(subcloud_ref, response.status_code)
LOG.error(message)
raise Exception(message)
def get_subcloud_list(self):
"""Get subcloud list."""
url = self.endpoint + '/subclouds'
headers = {"X-Auth-Token": self.token}
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
data = response.json()
return data.get('subclouds', [])
else:
message = "Get Subcloud list failed with RC: %d" % \
response.status_code
LOG.error(message)
raise Exception(message)
def get_subcloud_group_list(self):
"""Get subcloud group list."""
url = self.endpoint + '/subcloud-groups'
headers = {"X-Auth-Token": self.token}
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
data = response.json()
return data.get('subcloud_groups', [])
else:
message = "Get Subcloud Group list: failed with RC: %d" % \
response.status_code
LOG.error(message)
raise Exception(message)
def get_subcloud_peer_group_list(self):
"""Get subcloud peer group list."""
url = self.endpoint + '/subcloud-peer-groups'
headers = {"X-Auth-Token": self.token}
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
data = response.json()
return data.get('subcloud_peer_groups', [])
else:
message = "Get Subcloud Peer Group list: failed with RC: %d" % \
response.status_code
LOG.error(message)
raise Exception(message)
def get_subcloud_peer_group(self, peer_group_ref):
"""Get subcloud peer group."""
if peer_group_ref is None:
raise ValueError("peer_group_ref is required.")
url = f"{self.endpoint}/subcloud-peer-groups/{peer_group_ref}"
headers = {"X-Auth-Token": self.token}
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud Peer Group not found' in response.text:
raise exceptions.SubcloudPeerGroupNotFound(
peer_group_ref=peer_group_ref)
message = "Get Subcloud Peer Group: peer_group_ref %s " \
"failed with RC: %d" % (peer_group_ref, response.status_code)
LOG.error(message)
raise Exception(message)
def get_subcloud_list_by_peer_group(self, peer_group_ref):
"""Get subclouds in the specified subcloud peer group."""
if peer_group_ref is None:
raise ValueError("peer_group_ref is required.")
url = f"{self.endpoint}/subcloud-peer-groups/{peer_group_ref}/" \
"subclouds"
headers = {"X-Auth-Token": self.token}
response = requests.get(url, headers=headers, timeout=self.timeout)
if response.status_code == 200:
data = response.json()
return data.get('subclouds', [])
else:
if response.status_code == 404 and \
'Subcloud Peer Group not found' in response.text:
raise exceptions.SubcloudPeerGroupNotFound(
peer_group_ref=peer_group_ref)
message = "Get Subcloud list by Peer Group: peer_group_ref %s " \
"failed with RC: %d" % (peer_group_ref, response.status_code)
LOG.error(message)
raise Exception(message)
def add_subcloud_peer_group(self, **kwargs):
"""Add a subcloud peer group."""
url = self.endpoint + '/subcloud-peer-groups'
headers = {"X-Auth-Token": self.token,
"Content-Type": "application/json"}
response = requests.post(url, json=kwargs, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
message = "Add Subcloud Peer Group: %s, failed with RC: %d" % \
(kwargs, response.status_code)
LOG.error(message)
raise Exception(message)
def add_subcloud_with_secondary_status(self, files, data):
"""Add a subcloud with secondary status."""
url = self.endpoint + '/subclouds'
# If not explicitly specified, set 'secondary' to true by default.
# This action adds a secondary subcloud with rehoming data in the
# peer site without creating an actual subcloud.
if 'secondary' in data and data['secondary'] != "true":
raise ValueError("secondary in data must true.")
data['secondary'] = "true"
fields = dict()
if files is not None:
# If files are specified, add them to the fields.
for k, v in files.items():
fields.update({k: (v, open(v, 'rb'),)})
fields.update(data)
enc = MultipartEncoder(fields=fields)
headers = {"X-Auth-Token": self.token,
"Content-Type": enc.content_type}
response = requests.post(url, headers=headers, data=enc,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
message = "Add Subcloud with secondary status: files: %s, " \
"data: %s, failed with RC: %d" % (files, data,
response.status_code)
LOG.error(message)
raise Exception(message)
def update_subcloud_peer_group(self, peer_group_ref, **kwargs):
"""Update the subcloud peer group."""
if peer_group_ref is None:
raise ValueError("peer_group_ref is required.")
url = f"{self.endpoint}/subcloud-peer-groups/{peer_group_ref}"
headers = {"X-Auth-Token": self.token,
"Content-Type": "application/json"}
response = requests.patch(url, json=kwargs, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud Peer Group not found' in response.text:
raise exceptions.SubcloudPeerGroupNotFound(
peer_group_ref=peer_group_ref)
message = "Update Subcloud Peer Group: peer_group_ref %s, %s, " \
"failed with RC: %d" % (peer_group_ref, kwargs,
response.status_code)
LOG.error(message)
raise Exception(message)
def audit_subcloud_peer_group(self, peer_group_ref, **kwargs):
"""Audit the subcloud peer group."""
if peer_group_ref is None:
raise ValueError("peer_group_ref is required.")
url = f"{self.endpoint}/subcloud-peer-groups/{peer_group_ref}/audit"
headers = {"X-Auth-Token": self.token,
"Content-Type": "application/json"}
response = requests.patch(url, json=kwargs, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud Peer Group not found' in response.text:
raise exceptions.SubcloudPeerGroupNotFound(
peer_group_ref=peer_group_ref)
message = "Audit Subcloud Peer Group: peer_group_ref %s, %s, " \
"failed with RC: %d" % (peer_group_ref, kwargs,
response.status_code)
LOG.error(message)
raise Exception(message)
def update_subcloud(self, subcloud_ref, files, data, is_region_name=False):
"""Update the subcloud."""
if subcloud_ref is None:
raise ValueError("subcloud_ref is required.")
url = f"{self.endpoint}/subclouds/{subcloud_ref}"
fields = dict()
if files is not None:
# If files are specified, add them to the fields.
for k, v in files.items():
fields.update({k: (v, open(v, 'rb'),)})
fields.update(data)
enc = MultipartEncoder(fields=fields)
headers = {"X-Auth-Token": self.token,
"Content-Type": enc.content_type}
# Add header to flag the request is from another DC,
# server will treat subcloud_ref as a region_name
if is_region_name:
headers["User-Agent"] = consts.DCMANAGER_V1_HTTP_AGENT
response = requests.patch(url, headers=headers, data=enc,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud not found' in response.text:
raise exceptions.SubcloudNotFound(subcloud_ref=subcloud_ref)
message = "Update Subcloud: subcloud_ref: %s files: %s, " \
"data: %s, failed with RC: %d" % (subcloud_ref, files, data,
response.status_code)
LOG.error(message)
raise Exception(message)
def delete_subcloud_peer_group(self, peer_group_ref):
"""Delete the subcloud peer group."""
if peer_group_ref is None:
raise ValueError("peer_group_ref is required.")
url = f"{self.endpoint}/subcloud-peer-groups/{peer_group_ref}"
headers = {"X-Auth-Token": self.token}
response = requests.delete(url, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud Peer Group not found' in response.text:
raise exceptions.SubcloudPeerGroupNotFound(
peer_group_ref=peer_group_ref)
elif response.status_code == 400 and \
'a peer group which is associated with a system peer' in \
response.text:
raise exceptions.SubcloudPeerGroupDeleteFailedAssociated(
peer_group_ref=peer_group_ref)
message = "Delete Subcloud Peer Group: peer_group_ref %s " \
"failed with RC: %d" % (peer_group_ref, response.status_code)
LOG.error(message)
raise Exception(message)
def delete_subcloud(self, subcloud_ref):
"""Delete the subcloud."""
if subcloud_ref is None:
raise ValueError("subcloud_ref is required.")
url = f"{self.endpoint}/subclouds/{subcloud_ref}"
headers = {"X-Auth-Token": self.token}
response = requests.delete(url, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
return response.json()
else:
if response.status_code == 404 and \
'Subcloud not found' in response.text:
raise exceptions.SubcloudNotFound(subcloud_ref=subcloud_ref)
message = "Delete Subcloud: subcloud_ref %s failed with RC: %d" % \
(subcloud_ref, response.status_code)
LOG.error(message)
raise Exception(message)
View Git Blame Copy Permalink