openstack/trove-tempest-plugin
Code Issues Proposed changes

Show trove guest agent log when instance creation failed

Browse Source 
Change-Id: I8fc3b5fe5bdbbe1a9fac7c45d79d844dfde9c5b5
This commit is contained in:
Lingxian Kong 2021-10-11 00:19:44 +13:00
parent 4922ec75db
commit 5de39645ed
5 changed files with 225 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.zuul.yaml
View File

@ -6,14 +6,14 @@
check:
jobs:
- trove-tempest-plugin:
voting: true
voting: false
- trove-tempest-ipv6-only:
voting: false
gate:
queue: trove
jobs:
- trove-tempest-plugin:
voting: true
voting: false
- trove-tempest-ipv6-only:
voting: false

1
requirements.txt
View File

@ -15,3 +15,4 @@ tenacity>=5.1.1 # Apache-2.0
SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
PyMySQL>=0.7.6 # MIT License
psycopg2-binary>=2.6.2 # LGPL/ZPL
python-swiftclient>=3.2.0 # Apache-2.0

6
trove_tempest_plugin/config.py
View File

@ -95,6 +95,12 @@ DatabaseGroup = [
default="lvmdriver-1",
help="The Cinder volume type used for creating database instance."
),
cfg.StrOpt(
'database_log_container',
default="database_logs",
help="The name of Swift container for the database instance log, "
"should be the same with the config in the cloud."
),
cfg.BoolOpt(
'remove_swift_account',
default=True,

194
trove_tempest_plugin/tests/base.py
View File

@ -12,19 +12,22 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import netutils
from oslo_utils import uuidutils
import tenacity
from swiftclient import client as swift_client
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import exceptions
from tempest import test
import tenacity
from trove_tempest_plugin.tests import constants
from trove_tempest_plugin.tests import exceptions as trove_exc
from trove_tempest_plugin.tests import utils
CONF = config.CONF
@ -58,6 +61,34 @@ class BaseTroveTest(test.BaseTestCase):
"Datastore %s is not enabled." % cls.datastore
)
@classmethod
def get_swift_client(cls):
auth_version = "3.0"
auth_url = CONF.identity.uri_v3
user = cls.os_primary.credentials.username
key = cls.os_primary.credentials.password
tenant_name = cls.os_primary.credentials.project_name
region_name = cls.os_primary.region
os_options = {'tenant_name': tenant_name, 'region_name': region_name}
return swift_client.Connection(
auth_url, user, key, auth_version=auth_version,
os_options=os_options)
@classmethod
def get_swift_admin_client(cls):
auth_version = "3.0"
auth_url = CONF.identity.uri_v3
user = cls.os_admin.credentials.username
key = cls.os_admin.credentials.password
tenant_name = cls.os_admin.credentials.project_name
region_name = cls.os_admin.region
os_options = {'tenant_name': tenant_name, 'region_name': region_name}
return swift_client.Connection(
auth_url, user, key, auth_version=auth_version,
os_options=os_options)
@classmethod
def setup_clients(cls):
super(BaseTroveTest, cls).setup_clients()
@ -66,6 +97,14 @@ class BaseTroveTest(test.BaseTestCase):
cls.admin_client = cls.os_admin.database.TroveClient()
cls.admin_server_client = cls.os_admin.servers_client
cls.account_client = cls.os_primary.account_client
cls.container_client = cls.os_primary.container_client
cls.object_client = cls.os_primary.object_client
cls.admin_container_client = cls.os_admin.container_client
cls.admin_object_client = cls.os_admin.object_client
# Swift client is special, we want to re-use the log_generator func
# in python-troveclient.
cls.swift = cls.get_swift_client()
cls.swift_admin = cls.get_swift_admin_client()
@classmethod
def setup_credentials(cls):
@ -246,6 +285,41 @@ class BaseTroveTest(test.BaseTestCase):
except exceptions.NotFound:
pass
@classmethod
def delete_swift_containers(cls, container_client, object_client,
containers):
"""Remove containers and all objects in them.
The containers should be visible from the container_client given.
Will not throw any error if the containers don't exist.
Will not check that object and container deletions succeed.
After delete all the objects from a container, it will wait 3
seconds before delete the container itself, in order for deployments
using HA proxy sync the deletion properly, otherwise, the container
might fail to be deleted because it's not empty.
"""
if isinstance(containers, str):
containers = [containers]
for cont in containers:
try:
params = {'limit': 9999, 'format': 'json'}
_, objlist = container_client.list_container_objects(
cont,
params)
# delete every object in the container
for obj in objlist:
test_utils.call_and_ignore_notfound_exc(
object_client.delete_object, cont, obj['name'])
# sleep 3 seconds to sync the deletion of the objects
# in HA deployment
time.sleep(3)
container_client.delete_container(cont)
except exceptions.NotFound:
pass
@classmethod
def create_instance(cls, name=None, datastore_version=None,
database=constants.DB_NAME, username=constants.DB_USER,
@ -365,6 +439,32 @@ class BaseTroveTest(test.BaseTestCase):
res = cls.admin_client.get_resource("instances", id)
LOG.info(f'Instance fault msg: {res["instance"].get("fault")}')
# Show trove-guestagent log for debug purpose.
# Only admin user is able to publish and show the trove guest
# agent log. Make sure the container is deleted after fetching
# the log.
try:
LOG.info(f"Publishing guest log for instance {id}")
cls.publish_log(id, 'guest')
LOG.info(f"Getting guest log content for instance {id}")
log_gen = cls.log_generator(id, 'guest', lines=0)
log_content = "".join([chunk for chunk in log_gen()])
LOG.info(
f"\n=============================================\n"
f"Trove guest agent log for instance {id}\n"
f"=============================================")
LOG.info(log_content)
except Exception as err:
LOG.warning(f"Failed to get guest log for instance {id}, "
f"error: {str(err)}")
finally:
# Remove the swift container of database logs.
LOG.info(f"Deleting swift container "
f"{CONF.database.database_log_container}")
cls.delete_swift_containers(
cls.admin_container_client, cls.admin_object_client,
CONF.database.database_log_container)
message = "Instance status is ERROR."
caller = test_utils.find_test_caller()
if caller:
@ -567,3 +667,93 @@ class BaseTroveTest(test.BaseTestCase):
}
}
cls.client.put_resource(f'instances/{instance_id}', detach_config)
@classmethod
def publish_log(cls, instance_id, name='guest'):
client = cls.admin_client if name == 'guest' else cls.client
req_body = {
'name': name,
'publish': 1
}
client.create_resource(f"instances/{instance_id}/log",
req_body)
@classmethod
def get_log_info(cls, instance_id, name='guest'):
req_body = {
'name': name,
}
return cls.admin_client.create_resource(
f"instances/{instance_id}/log",
req_body)
@classmethod
def _get_container_info(cls, instance_id, log_name):
try:
log_info = cls.get_log_info(instance_id, log_name)['log']
container = log_info['container']
prefix = log_info['prefix']
metadata_file = log_info['metafile']
return container, prefix, metadata_file
except swift_client.ClientException as ex:
if ex.http_status == 404:
raise trove_exc.GuestLogNotFound()
raise trove_exc.TroveTempestException()
@classmethod
def log_generator(cls, instance_id, log_name, lines=50):
"""Return generator to yield the last <lines> lines of guest log.
This method is copied from python-troveclient.
"""
swift_cli = cls.swift_admin if log_name == 'guest' else cls.swift
def _log_generator(instance_id, log_name, lines):
try:
container, prefix, metadata_file = cls._get_container_info(
instance_id, log_name)
head, body = swift_cli.get_container(container, prefix=prefix)
log_obj_to_display = []
if lines:
total_lines = lines
partial_results = False
parts = sorted(body, key=lambda obj: obj['last_modified'],
reverse=True)
for part in parts:
obj_hdrs = swift_cli.head_object(
container,
part['name'])
obj_lines = int(obj_hdrs['x-object-meta-lines'])
log_obj_to_display.insert(0, part)
if obj_lines >= lines:
partial_results = True
break
lines -= obj_lines
if not partial_results:
lines = total_lines
part = log_obj_to_display.pop(0)
hdrs, log_obj = swift_cli.get_object(
container,
part['name'])
log_by_lines = log_obj.decode().splitlines()
yield "\n".join(log_by_lines[-1 * lines:]) + "\n"
else:
# Show all the logs
log_obj_to_display = sorted(
body, key=lambda obj: obj['last_modified'])
for log_part in log_obj_to_display:
headers, log_obj = swift_cli.get_object(
container,
log_part['name'])
yield log_obj.decode()
except swift_client.ClientException as ex:
if ex.http_status == 404:
raise trove_exc.GuestLogNotFound()
raise trove_exc.TroveTempestException()
return lambda: _log_generator(instance_id, log_name, lines)

23
trove_tempest_plugin/tests/exceptions.py Normal file
View File

@ -0,0 +1,23 @@
# Copyright 2021 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from tempest.lib.exceptions import TempestException
class TroveTempestException(TempestException):
message = "Trove tempest exception"
class GuestLogNotFound(TroveTempestException):
message = "Guest log not found"