From cff7420cfffce50cf70389d1808cbeea983973c9 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Mon, 2 Jul 2018 16:20:07 -0500 Subject: [PATCH] Support links for task status - Some status changes in a task may have additional information that is referenced by a URI link. Support describing these links and returning them via API. - Refactor alembic stuff to better handle table schema updates - Add unit tests Change-Id: Iae63a9716f2522578be0244925fc274a4338eac4 --- .../4713e7ebca9_add_task_status_links.py | 29 +++++ .../4a5bef3702b_create_build_data_table.py | 2 +- ...593a123e7c5_create_base_database_tables.py | 10 +- drydock_provisioner/control/util.py | 30 +++++ drydock_provisioner/drydock_client/session.py | 5 +- drydock_provisioner/objects/task.py | 36 ++++++ drydock_provisioner/statemgmt/db/tables.py | 29 ++++- etc/drydock/drydock.conf.sample | 3 + tests/unit/test_task_link.py | 114 ++++++++++++++++++ 9 files changed, 244 insertions(+), 14 deletions(-) create mode 100644 alembic/versions/4713e7ebca9_add_task_status_links.py create mode 100644 drydock_provisioner/control/util.py create mode 100644 tests/unit/test_task_link.py diff --git a/alembic/versions/4713e7ebca9_add_task_status_links.py b/alembic/versions/4713e7ebca9_add_task_status_links.py new file mode 100644 index 00000000..5318a55f --- /dev/null +++ b/alembic/versions/4713e7ebca9_add_task_status_links.py @@ -0,0 +1,29 @@ +"""add task status links + +Revision ID: 4713e7ebca9 +Revises: 4a5bef3702b +Create Date: 2018-07-05 14:54:18.381988 + +""" + +# revision identifiers, used by Alembic. +revision = '4713e7ebca9' +down_revision = '4a5bef3702b' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +from drydock_provisioner.statemgmt.db import tables + + +def upgrade(): + for c in tables.Tasks.__add_result_links__: + op.add_column(tables.Tasks.__tablename__, c) + + +def downgrade(): + for c in tables.Tasks.__add_result_links__: + op.drop_column(tables.Tasks.__tablename__, c.name) + diff --git a/alembic/versions/4a5bef3702b_create_build_data_table.py b/alembic/versions/4a5bef3702b_create_build_data_table.py index 79e0ec46..8408bc25 100644 --- a/alembic/versions/4a5bef3702b_create_build_data_table.py +++ b/alembic/versions/4a5bef3702b_create_build_data_table.py @@ -19,7 +19,7 @@ from drydock_provisioner.statemgmt.db import tables def upgrade(): op.create_table(tables.BuildData.__tablename__, - *tables.BuildData.__schema__) + *tables.BuildData.__baseschema__) def downgrade(): diff --git a/alembic/versions/9593a123e7c5_create_base_database_tables.py b/alembic/versions/9593a123e7c5_create_base_database_tables.py index ed2c357a..16b1a013 100644 --- a/alembic/versions/9593a123e7c5_create_base_database_tables.py +++ b/alembic/versions/9593a123e7c5_create_base_database_tables.py @@ -18,15 +18,15 @@ from drydock_provisioner.statemgmt.db import tables def upgrade(): - op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__schema__) + op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__baseschema__) op.create_table(tables.ResultMessage.__tablename__, - *tables.ResultMessage.__schema__) + *tables.ResultMessage.__baseschema__) op.create_table(tables.ActiveInstance.__tablename__, - *tables.ActiveInstance.__schema__) + *tables.ActiveInstance.__baseschema__) op.create_table(tables.BootAction.__tablename__, - *tables.BootAction.__schema__) + *tables.BootAction.__baseschema__) op.create_table(tables.BootActionStatus.__tablename__, - *tables.BootActionStatus.__schema__) + *tables.BootActionStatus.__baseschema__) def downgrade(): diff --git a/drydock_provisioner/control/util.py b/drydock_provisioner/control/util.py new file mode 100644 index 00000000..d5b9d840 --- /dev/null +++ b/drydock_provisioner/control/util.py @@ -0,0 +1,30 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Reusable utility functions for API access.""" +from drydock_provisioner.error import ApiError +from drydock_provisioner.drydock_client.session import KeystoneClient +from drydock_provisioner.util import KeystoneUtils + +def get_internal_api_href(ver): + """Get the internal API href for Drydock API version ``ver``.""" + + # TODO(sh8121att) Support versioned service registration + supported_versions = ['v1.0'] + if ver in supported_versions: + ks_sess = KeystoneUtils.get_session() + url = KeystoneClient.get_endpoint( + "physicalprovisioner", ks_sess=ks_sess, interface='internal') + return url + else: + raise ApiError("API version %s unknown." % ver) diff --git a/drydock_provisioner/drydock_client/session.py b/drydock_provisioner/drydock_client/session.py index cbb11950..55bf5006 100644 --- a/drydock_provisioner/drydock_client/session.py +++ b/drydock_provisioner/drydock_client/session.py @@ -180,12 +180,13 @@ class DrydockSession(object): class KeystoneClient(object): @staticmethod - def get_endpoint(endpoint, ks_sess=None, auth_info=None): + def get_endpoint(endpoint, ks_sess=None, auth_info=None, interface='internal'): """ Wraps calls to keystone for lookup of an endpoint by service type :param endpoint: The endpoint to look up :param ks_sess: A keystone session to use for accessing endpoint catalogue :param auth_info: Authentication info to use for building a token if a ``ks_sess`` is not specified + :param str interface: Which registered endpoint to return :returns: The url string of the endpoint :rtype: str """ @@ -193,7 +194,7 @@ class KeystoneClient(object): ks_sess = KeystoneClient.get_ks_session(**auth_info) return ks_sess.get_endpoint( - interface='internal', service_type=endpoint) + interface=interface, service_type=endpoint) @staticmethod def get_token(ks_sess=None, auth_info=None): diff --git a/drydock_provisioner/objects/task.py b/drydock_provisioner/objects/task.py index 72253bf4..6332c663 100644 --- a/drydock_provisioner/objects/task.py +++ b/drydock_provisioner/objects/task.py @@ -371,6 +371,8 @@ class Task(object): self.result.successes, 'result_failures': self.result.failures, + 'result_links': + self.result.links, 'status': self.status, 'created': @@ -486,6 +488,7 @@ class Task(object): i.result.status = d.get('result_status') i.result.successes = d.get('result_successes', []) i.result.failures = d.get('result_failures', []) + i.result.links = d.get('result_links', []) # Deserialize the request context for this task if i.request_context is not None: @@ -506,6 +509,8 @@ class TaskStatus(object): self.reason = None self.status = hd_fields.ActionResult.Incomplete + self.links = dict() + # For tasks operating on multiple contexts (nodes, networks, etc...) # track which contexts ended successfully and which failed self.successes = [] @@ -515,6 +520,31 @@ class TaskStatus(object): def obj_name(cls): return cls.__name__ + def add_link(self, relation, uri): + """Add a external reference link to this status. + + :param str relation: The relation of the link + :param str uri: A valid URI that references the external content + """ + self.links.setdefault(relation, []) + self.links[relation].append(uri) + + def get_links(self, relation=None): + """Get one or more links of this status. + + If ``relation`` is None, then return all links. + + :param str relation: Return only links that exhibit this relation + :returns: a list of str URIs or empty list + """ + if relation: + return self.links.get(relation, []) + else: + all_links = list() + for v in self.links.values(): + all_links.extend(v) + return all_links + def set_message(self, msg): self.message = msg @@ -560,6 +590,11 @@ class TaskStatus(object): return new_msg def to_dict(self): + links = list() + if self.links: + for k, v in self.links.items(): + for r in v: + links.append(dict(rel=k, href=r)) return { 'kind': 'Status', 'apiVersion': 'v1.0', @@ -569,6 +604,7 @@ class TaskStatus(object): 'status': self.status, 'successes': self.successes, 'failures': self.failures, + 'links': links, 'details': { 'errorCount': self.error_count, 'messageList': [x.to_dict() for x in self.message_list], diff --git a/drydock_provisioner/statemgmt/db/tables.py b/drydock_provisioner/statemgmt/db/tables.py index 5568fbf2..39ce7815 100644 --- a/drydock_provisioner/statemgmt/db/tables.py +++ b/drydock_provisioner/statemgmt/db/tables.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Definitions for Drydock database tables.""" +import copy from sqlalchemy.schema import Table, Column from sqlalchemy.types import Boolean, DateTime, String, Integer, Text @@ -30,7 +31,7 @@ class Tasks(ExtendTable): __tablename__ = 'tasks' - __schema__ = [ + __baseschema__ = [ Column('task_id', pg.BYTEA(16), primary_key=True), Column('parent_task_id', pg.BYTEA(16)), Column('subtask_id_list', pg.ARRAY(pg.BYTEA(16))), @@ -54,13 +55,19 @@ class Tasks(ExtendTable): Column('terminate', Boolean, default=False) ] + __add_result_links__ = [ + Column('result_links', pg.JSON), + ] + + __schema__ = copy.copy(__baseschema__) + __schema__.extend(__add_result_links__) class ResultMessage(ExtendTable): """Table for tracking result/status messages.""" __tablename__ = 'result_message' - __schema__ = [ + __baseschema__ = [ Column('sequence', Integer, primary_key=True), Column('task_id', pg.BYTEA(16)), Column('message', String(1024)), @@ -71,37 +78,43 @@ class ResultMessage(ExtendTable): Column('extra', pg.JSON) ] + __schema__ = copy.copy(__baseschema__) + class ActiveInstance(ExtendTable): """Table to organize multiple orchestrator instances.""" __tablename__ = 'active_instance' - __schema__ = [ + __baseschema__ = [ Column('dummy_key', Integer, primary_key=True), Column('identity', pg.BYTEA(16)), Column('last_ping', DateTime), ] + __schema__ = copy.copy(__baseschema__) + class BootAction(ExtendTable): """Table persisting node build data.""" __tablename__ = 'boot_action' - __schema__ = [ + __baseschema__ = [ Column('node_name', String(280), primary_key=True), Column('task_id', pg.BYTEA(16)), Column('identity_key', pg.BYTEA(32)), ] + __schema__ = copy.copy(__baseschema__) + class BootActionStatus(ExtendTable): """Table tracking status of node boot actions.""" __tablename__ = 'boot_action_status' - __schema__ = [ + __baseschema__ = [ Column('node_name', String(280), index=True), Column('action_id', pg.BYTEA(16), primary_key=True), Column('action_name', String(64)), @@ -110,13 +123,15 @@ class BootActionStatus(ExtendTable): Column('action_status', String(32)), ] + __schema__ = copy.copy(__baseschema__) + class BuildData(ExtendTable): """Table for persisting node build data.""" __tablename__ = 'build_data' - __schema__ = [ + __baseschema__ = [ Column('node_name', String(32), index=True), Column('task_id', pg.BYTEA(16), index=True), Column('collected_date', DateTime), @@ -124,3 +139,5 @@ class BuildData(ExtendTable): Column('data_format', String(32)), Column('data_element', Text), ] + + __schema__ = copy.copy(__baseschema__) diff --git a/etc/drydock/drydock.conf.sample b/etc/drydock/drydock.conf.sample index a8fbb556..09eb4b9f 100644 --- a/etc/drydock/drydock.conf.sample +++ b/etc/drydock/drydock.conf.sample @@ -26,6 +26,9 @@ # The URI database connect string. (string value) #database_connect_string = +# The SQLalchemy database connection pool size. (integer value) +#pool_size = 15 + [keystone_authtoken] diff --git a/tests/unit/test_task_link.py b/tests/unit/test_task_link.py new file mode 100644 index 00000000..2263e858 --- /dev/null +++ b/tests/unit/test_task_link.py @@ -0,0 +1,114 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +'''Tests the functions for adding and retrieving task status links.''' +from drydock_provisioner.objects import TaskStatus + + +class TestTaskStatusLinks(): + def test_links_add(self): + '''Add a link to a task status.''' + ts = TaskStatus() + + relation = 'test' + uri = 'http://foo.com/test' + + ts.add_link(relation, uri) + + assert relation in ts.links + assert uri in ts.links.get(relation, []) + + def test_links_get_empty(self): + '''Get links with an empty list.''' + ts = TaskStatus() + + links = ts.get_links() + + assert len(links) == 0 + + relation = 'test' + uri = 'http://foo.com/test' + + ts.add_link(relation, uri) + links = ts.get_links(relation='none') + + assert len(links) == 0 + + def test_links_get_all(self): + '''Get all links in a task status.''' + ts = TaskStatus() + + relation = 'test' + uri = 'http://foo.com/test' + + ts.add_link(relation, uri) + links = ts.get_links() + + assert len(links) == 1 + assert uri in links + + def test_links_get_all_duplicate_relation(self): + '''Get all links where a relation has multiple uris.''' + ts = TaskStatus() + + relation = 'test' + uri = 'http://foo.com/test' + uri2 = 'http://baz.com/test' + + ts.add_link(relation, uri) + ts.add_link(relation, uri2) + + links = ts.get_links() + + assert len(links) == 2 + assert uri in links + assert uri2 in links + + def test_links_get_filter(self): + '''Get links with a filter.''' + ts = TaskStatus() + + relation = 'test' + uri = 'http://foo.com/test' + + relation2 = 'test2' + uri2 = 'http://baz.com/test' + + ts.add_link(relation, uri) + ts.add_link(relation2, uri2) + + links = ts.get_links(relation=relation) + + assert len(links) == 1 + assert uri in links + + links = ts.get_links(relation=relation2) + + assert len(links) == 1 + assert uri2 in links + + def test_links_serialization(self): + '''Check that task status serilization contains links correctly.''' + ts = TaskStatus() + + relation = 'test' + uri = 'http://bar.com' + + ts.set_message('foo') + ts.set_reason('bar') + ts.add_link(relation, uri) + + ts_dict = ts.to_dict() + + assert isinstance(ts_dict.get('links'), list) + assert {'rel': relation, 'href': uri} in ts_dict.get('links', [])