airship/deckhand
Code Issues Proposed changes

Merge pull request #10 from att-comdev/revisions-api

Browse Source 
Revisions database and API implementation
This commit is contained in:
Scott Hussey 2017-08-02 15:10:34 -05:00 committed by GitHub
commit 9bbc767b0a
30 changed files with 1170 additions and 584 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
deckhand/common/timeutils.py
View File

@ -1,88 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Time related utilities and helper functions.
"""
import datetime
import iso8601
from monotonic import monotonic as now # noqa
from oslo_utils import encodeutils
# ISO 8601 extended time format with microseconds
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
def isotime(at=None, subsecond=False):
"""Stringify time in ISO 8601 format."""
if not at:
at = utcnow()
st = at.strftime(_ISO8601_TIME_FORMAT
if not subsecond
else _ISO8601_TIME_FORMAT_SUBSECOND)
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
st += ('Z' if tz == 'UTC' else tz)
return st
def parse_isotime(timestr):
"""Parse time from ISO 8601 format."""
try:
return iso8601.parse_date(timestr)
except iso8601.ParseError as e:
raise ValueError(encodeutils.exception_to_unicode(e))
except TypeError as e:
raise ValueError(encodeutils.exception_to_unicode(e))
def utcnow(with_timezone=False):
"""Overridable version of utils.utcnow that can return a TZ-aware datetime.
"""
if utcnow.override_time:
try:
return utcnow.override_time.pop(0)
except AttributeError:
return utcnow.override_time
if with_timezone:
return datetime.datetime.now(tz=iso8601.iso8601.UTC)
return datetime.datetime.utcnow()
def normalize_time(timestamp):
"""Normalize time in arbitrary timezone to UTC naive object."""
offset = timestamp.utcoffset()
if offset is None:
return timestamp
return timestamp.replace(tzinfo=None) - offset
def iso8601_from_timestamp(timestamp, microsecond=False):
"""Returns an iso8601 formatted date from timestamp."""
return isotime(datetime.datetime.utcfromtimestamp(timestamp), microsecond)
utcnow.override_time = None
def delta_seconds(before, after):
"""Return the difference between two timing objects.
Compute the difference in seconds between two date, time, or
datetime objects (as a float, to microsecond resolution).
"""
delta = after - before
return datetime.timedelta.total_seconds(delta)

155
deckhand/control/README.rst
View File

@ -7,13 +7,150 @@ Deckhand-managed data.
v1.0 Endpoints
--------------
/api/v1.0/documents
~~~~~~~~~~~~~~~~~~~
POST `/documents`
~~~~~~~~~~~~~~~~~
POST - Create a new YAML document and return a revision number. If the YAML
document already exists, then the document will be replaced and a new
revision number will be returned.
Accepts a multi-document YAML body and creates a new revision which adds
those documents. Updates are detected based on exact match to an existing
document of `schema` + `metadata.name`. Documents are "deleted" by including
documents with the tombstone metadata schema, such as:
```yaml
---
schema: any-namespace/AnyKind/v1
metadata:
schema: metadata/Tombstone/v1
name: name-to-delete
...
```
This endpoint is the only way to add, update, and delete documents. This
triggers Deckhand's internal schema validations for all documents.
If no changes are detected, a new revision should not be created. This allows
services to periodically re-register their schemas without creating
unnecessary revisions.
Sample response:
```yaml
---
created_at: '2017-07-31T14:46:46.119853'
data:
path:
to:
merge:
into:
ignored: {data: here}
parent: {foo: bar}
substitution: {target: null}
deleted: false
deleted_at: null
id: f99630d9-a89c-4aad-9aaa-7c44462047c1
metadata:
labels: {genesis: enabled, master: enabled}
layeringDefinition:
abstract: false
actions:
- {method: merge, path: .path.to.merge.into.parent}
- {method: delete, path: .path.to.delete}
layer: region
parentSelector: {required_key_a: required_label_a, required_key_b: required_label_b}
name: unique-name-given-schema
schema: metadata/Document/v1
storagePolicy: cleartext
substitutions:
- dest: {path: .substitution.target}
src: {name: name-of-source-document, path: .source.path, schema: another-service/SourceType/v1}
name: unique-name-given-schema
revision_id: 0206088a-c9e9-48e1-8725-c9bdac15d6b7
schema: some-service/ResourceType/v1
updated_at: '2017-07-31T14:46:46.119858'
```
GET `/revisions`
~~~~~~~~~~~~~~~~
Lists existing revisions and reports basic details including a summary of
validation status for each `deckhand/ValidationPolicy` that is part of that
revision.
Sample response:
```yaml
---
count: 7
next: https://deckhand/api/v1.0/revisions?limit=2&offset=2
prev: null
results:
- id: 0
url: https://deckhand/api/v1.0/revisions/0
createdAt: 2017-07-14T21:23Z
validationPolicies:
site-deploy-validation:
status: failed
- id: 1
url: https://deckhand/api/v1.0/revisions/1
createdAt: 2017-07-16T01:15Z
validationPolicies:
site-deploy-validation:
status: succeeded
...
```
GET `/revisions/{revision_id}/documents`
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Returns a multi-document YAML response containing all the documents matching
the filters specified via query string parameters. Returned documents will be
as originally posted with no substitutions or layering applied.
Supported query string parameters:
* `schema` - string, optional - The top-level `schema` field to select. This
may be partially specified by section, e.g., `schema=promenade` would select all
`kind` and `version` schemas owned by promenade, or `schema=promenade/Node`
which would select all versions of `promenade/Node` documents. One may not
partially specify the namespace or kind, so `schema=promenade/No` would not
select `promenade/Node/v1` documents, and `schema=prom` would not select
`promenade` documents.
* `metadata.name` - string, optional
* `metadata.layeringDefinition.abstract` - string, optional - Valid values are
the "true" and "false".
* `metadata.layeringDefinition.layer` - string, optional - Only return documents from
the specified layer.
* `metadata.label` - string, optional, repeatable - Uses the format
`metadata.label=key=value`. Repeating this parameter indicates all
requested labels must apply (AND not OR).
Sample response:
```yaml
created_at: '2017-07-31T14:36:00.352701'
data: {foo: bar}
deleted: false
deleted_at: null
id: ffba233a-326b-4eed-9b21-079ebd2a53f0
metadata:
labels: {genesis: enabled, master: enabled}
layeringDefinition:
abstract: false
actions:
- {method: merge, path: .path.to.merge.into.parent}
- {method: delete, path: .path.to.delete}
layer: region
parentSelector: {required_key_a: required_label_a, required_key_b: required_label_b}
name: foo-name-given-schema
schema: metadata/Document/v1
storagePolicy: cleartext
substitutions:
- dest: {path: .substitution.target}
src: {name: name-of-source-document, path: .source.path, schema: another-service/SourceType/v1}
name: foo-name-given-schema
revision_id: d3428d6a-d8c4-4a5b-8006-aba974cc36a2
schema: some-service/ResourceType/v1
updated_at: '2017-07-31T14:36:00.352705'
```
Testing
-------
@ -22,7 +159,9 @@ Document creation can be tested locally using (from root deckhand directory):
.. code-block:: console
curl -i -X POST localhost:9000/api/v1.0/documents \
-H "Content-Type: application/x-yaml" \
--data-binary "@deckhand/tests/unit/resources/sample.yaml"
$ curl -i -X POST localhost:9000/api/v1.0/documents \
-H "Content-Type: application/x-yaml" \
--data-binary "@deckhand/tests/unit/resources/sample.yaml"
# revision_id copy/pasted from previous response.
$ curl -i -X GET localhost:9000/api/v1.0/revisions/0e99c8b9-bab4-4fc7-8405-7dbd22c33a30/documents

6
deckhand/control/api.py
View File

@ -21,6 +21,8 @@ from oslo_log import log as logging
from deckhand.conf import config
from deckhand.control import base as api_base
from deckhand.control import documents
from deckhand.control import revision_documents
from deckhand.control import revisions
from deckhand.control import secrets
from deckhand.db.sqlalchemy import api as db_api
@ -68,6 +70,10 @@ def start_api(state_manager=None):
v1_0_routes = [
('documents', documents.DocumentsResource()),
('revisions', revisions.RevisionsResource()),
('revisions/{revision_id}', revisions.RevisionsResource()),
('revisions/{revision_id}/documents',
revision_documents.RevisionDocumentsResource()),
('secrets', secrets.SecretsResource())
]

25
deckhand/control/base.py
View File

@ -12,14 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import uuid
import yaml
import falcon
from falcon import request
from oslo_log import log as logging
from oslo_serialization import jsonutils as json
import six
from deckhand import errors
LOG = logging.getLogger(__name__)
class BaseResource(object):
"""Base resource class for implementing API resources."""
@ -72,15 +77,29 @@ class BaseResource(object):
def return_error(self, resp, status_code, message="", retry=False):
resp.body = json.dumps(
{'type': 'error', 'message': message, 'retry': retry})
{'type': 'error', 'message': six.text_type(message),
'retry': retry})
resp.status = status_code
def to_yaml_body(self, dict_body):
"""Converts JSON body into YAML response body.
:dict_body: response body to be converted to YAML.
:returns: YAML encoding of `dict_body`.
"""
if isinstance(dict_body, dict):
return yaml.safe_dump(dict_body)
elif isinstance(dict_body, list):
return yaml.safe_dump_all(dict_body)
raise TypeError('Unrecognized dict_body type when converting response '
'body to YAML format.')
class DeckhandRequestContext(object):
def __init__(self):
self.user = None
self.roles = ['anyone']
self.roles = []
self.request_id = str(uuid.uuid4())
def set_user(self, user):

31
deckhand/control/common.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import string
def to_camel_case(s):
return (s[0].lower() + string.capwords(s, sep='_').replace('_', '')[1:]
if s else s)
class ViewBuilder(object):
"""Model API responses as dictionaries."""
_collection_name = None
def _gen_url(self, revision):
# TODO: Use a config-based url for the base url below.
base_url = 'https://deckhand/api/v1.0/%s/%s'
return base_url % (self._collection_name, revision.get('id'))

21
deckhand/control/documents.py
View File

@ -32,16 +32,6 @@ LOG = logging.getLogger(__name__)
class DocumentsResource(api_base.BaseResource):
"""API resource for realizing CRUD endpoints for Documents."""
def __init__(self, **kwargs):
super(DocumentsResource, self).__init__(**kwargs)
self.authorized_roles = ['user']
def on_get(self, req, resp):
pass
def on_head(self, req, resp):
pass
def on_post(self, req, resp):
"""Create a document. Accepts YAML data only."""
if req.content_type != 'application/x-yaml':
@ -57,10 +47,11 @@ class DocumentsResource(api_base.BaseResource):
LOG.error(error_msg)
return self.return_error(resp, falcon.HTTP_400, message=error_msg)
# Validate the document before doing anything with it.
# All concrete documents in the payload must successfully pass their
# JSON schema validations. Otherwise raise an error.
try:
for doc in documents:
document_validation.DocumentValidation(doc)
document_validation.DocumentValidation(doc).pre_validate()
except deckhand_errors.InvalidFormat as e:
return self.return_error(resp, falcon.HTTP_400, message=e)
@ -72,7 +63,5 @@ class DocumentsResource(api_base.BaseResource):
return self.return_error(resp, falcon.HTTP_500, message=e)
resp.status = falcon.HTTP_201
resp.body = json.dumps(created_documents)
def _check_document_exists(self):
pass
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(created_documents)

42
deckhand/control/revision_documents.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from oslo_db import exception as db_exc
from deckhand.control import base as api_base
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
class RevisionDocumentsResource(api_base.BaseResource):
"""API resource for realizing CRUD endpoints for Document Revisions."""
def on_get(self, req, resp, revision_id):
"""Returns all documents for a `revision_id`.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will be as originally posted with no substitutions or
layering applied.
"""
params = req.params
try:
documents = db_api.revision_get_documents(revision_id, **params)
except errors.RevisionNotFound as e:
return self.return_error(resp, falcon.HTTP_404, message=e)
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(documents)

60
deckhand/control/revisions.py Normal file
View File

@ -0,0 +1,60 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from deckhand.control import base as api_base
from deckhand.control.views import revision as revision_view
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
class RevisionsResource(api_base.BaseResource):
"""API resource for realizing CRUD operations for revisions."""
def on_get(self, req, resp, revision_id=None):
"""Returns list of existing revisions.
Lists existing revisions and reports basic details including a summary
of validation status for each `deckhand/ValidationPolicy` that is part
of each revision.
"""
if revision_id:
self._show_revision(req, resp, revision_id=revision_id)
else:
self._list_revisions(req, resp)
def _show_revision(self, req, resp, revision_id):
"""Returns detailed description of a particular revision.
The status of each ValidationPolicy belonging to the revision is also
included.
"""
try:
revision = db_api.revision_get(revision_id)
except errors.RevisionNotFound as e:
return self.return_error(resp, falcon.HTTP_404, message=e)
revision_resp = revision_view.ViewBuilder().show(revision)
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(revision_resp)
def _list_revisions(self, req, resp):
revisions = db_api.revision_get_all()
revisions_resp = revision_view.ViewBuilder().list(revisions)
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(revisions_resp)

0
deckhand/common/__init__.py → deckhand/control/views/__init__.py
View File

47
deckhand/control/views/revision.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.control import common
class ViewBuilder(common.ViewBuilder):
"""Model revision API responses as a python dictionary."""
_collection_name = 'revisions'
def list(self, revisions):
resp_body = {
'count': len(revisions),
'next': None,
'prev': None,
'results': []
}
for revision in revisions:
result = {}
for attr in ('id', 'created_at'):
result[common.to_camel_case(attr)] = revision[attr]
result['count'] = len(revision.pop('documents'))
resp_body['results'].append(result)
return resp_body
def show(self, revision):
return {
'id': revision.get('id'),
'createdAt': revision.get('created_at'),
'url': self._gen_url(revision),
# TODO: Not yet implemented.
'validationPolicies': [],
}

155
deckhand/db/sqlalchemy/api.py
View File

@ -15,6 +15,8 @@
"""Defines interface for DB access."""
import ast
import copy
import datetime
import threading
@ -28,12 +30,15 @@ import six
from six.moves import range
import sqlalchemy
from sqlalchemy.ext.compiler import compiles
from sqlalchemy import desc
from sqlalchemy import MetaData, Table
import sqlalchemy.orm as sa_orm
from sqlalchemy import sql
import sqlalchemy.sql as sa_sql
from deckhand.db.sqlalchemy import models
from deckhand import errors
from deckhand import utils
sa_logger = None
LOG = logging.getLogger(__name__)
@ -112,16 +117,146 @@ def documents_create(documents, session=None):
return created_docs
def document_create(values, session=None):
"""Create a document."""
values = values.copy()
values['doc_metadata'] = values.pop('metadata')
values['schema_version'] = values.pop('schemaVersion')
def documents_create(values_list, session=None):
"""Create a set of documents and associated schema.
If no changes are detected, a new revision will not be created. This
allows services to periodically re-register their schemas without
creating unnecessary revisions.
"""
values_list = copy.deepcopy(values_list)
session = session or get_session()
document = models.Document()
with session.begin():
document.update(values)
document.save(session=session)
filters = models.Document.UNIQUE_CONSTRAINTS
return document.to_dict()
do_create = False
documents_created = []
def _document_changed(existing_document):
# The document has changed if at least one value in ``values`` differs.
for key, val in values.items():
if val != existing_document[key]:
return True
return False
def _document_create(values):
document = models.Document()
with session.begin():
document.update(values)
document.save(session=session)
return document.to_dict()
for values in values_list:
values['_metadata'] = values.pop('metadata')
values['name'] = values['_metadata']['name']
try:
existing_document = document_get(
raw_dict=True,
**{c: values[c] for c in filters if c != 'revision_id'})
except db_exception.DBError:
# Ignore bad data at this point. Allow creation to bubble up the
# error related to bad data.
existing_document = None
if not existing_document:
do_create = True
elif existing_document and _document_changed(existing_document):
do_create = True
if do_create:
revision = revision_create()
for values in values_list:
values['revision_id'] = revision['id']
doc = _document_create(values)
documents_created.append(doc)
return documents_created
def document_get(session=None, raw_dict=False, **filters):
session = session or get_session()
document = session.query(models.Document).filter_by(**filters).first()
return document.to_dict(raw_dict=raw_dict) if document else {}
####################
def revision_create(session=None):
session = session or get_session()
revision = models.Revision()
with session.begin():
revision.save(session=session)
return revision.to_dict()
def revision_get(revision_id, session=None):
"""Return the specified `revision_id`.
:raises: RevisionNotFound if the revision was not found.
"""
session = session or get_session()
try:
revision = session.query(models.Revision).filter_by(
id=revision_id).one().to_dict()
except sa_orm.exc.NoResultFound:
raise errors.RevisionNotFound(revision=revision_id)
return revision
def revision_get_all(session=None):
"""Return list of all revisions."""
session = session or get_session()
revisions = session.query(models.Revision).all()
return [r.to_dict() for r in revisions]
def revision_get_documents(revision_id, session=None, **filters):
"""Return the documents that match filters for the specified `revision_id`.
:raises: RevisionNotFound if the revision was not found.
"""
session = session or get_session()
try:
revision = session.query(models.Revision).filter_by(
id=revision_id).one().to_dict()
except sa_orm.exc.NoResultFound:
raise errors.RevisionNotFound(revision=revision_id)
filtered_documents = _filter_revision_documents(
revision['documents'], **filters)
return filtered_documents
def _filter_revision_documents(documents, **filters):
"""Return the list of documents that match filters.
:returns: list of documents that match specified filters.
"""
# TODO: Implement this as an sqlalchemy query.
filtered_documents = []
for document in documents:
match = True
for filter_key, filter_val in filters.items():
actual_val = utils.multi_getattr(filter_key, document)
if (isinstance(actual_val, bool)
and isinstance(filter_val, six.text_type)):
try:
filter_val = ast.literal_eval(filter_val.title())
except ValueError:
# If not True/False, set to None to avoid matching
# `actual_val` which is always boolean.
filter_val = None
if actual_val != filter_val:
match = False
if match:
filtered_documents.append(document)
return filtered_documents

92
deckhand/db/sqlalchemy/models.py
View File

@ -15,52 +15,27 @@
import uuid
from oslo_db.sqlalchemy import models
from oslo_log import log as logging
from oslo_serialization import jsonutils as json
from oslo_db.sqlalchemy import types as oslo_types
from oslo_utils import timeutils
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy.ext import declarative
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import orm
from sqlalchemy.orm import backref, relationship
from sqlalchemy import schema
from sqlalchemy import String
from sqlalchemy import Text
from sqlalchemy.types import TypeDecorator
from deckhand.common import timeutils
LOG = logging.getLogger(__name__)
# Declarative base class which maintains a catalog of classes and tables
# relative to that base.
BASE = declarative.declarative_base()
class JSONEncodedDict(TypeDecorator):
"""Represents an immutable structure as a json-encoded string.
Usage::
JSONEncodedDict(255)
"""
impl = Text
def process_bind_param(self, value, dialect):
if value is not None:
value = json.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = json.loads(value)
return value
class DeckhandBase(models.ModelBase, models.TimestampMixin):
"""Base class for Deckhand Models."""
@ -101,31 +76,68 @@ class DeckhandBase(models.ModelBase, models.TimestampMixin):
# CircularReference.
d.pop("_sa_instance_state")
if 'deleted_at' not in d:
d.setdefault('deleted_at', None)
for k in ["created_at", "updated_at", "deleted_at", "deleted"]:
if k in d and d[k]:
d[k] = d[k].isoformat()
return d
@staticmethod
def gen_unqiue_contraint(self, *fields):
constraint_name = 'ix_' + self.__class__.__name__.lower() + '_'
for field in fields:
constraint_name = constraint_name + '_%s' % field
return schema.UniqueConstraint(*fields, name=constraint_name)
class Document(BASE, DeckhandBase):
__tablename__ = 'document'
__table_args__ = (schema.UniqueConstraint('schema_version', 'kind',
name='ix_documents_schema_version_kind'),)
class Revision(BASE, DeckhandBase):
__tablename__ = 'revisions'
id = Column(String(36), primary_key=True,
default=lambda: str(uuid.uuid4()))
# TODO: the revision_index will be a foreign key to a Revision table.
revision_index = Column(String(36), nullable=False,
default=lambda: str(uuid.uuid4()))
schema_version = Column(String(64), nullable=False)
kind = Column(String(64), nullable=False)
parent_id = Column(Integer, ForeignKey('revisions.id'), nullable=True)
child_id = Column(Integer, ForeignKey('revisions.id'), nullable=True)
results = Column(oslo_types.JsonEncodedList(), nullable=True)
documents = relationship("Document")
def to_dict(self):
d = super(Revision, self).to_dict()
d['documents'] = [doc.to_dict() for doc in self.documents]
return d
class Document(BASE, DeckhandBase):
UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id')
__tablename__ = 'documents'
__table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),)
id = Column(String(36), primary_key=True,
default=lambda: str(uuid.uuid4()))
schema = Column(String(64), nullable=False)
name = Column(String(64), nullable=False)
# NOTE: Do not define a maximum length for these JSON data below. However,
# this approach is not compatible with all database types.
# "metadata" is reserved, so use "doc_metadata" instead.
doc_metadata = Column(JSONEncodedDict(), nullable=False)
data = Column(JSONEncodedDict(), nullable=False)
# "metadata" is reserved, so use "_metadata" instead.
_metadata = Column(oslo_types.JsonEncodedDict(), nullable=False)
data = Column(oslo_types.JsonEncodedDict(), nullable=False)
revision_id = Column(Integer, ForeignKey('revisions.id'), nullable=False)
def to_dict(self, raw_dict=False):
"""Convert the ``Document`` object into a dictionary format.
:param raw_dict: if True, returns unmodified data; else returns data
expected by users.
:returns: dictionary format of ``Document`` object.
"""
d = super(Document, self).to_dict()
# ``_metadata`` is used in the DB schema as ``metadata`` is reserved.
if not raw_dict:
d['metadata'] = d.pop('_metadata')
return d
def register_models(engine):
"""Create database tables for all models with the given engine."""

101
deckhand/engine/document_validation.py
View File

@ -13,16 +13,20 @@
# limitations under the License.
import jsonschema
from oslo_log import log as logging
import six
from deckhand.engine.schema.v1_0 import default_schema
from deckhand.engine.schema.v1_0 import default_policy_validation
from deckhand.engine.schema.v1_0 import default_schema_validation
from deckhand import errors
LOG = logging.getLogger(__name__)
class DocumentValidation(object):
"""Class for document validation logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file.
This class is responsible for performing built-in validations on Documents.
:param data: YAML data that requires secrets to be validated, merged and
consolidated.
@ -30,7 +34,6 @@ class DocumentValidation(object):
def __init__(self, data):
self.data = data
self.pre_validate_data()
class SchemaVersion(object):
"""Class for retrieving correct schema for pre-validation on YAML.
@ -38,81 +41,59 @@ class DocumentValidation(object):
Retrieves the schema that corresponds to "apiVersion" in the YAML
data. This schema is responsible for performing pre-validation on
YAML data.
The built-in validation schemas that are always executed include:
- `deckhand-document-schema-validation`
- `deckhand-policy-validation`
"""
# TODO: Update kind according to requirements.
schema_versions_info = [{'version': 'v1', 'kind': 'default',
'schema': default_schema}]
# TODO: Use the correct validation based on the Document's schema.
internal_validations = [
{'version': 'v1', 'fqn': 'deckhand-document-schema-validation',
'schema': default_schema_validation},
{'version': 'v1', 'fqn': 'deckhand-policy-validation',
'schema': default_policy_validation}]
def __init__(self, schema_version, kind):
def __init__(self, schema_version):
self.schema_version = schema_version
self.kind = kind
@property
def schema(self):
# TODO: return schema based on version and kind.
return [v['schema'] for v in self.schema_versions_info
# TODO: return schema based on Document's schema.
return [v['schema'] for v in self.internal_validations
if v['version'] == self.schema_version][0].schema
def pre_validate_data(self):
def pre_validate(self):
"""Pre-validate that the YAML file is correctly formatted."""
self._validate_with_schema()
# TODO(fm577c): Query Deckhand API to validate "src" values.
@property
def doc_name(self):
return (self.data['schemaVersion'] + self.data['kind'] +
self.data['metadata']['name'])
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.
# Validate the document using the document's ``schema``. Only validate
# concrete documents.
try:
schema_version = self.data['schemaVersion'].split('/')[-1]
doc_kind = self.data['kind']
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
abstract = self.data['metadata']['layeringDefinition'][
'abstract']
is_abstract = six.text_type(abstract).lower() == 'true'
except KeyError as e:
raise errors.InvalidFormat(
"Could not find 'abstract' property from document.")
# TODO: This should be done inside a different module.
if is_abstract:
LOG.info(
"Skipping validation for the document because it is abstract")
return
try:
schema_version = self.data['schema'].split('/')[-1]
doc_schema_version = self.SchemaVersion(schema_version)
except (AttributeError, IndexError, KeyError) as e:
raise errors.InvalidFormat(
'The provided schemaVersion is invalid or missing. Exception: '
'The provided schema is invalid or missing. Exception: '
'%s.' % e)
try:
jsonschema.validate(self.data, doc_schema_version.schema)
except jsonschema.exceptions.ValidationError as e:
raise errors.InvalidFormat('The provided YAML file is invalid. '
'Exception: %s.' % e.message)
def _multi_getattr(self, multi_key, substitutable_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: Tuple where first value is a boolean indicating that the
nested attribute was found and the second value is the attribute
that was not found, if applicable.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = substitutable_data
for attr in attrs:
if attr not in data:
return False, attr
data = data.get(attr)
return True, None

13
deckhand/engine/schema/v1_0/default_policy_validation.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

60
deckhand/engine/schema/v1_0/default_schema.py → deckhand/engine/schema/v1_0/default_schema_validation.py
View File

@ -18,8 +18,7 @@ substitution_schema = {
'dest': {
'type': 'object',
'properties': {
'path': {'type': 'string'},
'replacePattern': {'type': 'string'}
'path': {'type': 'string'}
},
'additionalProperties': False,
# 'replacePattern' is not required.
@ -28,12 +27,12 @@ substitution_schema = {
'src': {
'type': 'object',
'properties': {
'kind': {'type': 'string'},
'schema': {'type': 'string'},
'name': {'type': 'string'},
'path': {'type': 'string'}
},
'additionalProperties': False,
'required': ['kind', 'name', 'path']
'required': ['schema', 'name', 'path']
}
},
'additionalProperties': False,
@ -43,45 +42,46 @@ substitution_schema = {
schema = {
'type': 'object',
'properties': {
'schemaVersion': {
'schema': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
'pattern': '^(.*\/v[0-9]{1})$'
},
# TODO: The kind should be an enum.
'kind': {'type': 'string'},
'metadata': {
'type': 'object',
'properties': {
'metadataVersion': {
'schema': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
'pattern': '^(.*/v[0-9]{1})$'
},
'name': {'type': 'string'},
'storagePolicy': {'type': 'string'},
'labels': {
'type': 'object',
'properties': {
'component': {'type': 'string'},
'hostname': {'type': 'string'}
},
'additionalProperties': False,
'required': ['component', 'hostname']
'type': 'object'
},
'layerDefinition': {
'layeringDefinition': {
'type': 'object',
'properties': {
'layer': {'enum': ['global', 'region', 'site']},
'layer': {'type': 'string'},
'abstract': {'type': 'boolean'},
'childSelector': {
'type': 'object',
'properties': {
'label': {'type': 'string'}
},
'additionalProperties': False,
'required': ['label']
'parentSelector': {
'type': 'object'
},
'actions': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'method': {'enum': ['merge', 'delete',
'replace']},
'path': {'type': 'string'}
},
'additionalProperties': False,
'required': ['method', 'path']
}
}
},
'additionalProperties': False,
'required': ['layer', 'abstract', 'childSelector']
'required': ['layer', 'abstract', 'parentSelector']
},
'substitutions': {
'type': 'array',
@ -89,13 +89,13 @@ schema = {
}
},
'additionalProperties': False,
'required': ['metadataVersion', 'name', 'labels',
'layerDefinition', 'substitutions']
'required': ['schema', 'name', 'storagePolicy', 'labels',
'layeringDefinition', 'substitutions']
},
'data': {
'type': 'object'
}
},
'additionalProperties': False,
'required': ['schemaVersion', 'kind', 'metadata', 'data']
'required': ['schema', 'metadata', 'data']
}

138
deckhand/engine/secret_substitution.py
View File

@ -1,138 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import jsonschema
from deckhand.engine.schema.v1_0 import default_schema
from deckhand import errors
class SecretSubstitution(object):
"""Class for secret substitution logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file. Afterward, secret values will be
substituted or "forward-repalced" into the YAML file. The end result is a
YAML file containing all necessary secrets to be handed off to other
services.
:param data: YAML data that requires secrets to be validated, merged and
consolidated.
"""
def __init__(self, data):
try:
self.data = yaml.safe_load(data)
except yaml.YAMLError:
raise errors.InvalidFormat(
'The provided YAML file cannot be parsed.')
self.pre_validate_data()
class SchemaVersion(object):
"""Class for retrieving correct schema for pre-validation on YAML.
Retrieves the schema that corresponds to "apiVersion" in the YAML
data. This schema is responsible for performing pre-validation on
YAML data.
"""
# TODO: Update kind according to requirements.
schema_versions_info = [{'version': 'v1', 'kind': 'default',
'schema': default_schema}]
def __init__(self, schema_version, kind):
self.schema_version = schema_version
self.kind = kind
@property
def schema(self):
# TODO: return schema based on version and kind.
return [v['schema'] for v in self.schema_versions_info
if v['version'] == self.schema_version][0].schema
def pre_validate_data(self):
"""Pre-validate that the YAML file is correctly formatted."""
self._validate_with_schema()
# Validate that each "dest" field exists in the YAML data.
# FIXME(fm577c): Dest fields will be injected if not present - the
# validation below needs to be updated or removed.
substitutions = self.data['metadata']['substitutions']
destinations = [s['dest'] for s in substitutions]
sub_data = self.data['data']
for dest in destinations:
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
if not result:
raise errors.InvalidFormat(
'The attribute "%s" included in the "dest" field "%s" is '
'missing from the YAML data: "%s".' % (
missing_attr, dest, sub_data))
# TODO(fm577c): Query Deckhand API to validate "src" values.
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.
try:
schema_version = self.data['schemaVersion'].split('/')[-1]
doc_kind = self.data['kind']
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
except (AttributeError, IndexError, KeyError) as e:
raise errors.InvalidFormat(
'The provided schemaVersion is invalid or missing. Exception: '
'%s.' % e)
try:
jsonschema.validate(self.data, doc_schema_version.schema)
except jsonschema.exceptions.ValidationError as e:
raise errors.InvalidFormat('The provided YAML file is invalid. '
'Exception: %s.' % e.message)
def _multi_getattr(self, multi_key, substitutable_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: Tuple where first value is a boolean indicating that the
nested attribute was found and the second value is the attribute
that was not found, if applicable.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = substitutable_data
for attr in attrs:
if attr not in data:
return False, attr
data = data.get(attr)
return True, None

6
deckhand/errors.py
View File

@ -14,7 +14,7 @@
class DeckhandException(Exception):
"""Base Nova Exception
"""Base Deckhand Exception
To correctly use this class, inherit from it and define
a 'msg_fmt' property. That msg_fmt will get printf'd
with the keyword arguments provided to the constructor.
@ -57,3 +57,7 @@ class DocumentExists(DeckhandException):
msg_fmt = ("Document with kind %(kind)s and schemaVersion "
"%(schema_version)s already exists.")
class RevisionNotFound(DeckhandException):
msg_fmt = ("The requested revision %(revision)s was not found.")
code = 403

62
deckhand/tests/test_utils.py Normal file
View File

@ -0,0 +1,62 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import random
import uuid
def rand_uuid_hex():
"""Generate a random UUID hex string
:return: a random UUID (e.g. '0b98cf96d90447bda4b46f31aeb1508c')
:rtype: string
"""
return uuid.uuid4().hex
def rand_name(name='', prefix='deckhand'):
"""Generate a random name that includes a random number
:param str name: The name that you want to include
:param str prefix: The prefix that you want to include
:return: a random name. The format is
'<prefix>-<name>-<random number>'.
(e.g. 'prefixfoo-namebar-154876201')
:rtype: string
"""
randbits = str(random.randint(1, 0x7fffffff))
rand_name = randbits
if name:
rand_name = name + '-' + rand_name
if prefix:
rand_name = prefix + '-' + rand_name
return rand_name
def rand_bool():
"""Generate a random boolean value.
:return: a random boolean value.
:rtype: boolean
"""
return random.choice([True, False])
def rand_int(min, max):
"""Generate a random integer value between range (`min`, `max`).
:return: a random integer between the range(`min`, `max`).
:rtype: integer
"""
return random.randint(min, max)

3
deckhand/tests/unit/base.py
View File

@ -36,6 +36,9 @@ class DeckhandTestCase(testtools.TestCase):
CONF.set_override(name, override, group)
self.addCleanup(CONF.clear_override, name, group)
def assertEmpty(self, list):
self.assertEqual(0, len(list))
class DeckhandWithDBTestCase(DeckhandTestCase):

26
deckhand/tests/unit/control/test_api.py
View File

@ -18,17 +18,27 @@ import testtools
from deckhand.control import api
from deckhand.control import base as api_base
from deckhand.control import documents
from deckhand.control import revision_documents
from deckhand.control import revisions
from deckhand.control import secrets
class TestApi(testtools.TestCase):
def setUp(self):
super(TestApi, self).setUp()
for resource in (documents, revisions, revision_documents, secrets):
resource_name = resource.__name__.split('.')[-1]
resource_obj = mock.patch.object(
resource, '%sResource' % resource_name.title().replace(
'_', '')).start()
setattr(self, '%s_resource' % resource_name, resource_obj)
@mock.patch.object(api, 'db_api', autospec=True)
@mock.patch.object(api, 'config', autospec=True)
@mock.patch.object(api, 'secrets', autospec=True)
@mock.patch.object(api, 'documents', autospec=True)
@mock.patch.object(api, 'falcon', autospec=True)
def test_start_api(self, mock_falcon, mock_documents, mock_secrets,
def test_start_api(self, mock_falcon,
mock_config, mock_db_api):
mock_falcon_api = mock_falcon.API.return_value
@ -38,9 +48,13 @@ class TestApi(testtools.TestCase):
mock_falcon.API.assert_called_once_with(
request_type=api_base.DeckhandRequest)
mock_falcon_api.add_route.assert_has_calls([
mock.call(
'/api/v1.0/documents', mock_documents.DocumentsResource()),
mock.call('/api/v1.0/secrets', mock_secrets.SecretsResource())
mock.call('/api/v1.0/documents', self.documents_resource()),
mock.call('/api/v1.0/revisions', self.revisions_resource()),
mock.call('/api/v1.0/revisions/{revision_id}',
self.revisions_resource()),
mock.call('/api/v1.0/revisions/{revision_id}/documents',
self.revision_documents_resource()),
mock.call('/api/v1.0/secrets', self.secrets_resource())
])
mock_config.parse_args.assert_called_once_with()
mock_db_api.setup_db.assert_called_once_with()

115
deckhand/tests/unit/db/base.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import testtools
from testtools import matchers
from deckhand.db.sqlalchemy import api as db_api
from deckhand.tests import test_utils
from deckhand.tests.unit import base
BASE_EXPECTED_FIELDS = ("created_at", "updated_at", "deleted_at", "deleted")
DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
"id", "schema", "name", "metadata", "data", "revision_id")
REVISION_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
"id", "child_id", "parent_id", "documents")
class DocumentFixture(object):
@staticmethod
def get_minimal_fixture(**kwargs):
fixture = {
'data': {
test_utils.rand_name('key'): test_utils.rand_name('value')
},
'metadata': {
'name': test_utils.rand_name('metadata_data'),
'label': test_utils.rand_name('metadata_label'),
'layeringDefinition': {
'abstract': test_utils.rand_bool(),
'layer': test_utils.rand_name('layer')
}
},
'schema': test_utils.rand_name('schema')}
fixture.update(kwargs)
return fixture
@staticmethod
def get_minimal_multi_fixture(count=2, **kwargs):
return [DocumentFixture.get_minimal_fixture(**kwargs)
for _ in range(count)]
class TestDbBase(base.DeckhandWithDBTestCase):
def _create_documents(self, payload):
if not isinstance(payload, list):
payload = [payload]
docs = db_api.documents_create(payload)
for idx, doc in enumerate(docs):
self._validate_document(expected=payload[idx], actual=doc)
return docs
def _get_document(self, **fields):
doc = db_api.document_get(**fields)
self._validate_document(actual=doc)
return doc
def _get_revision(self, revision_id):
revision = db_api.revision_get(revision_id)
self._validate_revision(revision)
return revision
def _get_revision_documents(self, revision_id, **filters):
documents = db_api.revision_get_documents(revision_id, **filters)
for document in documents:
self._validate_document(document)
return documents
def _list_revisions(self):
return db_api.revision_get_all()
def _validate_object(self, obj):
for attr in BASE_EXPECTED_FIELDS:
if attr.endswith('_at'):
self.assertThat(obj[attr], matchers.MatchesAny(
matchers.Is(None), matchers.IsInstance(str)))
else:
self.assertIsInstance(obj[attr], bool)
def _validate_document(self, actual, expected=None, is_deleted=False):
self._validate_object(actual)
# Validate that the document has all expected fields and is a dict.
expected_fields = list(DOCUMENT_EXPECTED_FIELDS)
if not is_deleted:
expected_fields.remove('deleted_at')
self.assertIsInstance(actual, dict)
for field in expected_fields:
self.assertIn(field, actual)
if expected:
# Validate that the expected values are equivalent to actual
# values.
for key, val in expected.items():
self.assertEqual(val, actual[key])
def _validate_revision(self, revision):
self._validate_object(revision)
for attr in REVISION_EXPECTED_FIELDS:
self.assertIn(attr, revision)

92
deckhand/tests/unit/db/test_documents.py
View File

@ -12,38 +12,76 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from deckhand.db.sqlalchemy import api as db_api
from deckhand.tests.unit import base
from deckhand.tests.unit.db import base
class DocumentFixture(object):
class TestDocuments(base.TestDbBase):
def get_minimal_fixture(self, **kwargs):
fixture = {'data': 'fake document data',
'metadata': 'fake meta',
'kind': 'FakeConfigType',
'schemaVersion': 'deckhand/v1'}
fixture.update(kwargs)
return fixture
def test_create_and_get_document(self):
payload = base.DocumentFixture.get_minimal_fixture()
documents = self._create_documents(payload)
self.assertIsInstance(documents, list)
self.assertEqual(1, len(documents))
class TestDocumentsApi(base.DeckhandWithDBTestCase):
for document in documents:
retrieved_document = self._get_document(id=document['id'])
self.assertEqual(document, retrieved_document)
def _validate_document(self, expected, actual):
expected['doc_metadata'] = expected.pop('metadata')
expected['schema_version'] = expected.pop('schemaVersion')
def test_create_document_again_with_no_changes(self):
payload = base.DocumentFixture.get_minimal_fixture()
self._create_documents(payload)
documents = self._create_documents(payload)
# TODO: Validate "status" fields, like created_at.
self.assertIsInstance(actual, dict)
for key, val in expected.items():
self.assertIn(key, actual)
self.assertEqual(val, actual[key])
self.assertIsInstance(documents, list)
self.assertEmpty(documents)
def test_create_document(self):
fixture = DocumentFixture().get_minimal_fixture()
document = db_api.document_create(fixture)
self._validate_document(fixture, document)
def test_create_document_and_get_revision(self):
payload = base.DocumentFixture.get_minimal_fixture()
documents = self._create_documents(payload)
self.assertIsInstance(documents, list)
self.assertEqual(1, len(documents))
for document in documents:
revision = self._get_revision(document['revision_id'])
self._validate_revision(revision)
self.assertEqual(document['revision_id'], revision['id'])
def test_get_documents_by_revision_id(self):
payload = base.DocumentFixture.get_minimal_fixture()
documents = self._create_documents(payload)
revision = self._get_revision(documents[0]['revision_id'])
self.assertEqual(1, len(revision['documents']))
self.assertEqual(documents[0], revision['documents'][0])
def test_get_multiple_documents_by_revision_id(self):
payload = base.DocumentFixture.get_minimal_multi_fixture(count=3)
documents = self._create_documents(payload)
self.assertIsInstance(documents, list)
self.assertEqual(3, len(documents))
for document in documents:
revision = self._get_revision(document['revision_id'])
self._validate_revision(revision)
self.assertEqual(document['revision_id'], revision['id'])
def test_get_documents_by_revision_id_and_filters(self):
payload = base.DocumentFixture.get_minimal_fixture()
document = self._create_documents(payload)[0]
filters = {
'schema': document['schema'],
'metadata.name': document['metadata']['name'],
'metadata.layeringDefinition.abstract':
document['metadata']['layeringDefinition']['abstract'],
'metadata.layeringDefinition.layer':
document['metadata']['layeringDefinition']['layer'],
'metadata.label': document['metadata']['label']
}
documents = self._get_revision_documents(
document['revision_id'], **filters)
self.assertEqual(1, len(documents))
self.assertEqual(document, documents[0])

39
deckhand/tests/unit/db/test_documents_negative.py Normal file
View File

@ -0,0 +1,39 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.tests.unit.db import base
class TestDocumentsNegative(base.TestDbBase):
def test_get_documents_by_revision_id_and_wrong_filters(self):
payload = base.DocumentFixture.get_minimal_fixture()
document = self._create_documents(payload)[0]
filters = {
'schema': 'fake_schema',
'metadata.name': 'fake_meta_name',
'metadata.layeringDefinition.abstract':
not document['metadata']['layeringDefinition']['abstract'],
'metadata.layeringDefinition.layer': 'fake_layer',
'metadata.label': 'fake_label'
}
documents = self._get_revision_documents(
document['revision_id'], **filters)
self.assertEmpty(documents)
for filter_key, filter_val in filters.items():
documents = self._get_revision_documents(
document['revision_id'], filter_key=filter_val)
self.assertEmpty(documents)

28
deckhand/tests/unit/db/test_revisions.py Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.tests.unit.db import base
class TestRevisionViews(base.TestDbBase):
def test_list(self):
payload = [base.DocumentFixture.get_minimal_fixture()
for _ in range(4)]
self._create_documents(payload)
revisions = self._list_revisions()
self.assertIsInstance(revisions, list)
self.assertEqual(1, len(revisions))
self.assertEqual(4, len(revisions[0]['documents']))

44
deckhand/tests/unit/engine/test_document_validation.py
View File

@ -17,6 +17,7 @@ import os
import testtools
import yaml
import mock
import six
from deckhand.engine import document_validation
@ -70,19 +71,15 @@ class TestDocumentValidation(testtools.TestCase):
return corrupted_data
def test_initialization(self):
doc_validation = document_validation.DocumentValidation(
self.data)
self.assertIsInstance(doc_validation,
document_validation.DocumentValidation)
doc_validation = document_validation.DocumentValidation(self.data)
doc_validation.pre_validate() # Should not raise any errors.
def test_initialization_missing_sections(self):
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
"is a required property.")
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'),
'metadataVersion'),
(self._corrupt_data('metadata.schema'), 'schema'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
@ -92,4 +89,35 @@ class TestDocumentValidation(testtools.TestCase):
for invalid_entry, missing_key in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err % missing_key):
document_validation.DocumentValidation(invalid_entry)
doc_validation = document_validation.DocumentValidation(
invalid_entry)
doc_validation.pre_validate()
def test_initialization_missing_abstract_section(self):
expected_err = ("Could not find 'abstract' property from document.")
invalid_data = [
self._corrupt_data('metadata'),
self._corrupt_data('metadata.layeringDefinition'),
self._corrupt_data('metadata.layeringDefinition.abstract'),
]
for invalid_entry in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err):
doc_validation = document_validation.DocumentValidation(
invalid_entry)
doc_validation.pre_validate()
@mock.patch.object(document_validation, 'LOG', autospec=True)
def test_initialization_with_abstract_document(self, mock_log):
abstract_data = copy.deepcopy(self.data)
for true_val in (True, 'true', 'True'):
abstract_data['metadata']['layeringDefinition']['abstract'] = True
doc_validation = document_validation.DocumentValidation(
abstract_data)
doc_validation.pre_validate()
mock_log.info.assert_called_once_with(
"Skipping validation for the document because it is abstract")
mock_log.info.reset_mock()

123
deckhand/tests/unit/engine/test_secret_substitution.py
View File

@ -1,123 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import testtools
import yaml
import six
from deckhand.engine import secret_substitution
from deckhand import errors
class TestSecretSubtitution(testtools.TestCase):
def setUp(self):
super(TestSecretSubtitution, self).setUp()
dir_path = os.path.dirname(os.path.realpath(__file__))
test_yaml_path = os.path.abspath(os.path.join(
dir_path, os.pardir, 'resources', 'sample.yaml'))
with open(test_yaml_path, 'r') as yaml_file:
yaml_data = yaml_file.read()
self.data = yaml.safe_load(yaml_data)
def _corrupt_data(self, key, data=None):
"""Corrupt test data to check that pre-validation works.
Corrupt data by removing a key from the document. Each key must
correspond to a value that is a dictionary.
:param key: The document key to be removed. The key can have the
following formats:
* 'data' => document.pop('data')
* 'metadata.name' => document['metadata'].pop('name')
* 'metadata.substitutions.0.dest' =>
document['metadata']['substitutions'][0].pop('dest')
:returns: Corrupted YAML data.
"""
if data is None:
data = self.data
corrupted_data = copy.deepcopy(data)
if '.' in key:
_corrupted_data = corrupted_data
nested_keys = key.split('.')
for nested_key in nested_keys:
if nested_key == nested_keys[-1]:
break
if nested_key.isdigit():
_corrupted_data = _corrupted_data[int(nested_key)]
else:
_corrupted_data = _corrupted_data[nested_key]
_corrupted_data.pop(nested_keys[-1])
else:
corrupted_data.pop(key)
return self._format_data(corrupted_data)
def _format_data(self, data=None):
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
if data is None:
data = self.data
return yaml.safe_dump(data)
def test_initialization(self):
sub = secret_substitution.SecretSubstitution(self._format_data())
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
def test_initialization_missing_sections(self):
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
"is a required property.")
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
(self._corrupt_data('metadata.substitutions.0.src'), 'src')
]
for invalid_entry, missing_key in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err % missing_key):
secret_substitution.SecretSubstitution(invalid_entry)
def test_initialization_bad_substitutions(self):
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
'is missing from the YAML data')
invalid_data = []
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
invalid_data.append(self._format_data(data))
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {
'path': 'tls_endpoint.bar'}
invalid_data.append(self._format_data(data))
def _test(invalid_entry, field, dest):
_expected_err = expected_err % (field, dest)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
# Verify that invalid body dest reference is invalid.
_test(invalid_data[0], "foo", {'path': 'foo'})
# Verify that nested invalid body dest reference is invalid.
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})

57
deckhand/tests/unit/resources/sample.yaml
View File

@ -1,33 +1,38 @@
# Sample YAML file for testing forward replacement.
---
schemaVersion: promenade/v1
kind: SomeConfigType
schema: some-service/ResourceType/v1
metadata:
metadataVersion: deckhand/v1
name: a-unique-config-name-12345
schema: metadata/Document/v1
name: unique-name-given-schema
storagePolicy: cleartext
labels:
component: apiserver
hostname: server0
layerDefinition:
layer: global
abstract: True
childSelector:
label: value
genesis: enabled
master: enabled
layeringDefinition:
abstract: false
layer: region
parentSelector:
required_key_a: required_label_a
required_key_b: required_label_b
actions:
- method: merge
path: .path.to.merge.into.parent
- method: delete
path: .path.to.delete
substitutions:
- dest:
path: .tls_endpoint.certificate
replacePattern: 'test.pattern'
path: .substitution.target
src:
kind: Certificate
name: some-certificate-asdf-1234
path: .cert
- dest:
path: .tls_endpoint.key
src:
kind: CertificateKey
name: some-certificate-asdf-1234
path: .key
schema: another-service/SourceType/v1
name: name-of-source-document
path: .source.path
data:
tls_endpoint:
certificate: '.cert'
key: deckhand/v1:some-certificate-asdf-1234
path:
to:
merge:
into:
parent:
foo: bar
ignored: # Will not be part of the resultant document after layering.
data: here
substitution:
target: null # Paths do not need to exist to be specified as substitution destinations.

0
deckhand/tests/unit/views/__init__.py Normal file
View File

78
deckhand/tests/unit/views/test_views.py Normal file
View File

@ -0,0 +1,78 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.control.views import revision
from deckhand.tests.unit.db import base
from deckhand.tests import test_utils
class TestRevisionViews(base.TestDbBase):
def setUp(self):
super(TestRevisionViews, self).setUp()
self.view_builder = revision.ViewBuilder()
def test_list_revisions(self):
payload = [base.DocumentFixture.get_minimal_fixture()
for _ in range(4)]
self._create_documents(payload)
revisions = self._list_revisions()
revisions_view = self.view_builder.list(revisions)
expected_attrs = ('next', 'prev', 'results', 'count')
for attr in expected_attrs:
self.assertIn(attr, revisions_view)
# Validate that only 1 revision was returned.
self.assertEqual(1, revisions_view['count'])
# Validate that the first revision has 4 documents.
self.assertIn('id', revisions_view['results'][0])
self.assertIn('count', revisions_view['results'][0])
self.assertEqual(4, revisions_view['results'][0]['count'])
def test_list_many_revisions(self):
docs_count = []
for _ in range(3):
doc_count = test_utils.rand_int(3, 9)
docs_count.append(doc_count)
payload = [base.DocumentFixture.get_minimal_fixture()
for _ in range(doc_count)]
self._create_documents(payload)
revisions = self._list_revisions()
revisions_view = self.view_builder.list(revisions)
expected_attrs = ('next', 'prev', 'results', 'count')
for attr in expected_attrs:
self.assertIn(attr, revisions_view)
# Validate that only 1 revision was returned.
self.assertEqual(3, revisions_view['count'])
# Validate that each revision has correct number of documents.
for idx, doc_count in enumerate(docs_count):
self.assertIn('count', revisions_view['results'][idx])
self.assertIn('id', revisions_view['results'][idx])
self.assertEqual(doc_count, revisions_view['results'][idx][
'count'])
def test_show_revision(self):
payload = [base.DocumentFixture.get_minimal_fixture()
for _ in range(4)]
documents = self._create_documents(payload)
revision = self._get_revision(documents[0]['revision_id'])
revision_view = self.view_builder.show(revision)
expected_attrs = ('id', 'url', 'createdAt', 'validationPolicies')
for attr in expected_attrs:
self.assertIn(attr, revision_view)
self.assertIsInstance(revision_view['validationPolicies'], list)

47
deckhand/utils.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def multi_getattr(multi_key, dict_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: nested entry in ``dict_data`` if present; else None.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = dict_data
for attr in attrs:
if attr not in data:
return None
data = data.get(attr)
return data