deployment_swift_data property for server resources

Adds a new property, deployment_swift_data for server resources. The
property is a map containing the swift container and object name to use
for storing the deployment data and generating temp url's for the
resource.

Making this a property allows it to be created externally from Heat with
known values prior to stack creation. This allows the configuration of
the os-collect-config agent on deployed servers prior to starting the
Heat stack creation.

implements blueprint split-stack-default

Change-Id: Ia07e9374a4b95bd0e74fc47fb9df4bf6ad096715

heat/engine/resources/openstack/heat/deployed_server.py

@@ -37,9 +37,11 @@ class DeployedServer(server_base.BaseServer):
     """
 
     PROPERTIES = (
-        NAME, METADATA, SOFTWARE_CONFIG_TRANSPORT
+        NAME, METADATA, SOFTWARE_CONFIG_TRANSPORT,
+        DEPLOYMENT_SWIFT_DATA
     ) = (
-        'name', 'metadata', 'software_config_transport'
+        'name', 'metadata', 'software_config_transport',
+        'deployment_swift_data'
     )
 
     _SOFTWARE_CONFIG_TRANSPORTS = (
@@ -48,6 +50,12 @@ class DeployedServer(server_base.BaseServer):
         'POLL_SERVER_CFN', 'POLL_SERVER_HEAT', 'POLL_TEMP_URL', 'ZAQAR_MESSAGE'
     )
 
+    _DEPLOYMENT_SWIFT_DATA_KEYS = (
+        CONTAINER, OBJECT
+    ) = (
+        'container', 'object',
+    )
+
     properties_schema = {
         NAME: properties.Schema(
             properties.Schema.STRING,
@@ -79,6 +87,37 @@ class DeployedServer(server_base.BaseServer):
                 constraints.AllowedValues(_SOFTWARE_CONFIG_TRANSPORTS),
             ]
         ),
+        DEPLOYMENT_SWIFT_DATA: properties.Schema(
+            properties.Schema.MAP,
+            _('Swift container and object to use for storing deployment data '
+              'for the server resource. The parameter is a map value '
+              'with the keys "container" and "object", and the values '
+              'are the corresponding container and object names. The '
+              'software_config_transport parameter must be set to '
+              'POLL_TEMP_URL for swift to be used. If not specified, '
+              'and software_config_transport is set to POLL_TEMP_URL, a '
+              'container will be automatically created from the resource '
+              'name, and the object name will be a generated uuid.'),
+            support_status=support.SupportStatus(version='9.0.0'),
+            default={},
+            update_allowed=True,
+            schema={
+                CONTAINER: properties.Schema(
+                    properties.Schema.STRING,
+                    _('Name of the container.'),
+                    constraints=[
+                        constraints.Length(min=1)
+                    ]
+                ),
+                OBJECT: properties.Schema(
+                    properties.Schema.STRING,
+                    _('Name of the object.'),
+                    constraints=[
+                        constraints.Length(min=1)
+                    ]
+                )
+            }
+        )
     }
 
     ATTRIBUTES = (

heat/engine/resources/openstack/nova/server.py

@@ -54,7 +54,7 @@ class Server(server_base.BaseServer, sh.SchedulerHintsMixin,
         SCHEDULER_HINTS, METADATA, USER_DATA_FORMAT, USER_DATA,
         RESERVATION_ID, CONFIG_DRIVE, DISK_CONFIG, PERSONALITY,
         ADMIN_PASS, SOFTWARE_CONFIG_TRANSPORT, USER_DATA_UPDATE_POLICY,
-        TAGS
+        TAGS, DEPLOYMENT_SWIFT_DATA
     ) = (
         'name', 'image', 'block_device_mapping', 'block_device_mapping_v2',
         'flavor', 'flavor_update_policy', 'image_update_policy', 'key_name',
@@ -62,7 +62,7 @@ class Server(server_base.BaseServer, sh.SchedulerHintsMixin,
         'scheduler_hints', 'metadata', 'user_data_format', 'user_data',
         'reservation_id', 'config_drive', 'diskConfig', 'personality',
         'admin_pass', 'software_config_transport', 'user_data_update_policy',
-        'tags'
+        'tags', 'deployment_swift_data'
     )
 
     _BLOCK_DEVICE_MAPPING_KEYS = (
@@ -135,6 +135,12 @@ class Server(server_base.BaseServer, sh.SchedulerHintsMixin,
         'none', 'auto',
     )
 
+    _DEPLOYMENT_SWIFT_DATA_KEYS = (
+        CONTAINER, OBJECT
+    ) = (
+        'container', 'object',
+    )
+
     ATTRIBUTES = (
         NAME_ATTR, ADDRESSES, NETWORKS_ATTR, FIRST_ADDRESS,
         INSTANCE_NAME, ACCESSIPV4, ACCESSIPV6, CONSOLE_URLS, TAGS_ATTR,
@@ -568,6 +574,37 @@ class Server(server_base.BaseServer, sh.SchedulerHintsMixin,
             support_status=support.SupportStatus(version='8.0.0'),
             schema=properties.Schema(properties.Schema.STRING),
             update_allowed=True
+        ),
+        DEPLOYMENT_SWIFT_DATA: properties.Schema(
+            properties.Schema.MAP,
+            _('Swift container and object to use for storing deployment data '
+              'for the server resource. The parameter is a map value '
+              'with the keys "container" and "object", and the values '
+              'are the corresponding container and object names. The '
+              'software_config_transport parameter must be set to '
+              'POLL_TEMP_URL for swift to be used. If not specified, '
+              'and software_config_transport is set to POLL_TEMP_URL, a '
+              'container will be automatically created from the resource '
+              'name, and the object name will be a generated uuid.'),
+            support_status=support.SupportStatus(version='9.0.0'),
+            default={},
+            update_allowed=True,
+            schema={
+                CONTAINER: properties.Schema(
+                    properties.Schema.STRING,
+                    _('Name of the container.'),
+                    constraints=[
+                        constraints.Length(min=1)
+                    ]
+                ),
+                OBJECT: properties.Schema(
+                    properties.Schema.STRING,
+                    _('Name of the object.'),
+                    constraints=[
+                        constraints.Length(min=1)
+                    ]
+                )
+            }
         )
     }
 

heat/engine/resources/server_base.py

@@ -45,6 +45,22 @@ class BaseServer(stack_user.StackUser):
 
         return self.physical_resource_name()
 
+    def _container_and_object_name(self, props):
+        deployment_swift_data = props.get(
+            self.DEPLOYMENT_SWIFT_DATA,
+            self.properties[self.DEPLOYMENT_SWIFT_DATA])
+        container_name = deployment_swift_data[self.CONTAINER]
+        if container_name is None:
+            container_name = self.physical_resource_name()
+
+        object_name = deployment_swift_data[self.OBJECT]
+        if object_name is None:
+            object_name = self.data().get('metadata_object_name')
+        if object_name is None:
+            object_name = str(uuid.uuid4())
+
+        return container_name, object_name
+
     def _populate_deployments_metadata(self, meta, props):
         meta['deployments'] = meta.get('deployments', [])
         meta['os-collect-config'] = meta.get('os-collect-config', {})
@@ -94,17 +110,15 @@ class BaseServer(stack_user.StackUser):
             collectors.append('cfn')
 
         elif self.transport_poll_temp_url(props):
-            container = self.physical_resource_name()
+            container_name, object_name = self._container_and_object_name(
-            object_name = self.data().get('metadata_object_name')
+                props)
-            if not object_name:
-                object_name = str(uuid.uuid4())
 
-            self.client('swift').put_container(container)
+            self.client('swift').put_container(container_name)
 
             url = self.client_plugin('swift').get_temp_url(
-                container, object_name, method='GET')
+                container_name, object_name, method='GET')
             put_url = self.client_plugin('swift').get_temp_url(
-                container, object_name)
+                container_name, object_name)
             self.data_set('metadata_put_url', put_url)
             self.data_set('metadata_object_name', object_name)
 
@@ -126,9 +140,10 @@ class BaseServer(stack_user.StackUser):
 
         object_name = self.data().get('metadata_object_name')
         if object_name:
-            container = self.physical_resource_name()
+            container_name, object_name = self._container_and_object_name(
+                props)
             self.client('swift').put_object(
-                container, object_name, jsonutils.dumps(meta))
+                container_name, object_name, jsonutils.dumps(meta))
 
         self.attributes.reset_resolved_values()
 
@@ -278,7 +293,9 @@ class BaseServer(stack_user.StackUser):
         if not object_name:
             return
         with self.client_plugin('swift').ignore_not_found:
-            container = self.physical_resource_name()
+            container = self.properties[self.DEPLOYMENT_SWIFT_DATA].get(
+                'container')
+            container = container or self.physical_resource_name()
             swift = self.client('swift')
             swift.delete_object(container, object_name)
             headers = swift.head_container(container)

heat/tests/openstack/heat/test_deployed_server.py

@@ -17,6 +17,7 @@ from oslo_serialization import jsonutils
 from oslo_utils import uuidutils
 from six.moves.urllib import parse as urlparse
 
+from heat.common import exception
 from heat.common import template_format
 from heat.engine.clients.os import heat_plugin
 from heat.engine.clients.os import swift
@@ -65,6 +66,66 @@ resources:
       software_config_transport: ZAQAR_MESSAGE
 """
 
+ds_deployment_data_tmpl = """
+heat_template_version: 2015-10-15
+resources:
+  server:
+    type: OS::Heat::DeployedServer
+    properties:
+      software_config_transport: POLL_TEMP_URL
+      deployment_swift_data:
+        container: my-custom-container
+        object: my-custom-object
+"""
+
+ds_deployment_data_bad_container_tmpl = """
+heat_template_version: 2015-10-15
+resources:
+  server:
+    type: OS::Heat::DeployedServer
+    properties:
+      software_config_transport: POLL_TEMP_URL
+      deployment_swift_data:
+        container: ''
+        object: 'my-custom-object'
+"""
+
+ds_deployment_data_bad_object_tmpl = """
+heat_template_version: 2015-10-15
+resources:
+  server:
+    type: OS::Heat::DeployedServer
+    properties:
+      software_config_transport: POLL_TEMP_URL
+      deployment_swift_data:
+        container: 'my-custom-container'
+        object: ''
+"""
+
+ds_deployment_data_none_container_tmpl = """
+heat_template_version: 2015-10-15
+resources:
+  server:
+    type: OS::Heat::DeployedServer
+    properties:
+      software_config_transport: POLL_TEMP_URL
+      deployment_swift_data:
+        container: 0
+        object: 'my-custom-object'
+"""
+
+ds_deployment_data_none_object_tmpl = """
+heat_template_version: 2015-10-15
+resources:
+  server:
+    type: OS::Heat::DeployedServer
+    properties:
+      software_config_transport: POLL_TEMP_URL
+      deployment_swift_data:
+        container: 'my-custom-container'
+        object: 0
+"""
+
 
 class DeployedServersTest(common.HeatTestCase):
     def setUp(self):
@@ -128,6 +189,178 @@ class DeployedServersTest(common.HeatTestCase):
         sc.delete_container.assert_called_once_with(container_name)
         return metadata_url, server
 
+    def test_server_create_deployment_swift_data(self):
+        server_name = 'server'
+        stack_name = '%s_s' % server_name
+        (tmpl, stack) = self._setup_test_stack(
+            stack_name,
+            ds_deployment_data_tmpl)
+
+        props = tmpl.t['resources']['server']['properties']
+        props['software_config_transport'] = 'POLL_TEMP_URL'
+        self.server_props = props
+
+        resource_defns = tmpl.resource_definitions(stack)
+        server = deployed_server.DeployedServer(
+            server_name, resource_defns[server_name], stack)
+
+        sc = mock.Mock()
+        sc.head_account.return_value = {
+            'x-account-meta-temp-url-key': 'secrit'
+        }
+        sc.url = 'http://192.0.2.2'
+
+        self.patchobject(swift.SwiftClientPlugin, '_create',
+                         return_value=sc)
+        scheduler.TaskRunner(server.create)()
+        # self._create_test_server(server_name)
+        metadata_put_url = server.data().get('metadata_put_url')
+        md = server.metadata_get()
+        metadata_url = md['os-collect-config']['request']['metadata_url']
+        self.assertNotEqual(metadata_url, metadata_put_url)
+
+        container_name = 'my-custom-container'
+        object_name = 'my-custom-object'
+        test_path = '/v1/AUTH_test_tenant_id/%s/%s' % (
+            container_name, object_name)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_put_url).path)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_url).path)
+        sc.put_object.assert_called_once_with(
+            container_name, object_name, jsonutils.dumps(md))
+
+        sc.head_container.return_value = {'x-container-object-count': '0'}
+        server._delete_temp_url()
+        sc.delete_object.assert_called_once_with(container_name, object_name)
+        sc.head_container.assert_called_once_with(container_name)
+        sc.delete_container.assert_called_once_with(container_name)
+        return metadata_url, server
+
+    def test_server_create_deployment_swift_data_bad_container(self):
+        server_name = 'server'
+        stack_name = '%s_s' % server_name
+        (tmpl, stack) = self._setup_test_stack(
+            stack_name,
+            ds_deployment_data_bad_container_tmpl)
+
+        props = tmpl.t['resources']['server']['properties']
+        props['software_config_transport'] = 'POLL_TEMP_URL'
+        self.server_props = props
+
+        resource_defns = tmpl.resource_definitions(stack)
+        server = deployed_server.DeployedServer(
+            server_name, resource_defns[server_name], stack)
+
+        self.assertRaises(exception.StackValidationFailed, server.validate)
+
+    def test_server_create_deployment_swift_data_bad_object(self):
+        server_name = 'server'
+        stack_name = '%s_s' % server_name
+        (tmpl, stack) = self._setup_test_stack(
+            stack_name,
+            ds_deployment_data_bad_object_tmpl)
+
+        props = tmpl.t['resources']['server']['properties']
+        props['software_config_transport'] = 'POLL_TEMP_URL'
+        self.server_props = props
+
+        resource_defns = tmpl.resource_definitions(stack)
+        server = deployed_server.DeployedServer(
+            server_name, resource_defns[server_name], stack)
+
+        self.assertRaises(exception.StackValidationFailed, server.validate)
+
+    def test_server_create_deployment_swift_data_none_container(self):
+        server_name = 'server'
+        stack_name = '%s_s' % server_name
+        (tmpl, stack) = self._setup_test_stack(
+            stack_name,
+            ds_deployment_data_none_container_tmpl)
+
+        props = tmpl.t['resources']['server']['properties']
+        props['software_config_transport'] = 'POLL_TEMP_URL'
+        self.server_props = props
+
+        resource_defns = tmpl.resource_definitions(stack)
+        server = deployed_server.DeployedServer(
+            server_name, resource_defns[server_name], stack)
+
+        sc = mock.Mock()
+        sc.head_account.return_value = {
+            'x-account-meta-temp-url-key': 'secrit'
+        }
+        sc.url = 'http://192.0.2.2'
+
+        self.patchobject(swift.SwiftClientPlugin, '_create',
+                         return_value=sc)
+        scheduler.TaskRunner(server.create)()
+        # self._create_test_server(server_name)
+        metadata_put_url = server.data().get('metadata_put_url')
+        md = server.metadata_get()
+        metadata_url = md['os-collect-config']['request']['metadata_url']
+        self.assertNotEqual(metadata_url, metadata_put_url)
+
+        container_name = '0'
+        object_name = 'my-custom-object'
+        test_path = '/v1/AUTH_test_tenant_id/%s/%s' % (
+            container_name, object_name)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_put_url).path)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_url).path)
+        sc.put_object.assert_called_once_with(
+            container_name, object_name, jsonutils.dumps(md))
+
+        sc.head_container.return_value = {'x-container-object-count': '0'}
+        server._delete_temp_url()
+        sc.delete_object.assert_called_once_with(container_name, object_name)
+        sc.head_container.assert_called_once_with(container_name)
+        sc.delete_container.assert_called_once_with(container_name)
+        return metadata_url, server
+
+    def test_server_create_deployment_swift_data_none_object(self):
+        server_name = 'server'
+        stack_name = '%s_s' % server_name
+        (tmpl, stack) = self._setup_test_stack(
+            stack_name,
+            ds_deployment_data_none_object_tmpl)
+
+        props = tmpl.t['resources']['server']['properties']
+        props['software_config_transport'] = 'POLL_TEMP_URL'
+        self.server_props = props
+
+        resource_defns = tmpl.resource_definitions(stack)
+        server = deployed_server.DeployedServer(
+            server_name, resource_defns[server_name], stack)
+
+        sc = mock.Mock()
+        sc.head_account.return_value = {
+            'x-account-meta-temp-url-key': 'secrit'
+        }
+        sc.url = 'http://192.0.2.2'
+
+        self.patchobject(swift.SwiftClientPlugin, '_create',
+                         return_value=sc)
+        scheduler.TaskRunner(server.create)()
+        # self._create_test_server(server_name)
+        metadata_put_url = server.data().get('metadata_put_url')
+        md = server.metadata_get()
+        metadata_url = md['os-collect-config']['request']['metadata_url']
+        self.assertNotEqual(metadata_url, metadata_put_url)
+
+        container_name = 'my-custom-container'
+        object_name = '0'
+        test_path = '/v1/AUTH_test_tenant_id/%s/%s' % (
+            container_name, object_name)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_put_url).path)
+        self.assertEqual(test_path, urlparse.urlparse(metadata_url).path)
+        sc.put_object.assert_called_once_with(
+            container_name, object_name, jsonutils.dumps(md))
+
+        sc.head_container.return_value = {'x-container-object-count': '0'}
+        server._delete_temp_url()
+        sc.delete_object.assert_called_once_with(container_name, object_name)
+        sc.head_container.assert_called_once_with(container_name)
+        sc.delete_container.assert_called_once_with(container_name)
+        return metadata_url, server
+
     def test_server_create_software_config_poll_temp_url(self):
         metadata_url, server = (
             self._server_create_software_config_poll_temp_url())

releasenotes/notes/deployment-swift-data-server-property-51fd4f9d1671fc90.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - A new property, deployment_swift_data is added to the OS::Nova::Server
+    and OS::Heat::DeployedServer resources. The property is used to define
+    the Swift container and object name that is used for deployment data
+    for the server. If unset, the fallback is the previous behavior where
+    these values will be automatically generated.