openstack/ironic
Code Issues Proposed changes

Merge "Add RPC and object version pinning"

Browse Source
This commit is contained in:
Jenkins 2017-03-29 17:33:17 +00:00 committed by Gerrit Code Review
commit a84c2e0d8e
9 changed files with 372 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/source/dev/releasing.rst
View File

@ -47,11 +47,24 @@ Things to do before releasing
TEMPEST_BAREMETAL_MAX_MICROVERSION in devstack/lib/ironic to make sure that
unsupported API tempest tests are skipped on stable branches.
* To support rolling upgrades, add this new release version (and release name
if it is a named release) into ironic/common/release_mappings.py:
* in RELEASE_MAPPING, make a copy of the 'master' entry, and rename the first
'master' entry to the new semver release version.
* If this is a named release, add a RELEASE_MAPPING entry for the named
release. Its value should be the same as that of the latest semver one
(that you just added above).
* Regenerate the sample config file, so that the choices for the
``[DEFAULT]/pin_release_version`` configuration are accurate.
.. _`standards`: http://docs.openstack.org/developer/ironic/dev/faq.html#know-if-a-release-note-is-needed-for-my-change
Things to do after releasing
============================
When a release is done that results in a stable branch
------------------------------------------------------
When a release is done that results in a stable branch for the project, the
release automation will push a number of changes that need to be approved.
@ -82,8 +95,21 @@ Additionally, changes need to be made on master to:
and `pbr documentation
<http://docs.openstack.org/developer/pbr/#version>`_ for details.
For all releases
----------------
For all releases, whether or not it results in a stable branch:
* update the specs repo to mark any specs completed in the release as
implemented.
* remove any -2s on patches that were blocked until after the release.
* to support rolling upgrades, make these changes in
ironic/common/release_mappings.py:
* if the release was a named release, delete any entries from
RELEASE_MAPPING associated with the oldest named release. Since we
support upgrades between adjacent named releases, the master branch will
only support upgrades from the most recent named release to master.
* regenerate the sample config file, so that the choices for the
``[DEFAULT]/pin_release_version`` configuration are accurate.

9
etc/ironic/ironic.conf.sample
View File

@ -354,6 +354,15 @@
# value)
#host = localhost
# Used for rolling upgrades. Setting this option downgrades
# the internal ironic RPC communication to the specified
# version to enable communication with older services. When
# doing a rolling upgrade from version X to version Y, set
# this to X. Defaults to using the newest possible RPC
# behavior. (string value)
# Allowed values: ocata, 7.0
#pin_release_version = <None>
# Path to the rootwrap configuration file to use for running
# commands as root. (string value)
#rootwrap_config = /etc/ironic/rootwrap.conf

86
ironic/common/release_mappings.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2016 Intel Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# NOTE(xek): This decides the version cap of RPC messages sent to conductor
# and objects during rolling upgrades, when [DEFAULT]/pin_release_version
# configuration is set.
#
# Remember to add a new entry for the new version that is shipping in a new
# release.
#
# We support a rolling upgrade between adjacent named releases, as well as
# between a release and master, so old, unsupported releases can be removed,
# together with the supporting code, which is typically found in an object's
# make_compatible methods and RPC client code.
# NOTE(xek): The format of this dict is:
# { '<release version>': {
# 'rpc': '<RPC API version>',
# 'objects': {
# '<object class name>': '<object version>',
# }
# },
# }
# The list should contain all objects which are persisted in the database and
# sent over RPC. Notifications/Payloads are not being included here since we
# don't need to pin them during rolling upgrades.
#
# There should always be a 'master' entry that reflects the objects in the
# master branch.
#
# Just before doing a release, copy the 'master' entry, and rename the first
# 'master' entry to the (semver) version being released.
#
# Just after doing a named release, delete any entries associated with the
# oldest named release.
RELEASE_MAPPING = {
'7.0': {
'rpc': '1.40',
'objects': {
'Node': '1.21',
'Conductor': '1.2',
'Chassis': '1.3',
'Port': '1.6',
'Portgroup': '1.3',
'VolumeConnector': '1.0',
'VolumeTarget': '1.0',
}
},
'master': {
'rpc': '1.40',
'objects': {
'Node': '1.21',
'Conductor': '1.2',
'Chassis': '1.3',
'Port': '1.6',
'Portgroup': '1.3',
'VolumeConnector': '1.0',
'VolumeTarget': '1.0',
}
},
}
# NOTE(xek): Assign each named release to the appropriate semver.
#
# Just before we do a new named release, add a mapping for the new
# named release.
#
# Just after we do a new named release, delete the oldest named
# release (that we are no longer supporting for a rolling upgrade).
#
# There should be at most two named mappings here.
RELEASE_MAPPING['ocata'] = RELEASE_MAPPING['7.0']
# List of available versions with named versions first; 'master' is excluded.
RELEASE_VERSIONS = sorted(set(RELEASE_MAPPING) - {'master'}, reverse=True)

7
ironic/conductor/rpcapi.py
View File

@ -25,6 +25,7 @@ import oslo_messaging as messaging
from ironic.common import exception
from ironic.common import hash_ring
from ironic.common.i18n import _
from ironic.common import release_mappings as versions
from ironic.common import rpc
from ironic.conductor import manager
from ironic.conf import CONF
@ -103,8 +104,10 @@ class ConductorAPI(object):
target = messaging.Target(topic=self.topic,
version='1.0')
serializer = objects_base.IronicObjectSerializer()
self.client = rpc.get_client(target,
version_cap=self.RPC_API_VERSION,
release_ver = versions.RELEASE_MAPPING.get(CONF.pin_release_version)
version_cap = (release_ver['rpc'] if release_ver
else self.RPC_API_VERSION)
self.client = rpc.get_client(target, version_cap=version_cap,
serializer=serializer)
# NOTE(deva): this is going to be buggy
self.ring_manager = hash_ring.HashRingManager()

10
ironic/conf/default.py
View File

@ -25,6 +25,7 @@ from oslo_config import cfg
from oslo_utils import netutils
from ironic.common.i18n import _
from ironic.common import release_mappings as versions
_ENABLED_IFACE_HELP = _('Specify the list of {0} interfaces to load during '
@ -261,6 +262,15 @@ service_opts = [
'However, the node name must be valid within '
'an AMQP key, and if using ZeroMQ, a valid '
'hostname, FQDN, or IP address.')),
cfg.StrOpt('pin_release_version',
choices=versions.RELEASE_VERSIONS,
# TODO(xek): mutable=True,
help=_('Used for rolling upgrades. Setting this option '
'downgrades the internal ironic RPC communication to '
'the specified version to enable communication with '
'older services. When doing a rolling upgrade from '
'version X to version Y, set this to X. Defaults to '
'using the newest possible RPC behavior.')),
]
utils_opts = [

33
ironic/objects/base.py
View File

@ -14,12 +14,17 @@
"""Ironic common internal object model"""
from oslo_log import log
from oslo_utils import versionutils
from oslo_versionedobjects import base as object_base
from ironic.common import release_mappings as versions
from ironic.conf import CONF
from ironic import objects
from ironic.objects import fields as object_fields
LOG = log.getLogger(__name__)
class IronicObjectRegistry(object_base.VersionedObjectRegistry):
def registration_hook(self, cls, index):
@ -103,7 +108,35 @@ class IronicObject(object_base.VersionedObject):
return [cls._from_db_object(cls(context), db_obj)
for db_obj in db_objects]
def _get_target_version(self):
"""Returns the target version for this object.
If pinned, returns the version of this object corresponding to
the pin. Otherwise, returns this (latest) version of the object.
"""
pinned_version = None
pin = CONF.pin_release_version
if pin:
version_manifest = versions.RELEASE_MAPPING[pin]['objects']
pinned_version = version_manifest.get(self.obj_name())
return pinned_version or self.__class__.VERSION
class IronicObjectSerializer(object_base.VersionedObjectSerializer):
# Base class to use for object hydration
OBJ_BASE_CLASS = IronicObject
def serialize_entity(self, context, entity):
if isinstance(entity, (tuple, list, set, dict)):
entity = self._process_iterable(context, self.serialize_entity,
entity)
elif (hasattr(entity, 'obj_to_primitive')
and callable(entity.obj_to_primitive)):
target_version = entity._get_target_version()
# NOTE(xek): If the version is pinned, target_version is an older
# object version and entity's obj_make_compatible method is called
# to backport the object before serialization.
entity = entity.obj_to_primitive(target_version=target_version)
elif not isinstance(entity, (int, str, bool, float, type)) and entity:
LOG.warning("Entity %s was not serialized.", str(entity))
return entity

95
ironic/tests/unit/common/test_release_mappings.py Normal file
View File

@ -0,0 +1,95 @@
# Copyright 2016 Intel Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import versionutils
import six
from ironic.common.release_mappings import RELEASE_MAPPING
from ironic.conductor import rpcapi
from ironic.db.sqlalchemy import models
from ironic.objects import base as obj_base
from ironic.tests import base
from ironic import version
def _check_versions_compatibility(conf_version, actual_version):
"""Checks the configured version against the actual version.
Returns True if the configured version is <= the actual version;
otherwise returns False.
param conf_version: configured version, a string with dots
param actual_version: actual version, a string with dots
"""
conf_cap = versionutils.convert_version_to_tuple(conf_version)
actual_cap = versionutils.convert_version_to_tuple(actual_version)
return conf_cap <= actual_cap
class ReleaseMappingsTestCase(base.TestCase):
"""Tests whether the dict RELEASE_MAPPING is correct, valid and consistent.
"""
def test_structure(self):
for value in RELEASE_MAPPING.values():
self.assertTrue(isinstance(value, dict))
self.assertEqual({'rpc', 'objects'}, set(value))
self.assertTrue(isinstance(value['rpc'], six.string_types))
self.assertTrue(isinstance(value['objects'], dict))
for obj_value in value['objects'].values():
self.assertTrue(isinstance(obj_value, six.string_types))
tuple_ver = versionutils.convert_version_to_tuple(obj_value)
self.assertEqual(2, len(tuple_ver))
def test_object_names_are_registered(self):
registered_objects = set(obj_base.IronicObjectRegistry.obj_classes())
for mapping in RELEASE_MAPPING.values():
objects = set(mapping['objects'])
self.assertTrue(objects.issubset(registered_objects))
def test_contains_current_release_entry(self):
version_info = str(version.version_info)
# NOTE(sborkows): We only need first two values from version (like 5.1)
# and assume version_info is of the form 'x.y.z'.
current_release = version_info[:version_info.rfind('.')]
self.assertIn(current_release, RELEASE_MAPPING)
def test_current_rpc_version(self):
self.assertEqual(rpcapi.ConductorAPI.RPC_API_VERSION,
RELEASE_MAPPING['master']['rpc'])
def test_current_object_versions(self):
registered_objects = obj_base.IronicObjectRegistry.obj_classes()
for obj, objver in RELEASE_MAPPING['master']['objects'].items():
self.assertEqual(registered_objects[obj][0].VERSION, objver)
def test_contains_all_db_objects(self):
self.assertIn('master', RELEASE_MAPPING)
model_names = set((s.__name__ for s in models.Base.__subclasses__()))
exceptions = set(['NodeTag', 'ConductorHardwareInterfaces'])
# NOTE(xek): As a rule, all models which can be changed between
# releases or are sent through RPC should have their counterpart
# versioned objects. This means all, but very simple models.
model_names -= exceptions
object_names = set(RELEASE_MAPPING['master']['objects'])
self.assertEqual(model_names, object_names)
def test_rpc_and_objects_versions_supported(self):
registered_objects = obj_base.IronicObjectRegistry.obj_classes()
for versions in RELEASE_MAPPING.values():
self.assertTrue(_check_versions_compatibility(
versions['rpc'], rpcapi.ConductorAPI.RPC_API_VERSION))
for obj_name, obj_ver in versions['objects'].items():
self.assertTrue(_check_versions_compatibility(
obj_ver, registered_objects[obj_name][0].VERSION))

17
ironic/tests/unit/conductor/test_rpcapi.py
View File

@ -27,6 +27,7 @@ from oslo_messaging import _utils as messaging_utils
from ironic.common import boot_devices
from ironic.common import exception
from ironic.common import release_mappings
from ironic.common import states
from ironic.conductor import manager as conductor_manager
from ironic.conductor import rpcapi as conductor_rpcapi
@ -45,6 +46,22 @@ class ConductorRPCAPITestCase(tests_base.TestCase):
conductor_manager.ConductorManager.RPC_API_VERSION,
conductor_rpcapi.ConductorAPI.RPC_API_VERSION)
@mock.patch('ironic.common.rpc.get_client')
def test_version_cap(self, mock_get_client):
conductor_rpcapi.ConductorAPI()
self.assertEqual(conductor_rpcapi.ConductorAPI.RPC_API_VERSION,
mock_get_client.call_args[1]['version_cap'])
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
@mock.patch('ironic.common.rpc.get_client')
def test_version_capped(self, mock_get_client, mock_release_mapping):
CONF.set_override('pin_release_version',
release_mappings.RELEASE_VERSIONS[0],
enforce_type=True)
mock_release_mapping.get.return_value = {'rpc': '3'}
conductor_rpcapi.ConductorAPI()
self.assertEqual('3', mock_get_client.call_args[1]['version_cap'])
class RPCAPITestCase(base.DbTestCase):

91
ironic/tests/unit/objects/test_objects.py
View File

@ -23,6 +23,8 @@ from oslo_versionedobjects import fixture as object_fixture
import six
from ironic.common import context
from ironic.common import release_mappings
from ironic.conf import CONF
from ironic.objects import base
from ironic.objects import fields
from ironic.tests import base as test_base
@ -37,6 +39,11 @@ class MyObj(base.IronicObject, object_base.VersionedObjectDictCompat):
'missing': fields.StringField(),
}
def obj_make_compatible(self, primitive, target_version):
super(MyObj, self).obj_make_compatible(primitive, target_version)
if target_version == '1.4' and 'missing' in primitive:
del primitive['missing']
def obj_load_attr(self, attrname):
setattr(self, attrname, 'loaded!')
@ -518,6 +525,90 @@ class TestObjectSerializer(test_base.TestCase):
"Test object with unsupported (newer) version and revision"
self._test_deserialize_entity_newer('1.7', '1.6.1', my_version='1.6.1')
@mock.patch.object(MyObj, 'obj_make_compatible')
def test_serialize_entity_no_backport(self, make_compatible_mock):
"""Test single element serializer with no backport."""
serializer = base.IronicObjectSerializer()
obj = MyObj(self.context)
obj.foo = 1
obj.bar = 'text'
obj.missing = 'textt'
primitive = serializer.serialize_entity(self.context, obj)
self.assertEqual('1.5', primitive['ironic_object.version'])
data = primitive['ironic_object.data']
self.assertEqual(1, data['foo'])
self.assertEqual('text', data['bar'])
self.assertEqual('textt', data['missing'])
changes = primitive['ironic_object.changes']
self.assertEqual(set(['foo', 'bar', 'missing']), set(changes))
make_compatible_mock.assert_not_called()
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
def test_serialize_entity_backport(self, mock_release_mapping):
"""Test single element serializer with backport."""
CONF.set_override('pin_release_version',
release_mappings.RELEASE_VERSIONS[-1],
enforce_type=True)
mock_release_mapping.__getitem__.return_value = {
'objects': {
'MyObj': '1.4',
}
}
serializer = base.IronicObjectSerializer()
obj = MyObj(self.context)
obj.foo = 1
obj.bar = 'text'
obj.missing = 'textt'
primitive = serializer.serialize_entity(self.context, obj)
self.assertEqual('1.4', primitive['ironic_object.version'])
data = primitive['ironic_object.data']
self.assertEqual(1, data['foo'])
self.assertEqual('text', data['bar'])
self.assertNotIn('missing', data)
changes = primitive['ironic_object.changes']
self.assertEqual(set(['foo', 'bar']), set(changes))
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
def test_serialize_entity_invalid_pin(self, mock_release_mapping):
CONF.set_override('pin_release_version',
release_mappings.RELEASE_VERSIONS[-1],
enforce_type=True)
mock_release_mapping.__getitem__.return_value = {
'objects': {
'MyObj': '1.6',
}
}
serializer = base.IronicObjectSerializer()
obj = MyObj(self.context)
self.assertRaises(object_exception.InvalidTargetVersion,
serializer.serialize_entity, self.context, obj)
@mock.patch('ironic.common.release_mappings.RELEASE_MAPPING')
def test_serialize_entity_no_pin(self, mock_release_mapping):
CONF.set_override('pin_release_version',
release_mappings.RELEASE_VERSIONS[-1],
enforce_type=True)
mock_release_mapping.__getitem__.return_value = {
'objects': {}
}
serializer = base.IronicObjectSerializer()
obj = MyObj(self.context)
primitive = serializer.serialize_entity(self.context, obj)
self.assertEqual('1.5', primitive['ironic_object.version'])
@mock.patch('ironic.objects.base.IronicObject._get_target_version')
@mock.patch('ironic.objects.base.LOG.warning')
def test_serialize_entity_unknown_entity(self, mock_warn, mock_version):
class Foo(object):
fields = {'foobar': fields.IntegerField()}
serializer = base.IronicObjectSerializer()
obj = Foo()
primitive = serializer.serialize_entity(self.context, obj)
self.assertEqual(obj, primitive)
self.assertTrue(mock_warn.called)
mock_version.assert_not_called()
class TestRegistry(test_base.TestCase):
@mock.patch('ironic.objects.base.objects')