openstack/ironic
Code Issues Proposed changes

Automatically migrate nodes to hardware types

Browse Source 
This change adds a new data migration: migrate_to_hardware_types.
It works by walking through known classic drivers, detecting matching
hardware types and interfaces and updates nodes accordingly.

Nodes that cannot be updated (e.g. matching hardware type is not
enabled) are skipped. A new migration option reset_unsupported_interfaces
can be set to True to allow resetting optional interfaces to their
no-op versions.

The example implementation are provided for the community supported
IPMI and SNMP drivers, as well as for fake drivers based on them.

Change-Id: I732b44f2ab1ef73f56b352415ffd9cdd8a0e232b
Partial-Bug: #1690185
This commit is contained in:
Dmitry Tantsur 2018-01-16 17:27:17 +01:00
parent 98570dc6ad
commit cc6f7bc73e
12 changed files with 519 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ironic/cmd/dbsync.py
View File

@ -65,6 +65,8 @@ ONLINE_MIGRATIONS = (
# Added in Pike, modified in Queens
# TODO(rloo): remove in Rocky
(dbapi, 'backfill_version_column'),
# TODO(dtantsur): remove when classic drivers are removed (Rocky?)
(dbapi, 'migrate_to_hardware_types'),
)

98
ironic/common/driver_factory.py
View File

@ -17,6 +17,7 @@ import collections
from oslo_concurrency import lockutils
from oslo_log import log
import stevedore
from stevedore import named
from ironic.common import exception
@ -558,3 +559,100 @@ _INTERFACE_LOADERS = {
# refactor them later to use _INTERFACE_LOADERS.
NetworkInterfaceFactory = _INTERFACE_LOADERS['network']
StorageInterfaceFactory = _INTERFACE_LOADERS['storage']
def calculate_migration_delta(driver_name, driver_class,
reset_unsupported_interfaces=False):
"""Calculate an update for the given classic driver extension.
This function calculates a database update required to convert a node
with a classic driver to hardware types and interfaces.
This function is used in the data migrations and is not a part of the
public Python API.
:param driver_name: the entry point name of the driver
:param driver_class: class of classic driver.
:param reset_unsupported_interfaces: if set to True, target interfaces
that are not enabled will be replaced with a no-<interface name>,
if possible.
:returns: Node fields requiring update as a dict (field -> new value).
None if a migration is not possible.
"""
# NOTE(dtantsur): provide defaults for optional interfaces
defaults = {'console': 'no-console',
'inspect': 'no-inspect',
'raid': 'no-raid',
'rescue': 'no-rescue',
'vendor': 'no-vendor'}
try:
hw_type, new_ifaces = driver_class.to_hardware_type()
except NotImplementedError:
LOG.warning('Skipping migrating nodes with driver %s, '
'migration not supported', driver_name)
return None
else:
ifaces = dict(defaults, **new_ifaces)
if hw_type not in CONF.enabled_hardware_types:
LOG.warning('Skipping migrating nodes with driver %(drv)s: '
'hardware type %(hw_type)s is not enabled',
{'drv': driver_name, 'hw_type': hw_type})
return None
not_enabled = []
delta = {'driver': hw_type}
for iface, value in ifaces.items():
conf = 'enabled_%s_interfaces' % iface
if value not in getattr(CONF, conf):
not_enabled.append((iface, value))
else:
delta['%s_interface' % iface] = value
if not_enabled and reset_unsupported_interfaces:
still_not_enabled = []
for iface, value in not_enabled:
try:
default = defaults[iface]
except KeyError:
still_not_enabled.append((iface, value))
else:
conf = 'enabled_%s_interfaces' % iface
if default not in getattr(CONF, conf):
still_not_enabled.append((iface, value))
else:
delta['%s_interface' % iface] = default
not_enabled = still_not_enabled
if not_enabled:
LOG.warning('Skipping migrating nodes with driver %(drv)s, '
'the following interfaces are not supported: '
'%(ifaces)s',
{'drv': driver_name,
'ifaces': ', '.join('%s_interface=%s' % tpl
for tpl in not_enabled)})
return None
return delta
def classic_drivers_to_migrate():
"""Get drivers requiring migration.
This function is used in the data migrations and is not a part of the
public Python API.
:returns: a dict mapping driver names to driver classes
"""
def failure_callback(mgr, ep, exc):
LOG.warning('Unable to load classic driver %(drv)s: %(err)s',
{'drv': ep.name, 'err': exc})
extension_manager = (
stevedore.ExtensionManager(
'ironic.drivers',
invoke_on_load=False,
on_load_failure_callback=failure_callback))
return {ext.name: ext.plugin for ext in extension_manager}

19
ironic/db/api.py
View File

@ -923,6 +923,25 @@ class Connection(object):
"""
# TODO(rloo) Delete this in Rocky cycle.
@abc.abstractmethod
def migrate_to_hardware_types(self, context, max_count,
reset_unsupported_interfaces=False):
"""Migrate nodes from classic drivers to hardware types.
Go through all nodes with a classic driver and try to migrate them to a
corresponding hardware type and a correct set of hardware interfaces.
:param context: the admin context
:param max_count: The maximum number of objects to migrate. Must be
>= 0. If zero, all the objects will be migrated.
:param reset_unsupported_interfaces: whether to reset unsupported
optional interfaces to their no-XXX versions.
:returns: A 2-tuple, 1. the total number of objects that need to be
migrated (at the beginning of this call) and 2. the number
of migrated objects.
"""
# TODO(dtantsur) Delete this in Rocky cycle.
@abc.abstractmethod
def set_node_traits(self, node_id, traits, version):
"""Replace all of the node traits with specified list of traits.

77
ironic/db/sqlalchemy/api.py
View File

@ -33,6 +33,7 @@ from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy.orm import joinedload
from sqlalchemy import sql
from ironic.common import driver_factory
from ironic.common import exception
from ironic.common.i18n import _
from ironic.common import profiler
@ -1294,6 +1295,82 @@ class Connection(api.Connection):
return total_to_migrate, total_migrated
@oslo_db_api.retry_on_deadlock
def migrate_to_hardware_types(self, context, max_count,
reset_unsupported_interfaces=False):
"""Migrate nodes from classic drivers to hardware types.
Go through all nodes with a classic driver and try to migrate them to
a corresponding hardware type and a correct set of hardware interfaces.
If migration is not possible for any reason (e.g. the target hardware
type is not enabled), the nodes are skipped. An operator is expected to
correct the configuration and either rerun online_data_migration or
migrate the nodes manually.
:param context: the admin context (not used)
:param max_count: The maximum number of objects to migrate. Must be
>= 0. If zero, all the objects will be migrated.
:param reset_unsupported_interfaces: whether to reset unsupported
optional interfaces to their no-XXX versions.
:returns: A 2-tuple, 1. the total number of objects that need to be
migrated (at the beginning of this call) and 2. the number
of migrated objects.
"""
reset_unsupported_interfaces = strutils.bool_from_string(
reset_unsupported_interfaces, strict=True)
drivers = driver_factory.classic_drivers_to_migrate()
total_to_migrate = (model_query(models.Node)
.filter(models.Node.driver.in_(list(drivers)))
.count())
total_migrated = 0
for driver, driver_cls in drivers.items():
if max_count and total_migrated >= max_count:
return total_to_migrate, total_migrated
# UPDATE with LIMIT seems to be a MySQL-only feature, so first
# fetch the required number of Node IDs, then update them.
query = model_query(models.Node.id).filter_by(driver=driver)
if max_count:
query = query.limit(max_count - total_migrated)
ids = [obj.id for obj in query]
if not ids:
continue
delta = driver_factory.calculate_migration_delta(
driver, driver_cls, reset_unsupported_interfaces)
if delta is None:
# NOTE(dtantsur): mark unsupported nodes as migrated. Otherwise
# calling online_data_migration without --max-count will result
# in a infinite loop.
total_migrated += len(ids)
continue
# UPDATE with LIMIT seems to be a MySQL-only feature, so first
# fetch the required number of Node IDs, then update them.
query = model_query(models.Node.id).filter_by(driver=driver)
if max_count:
query = query.limit(max_count - total_migrated)
ids = [obj.id for obj in query]
if not ids:
LOG.debug('No nodes with driver %s', driver)
continue
LOG.info('Migrating nodes with driver %(drv)s to %(delta)s',
{'drv': driver, 'delta': delta})
with _session_for_write():
num_migrated = (model_query(models.Node)
.filter_by(driver=driver)
.filter(models.Node.id.in_(ids))
.update(delta, synchronize_session=False))
total_migrated += num_migrated
return total_to_migrate, total_migrated
@staticmethod
def _verify_max_traits_per_node(node_id, num_traits):
"""Verify that an operation would not exceed the per-node trait limit.

12
ironic/drivers/base.py
View File

@ -147,6 +147,18 @@ class BaseDriver(object):
properties.update(iface.get_properties())
return properties
@classmethod
def to_hardware_type(cls):
"""Return corresponding hardware type and hardware interfaces.
:returns: a tuple with two items:
* new driver field - the target hardware type
* dictionary containing interfaces to update, e.g.
{'deploy': 'iscsi', 'power': 'ipmitool'}
"""
raise NotImplementedError()
class BareDriver(BaseDriver):
"""A bare driver object which will have interfaces attached later.

69
ironic/drivers/fake.py
View File

@ -70,6 +70,14 @@ class FakeDriver(base.BaseDriver):
self.inspect = fake.FakeInspect()
self.raid = fake.FakeRAID()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
iface: 'fake'
for iface in ['boot', 'console', 'deploy', 'inspect',
'management', 'power', 'raid', 'rescue', 'vendor']
}
class FakeSoftPowerDriver(FakeDriver):
"""Example implementation of a Driver."""
@ -89,6 +97,17 @@ class FakeIPMIToolDriver(base.BaseDriver):
self.vendor = ipmitool.VendorPassthru()
self.management = ipmitool.IPMIManagement()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
'boot': 'fake',
'console': 'ipmitool-shellinabox',
'deploy': 'fake',
'management': 'ipmitool',
'power': 'ipmitool',
'vendor': 'ipmitool'
}
class FakeIPMIToolSocatDriver(base.BaseDriver):
"""Example implementation of a Driver."""
@ -100,6 +119,17 @@ class FakeIPMIToolSocatDriver(base.BaseDriver):
self.vendor = ipmitool.VendorPassthru()
self.management = ipmitool.IPMIManagement()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
'boot': 'fake',
'console': 'ipmitool-socat',
'deploy': 'fake',
'management': 'ipmitool',
'power': 'ipmitool',
'vendor': 'ipmitool'
}
class FakePXEDriver(base.BaseDriver):
"""Example implementation of a Driver."""
@ -109,6 +139,15 @@ class FakePXEDriver(base.BaseDriver):
self.boot = pxe.PXEBoot()
self.deploy = iscsi_deploy.ISCSIDeploy()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
'boot': 'pxe',
'deploy': 'iscsi',
'management': 'fake',
'power': 'fake',
}
class FakeAgentDriver(base.BaseDriver):
"""Example implementation of an AgentDriver."""
@ -119,6 +158,16 @@ class FakeAgentDriver(base.BaseDriver):
self.deploy = agent.AgentDeploy()
self.raid = agent.AgentRAID()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
'boot': 'pxe',
'deploy': 'direct',
'management': 'fake',
'power': 'fake',
'raid': 'agent'
}
class FakeIloDriver(base.BaseDriver):
"""Fake iLO driver, used in testing."""
@ -162,6 +211,15 @@ class FakeSNMPDriver(base.BaseDriver):
self.power = snmp.SNMPPower()
self.deploy = fake.FakeDeploy()
@classmethod
def to_hardware_type(cls):
return 'snmp', {
'boot': 'fake',
'deploy': 'fake',
'management': 'fake',
'power': 'snmp',
}
class FakeIRMCDriver(base.BaseDriver):
"""Fake iRMC driver."""
@ -191,6 +249,17 @@ class FakeIPMIToolInspectorDriver(base.BaseDriver):
# integration.
self.inspect = inspector.Inspector()
@classmethod
def to_hardware_type(cls):
return 'fake-hardware', {
'boot': 'fake',
'console': 'ipmitool-shellinabox',
'deploy': 'fake',
'inspect': 'inspector',
'management': 'ipmitool',
'power': 'ipmitool',
}
class FakeUcsDriver(base.BaseDriver):
"""Fake UCS driver."""

45
ironic/drivers/ipmi.py
View File

@ -14,6 +14,8 @@
Hardware types and classic drivers for IPMI (using ipmitool).
"""
from oslo_config import cfg
from ironic.drivers import base
from ironic.drivers import generic
from ironic.drivers.modules import agent
@ -24,6 +26,9 @@ from ironic.drivers.modules import noop
from ironic.drivers.modules import pxe
CONF = cfg.CONF
class IPMIHardware(generic.GenericHardware):
"""IPMI hardware type.
@ -53,6 +58,22 @@ class IPMIHardware(generic.GenericHardware):
return [ipmitool.VendorPassthru, noop.NoVendor]
def _to_hardware_type():
# NOTE(dtantsur): classic drivers are not affected by the
# enabled_inspect_interfaces configuration option.
if CONF.inspector.enabled:
inspect_interface = 'inspector'
else:
inspect_interface = 'no-inspect'
return {'boot': 'pxe',
'inspect': inspect_interface,
'management': 'ipmitool',
'power': 'ipmitool',
'raid': 'agent',
'vendor': 'ipmitool'}
class PXEAndIPMIToolDriver(base.BaseDriver):
"""PXE + IPMITool driver.
@ -74,6 +95,12 @@ class PXEAndIPMIToolDriver(base.BaseDriver):
self.vendor = ipmitool.VendorPassthru()
self.raid = agent.AgentRAID()
@classmethod
def to_hardware_type(cls):
return 'ipmi', dict(_to_hardware_type(),
console='ipmitool-shellinabox',
deploy='iscsi')
class PXEAndIPMIToolAndSocatDriver(PXEAndIPMIToolDriver):
"""PXE + IPMITool + socat driver.
@ -93,6 +120,12 @@ class PXEAndIPMIToolAndSocatDriver(PXEAndIPMIToolDriver):
PXEAndIPMIToolDriver.__init__(self)
self.console = ipmitool.IPMISocatConsole()
@classmethod
def to_hardware_type(cls):
return 'ipmi', dict(_to_hardware_type(),
console='ipmitool-socat',
deploy='iscsi')
class AgentAndIPMIToolDriver(base.BaseDriver):
"""Agent + IPMITool driver.
@ -116,6 +149,12 @@ class AgentAndIPMIToolDriver(base.BaseDriver):
self.inspect = inspector.Inspector.create_if_enabled(
'AgentAndIPMIToolDriver')
@classmethod
def to_hardware_type(cls):
return 'ipmi', dict(_to_hardware_type(),
console='ipmitool-shellinabox',
deploy='direct')
class AgentAndIPMIToolAndSocatDriver(AgentAndIPMIToolDriver):
"""Agent + IPMITool + socat driver.
@ -134,3 +173,9 @@ class AgentAndIPMIToolAndSocatDriver(AgentAndIPMIToolDriver):
def __init__(self):
AgentAndIPMIToolDriver.__init__(self)
self.console = ipmitool.IPMISocatConsole()
@classmethod
def to_hardware_type(cls):
return 'ipmi', dict(_to_hardware_type(),
console='ipmitool-socat',
deploy='direct')

9
ironic/drivers/pxe.py
View File

@ -100,6 +100,15 @@ class PXEAndSNMPDriver(base.BaseDriver):
# Only PXE as a boot device is supported.
self.management = None
@classmethod
def to_hardware_type(cls):
return 'snmp', {
'boot': 'pxe',
'deploy': 'iscsi',
'management': 'fake',
'power': 'snmp',
}
class PXEAndIRMCDriver(base.BaseDriver):
"""PXE + iRMC driver using SCCI.

76
ironic/tests/unit/common/test_driver_factory.py
View File

@ -14,6 +14,7 @@
import mock
from oslo_utils import uuidutils
import stevedore
from stevedore import named
from ironic.common import driver_factory
@ -796,3 +797,78 @@ class HardwareTypeLoadTestCase(db_base.DbTestCase):
def test_enabled_supported_interfaces_non_default(self):
self._test_enabled_supported_interfaces(True)
class ClassicDriverMigrationTestCase(base.TestCase):
def setUp(self):
super(ClassicDriverMigrationTestCase, self).setUp()
self.driver_cls = mock.Mock(spec=['to_hardware_type'])
self.driver_cls2 = mock.Mock(spec=['to_hardware_type'])
self.new_ifaces = {
'console': 'new-console',
'inspect': 'new-inspect'
}
self.driver_cls.to_hardware_type.return_value = ('hw-type',
self.new_ifaces)
self.ext = mock.Mock(plugin=self.driver_cls)
self.ext.name = 'drv1'
self.ext2 = mock.Mock(plugin=self.driver_cls2)
self.ext2.name = 'drv2'
self.config(enabled_hardware_types=['hw-type'],
enabled_console_interfaces=['no-console', 'new-console'],
enabled_inspect_interfaces=['no-inspect', 'new-inspect'],
enabled_raid_interfaces=['no-raid'],
enabled_rescue_interfaces=['no-rescue'],
enabled_vendor_interfaces=['no-vendor'])
def test_calculate_migration_delta(self):
delta = driver_factory.calculate_migration_delta(
'drv', self.driver_cls, False)
self.assertEqual({'driver': 'hw-type',
'console_interface': 'new-console',
'inspect_interface': 'new-inspect',
'raid_interface': 'no-raid',
'rescue_interface': 'no-rescue',
'vendor_interface': 'no-vendor'},
delta)
def test_calculate_migration_delta_not_implemeted(self):
self.driver_cls.to_hardware_type.side_effect = NotImplementedError()
delta = driver_factory.calculate_migration_delta(
'drv', self.driver_cls, False)
self.assertIsNone(delta)
def test_calculate_migration_delta_unsupported_hw_type(self):
self.driver_cls.to_hardware_type.return_value = ('hw-type2',
self.new_ifaces)
delta = driver_factory.calculate_migration_delta(
'drv', self.driver_cls, False)
self.assertIsNone(delta)
def test__calculate_migration_delta_unsupported_interface(self):
self.new_ifaces['inspect'] = 'unsupported inspect'
delta = driver_factory.calculate_migration_delta(
'drv', self.driver_cls, False)
self.assertIsNone(delta)
def test_calculate_migration_delta_unsupported_interface_reset(self):
self.new_ifaces['inspect'] = 'unsupported inspect'
delta = driver_factory.calculate_migration_delta(
'drv', self.driver_cls, True)
self.assertEqual({'driver': 'hw-type',
'console_interface': 'new-console',
'inspect_interface': 'no-inspect',
'raid_interface': 'no-raid',
'rescue_interface': 'no-rescue',
'vendor_interface': 'no-vendor'},
delta)
@mock.patch.object(stevedore, 'ExtensionManager', autospec=True)
def test_classic_drivers_to_migrate(self, mock_ext_mgr):
mock_ext_mgr.return_value.__iter__.return_value = iter([self.ext,
self.ext2])
self.assertEqual({'drv1': self.driver_cls,
'drv2': self.driver_cls2},
driver_factory.classic_drivers_to_migrate())

34
ironic/tests/unit/db/test_api.py
View File

@ -10,9 +10,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
from oslo_utils import uuidutils
from ironic.common import context
from ironic.common import driver_factory
from ironic.common import release_mappings
from ironic.db import api as db_api
from ironic.tests.unit.db import base
@ -137,3 +139,35 @@ class BackfillVersionTestCase(base.DbTestCase):
for hostname in conductors:
conductor = self.dbapi.get_conductor(hostname)
self.assertEqual(self.conductor_ver, conductor.version)
@mock.patch.object(driver_factory, 'calculate_migration_delta', autospec=True)
@mock.patch.object(driver_factory, 'classic_drivers_to_migrate', autospec=True)
class MigrateToHardwareTypesTestCase(base.DbTestCase):
def setUp(self):
super(MigrateToHardwareTypesTestCase, self).setUp()
self.context = context.get_admin_context()
self.dbapi = db_api.get_instance()
self.node = utils.create_test_node(uuid=uuidutils.generate_uuid(),
driver='classic_driver')
def test_migrate(self, mock_drivers, mock_delta):
mock_drivers.return_value = {'classic_driver': mock.sentinel.drv1,
'another_driver': mock.sentinel.drv2}
mock_delta.return_value = {'driver': 'new_driver',
'inspect_interface': 'new_inspect'}
result = self.dbapi.migrate_to_hardware_types(self.context, 0)
self.assertEqual((1, 1), result)
node = self.dbapi.get_node_by_id(self.node.id)
self.assertEqual('new_driver', node.driver)
self.assertEqual('new_inspect', node.inspect_interface)
def test_migrate_unsupported(self, mock_drivers, mock_delta):
mock_drivers.return_value = {'classic_driver': mock.sentinel.drv1,
'another_driver': mock.sentinel.drv2}
mock_delta.return_value = None
result = self.dbapi.migrate_to_hardware_types(self.context, 0)
self.assertEqual((1, 1), result)
node = self.dbapi.get_node_by_id(self.node.id)
self.assertEqual('classic_driver', node.driver)

55
ironic/tests/unit/drivers/test_base.py
View File

@ -16,7 +16,9 @@
import json
import mock
import stevedore
from ironic.common import driver_factory
from ironic.common import exception
from ironic.common import raid
from ironic.drivers import base as driver_base
@ -451,3 +453,56 @@ class TestBareDriver(base.TestCase):
'rescue', 'storage'),
driver_base.BareDriver.standard_interfaces
)
class TestToHardwareType(base.TestCase):
def setUp(self):
super(TestToHardwareType, self).setUp()
self.driver_classes = list(
driver_factory.classic_drivers_to_migrate().values())
self.existing_ifaces = {}
for iface in driver_base.ALL_INTERFACES:
self.existing_ifaces[iface] = stevedore.ExtensionManager(
'ironic.hardware.interfaces.%s' % iface,
invoke_on_load=False).names()
self.hardware_types = stevedore.ExtensionManager(
'ironic.hardware.types', invoke_on_load=False).names()
# These are the interfaces that don't have a no-op version
self.mandatory_interfaces = ['boot', 'deploy', 'management', 'power']
def test_to_hardware_type_returns_hardware_type(self):
for driver in self.driver_classes:
try:
hw_type = driver.to_hardware_type()[0]
except NotImplementedError:
continue
self.assertIn(hw_type, self.hardware_types,
'%s returns unknown hardware type %s' %
(driver, hw_type))
def test_to_hardware_type_returns_existing_interfaces(self):
# Check that all defined implementations of to_hardware_type
# contain only existing interface types
for driver in self.driver_classes:
try:
delta = driver.to_hardware_type()[1]
except NotImplementedError:
continue
for iface, value in delta.items():
self.assertIn(iface, self.existing_ifaces,
'%s returns unknown interface %s' %
(driver, iface))
self.assertIn(value, self.existing_ifaces[iface],
'%s returns unknown %s interface %s' %
(driver, iface, value))
def test_to_hardware_type_mandatory_interfaces(self):
for driver in self.driver_classes:
try:
delta = driver.to_hardware_type()[1]
except NotImplementedError:
continue
for iface in self.mandatory_interfaces:
self.assertIn(iface, delta,
'%s does not return mandatory interface %s' %
(driver, iface))

23
releasenotes/notes/migrate_to_hardware_types-0c85c6707c4f296d.yaml Normal file
View File

@ -0,0 +1,23 @@
---
upgrade:
- |
Adds new data migration ``migrate_to_hardware_types`` that will try to
migrate nodes from classic drivers to hardware types on upgrade. Matching
hardware types and interfaces have to be provided on classic drivers
themselves. Nodes that cannot be migrated are skipped. This can primary
happen for three reasons:
* migration is not implemented for the classic driver,
* the matching hardware type is not enabled,
* one or more matching hardware interfaces are not enabled.
In the latter case, the new migration command line option
``reset_unsupported_interfaces`` can be used to reset optional interfaces
(all except for ``boot``, ``deploy``, ``management`` and ``power``) to
their no-op implementations (e.g. ``no-inspect``) if the matching
implementation is not enabled. Use it like::
ironic-dbsync online_data_migrations --option migrate_to_hardware_types.reset_unsupported_interfaces=true
This migration can be repeated several times to migrate skipped nodes
after the configuration is changed.