diff --git a/devstack/lib/ironic b/devstack/lib/ironic index 4a8133c1d4..b712a65377 100644 --- a/devstack/lib/ironic +++ b/devstack/lib/ironic @@ -510,6 +510,10 @@ function configure_ironic_conductor { iniset $IRONIC_CONF_FILE deploy http_root $IRONIC_HTTP_DIR iniset $IRONIC_CONF_FILE deploy http_url "http://$IRONIC_HTTP_SERVER:$IRONIC_HTTP_PORT" fi + + if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then + iniset $IRONIC_CONF_FILE neutron port_setup_delay 15 + fi } # create_ironic_cache_dir() - Part of the init_ironic() process diff --git a/etc/ironic/ironic.conf.sample b/etc/ironic/ironic.conf.sample index afb11f3088..a287f9c293 100644 --- a/etc/ironic/ironic.conf.sample +++ b/etc/ironic/ironic.conf.sample @@ -1548,6 +1548,11 @@ # value) #url_timeout = 30 +# Delay value to wait for Neutron agents to setup sufficient +# DHCP configuration for port. (integer value) +# Minimum value: 0 +#port_setup_delay = 0 + # Client retries in the case of a failed request. (integer # value) #retries = 3 diff --git a/ironic/dhcp/neutron.py b/ironic/dhcp/neutron.py index 63fac876e2..7d55cefd9c 100644 --- a/ironic/dhcp/neutron.py +++ b/ironic/dhcp/neutron.py @@ -40,6 +40,11 @@ neutron_opts = [ cfg.IntOpt('url_timeout', default=30, help=_('Timeout value for connecting to neutron in seconds.')), + cfg.IntOpt('port_setup_delay', + default=0, + min=0, + help=_('Delay value to wait for Neutron agents to setup ' + 'sufficient DHCP configuration for port.')), cfg.IntOpt('retries', default=3, help=_('Client retries in the case of a failed request.')), @@ -193,13 +198,22 @@ class NeutronDHCPApi(base.BaseDHCP): {'node': task.node.uuid, 'ports': failures}) # TODO(adam_g): Hack to workaround bug 1334447 until we have a - # mechanism for synchronizing events with Neutron. We need to sleep - # only if we are booting VMs, which is implied by SSHPower, to ensure - # they do not boot before Neutron agents have setup sufficient DHCP - # config for netboot. - if isinstance(task.driver.power, ssh.SSHPower): - LOG.debug("Waiting 15 seconds for Neutron.") - time.sleep(15) + # mechanism for synchronizing events with Neutron. We need to sleep + # only if server gets to PXE faster than Neutron agents have setup + # sufficient DHCP config for netboot. It may occur when we are using + # VMs or hardware server with fast boot enabled. + port_delay = CONF.neutron.port_setup_delay + # TODO(vsaienko) remove hardcoded value for SSHPower driver + # after Newton release. + if isinstance(task.driver.power, ssh.SSHPower) and port_delay == 0: + LOG.warning(_LW("Setting the port delay to 15 for SSH power " + "driver by default, this will be removed in " + "Ocata release. Please set configuration " + "parameter port_setup_delay to 15.")) + port_delay = 15 + if port_delay != 0: + LOG.debug("Waiting %d seconds for Neutron.", port_delay) + time.sleep(port_delay) def _get_fixed_ip_address(self, port_uuid, client): """Get a Neutron port's fixed ip address. diff --git a/ironic/tests/unit/dhcp/test_neutron.py b/ironic/tests/unit/dhcp/test_neutron.py index 1b0283469a..2647639e71 100644 --- a/ironic/tests/unit/dhcp/test_neutron.py +++ b/ironic/tests/unit/dhcp/test_neutron.py @@ -26,6 +26,7 @@ from ironic.common import exception from ironic.common import pxe_utils from ironic.conductor import task_manager from ironic.dhcp import neutron +from ironic.drivers.modules import ssh from ironic.tests.unit.conductor import mgr_utils from ironic.tests.unit.db import base as db_base from ironic.tests.unit.objects import utils as object_utils @@ -250,6 +251,87 @@ class TestNeutron(db_base.DbTestCase): mock_gnvi.assert_called_once_with(task) self.assertEqual(2, mock_updo.call_count) + @mock.patch('time.sleep', autospec=True) + @mock.patch.object(neutron.NeutronDHCPApi, 'update_port_dhcp_opts', + autospec=True) + @mock.patch('ironic.common.network.get_node_vif_ids', autospec=True) + def test_update_dhcp_set_sleep_and_ssh(self, mock_gnvi, mock_updo, + mock_ts): + mock_gnvi.return_value = {'ports': {'port-uuid': 'vif-uuid'}, + 'portgroups': {}} + self.config(port_setup_delay=30, group='neutron') + with task_manager.acquire(self.context, + self.node.uuid) as task: + task.driver.power = ssh.SSHPower() + opts = pxe_utils.dhcp_options_for_instance(task) + api = dhcp_factory.DHCPFactory() + api.update_dhcp(task, opts) + mock_ts.assert_called_with(30) + mock_updo.assert_called_once_with(mock.ANY, 'vif-uuid', opts, + token=self.context.auth_token) + + @mock.patch.object(neutron, 'LOG', autospec=True) + @mock.patch('time.sleep', autospec=True) + @mock.patch.object(neutron.NeutronDHCPApi, 'update_port_dhcp_opts', + autospec=True) + @mock.patch('ironic.common.network.get_node_vif_ids', autospec=True) + def test_update_dhcp_unset_sleep_and_ssh(self, mock_gnvi, mock_updo, + mock_ts, mock_log): + mock_gnvi.return_value = {'ports': {'port-uuid': 'vif-uuid'}, + 'portgroups': {}} + with task_manager.acquire(self.context, + self.node.uuid) as task: + opts = pxe_utils.dhcp_options_for_instance(task) + task.driver.power = ssh.SSHPower() + api = dhcp_factory.DHCPFactory() + api.update_dhcp(task, opts) + self.assertTrue(mock_log.warning.called) + self.assertIn('Setting the port delay to 15 for SSH', + mock_log.warning.call_args[0][0]) + mock_ts.assert_called_with(15) + mock_updo.assert_called_once_with(mock.ANY, 'vif-uuid', opts, + token=self.context.auth_token) + + @mock.patch.object(neutron, 'LOG', autospec=True) + @mock.patch('time.sleep', autospec=True) + @mock.patch.object(neutron.NeutronDHCPApi, 'update_port_dhcp_opts', + autospec=True) + @mock.patch('ironic.common.network.get_node_vif_ids', autospec=True) + def test_update_dhcp_set_sleep_and_fake(self, mock_gnvi, mock_updo, + mock_ts, mock_log): + mock_gnvi.return_value = {'ports': {'port-uuid': 'vif-uuid'}, + 'portgroups': {}} + self.config(port_setup_delay=30, group='neutron') + with task_manager.acquire(self.context, + self.node.uuid) as task: + opts = pxe_utils.dhcp_options_for_instance(task) + api = dhcp_factory.DHCPFactory() + api.update_dhcp(task, opts) + mock_log.debug.assert_called_once_with( + "Waiting %d seconds for Neutron.", 30) + mock_log.warning.assert_not_called() + mock_ts.assert_called_with(30) + mock_updo.assert_called_once_with(mock.ANY, 'vif-uuid', opts, + token=self.context.auth_token) + + @mock.patch.object(neutron, 'LOG', autospec=True) + @mock.patch.object(neutron.NeutronDHCPApi, 'update_port_dhcp_opts', + autospec=True) + @mock.patch('ironic.common.network.get_node_vif_ids', autospec=True) + def test_update_dhcp_unset_sleep_and_fake(self, mock_gnvi, mock_updo, + mock_log): + mock_gnvi.return_value = {'ports': {'port-uuid': 'vif-uuid'}, + 'portgroups': {}} + with task_manager.acquire(self.context, + self.node.uuid) as task: + opts = pxe_utils.dhcp_options_for_instance(task) + api = dhcp_factory.DHCPFactory() + api.update_dhcp(task, opts) + mock_log.debug.assert_not_called() + mock_log.warning.assert_not_called() + mock_updo.assert_called_once_with(mock.ANY, 'vif-uuid', opts, + token=self.context.auth_token) + def test__get_fixed_ip_address(self): port_id = 'fake-port-id' expected = "192.168.1.3" diff --git a/releasenotes/notes/neutron-port-timeout-cbd82e1d09c6a46c.yaml b/releasenotes/notes/neutron-port-timeout-cbd82e1d09c6a46c.yaml new file mode 100644 index 0000000000..9b40ca35e1 --- /dev/null +++ b/releasenotes/notes/neutron-port-timeout-cbd82e1d09c6a46c.yaml @@ -0,0 +1,7 @@ +--- +other: + - Add Neutron ``port_setup_delay`` configuration + option. This delay allows Ironic to wait for + Neutron port operations until we have a + mechanism for synchronizing events with Neutron. + Set to 0 by default.