Merge "Add address scopes support to the L3 agent"

doc/source/devref/address_scopes.rst

@@ -170,3 +170,43 @@ references a single subnet pool::
     |                |        |                  |        |              |
     |                |        |                  |        |              |
     +----------------+        +------------------+        +--------------+
+
+L3 Agent
+~~~~~~~~
+
+The L3 agent is limited in its support for multiple address scopes.  Within a
+router in the reference implementation, traffic is marked on ingress with the
+address scope corresponding to the network it is coming from.  If that traffic
+would route to an interface in a different address scope, the traffic is
+blocked unless an exception is made.
+
+One exception is made for floating IP traffic.  When traffic is headed to a
+floating IP, DNAT is applied and the traffic is allowed to route to the private
+IP address potentially crossing the address scope boundary.  When traffic
+flows from an internal port to the external network and a floating IP is
+assigned, that traffic is also allowed.
+
+Another exception is made for traffic from an internal network to the external
+network when SNAT is enabled.  In this case, SNAT to the router's fixed IP
+address is applied to the traffic.  However, SNAT is not used if the external
+network has an explicit address scope assigned and it matches the internal
+network's.  In that case, traffic routes straight through without NAT.  The
+internal network's addresses are viable on the external network in this case.
+
+The reference implementation has limitations.  Even with multiple address
+scopes, a router implementation is unable to connect to two networks with
+overlapping IP addresses.  There are two reasons for this.
+
+First, a single routing table is used inside the namespace.  An implementation
+using multiple routing tables has been in the works but there are some
+unresolved issues with it.
+
+Second, the default SNAT feature cannot be supported with the current Linux
+conntrack implementation unless a double NAT is used (one NAT to get from the
+address scope to an intermediate address specific to the scope and a second NAT
+to get from that intermediate address to an external address).  Single NAT
+won't work if there are duplicate addresses across the scopes.
+
+Due to these complications the router will still refuse to connect to
+overlapping subnets.  We can look in to an implementation that overcomes these
+limitations in the future.

neutron/agent/l3/dvr_edge_router.py

@@ -20,6 +20,7 @@ from neutron.agent.l3 import dvr_snat_ns
 from neutron.agent.l3 import router_info as router
 from neutron.agent.linux import ip_lib
 from neutron.agent.linux import iptables_manager
+from neutron.common import constants as l3_constants
 
 LOG = logging.getLogger(__name__)
 
@@ -207,3 +208,28 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter):
         super(DvrEdgeRouter, self).delete(agent)
         if self.snat_namespace:
             self.snat_namespace.delete()
+
+    def process_address_scope(self):
+        super(DvrEdgeRouter, self).process_address_scope()
+
+        if not self._is_this_snat_host():
+            return
+
+        snat_iptables_manager = self.snat_iptables_manager
+        # Prepare address scope iptables rule for dvr snat interfaces
+        internal_ports = self.get_snat_interfaces()
+        ports_scopemark = self._get_port_devicename_scopemark(
+            internal_ports, self._get_snat_int_device_name)
+        # Prepare address scope iptables rule for external port
+        external_port = self.get_ex_gw_port()
+        if external_port:
+            external_port_scopemark = self._get_port_devicename_scopemark(
+                [external_port], self.get_external_device_name)
+            for ip_version in (l3_constants.IP_VERSION_4,
+                               l3_constants.IP_VERSION_6):
+                ports_scopemark[ip_version].update(
+                    external_port_scopemark[ip_version])
+
+        with snat_iptables_manager.defer_apply():
+            self._add_address_scope_mark(
+                snat_iptables_manager, ports_scopemark)

neutron/agent/l3/dvr_local_router.py

@@ -410,6 +410,14 @@ class DvrLocalRouter(dvr_router_base.DvrRouterBase):
     def _handle_router_snat_rules(self, ex_gw_port, interface_name):
         pass
 
+    def _get_address_scope_mark(self):
+        # Prepare address scope iptables rule for internal ports
+        internal_ports = self.router.get(l3_constants.INTERFACE_KEY, [])
+        ports_scopemark = self._get_port_devicename_scopemark(
+            internal_ports, self.get_internal_device_name)
+        # DVR local router don't need to consider external port
+        return ports_scopemark
+
     def process_external(self, agent):
         ex_gw_port = self.get_ex_gw_port()
         if ex_gw_port:

neutron/agent/l3/router_info.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import netaddr
 from oslo_log import log as logging
 
@@ -30,6 +31,9 @@ INTERNAL_DEV_PREFIX = namespaces.INTERNAL_DEV_PREFIX
 EXTERNAL_DEV_PREFIX = namespaces.EXTERNAL_DEV_PREFIX
 
 FLOATINGIP_STATUS_NOCHANGE = object()
+ADDRESS_SCOPE_MARK_MASK = "0xffff0000"
+ADDRESS_SCOPE_MARK_IDS = set(range(1024, 2048))
+DEFAULT_ADDRESS_SCOPE = "noscope"
 
 
 class RouterInfo(object):
@@ -52,6 +56,8 @@ class RouterInfo(object):
             router_id, agent_conf, interface_driver, use_ipv6)
         self.router_namespace = ns
         self.ns_name = ns.name
+        self._address_scope_to_mark_id = {
+            DEFAULT_ADDRESS_SCOPE: ADDRESS_SCOPE_MARK_IDS.pop()}
         self.iptables_manager = iptables_manager.IptablesManager(
             use_ipv6=use_ipv6,
             namespace=self.ns_name)
@@ -140,6 +146,44 @@ class RouterInfo(object):
                 ('float-snat', '-s %s -j SNAT --to %s' %
                  (fixed_ip, floating_ip))]
 
+    def floating_mangle_rules(self, floating_ip, fixed_ip,
+                              external_devicename, internal_mark):
+        mark_traffic_to_floating_ip = (
+            'floatingip', '-d %s -j MARK --set-mark %s' % (
+                floating_ip, internal_mark))
+        mark_traffic_from_fixed_ip = (
+            'FORWARD', '-s %s -o %s -j $float-snat' % (
+                fixed_ip, external_devicename))
+        return [mark_traffic_to_floating_ip, mark_traffic_from_fixed_ip]
+
+    def get_address_scope_mark_mask(self, address_scope=None):
+        if not address_scope:
+            address_scope = DEFAULT_ADDRESS_SCOPE
+
+        if address_scope not in self._address_scope_to_mark_id:
+            mark_ids = set(self._address_scope_to_mark_id.values())
+            available_ids = ADDRESS_SCOPE_MARK_IDS - mark_ids
+            self._address_scope_to_mark_id[address_scope] = (
+                available_ids.pop())
+
+        mark_id = self._address_scope_to_mark_id[address_scope]
+        # NOTE: Address scopes use only the upper 16 bits of the 32 fwmark
+        return "%s/%s" % (hex(mark_id << 16), ADDRESS_SCOPE_MARK_MASK)
+
+    def get_port_address_scope_mark(self, port):
+        """Get the IP version 4 and 6 address scope mark for the port
+
+        :param port: A port dict from the RPC call
+        :returns: A dict mapping the address family to the address scope mark
+        """
+        port_scopes = port.get('address_scopes', {})
+
+        address_scope_mark_masks = (
+            (int(k), self.get_address_scope_mark_mask(v))
+            for k, v in port_scopes.items())
+        return collections.defaultdict(self.get_address_scope_mark_mask,
+                                       address_scope_mark_masks)
+
     def process_floating_ip_nat_rules(self):
         """Configure NAT rules for the router's floating IPs.
 
@@ -160,6 +204,42 @@ class RouterInfo(object):
 
         self.iptables_manager.apply()
 
+    def process_floating_ip_address_scope_rules(self):
+        """Configure address scope related iptables rules for the router's
+         floating IPs.
+        """
+
+        # Clear out all iptables rules for floating ips
+        self.iptables_manager.ipv4['mangle'].clear_rules_by_tag('floating_ip')
+
+        floating_ips = self.get_floating_ips()
+        if floating_ips:
+            ext_scope = self._get_external_address_scope()
+            # Add address scope for floatingip egress
+            self.iptables_manager.ipv4['mangle'].add_rule(
+                'float-snat',
+                '-j MARK --set-xmark %s'
+                % self.get_address_scope_mark_mask(ext_scope),
+                tag='floating_ip')
+
+        # Loop once to ensure that floating ips are configured.
+        for fip in floating_ips:
+            # Rebuild iptables rules for the floating ip.
+            fip_ip = fip['floating_ip_address']
+            # Send the floating ip traffic to the right address scope
+            fixed_ip = fip['fixed_ip_address']
+            fixed_scope = fip.get('fixed_ip_address_scope')
+            internal_mark = self.get_address_scope_mark_mask(fixed_scope)
+            external_port = self.get_ex_gw_port()
+            external_devicename = self.get_external_device_interface_name(
+                external_port)
+            mangle_rules = self.floating_mangle_rules(fip_ip, fixed_ip,
+                                                      external_devicename,
+                                                      internal_mark)
+            for chain, rule in mangle_rules:
+                self.iptables_manager.ipv4['mangle'].add_rule(
+                    chain, rule, tag='floating_ip')
+
     def process_snat_dnat_for_fip(self):
         try:
             self.process_floating_ip_nat_rules()
@@ -357,9 +437,26 @@ class RouterInfo(object):
         self.radvd.disable()
 
     def internal_network_updated(self, interface_name, ip_cidrs):
-        self.driver.init_l3(interface_name, ip_cidrs=ip_cidrs,
+        self.driver.init_router_port(
+            interface_name,
+            ip_cidrs=ip_cidrs,
             namespace=self.ns_name)
 
+    def address_scope_mangle_rule(self, device_name, mark_mask):
+        external_port = self.get_ex_gw_port()
+        if external_port:
+            external_device_name = self.get_external_device_name(
+                external_port['id'])
+            if external_device_name == device_name:
+                return ('-i %s -m connmark --mark 0x0/0xffff0000 '
+                        '-j CONNMARK --set-mark %s'
+                        % (device_name, mark_mask))
+        return '-i %s -j MARK --set-mark %s' % (device_name, mark_mask)
+
+    def address_scope_filter_rule(self, device_name, mark_mask):
+        return '-o %s -m mark ! --mark %s -j DROP' % (
+            device_name, mark_mask)
+
     def _process_internal_ports(self, pd):
         existing_port_ids = set(p['id'] for p in self.internal_ports)
 
@@ -613,9 +710,12 @@ class RouterInfo(object):
         iptables_manager.ipv4['nat'].empty_chain('POSTROUTING')
         iptables_manager.ipv4['nat'].empty_chain('snat')
         iptables_manager.ipv4['mangle'].empty_chain('mark')
+        iptables_manager.ipv4['mangle'].empty_chain('POSTROUTING')
 
     def _add_snat_rules(self, ex_gw_port, iptables_manager,
                         interface_name):
+        self.process_external_port_address_scope_routing(iptables_manager)
+
         if ex_gw_port:
             # ex_gw_port should not be None in this case
             # NAT rules are added only if ex_gw_port has an IPv4 address
@@ -707,6 +807,110 @@ class RouterInfo(object):
         agent.plugin_rpc.update_floatingip_statuses(
             agent.context, self.router_id, fip_statuses)
 
+    def _get_port_devicename_scopemark(self, ports, name_generator):
+        devicename_scopemark = {l3_constants.IP_VERSION_4: dict(),
+                                l3_constants.IP_VERSION_6: dict()}
+        for p in ports:
+            device_name = name_generator(p['id'])
+            ip_cidrs = common_utils.fixed_ip_cidrs(p['fixed_ips'])
+            port_as_marks = self.get_port_address_scope_mark(p)
+            for ip_version in {ip_lib.get_ip_version(cidr)
+                               for cidr in ip_cidrs}:
+                devicename_scopemark[ip_version][device_name] = (
+                    port_as_marks[ip_version])
+
+        return devicename_scopemark
+
+    def _get_address_scope_mark(self):
+        # Prepare address scope iptables rule for internal ports
+        internal_ports = self.router.get(l3_constants.INTERFACE_KEY, [])
+        ports_scopemark = self._get_port_devicename_scopemark(
+            internal_ports, self.get_internal_device_name)
+
+        # Prepare address scope iptables rule for external port
+        external_port = self.get_ex_gw_port()
+        if external_port:
+            external_port_scopemark = self._get_port_devicename_scopemark(
+                [external_port], self.get_external_device_name)
+            for ip_version in (l3_constants.IP_VERSION_4,
+                               l3_constants.IP_VERSION_6):
+                ports_scopemark[ip_version].update(
+                    external_port_scopemark[ip_version])
+        return ports_scopemark
+
+    def _add_address_scope_mark(self, iptables_manager, ports_scopemark):
+        external_device_name = None
+        external_port = self.get_ex_gw_port()
+        if external_port:
+            external_device_name = self.get_external_device_name(
+                external_port['id'])
+
+        # Process address scope iptables rules
+        for ip_version in (l3_constants.IP_VERSION_4,
+                           l3_constants.IP_VERSION_6):
+            scopemarks = ports_scopemark[ip_version]
+            iptables = iptables_manager.get_tables(ip_version)
+            iptables['mangle'].empty_chain('scope')
+            iptables['filter'].empty_chain('scope')
+            dont_block_external = (ip_version == l3_constants.IP_VERSION_4
+                                   and self._snat_enabled and external_port)
+            for device_name, mark in scopemarks.items():
+                # Add address scope iptables rule
+                iptables['mangle'].add_rule(
+                    'scope',
+                    self.address_scope_mangle_rule(device_name, mark))
+                if dont_block_external and device_name == external_device_name:
+                    continue
+                iptables['filter'].add_rule(
+                    'scope',
+                    self.address_scope_filter_rule(device_name, mark))
+
+    def process_ports_address_scope_iptables(self):
+        ports_scopemark = self._get_address_scope_mark()
+        self._add_address_scope_mark(self.iptables_manager, ports_scopemark)
+
+    def _get_external_address_scope(self):
+        external_port = self.get_ex_gw_port()
+        if not external_port:
+            return
+
+        scopes = external_port.get('address_scopes', {})
+        return scopes.get(str(l3_constants.IP_VERSION_4))
+
+    def process_external_port_address_scope_routing(self, iptables_manager):
+        if not self._snat_enabled:
+            return
+
+        external_port = self.get_ex_gw_port()
+        if not external_port:
+            return
+
+        external_devicename = self.get_external_device_name(
+            external_port['id'])
+
+        # Records snat egress address scope
+        rule = ('-o %s -m connmark --mark 0x0/0xffff0000 '
+                '-j CONNMARK --save-mark '
+                '--nfmask 0xffff0000 --ctmask 0xffff0000' %
+                external_devicename)
+
+        iptables_manager.ipv4['mangle'].add_rule('POSTROUTING', rule)
+
+        address_scope = self._get_external_address_scope()
+        if not address_scope:
+            return
+
+        # Prevents snat within the same address scope
+        rule = '-o %s -m connmark --mark %s -j ACCEPT' % (
+            external_devicename,
+            self.get_address_scope_mark_mask(address_scope))
+        iptables_manager.ipv4['nat'].add_rule('snat', rule)
+
+    def process_address_scope(self):
+        with self.iptables_manager.defer_apply():
+            self.process_ports_address_scope_iptables()
+            self.process_floating_ip_address_scope_rules()
+
     @common_utils.exception_logger()
     def process_delete(self, agent):
         """Process the delete of this router
@@ -736,6 +940,7 @@ class RouterInfo(object):
         self._process_internal_ports(agent.pd)
         agent.pd.sync_router(self.router['id'])
         self.process_external(agent)
+        self.process_address_scope()
         # Process static routes for router
         self.routes_updated(self.routes, self.router['routes'])
         self.routes = self.router['routes']

neutron/agent/linux/iptables_manager.py

@@ -336,6 +336,11 @@ class IptablesManager(object):
             builtin_chains[4].update(
                 {'mangle': ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT',
                             'POSTROUTING']})
+            self.ipv6.update(
+                {'mangle': IptablesTable(binary_name=self.wrap_name)})
+            builtin_chains[6].update(
+                {'mangle': ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT',
+                            'POSTROUTING']})
             self.ipv4.update(
                 {'nat': IptablesTable(binary_name=self.wrap_name)})
             builtin_chains[4].update({'nat': ['PREROUTING',
@@ -385,9 +390,55 @@ class IptablesManager(object):
             self.ipv4['mangle'].add_chain('mark')
             self.ipv4['mangle'].add_rule('PREROUTING', '-j $mark')
 
+            # Add address scope related chains
+            self.ipv4['mangle'].add_chain('scope')
+            self.ipv6['mangle'].add_chain('scope')
+
+            self.ipv4['mangle'].add_chain('floatingip')
+            self.ipv4['mangle'].add_chain('float-snat')
+
+            self.ipv4['filter'].add_chain('scope')
+            self.ipv6['filter'].add_chain('scope')
+            self.ipv4['filter'].add_rule('FORWARD', '-j $scope')
+            self.ipv6['filter'].add_rule('FORWARD', '-j $scope')
+
+            # Add rules for marking traffic for address scopes
+            mark_new_ingress_address_scope_by_interface = (
+                '-j $scope')
+            copy_address_scope_for_existing = (
+                '-m connmark ! --mark 0x0/0xffff0000 '
+                '-j CONNMARK --restore-mark '
+                '--nfmask 0xffff0000 --ctmask 0xffff0000')
+            mark_new_ingress_address_scope_by_floatingip = (
+                '-j $floatingip')
+            save_mark_to_connmark = (
+                '-m connmark --mark 0x0/0xffff0000 '
+                '-j CONNMARK --save-mark '
+                '--nfmask 0xffff0000 --ctmask 0xffff0000')
+
+            self.ipv4['mangle'].add_rule(
+                'PREROUTING', mark_new_ingress_address_scope_by_interface)
+            self.ipv4['mangle'].add_rule(
+                'PREROUTING', copy_address_scope_for_existing)
+            # The floating ip scope rules must come after the CONNTRACK rules
+            # because the (CONN)MARK targets are non-terminating (this is true
+            # despite them not being documented as such) and the floating ip
+            # rules need to override the mark from CONNMARK to cross scopes.
+            self.ipv4['mangle'].add_rule(
+                'PREROUTING', mark_new_ingress_address_scope_by_floatingip)
+            self.ipv4['mangle'].add_rule(
+                'float-snat', save_mark_to_connmark)
+            self.ipv6['mangle'].add_rule(
+                'PREROUTING', mark_new_ingress_address_scope_by_interface)
+            self.ipv6['mangle'].add_rule(
+                'PREROUTING', copy_address_scope_for_existing)
+
+    def get_tables(self, ip_version):
+        return {4: self.ipv4, 6: self.ipv6}[ip_version]
+
     def get_chain(self, table, chain, ip_version=4, wrap=True):
         try:
-            requested_table = {4: self.ipv4, 6: self.ipv6}[ip_version][table]
+            requested_table = self.get_tables(ip_version)[table]
         except KeyError:
             return []
         return requested_table._get_chain_rules(chain, wrap)

neutron/tests/unit/agent/l3/test_agent.py

@@ -744,7 +744,13 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
                 self.assertIn(r.rule, expected_rules)
         expected_rules = [
             '-i %s -j MARK --set-xmark 0x2/%s' %
-            (interface_name, l3_constants.ROUTER_MARK_MASK)]
+            (interface_name, l3_constants.ROUTER_MARK_MASK),
+            '-o %s -m connmark --mark 0x0/%s -j CONNMARK '
+            '--save-mark --nfmask %s --ctmask %s' %
+            (interface_name,
+             l3router.ADDRESS_SCOPE_MARK_MASK,
+             l3router.ADDRESS_SCOPE_MARK_MASK,
+             l3router.ADDRESS_SCOPE_MARK_MASK)]
         for r in mangle_rules:
             if negate:
                 self.assertNotIn(r.rule, expected_rules)
@@ -809,6 +815,7 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
             fake_fip_id: 'ACTIVE'}
         ri.external_gateway_added = mock.Mock()
         ri.external_gateway_updated = mock.Mock()
+        ri.process_address_scope = mock.Mock()
         fake_floatingips1 = {'floatingips': [
             {'id': fake_fip_id,
              'floating_ip_address': '8.8.8.8',
@@ -1140,7 +1147,7 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         mangle_rules_delta = [
             r for r in orig_mangle_rules
             if r not in ri.iptables_manager.ipv4['mangle'].rules]
-        self.assertEqual(1, len(mangle_rules_delta))
+        self.assertEqual(2, len(mangle_rules_delta))
         self._verify_snat_mangle_rules(nat_rules_delta, mangle_rules_delta,
                                        router)
         self.assertEqual(1, self.send_adv_notif.call_count)
@@ -1167,7 +1174,7 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         mangle_rules_delta = [
             r for r in ri.iptables_manager.ipv4['mangle'].rules
             if r not in orig_mangle_rules]
-        self.assertEqual(1, len(mangle_rules_delta))
+        self.assertEqual(2, len(mangle_rules_delta))
         self._verify_snat_mangle_rules(nat_rules_delta, mangle_rules_delta,
                                        router)
         self.assertEqual(1, self.send_adv_notif.call_count)

neutron/tests/unit/agent/l3/test_router_info.py

@@ -122,6 +122,27 @@ class TestRouterInfo(base.BaseTestCase):
                     'via', '10.100.10.30']]
         self._check_agent_method_called(expected)
 
+    def test_add_ports_address_scope_iptables(self):
+        ri = router_info.RouterInfo(_uuid(), {}, **self.ri_kwargs)
+        port = {
+            'id': _uuid(),
+            'fixed_ips': [{'ip_address': '172.9.9.9'}],
+            'address_scopes': {l3_constants.IP_VERSION_4: '1234'}
+        }
+        ipv4_mangle = ri.iptables_manager.ipv4['mangle'] = mock.MagicMock()
+        ri.get_address_scope_mark_mask = mock.Mock(return_value='fake_mark')
+        ri.get_internal_device_name = mock.Mock(return_value='fake_device')
+        ri.rt_tables_manager = mock.MagicMock()
+        ri.process_external_port_address_scope_routing = mock.Mock()
+        ri.process_floating_ip_address_scope_rules = mock.Mock()
+        ri.iptables_manager._apply = mock.Mock()
+
+        ri.router[l3_constants.INTERFACE_KEY] = [port]
+        ri.process_address_scope()
+
+        ipv4_mangle.add_rule.assert_called_once_with(
+            'scope', ri.address_scope_mangle_rule('fake_device', 'fake_mark'))
+
 
 class BasicRouterTestCaseFramework(base.BaseTestCase):
     def _create_router(self, router=None, **kwargs):
@@ -185,6 +206,42 @@ class TestBasicRouterOperations(BasicRouterTestCaseFramework):
         # Be sure that add_rule is called somewhere in the middle
         self.assertFalse(ipv4_nat.add_rule.called)
 
+    def test_process_floating_ip_address_scope_rules(self):
+        ri = self._create_router()
+        fips = [{'fixed_ip_address': mock.sentinel.ip,
+                 'floating_ip_address': mock.sentinel.fip}]
+        ri.get_floating_ips = mock.Mock(return_value=fips)
+        ipv4_mangle = ri.iptables_manager.ipv4['mangle'] = mock.MagicMock()
+        ri.floating_mangle_rules = mock.Mock(
+            return_value=[(mock.sentinel.chain1, mock.sentinel.rule1)])
+        ri.get_external_device_name = mock.Mock()
+
+        ri.process_floating_ip_address_scope_rules()
+
+        # Be sure that the rules are cleared first
+        self.assertEqual(mock.call.clear_rules_by_tag('floating_ip'),
+                         ipv4_mangle.mock_calls[0])
+        # Be sure that add_rule is called somewhere in the middle
+        self.assertEqual(2, ipv4_mangle.add_rule.call_count)
+        self.assertEqual(mock.call.add_rule(mock.sentinel.chain1,
+                                            mock.sentinel.rule1,
+                                            tag='floating_ip'),
+                         ipv4_mangle.mock_calls[2])
+
+    def test_process_floating_ip_mangle_rules_removed(self):
+        ri = self._create_router()
+        ri.get_floating_ips = mock.Mock(return_value=[])
+        ipv4_mangle = ri.iptables_manager.ipv4['mangle'] = mock.MagicMock()
+
+        ri.process_floating_ip_address_scope_rules()
+
+        # Be sure that the rules are cleared first
+        self.assertEqual(mock.call.clear_rules_by_tag('floating_ip'),
+                         ipv4_mangle.mock_calls[0])
+
+        # Be sure that add_rule is not called somewhere in the middle
+        self.assertFalse(ipv4_mangle.add_rule.called)
+
     def _test_add_fip_addr_to_device_error(self, device):
         ri = self._create_router()
         ip = '15.1.2.3'

neutron/tests/unit/agent/linux/test_iptables_manager.py

@@ -67,12 +67,14 @@ FILTER_TEMPLATE = ('# Generated by iptables_manager\n'
                    ':%(bn)s-INPUT - [0:0]\n'
                    ':%(bn)s-OUTPUT - [0:0]\n'
                    ':%(bn)s-local - [0:0]\n'
+                   ':%(bn)s-scope - [0:0]\n'
                    '-I FORWARD 1 -j neutron-filter-top\n'
                    '-I FORWARD 2 -j %(bn)s-FORWARD\n'
                    '-I INPUT 1 -j %(bn)s-INPUT\n'
                    '-I OUTPUT 1 -j neutron-filter-top\n'
                    '-I OUTPUT 2 -j %(bn)s-OUTPUT\n'
                    '-I neutron-filter-top 1 -j %(bn)s-local\n'
+                   '-I %(bn)s-FORWARD 1 -j %(bn)s-scope\n'
                    'COMMIT\n'
                    '# Completed by iptables_manager\n')
 
@@ -90,12 +92,14 @@ FILTER_WITH_RULES_TEMPLATE = (
     ':%(bn)s-OUTPUT - [0:0]\n'
     ':%(bn)s-filter - [0:0]\n'
     ':%(bn)s-local - [0:0]\n'
+    ':%(bn)s-scope - [0:0]\n'
     '-I FORWARD 1 -j neutron-filter-top\n'
     '-I FORWARD 2 -j %(bn)s-FORWARD\n'
     '-I INPUT 1 -j %(bn)s-INPUT\n'
     '-I OUTPUT 1 -j neutron-filter-top\n'
     '-I OUTPUT 2 -j %(bn)s-OUTPUT\n'
     '-I neutron-filter-top 1 -j %(bn)s-local\n'
+    '-I %(bn)s-FORWARD 1 -j %(bn)s-scope\n'
     '%(filter_rules)s'
     'COMMIT\n'
     '# Completed by iptables_manager\n')
@@ -235,13 +239,51 @@ def _generate_mangle_dump(iptables_args):
             ':%(bn)s-OUTPUT - [0:0]\n'
             ':%(bn)s-POSTROUTING - [0:0]\n'
             ':%(bn)s-PREROUTING - [0:0]\n'
+            ':%(bn)s-float-snat - [0:0]\n'
+            ':%(bn)s-floatingip - [0:0]\n'
             ':%(bn)s-mark - [0:0]\n'
+            ':%(bn)s-scope - [0:0]\n'
             '-I FORWARD 1 -j %(bn)s-FORWARD\n'
             '-I INPUT 1 -j %(bn)s-INPUT\n'
             '-I OUTPUT 1 -j %(bn)s-OUTPUT\n'
             '-I POSTROUTING 1 -j %(bn)s-POSTROUTING\n'
             '-I PREROUTING 1 -j %(bn)s-PREROUTING\n'
             '-I %(bn)s-PREROUTING 1 -j %(bn)s-mark\n'
+            '-I %(bn)s-PREROUTING 2 -j %(bn)s-scope\n'
+            '-I %(bn)s-PREROUTING 3 -m connmark ! --mark 0x0/0xffff0000 '
+            '-j CONNMARK --restore-mark '
+            '--nfmask 0xffff0000 --ctmask 0xffff0000\n'
+            '-I %(bn)s-PREROUTING 4 -j %(bn)s-floatingip\n'
+            '-I %(bn)s-float-snat 1 -m connmark --mark 0x0/0xffff0000 '
+            '-j CONNMARK --save-mark '
+            '--nfmask 0xffff0000 --ctmask 0xffff0000\n'
+            'COMMIT\n'
+            '# Completed by iptables_manager\n' % iptables_args)
+
+
+def _generate_mangle_dump_v6(iptables_args):
+    return ('# Generated by iptables_manager\n'
+            '*mangle\n'
+            ':FORWARD - [0:0]\n'
+            ':INPUT - [0:0]\n'
+            ':OUTPUT - [0:0]\n'
+            ':POSTROUTING - [0:0]\n'
+            ':PREROUTING - [0:0]\n'
+            ':%(bn)s-FORWARD - [0:0]\n'
+            ':%(bn)s-INPUT - [0:0]\n'
+            ':%(bn)s-OUTPUT - [0:0]\n'
+            ':%(bn)s-POSTROUTING - [0:0]\n'
+            ':%(bn)s-PREROUTING - [0:0]\n'
+            ':%(bn)s-scope - [0:0]\n'
+            '-I FORWARD 1 -j %(bn)s-FORWARD\n'
+            '-I INPUT 1 -j %(bn)s-INPUT\n'
+            '-I OUTPUT 1 -j %(bn)s-OUTPUT\n'
+            '-I POSTROUTING 1 -j %(bn)s-POSTROUTING\n'
+            '-I PREROUTING 1 -j %(bn)s-PREROUTING\n'
+            '-I %(bn)s-PREROUTING 1 -j %(bn)s-scope\n'
+            '-I %(bn)s-PREROUTING 2 -m connmark ! --mark 0x0/0xffff0000 '
+            '-j CONNMARK --restore-mark '
+            '--nfmask 0xffff0000 --ctmask 0xffff0000\n'
             'COMMIT\n'
             '# Completed by iptables_manager\n' % iptables_args)
 
@@ -260,6 +302,7 @@ def _generate_raw_dump(iptables_args):
 
 
 MANGLE_DUMP = _generate_mangle_dump(IPTABLES_ARG)
+MANGLE_DUMP_V6 = _generate_mangle_dump_v6(IPTABLES_ARG)
 RAW_DUMP = _generate_raw_dump(IPTABLES_ARG)
 
 
@@ -350,8 +393,10 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            mangle_dump_v6 = _generate_mangle_dump_v6(iptables_args)
-                                               filter_dump_ipv6 + raw_dump)
+            self._extend_with_ip6tables_filter(
+                expected_calls_and_values,
+                filter_dump_ipv6 + mangle_dump_v6 + raw_dump)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -410,8 +455,10 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            mangle_dump_v6 = _generate_mangle_dump_v6(iptables_args)
-                                               filter_dump + raw_dump)
+            self._extend_with_ip6tables_filter(
+                expected_calls_and_values,
+                filter_dump + mangle_dump_v6 + raw_dump)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -457,8 +504,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + RAW_DUMP)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + RAW_DUMP)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -511,8 +559,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + raw_dump)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + raw_dump)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -563,12 +612,14 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
                            ':%(bn)s-INPUT - [0:0]\n'
                            ':%(bn)s-OUTPUT - [0:0]\n'
                            ':%(bn)s-local - [0:0]\n'
+                           ':%(bn)s-scope - [0:0]\n'
                            '-I FORWARD 1 -j neutron-filter-top\n'
                            '-I FORWARD 2 -j %(bn)s-FORWARD\n'
                            '-I INPUT 1 -j %(bn)s-INPUT\n'
                            '-I OUTPUT 1 -j neutron-filter-top\n'
                            '-I OUTPUT 2 -j %(bn)s-OUTPUT\n'
                            '-I neutron-filter-top 1 -j %(bn)s-local\n'
+                           '-I %(bn)s-FORWARD 1 -j %(bn)s-scope\n'
                            '-I %(bn)s-INPUT 1 -s 0/0 -d 192.168.0.2 -j '
                            '%(wrap)s\n'
                            'COMMIT\n'
@@ -596,8 +647,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + raw_dump)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + raw_dump)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -640,15 +692,26 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
             ':%(bn)s-OUTPUT - [0:0]\n'
             ':%(bn)s-POSTROUTING - [0:0]\n'
             ':%(bn)s-PREROUTING - [0:0]\n'
+            ':%(bn)s-float-snat - [0:0]\n'
+            ':%(bn)s-floatingip - [0:0]\n'
             ':%(bn)s-mangle - [0:0]\n'
             ':%(bn)s-mark - [0:0]\n'
+            ':%(bn)s-scope - [0:0]\n'
             '-I FORWARD 1 -j %(bn)s-FORWARD\n'
             '-I INPUT 1 -j %(bn)s-INPUT\n'
             '-I OUTPUT 1 -j %(bn)s-OUTPUT\n'
             '-I POSTROUTING 1 -j %(bn)s-POSTROUTING\n'
             '-I PREROUTING 1 -j %(bn)s-PREROUTING\n'
             '-I %(bn)s-PREROUTING 1 -j %(bn)s-mark\n'
-            '-I %(bn)s-PREROUTING 2 -j MARK --set-xmark 0x1/%(mark)s\n'
+            '-I %(bn)s-PREROUTING 2 -j %(bn)s-scope\n'
+            '-I %(bn)s-PREROUTING 3 -m connmark ! --mark 0x0/0xffff0000 '
+            '-j CONNMARK --restore-mark '
+            '--nfmask 0xffff0000 --ctmask 0xffff0000\n'
+            '-I %(bn)s-PREROUTING 4 -j %(bn)s-floatingip\n'
+            '-I %(bn)s-PREROUTING 5 -j MARK --set-xmark 0x1/%(mark)s\n'
+            '-I %(bn)s-float-snat 1 -m connmark --mark 0x0/0xffff0000 '
+            '-j CONNMARK --save-mark '
+            '--nfmask 0xffff0000 --ctmask 0xffff0000\n'
             'COMMIT\n'
             '# Completed by iptables_manager\n' % IPTABLES_ARG)
 
@@ -671,8 +734,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + RAW_DUMP)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + RAW_DUMP)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -751,8 +815,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + raw_dump)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + raw_dump)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -822,8 +887,9 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
              None),
         ]
         if use_ipv6:
-            self._extend_with_ip6tables_filter(expected_calls_and_values,
+            self._extend_with_ip6tables_filter(
-                                               FILTER_DUMP + RAW_DUMP)
+                expected_calls_and_values,
+                FILTER_DUMP + MANGLE_DUMP_V6 + RAW_DUMP)
 
         tools.setup_mock_calls(self.execute, expected_calls_and_values)
 
@@ -974,6 +1040,10 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
                            '-n', '-v', '-x'],
                            run_as_root=True),
                  TRAFFIC_COUNTERS_DUMP))
+            expected_calls_and_values.append(
+                (mock.call(['ip6tables', '-t', 'mangle', '-L', 'OUTPUT',
+                           '-n', '-v', '-x'], run_as_root=True),
+                 ''))
             exp_packets *= 2
             exp_bytes *= 2
 
@@ -1027,6 +1097,10 @@ class IptablesManagerStateFulTestCase(base.BaseTestCase):
                             '-n', '-v', '-x', '-Z'],
                            run_as_root=True),
                  TRAFFIC_COUNTERS_DUMP))
+            expected_calls_and_values.append(
+                (mock.call(['ip6tables', '-t', 'mangle', '-L', 'OUTPUT',
+                            '-n', '-v', '-x', '-Z'], run_as_root=True),
+                 ''))
             exp_packets *= 2
             exp_bytes *= 2
 

neutron/tests/unit/agent/test_securitygroups_rpc.py

@@ -1692,7 +1692,8 @@ IPTABLES_ARG = {'bn': iptables_manager.binary_name,
                 'physdev_mod': PHYSDEV_MOD,
                 'physdev_is_bridged': PHYSDEV_IS_BRIDGED}
 
-CHAINS_MANGLE = 'FORWARD|INPUT|OUTPUT|POSTROUTING|PREROUTING|mark'
+CHAINS_MANGLE = ('FORWARD|INPUT|OUTPUT|POSTROUTING|PREROUTING|mark|scope'
+                 '|float-snat|floatingip')
 IPTABLES_ARG['chains'] = CHAINS_MANGLE
 
 IPTABLES_MANGLE = """# Generated by iptables_manager
@@ -1708,12 +1709,48 @@ IPTABLES_MANGLE = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+-I FORWARD 1 -j %(bn)s-FORWARD
+-I INPUT 1 -j %(bn)s-INPUT
+-I OUTPUT 1 -j %(bn)s-OUTPUT
+-I POSTROUTING 1 -j %(bn)s-POSTROUTING
+-I PREROUTING 1 -j %(bn)s-PREROUTING
+-I %(bn)s-PREROUTING 1 -j %(bn)s-mark
+-I %(bn)s-PREROUTING 2 -j %(bn)s-scope
+-I %(bn)s-PREROUTING 3 -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK \
+--restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000
+-I %(bn)s-PREROUTING 4 -j %(bn)s-floatingip
+-I %(bn)s-float-snat 1 -m connmark --mark 0x0/0xffff0000 \
+-j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000
+COMMIT
+# Completed by iptables_manager
+""" % IPTABLES_ARG
+
+CHAINS_MANGLE_V6 = 'FORWARD|INPUT|OUTPUT|POSTROUTING|PREROUTING|scope'
+IPTABLES_ARG['chains'] = CHAINS_MANGLE_V6
+IPTABLES_MANGLE_V6 = """# Generated by iptables_manager
+*mangle
+:FORWARD - [0:0]
+:INPUT - [0:0]
+:OUTPUT - [0:0]
+:POSTROUTING - [0:0]
+:PREROUTING - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j %(bn)s-OUTPUT
 -I POSTROUTING 1 -j %(bn)s-POSTROUTING
 -I PREROUTING 1 -j %(bn)s-PREROUTING
--I %(bn)s-PREROUTING 1 -j %(bn)s-mark
+-I %(bn)s-PREROUTING 1 -j %(bn)s-scope
+-I %(bn)s-PREROUTING 2 -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK \
+--restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000
 COMMIT
 # Completed by iptables_manager
 """ % IPTABLES_ARG
@@ -1808,7 +1845,7 @@ COMMIT
 # Completed by iptables_manager
 """ % IPTABLES_ARG
 
-CHAINS_EMPTY = 'FORWARD|INPUT|OUTPUT|local|sg-chain|sg-fallback'
+CHAINS_EMPTY = ('FORWARD|INPUT|OUTPUT|local|scope|sg-chain|sg-fallback')
 CHAINS_1 = CHAINS_EMPTY + '|i_port1|o_port1|s_port1'
 CHAINS_2 = CHAINS_1 + '|i_port2|o_port2|s_port2'
 
@@ -1829,15 +1866,17 @@ IPSET_FILTER_1 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_port1 \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_port1 \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-o_port1
@@ -1885,15 +1924,17 @@ IPTABLES_FILTER_1 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_port1 \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_port1 \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-o_port1
@@ -1940,15 +1981,17 @@ IPTABLES_FILTER_1_2 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_port1 \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_port1 \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-o_port1
@@ -2000,19 +2043,21 @@ IPSET_FILTER_2 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2082,19 +2127,21 @@ IPSET_FILTER_2_3 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2166,19 +2213,21 @@ IPTABLES_FILTER_2 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2250,19 +2299,21 @@ IPTABLES_FILTER_2_2 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2331,19 +2382,21 @@ IPTABLES_FILTER_2_3 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2411,12 +2464,14 @@ IPTABLES_FILTER_EMPTY = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
 -I %(bn)s-sg-chain 1 -j ACCEPT
 -I %(bn)s-sg-fallback 1 -j DROP
 COMMIT
@@ -2438,15 +2493,17 @@ IPTABLES_FILTER_V6_1 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_port1 \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_port1 \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_port1 \
 %(physdev_is_bridged)s -j %(bn)s-o_port1
@@ -2494,19 +2551,21 @@ IPTABLES_FILTER_V6_2 = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
--I %(bn)s-FORWARD 1 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
+-I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-INGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 2 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
+-I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 3 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-INGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
--I %(bn)s-FORWARD 4 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
+-I %(bn)s-FORWARD 5 %(physdev_mod)s --physdev-EGRESS tap_%(port2)s \
 %(physdev_is_bridged)s -j %(bn)s-sg-chain
 -I %(bn)s-INPUT 1 %(physdev_mod)s --physdev-EGRESS tap_%(port1)s \
 %(physdev_is_bridged)s -j %(bn)s-o_%(port1)s
@@ -2569,12 +2628,14 @@ IPTABLES_FILTER_V6_EMPTY = """# Generated by iptables_manager
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
 :%(bn)s-(%(chains)s) - [0:0]
+:%(bn)s-(%(chains)s) - [0:0]
 -I FORWARD 1 -j neutron-filter-top
 -I FORWARD 2 -j %(bn)s-FORWARD
 -I INPUT 1 -j %(bn)s-INPUT
 -I OUTPUT 1 -j neutron-filter-top
 -I OUTPUT 2 -j %(bn)s-OUTPUT
 -I neutron-filter-top 1 -j %(bn)s-local
+-I %(bn)s-FORWARD 1 -j %(bn)s-scope
 -I %(bn)s-sg-chain 1 -j ACCEPT
 -I %(bn)s-sg-fallback 1 -j DROP
 COMMIT
@@ -2759,7 +2820,7 @@ class TestSecurityGroupAgentWithIptables(base.BaseTestCase):
             return_value='')
         self._register_mock_call(
             ['ip6tables-restore', '-n'],
-            process_input=self._regex(v6_filter + raw),
+            process_input=self._regex(v6_filter + IPTABLES_MANGLE_V6 + raw),
             run_as_root=True,
             return_value='')