NSXv: Fine grained control for logging security-group rules

Allows admin to control security-groups rule logging

NSXv distributed firewall expose an API to control rule logging,
as for the moment, admin user can use this feature only from inside of
the distributed firewall.
This patch make use of this API to provide the cloud admin with three ways
to control security-group logging:

    - log whenever security-group rule is matched
    - log when a packet doesn't match any security-group rule
    - log whenever security-group rule is matched for selected
      security-groups

Change-Id: I2a4dbff2ecba4c6041b4aaad1f20941440a5f6b6

etc/nsx.ini

@@ -191,6 +191,14 @@
 # (Optional) DHCP lease time
 # dhcp_lease_time = 86400
 
+# (Optional) Indicates whether distributed-firewall rule for security-groups
+# blocked traffic is logged.
+# log_security_groups_blocked_traffic = False
+
+# (Optional) Indicates whether distributed-firewall security-groups rules are
+# logged.
+# log_security_groups_allowed_traffic = False
+
 [nsx]
 # Maximum number of ports for each bridged logical switch
 # The recommended value for this parameter varies with NSX version

etc/policy.json

@@ -139,5 +139,9 @@
 
     "get_service_provider": "rule:regular_user",
     "get_lsn": "rule:admin_only",
-    "create_lsn": "rule:admin_only"
+    "create_lsn": "rule:admin_only",
+
+    "create_security_group:logging": "rule:admin_only",
+    "update_security_group:logging": "rule:admin_only",
+    "get_security_group:logging": "rule:admin_only"
 }

vmware_nsx/common/config.py

@@ -401,6 +401,14 @@ nsxv_opts = [
                        'involves configuring the dvs backing nsx_v directly. '
                        'If False, only features exposed via nsx_v will be '
                        'supported')),
+    cfg.BoolOpt('log_security_groups_blocked_traffic',
+                default=False,
+                help=_("Indicates whether distributed-firewall rule for "
+                       "security-groups blocked traffic is logged")),
+    cfg.BoolOpt('log_security_groups_allowed_traffic',
+                default=False,
+                help=_("Indicates whether distributed-firewall "
+                       "security-groups allowed traffic is logged")),
 ]
 
 # Register the configuration options

vmware_nsx/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -1 +1 @@
-4c45bcadccf9
+2c87aedb206f

vmware_nsx/db/migration/alembic_migrations/versions/newton/expand/2c87aedb206f_nsxv_security_group_logging.py

@@ -0,0 +1,34 @@
+# Copyright 2016 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""nsxv_security_group_logging
+
+Revision ID: 2c87aedb206f
+Revises: 4c45bcadccf9
+Create Date: 2016-03-15 06:06:06.680092
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '2c87aedb206f'
+down_revision = '4c45bcadccf9'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('nsxv_security_group_section_mappings',
+                  sa.Column('logging', sa.Boolean(), nullable=False))

vmware_nsx/db/nsxv_db.py

@@ -366,10 +366,10 @@ def delete_nsxv_internal_edge(session, ext_ip_address):
                 filter_by(ext_ip_address=ext_ip_address).delete())
 
 
-def add_neutron_nsx_section_mapping(session, neutron_id, ip_section_id):
+def add_neutron_nsx_section_mapping(session, neutron_id, section_id, logging):
     with session.begin(subtransactions=True):
         mapping = nsxv_models.NsxvSecurityGroupSectionMapping(
-            neutron_id=neutron_id, ip_section_id=ip_section_id)
+            neutron_id=neutron_id, ip_section_id=section_id, logging=logging)
         session.add(mapping)
     return mapping
 

vmware_nsx/db/nsxv_models.py

@@ -114,6 +114,7 @@ class NsxvSecurityGroupSectionMapping(model_base.BASEV2):
                                          ondelete="CASCADE"),
                            primary_key=True)
     ip_section_id = sa.Column(sa.String(100))
+    logging = sa.Column(sa.Boolean, default=False, nullable=False)
 
 
 class NsxvRuleMapping(model_base.BASEV2):

vmware_nsx/extensions/securitygrouplogging.py

@@ -0,0 +1,67 @@
+# Copyright 2016 VMware, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.api import extensions
+from neutron.api.v2 import attributes
+
+RESOURCE_ATTRIBUTE_MAP = {
+    'security_groups': {
+        'logging': {
+            'allow_post': True,
+            'allow_put': True,
+            'convert_to': attributes.convert_to_boolean,
+            'default': False,
+            'enforce_policy': True,
+            'is_visible': True}
+    }
+}
+
+
+class Securitygrouplogging(extensions.ExtensionDescriptor):
+    """Security group logging extension."""
+
+    @classmethod
+    def get_name(cls):
+        return "Security group logging"
+
+    @classmethod
+    def get_alias(cls):
+        return "security-group-logging"
+
+    @classmethod
+    def get_description(cls):
+        return "Security group logging extension."
+
+    @classmethod
+    def get_namespace(cls):
+        # todo
+        return "http://docs.openstack.org/ext/security_group_logging/api/v2.0"
+
+    @classmethod
+    def get_updated(cls):
+        return "2015-04-13T10:00:00-00:00"
+
+    def get_required_extensions(self):
+        return ["security-group"]
+
+    @classmethod
+    def get_resources(cls):
+        """Returns Ext Resources."""
+        return []
+
+    def get_extended_resources(self, version):
+        if version == "2.0":
+            return RESOURCE_ATTRIBUTE_MAP
+        else:
+            return {}

vmware_nsx/plugins/nsx_v/plugin.py

@@ -31,6 +31,7 @@ from neutron.api.v2 import attributes as attr
 from neutron.callbacks import events
 from neutron.callbacks import registry
 from neutron.callbacks import resources
+from neutron import context as n_context
 from neutron.db import agents_db
 from neutron.db import allowedaddresspairs_db as addr_pair_db
 from neutron.db import db_base_plugin_v2
@@ -122,6 +123,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                                    "router",
                                    "security-group",
                                    "secgroup-rule-local-ip-prefix",
+                                   "security-group-logging",
                                    "nsxv-router-type",
                                    "nsxv-router-size",
                                    "vnic-index",
@@ -170,6 +172,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         self._validate_config()
         self.sg_container_id = self._create_security_group_container()
         self.default_section = self._create_cluster_default_fw_section()
+        self._process_security_groups_rules_logging()
         self._router_managers = managers.RouterTypeManager(self)
 
         if cfg.CONF.nsxv.use_dvs_features:
@@ -264,7 +267,8 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
 
         # Default security-group rules
         block_rule = self.nsx_sg_utils.get_rule_config(
-            [self.sg_container_id], 'Block All', 'deny')
+            [self.sg_container_id], 'Block All', 'deny',
+            logged=cfg.CONF.nsxv.log_security_groups_blocked_traffic)
         rule_list.append(block_rule)
 
         with locking.LockManager.get_lock('default-section-init'):
@@ -282,6 +286,38 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                 section_id = self.nsx_sg_utils.parse_and_get_section_id(c)
         return section_id
 
+    def _process_security_groups_rules_logging(self):
+
+        with locking.LockManager.get_lock('nsx-dfw-section',
+                                          lock_file_prefix='dfw-section',
+                                          external=True):
+            context = n_context.get_admin_context()
+            log_all_rules = cfg.CONF.nsxv.log_security_groups_allowed_traffic
+
+            for sg in self.get_security_groups(context, fields=['id']):
+                fw_section = self._get_section(context.session, sg['id'])
+                # If the section/sg is already logged, then no action is
+                # required.
+                if fw_section is None or fw_section['logging']:
+                    continue
+
+                # Section/sg is not logged, update rules logging according to
+                # the 'log_security_groups_allowed_traffic' config option.
+                try:
+                    section_uri = fw_section['ip_section_id']
+                    h, c = self.nsx_v.vcns.get_section(section_uri)
+                    section = self.nsx_sg_utils.parse_section(c)
+                    section_needs_update = (
+                        self.nsx_sg_utils.set_rules_logged_option(
+                            section, log_all_rules))
+                    if section_needs_update:
+                        self.nsx_v.vcns.update_section(
+                            section_uri,
+                            self.nsx_sg_utils.to_xml_string(section), h)
+                except Exception as exc:
+                    LOG.error(_LE('Unable to update section for logging. %s'),
+                              exc)
+
     def _create_dhcp_static_binding(self, context, neutron_port_db):
 
         network_id = neutron_port_db['network_id']
@@ -1926,6 +1962,22 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         if mapping is not None:
             return mapping['ip_section_id']
 
+    def _get_section(self, session, security_group_id):
+        return nsxv_db.get_nsx_section(session, security_group_id)
+
+    def _update_section_logging(self, session, section, section_db):
+        logging = not section_db['logging']
+        # Update the DB for the new logging settings.
+        with session.begin(subtransactions=True):
+            section_db['logging'] = logging
+        # Update section rules logging only if we are not already logging them.
+        log_all_rules = cfg.CONF.nsxv.log_security_groups_allowed_traffic
+        section_needs_update = False
+        if not log_all_rules:
+            section_needs_update = (
+                self.nsx_sg_utils.set_rules_logged_option(section, logging))
+        return section_needs_update
+
     def create_security_group(self, context, security_group,
                               default_sg=False):
         """Create a security group."""
@@ -1934,6 +1986,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
 
         new_security_group = super(NsxVPluginV2, self).create_security_group(
             context, security_group, default_sg)
+        sg_id = new_security_group['id']
 
         nsx_sg_name = self.nsx_sg_utils.get_nsx_sg_name(sg_data)
         # NSX security-group config
@@ -1943,12 +1996,17 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         # Translate Neutron rules to NSXv fw rules and construct the fw section
         nsx_sg_id = section_uri = None
         try:
+            log_all_rules = cfg.CONF.nsxv.log_security_groups_allowed_traffic
             # Create the nsx security group
             h, nsx_sg_id = self.nsx_v.vcns.create_security_group(sg_dict)
 
             section_name = self.nsx_sg_utils.get_nsx_section_name(nsx_sg_name)
-            nsx_rules = [self._create_nsx_rule(context, rule, nsx_sg_id) for
-                         rule in new_security_group['security_group_rules']]
+            logging = sg_data.get('logging', False)
+            nsx_rules = []
+            for rule in new_security_group['security_group_rules']:
+                nsx_rule = self._create_nsx_rule(
+                    context, rule, nsx_sg_id, logged=log_all_rules or logging)
+                nsx_rules.append(nsx_rule)
             section = self.nsx_sg_utils.get_section_with_rules(
                 section_name, nsx_rules)
 
@@ -1961,10 +2019,10 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
 
             # Save moref in the DB for future access
             nsx_db.add_neutron_nsx_security_group_mapping(
-                context.session, new_security_group['id'], nsx_sg_id)
+                context.session, sg_id, nsx_sg_id)
             # Add database associations for fw section and rules
             nsxv_db.add_neutron_nsx_section_mapping(
-                context.session, new_security_group['id'], section_uri)
+                context.session, sg_id, section_uri, logging)
             for pair in rule_pairs:
                 # Save nsx rule id in the DB for future access
                 nsxv_db.add_neutron_nsx_rule_mapping(context.session,
@@ -1979,31 +2037,46 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                 # Only admin can delete the default security-group
                 if default_sg:
                     context = context.elevated()
-                super(NsxVPluginV2, self).delete_security_group(
-                    context, new_security_group['id'])
+                super(NsxVPluginV2, self).delete_security_group(context, sg_id)
                 # Delete the created nsx security-group and the fw section
                 self._delete_section(section_uri)
                 self._delete_nsx_security_group(nsx_sg_id)
                 LOG.exception(_LE('Failed to create security group'))
+        if context.is_admin:
+            new_security_group['logging'] = logging
         return new_security_group
 
     def update_security_group(self, context, id, security_group):
         s = security_group['security_group']
         nsx_sg_id = nsx_db.get_nsx_security_group_id(context.session, id)
-        section_uri = self._get_section_uri(context.session, id)
-        h, c = self.nsx_v.vcns.get_section(section_uri)
-        section = self.nsx_sg_utils.parse_section(c)
+        section_db = self._get_section(context.session, id)
+        section_uri = section_db['ip_section_id']
+        section_needs_update = False
 
         sg_data = super(NsxVPluginV2, self).update_security_group(
             context, id, security_group)
+
+        # Reflect security-group name or description changes in the backend,
+        # dfw section name needs to be updated as well.
+        h, c = self.nsx_v.vcns.get_section(section_uri)
+        section = self.nsx_sg_utils.parse_section(c)
         if set(['name', 'description']) & set(s.keys()):
             nsx_sg_name = self.nsx_sg_utils.get_nsx_sg_name(sg_data)
+            section_name = self.nsx_sg_utils.get_nsx_section_name(nsx_sg_name)
             self.nsx_v.vcns.update_security_group(
                 nsx_sg_id, nsx_sg_name, sg_data['description'])
-            section_name = self.nsx_sg_utils.get_nsx_section_name(nsx_sg_name)
             section.attrib['name'] = section_name
+            section_needs_update = True
+        # Update the dfw section if security-group logging option has changed.
+        # TBD: enforce admin only?
+        if 'logging' in s and s['logging'] != section_db['logging']:
+            section_needs_update = self._update_section_logging(
+                context.session, section, section_db)
+        if section_needs_update:
             self.nsx_v.vcns.update_section(
                 section_uri, self.nsx_sg_utils.to_xml_string(section), h)
+        if context.is_admin:
+            sg_data['logging'] = section_db['logging']
         return sg_data
 
     def delete_security_group(self, context, id):
@@ -2028,7 +2101,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
             with excutils.save_and_reraise_exception():
                 LOG.exception(_LE("Failed to delete security group"))
 
-    def _create_nsx_rule(self, context, rule, nsx_sg_id=None):
+    def _create_nsx_rule(self, context, rule, nsx_sg_id=None, logged=False):
         src = None
         dest = None
         port = None
@@ -2089,7 +2162,8 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
             source=src,
             destination=dest,
             services=services,
-            flags=flags)
+            flags=flags,
+            logged=logged)
         return nsx_rule
 
     def create_security_group_rule(self, context, security_group_rule):
@@ -2103,10 +2177,18 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
         :param security_group_rules: list of rules to create
         """
         sg_rules = security_group_rules['security_group_rules']
+        sg_id = sg_rules[0]['security_group_rule']['security_group_id']
         ruleids = set()
         nsx_rules = []
 
         self._validate_security_group_rules(context, security_group_rules)
+
+        # Fetching the the dfw section associated with the security-group
+        section_db = self._get_section(context.session, sg_id)
+        section_uri = section_db['ip_section_id']
+        logging = section_db['logging']
+        log_all_rules = cfg.CONF.nsxv.log_security_groups_allowed_traffic
+
         # Translating Neutron rules to Nsx DFW rules
         for r in sg_rules:
             rule = r['security_group_rule']
@@ -2114,12 +2196,9 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                 rule[ext_loip.LOCAL_IP_PREFIX] = None
             rule['id'] = uuidutils.generate_uuid()
             ruleids.add(rule['id'])
-            nsx_rules.append(self._create_nsx_rule(context, rule))
+            nsx_rules.append(self._create_nsx_rule(
+                context, rule, logged=log_all_rules or logging))
 
-        # Find section uri for the security group, retrieve it and update with
-        # the new rules
-        section_uri = self._get_section_uri(
-            context.session, rule['security_group_id'])
         _h, _c = self.nsx_v.vcns.get_section(section_uri)
         section = self.nsx_sg_utils.parse_section(_c)
         self.nsx_sg_utils.extend_section_with_rules(section, nsx_rules)

vmware_nsx/plugins/nsx_v/vshield/securitygroup_utils.py

@@ -58,9 +58,10 @@ class NsxSecurityGroupUtils(object):
     def get_rule_config(self, applied_to_ids, name, action='allow',
                         applied_to='SecurityGroup',
                         source=None, destination=None, services=None,
-                        flags=None):
+                        flags=None, logged=False):
         """Helper method to create a nsx rule dict."""
         ruleTag = et.Element('rule')
+        ruleTag.attrib['logged'] = 'true' if logged else 'false'
         nameTag = et.SubElement(ruleTag, 'name')
         nameTag.text = name
         actionTag = et.SubElement(ruleTag, 'action')
@@ -146,3 +147,21 @@ class NsxSecurityGroupUtils(object):
     def parse_and_get_section_id(self, section_xml):
         section = et.fromstring(section_xml)
         return section.attrib['id']
+
+    def is_section_logged(self, section):
+        # Determine if this section rules are being logged by the first rule
+        # 'logged' value.
+        rule = section.find('rule')
+        if rule is not None:
+            return rule.attrib.get('logged') == 'true'
+        return False
+
+    def set_rules_logged_option(self, section, logged):
+        value = 'true' if logged else 'false'
+        rules = section.findall('rule')
+        updated = False
+        for rule in rules:
+            if rule.attrib['logged'] != value:
+                rule.attrib['logged'] = value
+                updated = True
+        return updated

vmware_nsx/tests/unit/extensions/test_secgroup_rule_local_ip_prefix.py

@@ -120,5 +120,6 @@ class TestNsxVExtendedSGRule(test_nsxv_plugin.NsxVSecurityGroupsTestCase,
         super(TestNsxVExtendedSGRule,
               self).test_create_rule_with_local_ip_prefix()
         plugin.nsx_sg_utils.get_rule_config.assert_called_with(
-            destination=dest, applied_to_ids=mock.ANY, name=mock.ANY,
-            services=mock.ANY, source=mock.ANY, flags=mock.ANY)
+            source=mock.ANY, destination=dest, services=mock.ANY,
+            name=mock.ANY, applied_to_ids=mock.ANY, flags=mock.ANY,
+            logged=mock.ANY)

vmware_nsx/tests/unit/nsx_v/test_plugin.py

@@ -50,6 +50,7 @@ from vmware_nsx.common import nsx_constants
 from vmware_nsx.db import nsxv_db
 from vmware_nsx.extensions import routersize as router_size
 from vmware_nsx.extensions import routertype as router_type
+from vmware_nsx.extensions import securitygrouplogging
 from vmware_nsx.extensions import vnicindex as ext_vnic_idx
 from vmware_nsx.plugins.nsx_v.vshield.common import constants as vcns_const
 from vmware_nsx.plugins.nsx_v.vshield import edge_utils
@@ -2368,6 +2369,8 @@ class NsxVSecurityGroupsTestCase(ext_sg.SecurityGroupDBTestCase):
               ext_mgr=None,
               service_plugins=None):
         test_utils.override_nsx_ini_test()
+        attributes.RESOURCE_ATTRIBUTE_MAP.update(
+            securitygrouplogging.RESOURCE_ATTRIBUTE_MAP)
         mock_vcns = mock.patch(vmware.VCNS_NAME, autospec=True)
         mock_vcns_instance = mock_vcns.start()
         self.fc2 = fake_vcns.FakeVcns()
@@ -2510,6 +2513,43 @@ class NsxVTestSecurityGroup(ext_sg.TestSecurityGroups,
         # This test is aimed to test the security-group db mixin
         pass
 
+    def _plugin_update_security_group(self, context, id, logging):
+        data = {'security_group': {'logging': logging}}
+        security_group = (
+            self.plugin.update_security_group(context, id, data))
+        return security_group
+
+    def _plugin_create_security_group(self, context, logging=False):
+        data = {'security_group': {'name': 'SG',
+                                   'tenant_id': 'tenant_id',
+                                   'description': ''}}
+        if logging:
+            data['security_group']['logging'] = True
+        security_group = (
+            self.plugin.create_security_group(context, data, False))
+        return security_group
+
+    def test_create_security_group_default_logging(self):
+        _context = context.get_admin_context()
+        sg = self._plugin_create_security_group(_context)
+        self.assertFalse(sg['logging'])
+
+    def test_create_security_group_with_logging(self):
+        _context = context.get_admin_context()
+        sg = self._plugin_create_security_group(_context, logging=True)
+        self.assertTrue(sg['logging'])
+
+    def test_update_security_group_with_logging(self):
+        _context = context.get_admin_context()
+        sg = self._plugin_create_security_group(_context)
+        sg = self._plugin_update_security_group(_context, sg['id'], True)
+        self.assertTrue(sg['logging'])
+
+    def test_security_group_logging_not_visible_for_user(self):
+        _context = context.Context('user', 'tenant_id')
+        sg = self._plugin_create_security_group(_context)
+        self.assertFalse('logging' in sg)
+
 
 class TestVdrTestCase(L3NatTest, L3NatTestCaseBase,
                       test_l3_plugin.L3NatDBIntTestCase,