[Admin-Util] NSXv: fix plugin issues

1. Since the nsxv plugin was defined globally in one of the utilities files,
it was actually initialized for all the nsxv utilities, which has some side
affects (like deploying backup edges during unrelated admin utilities in case
some of the pools are not full), and also takes a long time.
Now the plugin is initialized only when needed.

2. When the plugin is initialized during an admin-util call,
we mustn't exit while spawn jobs for creating edges are still running,
or else - those edges will be in PENDING-CREATE state for ever.
Initializing the plugin using "with" and adding an exit method which waits for
the jobs to finish solved this issue.

Change-Id: Ia1fa557a8da149f79a46b3ca49b122f991b2ca9b

vmware_nsx/shell/admin/plugins/nsxv/resources/dhcp_binding.py

@@ -118,7 +118,7 @@ def nsx_update_dhcp_edge_binding(resource, event, trigger, **kwargs):
         LOG.info(_LI("Updating NSXv Edge: %s"), edge_id)
         # Need to create a plugin object; so that we are able to
         # do neutron list-ports.
-        plugin = utils.NsxVPluginWrapper()
+        with utils.NsxVPluginWrapper() as plugin:
             nsxv_manager = vcns_driver.VcnsDriver(
                                edge_utils.NsxVCallbacks(plugin))
             edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
@@ -260,13 +260,6 @@ def nsx_recreate_dhcp_edge(resource, event, trigger, **kwargs):
         return
     LOG.info(_LI("ReCreating NSXv Edge: %s"), old_edge_id)
 
-    # init the plugin and edge manager
-    cfg.CONF.set_override('core_plugin',
-                          'vmware_nsx.shell.admin.plugins.nsxv.resources'
-                          '.utils.NsxVPluginWrapper')
-    plugin = utils.NsxVPluginWrapper()
-    nsxv_manager = vcns_driver.VcnsDriver(edge_utils.NsxVCallbacks(plugin))
-    edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
     context = n_context.get_admin_context()
 
     # verify that this is a DHCP edge
@@ -279,6 +272,15 @@ def nsx_recreate_dhcp_edge(resource, event, trigger, **kwargs):
                  {'edge_id': old_edge_id})
         return
 
+    # init the plugin and edge manager
+    cfg.CONF.set_override('core_plugin',
+                          'vmware_nsx.shell.admin.plugins.nsxv.resources'
+                          '.utils.NsxVPluginWrapper')
+    with utils.NsxVPluginWrapper() as plugin:
+        nsxv_manager = vcns_driver.VcnsDriver(
+            edge_utils.NsxVCallbacks(plugin))
+        edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
+
         # find the networks bound to this DHCP edge
         networks_binding = nsxv_db.get_edge_vnic_bindings_by_edge(
             context.session, old_edge_id)
@@ -322,13 +324,6 @@ def nsx_recreate_dhcp_edge_by_net_id(net_id):
     """Recreate a dhcp edge for a specific network without an edge"""
     LOG.info(_LI("ReCreating NSXv Edge for network: %s"), net_id)
 
-    # init the plugin and edge manager
-    cfg.CONF.set_override('core_plugin',
-                          'vmware_nsx.shell.admin.plugins.nsxv.resources'
-                          '.utils.NsxVPluginWrapper')
-    plugin = utils.NsxVPluginWrapper()
-    nsxv_manager = vcns_driver.VcnsDriver(edge_utils.NsxVCallbacks(plugin))
-    edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
     context = n_context.get_admin_context()
 
     # verify that there is no DHCP edge for this network at the moment
@@ -346,6 +341,14 @@ def nsx_recreate_dhcp_edge_by_net_id(net_id):
         # delete this old entry
         nsxv_db.delete_nsxv_router_binding(context.session, resource_id)
 
+    # init the plugin and edge manager
+    cfg.CONF.set_override('core_plugin',
+                          'vmware_nsx.shell.admin.plugins.nsxv.resources'
+                          '.utils.NsxVPluginWrapper')
+    with utils.NsxVPluginWrapper() as plugin:
+        nsxv_manager = vcns_driver.VcnsDriver(edge_utils.NsxVCallbacks(plugin))
+        edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
+
         # check if this network is attached to a distributed router
         vdr_router_id = _get_net_vdr_router_id(plugin, context, net_id)
         if vdr_router_id:

vmware_nsx/shell/admin/plugins/nsxv/resources/routers.py

@@ -76,8 +76,9 @@ def nsx_recreate_router_edge(resource, event, trigger, **kwargs):
     cfg.CONF.set_override('core_plugin',
                           'vmware_nsx.shell.admin.plugins.nsxv.resources'
                           '.utils.NsxVPluginWrapper')
-    plugin = utils.NsxVPluginWrapper()
+    with utils.NsxVPluginWrapper() as plugin:
-    nsxv_manager = vcns_driver.VcnsDriver(edge_utils.NsxVCallbacks(plugin))
+        nsxv_manager = vcns_driver.VcnsDriver(
+            edge_utils.NsxVCallbacks(plugin))
         edge_manager = edge_utils.EdgeManager(nsxv_manager, plugin)
         context = n_context.get_admin_context()
 
@@ -120,7 +121,8 @@ def nsx_recreate_router_edge(resource, event, trigger, **kwargs):
                                         {'router': router},
                                         appliance_size=appliance_size)
             # find out who is the new edge to print it
-        new_edge_id = router_driver._get_edge_id_or_raise(context, router_id)
+            new_edge_id = router_driver._get_edge_id_or_raise(
+                context, router_id)
             LOG.info(_LI("Router %(router)s was attached to edge %(edge)s"),
                      {'router': router_id, 'edge': new_edge_id})
 

vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py

@@ -86,6 +86,7 @@ class NeutronSecurityGroupDB(utils.NeutronDbClient,
                 self.context.session.delete(sg_mapping)
 
     def get_vnics_in_security_group(self, security_group_id):
+        with utils.NsxVPluginWrapper() as plugin:
             vnics = []
             query = self.context.session.query(
                 models_v2.Port.id, models_v2.Port.device_id
@@ -104,6 +105,8 @@ class NsxFirewallAPI(object):
 
     def list_security_groups(self):
         h, secgroups = self.vcns.list_security_groups()
+        if not secgroups:
+            return []
         root = et.fromstring(secgroups)
         secgroups = []
         for sg in root.iter('securitygroup'):
@@ -117,6 +120,8 @@ class NsxFirewallAPI(object):
 
     def list_fw_sections(self):
         h, firewall_config = self.vcns.get_dfw_config()
+        if not firewall_config:
+            return []
         root = et.fromstring(firewall_config)
         sections = []
         for sec in root.iter('section'):
@@ -131,6 +136,10 @@ class NsxFirewallAPI(object):
     def reorder_fw_sections(self):
         # read all the sections
         h, firewall_config = self.vcns.get_dfw_config()
+        if not firewall_config:
+            LOG.info(_LI("No firewall sections were found."))
+            return
+
         root = et.fromstring(firewall_config)
 
         for child in root:
@@ -163,7 +172,6 @@ class NsxFirewallAPI(object):
 
 neutron_sg = NeutronSecurityGroupDB()
 nsxv_firewall = NsxFirewallAPI()
-plugin = utils.NsxVPluginWrapper()
 
 
 def _log_info(resource, data, attrs=['name', 'id']):
@@ -263,7 +271,11 @@ def fix_security_groups(resource, event, trigger, **kwargs):
     context_ = context.get_admin_context()
     sgs_with_missing_section = _find_missing_sections()
     sgs_with_missing_nsx_group = _find_missing_security_groups()
-    plugin = utils.NsxVPluginWrapper()
+    if not sgs_with_missing_section and not sgs_with_missing_nsx_group:
+        # no mismatches
+        return
+
+    with utils.NsxVPluginWrapper() as plugin:
         # If only the fw section is missing then create it.
         for sg_id in (set(sgs_with_missing_section.keys()) -
                       set(sgs_with_missing_nsx_group.keys())):
@@ -273,8 +285,8 @@ def fix_security_groups(resource, event, trigger, **kwargs):
                 context_, secgroup,
                 sgs_with_missing_section[sg_id]['nsx-securitygroup-id'])
 
-    # If nsx security-group is missing then create both nsx security-group and
+        # If nsx security-group is missing then create both nsx security-group
-    # a new fw section (remove old one).
+        # and a new fw section (remove old one).
         for sg_id, sg in sgs_with_missing_nsx_group.items():
             secgroup = plugin.get_security_group(context_, sg_id)
             if sg_id not in sgs_with_missing_section:
@@ -309,7 +321,7 @@ def migrate_sg_to_policy(resource, event, trigger, **kwargs):
 
     # validate that the security group exist and contains rules and no policy
     context_ = context.get_admin_context()
-    plugin = utils.NsxVPluginWrapper()
+    with utils.NsxVPluginWrapper() as plugin:
         try:
             secgroup = plugin.get_security_group(context_, sg_id)
         except ext_sg.SecurityGroupNotFound:
@@ -330,8 +342,8 @@ def migrate_sg_to_policy(resource, event, trigger, **kwargs):
             try:
                 plugin.delete_security_group_rule(context_, rule['id'])
             except Exception as e:
-            LOG.warning(_LW("Failed to delete rule %(r)s from security group "
+                LOG.warning(_LW("Failed to delete rule %(r)s from security "
-                          "%(sg)s: %(e)s"),
+                                "group %(sg)s: %(e)s"),
                             {'r': rule['id'], 'sg': sg_id, 'e': e})
                 # continue anyway
 
@@ -340,16 +352,18 @@ def migrate_sg_to_policy(resource, event, trigger, **kwargs):
         try:
             section_uri = plugin._get_section_uri(context_.session, sg_id)
             plugin._delete_section(section_uri)
-        nsxv_db.delete_neutron_nsx_section_mapping(context_.session, sg_id)
+            nsxv_db.delete_neutron_nsx_section_mapping(
+                context_.session, sg_id)
         except Exception as e:
-        LOG.warning(_LW("Failed to delete firewall section of security group "
+            LOG.warning(_LW("Failed to delete firewall section of security "
-                      "%(sg)s: %(e)s"),
+                            "group %(sg)s: %(e)s"),
                         {'sg': sg_id, 'e': e})
             # continue anyway
 
         # bind this security group to the policy in the backend and DB
         nsx_sg_id = nsx_db.get_nsx_security_group_id(context_.session, sg_id)
-    LOG.info(_LI("Binding the NSX security group %(nsx)s to policy %(pol)s"),
+        LOG.info(_LI("Binding the NSX security group %(nsx)s to policy "
+                     "%(pol)s"),
                  {'nsx': nsx_sg_id, 'pol': policy_id})
         plugin._update_nsx_security_group_policies(
             policy_id, None, nsx_sg_id)

vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py

@@ -12,16 +12,21 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import time
 
 from oslo_config import cfg
+from oslo_log import log as logging
 
 from neutron import context as neutron_context
 from neutron.db import common_db_mixin as common_db
 
+from vmware_nsx._i18n import _LW
 from vmware_nsx.common import config
 from vmware_nsx import plugin
 from vmware_nsx.plugins.nsx_v.vshield import vcns
 
+LOG = logging.getLogger(__name__)
+
 
 def get_nsxv_client():
     return vcns.Vcns(
@@ -55,6 +60,43 @@ class NsxVPluginWrapper(plugin.NsxVPlugin):
         # skip getting the Qos policy ID because get_object calls
         # plugin init again on admin-util environment
 
+    def count_spawn_jobs(self):
+        # check if there are any spawn jobs running
+        return self.edge_manager._get_worker_pool().running()
+
+    # Define enter & exit to be used in with statements
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        """Wait until no more jobs are pending
+
+        We want to wait until all spawn edge creation are done, or else the
+        edges might be in PERNDING_CREATE state in the nsx DB
+        """
+        if not self.count_spawn_jobs():
+            return
+
+        LOG.warning(_LW("Waiting for plugin jobs to finish properly..."))
+        sleep_time = 1
+        print_time = 20
+        max_loop = 600
+        for print_index in range(1, max_loop):
+            n_jobs = self.count_spawn_jobs()
+            if n_jobs > 0:
+                if (print_index % print_time) == 0:
+                    LOG.warning(_LW("Still Waiting on %(jobs)s "
+                                    "job%(plural)s"),
+                                {'jobs': n_jobs,
+                                 'plural': 's' if n_jobs > 1 else ''})
+                time.sleep(sleep_time)
+            else:
+                LOG.warning(_LW("Done."))
+                return
+
+        LOG.warning(_LW("Sorry. Waited for too long. Some jobs are still "
+                        "running."))
+
 
 def get_nsxv_backend_edges():
     """Get a list of all the backend edges and some of their attributes

vmware_nsx/tests/unit/shell/test_admin_utils.py

@@ -118,7 +118,7 @@ class TestNsxvAdminUtils(AbstractTestAdminUtils,
     # This is an example how to test a specific utility with arguments
     def test_with_args(self):
         args = {'property': ["xxx=yyy"]}
-        self._test_resource('networks', 'list', **args)
+        self._test_resource('security-groups', 'fix-mismatch', **args)
 
     def test_bad_args(self):
         args = {'property': ["xxx"]}