Merge "Reduce the # of SQL queries during AXFRs"

designate/mdns/handler.py

@@ -207,6 +207,41 @@ class RequestHandler(xfr.XFRMixin):
 
         return r_rrset
 
+    def _prep_rrsets(self, raw_records, domain_ttl):
+        rrsets = []
+        rrset_id = None
+        current_rrset = None
+
+        for record in raw_records:
+            # If we're looking at the first, or a new rrset
+            if record[0] != rrset_id:
+                if current_rrset is not None:
+                    # If this isn't the first iteration
+                    rrsets.append(current_rrset)
+                # Set up a new rrset
+                rrset_id = record[0]
+                rrtype = str(record[1])
+                # gross
+                ttl = int(record[2]) if record[2] is not None else domain_ttl
+                name = str(record[3])
+                rdata = str(record[4])
+                current_rrset = dns.rrset.from_text_list(
+                    name, ttl, dns.rdataclass.IN, rrtype, [rdata])
+            else:
+                # We've already got an rrset, add the rdata
+                rrtype = str(record[1])
+                rdata = str(record[4])
+                rd = dns.rdata.from_text(dns.rdataclass.IN,
+                    dns.rdatatype.from_text(rrtype), rdata)
+                current_rrset.add(rd)
+
+        # If the last record examined was a new rrset, or there is only 1 rrset
+        if rrsets == [] or (rrsets != [] and rrsets[-1] != current_rrset):
+            if current_rrset is not None:
+                rrsets.append(current_rrset)
+
+        return rrsets
+
     def _handle_axfr(self, request):
         context = request.environ['context']
 
@@ -243,12 +278,10 @@ class RequestHandler(xfr.XFRMixin):
 
         # Get all the recordsets other than SOA
         criterion = {'domain_id': domain.id, 'type': '!SOA'}
-        recordsets = self.storage.find_recordsets(context, criterion)
 
-        for recordset in recordsets:
+        # Get the raw record data out of storage and parse it
-            r_rrset = self._convert_to_rrset(domain, recordset)
+        raw_records = self.storage.find_recordsets_axfr(context, criterion)
-            if r_rrset:
+        r_rrsets.extend(self._prep_rrsets(raw_records, domain.ttl))
-                r_rrsets.append(r_rrset)
 
         # Append the SOA recordset at the end
         for recordset in soa_recordsets:

designate/sqlalchemy/base.py

@@ -328,3 +328,20 @@ class SQLAlchemy(object):
         resultproxy = self.session.execute(query)
 
         return _set_object_from_model(obj, resultproxy.fetchone())
+
+    def _select_raw(self, context, table, criterion, query=None):
+        # Build the query
+        if query is None:
+            query = select([table])
+
+        query = self._apply_criterion(table, query, criterion)
+        query = self._apply_deleted_criteria(context, table, query)
+
+        try:
+            resultproxy = self.session.execute(query)
+            return resultproxy.fetchall()
+        # Any ValueErrors are propagated back to the user as is.
+        # If however central or storage is called directly, invalid values
+        # show up as ValueError
+        except ValueError as value_error:
+            raise exceptions.ValueError(value_error.message)

designate/storage/base.py

@@ -331,6 +331,15 @@ class Storage(DriverPlugin):
         :param sort_dir: Direction to sort after using sort_key.
         """
 
+    @abc.abstractmethod
+    def find_recordsets_axfr(self, context, criterion=None):
+        """
+        Find RecordSets.
+
+        :param context: RPC Context.
+        :param criterion: Criteria to filter by.
+        """
+
     @abc.abstractmethod
     def find_recordset(self, context, criterion):
         """

designate/storage/impl_sqlalchemy/__init__.py

@@ -474,6 +474,28 @@ class SQLAlchemyStorage(sqlalchemy_base.SQLAlchemy, storage_base.Storage):
 
         return recordsets
 
+    def find_recordsets_axfr(self, context, criterion=None):
+        query = None
+
+        # Check to see if the criterion can use the reverse_name column
+        criterion = self._rname_check(criterion)
+
+        rjoin = tables.records.join(
+            tables.recordsets,
+            tables.records.c.recordset_id == tables.recordsets.c.id)
+
+        query = select([tables.recordsets.c.id, tables.recordsets.c.type,
+                        tables.recordsets.c.ttl, tables.recordsets.c.name,
+                        tables.records.c.data, tables.records.c.action]).\
+            select_from(rjoin).where(tables.records.c.action != 'DELETE')
+
+        query = query.order_by(tables.recordsets.c.id)
+
+        raw_rows = self._select_raw(
+            context, tables.recordsets, criterion, query)
+
+        return raw_rows
+
     def create_recordset(self, context, domain_id, recordset):
         # Fetch the domain as we need the tenant_id
         domain = self._find_domains(context, {'id': domain_id}, one=True)