From b8f030feaf5ab1bf074f1c4d4ee563aad0d4920b Mon Sep 17 00:00:00 2001 From: Tim Simmons Date: Wed, 25 Mar 2015 21:38:49 +0000 Subject: [PATCH] Reduce the # of SQL queries during AXFRs - Adds a new SQL base function _select_raw that does no object serialization, or much of anything else - Adds a storage method to get all of the record data necessary for an AXFR in one query - Add a function in MiniDNS that handles the data from the above, and prepares it in DNSPython format for AXFRs Change-Id: Iba2d26a5d442443ced932b18bd1091f8a17d9b47 Partial-Bug: 1434479 Closes-Bug: 1435888 --- designate/mdns/handler.py | 43 ++++++++++++++++--- designate/sqlalchemy/base.py | 17 ++++++++ designate/storage/base.py | 9 ++++ designate/storage/impl_sqlalchemy/__init__.py | 22 ++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/designate/mdns/handler.py b/designate/mdns/handler.py index c88c3dd02..8d8284b05 100644 --- a/designate/mdns/handler.py +++ b/designate/mdns/handler.py @@ -207,6 +207,41 @@ class RequestHandler(xfr.XFRMixin): return r_rrset + def _prep_rrsets(self, raw_records, domain_ttl): + rrsets = [] + rrset_id = None + current_rrset = None + + for record in raw_records: + # If we're looking at the first, or a new rrset + if record[0] != rrset_id: + if current_rrset is not None: + # If this isn't the first iteration + rrsets.append(current_rrset) + # Set up a new rrset + rrset_id = record[0] + rrtype = str(record[1]) + # gross + ttl = int(record[2]) if record[2] is not None else domain_ttl + name = str(record[3]) + rdata = str(record[4]) + current_rrset = dns.rrset.from_text_list( + name, ttl, dns.rdataclass.IN, rrtype, [rdata]) + else: + # We've already got an rrset, add the rdata + rrtype = str(record[1]) + rdata = str(record[4]) + rd = dns.rdata.from_text(dns.rdataclass.IN, + dns.rdatatype.from_text(rrtype), rdata) + current_rrset.add(rd) + + # If the last record examined was a new rrset, or there is only 1 rrset + if rrsets == [] or (rrsets != [] and rrsets[-1] != current_rrset): + if current_rrset is not None: + rrsets.append(current_rrset) + + return rrsets + def _handle_axfr(self, request): context = request.environ['context'] @@ -243,12 +278,10 @@ class RequestHandler(xfr.XFRMixin): # Get all the recordsets other than SOA criterion = {'domain_id': domain.id, 'type': '!SOA'} - recordsets = self.storage.find_recordsets(context, criterion) - for recordset in recordsets: - r_rrset = self._convert_to_rrset(domain, recordset) - if r_rrset: - r_rrsets.append(r_rrset) + # Get the raw record data out of storage and parse it + raw_records = self.storage.find_recordsets_axfr(context, criterion) + r_rrsets.extend(self._prep_rrsets(raw_records, domain.ttl)) # Append the SOA recordset at the end for recordset in soa_recordsets: diff --git a/designate/sqlalchemy/base.py b/designate/sqlalchemy/base.py index 893cb366e..90ea54972 100644 --- a/designate/sqlalchemy/base.py +++ b/designate/sqlalchemy/base.py @@ -328,3 +328,20 @@ class SQLAlchemy(object): resultproxy = self.session.execute(query) return _set_object_from_model(obj, resultproxy.fetchone()) + + def _select_raw(self, context, table, criterion, query=None): + # Build the query + if query is None: + query = select([table]) + + query = self._apply_criterion(table, query, criterion) + query = self._apply_deleted_criteria(context, table, query) + + try: + resultproxy = self.session.execute(query) + return resultproxy.fetchall() + # Any ValueErrors are propagated back to the user as is. + # If however central or storage is called directly, invalid values + # show up as ValueError + except ValueError as value_error: + raise exceptions.ValueError(value_error.message) diff --git a/designate/storage/base.py b/designate/storage/base.py index 8312bdda9..4d6636be9 100644 --- a/designate/storage/base.py +++ b/designate/storage/base.py @@ -331,6 +331,15 @@ class Storage(DriverPlugin): :param sort_dir: Direction to sort after using sort_key. """ + @abc.abstractmethod + def find_recordsets_axfr(self, context, criterion=None): + """ + Find RecordSets. + + :param context: RPC Context. + :param criterion: Criteria to filter by. + """ + @abc.abstractmethod def find_recordset(self, context, criterion): """ diff --git a/designate/storage/impl_sqlalchemy/__init__.py b/designate/storage/impl_sqlalchemy/__init__.py index 89a5884f1..cd505aeda 100644 --- a/designate/storage/impl_sqlalchemy/__init__.py +++ b/designate/storage/impl_sqlalchemy/__init__.py @@ -474,6 +474,28 @@ class SQLAlchemyStorage(sqlalchemy_base.SQLAlchemy, storage_base.Storage): return recordsets + def find_recordsets_axfr(self, context, criterion=None): + query = None + + # Check to see if the criterion can use the reverse_name column + criterion = self._rname_check(criterion) + + rjoin = tables.records.join( + tables.recordsets, + tables.records.c.recordset_id == tables.recordsets.c.id) + + query = select([tables.recordsets.c.id, tables.recordsets.c.type, + tables.recordsets.c.ttl, tables.recordsets.c.name, + tables.records.c.data, tables.records.c.action]).\ + select_from(rjoin).where(tables.records.c.action != 'DELETE') + + query = query.order_by(tables.recordsets.c.id) + + raw_rows = self._select_raw( + context, tables.recordsets, criterion, query) + + return raw_rows + def create_recordset(self, context, domain_id, recordset): # Fetch the domain as we need the tenant_id domain = self._find_domains(context, {'id': domain_id}, one=True)