From 41e861987685fcedb8ab5186e9cbcaad8884f1e9 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 4 Jan 2020 16:21:43 +0000 Subject: [PATCH] accounts: Implement page searches for LDAP Since we have so many accounts, we can no longer retrieve them in one single search operation without disabling all safety measures that OpenLDAP provides. Hence we perform all search now as paged search retrieving up to 512 results at a time. Signed-off-by: Michael Tremer --- src/backend/accounts.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/backend/accounts.py b/src/backend/accounts.py index ee7fca08..b1160ac2 100644 --- a/src/backend/accounts.py +++ b/src/backend/accounts.py @@ -220,16 +220,47 @@ class Accounts(Object): t = time.time() - results = self.ldap.search_ext_s(search_base or self.search_base, - ldap.SCOPE_SUBTREE, query, attrlist=attrlist, sizelimit=limit) + # Ask for up to 512 results being returned at a time + page_control = ldap.controls.SimplePagedResultsControl(True, size=512, cookie="") + + results = [] + pages = 0 + + # Perform the search + while True: + response = self.ldap.search_ext(search_base or self.search_base, + ldap.SCOPE_SUBTREE, query, attrlist=attrlist, sizelimit=limit, + serverctrls=[page_control], + ) + + # Fetch all results + type, data, rmsgid, serverctrls = self.ldap.result3(response) + + # Append to local copy + results += data + pages += 1 + + controls = [c for c in serverctrls + if c.controlType == ldap.controls.SimplePagedResultsControl.controlType] + + if not controls: + logging.warning("The server ignores RFC 2696 control") + break + + # Set the cookie for more results + page_control.cookie = controls[0].cookie + + # There are no more results + if not page_control.cookie: + break # Log time it took to perform the query - logging.debug("Query took %.2fms" % ((time.time() - t) * 1000.0)) + logging.debug("Query took %.2fms (%s page(s))" % ((time.time() - t) * 1000.0, pages)) return results def _count(self, query): - res = self._query(query, attrlist=["dn"], limit=INT_MAX) + res = self._query(query, attrlist=["dn"]) return len(res) -- 2.47.3