From: Ondřej Kuzník Date: Thu, 13 May 2021 09:48:04 +0000 (+0100) Subject: ITS#9596 First take on Python test suite X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=40ef1b963c7ca595d4356bf37b82f44bcf00e4bd;p=thirdparty%2Fopenldap.git ITS#9596 First take on Python test suite --- diff --git a/tests/python/.gitignore b/tests/python/.gitignore new file mode 100644 index 0000000000..8d35cb3277 --- /dev/null +++ b/tests/python/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*.pyc diff --git a/tests/python/__init__.py b/tests/python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/python/backends.py b/tests/python/backends.py new file mode 100755 index 0000000000..b3cabfa95c --- /dev/null +++ b/tests/python/backends.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# This work is part of OpenLDAP Software . +# +# Copyright 2021 The OpenLDAP Foundation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted only as authorized by the OpenLDAP +# Public License. +# +# A copy of this license is available in the file LICENSE in the +# top-level directory of the distribution or, alternatively, at +# . +# +# ACKNOWLEDGEMENTS: +# This work was initially developed by Ondřej Kuzník +# for inclusion in OpenLDAP Software. +""" +OpenLDAP fixtures for backends +""" + +import ldap0 +import logging +import os +import pathlib +import pytest +import secrets +import tempfile + +from ldap0.controls.readentry import PostReadControl + +from .slapd import server + + +SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute() +BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute() + + +logger = logging.getLogger(__name__) + + +class Database: + have_directory = True + + def __init__(self, server, suffix, backend): + self.server = server + self.suffix = suffix + self.rootdn = suffix + self.secret = secrets.token_urlsafe() + self.overlays = [] + + if suffix in server.suffixes: + raise RuntimeError(f"Suffix {suffix} already configured in server") + + if self.have_directory: + self.directory = tempfile.TemporaryDirectory(dir=server.home) + + conn = server.connect() + conn.simple_bind_s("cn=config", server.secret) + + # We're just after the generated DN, no other attributes at the moment + control = PostReadControl(True, []) + + result = conn.add_s( + f"olcDatabase={backend},cn=config", self._entry(), + req_ctrls=[control]) + dn = result.ctrls[0].res.dn_s + + self.dn = dn + server.suffixes[suffix] = self + + def _entry(self): + entry = { + "objectclass": [self.objectclass.encode()], + "olcSuffix": [self.suffix.encode()], + "olcRootDN": [self.suffix.encode()], + "olcRootPW": [self.secret.encode()], + } + if self.have_directory: + entry["olcDbDirectory"] = [self.directory.name.encode()] + return entry + + +class MDB(Database): + have_directory = True + objectclass = "olcMdbConfig" + + _size = 10 * (1024 ** 3) + + def __init__(self, server, suffix): + super().__init__(server, suffix, "mdb") + + def _entry(self): + entry = { + "olcDbMaxSize": [str(self._size).encode()], + } + return {**super()._entry(), **entry} + + +class LDAP(Database): + have_directory = False + objectclass = "olcLDAPConfig" + + def __init__(self, server, suffix, uris): + self.uris = uris + super().__init__(server, suffix, "ldap") + + def _entry(self): + entry = { + "olcDbURI": [" ".join(self.uris).encode()], + } + return {**super()._entry(), **entry} + + +backend_types = { + "mdb": MDB, + "ldap": LDAP, +} + + +@pytest.fixture(scope="class") +def db(request, server): + marker = request.node.get_closest_marker("db") + database_type = marker.args[0] if marker else "mdb" + klass = backend_types[database_type] + + conn = server.connect() + conn.simple_bind_s("cn=config", server.secret) + + db = klass(server, "cn=test") + yield db + + conn.delete_s(db.dn) + + +class TestDB: + def test_db_setup(self, db): + pass diff --git a/tests/python/conftest.py b/tests/python/conftest.py new file mode 100755 index 0000000000..988f21f296 --- /dev/null +++ b/tests/python/conftest.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# This work is part of OpenLDAP Software . +# +# Copyright 2021 The OpenLDAP Foundation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted only as authorized by the OpenLDAP +# Public License. +# +# A copy of this license is available in the file LICENSE in the +# top-level directory of the distribution or, alternatively, at +# . +# +# ACKNOWLEDGEMENTS: +# This work was initially developed by Ondřej Kuzník +# for inclusion in OpenLDAP Software. +""" +OpenLDAP test suite fixtures +""" + +import pytest + +from .slapd import temp, server_factory, server +from .backends import db diff --git a/tests/python/overlays.py b/tests/python/overlays.py new file mode 100755 index 0000000000..f12587f9b6 --- /dev/null +++ b/tests/python/overlays.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# This work is part of OpenLDAP Software . +# +# Copyright 2021 The OpenLDAP Foundation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted only as authorized by the OpenLDAP +# Public License. +# +# A copy of this license is available in the file LICENSE in the +# top-level directory of the distribution or, alternatively, at +# . +# +# ACKNOWLEDGEMENTS: +# This work was initially developed by Ondřej Kuzník +# for inclusion in OpenLDAP Software. +""" +OpenLDAP fixtures for overlays +""" + +import logging +import os +import pathlib + +from ldap0.controls.readentry import PostReadControl + + +SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute() +BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute() + + +logger = logging.getLogger(__name__) + + +class Overlay: + def __init__(self, database, overlay, order=-1): + self.database = database + server = database.server + + conn = server.connect() + conn.simple_bind_s("cn=config", server.secret) + + if isinstance(overlay, pathlib.Path): + overlay_name = overlay.stem + else: + overlay_name = overlay + overlay = BUILDROOT/"servers"/"slapd"/"overlays"/overlay_name + + server.load_module(overlay) + + # We're just after the generated DN, no other attributes at the moment + control = PostReadControl(True, []) + + result = conn.add_s( + f"olcOverlay={overlay_name},{database.dn}", self._entry(), + req_ctrls=[control]) + self.dn = result.ctrls[0].res.dn_s + + if order == -1: + database.overlays.append(self) + else: + raise NotImplementedError + database.overlays.insert(order, self) + + def _entry(self): + entry = { + "objectclass": [self.objectclass.encode()], + } + return entry diff --git a/tests/python/slapd.py b/tests/python/slapd.py new file mode 100755 index 0000000000..5446a4d543 --- /dev/null +++ b/tests/python/slapd.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# This work is part of OpenLDAP Software . +# +# Copyright 2021 The OpenLDAP Foundation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted only as authorized by the OpenLDAP +# Public License. +# +# A copy of this license is available in the file LICENSE in the +# top-level directory of the distribution or, alternatively, at +# . +# +# ACKNOWLEDGEMENTS: +# This work was initially developed by Ondřej Kuzník +# for inclusion in OpenLDAP Software. +""" +OpenLDAP server fixtures +""" + +import ldap0 +import ldapurl +import logging +import os +import pathlib +import pytest +import re +import secrets +import signal +import socket +import subprocess +import tempfile +import textwrap + +from ldap0.ldapobject import LDAPObject + + +SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute() +BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute() + + +logger = logging.getLogger(__name__) + + +class Server: + def __init__(self, where, manager, cnconfig=True, schemas=None): + self.path = where + self.home = pathlib.Path(self.path.name) + self.executable = BUILDROOT/'servers'/'slapd'/'slapd' + + self.manager = manager + self.cnconfig = cnconfig + + self.token = secrets.token_urlsafe() + self.secret = None + self.level = "-1" + self.port = 0 + self.pid = None + + if schemas is None: + schemas = ["core", "cosine", "inetorgperson", "openldap", "nis"] + + if cnconfig and not (self.home/'slapd.d').is_dir(): + self.create_config(schemas) + elif not cnconfig and not (self.home/'slapd.conf').is_file(): + self.create_config(schemas) + + self.process = None + self.schema = [] + self.suffixes = {} + + def create_config(self, schemas): + mod_harness = BUILDROOT/"tests"/"modules"/"mod-harness"/"mod_harness" + schemadir = SOURCEROOT/"servers"/"slapd"/"schema" + if not self.secret: + self.secret = secrets.token_urlsafe() + + if self.cnconfig: + confdir = self.home/'slapd.d' + confdir.mkdir() + includes = [] + + config = """ + dn: cn=config + objectClass: olcGlobal + cn: config + + dn: cn=module{{0}},cn=config + objectClass: olcModuleList + olcModuleLoad: {mod_harness} + + dn: cn=schema,cn=config + objectClass: olcSchemaConfig + cn: schema + + dn: olcBackend={{0}}harness,cn=config + objectClass: olcBkHarnessConfig + olcBkHarnessHost: {self.manager.host} + olcBkHarnessPort: {self.manager.port} + olcBkHarnessIdentifier: {self.token} + + dn: olcDatabase={{0}}config,cn=config + objectClass: olcDatabaseConfig + olcRootPW: {self.secret} + """.format(self=self, mod_harness=mod_harness) + + for schema in schemas: + if not isinstance(schema, pathlib.Path): + schema = schemadir / (schema + ".ldif") + includes.append(f"include: file://{schema}") + + config = "\n".join([textwrap.dedent(config), "\n", *includes]) + + args = [self.executable, '-T', 'add', '-d', self.level, + '-n0', '-F', confdir] + args = [str(arg) for arg in args] + subprocess.run(args, capture_output=True, check=True, + cwd=self.home, text=True, input=config) + else: + with open(self.home/'slapd.conf', mode='w') as config: + config.write(textwrap.dedent(""" + moduleload {mod_harness} + + backend harness + host {self.manager.host} + port {self.manager.port} + identifier {self.token} + + database config + rootpw {self.secret} + """.format(self=self, mod_harness=mod_harness))) + + includes = [] + for schema in schemas: + if not isinstance(schema, pathlib.Path): + schema = schemadir / (schema + ".schema") + includes.append(f"include {schema}\n") + + config.write("".join(includes)) + + def test(self): + args = [self.executable, '-T', 'test', '-d', self.level] + if self.cnconfig: + args += ['-F', self.home/'slapd.d'] + else: + args += ['-f', self.home/'slapd.conf'] + + args = [str(arg) for arg in args] + return subprocess.run(args, capture_output=True, check=True, + cwd=self.home) + + def start(self, port=None): + if self.process: + raise RuntimeError("process %d still running" % self.process.pid) + + self.test() + + if port is not None: + self.port = port + + listeners = [ + 'ldapi://socket', + 'ldap://localhost:%d' % self.port, + ] + args = [self.executable, '-d', self.level] + if self.cnconfig: + args += ['-F', self.home/'slapd.d'] + else: + args += ['-f', self.home/'slapd.conf'] + args += ['-h', ' '.join(listeners)] + + with open(self.home/'slapd.log', 'a+') as log: + args = [str(arg) for arg in args] + self.process = subprocess.Popen(args, stderr=log, cwd=self.home) + self.log = open(self.home/'slapd.log', 'r+') + + self.connection, self.pid = self.manager.wait(self.token) + + line = self.connection.readline().strip() + while line: + if line == 'SLAPD READY': + break + elif line.startswith("URI="): + uri, name = line[4:].split() + line = self.connection.readline().strip() + + def stop(self): + if self.process: + os.kill(self.pid, signal.SIGHUP) + self.process.terminate() + self.process.wait() + self.process = None + + def connect(self): + return LDAPObject(str(self.uri)) + + def load_module(self, module): + if not self.cnconfig: + raise NotImplementedError + + if not isinstance(module, pathlib.Path): + raise NotImplementedError + module_name = module.stem + + conn = self.connect() + conn.simple_bind_s('cn=config', self.secret) + + moduleload_object = None + for entry in conn.search_s('cn=config', ldap0.SCOPE_SUBTREE, + 'objectclass=olcModuleList', + ['olcModuleLoad']): + if not moduleload_object: + moduleload_object = entry.dn_s + for value in entry.entry_s.get('olcModuleLoad', []): + if value[0] == '{': + value = value[value.find('}')+1:] + if pathlib.Path(value).stem == module_name: + logger.warning("Module %s already loaded, ignoring", + module_name) + return + + if moduleload_object: + conn.modify_s( + moduleload_object, + [(ldap0.MOD_ADD, b'olcModuleLoad', [str(module).encode()])]) + else: + conn.add_s('cn=module,cn=config', + {'objectClass': [b'olcModuleList'], + 'olcModuleLoad': [str(module).encode()]}) + + @property + def uri(self): + return ldapurl.LDAPUrl(urlscheme="ldapi", + hostport=str(self.home/'socket')) + + +class ServerManager: + def __init__(self, tmp_path): + self.tmpdir = tmp_path + self.waiter = socket.create_server(('localhost', 0)) + self.address = self.waiter.getsockname() + + @property + def host(self): + return self.address[0] + + @property + def port(self): + return self.address[1] + + def new_server(self): + path = tempfile.TemporaryDirectory(dir=self.tmpdir) + return Server(path, self) + + def wait(self, token): + s, _ = self.waiter.accept() + f = s.makefile('r') + response = f.readline().split() + if response[0] != 'PID': + response.close() + raise RuntimeError("Unexpected response") + if response[2] != token: + raise NotImplementedError("Concurrent startup not implemented yet") + return f, int(response[1]) + + +@pytest.fixture(scope="module") +def temp(request, tmp_path_factory): + # Stolen from pytest.tmpdir._mk_tmp + name = request.node.name + name = re.sub(r"[\W]", "_", name) + MAXVAL = 30 + name = name[:MAXVAL] + return tmp_path_factory.mktemp(name, numbered=True) + + +@pytest.fixture(scope="module") +def server_factory(temp): + return ServerManager(temp) + + +@pytest.fixture(scope="class") +def server(server_factory): + server = server_factory.new_server() + server.start() + yield server + server.stop() + server.path.cleanup() + + +def test_rootdse(server): + conn = server.connect() + conn.search_s("", scope=ldap0.SCOPE_BASE) diff --git a/tests/python/syncrepl.py b/tests/python/syncrepl.py new file mode 100755 index 0000000000..72c278e718 --- /dev/null +++ b/tests/python/syncrepl.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# This work is part of OpenLDAP Software . +# +# Copyright 2021 The OpenLDAP Foundation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted only as authorized by the OpenLDAP +# Public License. +# +# A copy of this license is available in the file LICENSE in the +# top-level directory of the distribution or, alternatively, at +# . +# +# ACKNOWLEDGEMENTS: +# This work was initially developed by Ondřej Kuzník +# for inclusion in OpenLDAP Software. +""" +OpenLDAP fixtures for overlays +""" + +import ldap0 +import logging +import os +import pathlib +import pytest +import subprocess + + +from .slapd import server +from .backends import db, backend_types +from .overlays import Overlay + + +SOURCEROOT = pathlib.Path(os.environ.get('TOP_SRCDIR', "..")).absolute() +BUILDROOT = pathlib.Path(os.environ.get('TOP_BUILDDIR', SOURCEROOT)).absolute() + + +logger = logging.getLogger(__name__) + + +class Syncprov(Overlay): + objectclass = 'olcSyncprovConfig' + + def __init__(self, backend, *args, **kwargs): + super().__init__(backend, 'syncprov', *args, **kwargs) + + +@pytest.fixture(scope="class") +def provider(request, db): + conn = server.connect() + conn.simple_bind_s("cn=config", server.secret) + + syncprov = Syncprov(db) + yield db.server + + conn.delete_s(syncprov.dn) + + +@pytest.fixture(scope="class") +def replica(request, server_factory, provider): + raise NotImplementedError + + +@pytest.fixture(scope="class") +def mmr(request, server_factory): + mmr_marker = request.node.get_closest_marker("mmr") + mmr_args = mmr_marker and mmr_marker.args or {} + server_count = mmr_args.get("mmr", 4) + serverids = mmr_args.get("serverids", range(1, server_count+1)) + server_connections = mmr_args.get("connections") or \ + {consumer: {provider for provider in serverids if provider != consumer} + for consumer in serverids} + + database_marker = request.node.get_closest_marker("db") + database_type = database_marker.args[0] if database_marker else "mdb" + db_class = backend_types[database_type] + + servers = {} + connections = {} + for serverid in serverids: + server = server_factory.new_server() + server.start() + conn = server.connect() + conn.simple_bind_s("cn=config", server.secret) + + conn.modify_s("cn=config", [ + (ldap0.MOD_REPLACE, b"olcServerId", [str(serverid).encode()])]) + + server.serverid = serverid + servers[serverid] = server + connections[serverid] = conn + + db = db_class(server, "dc=example,dc=com") + syncprov = Syncprov(db) + + for serverid, server in servers.items(): + suffix = db.suffix + + syncrepl = [] + for providerid in server_connections[serverid]: + provider = servers[providerid] + db = provider.suffixes[suffix] + syncrepl.append(( + f'rid={providerid} provider={provider.uri} ' + f'searchbase="{db.suffix}" ' + f'type=refreshAndPersist retry="1 +" ' + f'bindmethod=simple ' + f'binddn="{db.suffix}" credentials="{db.secret}"').encode()) + + connections[serverid].modify_s(db.dn, [ + (ldap0.MOD_REPLACE, b"olcSyncrepl", syncrepl), + (ldap0.MOD_REPLACE, b"olcMultiprovider", [b"TRUE"])]) + + yield servers + + for serverid, server in servers.items(): + server.stop() + server.path.cleanup() + + +# TODO: after we switch to asyncio, make use of the syncmonitor module +# directly. +# We should even wrap this in a class to allow finer grained control +# over the behaviour like waiting for partial syncs etc. +def wait_for_resync(searchbase, servers, timeout=30): + subprocess.check_call(["synccheck", "-p", "--base", searchbase, + "--timeout", str(timeout), + *[str(server.uri) for server in servers], + ], timeout=timeout+5) + + +def test_mmr(mmr): + suffix = "dc=example,dc=com" + entries_added = set() + + connections = [] + for serverid, server in mmr.items(): + db = server.suffixes[suffix] + conn = server.connect() + conn.simple_bind_s(db.rootdn, db.secret) + + if not entries_added: + conn.add_s(suffix, { + "objectClass": [b"organization", + b"domainRelatedObject", + b"dcobject"], + "o": [b"Example, Inc."], + "associatedDomain": [b"example.com"]}) + entries_added.add(suffix) + # Make sure all hosts have the suffix entry + wait_for_resync(suffix, mmr.values()) + + dn = f"cn=entry{serverid},{suffix}" + conn.add_s(dn, {"objectClass": [b"device"], + "description": [(f"Entry created on serverid " + f"{serverid}").encode()]}) + entries_added.add(dn) + connections.append(conn) + + wait_for_resync(suffix, mmr.values()) + + for conn in connections: + result = conn.search_s(suffix, ldap0.SCOPE_SUBTREE, attrlist=['1.1']) + dns = {entry.dn_s for entry in result} + assert dns == entries_added, \ + f"Server {serverid} contents do not match expectations"