From: Daniele Varrazzo Date: Mon, 9 Nov 2020 00:55:11 +0000 (+0000) Subject: Load the uuid and network modules lazily X-Git-Tag: 3.0.dev0~389 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a82f55831f00c73c71f10e42b1250e82d64db5c2;p=thirdparty%2Fpsycopg.git Load the uuid and network modules lazily The uuid module is slow to import (was fixed in psycopg2 2.4.3, but there is no ticket open for it). Similarly, lazy load other non-default modules. --- diff --git a/psycopg3/psycopg3/_transform.py b/psycopg3/psycopg3/_transform.py index dfe79fbd4..383b3c159 100644 --- a/psycopg3/psycopg3/_transform.py +++ b/psycopg3/psycopg3/_transform.py @@ -157,9 +157,25 @@ class Transformer: for dmap in self._dumpers_maps: for scls in cls.__mro__: key = (scls, format) - if key in dmap: - self._dumpers_cache[key] = dumper = dmap[key](scls, self) - return dumper + dumper_class = dmap.get(key) + if not dumper_class: + continue + + self._dumpers_cache[key] = dumper = dumper_class(scls, self) + return dumper + + # If the adapter is not found, look for its name as a string + for dmap in self._dumpers_maps: + for scls in cls.__mro__: + fqn = f"{cls.__module__}.{scls.__qualname__}" + dumper_class = dmap.get((fqn, format)) + if dumper_class is None: + continue + + key = (scls, format) + dmap[key] = dumper_class + self._dumpers_cache[key] = dumper = dumper_class(scls, self) + return dumper raise e.ProgrammingError( f"cannot adapt type {type(obj).__name__}" diff --git a/psycopg3/psycopg3/adapt.py b/psycopg3/psycopg3/adapt.py index 6b40fee27..93f26a774 100644 --- a/psycopg3/psycopg3/adapt.py +++ b/psycopg3/psycopg3/adapt.py @@ -4,7 +4,7 @@ Entry point into the adaptation system. # Copyright (C) 2020 The Psycopg Team -from typing import Any, Callable, Optional, Type +from typing import Any, Callable, Optional, Type, Union from . import pq from . import proto @@ -46,11 +46,11 @@ class Dumper: @classmethod def register( cls, - src: type, + src: Union[type, str], context: AdaptContext = None, format: Format = Format.TEXT, ) -> None: - if not isinstance(src, type): + if not isinstance(src, (str, type)): raise TypeError( f"dumpers should be registered on classes, got {src} instead" ) @@ -59,11 +59,13 @@ class Dumper: where[src, format] = cls @classmethod - def register_binary(cls, src: type, context: AdaptContext = None) -> None: + def register_binary( + cls, src: Union[type, str], context: AdaptContext = None + ) -> None: cls.register(src, context, format=Format.BINARY) @classmethod - def text(cls, src: type) -> Callable[[DumperType], DumperType]: + def text(cls, src: Union[type, str]) -> Callable[[DumperType], DumperType]: def text_(dumper: DumperType) -> DumperType: dumper.register(src) return dumper @@ -71,7 +73,9 @@ class Dumper: return text_ @classmethod - def binary(cls, src: type) -> Callable[[DumperType], DumperType]: + def binary( + cls, src: Union[type, str] + ) -> Callable[[DumperType], DumperType]: def binary_(dumper: DumperType) -> DumperType: dumper.register_binary(src) return dumper diff --git a/psycopg3/psycopg3/proto.py b/psycopg3/psycopg3/proto.py index 3a0130e4b..c0bdfd664 100644 --- a/psycopg3/psycopg3/proto.py +++ b/psycopg3/psycopg3/proto.py @@ -35,7 +35,7 @@ AdaptContext = Union[None, "BaseConnection", "BaseCursor", "Transformer"] DumpFunc = Callable[[Any], bytes] DumperType = Type["Dumper"] -DumpersMap = Dict[Tuple[type, Format], DumperType] +DumpersMap = Dict[Tuple[Union[type, str], Format], DumperType] LoadFunc = Callable[[bytes], Any] LoaderType = Type["Loader"] diff --git a/psycopg3/psycopg3/types/network.py b/psycopg3/psycopg3/types/network.py index b192aeee5..6fdd4f8b9 100644 --- a/psycopg3/psycopg3/types/network.py +++ b/psycopg3/psycopg3/types/network.py @@ -4,30 +4,29 @@ Adapters for network types. # Copyright (C) 2020 The Psycopg Team -# TODO: consiter lazy dumper registration. -import ipaddress -from ipaddress import IPv4Address, IPv4Interface, IPv4Network -from ipaddress import IPv6Address, IPv6Interface, IPv6Network - -from typing import cast, Callable, Union +from typing import Callable, Union, TYPE_CHECKING from ..oids import builtins from ..adapt import Dumper, Loader +from ..proto import AdaptContext + +if TYPE_CHECKING: + import ipaddress -Address = Union[IPv4Address, IPv6Address] -Interface = Union[IPv4Interface, IPv6Interface] -Network = Union[IPv4Network, IPv6Network] +Address = Union["ipaddress.IPv4Address", "ipaddress.IPv6Address"] +Interface = Union["ipaddress.IPv4Interface", "ipaddress.IPv6Interface"] +Network = Union["ipaddress.IPv4Network", "ipaddress.IPv6Network"] -# in typeshed these types are commented out -ip_address = cast(Callable[[str], Address], ipaddress.ip_address) -ip_interface = cast(Callable[[str], Interface], ipaddress.ip_interface) -ip_network = cast(Callable[[str], Network], ipaddress.ip_network) +# These functions will be imported lazily +ip_address: Callable[[str], Address] +ip_interface: Callable[[str], Interface] +ip_network: Callable[[str], Network] -@Dumper.text(IPv4Address) -@Dumper.text(IPv6Address) -@Dumper.text(IPv4Interface) -@Dumper.text(IPv6Interface) +@Dumper.text("ipaddress.IPv4Address") +@Dumper.text("ipaddress.IPv6Address") +@Dumper.text("ipaddress.IPv4Interface") +@Dumper.text("ipaddress.IPv6Interface") class InterfaceDumper(Dumper): oid = builtins["inet"].oid @@ -36,8 +35,8 @@ class InterfaceDumper(Dumper): return str(obj).encode("utf8") -@Dumper.text(IPv4Network) -@Dumper.text(IPv6Network) +@Dumper.text("ipaddress.IPv4Network") +@Dumper.text("ipaddress.IPv6Network") class NetworkDumper(Dumper): oid = builtins["cidr"].oid @@ -46,8 +45,15 @@ class NetworkDumper(Dumper): return str(obj).encode("utf8") +class _LazyIpaddress(Loader): + def __init__(self, oid: int, context: AdaptContext = None): + super().__init__(oid, context) + global ip_address, ip_interface, ip_network + from ipaddress import ip_address, ip_interface, ip_network + + @Loader.text(builtins["inet"].oid) -class InetLoader(Loader): +class InetLoader(_LazyIpaddress): def load(self, data: bytes) -> Union[Address, Interface]: if b"/" in data: return ip_interface(data.decode("utf8")) @@ -56,6 +62,6 @@ class InetLoader(Loader): @Loader.text(builtins["cidr"].oid) -class CidrLoader(Loader): +class CidrLoader(_LazyIpaddress): def load(self, data: bytes) -> Network: return ip_network(data.decode("utf8")) diff --git a/psycopg3/psycopg3/types/uuid.py b/psycopg3/psycopg3/types/uuid.py index 73cb30ef2..eb50a1eb8 100644 --- a/psycopg3/psycopg3/types/uuid.py +++ b/psycopg3/psycopg3/types/uuid.py @@ -4,36 +4,46 @@ Adapters for the UUID type. # Copyright (C) 2020 The Psycopg Team -# TODO: importing uuid is slow. Don't import it at module level. -# Should implement lazy dumper registration. -from uuid import UUID +from typing import Callable, TYPE_CHECKING from ..oids import builtins from ..adapt import Dumper, Loader +from ..proto import AdaptContext +if TYPE_CHECKING: + import uuid -@Dumper.text(UUID) +# Importing the uuid module is slow, so import it only on request. +UUID: Callable[..., "uuid.UUID"] + + +@Dumper.text("uuid.UUID") class UUIDDumper(Dumper): oid = builtins["uuid"].oid - def dump(self, obj: UUID) -> bytes: + def dump(self, obj: "uuid.UUID") -> bytes: return obj.hex.encode("utf8") -@Dumper.binary(UUID) +@Dumper.binary("uuid.UUID") class UUIDBinaryDumper(UUIDDumper): - def dump(self, obj: UUID) -> bytes: + def dump(self, obj: "uuid.UUID") -> bytes: return obj.bytes @Loader.text(builtins["uuid"].oid) class UUIDLoader(Loader): - def load(self, data: bytes) -> UUID: + def __init__(self, oid: int, context: AdaptContext = None): + super().__init__(oid, context) + global UUID + from uuid import UUID + + def load(self, data: bytes) -> "uuid.UUID": return UUID(data.decode("utf8")) @Loader.binary(builtins["uuid"].oid) -class UUIDBinaryLoader(Loader): - def load(self, data: bytes) -> UUID: +class UUIDBinaryLoader(UUIDLoader): + def load(self, data: bytes) -> "uuid.UUID": return UUID(bytes=data) diff --git a/psycopg3_c/psycopg3_c/transform.pyx b/psycopg3_c/psycopg3_c/transform.pyx index a8cde1832..7d7e79fc6 100644 --- a/psycopg3_c/psycopg3_c/transform.pyx +++ b/psycopg3_c/psycopg3_c/transform.pyx @@ -192,9 +192,25 @@ cdef class Transformer: for dmap in self._dumpers_maps: for scls in cls.__mro__: key = (scls, format) - if key in dmap: - self._dumpers_cache[key] = dumper = dmap[key](scls, self) - return dumper + dumper_class = dmap.get(key) + if not dumper_class: + continue + + self._dumpers_cache[key] = dumper = dumper_class(scls, self) + return dumper + + # If the adapter is not found, look for its name as a string + for dmap in self._dumpers_maps: + for scls in cls.__mro__: + fqn = f"{cls.__module__}.{scls.__qualname__}" + dumper_class = dmap.get((fqn, format)) + if dumper_class is None: + continue + + key = (scls, format) + dmap[key] = dumper_class + self._dumpers_cache[key] = dumper = dumper_class(scls, self) + return dumper raise e.ProgrammingError( f"cannot adapt type {type(obj).__name__}" diff --git a/tests/types/test_network.py b/tests/types/test_network.py index c04a9113f..3d3bbf4aa 100644 --- a/tests/types/test_network.py +++ b/tests/types/test_network.py @@ -1,4 +1,7 @@ +import os +import sys import ipaddress +import subprocess as sp import pytest @@ -87,3 +90,28 @@ def test_cidr_load(conn, fmt_out, val): def binary_check(fmt): if fmt == Format.BINARY: pytest.xfail("inet binary not implemented") + + +def test_lazy_load(dsn): + script = f"""\ +import sys +import psycopg3 + +# In 3.6 it seems already loaded (at least on Travis). +if sys.version_info >= (3, 7): + assert 'ipaddress' not in sys.modules + +conn = psycopg3.connect({dsn!r}) +with conn.cursor() as cur: + cur.execute("select '127.0.0.1'::inet") + cur.fetchone() + +conn.close() +assert 'ipaddress' in sys.modules +""" + + # TODO: debug this. Importing c module fails on travis in this scenario + env = dict(os.environ) + env.pop("PSYCOPG3_IMPL", None) + + sp.check_call([sys.executable, "-s", "-c", script], env=env) diff --git a/tests/types/test_uuid.py b/tests/types/test_uuid.py index 7352f3c7f..a231041c3 100644 --- a/tests/types/test_uuid.py +++ b/tests/types/test_uuid.py @@ -1,4 +1,7 @@ +import os +import sys from uuid import UUID +import subprocess as sp import pytest @@ -20,3 +23,26 @@ def test_uuid_load(conn, fmt_out): val = "12345678123456781234567812345679" cur.execute("select %s::uuid", (val,)) assert cur.fetchone()[0] == UUID(val) + + +def test_lazy_load(dsn): + script = f"""\ +import sys +import psycopg3 + +assert 'uuid' not in sys.modules + +conn = psycopg3.connect({dsn!r}) +with conn.cursor() as cur: + cur.execute("select repeat('1', 32)::uuid") + cur.fetchone() + +conn.close() +assert 'uuid' in sys.modules +""" + + # TODO: debug this. Importing c module fails on travis in this scenario + env = dict(os.environ) + env.pop("PSYCOPG3_IMPL", None) + + sp.check_call([sys.executable, "-c", script], env=env)