From 9b4885e2a0fdd8aa9f22edb15bd38e8509c85c00 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 4 Nov 2020 15:15:40 +0100 Subject: [PATCH] Fast codec functions moved to an utility module --- psycopg3/psycopg3/proto.py | 3 --- psycopg3/psycopg3/types/date.py | 29 +++++++++++++---------------- psycopg3/psycopg3/types/json.py | 9 +++------ psycopg3/psycopg3/types/numeric.py | 12 ++++-------- psycopg3/psycopg3/types/text.py | 14 +++++++------- psycopg3/psycopg3/types/uuid.py | 11 +++-------- psycopg3/psycopg3/utils/codecs.py | 16 ++++++++++++++++ 7 files changed, 46 insertions(+), 48 deletions(-) create mode 100644 psycopg3/psycopg3/utils/codecs.py diff --git a/psycopg3/psycopg3/proto.py b/psycopg3/psycopg3/proto.py index da7137982..c8f04b24c 100644 --- a/psycopg3/psycopg3/proto.py +++ b/psycopg3/psycopg3/proto.py @@ -20,9 +20,6 @@ if TYPE_CHECKING: from .waiting import Wait, Ready from .sql import Composable -EncodeFunc = Callable[[str], Tuple[bytes, int]] -DecodeFunc = Callable[[bytes], Tuple[str, int]] - Query = Union[str, bytes, "Composable"] Params = Union[Sequence[Any], Mapping[str, Any]] diff --git a/psycopg3/psycopg3/types/date.py b/psycopg3/psycopg3/types/date.py index 47861903f..9cf039353 100644 --- a/psycopg3/psycopg3/types/date.py +++ b/psycopg3/psycopg3/types/date.py @@ -6,17 +6,14 @@ Adapters for date/time types. import re import sys -import codecs from datetime import date, datetime, time, timedelta from typing import cast from ..oids import builtins from ..adapt import Dumper, Loader -from ..proto import AdaptContext, EncodeFunc, DecodeFunc +from ..proto import AdaptContext from ..errors import InterfaceError, DataError - -_encode_ascii = codecs.lookup("ascii").encode -_decode_ascii = codecs.lookup("ascii").decode +from ..utils.codecs import EncodeFunc, DecodeFunc, encode_ascii, decode_ascii @Dumper.text(date) @@ -24,7 +21,7 @@ class DateDumper(Dumper): oid = builtins["date"].oid - def dump(self, obj: date, __encode: EncodeFunc = _encode_ascii) -> bytes: + def dump(self, obj: date, __encode: EncodeFunc = encode_ascii) -> bytes: # NOTE: whatever the PostgreSQL DateStyle input format (DMY, MDY, YMD) # the YYYY-MM-DD is always understood correctly. return __encode(str(obj))[0] @@ -35,7 +32,7 @@ class TimeDumper(Dumper): oid = builtins["timetz"].oid - def dump(self, obj: time, __encode: EncodeFunc = _encode_ascii) -> bytes: + def dump(self, obj: time, __encode: EncodeFunc = encode_ascii) -> bytes: return __encode(str(obj))[0] @@ -44,7 +41,7 @@ class DateTimeDumper(Dumper): oid = builtins["timestamptz"].oid - def dump(self, obj: date, __encode: EncodeFunc = _encode_ascii) -> bytes: + def dump(self, obj: date, __encode: EncodeFunc = encode_ascii) -> bytes: # NOTE: whatever the PostgreSQL DateStyle input format (DMY, MDY, YMD) # the YYYY-MM-DD is always understood correctly. return __encode(str(obj))[0] @@ -65,7 +62,7 @@ class TimeDeltaDumper(Dumper): setattr(self, "dump", self._dump_sql) def dump( - self, obj: timedelta, __encode: EncodeFunc = _encode_ascii + self, obj: timedelta, __encode: EncodeFunc = encode_ascii ) -> bytes: return __encode(str(obj))[0] @@ -85,7 +82,7 @@ class DateLoader(Loader): super().__init__(oid, context) self._format = self._format_from_context() - def load(self, data: bytes, __decode: DecodeFunc = _decode_ascii) -> date: + def load(self, data: bytes, __decode: DecodeFunc = decode_ascii) -> date: try: return datetime.strptime(__decode(data)[0], self._format).date() except ValueError as e: @@ -143,7 +140,7 @@ class TimeLoader(Loader): _format = "%H:%M:%S.%f" _format_no_micro = _format.replace(".%f", "") - def load(self, data: bytes, __decode: DecodeFunc = _decode_ascii) -> time: + def load(self, data: bytes, __decode: DecodeFunc = decode_ascii) -> time: # check if the data contains microseconds fmt = self._format if b"." in data else self._format_no_micro try: @@ -173,7 +170,7 @@ class TimeTzLoader(TimeLoader): super().__init__(oid, context) - def load(self, data: bytes, __decode: DecodeFunc = _decode_ascii) -> time: + def load(self, data: bytes, __decode: DecodeFunc = decode_ascii) -> time: # Hack to convert +HH in +HHMM if data[-3] in (43, 45): data += b"00" @@ -187,7 +184,7 @@ class TimeTzLoader(TimeLoader): return dt.time().replace(tzinfo=dt.tzinfo) def _load_py36( - self, data: bytes, __decode: DecodeFunc = _decode_ascii + self, data: bytes, __decode: DecodeFunc = decode_ascii ) -> time: # Drop seconds from timezone for Python 3.6 # Also, Python 3.6 doesn't support HHMM, only HH:MM @@ -206,7 +203,7 @@ class TimestampLoader(DateLoader): self._format_no_micro = self._format.replace(".%f", "") def load( - self, data: bytes, __decode: DecodeFunc = _decode_ascii + self, data: bytes, __decode: DecodeFunc = decode_ascii ) -> datetime: # check if the data contains microseconds fmt = ( @@ -288,7 +285,7 @@ class TimestamptzLoader(TimestampLoader): return "" def load( - self, data: bytes, __decode: DecodeFunc = _decode_ascii + self, data: bytes, __decode: DecodeFunc = decode_ascii ) -> datetime: # Hack to convert +HH in +HHMM if data[-3] in (43, 45): @@ -297,7 +294,7 @@ class TimestamptzLoader(TimestampLoader): return super().load(data) def _load_py36( - self, data: bytes, __decode: DecodeFunc = _decode_ascii + self, data: bytes, __decode: DecodeFunc = decode_ascii ) -> datetime: # Drop seconds from timezone for Python 3.6 # Also, Python 3.6 doesn't support HHMM, only HH:MM diff --git a/psycopg3/psycopg3/types/json.py b/psycopg3/psycopg3/types/json.py index 296082b8e..2e7ee5ffd 100644 --- a/psycopg3/psycopg3/types/json.py +++ b/psycopg3/psycopg3/types/json.py @@ -5,15 +5,12 @@ Adapers for JSON types. # Copyright (C) 2020 The Psycopg Team import json -import codecs from typing import Any, Callable, Optional from ..oids import builtins from ..adapt import Dumper, Loader -from ..proto import EncodeFunc from ..errors import DataError - -_encode_utf8 = codecs.lookup("utf8").encode +from ..utils.codecs import EncodeFunc, encode_utf8 JSON_OID = builtins["json"].oid JSONB_OID = builtins["jsonb"].oid @@ -40,7 +37,7 @@ class Jsonb(_JsonWrapper): class _JsonDumper(Dumper): def dump( - self, obj: _JsonWrapper, __encode: EncodeFunc = _encode_utf8 + self, obj: _JsonWrapper, __encode: EncodeFunc = encode_utf8 ) -> bytes: return __encode(obj.dumps())[0] @@ -59,7 +56,7 @@ class JsonbDumper(_JsonDumper): @Dumper.binary(Jsonb) class JsonbBinaryDumper(JsonbDumper): def dump( - self, obj: _JsonWrapper, __encode: EncodeFunc = _encode_utf8 + self, obj: _JsonWrapper, __encode: EncodeFunc = encode_utf8 ) -> bytes: return b"\x01" + __encode(obj.dumps())[0] diff --git a/psycopg3/psycopg3/types/numeric.py b/psycopg3/psycopg3/types/numeric.py index 50b07e557..e32f7f638 100644 --- a/psycopg3/psycopg3/types/numeric.py +++ b/psycopg3/psycopg3/types/numeric.py @@ -4,26 +4,22 @@ Adapers for numeric types. # Copyright (C) 2020 The Psycopg Team -import codecs import struct from typing import Any, Callable, Dict, Tuple, cast from decimal import Decimal from ..oids import builtins from ..adapt import Dumper, Loader -from ..proto import EncodeFunc, DecodeFunc +from ..utils.codecs import EncodeFunc, DecodeFunc, encode_ascii, decode_ascii UnpackInt = Callable[[bytes], Tuple[int]] UnpackFloat = Callable[[bytes], Tuple[float]] -_encode_ascii = codecs.lookup("ascii").encode -_decode_ascii = codecs.lookup("ascii").decode - class NumberDumper(Dumper): _special: Dict[bytes, bytes] = {} - def dump(self, obj: Any, __encode: EncodeFunc = _encode_ascii) -> bytes: + def dump(self, obj: Any, __encode: EncodeFunc = encode_ascii) -> bytes: return __encode(str(obj))[0] def quote(self, obj: Any) -> bytes: @@ -70,7 +66,7 @@ class DecimalDumper(NumberDumper): @Loader.text(builtins["int8"].oid) @Loader.text(builtins["oid"].oid) class IntLoader(Loader): - def load(self, data: bytes, __decode: DecodeFunc = _decode_ascii) -> int: + def load(self, data: bytes, __decode: DecodeFunc = decode_ascii) -> int: return int(__decode(data)[0]) @@ -145,6 +141,6 @@ class Float8BinaryLoader(Loader): @Loader.text(builtins["numeric"].oid) class NumericLoader(Loader): def load( - self, data: bytes, __decode: DecodeFunc = _decode_ascii + self, data: bytes, __decode: DecodeFunc = decode_ascii ) -> Decimal: return Decimal(__decode(data)[0]) diff --git a/psycopg3/psycopg3/types/text.py b/psycopg3/psycopg3/types/text.py index 8d82c963d..c5988778c 100644 --- a/psycopg3/psycopg3/types/text.py +++ b/psycopg3/psycopg3/types/text.py @@ -4,14 +4,14 @@ Adapters for textual types. # Copyright (C) 2020 The Psycopg Team -import codecs from typing import Optional, Union, TYPE_CHECKING +from ..pq import Escaping from ..oids import builtins, INVALID_OID from ..adapt import Dumper, Loader -from ..proto import AdaptContext, EncodeFunc, DecodeFunc +from ..proto import AdaptContext from ..errors import DataError -from ..pq import Escaping +from ..utils.codecs import EncodeFunc, DecodeFunc, encode_utf8, decode_utf8 if TYPE_CHECKING: from ..pq.proto import Escaping as EscapingProto @@ -26,9 +26,9 @@ class _StringDumper(Dumper): if self.connection.client_encoding != "SQL_ASCII": self._encode = self.connection.codec.encode else: - self._encode = codecs.lookup("utf8").encode + self._encode = encode_utf8 else: - self._encode = codecs.lookup("utf8").encode + self._encode = encode_utf8 @Dumper.binary(str) @@ -66,7 +66,7 @@ class TextLoader(Loader): else: self.decode = None else: - self.decode = codecs.lookup("utf8").decode + self.decode = decode_utf8 def load(self, data: bytes) -> Union[bytes, str]: if self.decode is not None: @@ -88,7 +88,7 @@ class UnknownLoader(Loader): if self.connection is not None: self.decode = self.connection.codec.decode else: - self.decode = codecs.lookup("utf8").decode + self.decode = decode_utf8 def load(self, data: bytes) -> str: return self.decode(data)[0] diff --git a/psycopg3/psycopg3/types/uuid.py b/psycopg3/psycopg3/types/uuid.py index 37ff17cae..6076b12f4 100644 --- a/psycopg3/psycopg3/types/uuid.py +++ b/psycopg3/psycopg3/types/uuid.py @@ -8,14 +8,9 @@ Adapters for the UUID type. # Should implement lazy dumper registration. from uuid import UUID -import codecs - from ..oids import builtins from ..adapt import Dumper, Loader -from ..proto import DecodeFunc, EncodeFunc - -_encode_ascii = codecs.lookup("ascii").encode -_decode_ascii = codecs.lookup("ascii").decode +from ..utils.codecs import EncodeFunc, DecodeFunc, encode_ascii, decode_ascii @Dumper.text(UUID) @@ -23,7 +18,7 @@ class UUIDDumper(Dumper): oid = builtins["uuid"].oid - def dump(self, obj: UUID, __encode: EncodeFunc = _encode_ascii) -> bytes: + def dump(self, obj: UUID, __encode: EncodeFunc = encode_ascii) -> bytes: return __encode(obj.hex)[0] @@ -38,7 +33,7 @@ class UUIDBinaryDumper(Dumper): @Loader.text(builtins["uuid"].oid) class UUIDLoader(Loader): - def load(self, data: bytes, __decode: DecodeFunc = _decode_ascii) -> UUID: + def load(self, data: bytes, __decode: DecodeFunc = decode_ascii) -> UUID: return UUID(__decode(data)[0]) diff --git a/psycopg3/psycopg3/utils/codecs.py b/psycopg3/psycopg3/utils/codecs.py new file mode 100644 index 000000000..e9c31b98e --- /dev/null +++ b/psycopg3/psycopg3/utils/codecs.py @@ -0,0 +1,16 @@ +""" +Utility module to access fast encoders/decoders +""" + +# Copyright (C) 2020 The Psycopg Team + +import codecs +from typing import Callable, Tuple + +EncodeFunc = Callable[[str], Tuple[bytes, int]] +DecodeFunc = Callable[[bytes], Tuple[str, int]] + +encode_ascii = codecs.lookup("ascii").encode +decode_ascii = codecs.lookup("ascii").decode +encode_utf8 = codecs.lookup("utf8").encode +decode_utf8 = codecs.lookup("utf8").decode -- 2.47.2