- PGVER=9.5
- PSYCOPG3_IMPL=c
- PGPORT=5432
+ # skip tests failing on importing psycopg3_c.pq on subprocess
+ # they only fail on Travis, work ok locally under tox too.
+ - PYTEST_ADDOPTS="-m 'not subprocess'"
- python: 3.6
addons:
- PGVER=10
- PSYCOPG3_IMPL=c
- PGPORT=5432
+ - PYTEST_ADDOPTS="-m 'not subprocess'"
- python: 3.7
addons:
- PGVER=12
- PSYCOPG3_IMPL=c
- PGPORT=5433
+ - PYTEST_ADDOPTS="-m 'not subprocess'"
- python: 3.8
addons:
if not impl or impl == "c":
try:
# TODO: extension module not recognised by mypy?
- from psycopg3_c import pq_cython as module # type: ignore
+ from psycopg3_c import pq as module # type: ignore
except Exception as e:
if not impl:
logger.debug("C pq wrapper not available: %s", e)
# Second best implementation: fast and stand-alone
if not module and (not impl or impl == "binary"):
try:
- from psycopg3_binary import pq_cython as module # type: ignore
+ from psycopg3_binary import pq as module # type: ignore
except Exception as e:
if not impl:
logger.debug("C pq wrapper not available: %s", e)
/*.so
_psycopg3.c
-pq_cython.c
+pq.c
*.html
"""
-psycopg3._psycopg3 optimization module.
+psycopg3_c._psycopg3 optimization module.
The module contains optimized C code used in preference to Python code
if a compiler is available.
# Copyright (C) 2020 The Psycopg Team
+from psycopg3_c cimport pq
+from psycopg3_c.pq cimport libpq
+from psycopg3_c._psycopg3 cimport oids
+
+include "_psycopg3/adapt.pyx"
+include "_psycopg3/generators.pyx"
+include "_psycopg3/transform.pyx"
+
include "types/numeric.pyx"
include "types/singletons.pyx"
include "types/text.pyx"
-include "generators.pyx"
-include "adapt.pyx"
-include "transform.pyx"
--- /dev/null
+"""
+psycopg3_c.pq cython module.
+
+This file is necessary to allow c-importing pxd files from this directory.
+"""
+
+# Copyright (C) 2020 The Psycopg Team
+
+from psycopg3_c._psycopg3 cimport oids
from cpython.bytearray cimport PyByteArray_FromStringAndSize, PyByteArray_Resize
from cpython.bytearray cimport PyByteArray_AS_STRING
-from psycopg3_c cimport oids
-from psycopg3_c.pq cimport libpq
-from psycopg3_c.adapt cimport cloader_func, get_context_func
-from psycopg3_c.pq_cython cimport Escaping, _buffer_as_string_and_size
+from psycopg3_c.pq cimport _buffer_as_string_and_size
from psycopg3 import errors as e
from psycopg3.pq import Format
cdef object src
cdef public libpq.Oid oid
cdef readonly object connection
- cdef PGconn _pgconn
+ cdef pq.PGconn _pgconn
def __init__(self, src: type, context: Optional["AdaptContext"] = None):
self.src = src
from typing import List
from psycopg3 import errors as e
+from psycopg3.pq import proto, error_message, PQerror
from psycopg3.proto import PQGen
from psycopg3.waiting import Wait, Ready
-from psycopg3 import pq
-from psycopg3_c.pq cimport libpq
-from psycopg3_c.pq_cython cimport PGconn, PGresult
cdef object WAIT_W = Wait.W
cdef object WAIT_R = Wait.R
cdef object WAIT_RW = Wait.RW
cdef int READY_R = Ready.R
-def connect(conninfo: str) -> PQGenConn[pq.proto.PGconn]:
+def connect(conninfo: str) -> PQGenConn[proto.PGconn]:
"""
Generator to create a database connection without blocking.
"""
- cdef PGconn conn = PGconn.connect_start(conninfo.encode("utf8"))
+ cdef pq.PGconn conn = pq.PGconn.connect_start(conninfo.encode("utf8"))
logger.debug("connection started, status %s", conn.status.name)
cdef libpq.PGconn *pgconn_ptr = conn.pgconn_ptr
cdef int conn_status = libpq.PQstatus(pgconn_ptr)
while 1:
if conn_status == libpq.CONNECTION_BAD:
raise e.OperationalError(
- f"connection is bad: {pq.error_message(conn)}"
+ f"connection is bad: {error_message(conn)}"
)
poll_status = libpq.PQconnectPoll(pgconn_ptr)
yield (libpq.PQsocket(pgconn_ptr), WAIT_W)
elif poll_status == libpq.PGRES_POLLING_FAILED:
raise e.OperationalError(
- f"connection failed: {pq.error_message(conn)}"
+ f"connection failed: {error_message(conn)}"
)
else:
raise e.InternalError(f"unexpected poll status: {poll_status}")
return conn
-def execute(PGconn pgconn) -> PQGen[List[pq.proto.PGresult]]:
+def execute(pq.PGconn pgconn) -> PQGen[List[proto.PGresult]]:
"""
Generator sending a query and returning results without blocking.
Return the list of results returned by the database (whether success
or error).
"""
- results: List[pq.proto.PGresult] = []
+ results: List[proto.PGresult] = []
cdef libpq.PGconn *pgconn_ptr = pgconn.pgconn_ptr
cdef int status
cdef libpq.PGnotify *notify
# PGconn buffer and passed to Python later.
cires = libpq.PQconsumeInput(pgconn_ptr)
if 1 != cires:
- raise pq.PQerror(
- f"consuming input failed: {pq.error_message(pgconn)}")
+ raise PQerror(
+ f"consuming input failed: {error_message(pgconn)}")
continue
# Fetching the result
ibres = libpq.PQisBusy(pgconn_ptr)
if 1 != cires:
- raise pq.PQerror(
- f"consuming input failed: {pq.error_message(pgconn)}")
+ raise PQerror(
+ f"consuming input failed: {error_message(pgconn)}")
if ibres:
yield WAIT_R
continue
pgres = libpq.PQgetResult(pgconn_ptr)
if pgres is NULL:
break
- results.append(PGresult._from_ptr(pgres))
+ results.append(pq.PGresult._from_ptr(pgres))
status = libpq.PQresultStatus(pgres)
if status in (libpq.PGRES_COPY_IN, libpq.PGRES_COPY_OUT, libpq.PGRES_COPY_BOTH):
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
-from psycopg3_c cimport oids
-from psycopg3_c.pq cimport libpq
-from psycopg3_c.pq_cython cimport PGresult
-
from psycopg3 import errors as e
from psycopg3.pq import Format
cdef dict _dumpers_cache
cdef dict _text_loaders
cdef dict _binary_loaders
- cdef PGresult _pgresult
+ cdef pq.PGresult _pgresult
cdef int _nfields, _ntuples
cdef list _row_loaders
+++ /dev/null
-"""
-C definitions of the adaptation system.
-"""
-
-# Copyright (C) 2020 The Psycopg Team
-
-from cpython.object cimport PyObject
-
-# The type of a function reading a result from the database and returning
-# a Python object.
-ctypedef object (*cloader_func)(const char *data, size_t length, void *context)
-
-# Take in input a Loader instance and return a context for a `cloader_func`.
-ctypedef void * (*get_context_func)(object conn)
from psycopg3_c.pq cimport libpq
-from psycopg3.pq.misc import PQerror, error_message
-
from psycopg3.pq import ConnStatus, PollingStatus, ExecStatus
from psycopg3.pq import TransactionStatus, Ping, DiagnosticField, Format
-
+from psycopg3.pq.misc import PQerror, error_message
__impl__ = 'c'
--- /dev/null
+"""
+psycopg3_c.pq cython module.
+
+This file is necessary to allow c-importing pxd files from this directory.
+"""
+
+# Copyright (C) 2020 The Psycopg Team
+
+from psycopg3_c.pq cimport libpq
"""
-psycopg3_c.pq_cython.Conninfo object implementation.
+psycopg3_c.pq.Conninfo object implementation.
"""
# Copyright (C) 2020 The Psycopg Team
"""
-psycopg3_c.pq_cython.Escaping object implementation.
+psycopg3_c.pq.Escaping object implementation.
"""
# Copyright (C) 2020 The Psycopg Team
"""
-psycopg3_c.pq_cython.PGcancel object implementation.
+psycopg3_c.pq.PGcancel object implementation.
"""
# Copyright (C) 2020 The Psycopg Team
"""
-psycopg3_c.pq_cython.PGconn object implementation.
+psycopg3_c.pq.PGconn object implementation.
"""
# Copyright (C) 2020 The Psycopg Team
import logging
-from psycopg3_c.pq.libpq cimport Oid
from psycopg3.pq.misc import PGnotify, connection_summary
logger = logging.getLogger('psycopg3')
_ensure_pgconn(self)
cdef int cnparams
- cdef Oid *ctypes
+ cdef libpq.Oid *ctypes
cdef char *const *cvalues
cdef int *clengths
cdef int *cformats
_ensure_pgconn(self)
cdef int cnparams
- cdef Oid *ctypes
+ cdef libpq.Oid *ctypes
cdef char *const *cvalues
cdef int *clengths
cdef int *cformats
cdef int i
cdef int nparams = len(param_types) if param_types else 0
- cdef Oid *atypes = NULL
+ cdef libpq.Oid *atypes = NULL
if nparams:
- atypes = <Oid *>PyMem_Malloc(nparams * sizeof(Oid))
+ atypes = <libpq.Oid *>PyMem_Malloc(nparams * sizeof(libpq.Oid))
for i in range(nparams):
atypes[i] = param_types[i]
_ensure_pgconn(self)
cdef int cnparams
- cdef Oid *ctypes
+ cdef libpq.Oid *ctypes
cdef char *const *cvalues
cdef int *clengths
cdef int *cformats
cdef int i
cdef int nparams = len(param_types) if param_types else 0
- cdef Oid *atypes = NULL
+ cdef libpq.Oid *atypes = NULL
if nparams:
- atypes = <Oid *>PyMem_Malloc(nparams * sizeof(Oid))
+ atypes = <libpq.Oid *>PyMem_Malloc(nparams * sizeof(libpq.Oid))
for i in range(nparams):
atypes[i] = param_types[i]
_ensure_pgconn(self)
cdef int cnparams
- cdef Oid *ctypes
+ cdef libpq.Oid *ctypes
cdef char *const *cvalues
cdef int *clengths
cdef int *cformats
res.pgresult_ptr = NULL # avoid destroying the pgresult_ptr
-cdef (int, Oid *, char * const*, int *, int *) _query_params_args(
+cdef (int, libpq.Oid *, char * const*, int *, int *) _query_params_args(
list param_values: Optional[Sequence[Optional[bytes]]],
param_types: Optional[Sequence[int]],
list param_formats: Optional[Sequence[Format]],
aparams[i] = ptr
alenghts[i] = length
- cdef Oid *atypes = NULL
+ cdef libpq.Oid *atypes = NULL
if tparam_types:
- atypes = <Oid *>PyMem_Malloc(nparams * sizeof(Oid))
+ atypes = <libpq.Oid *>PyMem_Malloc(nparams * sizeof(libpq.Oid))
for i in range(nparams):
atypes[i] = tparam_types[i]
cdef void _clear_query_params(
- Oid *ctypes, char *const *cvalues, int *clenghst, int *cformats
+ libpq.Oid *ctypes, char *const *cvalues, int *clenghst, int *cformats
):
PyMem_Free(ctypes)
PyMem_Free(<char **>cvalues)
"""
-psycopg3_c.pq_cython.PGresult object implementation.
+psycopg3_c.pq.PGresult object implementation.
"""
# Copyright (C) 2020 The Psycopg Team
from cpython.long cimport PyLong_FromLongLong, PyLong_FromUnsignedLong
from cpython.float cimport PyFloat_FromDouble
-from psycopg3_c cimport oids
-from psycopg3_c.endian cimport be16toh, be32toh, be64toh, htobe64
+from endian cimport be16toh, be32toh, be64toh, htobe64
cdef extern from "Python.h":
# work around https://github.com/cython/cython/issues/3909
# Copyright (C) 2020 The Psycopg Team
from psycopg3.pq import Format
-from psycopg3_c cimport oids
cdef class BoolDumper(CDumper):
from cpython.unicode cimport PyUnicode_Decode, PyUnicode_DecodeUTF8
from cpython.unicode cimport PyUnicode_AsUTF8String, PyUnicode_AsEncodedString
-from psycopg3_c cimport oids
-from psycopg3_c.pq cimport libpq
-from psycopg3_c.pq_cython cimport Escaping
+from psycopg3_c.pq cimport Escaping
cdef class _StringDumper(CDumper):
if cythonize is not None:
for ext in self.distribution.ext_modules:
- ext.sources[0] = os.path.splitext(ext.sources[0])[0] + ".pyx"
+ for i in range(len(ext.sources)):
+ base, fext = os.path.splitext(ext.sources[i])
+ if fext == ".c":
+ ext.sources[i] = base + ".pyx"
self.distribution.ext_modules = cythonize(
self.distribution.ext_modules,
)
pqext = Extension(
- "psycopg3_c.pq_cython",
- ["psycopg3_c/pq_cython.c"],
+ "psycopg3_c.pq",
+ ["psycopg3_c/pq.c"],
libraries=["pq"],
include_dirs=[],
)
config.addinivalue_line(
"markers", "slow: this test is kinda slow (skip with -m 'not slow')"
)
+
+ # There are troubles on travis with these kind of tests and I cannot
+ # catch the exception for my life.
+ config.addinivalue_line(
+ "markers", "subprocess: the test import psycopg3 after subprocess"
+ )
pgconn.host
-# TODO: to implement in pq_cython
+# TODO: to implement in psycopg3_c.pq
@pytest.mark.xfail
@pytest.mark.libpq(">= 12")
def test_hostaddr(pgconn):
@pytest.mark.slow
+@pytest.mark.subprocess
def test_multiprocess_close(dsn, tmpdir):
# Check the problem reported in psycopg2#829
# Subprocess gcs the copy of the fd after fork so it closes connection.
f.write(module)
env = dict(os.environ)
env["PYTHONPATH"] = str(tmpdir + os.pathsep + env.get("PYTHONPATH", ""))
- # TODO: debug this. Importing c module fails on travis in this scenario
- env.pop("PSYCOPG3_IMPL", None)
out = sp.check_output(
[sys.executable, "-c", script], stderr=sp.STDOUT, env=env
).decode("utf8", "replace")
-import os
import sys
import ipaddress
import subprocess as sp
pytest.xfail("inet binary not implemented")
+@pytest.mark.subprocess
def test_lazy_load(dsn):
script = f"""\
import sys
assert 'ipaddress' in sys.modules
"""
- # TODO: debug this. Importing c module fails on travis in this scenario
- env = dict(os.environ)
- env.pop("PSYCOPG3_IMPL", None)
-
- sp.check_call([sys.executable, "-s", "-c", script], env=env)
+ sp.check_call([sys.executable, "-s", "-c", script])
-import os
import sys
from uuid import UUID
import subprocess as sp
assert cur.fetchone()[0] == UUID(val)
+@pytest.mark.subprocess
def test_lazy_load(dsn):
script = f"""\
import sys
assert 'uuid' in sys.modules
"""
- # TODO: debug this. Importing c module fails on travis in this scenario
- env = dict(os.environ)
- env.pop("PSYCOPG3_IMPL", None)
-
- sp.check_call([sys.executable, "-c", script], env=env)
+ sp.check_call([sys.executable, "-c", script])
mv /psycopg3_binary/{psycopg3_c,psycopg3_binary}/
sed -i 's/psycopg3-c/psycopg3-binary/' /psycopg3_binary/setup.cfg
sed -i "s/__impl__[[:space:]]*=.*/__impl__ = 'binary'/" \
- /psycopg3_binary/psycopg3_binary/pq_cython.pyx
+ /psycopg3_binary/psycopg3_binary/pq.pyx
find /psycopg3_binary/ -name \*.pyx -or -name \*.pxd -or -name \*.py \
| xargs sed -i 's/\bpsycopg3_c\b/psycopg3_binary/'
You can update this file by executing it, using the PG* env var to connect
"""
-import os
import re
import subprocess as sp
from typing import List
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent
version_sql = """
def update_python_oids() -> None:
queries = [version_sql, py_oids_sql]
- fn = os.path.dirname(__file__) + "/../psycopg3/psycopg3/oids.py"
+ fn = ROOT / "psycopg3/psycopg3/oids.py"
update_file(fn, queries)
def update_cython_oids() -> None:
queries = [version_sql, cython_oids_sql]
- fn = os.path.dirname(__file__) + "/../psycopg3_c/psycopg3_c/oids.pxd"
+ fn = ROOT / "psycopg3_c/psycopg3_c/_psycopg3/oids.pxd"
update_file(fn, queries)
-def update_file(fn: str, queries: List[str]) -> None:
- with open(fn, "rb") as f:
+def update_file(fn: Path, queries: List[str]) -> None:
+ with fn.open("rb") as f:
lines = f.read().splitlines()
new = []
]
lines[istart + 1 : iend] = new
- with open(fn, "wb") as f:
+ with fn.open("wb") as f:
f.write(b"\n".join(lines))
f.write(b"\n")
projects / thirdparty / psycopg.git / commitdiff
