From: Mike Bayer Date: Sun, 26 May 2024 15:34:27 +0000 (-0400) Subject: skip dunders for langhelper symbol redefine; update tox X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=754804635bc922c20d0b0075e0ed2da0add38742;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git skip dunders for langhelper symbol redefine; update tox Set up full Python 3.13 support to the extent currently possible, repairing issues within internal language helpers as well as the serializer extension module. update tox for what will be a regular condition: greenlet is not available (and possibly other things like pymssql): 1. dont use "sqlalchemy[asyncio]" in pyproejct.toml as an extra; this goes out to pypi and ignores the local file 2. add py{3,38,39,...} qualifiers for asyncio deps in tox.ini. After many attempts I seem to have something that's fairly non-repetetive though I'd still prefer a single variable for this, somehow Fixes: #11417 Change-Id: Ib2ceccd9583d8776700f0da5b591906efcfe6e6f --- diff --git a/doc/build/changelog/unreleased_20/11417.rst b/doc/build/changelog/unreleased_20/11417.rst new file mode 100644 index 0000000000..8e27d05923 --- /dev/null +++ b/doc/build/changelog/unreleased_20/11417.rst @@ -0,0 +1,7 @@ +.. change:: + :tags: bug, general + :tickets: 11417 + + Set up full Python 3.13 support to the extent currently possible, repairing + issues within internal language helpers as well as the serializer extension + module. diff --git a/lib/sqlalchemy/ext/serializer.py b/lib/sqlalchemy/ext/serializer.py index f21e997a22..130d253747 100644 --- a/lib/sqlalchemy/ext/serializer.py +++ b/lib/sqlalchemy/ext/serializer.py @@ -82,10 +82,9 @@ from ..util import b64encode __all__ = ["Serializer", "Deserializer", "dumps", "loads"] -def Serializer(*args, **kw): - pickler = pickle.Pickler(*args, **kw) +class Serializer(pickle.Pickler): - def persistent_id(obj): + def persistent_id(self, obj): # print "serializing:", repr(obj) if isinstance(obj, Mapper) and not obj.non_primary: id_ = "mapper:" + b64encode(pickle.dumps(obj.class_)) @@ -113,9 +112,6 @@ def Serializer(*args, **kw): return None return id_ - pickler.persistent_id = persistent_id - return pickler - our_ids = re.compile( r"(mapperprop|mapper|mapper_selectable|table|column|" @@ -123,20 +119,23 @@ our_ids = re.compile( ) -def Deserializer(file, metadata=None, scoped_session=None, engine=None): - unpickler = pickle.Unpickler(file) +class Deserializer(pickle.Unpickler): + + def __init__(self, file, metadata=None, scoped_session=None, engine=None): + super().__init__(file) + self.metadata = metadata + self.scoped_session = scoped_session + self.engine = engine - def get_engine(): - if engine: - return engine - elif scoped_session and scoped_session().bind: - return scoped_session().bind - elif metadata and metadata.bind: - return metadata.bind + def get_engine(self): + if self.engine: + return self.engine + elif self.scoped_session and self.scoped_session().bind: + return self.scoped_session().bind else: return None - def persistent_load(id_): + def persistent_load(self, id_): m = our_ids.match(str(id_)) if not m: return None @@ -157,20 +156,17 @@ def Deserializer(file, metadata=None, scoped_session=None, engine=None): cls = pickle.loads(b64decode(mapper)) return class_mapper(cls).attrs[keyname] elif type_ == "table": - return metadata.tables[args] + return self.metadata.tables[args] elif type_ == "column": table, colname = args.split(":") - return metadata.tables[table].c[colname] + return self.metadata.tables[table].c[colname] elif type_ == "session": - return scoped_session() + return self.scoped_session() elif type_ == "engine": - return get_engine() + return self.get_engine() else: raise Exception("Unknown token: %s" % type_) - unpickler.persistent_load = persistent_load - return unpickler - def dumps(obj, protocol=pickle.HIGHEST_PROTOCOL): buf = BytesIO() diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index c97fa7d629..9a02e7d71a 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -1657,6 +1657,8 @@ class _IntFlagMeta(type): items: List[symbol] cls._items = items = [] for k, v in dict_.items(): + if re.match(r"^__.*__$", k): + continue if isinstance(v, int): sym = symbol(k, canonical=v) elif not k.startswith("_"): diff --git a/pyproject.toml b/pyproject.toml index 08d2259fdf..903d793d58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ oracle-oracledb = ["oracledb>=1.0.1"] postgresql = ["psycopg2>=2.7"] postgresql-pg8000 = ["pg8000>=1.29.3"] postgresql-asyncpg = [ - "sqlalchemy[asyncio]", + "greenlet!=0.4.17", # same as ".[asyncio]" if this syntax were supported "asyncpg", ] postgresql-psycopg2binary = ["psycopg2-binary"] @@ -68,19 +68,19 @@ postgresql-psycopg = ["psycopg>=3.0.7,!=3.1.15"] postgresql-psycopgbinary = ["psycopg[binary]>=3.0.7,!=3.1.15"] pymysql = ["pymysql"] aiomysql = [ - "sqlalchemy[asyncio]", + "greenlet!=0.4.17", # same as ".[asyncio]" if this syntax were supported "aiomysql", ] aioodbc = [ - "sqlalchemy[asyncio]", + "greenlet!=0.4.17", # same as ".[asyncio]" if this syntax were supported "aioodbc", ] asyncmy = [ - "sqlalchemy[asyncio]", + "greenlet!=0.4.17", # same as ".[asyncio]" if this syntax were supported "asyncmy>=0.2.3,!=0.2.4,!=0.2.6", ] aiosqlite = [ - "sqlalchemy[asyncio]", + "greenlet!=0.4.17", # same as ".[asyncio]" if this syntax were supported "aiosqlite", ] sqlcipher = ["sqlcipher3_binary"] diff --git a/test/orm/test_mapper.py b/test/orm/test_mapper.py index 64d0ac9abd..4b3bb99c5b 100644 --- a/test/orm/test_mapper.py +++ b/test/orm/test_mapper.py @@ -2010,12 +2010,12 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL): ) # object gracefully handles this condition - assert not hasattr(User.x, "__name__") + assert not hasattr(User.x, "foobar") assert not hasattr(User.x, "comparator") m.add_property("some_attr", column_property(users.c.name)) - assert not hasattr(User.x, "__name__") + assert not hasattr(User.x, "foobar") assert hasattr(User.x, "comparator") def test_synonym_of_non_property_raises(self): diff --git a/tox.ini b/tox.ini index 14a873844c..28cae3e020 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,20 @@ [tox] envlist = py +[greenletextras] +extras= + asyncio + sqlite: aiosqlite + sqlite_file: aiosqlite + postgresql: postgresql_asyncpg + mysql: asyncmy + mysql: aiomysql + mssql: aioodbc + + # not greenlet, but tends to not have packaging until the py version + # has been fully released + mssql: mssql_pymssql + [testenv] cov_args=--cov=sqlalchemy --cov-report term --cov-append --cov-report xml --exclude-tag memory-intensive --exclude-tag timing-intensive -k "not aaa_profiling" @@ -14,26 +28,20 @@ usedevelop= cov: True extras= - asyncio - sqlite: aiosqlite - sqlite_file: aiosqlite - sqlite_file: sqlcipher; python_version < '3.10' + py{3,38,39,310,311,312}: {[greenletextras]extras} + + py{38,39,310}-sqlite_file: sqlcipher postgresql: postgresql - postgresql: postgresql_asyncpg postgresql: postgresql_pg8000 postgresql: postgresql_psycopg mysql: mysql mysql: pymysql - mysql: asyncmy - mysql: aiomysql mysql: mariadb_connector oracle: oracle oracle: oracle_oracledb mssql: mssql - mssql: aioodbc - py{3,37,38,39,310,311}-mssql: mssql_pymssql install_command= # TODO: I can find no way to get pip / tox / anyone to have this @@ -46,8 +54,6 @@ deps= # tracked by https://github.com/pytest-dev/pytest-xdist/issues/907 pytest-xdist!=3.3.0 - py312: greenlet>=3.0.0a1 - dbapimain-sqlite: git+https://github.com/omnilib/aiosqlite.git\#egg=aiosqlite dbapimain-sqlite: git+https://github.com/coleifer/sqlcipher3.git\#egg=sqlcipher3 @@ -64,7 +70,7 @@ deps= dbapimain-oracle: git+https://github.com/oracle/python-cx_Oracle.git\#egg=cx_Oracle - py312-mssql: git+https://github.com/mkleehammer/pyodbc.git\#egg=pyodbc + py313-mssql: git+https://github.com/mkleehammer/pyodbc.git\#egg=pyodbc dbapimain-mssql: git+https://github.com/mkleehammer/pyodbc.git\#egg=pyodbc cov: pytest-cov @@ -101,8 +107,6 @@ setenv= WORKERS={env:TOX_WORKERS:-n4 --max-worker-restart=5} - - nocext: DISABLE_SQLALCHEMY_CEXT=1 cext: REQUIRE_SQLALCHEMY_CEXT=1 cov: COVERAGE={[testenv]cov_args} @@ -111,12 +115,16 @@ setenv= oracle: WORKERS={env:TOX_WORKERS:-n2 --max-worker-restart=5} oracle: ORACLE={env:TOX_ORACLE:--db oracle} + oracle: EXTRA_ORACLE_DRIVERS={env:EXTRA_ORACLE_DRIVERS:--dbdriver cx_oracle --dbdriver oracledb --dbdriver oracledb_async} + py{313,314}-oracle: EXTRA_ORACLE_DRIVERS={env:EXTRA_ORACLE_DRIVERS:--dbdriver cx_oracle --dbdriver oracledb} sqlite: SQLITE={env:TOX_SQLITE:--db sqlite} sqlite_file: SQLITE={env:TOX_SQLITE_FILE:--db sqlite_file} sqlite: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver pysqlite_numeric --dbdriver aiosqlite} + py{313,314}-sqlite: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver pysqlite_numeric} + sqlite-nogreenlet: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver pysqlite_numeric} py{37,38,39}-sqlite_file: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver aiosqlite --dbdriver pysqlcipher} @@ -140,10 +148,12 @@ setenv= mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector} mssql: MSSQL={env:TOX_MSSQL:--db mssql} - py{3,38,39,310,311}-mssql: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver aioodbc --dbdriver pymssql} - py{3,38,39,310,311}-mssql-nogreenlet: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver pymssql} - py312-mssql: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver aioodbc} - py312-mssql-nogreenlet: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc} + + mssql: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver aioodbc --dbdriver pymssql} + py{313,314}-mssql: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver aioodbc} + + mssql-nogreenlet: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver pymssql} + py{313,314}-mssql-nogreenlet: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc} oracle,mssql,sqlite_file: IDENTS=--write-idents db_idents.txt @@ -188,6 +198,9 @@ commands = # suddently appearing for it to be stable enough for CI # pyright +extras = + {[greenletextras]extras} + [testenv:mypy] deps= pytest>=7.0.0rc1,<8 @@ -196,6 +209,9 @@ deps= mypy >= 1.7.0 patch==1.* types-greenlet +extras= + {[greenletextras]extras} + commands = pytest {env:PYTEST_COLOR} -m mypy {posargs} @@ -205,6 +221,9 @@ deps= {[testenv:mypy]deps} pytest-cov +extras= + {[greenletextras]extras} + commands = pytest {env:PYTEST_COLOR} -m mypy {env:COVERAGE} {posargs} @@ -214,6 +233,10 @@ setenv= # thanks to https://julien.danjou.info/the-best-flake8-extensions/ [testenv:lint] basepython = python3 + +extras= + {[greenletextras]extras} + deps= flake8==6.1.0 flake8-import-order @@ -259,10 +282,15 @@ basepython = {[testenv:lint]basepython} deps = {[testenv:lint]deps} allowlist_externals = {[testenv:lint]allowlist_externals} commands = {[testenv:lint]commands} +extras = {[testenv:lint]extras} + # command run in the github action when cext are active. [testenv:github-cext] +extras= + {[greenletextras]extras} + deps = {[testenv]deps} .[aiosqlite] commands= @@ -271,6 +299,9 @@ commands= # command run in the github action when cext are not active. [testenv:github-nocext] +extras= + {[greenletextras]extras} + deps = {[testenv]deps} .[aiosqlite] commands=