From: Mike Bayer Date: Mon, 5 Feb 2024 17:02:19 +0000 (-0500) Subject: add additional IMV UUID tests, fix pymssql case X-Git-Tag: rel_2_0_26~14^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e766aa473f983dfbf926246ec14265220aa97103;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git add additional IMV UUID tests, fix pymssql case Fixed an issue regarding the use of the :class:`.Uuid` datatype with the :paramref:`.Uuid.as_uuid` parameter set to False, when using the pymssql dialect. ORM-optimized INSERT statements (e.g. the "insertmanyvalues" feature) would not correctly align primary key UUID values for bulk INSERT statements, resulting in errors. This change also adds a small degree of generalization to the Uuid datatype by adding the native/non-native compilation conditional to the base compiler. Patch is originally part of Ib920871102b9b64f2cba9697f5cb72b6263e4ed8 which is implementing native UUID for mariadb in 2.1 only. Change-Id: I96cbec5c0ece312b345206aa5a5db2ffcf732d41 (cherry picked from commit 9b1c9d5d2e2f9a1e83cf80ca5cd834de213e59ea) --- diff --git a/doc/build/changelog/unreleased_20/uuid_imv_fixes.rst b/doc/build/changelog/unreleased_20/uuid_imv_fixes.rst new file mode 100644 index 0000000000..79aa132b21 --- /dev/null +++ b/doc/build/changelog/unreleased_20/uuid_imv_fixes.rst @@ -0,0 +1,20 @@ +.. change:: + :tags: bug, mssql + + Fixed an issue regarding the use of the :class:`.Uuid` datatype with the + :paramref:`.Uuid.as_uuid` parameter set to False, when using the pymssql + dialect. ORM-optimized INSERT statements (e.g. the "insertmanyvalues" + feature) would not correctly align primary key UUID values for bulk INSERT + statements, resulting in errors. Similar issues were fixed for the + PostgreSQL drivers as well. + + +.. change:: + :tags: bug, postgresql + + Fixed an issue regarding the use of the :class:`.Uuid` datatype with the + :paramref:`.Uuid.as_uuid` parameter set to False, when using the pymssql + dialect. ORM-optimized INSERT statements (e.g. the "insertmanyvalues" + feature) would not correctly align primary key UUID values for bulk INSERT + statements, resulting in errors. Similar issues were fixed for the + pymssql driver as well. diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index e015dccdc9..83327899fa 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -1426,7 +1426,6 @@ class ROWVERSION(TIMESTAMP): class NTEXT(sqltypes.UnicodeText): - """MSSQL NTEXT type, for variable-length unicode text up to 2^30 characters.""" @@ -1557,36 +1556,26 @@ class MSUUid(sqltypes.Uuid): return process def _sentinel_value_resolver(self, dialect): - """Return a callable that will receive the uuid object or string - as it is normally passed to the DB in the parameter set, after - bind_processor() is called. Convert this value to match - what it would be as coming back from an INSERT..OUTPUT inserted. + if not self.native_uuid: + # dealing entirely with strings going in and out of + # CHAR(32) + return None - for the UUID type, there are four varieties of settings so here - we seek to convert to the string or UUID representation that comes - back from the driver. - - """ - character_based_uuid = ( - not dialect.supports_native_uuid or not self.native_uuid - ) + # true if we expect the returned UUID values to be strings + # pymssql sends UUID objects back, pyodbc sends strings, + # however pyodbc converts them to uppercase coming back, so + # need special logic here + character_based_uuid = not dialect.supports_native_uuid if character_based_uuid: - if self.native_uuid: - # for pyodbc, uuid.uuid() objects are accepted for incoming - # data, as well as strings. but the driver will always return - # uppercase strings in result sets. - def process(value): - return str(value).upper() - - else: - - def process(value): - return str(value) + # we sent UUID objects in all cases, see bind_processor() + def process(uuid_value): + return str(uuid_value).upper() return process + elif not self.as_uuid: + return _python_UUID else: - # for pymssql, we get uuid.uuid() objects back. return None @@ -2483,10 +2472,12 @@ class MSSQLCompiler(compiler.SQLCompiler): type_expression = "ELSE CAST(JSON_VALUE(%s, %s) AS %s)" % ( self.process(binary.left, **kw), self.process(binary.right, **kw), - "FLOAT" - if isinstance(binary.type, sqltypes.Float) - else "NUMERIC(%s, %s)" - % (binary.type.precision, binary.type.scale), + ( + "FLOAT" + if isinstance(binary.type, sqltypes.Float) + else "NUMERIC(%s, %s)" + % (binary.type.precision, binary.type.scale) + ), ) elif binary.type._type_affinity is sqltypes.Boolean: # the NULL handling is particularly weird with boolean, so @@ -2522,7 +2513,6 @@ class MSSQLCompiler(compiler.SQLCompiler): class MSSQLStrictCompiler(MSSQLCompiler): - """A subclass of MSSQLCompiler which disables the usage of bind parameters where not allowed natively by MS-SQL. diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 6c82bab831..e148ff6052 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -5747,7 +5747,6 @@ class SQLCompiler(Compiled): returning_cols = self.implicit_returning or insert_stmt._returning if returning_cols: add_sentinel_cols = crud_params_struct.use_sentinel_columns - if add_sentinel_cols is not None: assert use_insertmanyvalues @@ -7052,6 +7051,9 @@ class GenericTypeCompiler(TypeCompiler): def visit_TEXT(self, type_, **kw): return self._render_string_type(type_, "TEXT") + def visit_UUID(self, type_, **kw): + return "UUID" + def visit_BLOB(self, type_, **kw): return "BLOB" @@ -7065,7 +7067,10 @@ class GenericTypeCompiler(TypeCompiler): return "BOOLEAN" def visit_uuid(self, type_, **kw): - return self._render_string_type(type_, "CHAR", length_override=32) + if not type_.native_uuid or not self.dialect.supports_native_uuid: + return self._render_string_type(type_, "CHAR", length_override=32) + else: + return self.visit_UUID(type_, **kw) def visit_large_binary(self, type_, **kw): return self.visit_BLOB(type_, **kw) diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 0963e8ed20..b359fe97bd 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -3723,6 +3723,31 @@ class Uuid(Emulated, TypeEngine[_UUID_RETURN]): return process + def _sentinel_value_resolver(self, dialect): + """For the "insertmanyvalues" feature only, return a callable that + will receive the uuid object or string + as it is normally passed to the DB in the parameter set, after + bind_processor() is called. Convert this value to match + what it would be as coming back from a RETURNING or similar + statement for the given backend. + + Individual dialects and drivers may need their own implementations + based on how their UUID types send data and how the drivers behave + (e.g. pyodbc) + + """ + if not self.native_uuid or not dialect.supports_native_uuid: + # dealing entirely with strings going in and out of + # CHAR(32) + return None + + elif self.as_uuid: + # we sent UUID objects and we are getting UUID objects back + return None + else: + # we sent strings and we are getting UUID objects back + return _python_UUID + class UUID(Uuid[_UUID_RETURN], type_api.NativeForEmulated): diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index 31aac741d4..c5dc52be88 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -62,7 +62,10 @@ class SuiteRequirements(Requirements): def uuid_data_type(self): """Return databases that support the UUID datatype.""" - return exclusions.closed() + return exclusions.skip_if( + lambda config: not config.db.dialect.supports_native_uuid, + "backend does not have a UUID datatype", + ) @property def foreign_keys(self): diff --git a/lib/sqlalchemy/testing/suite/test_insert.py b/lib/sqlalchemy/testing/suite/test_insert.py index cc30945cab..09e9473365 100644 --- a/lib/sqlalchemy/testing/suite/test_insert.py +++ b/lib/sqlalchemy/testing/suite/test_insert.py @@ -551,6 +551,12 @@ class ReturningTest(fixtures.TablesTest): uuid.uuid4(), testing.requires.uuid_data_type, ), + ( + "generic_native_uuid_str", + Uuid(as_uuid=False, native_uuid=True), + str(uuid.uuid4()), + testing.requires.uuid_data_type, + ), ("UUID", UUID(), uuid.uuid4(), testing.requires.uuid_data_type), ( "LargeBinary1", diff --git a/test/sql/test_insert_exec.py b/test/sql/test_insert_exec.py index e9eda0e5bd..b60c5cfec9 100644 --- a/test/sql/test_insert_exec.py +++ b/test/sql/test_insert_exec.py @@ -1445,6 +1445,7 @@ class IMVSentinelTest(fixtures.TestBase): (ARRAY(Integer()), testing.requires.array_type), DateTime(), Uuid(), + Uuid(native_uuid=False), argnames="datatype", ) def test_inserts_w_all_nulls( @@ -1987,6 +1988,8 @@ class IMVSentinelTest(fixtures.TestBase): "return_type", ["include_sentinel", "default_only", "return_defaults"] ) @testing.variation("add_sentinel_flag_to_col", [True, False]) + @testing.variation("native_uuid", [True, False]) + @testing.variation("as_uuid", [True, False]) def test_sentinel_on_non_autoinc_primary_key( self, metadata, @@ -1995,8 +1998,13 @@ class IMVSentinelTest(fixtures.TestBase): sort_by_parameter_order, randomize_returning, add_sentinel_flag_to_col, + native_uuid, + as_uuid, ): uuids = [uuid.uuid4() for i in range(10)] + if not as_uuid: + uuids = [str(u) for u in uuids] + _some_uuids = iter(uuids) t1 = Table( @@ -2004,7 +2012,7 @@ class IMVSentinelTest(fixtures.TestBase): metadata, Column( "id", - Uuid(), + Uuid(native_uuid=bool(native_uuid), as_uuid=bool(as_uuid)), default=functools.partial(next, _some_uuids), primary_key=True, insert_sentinel=bool(add_sentinel_flag_to_col), @@ -2096,6 +2104,8 @@ class IMVSentinelTest(fixtures.TestBase): else: return_type.fail() + @testing.variation("native_uuid", [True, False]) + @testing.variation("as_uuid", [True, False]) def test_client_composite_pk( self, metadata, @@ -2103,15 +2113,19 @@ class IMVSentinelTest(fixtures.TestBase): randomize_returning, sort_by_parameter_order, warn_for_downgrades, + native_uuid, + as_uuid, ): uuids = [uuid.uuid4() for i in range(10)] + if not as_uuid: + uuids = [str(u) for u in uuids] t1 = Table( "data", metadata, Column( "id1", - Uuid(), + Uuid(as_uuid=bool(as_uuid), native_uuid=bool(native_uuid)), default=functools.partial(next, iter(uuids)), primary_key=True, ),