From: Mike Bayer Date: Thu, 7 May 2020 17:56:38 +0000 (-0400) Subject: Use cx_oracle.LONG_STRING /LONG_BINARY for CLOB/BLOB X-Git-Tag: rel_1_3_17~5^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=02a5abb745606021bfc7e56c4afdab13c7e8bd66;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Use cx_oracle.LONG_STRING /LONG_BINARY for CLOB/BLOB Changed the implementation of fetching CLOB and BLOB objects to use cx_Oracle's native implementation which fetches CLOB/BLOB objects inline with other result columns, rather than performing a separate fetch. As always, this can be disabled by setting auto_convert_lobs to False. As part of this change, the behavior of a CLOB that was given a blank string on INSERT now returns None on SELECT, which is now consistent with that of VARCHAR on Oracle. Fixes: #5314 Change-Id: I7b46c91704b6f5d6c157e083505dac6e0cb3ef6e (cherry picked from commit fb28e40b31797dc9ad72f11a8edd4f2eb555a36d) --- diff --git a/doc/build/changelog/unreleased_13/5314.rst b/doc/build/changelog/unreleased_13/5314.rst new file mode 100644 index 0000000000..a8ff9d30c2 --- /dev/null +++ b/doc/build/changelog/unreleased_13/5314.rst @@ -0,0 +1,13 @@ +.. change:: + :tags: bug, oracle, performance + :tickets: 5314 + + Changed the implementation of fetching CLOB and BLOB objects to use + cx_Oracle's native implementation which fetches CLOB/BLOB objects inline + with other result columns, rather than performing a separate fetch. As + always, this can be disabled by setting auto_convert_lobs to False. + + As part of this change, the behavior of a CLOB that was given a blank + string on INSERT now returns None on SELECT, which is now consistent with + that of VARCHAR on Oracle. + diff --git a/lib/sqlalchemy/dialects/oracle/cx_oracle.py b/lib/sqlalchemy/dialects/oracle/cx_oracle.py index 586fe7c97f..1068052da5 100644 --- a/lib/sqlalchemy/dialects/oracle/cx_oracle.py +++ b/lib/sqlalchemy/dialects/oracle/cx_oracle.py @@ -1032,17 +1032,16 @@ class OracleDialect_cx_oracle(OracleDialect): dialect.encoding, errors=dialect.encoding_errors ) return cursor.var( - default_type, + cx_Oracle.LONG_STRING, size, cursor.arraysize, - outconverter=lambda value: outconverter(value.read()), + outconverter=outconverter, ) else: return cursor.var( - default_type, + cx_Oracle.LONG_STRING, size, cursor.arraysize, - outconverter=lambda value: value.read(), **dialect._cursor_var_unicode_kwargs ) @@ -1050,10 +1049,7 @@ class OracleDialect_cx_oracle(OracleDialect): cx_Oracle.BLOB, ): return cursor.var( - default_type, - size, - cursor.arraysize, - outconverter=lambda value: value.read(), + cx_Oracle.LONG_BINARY, size, cursor.arraysize, ) return output_type_handler diff --git a/lib/sqlalchemy/testing/suite/test_types.py b/lib/sqlalchemy/testing/suite/test_types.py index a334b8ebcd..e3f362f320 100644 --- a/lib/sqlalchemy/testing/suite/test_types.py +++ b/lib/sqlalchemy/testing/suite/test_types.py @@ -134,11 +134,22 @@ class _UnicodeFixture(_LiteralRoundTripFixture): for row in rows: assert isinstance(row[0], util.text_type) - def _test_empty_strings(self): + def _test_null_strings(self, connection): unicode_table = self.tables.unicode_table - config.db.execute(unicode_table.insert(), {"unicode_data": u("")}) - row = config.db.execute(select([unicode_table.c.unicode_data])).first() + connection.execute(unicode_table.insert(), {"unicode_data": None}) + row = connection.execute( + select([unicode_table.c.unicode_data]) + ).first() + eq_(row, (None,)) + + def _test_empty_strings(self, connection): + unicode_table = self.tables.unicode_table + + connection.execute(unicode_table.insert(), {"unicode_data": u("")}) + row = connection.execute( + select([unicode_table.c.unicode_data]) + ).first() eq_(row, (u(""),)) def test_literal(self): @@ -157,8 +168,11 @@ class UnicodeVarcharTest(_UnicodeFixture, fixtures.TablesTest): datatype = Unicode(255) @requirements.empty_strings_varchar - def test_empty_strings_varchar(self): - self._test_empty_strings() + def test_empty_strings_varchar(self, connection): + self._test_empty_strings(connection) + + def test_null_strings_varchar(self, connection): + self._test_null_strings(connection) class UnicodeTextTest(_UnicodeFixture, fixtures.TablesTest): @@ -168,8 +182,11 @@ class UnicodeTextTest(_UnicodeFixture, fixtures.TablesTest): datatype = UnicodeText() @requirements.empty_strings_text - def test_empty_strings_text(self): - self._test_empty_strings() + def test_empty_strings_text(self, connection): + self._test_empty_strings(connection) + + def test_null_strings_text(self, connection): + self._test_null_strings(connection) class TextTest(_LiteralRoundTripFixture, fixtures.TablesTest): @@ -198,13 +215,21 @@ class TextTest(_LiteralRoundTripFixture, fixtures.TablesTest): row = config.db.execute(select([text_table.c.text_data])).first() eq_(row, ("some text",)) - def test_text_empty_strings(self): + @testing.requires.empty_strings_text + def test_text_empty_strings(self, connection): text_table = self.tables.text_table - config.db.execute(text_table.insert(), {"text_data": ""}) - row = config.db.execute(select([text_table.c.text_data])).first() + connection.execute(text_table.insert(), {"text_data": ""}) + row = connection.execute(select([text_table.c.text_data])).first() eq_(row, ("",)) + def test_text_null_strings(self, connection): + text_table = self.tables.text_table + + connection.execute(text_table.insert(), {"text_data": None}) + row = connection.execute(select([text_table.c.text_data])).first() + eq_(row, (None,)) + def test_literal(self): self._literal_round_trip(Text, ["some text"], ["some text"]) diff --git a/test/dialect/oracle/test_dialect.py b/test/dialect/oracle/test_dialect.py index e226ca7fbb..181667e63f 100644 --- a/test/dialect/oracle/test_dialect.py +++ b/test/dialect/oracle/test_dialect.py @@ -103,24 +103,19 @@ class EncodingErrorsTest(fixtures.TestBase): ) _oracle_char_combinations = testing.combinations( - ("STRING", cx_Oracle_STRING, False), - ("FIXED_CHAR", cx_Oracle_FIXED_CHAR, False), - ("CLOB", cx_Oracle_CLOB, True), - ("NCLOB", cx_Oracle_NCLOB, True), - argnames="cx_oracle_type,use_read", - id_="iaa", + ("STRING", cx_Oracle_STRING,), + ("FIXED_CHAR", cx_Oracle_FIXED_CHAR,), + ("CLOB", cx_Oracle_CLOB,), + ("NCLOB", cx_Oracle_NCLOB,), + argnames="cx_oracle_type", + id_="ia", ) - def _assert_errorhandler(self, outconverter, use_read, has_errorhandler): + def _assert_errorhandler(self, outconverter, has_errorhandler): data = ue("\uee2c\u9a66") # this is u"\uee2c\u9a66" utf8_w_errors = data.encode("utf-16") - if use_read: - utf8_w_errors = mock.Mock( - read=mock.Mock(return_value=utf8_w_errors) - ) - if has_errorhandler: eq_( @@ -132,9 +127,7 @@ class EncodingErrorsTest(fixtures.TestBase): @_oracle_char_combinations @testing.requires.python3 - def test_older_cx_oracle_warning( - self, cx_Oracle, cx_oracle_type, use_read - ): + def test_older_cx_oracle_warning(self, cx_Oracle, cx_oracle_type): cx_Oracle.version = "6.3" ignore_dialect = cx_oracle.dialect( @@ -156,7 +149,7 @@ class EncodingErrorsTest(fixtures.TestBase): @_oracle_char_combinations @testing.requires.python2 def test_encoding_errors_sqla_py2k( - self, cx_Oracle, cx_oracle_type, use_read + self, cx_Oracle, cx_oracle_type, ): ignore_dialect = cx_oracle.dialect( dbapi=cx_Oracle, encoding_errors="ignore" @@ -169,12 +162,12 @@ class EncodingErrorsTest(fixtures.TestBase): cursor = mock.Mock() ignore_outputhandler(cursor, "foo", cx_oracle_type, None, None, None) outconverter = cursor.mock_calls[0][2]["outconverter"] - self._assert_errorhandler(outconverter, use_read, True) + self._assert_errorhandler(outconverter, True) @_oracle_char_combinations @testing.requires.python2 def test_no_encoding_errors_sqla_py2k( - self, cx_Oracle, cx_oracle_type, use_read + self, cx_Oracle, cx_oracle_type, ): plain_dialect = cx_oracle.dialect(dbapi=cx_Oracle) @@ -185,12 +178,12 @@ class EncodingErrorsTest(fixtures.TestBase): cursor = mock.Mock() plain_outputhandler(cursor, "foo", cx_oracle_type, None, None, None) outconverter = cursor.mock_calls[0][2]["outconverter"] - self._assert_errorhandler(outconverter, use_read, False) + self._assert_errorhandler(outconverter, False) @_oracle_char_combinations @testing.requires.python3 def test_encoding_errors_cx_oracle_py3k( - self, cx_Oracle, cx_oracle_type, use_read + self, cx_Oracle, cx_oracle_type, ): ignore_dialect = cx_oracle.dialect( dbapi=cx_Oracle, encoding_errors="ignore" @@ -203,36 +196,19 @@ class EncodingErrorsTest(fixtures.TestBase): cursor = mock.Mock() ignore_outputhandler(cursor, "foo", cx_oracle_type, None, None, None) - if use_read: - eq_( - cursor.mock_calls, - [ - mock.call.var( - mock.ANY, - None, - cursor.arraysize, - encodingErrors="ignore", - outconverter=mock.ANY, - ) - ], - ) - else: - eq_( - cursor.mock_calls, - [ - mock.call.var( - mock.ANY, - None, - cursor.arraysize, - encodingErrors="ignore", - ) - ], - ) + eq_( + cursor.mock_calls, + [ + mock.call.var( + mock.ANY, None, cursor.arraysize, encodingErrors="ignore", + ) + ], + ) @_oracle_char_combinations @testing.requires.python3 def test_no_encoding_errors_cx_oracle_py3k( - self, cx_Oracle, cx_oracle_type, use_read + self, cx_Oracle, cx_oracle_type, ): plain_dialect = cx_oracle.dialect(dbapi=cx_Oracle) @@ -243,20 +219,10 @@ class EncodingErrorsTest(fixtures.TestBase): cursor = mock.Mock() plain_outputhandler(cursor, "foo", cx_oracle_type, None, None, None) - if use_read: - eq_( - cursor.mock_calls, - [ - mock.call.var( - mock.ANY, None, cursor.arraysize, outconverter=mock.ANY - ) - ], - ) - else: - eq_( - cursor.mock_calls, - [mock.call.var(mock.ANY, None, cursor.arraysize)], - ) + eq_( + cursor.mock_calls, + [mock.call.var(mock.ANY, None, cursor.arraysize)], + ) class ComputedReturningTest(fixtures.TablesTest): diff --git a/test/requirements.py b/test/requirements.py index ea6eee6fb8..d61b8ec524 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -741,7 +741,9 @@ class DefaultRequirements(SuiteRequirements): """target database can persist/return an empty string with an unbounded text.""" - return exclusions.open() + return fails_if( + ["oracle"], "oracle converts empty strings to a blank space" + ) @property def expressions_against_unbounded_text(self):