From: Mike Bayer Date: Thu, 28 Jan 2021 16:04:29 +0000 (-0500) Subject: Allow Oracle CLOB/NCLOB/BLOB in returning X-Git-Tag: rel_1_3_23~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=893c4e7f89ae4018f286216e1cfc1b1b9341c939;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Allow Oracle CLOB/NCLOB/BLOB in returning Fixed bug in Oracle dialect where retriving a CLOB/BLOB column via :meth:`_dml.Insert.returning` would fail as the LOB value would need to be read when returned; additionally, repaired support for retrieval of Unicode values via RETURNING under Python 2. As of yet, we still don't know how to reproduce the ORA-24813 error indicated in the issue. Also backporting the statement cache clear added to master in f1e96cb087 , as we are testing in CI against two oracle versions now there are sporadic failures that appear to be memory related. Fixes: #5812 Change-Id: I666f893e762dfa4d34dd2e324480565b226fb3a4 (cherry picked from commit 03179a96bfb9dd7ce17274fed44908c25229dedf) --- diff --git a/doc/build/changelog/unreleased_13/5812.rst b/doc/build/changelog/unreleased_13/5812.rst new file mode 100644 index 0000000000..e354a8ce47 --- /dev/null +++ b/doc/build/changelog/unreleased_13/5812.rst @@ -0,0 +1,8 @@ +.. change:: + :tags: bug, oracle + :tickets: 5812 + + Fixed bug in Oracle dialect where retriving a CLOB/BLOB column via + :meth:`_dml.Insert.returning` would fail as the LOB value would need to be + read when returned; additionally, repaired support for retrieval of Unicode + values via RETURNING under Python 2. \ No newline at end of file diff --git a/lib/sqlalchemy/dialects/oracle/cx_oracle.py b/lib/sqlalchemy/dialects/oracle/cx_oracle.py index 6ba824f109..b02f497060 100644 --- a/lib/sqlalchemy/dialects/oracle/cx_oracle.py +++ b/lib/sqlalchemy/dialects/oracle/cx_oracle.py @@ -514,7 +514,7 @@ class _OracleUnicodeStringNCHAR(oracle.NVARCHAR2): class _OracleUnicodeStringCHAR(sqltypes.Unicode): def get_dbapi_type(self, dbapi): - return None + return dbapi.LONG_STRING class _OracleUnicodeTextNCLOB(oracle.NCLOB): @@ -624,19 +624,63 @@ class OracleExecutionContext_cx_oracle(OracleExecutionContext): if bindparam.isoutparam: name = self.compiled.bind_names[bindparam] type_impl = bindparam.type.dialect_impl(self.dialect) + if hasattr(type_impl, "_cx_oracle_var"): self.out_parameters[name] = type_impl._cx_oracle_var( self.dialect, self.cursor ) else: dbtype = type_impl.get_dbapi_type(self.dialect.dbapi) + + cx_Oracle = self.dialect.dbapi + if dbtype is None: raise exc.InvalidRequestError( - "Cannot create out parameter for parameter " + "Cannot create out parameter for " + "parameter " "%r - its type %r is not supported by" " cx_oracle" % (bindparam.key, bindparam.type) ) - self.out_parameters[name] = self.cursor.var(dbtype) + + if compat.py2k and dbtype in ( + cx_Oracle.CLOB, + cx_Oracle.NCLOB, + ): + outconverter = ( + processors.to_unicode_processor_factory( + self.dialect.encoding, + errors=self.dialect.encoding_errors, + ) + ) + self.out_parameters[name] = self.cursor.var( + dbtype, + outconverter=lambda value: outconverter( + value.read() + ), + ) + + elif dbtype in ( + cx_Oracle.BLOB, + cx_Oracle.CLOB, + cx_Oracle.NCLOB, + ): + self.out_parameters[name] = self.cursor.var( + dbtype, outconverter=lambda value: value.read() + ) + elif compat.py2k and isinstance( + type_impl, sqltypes.Unicode + ): + outconverter = ( + processors.to_unicode_processor_factory( + self.dialect.encoding, + errors=self.dialect.encoding_errors, + ) + ) + self.out_parameters[name] = self.cursor.var( + dbtype, outconverter=outconverter + ) + else: + self.out_parameters[name] = self.cursor.var(dbtype) self.parameters[0][ quoted_bind_names.get(name, name) ] = self.out_parameters[name] diff --git a/lib/sqlalchemy/dialects/oracle/provision.py b/lib/sqlalchemy/dialects/oracle/provision.py index bf6116b07a..539725ddab 100644 --- a/lib/sqlalchemy/dialects/oracle/provision.py +++ b/lib/sqlalchemy/dialects/oracle/provision.py @@ -6,6 +6,7 @@ from ...testing.provision import create_db from ...testing.provision import drop_db from ...testing.provision import follower_url_from_main from ...testing.provision import log +from ...testing.provision import post_configure_engine from ...testing.provision import run_reap_dbs from ...testing.provision import stop_test_class from ...testing.provision import temp_table_keyword_args @@ -72,6 +73,32 @@ def stop_test_class(config, db, cls): with db.begin() as conn: conn.execute("purge recyclebin") + # clear statement cache on all connections that were used + # https://github.com/oracle/python-cx_Oracle/issues/519 + + for cx_oracle_conn in _all_conns: + try: + sc = cx_oracle_conn.stmtcachesize + except db.dialect.dbapi.InterfaceError: + # connection closed + pass + else: + cx_oracle_conn.stmtcachesize = 0 + cx_oracle_conn.stmtcachesize = sc + _all_conns.clear() + + +_all_conns = set() + + +@post_configure_engine.for_db("oracle") +def _oracle_post_configure_engine(url, engine, follower_ident): + from sqlalchemy import event + + @event.listens_for(engine, "checkout") + def checkout(dbapi_con, con_record, con_proxy): + _all_conns.add(dbapi_con) + @run_reap_dbs.for_db("oracle") def _reap_oracle_dbs(url, idents): diff --git a/test/dialect/oracle/test_types.py b/test/dialect/oracle/test_types.py index 4c61f08b44..59bed64f8a 100644 --- a/test/dialect/oracle/test_types.py +++ b/test/dialect/oracle/test_types.py @@ -4,6 +4,7 @@ import datetime import decimal import os +import random from sqlalchemy import bindparam from sqlalchemy import cast @@ -999,7 +1000,76 @@ class LOBFetchTest(fixtures.TablesTest): self.data, ) - def test_large_stream(self): + @testing.combinations( + (UnicodeText(),), (Text(),), (LargeBinary(),), argnames="datatype" + ) + @testing.combinations((10,), (100,), (250,), argnames="datasize") + @testing.combinations( + ("x,y,z"), ("y"), ("y,x,z"), ("x,z,y"), argnames="retcols" + ) + @testing.provide_metadata + def test_insert_returning_w_lobs( + self, datatype, datasize, retcols, connection + ): + metadata = self.metadata + long_text = Table( + "long_text", + metadata, + Column("x", Integer), + Column("y", datatype), + Column("z", Integer), + ) + long_text.create(connection) + + if isinstance(datatype, UnicodeText): + word_seed = u"ab🐍’«cdefg" + else: + word_seed = "abcdef" + + some_text = u" ".join( + "".join(random.choice(word_seed) for j in range(150)) + for i in range(datasize) + ) + if isinstance(datatype, LargeBinary): + some_text = some_text.encode("ascii") + + data = {"x": 5, "y": some_text, "z": 10} + return_columns = [long_text.c[col] for col in retcols.split(",")] + expected = tuple(data[col] for col in retcols.split(",")) + result = connection.execute( + long_text.insert().returning(*return_columns), + data, + ) + + eq_(result.fetchall(), [expected]) + + @testing.provide_metadata + def test_insert_returning_w_unicode(self, connection): + metadata = self.metadata + long_text = Table( + "long_text", + metadata, + Column("x", Integer), + Column("y", Unicode(255)), + ) + long_text.create(connection) + + word_seed = u"ab🐍’«cdefg" + + some_text = u" ".join( + "".join(random.choice(word_seed) for j in range(10)) + for i in range(15) + ) + + data = {"x": 5, "y": some_text} + result = connection.execute( + long_text.insert().returning(long_text.c.y), + data, + ) + + eq_(result.fetchall(), [(some_text,)]) + + def test_large_stream(self, connection): binary_table = self.tables.binary_table result = ( binary_table.select()