allow executemany values for ON CONFLICT DO NOTHING

Scaled back a fix made for :ticket:`6581` where "executemany values" mode
for psycopg2 were disabled for all "ON CONFLICT" styles of INSERT, to
not apply to the "ON CONFLICT DO NOTHING" clause, which does not include
any parameters and is safe for "executemany values" mode.  "ON CONFLICT
DO UPDATE" is still blocked from "executemany values" as there may
be additional parameters in the DO UPDATE clause that cannot be batched
(which is the original issue fixed by :ticket:`6581`).

Fixes: #7880
Change-Id: Id3e23a0c6699333409a50148fa8923cb8e564bdc
(cherry picked from commit cc8c5835a92b0035530b541c81b0c714b570b095)

diff --git a/doc/build/changelog/unreleased_14/7880.rst b/doc/build/changelog/unreleased_14/7880.rst
new file mode 100644 (file)
index 0000000..9abbac1
--- /dev/null
+++ b/doc/build/changelog/unreleased_14/7880.rst
@@ -0,0 +1,11 @@
+.. change::
+    :tags: bug, postgresql
+    :tickets: 7880
+
+    Scaled back a fix made for :ticket:`6581` where "executemany values" mode
+    for psycopg2 were disabled for all "ON CONFLICT" styles of INSERT, to
+    not apply to the "ON CONFLICT DO NOTHING" clause, which does not include
+    any parameters and is safe for "executemany values" mode.  "ON CONFLICT
+    DO UPDATE" is still blocked from "executemany values" as there may
+    be additional parameters in the DO UPDATE clause that cannot be batched
+    (which is the original issue fixed by :ticket:`6581`).

diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py
index 37b2113e9896f65fb9dea02ee064854c8931d68a..4f63002bc9bc897b2357fa3117d80af5aa0e8a40 100644 (file)
--- a/lib/sqlalchemy/dialects/postgresql/base.py
+++ b/lib/sqlalchemy/dialects/postgresql/base.py
@@ -1429,6 +1429,7 @@ import re
 from uuid import UUID as _python_UUID
 
 from . import array as _array
+from . import dml
 from . import hstore as _hstore
 from . import json as _json
 from . import ranges as _ranges
@@ -2463,6 +2464,24 @@ class PGCompiler(compiler.SQLCompiler):
 
         return target_text
 
+    @util.memoized_property
+    def _is_safe_for_fast_insert_values_helper(self):
+        # don't allow fast executemany if _post_values_clause is
+        # present and is not an OnConflictDoNothing. what this means
+        # concretely is that the
+        # "fast insert executemany helper" won't be used, in other
+        # words we won't convert "executemany()" of many parameter
+        # sets into a single INSERT with many elements in VALUES.
+        # We can't apply that optimization safely if for example the
+        # statement includes a clause like "ON CONFLICT DO UPDATE"
+
+        return self.insert_single_values_expr is not None and (
+            self.statement._post_values_clause is None
+            or isinstance(
+                self.statement._post_values_clause, dml.OnConflictDoNothing
+            )
+        )
+
     def visit_on_conflict_do_nothing(self, on_conflict, **kw):
 
         target_text = self._on_conflict_target(on_conflict, **kw)

diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
index 19d7b06ac9f00bc4140316360629195387b3f4f5..adebc9b67693ad2dd9c4e1ba1c23e4b483a997e4 100644 (file)
--- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py
+++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
@@ -927,7 +927,7 @@ class PGDialect_psycopg2(PGDialect):
             self.executemany_mode & EXECUTEMANY_VALUES
             and context
             and context.isinsert
-            and context.compiled.insert_single_values_expr
+            and context.compiled._is_safe_for_fast_insert_values_helper
         ):
             executemany_values = (
                 "(%s)" % context.compiled.insert_single_values_expr

diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index 671ca6749242234729e5a054ac9b189baff5e6e7..7393629a4067e8cac1cc3efd27a5283e49791d80 100644 (file)
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -3930,14 +3930,7 @@ class SQLCompiler(Compiled):
                 [value for c, expr, value in crud_params]
             )
             text += " VALUES (%s)" % insert_single_values_expr
-            if toplevel and insert_stmt._post_values_clause is None:
-                # don't assign insert_single_values_expr if _post_values_clause
-                # is present.  what this means concretely is that the
-                # "fast insert executemany helper" won't be used, in other
-                # words we won't convert "executemany()" of many parameter
-                # sets into a single INSERT with many elements in VALUES.
-                # We can't apply that optimization safely if for example the
-                # statement includes a clause like "ON CONFLICT DO UPDATE"
+            if toplevel:
                 self.insert_single_values_expr = insert_single_values_expr
 
         if insert_stmt._post_values_clause is not None:

diff --git a/test/dialect/postgresql/test_dialect.py b/test/dialect/postgresql/test_dialect.py
index c0eb4410cf972fd3dd92312788df304977ab8e0d..1d797a697d72d5cc880c39e826b511f49c783d31 100644 (file)
--- a/test/dialect/postgresql/test_dialect.py
+++ b/test/dialect/postgresql/test_dialect.py
@@ -30,6 +30,7 @@ from sqlalchemy import text
 from sqlalchemy import TypeDecorator
 from sqlalchemy import util
 from sqlalchemy.dialects.postgresql import base as postgresql
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.dialects.postgresql import psycopg2 as psycopg2_dialect
 from sqlalchemy.dialects.postgresql.psycopg2 import EXECUTEMANY_BATCH
 from sqlalchemy.dialects.postgresql.psycopg2 import EXECUTEMANY_PLAIN
@@ -357,7 +358,10 @@ class ExecuteManyMode(object):
             Column(ue("\u6e2c\u8a66"), Integer),
         )
 
-    def test_insert(self, connection):
+    @testing.combinations(
+        "insert", "pg_insert", "pg_insert_on_conflict", argnames="insert_type"
+    )
+    def test_insert(self, connection, insert_type):
         from psycopg2 import extras
 
         values_page_size = connection.dialect.executemany_values_page_size
@@ -377,11 +381,23 @@ class ExecuteManyMode(object):
         else:
             assert False
 
+        if insert_type == "pg_insert_on_conflict":
+            stmt += " ON CONFLICT DO NOTHING"
+
         with mock.patch.object(
             extras, meth.__name__, side_effect=meth
         ) as mock_exec:
+            if insert_type == "insert":
+                ins_stmt = self.tables.data.insert()
+            elif insert_type == "pg_insert":
+                ins_stmt = pg_insert(self.tables.data)
+            elif insert_type == "pg_insert_on_conflict":
+                ins_stmt = pg_insert(self.tables.data).on_conflict_do_nothing()
+            else:
+                assert False
+
             connection.execute(
-                self.tables.data.insert(),
+                ins_stmt,
                 [
                     {"x": "x1", "y": "y1"},
                     {"x": "x2", "y": "y2"},