gcn: Fix CDNA3 atomics' buffer invalidation

For device (agent) scope atomics - as needed when there is more than one teams,
a buffer_wbl2 followed by s_waitcnt is required. When doing the initial porting,
the pre-atomic instruction got accidentally replaced by buffer_inv sc1, which is
not quite the right instruction.

gcc/ChangeLog:

	* config/gcn/gcn.md (atomic_load, atomic_store, atomic_exchange):
	Fix CDNA3 L2 cache write-back before atomic instructions.

diff --git a/gcc/config/gcn/gcn.md b/gcc/config/gcn/gcn.md
index 67a45edba363674ac0a6f015b0371ffb37a4c932..4130cf63dbc5ac826e73361d4acc7d27185165b9 100644 (file)
--- a/gcc/config/gcn/gcn.md
+++ b/gcc/config/gcn/gcn.md
@@ -2183,7 +2183,7 @@
                    ? "buffer_gl1_inv\;buffer_gl0_inv\;flat_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\t0\;buffer_gl1_inv\;buffer_gl0_inv"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;flat_load%o0\t%0, %A1%O1 %G1\;"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\t0\;buffer_inv sc1"
                    : "buffer_wbinvl1_vol\;flat_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\t0\;buffer_wbinvl1_vol");
@@ -2195,7 +2195,7 @@
                    ? "buffer_gl1_inv\;buffer_gl0_inv\;global_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_gl1_inv\;buffer_gl0_inv"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;global_load%o0\t%0, %A1%O1 %G1\;"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
                    : "buffer_wbinvl1_vol\;global_load%o0\t%0, %A1%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol");
@@ -2242,7 +2242,7 @@
                    : TARGET_WBINVL1_CACHE
                    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_store%o1\t%A0, %1%O0 %G1"
                    : "error: cache architectire unspecified");
          case 2:
            return (TARGET_GLn_CACHE
@@ -2250,7 +2250,7 @@
                    : TARGET_WBINVL1_CACHE
                    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_store%o1\t%A0, %1%O0 %G1"
                    : "error: cache architecture unspecified");
          }
        break;
@@ -2270,7 +2270,8 @@
                    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1\;"
                      "s_waitcnt\t0\;buffer_wbinvl1_vol"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1\;"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;"
+                     "flat_store%o1\t%A0, %1%O0 %G1\;"
                      "s_waitcnt\t0\;buffer_inv sc1"
                    : "error: cache architecture unspecified");
          case 2:
@@ -2281,7 +2282,8 @@
                    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
                    : TARGET_TARGET_SC_CACHE
-                   ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1\;"
+                   ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
+                     "global_store%o1\t%A0, %1%O0 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
                    : "error: cache architecture unspecified");
          }
@@ -2365,7 +2367,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
                      "s_waitcnt\t0"
            : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
                      "s_waitcnt\t0"
             : "error: cache architecture unspecified");
          case 2:
@@ -2378,7 +2380,7 @@
                      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)"
            : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
                      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)"
             : "error: cache architecture unspecified");
@@ -2400,7 +2402,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
                      "s_waitcnt\t0\;buffer_wbinvl1_vol"
            : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
                      "s_waitcnt\t0\;buffer_inv sc1"
             : "error: cache architecture unspecified");
          case 2:
@@ -2413,7 +2415,7 @@
                      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
            : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
                      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
                      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
             : "error: cache architecture unspecified");