AArch64: Fix __sync_val_compare_and_swap [PR111404]

__sync_val_compare_and_swap may be used on 128-bit types and either calls the
outline atomic code or uses an inline loop.  On AArch64 LDXP is only atomic if
the value is stored successfully using STXP, but the current implementations
do not perform the store if the comparison fails.  In this case the value
returned is not read atomically.

gcc/ChangeLog:
	PR target/111404
	* config/aarch64/aarch64.cc (aarch64_split_compare_and_swap):
	For 128-bit store the loaded value and loop if needed.

libgcc/ChangeLog:
	PR target/111404
	* config/aarch64/lse.S (__aarch64_cas16_acq_rel): Execute STLXP using
	either new value or loaded value.

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index bc126cc60fc3cf9556b0f83e725f27df08f4ae7f..bde21f7f7f5e5572d705c8397836904f6ca47814 100644 (file)
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -21272,11 +21272,11 @@ aarch64_split_compare_and_swap (rtx operands[])
   mem = operands[1];
   oldval = operands[2];
   newval = operands[3];
-  is_weak = (operands[4] != const0_rtx);
   model_rtx = operands[5];
   scratch = operands[7];
   mode = GET_MODE (mem);
   model = memmodel_from_int (INTVAL (model_rtx));
+  is_weak = operands[4] != const0_rtx && mode != TImode;
 
   /* When OLDVAL is zero and we want the strong version we can emit a tighter
     loop:
@@ -21337,6 +21337,33 @@ aarch64_split_compare_and_swap (rtx operands[])
   else
     aarch64_gen_compare_reg (NE, scratch, const0_rtx);
 
+  /* 128-bit LDAXP is not atomic unless STLXP succeeds.  So for a mismatch,
+     store the returned value and loop if the STLXP fails.  */
+  if (mode == TImode)
+    {
+      rtx_code_label *label3 = gen_label_rtx ();
+      emit_jump_insn (gen_rtx_SET (pc_rtx, gen_rtx_LABEL_REF (Pmode, label3)));
+      emit_barrier ();
+
+      emit_label (label2);
+      aarch64_emit_store_exclusive (mode, scratch, mem, rval, model_rtx);
+
+      if (aarch64_track_speculation)
+       {
+         /* Emit an explicit compare instruction, so that we can correctly
+            track the condition codes.  */
+         rtx cc_reg = aarch64_gen_compare_reg (NE, scratch, const0_rtx);
+         x = gen_rtx_NE (GET_MODE (cc_reg), cc_reg, const0_rtx);
+       }
+      else
+       x = gen_rtx_NE (VOIDmode, scratch, const0_rtx);
+      x = gen_rtx_IF_THEN_ELSE (VOIDmode, x,
+                               gen_rtx_LABEL_REF (Pmode, label1), pc_rtx);
+      aarch64_emit_unlikely_jump (gen_rtx_SET (pc_rtx, x));
+
+      label2 = label3;
+    }
+
   emit_label (label2);
 
   /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL

diff --git a/libgcc/config/aarch64/lse.S b/libgcc/config/aarch64/lse.S
index f64a3e98bfaee72a473d094764cde08446c0152b..046dd40659d9d4294e3304e42ed3b7b4a568a8dc 100644 (file)
--- a/libgcc/config/aarch64/lse.S
+++ b/libgcc/config/aarch64/lse.S
@@ -163,6 +163,8 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define tmp0   16
 #define tmp1   17
 #define tmp2   15
+#define tmp3   14
+#define tmp4   13
 
 /* Start and end a function.  */
 .macro STARTFN name
@@ -233,10 +235,11 @@ STARTFN   NAME(cas)
 0:     LDXP            x0, x1, [x4]
        cmp             x0, x(tmp0)
        ccmp            x1, x(tmp1), #0, eq
-       bne             1f
-       STXP            w(tmp2), x2, x3, [x4]
-       cbnz            w(tmp2), 0b
-1:     BARRIER
+       csel            x(tmp2), x2, x0, eq
+       csel            x(tmp3), x3, x1, eq
+       STXP            w(tmp4), x(tmp2), x(tmp3), [x4]
+       cbnz            w(tmp4), 0b
+       BARRIER
        ret
 
 #endif