350 lines
12 KiB
Diff
350 lines
12 KiB
Diff
|
From 71506544eef580f59e5816f0a48a67aebbe5eed5 Mon Sep 17 00:00:00 2001
|
||
|
From: Patrick O'Neill <patrick@rivosinc.com>
|
||
|
Date: Wed, 5 Apr 2023 09:49:20 -0700
|
||
|
Subject: [PATCH] RISC-V: Weaken LR/SC pairs
|
||
|
|
||
|
Introduce the %I and %J flags for setting the .aqrl bits on LR/SC pairs
|
||
|
as needed.
|
||
|
|
||
|
Atomic compare and exchange ops provide success and failure memory
|
||
|
models. C++17 and later place no restrictions on the relative strength
|
||
|
of each model, so ensure we cover both by using a model that enforces
|
||
|
the ordering of both given models.
|
||
|
|
||
|
This change brings LR/SC ops in line with table A.6 of the ISA manual.
|
||
|
|
||
|
2023-04-27 Patrick O'Neill <patrick@rivosinc.com>
|
||
|
|
||
|
gcc/ChangeLog:
|
||
|
|
||
|
* config/riscv/riscv-protos.h (riscv_union_memmodels): Expose
|
||
|
riscv_union_memmodels function to sync.md.
|
||
|
* config/riscv/riscv.cc (riscv_union_memmodels): Add function to
|
||
|
get the union of two memmodels in sync.md.
|
||
|
(riscv_print_operand): Add %I and %J flags that output the
|
||
|
optimal LR/SC flag bits for a given memory model.
|
||
|
* config/riscv/sync.md: Remove static .aqrl bits on LR op/.rl
|
||
|
bits on SC op and replace with optimized %I, %J flags.
|
||
|
|
||
|
Signed-off-by: Patrick O'Neill <patrick@rivosinc.com>
|
||
|
---
|
||
|
gcc/config/riscv/riscv-protos.h | 3 +
|
||
|
gcc/config/riscv/riscv.cc | 44 ++++++++++++
|
||
|
gcc/config/riscv/sync.md | 114 +++++++++++++++++++-------------
|
||
|
3 files changed, 114 insertions(+), 47 deletions(-)
|
||
|
|
||
|
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
|
||
|
index 02b33e020207..b5616fb3e88c 100644
|
||
|
--- a/gcc/config/riscv/riscv-protos.h
|
||
|
+++ b/gcc/config/riscv/riscv-protos.h
|
||
|
@@ -22,6 +22,8 @@ along with GCC; see the file COPYING3. If not see
|
||
|
#ifndef GCC_RISCV_PROTOS_H
|
||
|
#define GCC_RISCV_PROTOS_H
|
||
|
|
||
|
+#include "memmodel.h"
|
||
|
+
|
||
|
/* Symbol types we understand. The order of this list must match that of
|
||
|
the unspec enum in riscv.md, subsequent to UNSPEC_ADDRESS_FIRST. */
|
||
|
enum riscv_symbol_type {
|
||
|
@@ -81,6 +83,7 @@ extern bool riscv_v_ext_vector_mode_p (machine_mode);
|
||
|
extern bool riscv_shamt_matches_mask_p (int, HOST_WIDE_INT);
|
||
|
extern void riscv_subword_address (rtx, rtx *, rtx *, rtx *, rtx *);
|
||
|
extern void riscv_lshift_subword (machine_mode, rtx, rtx, rtx *);
|
||
|
+extern enum memmodel riscv_union_memmodels (enum memmodel, enum memmodel);
|
||
|
|
||
|
/* Routines implemented in riscv-c.cc. */
|
||
|
void riscv_cpu_cpp_builtins (cpp_reader *);
|
||
|
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
|
||
|
index 951f6b5cf42d..59899268918d 100644
|
||
|
--- a/gcc/config/riscv/riscv.cc
|
||
|
+++ b/gcc/config/riscv/riscv.cc
|
||
|
@@ -4284,6 +4284,36 @@ riscv_print_operand_reloc (FILE *file, rtx op, bool hi_reloc)
|
||
|
fputc (')', file);
|
||
|
}
|
||
|
|
||
|
+/* Return the memory model that encapuslates both given models. */
|
||
|
+
|
||
|
+enum memmodel
|
||
|
+riscv_union_memmodels (enum memmodel model1, enum memmodel model2)
|
||
|
+{
|
||
|
+ model1 = memmodel_base (model1);
|
||
|
+ model2 = memmodel_base (model2);
|
||
|
+
|
||
|
+ enum memmodel weaker = model1 <= model2 ? model1: model2;
|
||
|
+ enum memmodel stronger = model1 > model2 ? model1: model2;
|
||
|
+
|
||
|
+ switch (stronger)
|
||
|
+ {
|
||
|
+ case MEMMODEL_SEQ_CST:
|
||
|
+ case MEMMODEL_ACQ_REL:
|
||
|
+ return stronger;
|
||
|
+ case MEMMODEL_RELEASE:
|
||
|
+ if (weaker == MEMMODEL_ACQUIRE || weaker == MEMMODEL_CONSUME)
|
||
|
+ return MEMMODEL_ACQ_REL;
|
||
|
+ else
|
||
|
+ return stronger;
|
||
|
+ case MEMMODEL_ACQUIRE:
|
||
|
+ case MEMMODEL_CONSUME:
|
||
|
+ case MEMMODEL_RELAXED:
|
||
|
+ return stronger;
|
||
|
+ default:
|
||
|
+ gcc_unreachable ();
|
||
|
+ }
|
||
|
+}
|
||
|
+
|
||
|
/* Return true if the .AQ suffix should be added to an AMO to implement the
|
||
|
acquire portion of memory model MODEL. */
|
||
|
|
||
|
@@ -4337,6 +4367,8 @@ riscv_memmodel_needs_amo_release (enum memmodel model)
|
||
|
'R' Print the low-part relocation associated with OP.
|
||
|
'C' Print the integer branch condition for comparison OP.
|
||
|
'A' Print the atomic operation suffix for memory model OP.
|
||
|
+ 'I' Print the LR suffix for memory model OP.
|
||
|
+ 'J' Print the SC suffix for memory model OP.
|
||
|
'z' Print x0 if OP is zero, otherwise print OP normally.
|
||
|
'i' Print i if the operand is not a register.
|
||
|
'S' Print shift-index of single-bit mask OP.
|
||
|
@@ -4506,6 +4538,18 @@ riscv_print_operand (FILE *file, rtx op, int letter)
|
||
|
fputs (".rl", file);
|
||
|
break;
|
||
|
|
||
|
+ case 'I':
|
||
|
+ if (model == MEMMODEL_SEQ_CST)
|
||
|
+ fputs (".aqrl", file);
|
||
|
+ else if (riscv_memmodel_needs_amo_acquire (model))
|
||
|
+ fputs (".aq", file);
|
||
|
+ break;
|
||
|
+
|
||
|
+ case 'J':
|
||
|
+ if (riscv_memmodel_needs_amo_release (model))
|
||
|
+ fputs (".rl", file);
|
||
|
+ break;
|
||
|
+
|
||
|
case 'i':
|
||
|
if (code != REG)
|
||
|
fputs ("i", file);
|
||
|
diff --git a/gcc/config/riscv/sync.md b/gcc/config/riscv/sync.md
|
||
|
index 9a3b57bd09fd..3e6345e83a35 100644
|
||
|
--- a/gcc/config/riscv/sync.md
|
||
|
+++ b/gcc/config/riscv/sync.md
|
||
|
@@ -116,21 +116,22 @@
|
||
|
(unspec_volatile:SI
|
||
|
[(any_atomic:SI (match_dup 1)
|
||
|
(match_operand:SI 2 "register_operand" "rI")) ;; value for op
|
||
|
- (match_operand:SI 3 "register_operand" "rI")] ;; mask
|
||
|
+ (match_operand:SI 3 "const_int_operand")] ;; model
|
||
|
UNSPEC_SYNC_OLD_OP_SUBWORD))
|
||
|
- (match_operand:SI 4 "register_operand" "rI") ;; not_mask
|
||
|
- (clobber (match_scratch:SI 5 "=&r")) ;; tmp_1
|
||
|
- (clobber (match_scratch:SI 6 "=&r"))] ;; tmp_2
|
||
|
+ (match_operand:SI 4 "register_operand" "rI") ;; mask
|
||
|
+ (match_operand:SI 5 "register_operand" "rI") ;; not_mask
|
||
|
+ (clobber (match_scratch:SI 6 "=&r")) ;; tmp_1
|
||
|
+ (clobber (match_scratch:SI 7 "=&r"))] ;; tmp_2
|
||
|
"TARGET_ATOMIC && TARGET_INLINE_SUBWORD_ATOMIC"
|
||
|
{
|
||
|
return "1:\;"
|
||
|
- "lr.w.aqrl\t%0, %1\;"
|
||
|
- "<insn>\t%5, %0, %2\;"
|
||
|
- "and\t%5, %5, %3\;"
|
||
|
- "and\t%6, %0, %4\;"
|
||
|
- "or\t%6, %6, %5\;"
|
||
|
- "sc.w.rl\t%5, %6, %1\;"
|
||
|
- "bnez\t%5, 1b";
|
||
|
+ "lr.w%I3\t%0, %1\;"
|
||
|
+ "<insn>\t%6, %0, %2\;"
|
||
|
+ "and\t%6, %6, %4\;"
|
||
|
+ "and\t%7, %0, %5\;"
|
||
|
+ "or\t%7, %7, %6\;"
|
||
|
+ "sc.w%J3\t%6, %7, %1\;"
|
||
|
+ "bnez\t%6, 1b";
|
||
|
}
|
||
|
[(set (attr "length") (const_int 28))])
|
||
|
|
||
|
@@ -151,6 +152,7 @@
|
||
|
rtx old = gen_reg_rtx (SImode);
|
||
|
rtx mem = operands[1];
|
||
|
rtx value = operands[2];
|
||
|
+ rtx model = operands[3];
|
||
|
rtx aligned_mem = gen_reg_rtx (SImode);
|
||
|
rtx shift = gen_reg_rtx (SImode);
|
||
|
rtx mask = gen_reg_rtx (SImode);
|
||
|
@@ -162,7 +164,7 @@
|
||
|
riscv_lshift_subword (<MODE>mode, value, shift, &shifted_value);
|
||
|
|
||
|
emit_insn (gen_subword_atomic_fetch_strong_nand (old, aligned_mem,
|
||
|
- shifted_value,
|
||
|
+ shifted_value, model,
|
||
|
mask, not_mask));
|
||
|
|
||
|
emit_move_insn (old, gen_rtx_ASHIFTRT (SImode, old,
|
||
|
@@ -180,22 +182,23 @@
|
||
|
(unspec_volatile:SI
|
||
|
[(not:SI (and:SI (match_dup 1)
|
||
|
(match_operand:SI 2 "register_operand" "rI"))) ;; value for op
|
||
|
- (match_operand:SI 3 "register_operand" "rI")] ;; mask
|
||
|
+ (match_operand:SI 3 "const_int_operand")] ;; mask
|
||
|
UNSPEC_SYNC_OLD_OP_SUBWORD))
|
||
|
- (match_operand:SI 4 "register_operand" "rI") ;; not_mask
|
||
|
- (clobber (match_scratch:SI 5 "=&r")) ;; tmp_1
|
||
|
- (clobber (match_scratch:SI 6 "=&r"))] ;; tmp_2
|
||
|
+ (match_operand:SI 4 "register_operand" "rI") ;; mask
|
||
|
+ (match_operand:SI 5 "register_operand" "rI") ;; not_mask
|
||
|
+ (clobber (match_scratch:SI 6 "=&r")) ;; tmp_1
|
||
|
+ (clobber (match_scratch:SI 7 "=&r"))] ;; tmp_2
|
||
|
"TARGET_ATOMIC && TARGET_INLINE_SUBWORD_ATOMIC"
|
||
|
{
|
||
|
return "1:\;"
|
||
|
- "lr.w.aqrl\t%0, %1\;"
|
||
|
- "and\t%5, %0, %2\;"
|
||
|
- "not\t%5, %5\;"
|
||
|
- "and\t%5, %5, %3\;"
|
||
|
- "and\t%6, %0, %4\;"
|
||
|
- "or\t%6, %6, %5\;"
|
||
|
- "sc.w.rl\t%5, %6, %1\;"
|
||
|
- "bnez\t%5, 1b";
|
||
|
+ "lr.w%I3\t%0, %1\;"
|
||
|
+ "and\t%6, %0, %2\;"
|
||
|
+ "not\t%6, %6\;"
|
||
|
+ "and\t%6, %6, %4\;"
|
||
|
+ "and\t%7, %0, %5\;"
|
||
|
+ "or\t%7, %7, %6\;"
|
||
|
+ "sc.w%J3\t%6, %7, %1\;"
|
||
|
+ "bnez\t%6, 1b";
|
||
|
}
|
||
|
[(set (attr "length") (const_int 32))])
|
||
|
|
||
|
@@ -216,6 +219,7 @@
|
||
|
rtx old = gen_reg_rtx (SImode);
|
||
|
rtx mem = operands[1];
|
||
|
rtx value = operands[2];
|
||
|
+ rtx model = operands[3];
|
||
|
rtx aligned_mem = gen_reg_rtx (SImode);
|
||
|
rtx shift = gen_reg_rtx (SImode);
|
||
|
rtx mask = gen_reg_rtx (SImode);
|
||
|
@@ -228,7 +232,8 @@
|
||
|
|
||
|
emit_insn (gen_subword_atomic_fetch_strong_<atomic_optab> (old, aligned_mem,
|
||
|
shifted_value,
|
||
|
- mask, not_mask));
|
||
|
+ model, mask,
|
||
|
+ not_mask));
|
||
|
|
||
|
emit_move_insn (old, gen_rtx_ASHIFTRT (SImode, old,
|
||
|
gen_lowpart (QImode, shift)));
|
||
|
@@ -261,6 +266,7 @@
|
||
|
rtx old = gen_reg_rtx (SImode);
|
||
|
rtx mem = operands[1];
|
||
|
rtx value = operands[2];
|
||
|
+ rtx model = operands[3];
|
||
|
rtx aligned_mem = gen_reg_rtx (SImode);
|
||
|
rtx shift = gen_reg_rtx (SImode);
|
||
|
rtx mask = gen_reg_rtx (SImode);
|
||
|
@@ -272,7 +278,8 @@
|
||
|
riscv_lshift_subword (<MODE>mode, value, shift, &shifted_value);
|
||
|
|
||
|
emit_insn (gen_subword_atomic_exchange_strong (old, aligned_mem,
|
||
|
- shifted_value, not_mask));
|
||
|
+ shifted_value, model,
|
||
|
+ not_mask));
|
||
|
|
||
|
emit_move_insn (old, gen_rtx_ASHIFTRT (SImode, old,
|
||
|
gen_lowpart (QImode, shift)));
|
||
|
@@ -286,18 +293,19 @@
|
||
|
(match_operand:SI 1 "memory_operand" "+A")) ;; mem location
|
||
|
(set (match_dup 1)
|
||
|
(unspec_volatile:SI
|
||
|
- [(match_operand:SI 2 "reg_or_0_operand" "rI") ;; value
|
||
|
- (match_operand:SI 3 "reg_or_0_operand" "rI")] ;; not_mask
|
||
|
+ [(match_operand:SI 2 "reg_or_0_operand" "rI") ;; value
|
||
|
+ (match_operand:SI 3 "const_int_operand")] ;; model
|
||
|
UNSPEC_SYNC_EXCHANGE_SUBWORD))
|
||
|
- (clobber (match_scratch:SI 4 "=&r"))] ;; tmp_1
|
||
|
+ (match_operand:SI 4 "reg_or_0_operand" "rI") ;; not_mask
|
||
|
+ (clobber (match_scratch:SI 5 "=&r"))] ;; tmp_1
|
||
|
"TARGET_ATOMIC && TARGET_INLINE_SUBWORD_ATOMIC"
|
||
|
{
|
||
|
return "1:\;"
|
||
|
- "lr.w.aqrl\t%0, %1\;"
|
||
|
- "and\t%4, %0, %3\;"
|
||
|
- "or\t%4, %4, %2\;"
|
||
|
- "sc.w.rl\t%4, %4, %1\;"
|
||
|
- "bnez\t%4, 1b";
|
||
|
+ "lr.w%I3\t%0, %1\;"
|
||
|
+ "and\t%5, %0, %4\;"
|
||
|
+ "or\t%5, %5, %2\;"
|
||
|
+ "sc.w%J3\t%5, %5, %1\;"
|
||
|
+ "bnez\t%5, 1b";
|
||
|
}
|
||
|
[(set (attr "length") (const_int 20))])
|
||
|
|
||
|
@@ -313,10 +321,15 @@
|
||
|
(clobber (match_scratch:GPR 6 "=&r"))]
|
||
|
"TARGET_ATOMIC"
|
||
|
{
|
||
|
+ enum memmodel model_success = (enum memmodel) INTVAL (operands[4]);
|
||
|
+ enum memmodel model_failure = (enum memmodel) INTVAL (operands[5]);
|
||
|
+ /* Find the union of the two memory models so we can satisfy both success
|
||
|
+ and failure memory models. */
|
||
|
+ operands[5] = GEN_INT (riscv_union_memmodels (model_success, model_failure));
|
||
|
return "1:\;"
|
||
|
- "lr.<amo>.aqrl\t%0,%1\;"
|
||
|
+ "lr.<amo>%I5\t%0,%1\;"
|
||
|
"bne\t%0,%z2,1f\;"
|
||
|
- "sc.<amo>.rl\t%6,%z3,%1\;"
|
||
|
+ "sc.<amo>%J5\t%6,%z3,%1\;"
|
||
|
"bnez\t%6,1b\;"
|
||
|
"1:";
|
||
|
}
|
||
|
@@ -440,9 +453,15 @@
|
||
|
emit_move_insn (shifted_o, gen_rtx_AND (SImode, shifted_o, mask));
|
||
|
emit_move_insn (shifted_n, gen_rtx_AND (SImode, shifted_n, mask));
|
||
|
|
||
|
+ enum memmodel model_success = (enum memmodel) INTVAL (operands[4]);
|
||
|
+ enum memmodel model_failure = (enum memmodel) INTVAL (operands[5]);
|
||
|
+ /* Find the union of the two memory models so we can satisfy both success
|
||
|
+ and failure memory models. */
|
||
|
+ rtx model = GEN_INT (riscv_union_memmodels (model_success, model_failure));
|
||
|
+
|
||
|
emit_insn (gen_subword_atomic_cas_strong (old, aligned_mem,
|
||
|
shifted_o, shifted_n,
|
||
|
- mask, not_mask));
|
||
|
+ model, mask, not_mask));
|
||
|
|
||
|
emit_move_insn (old, gen_rtx_ASHIFTRT (SImode, old,
|
||
|
gen_lowpart (QImode, shift)));
|
||
|
@@ -459,19 +478,20 @@
|
||
|
(unspec_volatile:SI [(match_operand:SI 2 "reg_or_0_operand" "rJ") ;; expected value
|
||
|
(match_operand:SI 3 "reg_or_0_operand" "rJ")] ;; desired value
|
||
|
UNSPEC_COMPARE_AND_SWAP_SUBWORD))
|
||
|
- (match_operand:SI 4 "register_operand" "rI") ;; mask
|
||
|
- (match_operand:SI 5 "register_operand" "rI") ;; not_mask
|
||
|
- (clobber (match_scratch:SI 6 "=&r"))] ;; tmp_1
|
||
|
+ (match_operand:SI 4 "const_int_operand") ;; model
|
||
|
+ (match_operand:SI 5 "register_operand" "rI") ;; mask
|
||
|
+ (match_operand:SI 6 "register_operand" "rI") ;; not_mask
|
||
|
+ (clobber (match_scratch:SI 7 "=&r"))] ;; tmp_1
|
||
|
"TARGET_ATOMIC && TARGET_INLINE_SUBWORD_ATOMIC"
|
||
|
{
|
||
|
return "1:\;"
|
||
|
- "lr.w.aqrl\t%0, %1\;"
|
||
|
- "and\t%6, %0, %4\;"
|
||
|
- "bne\t%6, %z2, 1f\;"
|
||
|
- "and\t%6, %0, %5\;"
|
||
|
- "or\t%6, %6, %3\;"
|
||
|
- "sc.w.rl\t%6, %6, %1\;"
|
||
|
- "bnez\t%6, 1b\;"
|
||
|
+ "lr.w%I4\t%0, %1\;"
|
||
|
+ "and\t%7, %0, %5\;"
|
||
|
+ "bne\t%7, %z2, 1f\;"
|
||
|
+ "and\t%7, %0, %6\;"
|
||
|
+ "or\t%7, %7, %3\;"
|
||
|
+ "sc.w%J4\t%7, %7, %1\;"
|
||
|
+ "bnez\t%7, 1b\;"
|
||
|
"1:";
|
||
|
}
|
||
|
[(set (attr "length") (const_int 28))])
|
||
|
--
|
||
|
2.39.3
|
||
|
|