From abbfc8ddd864b1b4d0dea3ea6bb8431d12e23ec2 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Tue, 30 Jul 2019 23:56:30 +0000 Subject: [PATCH] GlobalISel: Add G_ATOMICRMW_{FADD|FSUB} git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@367369 91177308-0d34-0410-b5e6-96231b3b80d8 --- .../CodeGen/GlobalISel/MachineIRBuilder.h | 14 +++++- include/llvm/Support/TargetOpcodes.def | 2 + include/llvm/Target/GenericOpcodes.td | 2 + .../Target/GlobalISel/SelectionDAGCompat.td | 2 + lib/CodeGen/GlobalISel/IRTranslator.cpp | 7 ++- lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 43 ++++++++++++----- .../GlobalISel/legalizer-info-validation.mir | 6 +++ .../GlobalISel/irtranslator-atomicrmw.ll | 48 +++++++++++++++++++ .../GlobalISel/MachineIRBuilderTest.cpp | 30 ++++++++++++ 9 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index a88c212d024..9823f4ab62f 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -963,8 +963,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes, - Register Addr, Register Val, + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, + const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_XCHG Addr, Val, MMO`. @@ -1137,6 +1137,16 @@ public: MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr, Register Val, MachineMemOperand &MMO); + /// Build and insert `OldValRes = G_ATOMICRMW_FADD Addr, Val, MMO`. + MachineInstrBuilder buildAtomicRMWFAdd( + const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes = G_ATOMICRMW_FSUB Addr, Val, MMO`. + MachineInstrBuilder buildAtomicRMWFSub( + const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO); + /// Build and insert `G_FENCE Ordering, Scope`. MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); diff --git a/include/llvm/Support/TargetOpcodes.def b/include/llvm/Support/TargetOpcodes.def index 598c1064efd..8d4ce176561 100644 --- a/include/llvm/Support/TargetOpcodes.def +++ b/include/llvm/Support/TargetOpcodes.def @@ -315,6 +315,8 @@ HANDLE_TARGET_OPCODE(G_ATOMICRMW_MAX) HANDLE_TARGET_OPCODE(G_ATOMICRMW_MIN) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMAX) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMIN) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_FADD) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_FSUB) // Generic atomic fence HANDLE_TARGET_OPCODE(G_FENCE) diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td index 45718327b4a..a9821b62893 100644 --- a/include/llvm/Target/GenericOpcodes.td +++ b/include/llvm/Target/GenericOpcodes.td @@ -798,6 +798,8 @@ def G_ATOMICRMW_MAX : G_ATOMICRMW_OP; def G_ATOMICRMW_MIN : G_ATOMICRMW_OP; def G_ATOMICRMW_UMAX : G_ATOMICRMW_OP; def G_ATOMICRMW_UMIN : G_ATOMICRMW_OP; +def G_ATOMICRMW_FADD : G_ATOMICRMW_OP; +def G_ATOMICRMW_FSUB : G_ATOMICRMW_OP; def G_FENCE : GenericInstruction { let OutOperandList = (outs); diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 5ada1b7ca2e..3da336341bc 100644 --- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -143,6 +143,8 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index 7c90d57f697..3dc2200d37f 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1984,7 +1984,6 @@ bool IRTranslator::translateAtomicRMW(const User &U, unsigned Opcode = 0; switch (I.getOperation()) { default: - llvm_unreachable("Unknown atomicrmw op"); return false; case AtomicRMWInst::Xchg: Opcode = TargetOpcode::G_ATOMICRMW_XCHG; @@ -2019,6 +2018,12 @@ bool IRTranslator::translateAtomicRMW(const User &U, case AtomicRMWInst::UMin: Opcode = TargetOpcode::G_ATOMICRMW_UMIN; break; + case AtomicRMWInst::FAdd: + Opcode = TargetOpcode::G_ATOMICRMW_FADD; + break; + case AtomicRMWInst::FSub: + Opcode = TargetOpcode::G_ATOMICRMW_FSUB; + break; } MIRBuilder.buildAtomicRMW( diff --git a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 34355f58399..7678110d547 100644 --- a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -776,26 +776,28 @@ MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr, .addMemOperand(&MMO); } -MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(unsigned Opcode, - Register OldValRes, - Register Addr, - Register Val, - MachineMemOperand &MMO) { +MachineInstrBuilder MachineIRBuilder::buildAtomicRMW( + unsigned Opcode, const DstOp &OldValRes, + const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO) { + #ifndef NDEBUG - LLT OldValResTy = getMRI()->getType(OldValRes); - LLT AddrTy = getMRI()->getType(Addr); - LLT ValTy = getMRI()->getType(Val); + LLT OldValResTy = OldValRes.getLLTTy(*getMRI()); + LLT AddrTy = Addr.getLLTTy(*getMRI()); + LLT ValTy = Val.getLLTTy(*getMRI()); assert(OldValResTy.isScalar() && "invalid operand type"); assert(AddrTy.isPointer() && "invalid operand type"); assert(ValTy.isValid() && "invalid operand type"); assert(OldValResTy == ValTy && "type mismatch"); + assert(MMO.isAtomic() && "not atomic mem operand"); #endif - return buildInstr(Opcode) - .addDef(OldValRes) - .addUse(Addr) - .addUse(Val) - .addMemOperand(&MMO); + auto MIB = buildInstr(Opcode); + OldValRes.addDefToMIB(*getMRI(), MIB); + Addr.addSrcToMIB(MIB); + Val.addSrcToMIB(MIB); + MIB.addMemOperand(&MMO); + return MIB; } MachineInstrBuilder @@ -866,6 +868,21 @@ MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr, MMO); } +MachineInstrBuilder +MachineIRBuilder::buildAtomicRMWFAdd( + const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val, + MMO); +} + +MachineInstrBuilder +MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, + MachineMemOperand &MMO) { + return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val, + MMO); +} + MachineInstrBuilder MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) { return buildInstr(TargetOpcode::G_FENCE) diff --git a/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index 381b889976f..bd3f36d1ba9 100644 --- a/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -144,6 +144,12 @@ # DEBUG-NEXT: G_ATOMICRMW_UMIN (opcode {{[0-9]+}}): 2 type indices # DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected # +# DEBUG-NEXT: G_ATOMICRMW_FADD (opcode {{[0-9]+}}): 2 type indices +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# +# DEBUG-NEXT: G_ATOMICRMW_FSUB (opcode {{[0-9]+}}): 2 type indices +# DEBUG: .. type index coverage check SKIPPED: no rules defined +# # DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices # DEBUG: .. type index coverage check SKIPPED: no rules defined # diff --git a/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll b/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll new file mode 100644 index 00000000000..34c16787840 --- /dev/null +++ b/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll @@ -0,0 +1,48 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -O0 -stop-after=irtranslator -o - %s | FileCheck %s + +define float @test_atomicrmw_fadd(float addrspace(3)* %addr) { + ; CHECK-LABEL: name: test_atomicrmw_fadd + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $vgpr0, $sgpr30_sgpr31 + ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 + ; CHECK: [[ATOMICRMW_FADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[C]] :: (load store seq_cst 4 on %ir.addr, addrspace 3) + ; CHECK: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32) + ; CHECK: [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]] + ; CHECK: S_SETPC_B64_return [[COPY2]], implicit $vgpr0 + %oldval = atomicrmw fadd float addrspace(3)* %addr, float 1.0 seq_cst + ret float %oldval +} + +define float @test_atomicrmw_fsub(float addrspace(3)* %addr) { + ; CHECK-LABEL: name: test_atomicrmw_fsub + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: liveins: $vgpr0, $sgpr30_sgpr31 + ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00 + ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 4 from %ir.addr, addrspace 3) + ; CHECK: bb.2.atomicrmw.start: + ; CHECK: successors: %bb.3(0x40000000), %bb.2(0x40000000) + ; CHECK: [[PHI:%[0-9]+]]:_(s64) = G_PHI %9(s64), %bb.2, [[C1]](s64), %bb.1 + ; CHECK: [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[LOAD]](s32), %bb.1, %7(s32), %bb.2 + ; CHECK: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[PHI1]], [[C]] + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p3), [[PHI1]], [[FSUB]] :: (load store seq_cst seq_cst 4 on %ir.2, addrspace 3) + ; CHECK: [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ATOMIC_CMPXCHG_WITH_SUCCESS1]](s1), [[PHI]](s64) + ; CHECK: [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.loop), [[INT]](s64) + ; CHECK: G_BRCOND [[INT1]](s1), %bb.3 + ; CHECK: G_BR %bb.2 + ; CHECK: bb.3.atomicrmw.end: + ; CHECK: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32), %bb.2 + ; CHECK: [[PHI3:%[0-9]+]]:_(s64) = G_PHI [[INT]](s64), %bb.2 + ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s64) + ; CHECK: $vgpr0 = COPY [[PHI2]](s32) + ; CHECK: [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]] + ; CHECK: S_SETPC_B64_return [[COPY2]], implicit $vgpr0 + %oldval = atomicrmw fsub float addrspace(3)* %addr, float 1.0 seq_cst + ret float %oldval +} diff --git a/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp index f6d811240f4..92d1c46d355 100644 --- a/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp +++ b/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp @@ -278,3 +278,33 @@ TEST_F(GISelMITest, BuildMinMax) { EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; } + +TEST_F(GISelMITest, BuildAtomicRMW) { + if (!TM) + return; + + LLT S64 = LLT::scalar(64); + LLT P0 = LLT::pointer(0, 64); + SmallVector Copies; + collectCopies(Copies, MF); + + MachineMemOperand *MMO = + MF->getMachineMemOperand( + MachinePointerInfo(), + MachineMemOperand::MOLoad | MachineMemOperand::MOStore, + 8, 8, AAMDNodes(), nullptr, SyncScope::System, AtomicOrdering::Unordered); + + auto Ptr = B.buildUndef(P0); + B.buildAtomicRMWFAdd(S64, Ptr, Copies[0], *MMO); + B.buildAtomicRMWFSub(S64, Ptr, Copies[0], *MMO); + + auto CheckStr = R"( + ; CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 + ; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF + ; CHECK: [[FADD:%[0-9]+]]:_(s64) = G_ATOMICRMW_FADD [[PTR]]:_(p0), [[COPY0]]:_ :: (load store unordered 8) + ; CHECK: [[FSUB:%[0-9]+]]:_(s64) = G_ATOMICRMW_FSUB [[PTR]]:_(p0), [[COPY0]]:_ :: (load store unordered 8) + )"; + + EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF; +} -- 2.40.0