From e3696113b639c8bf0b72d6c27dd76d6fdd8ebf61 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 8 Feb 2019 20:48:56 +0000 Subject: [PATCH] Implementation of asm-goto support in LLVM This patch accompanies the RFC posted here: http://lists.llvm.org/pipermail/llvm-dev/2018-October/127239.html This patch adds a new CallBr IR instruction to support asm-goto inline assembly like gcc as used by the linux kernel. This instruction is both a call instruction and a terminator instruction with multiple successors. Only inline assembly usage is supported today. This also adds a new INLINEASM_BR opcode to SelectionDAG and MachineIR to represent an INLINEASM block that is also considered a terminator instruction. There will likely be more bug fixes and optimizations to follow this, but we felt it had reached a point where we would like to switch to an incremental development model. Patch by Craig Topper, Alexander Ivchenko, Mikhail Dvoretckii Differential Revision: https://reviews.llvm.org/D53765 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@353563 91177308-0d34-0410-b5e6-96231b3b80d8 --- docs/LangRef.rst | 80 ++++++ include/llvm-c/Core.h | 1 + include/llvm/Analysis/SparsePropagation.h | 8 +- include/llvm/Bitcode/LLVMBitCodes.h | 2 + .../llvm/CodeGen/GlobalISel/IRTranslator.h | 2 + include/llvm/CodeGen/ISDOpcodes.h | 3 + include/llvm/CodeGen/MachineInstr.h | 5 +- include/llvm/CodeGen/SelectionDAGISel.h | 2 +- include/llvm/IR/CallSite.h | 107 ++++---- include/llvm/IR/IRBuilder.h | 36 +++ include/llvm/IR/InstVisitor.h | 13 +- include/llvm/IR/InstrTypes.h | 9 +- include/llvm/IR/Instruction.def | 139 +++++----- include/llvm/IR/Instruction.h | 14 + include/llvm/IR/Instructions.h | 243 ++++++++++++++++++ include/llvm/Support/TargetOpcodes.def | 1 + include/llvm/Target/Target.td | 9 + lib/Analysis/ValueTracking.cpp | 1 + lib/AsmParser/LLLexer.cpp | 1 + lib/AsmParser/LLParser.cpp | 127 +++++++++ lib/AsmParser/LLParser.h | 1 + lib/AsmParser/LLToken.h | 1 + lib/Bitcode/Reader/BitcodeReader.cpp | 68 +++++ lib/Bitcode/Writer/BitcodeWriter.cpp | 35 +++ lib/Bitcode/Writer/ValueEnumerator.cpp | 6 +- lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 1 + .../AsmPrinter/AsmPrinterInlineAsm.cpp | 13 +- lib/CodeGen/CodeGenPrepare.cpp | 10 + lib/CodeGen/GlobalISel/IRTranslator.cpp | 6 + lib/CodeGen/IndirectBrExpandPass.cpp | 8 +- lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 10 +- .../SelectionDAG/ResourcePriorityQueue.cpp | 4 + lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp | 3 +- .../SelectionDAG/ScheduleDAGRRList.cpp | 4 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 41 ++- .../SelectionDAG/SelectionDAGBuilder.h | 2 + .../SelectionDAG/SelectionDAGDumper.cpp | 1 + lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 8 +- lib/CodeGen/SelectionDAG/TargetLowering.cpp | 6 +- lib/CodeGen/TargetLoweringBase.cpp | 1 + lib/IR/AsmWriter.cpp | 45 ++++ lib/IR/Instruction.cpp | 11 +- lib/IR/Instructions.cpp | 83 ++++++ lib/IR/Value.cpp | 3 +- lib/IR/Verifier.cpp | 21 ++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 1 + lib/Target/AMDGPU/SIISelLowering.cpp | 3 +- lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +- lib/Target/ARM/ARMISelDAGToDAG.cpp | 3 +- lib/Target/AVR/AVRInstrInfo.cpp | 3 +- lib/Target/Hexagon/HexagonISelLowering.cpp | 6 +- .../Hexagon/HexagonMachineScheduler.cpp | 2 + lib/Target/MSP430/MSP430InstrInfo.cpp | 3 +- lib/Target/Mips/MipsInstrInfo.cpp | 3 +- lib/Target/PowerPC/PPCRegisterInfo.cpp | 3 +- lib/Target/RISCV/RISCVInstrInfo.cpp | 3 +- lib/Target/Sparc/SparcISelDAGToDAG.cpp | 5 +- lib/Target/X86/X86AsmPrinter.cpp | 5 + lib/Target/X86/X86FloatingPoint.cpp | 3 +- .../InstCombine/InstCombineCalls.cpp | 57 ++-- .../InstCombine/InstCombineInternal.h | 1 + .../InstCombine/InstructionCombining.cpp | 4 +- lib/Transforms/Scalar/GVN.cpp | 17 +- lib/Transforms/Scalar/JumpThreading.cpp | 11 +- lib/Transforms/Scalar/SCCP.cpp | 13 + lib/Transforms/Utils/BasicBlockUtils.cpp | 2 + lib/Transforms/Utils/BreakCriticalEdges.cpp | 4 + lib/Transforms/Utils/InlineFunction.cpp | 8 + lib/Transforms/Utils/Local.cpp | 12 + lib/Transforms/Utils/LoopSimplify.cpp | 16 +- lib/Transforms/Utils/LoopUtils.cpp | 3 + lib/Transforms/Utils/SimplifyCFG.cpp | 13 +- test/Bitcode/callbr.ll | 14 + test/Bitcode/callbr.ll.bc | Bin 0 -> 1036 bytes test/CodeGen/X86/callbr-asm-blockplacement.ll | 106 ++++++++ test/CodeGen/X86/callbr-asm-branch-folding.ll | 151 +++++++++++ test/CodeGen/X86/callbr-asm-destinations.ll | 15 ++ test/CodeGen/X86/callbr-asm-errors.ll | 18 ++ test/CodeGen/X86/callbr-asm-outputs.ll | 18 ++ test/CodeGen/X86/callbr-asm.ll | 133 ++++++++++ .../Transforms/GVN/callbr-loadpre-critedge.ll | 49 ++++ .../GVN/callbr-scalarpre-critedge.ll | 43 ++++ .../JumpThreading/callbr-edge-split.ll | 58 +++++ .../MergeFunc/call-and-invoke-with-ranges.ll | 15 +- test/Transforms/MergeFunc/inline-asm.ll | 6 +- tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp | 1 + utils/vim/syntax/llvm.vim | 2 +- 87 files changed, 1812 insertions(+), 219 deletions(-) create mode 100644 test/Bitcode/callbr.ll create mode 100644 test/Bitcode/callbr.ll.bc create mode 100644 test/CodeGen/X86/callbr-asm-blockplacement.ll create mode 100644 test/CodeGen/X86/callbr-asm-branch-folding.ll create mode 100644 test/CodeGen/X86/callbr-asm-destinations.ll create mode 100644 test/CodeGen/X86/callbr-asm-errors.ll create mode 100644 test/CodeGen/X86/callbr-asm-outputs.ll create mode 100644 test/CodeGen/X86/callbr-asm.ll create mode 100644 test/Transforms/GVN/callbr-loadpre-critedge.ll create mode 100644 test/Transforms/GVN/callbr-scalarpre-critedge.ll create mode 100644 test/Transforms/JumpThreading/callbr-edge-split.ll diff --git a/docs/LangRef.rst b/docs/LangRef.rst index 776271d2451..ea941ebdb10 100644 --- a/docs/LangRef.rst +++ b/docs/LangRef.rst @@ -6513,6 +6513,7 @@ control flow, not values (the one exception being the The terminator instructions are: ':ref:`ret `', ':ref:`br `', ':ref:`switch `', ':ref:`indirectbr `', ':ref:`invoke `', +':ref:`callbr `' ':ref:`resume `', ':ref:`catchswitch `', ':ref:`catchret `', ':ref:`cleanupret `', @@ -6837,6 +6838,85 @@ Example: %retval = invoke coldcc i32 %Testfnptr(i32 15) to label %Continue unwind label %TestCleanup ; i32:retval set +.. _i_callbr: + +'``callbr``' Instruction +^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + = callbr [cconv] [ret attrs] [addrspace()] [| () [fn attrs] + [operand bundles] to label or jump [other labels] + +Overview: +""""""""" + +The '``callbr``' instruction causes control to transfer to a specified +function, with the possibility of control flow transfer to either the +'``normal``' label or one of the '``other``' labels. + +This instruction should only be used to implement the "goto" feature of gcc +style inline assembly. Any other usage is an error in the IR verifier. + +Arguments: +"""""""""" + +This instruction requires several arguments: + +#. The optional "cconv" marker indicates which :ref:`calling + convention ` the call should use. If none is + specified, the call defaults to using C calling conventions. +#. The optional :ref:`Parameter Attributes ` list for return + values. Only '``zeroext``', '``signext``', and '``inreg``' attributes + are valid here. +#. The optional addrspace attribute can be used to indicate the address space + of the called function. If it is not specified, the program address space + from the :ref:`datalayout string` will be used. +#. '``ty``': the type of the call instruction itself which is also the + type of the return value. Functions that return no value are marked + ``void``. +#. '``fnty``': shall be the signature of the function being called. The + argument types must match the types implied by this signature. This + type can be omitted if the function is not varargs. +#. '``fnptrval``': An LLVM value containing a pointer to a function to + be called. In most cases, this is a direct function call, but + indirect ``callbr``'s are just as possible, calling an arbitrary pointer + to function value. +#. '``function args``': argument list whose types match the function + signature argument types and parameter attributes. All arguments must + be of :ref:`first class ` type. If the function signature + indicates the function accepts a variable number of arguments, the + extra arguments can be specified. +#. '``normal label``': the label reached when the called function + executes a '``ret``' instruction. +#. '``other labels``': the labels reached when a callee transfers control + to a location other than the normal '``normal label``' +#. The optional :ref:`function attributes ` list. +#. The optional :ref:`operand bundles ` list. + +Semantics: +"""""""""" + +This instruction is designed to operate as a standard '``call``' +instruction in most regards. The primary difference is that it +establishes an association with additional labels to define where control +flow goes after the call. + +The only use of this today is to implement the "goto" feature of gcc inline +assembly where additional labels can be provided as locations for the inline +assembly to jump to. + +Example: +"""""""" + +.. code-block:: llvm + + callbr void asm "", "r,x"(i32 %x, i8 *blockaddress(@foo, %fail)) + to label %normal or jump [label %fail] + .. _i_resume: '``resume``' Instruction diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h index d84a3a127ed..9c521899c95 100644 --- a/include/llvm-c/Core.h +++ b/include/llvm-c/Core.h @@ -65,6 +65,7 @@ typedef enum { LLVMInvoke = 5, /* removed 6 due to API changes */ LLVMUnreachable = 7, + LLVMCallBr = 67, /* Standard Unary Operators */ LLVMFNeg = 66, diff --git a/include/llvm/Analysis/SparsePropagation.h b/include/llvm/Analysis/SparsePropagation.h index b6e40eed8e3..fac92e4a25a 100644 --- a/include/llvm/Analysis/SparsePropagation.h +++ b/include/llvm/Analysis/SparsePropagation.h @@ -329,12 +329,8 @@ void SparseSolver::getFeasibleSuccessors( return; } - if (TI.isExceptionalTerminator()) { - Succs.assign(Succs.size(), true); - return; - } - - if (isa(TI)) { + if (TI.isExceptionalTerminator() || + TI.isIndirectTerminator()) { Succs.assign(Succs.size(), true); return; } diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h index ce853cd3998..71148ac55c3 100644 --- a/include/llvm/Bitcode/LLVMBitCodes.h +++ b/include/llvm/Bitcode/LLVMBitCodes.h @@ -535,6 +535,8 @@ enum FunctionCodes { // 54 is unused. FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] + FUNC_CODE_INST_CALLBR = 57, // CALLBR: [attr, cc, norm, transfs, + // fnty, fnid, args...] }; enum UseListCodes { diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index f061c2dd05a..8844260116c 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -252,6 +252,8 @@ private: bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); + bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); /// Translate one of LLVM's cast instructions into MachineInstrs, with the diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 89356641b7b..ed4bfe7ad2d 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -667,6 +667,9 @@ namespace ISD { /// SDOperands. INLINEASM, + /// INLINEASM_BR - Terminator version of inline asm. Used by asm-goto. + INLINEASM_BR, + /// EH_LABEL - Represents a label in mid basic block used to track /// locations needed for debug and exception handling tables. These nodes /// take a chain as input and return a chain. diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index 5267b9db21a..4cb39c5f0a6 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -1011,7 +1011,10 @@ public: } bool isKill() const { return getOpcode() == TargetOpcode::KILL; } bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } - bool isInlineAsm() const { return getOpcode() == TargetOpcode::INLINEASM; } + bool isInlineAsm() const { + return getOpcode() == TargetOpcode::INLINEASM || + getOpcode() == TargetOpcode::INLINEASM_BR; + } bool isMSInlineAsm() const { return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index 84bd5c05d5a..2acb9226d7d 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -302,7 +302,7 @@ public: private: // Calls to these functions are generated by tblgen. - void Select_INLINEASM(SDNode *N); + void Select_INLINEASM(SDNode *N, bool Branch); void Select_READ_REGISTER(SDNode *Op); void Select_WRITE_REGISTER(SDNode *Op); void Select_UNDEF(SDNode *N); diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h index 6d27005e881..1454874e4ef 100644 --- a/include/llvm/IR/CallSite.h +++ b/include/llvm/IR/CallSite.h @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// // // This file defines the CallSite class, which is a handy wrapper for code that -// wants to treat Call and Invoke instructions in a generic way. When in non- -// mutation context (e.g. an analysis) ImmutableCallSite should be used. +// wants to treat Call, Invoke and CallBr instructions in a generic way. When +// in non-mutation context (e.g. an analysis) ImmutableCallSite should be used. // Finally, when some degree of customization is necessary between these two // extremes, CallSiteBase<> can be supplied with fine-tuned parameters. // @@ -17,7 +17,7 @@ // They are efficiently copyable, assignable and constructable, with cost // equivalent to copying a pointer (notice that they have only a single data // member). The internal representation carries a flag which indicates which of -// the two variants is enclosed. This allows for cheaper checks when various +// the three variants is enclosed. This allows for cheaper checks when various // accessors of CallSite are employed. // //===----------------------------------------------------------------------===// @@ -48,45 +48,50 @@ namespace Intrinsic { enum ID : unsigned; } -template class CallSiteBase { protected: - PointerIntPair I; + PointerIntPair I; CallSiteBase() = default; - CallSiteBase(CallTy *CI) : I(CI, true) { assert(CI); } - CallSiteBase(InvokeTy *II) : I(II, false) { assert(II); } + CallSiteBase(CallTy *CI) : I(CI, 1) { assert(CI); } + CallSiteBase(InvokeTy *II) : I(II, 0) { assert(II); } + CallSiteBase(CallBrTy *CBI) : I(CBI, 2) { assert(CBI); } explicit CallSiteBase(ValTy *II) { *this = get(II); } private: /// This static method is like a constructor. It will create an appropriate - /// call site for a Call or Invoke instruction, but it can also create a null - /// initialized CallSiteBase object for something which is NOT a call site. + /// call site for a Call, Invoke or CallBr instruction, but it can also create + /// a null initialized CallSiteBase object for something which is NOT a call + /// site. static CallSiteBase get(ValTy *V) { if (InstrTy *II = dyn_cast(V)) { if (II->getOpcode() == Instruction::Call) return CallSiteBase(static_cast(II)); - else if (II->getOpcode() == Instruction::Invoke) + if (II->getOpcode() == Instruction::Invoke) return CallSiteBase(static_cast(II)); + if (II->getOpcode() == Instruction::CallBr) + return CallSiteBase(static_cast(II)); } return CallSiteBase(); } public: - /// Return true if a CallInst is enclosed. Note that !isCall() does not mean - /// an InvokeInst is enclosed. It may also signify a NULL instruction pointer. - bool isCall() const { return I.getInt(); } + /// Return true if a CallInst is enclosed. + bool isCall() const { return I.getInt() == 1; } - /// Return true if a InvokeInst is enclosed. - bool isInvoke() const { return getInstruction() && !I.getInt(); } + /// Return true if a InvokeInst is enclosed. !I.getInt() may also signify a + /// NULL instruction pointer, so check that. + bool isInvoke() const { return getInstruction() && I.getInt() == 0; } + + /// Return true if a CallBrInst is enclosed. + bool isCallBr() const { return I.getInt() == 2; } InstrTy *getInstruction() const { return I.getPointer(); } InstrTy *operator->() const { return I.getPointer(); } @@ -97,7 +102,7 @@ public: /// Return the pointer to function that is being called. ValTy *getCalledValue() const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); return *getCallee(); } @@ -114,17 +119,16 @@ public: return false; if (isa(V) || isa(V)) return false; - if (const CallInst *CI = dyn_cast(getInstruction())) { - if (CI->isInlineAsm()) + if (const CallBase *CB = dyn_cast(getInstruction())) + if (CB->isInlineAsm()) return false; - } return true; } /// Set the callee to the specified value. Unlike the function of the same /// name on CallBase, does not modify the type! void setCalledFunction(Value *V) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, callbr, or invoke instruction!"); assert(cast(V->getType())->getElementType() == cast(getInstruction())->getFunctionType() && "New callee type does not match FunctionType on call"); @@ -192,7 +196,7 @@ public: } void setArgument(unsigned ArgNo, Value* newVal) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(arg_begin() + ArgNo < arg_end() && "Argument # out of range!"); getInstruction()->setOperand(ArgNo, newVal); } @@ -206,7 +210,7 @@ public: /// Given a use for an argument, get the argument number that corresponds to /// it. unsigned getArgumentNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isArgOperand(U) && "Argument # out of range!"); return U - arg_begin(); } @@ -230,7 +234,7 @@ public: /// Given a use for a data operand, get the data operand number that /// corresponds to it. unsigned getDataOperandNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isDataOperand(U) && "Data operand # out of range!"); return U - data_operands_begin(); } @@ -240,10 +244,11 @@ public: using data_operand_iterator = IterTy; /// data_operands_begin/data_operands_end - Return iterators iterating over - /// the call / invoke argument list and bundle operands. For invokes, this is - /// the set of instruction operands except the invoke target and the two - /// successor blocks; and for calls this is the set of instruction operands - /// except the call target. + /// the call / invoke / callbr argument list and bundle operands. For invokes, + /// this is the set of instruction operands except the invoke target and the + /// two successor blocks; for calls this is the set of instruction operands + /// except the call target; for callbrs the number of labels to skip must be + /// determined first. IterTy data_operands_begin() const { assert(getInstruction() && "Not a call or invoke instruction!"); @@ -280,17 +285,19 @@ public: return isCall() && cast(getInstruction())->isTailCall(); } -#define CALLSITE_DELEGATE_GETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - return isCall() \ - ? cast(II)->METHOD \ - : cast(II)->METHOD +#define CALLSITE_DELEGATE_GETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + return isCall() ? cast(II)->METHOD \ + : isCallBr() ? cast(II)->METHOD \ + : cast(II)->METHOD -#define CALLSITE_DELEGATE_SETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - if (isCall()) \ - cast(II)->METHOD; \ - else \ +#define CALLSITE_DELEGATE_SETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + if (isCall()) \ + cast(II)->METHOD; \ + else if (isCallBr()) \ + cast(II)->METHOD; \ + else \ cast(II)->METHOD unsigned getNumArgOperands() const { @@ -306,9 +313,7 @@ public: } bool isInlineAsm() const { - if (isCall()) - return cast(getInstruction())->isInlineAsm(); - return false; + return cast(getInstruction())->isInlineAsm(); } /// Get the calling convention of the call. @@ -392,10 +397,10 @@ public: /// Return true if the data operand at index \p i directly or indirectly has /// the attribute \p A. /// - /// Normal call or invoke arguments have per operand attributes, as specified - /// in the attribute set attached to this instruction, while operand bundle - /// operands may have some attributes implied by the type of its containing - /// operand bundle. + /// Normal call, invoke or callbr arguments have per operand attributes, as + /// specified in the attribute set attached to this instruction, while operand + /// bundle operands may have some attributes implied by the type of its + /// containing operand bundle. bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, Kind)); } @@ -661,12 +666,13 @@ private: class CallSite : public CallSiteBase { + CallBrInst, User::op_iterator> { public: CallSite() = default; CallSite(CallSiteBase B) : CallSiteBase(B) {} CallSite(CallInst *CI) : CallSiteBase(CI) {} CallSite(InvokeInst *II) : CallSiteBase(II) {} + CallSite(CallBrInst *CBI) : CallSiteBase(CBI) {} explicit CallSite(Instruction *II) : CallSiteBase(II) {} explicit CallSite(Value *V) : CallSiteBase(V) {} @@ -888,6 +894,7 @@ public: ImmutableCallSite() = default; ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {} ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {} + ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {} explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {} explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {} ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index 863171ad741..2e0619d54f8 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -943,6 +943,42 @@ public: Callee, NormalDest, UnwindDest, Args, Name); } + /// \brief Create a callbr instruction. + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, + Args), Name); + } + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return Insert( + CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, + OpBundles), Name); + } + + CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), + DefaultDest, IndirectDests, Args, Name); + } + CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), + DefaultDest, IndirectDests, Args, Name); + } + ResumeInst *CreateResume(Value *Exn) { return Insert(ResumeInst::Create(Exn)); } diff --git a/include/llvm/IR/InstVisitor.h b/include/llvm/IR/InstVisitor.h index 18d40d8c120..fbeb2caf14e 100644 --- a/include/llvm/IR/InstVisitor.h +++ b/include/llvm/IR/InstVisitor.h @@ -217,14 +217,17 @@ public: RetTy visitVACopyInst(VACopyInst &I) { DELEGATE(IntrinsicInst); } RetTy visitIntrinsicInst(IntrinsicInst &I) { DELEGATE(CallInst); } - // Call and Invoke are slightly different as they delegate first through - // a generic CallSite visitor. + // Call, Invoke and CallBr are slightly different as they delegate first + // through a generic CallSite visitor. RetTy visitCallInst(CallInst &I) { return static_cast(this)->visitCallSite(&I); } RetTy visitInvokeInst(InvokeInst &I) { return static_cast(this)->visitCallSite(&I); } + RetTy visitCallBrInst(CallBrInst &I) { + return static_cast(this)->visitCallSite(&I); + } // While terminators don't have a distinct type modeling them, we support // intercepting them with dedicated a visitor callback. @@ -270,14 +273,14 @@ public: // The next level delegation for `CallBase` is slightly more complex in order // to support visiting cases where the call is also a terminator. RetTy visitCallBase(CallBase &I) { - if (isa(I)) + if (isa(I) || isa(I)) return static_cast(this)->visitTerminator(I); DELEGATE(Instruction); } - // Provide a legacy visitor for a 'callsite' that visits both calls and - // invokes. + // Provide a legacy visitor for a 'callsite' that visits calls, invokes, + // and calbrs. // // Prefer overriding the type system based `CallBase` instead. RetTy visitCallSite(CallSite CS) { diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h index 351b79a09e6..6f506687ee5 100644 --- a/include/llvm/IR/InstrTypes.h +++ b/include/llvm/IR/InstrTypes.h @@ -1033,16 +1033,23 @@ protected: return 0; case Instruction::Invoke: return 2; + case Instruction::CallBr: + return getNumSubclassExtraOperandsDynamic(); } llvm_unreachable("Invalid opcode!"); } + /// Get the number of extra operands for instructions that don't have a fixed + /// number of extra operands. + unsigned getNumSubclassExtraOperandsDynamic() const; + public: using Instruction::getContext; static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || - I->getOpcode() == Instruction::Invoke; + I->getOpcode() == Instruction::Invoke || + I->getOpcode() == Instruction::CallBr; } static bool classof(const Value *V) { return isa(V) && classof(cast(V)); diff --git a/include/llvm/IR/Instruction.def b/include/llvm/IR/Instruction.def index 4767d5b0021..41cdf613ad6 100644 --- a/include/llvm/IR/Instruction.def +++ b/include/llvm/IR/Instruction.def @@ -134,89 +134,90 @@ HANDLE_TERM_INST ( 7, Unreachable , UnreachableInst) HANDLE_TERM_INST ( 8, CleanupRet , CleanupReturnInst) HANDLE_TERM_INST ( 9, CatchRet , CatchReturnInst) HANDLE_TERM_INST (10, CatchSwitch , CatchSwitchInst) - LAST_TERM_INST (10) +HANDLE_TERM_INST (11, CallBr , CallBrInst) // A call-site terminator + LAST_TERM_INST (11) // Standard unary operators... - FIRST_UNARY_INST(11) -HANDLE_UNARY_INST(11, FNeg , UnaryOperator) - LAST_UNARY_INST(11) + FIRST_UNARY_INST(12) +HANDLE_UNARY_INST(12, FNeg , UnaryOperator) + LAST_UNARY_INST(12) // Standard binary operators... - FIRST_BINARY_INST(12) -HANDLE_BINARY_INST(12, Add , BinaryOperator) -HANDLE_BINARY_INST(13, FAdd , BinaryOperator) -HANDLE_BINARY_INST(14, Sub , BinaryOperator) -HANDLE_BINARY_INST(15, FSub , BinaryOperator) -HANDLE_BINARY_INST(16, Mul , BinaryOperator) -HANDLE_BINARY_INST(17, FMul , BinaryOperator) -HANDLE_BINARY_INST(18, UDiv , BinaryOperator) -HANDLE_BINARY_INST(19, SDiv , BinaryOperator) -HANDLE_BINARY_INST(20, FDiv , BinaryOperator) -HANDLE_BINARY_INST(21, URem , BinaryOperator) -HANDLE_BINARY_INST(22, SRem , BinaryOperator) -HANDLE_BINARY_INST(23, FRem , BinaryOperator) + FIRST_BINARY_INST(13) +HANDLE_BINARY_INST(13, Add , BinaryOperator) +HANDLE_BINARY_INST(14, FAdd , BinaryOperator) +HANDLE_BINARY_INST(15, Sub , BinaryOperator) +HANDLE_BINARY_INST(16, FSub , BinaryOperator) +HANDLE_BINARY_INST(17, Mul , BinaryOperator) +HANDLE_BINARY_INST(18, FMul , BinaryOperator) +HANDLE_BINARY_INST(19, UDiv , BinaryOperator) +HANDLE_BINARY_INST(20, SDiv , BinaryOperator) +HANDLE_BINARY_INST(21, FDiv , BinaryOperator) +HANDLE_BINARY_INST(22, URem , BinaryOperator) +HANDLE_BINARY_INST(23, SRem , BinaryOperator) +HANDLE_BINARY_INST(24, FRem , BinaryOperator) // Logical operators (integer operands) -HANDLE_BINARY_INST(24, Shl , BinaryOperator) // Shift left (logical) -HANDLE_BINARY_INST(25, LShr , BinaryOperator) // Shift right (logical) -HANDLE_BINARY_INST(26, AShr , BinaryOperator) // Shift right (arithmetic) -HANDLE_BINARY_INST(27, And , BinaryOperator) -HANDLE_BINARY_INST(28, Or , BinaryOperator) -HANDLE_BINARY_INST(29, Xor , BinaryOperator) - LAST_BINARY_INST(29) +HANDLE_BINARY_INST(25, Shl , BinaryOperator) // Shift left (logical) +HANDLE_BINARY_INST(26, LShr , BinaryOperator) // Shift right (logical) +HANDLE_BINARY_INST(27, AShr , BinaryOperator) // Shift right (arithmetic) +HANDLE_BINARY_INST(28, And , BinaryOperator) +HANDLE_BINARY_INST(29, Or , BinaryOperator) +HANDLE_BINARY_INST(30, Xor , BinaryOperator) + LAST_BINARY_INST(30) // Memory operators... - FIRST_MEMORY_INST(30) -HANDLE_MEMORY_INST(30, Alloca, AllocaInst) // Stack management -HANDLE_MEMORY_INST(31, Load , LoadInst ) // Memory manipulation instrs -HANDLE_MEMORY_INST(32, Store , StoreInst ) -HANDLE_MEMORY_INST(33, GetElementPtr, GetElementPtrInst) -HANDLE_MEMORY_INST(34, Fence , FenceInst ) -HANDLE_MEMORY_INST(35, AtomicCmpXchg , AtomicCmpXchgInst ) -HANDLE_MEMORY_INST(36, AtomicRMW , AtomicRMWInst ) - LAST_MEMORY_INST(36) + FIRST_MEMORY_INST(31) +HANDLE_MEMORY_INST(31, Alloca, AllocaInst) // Stack management +HANDLE_MEMORY_INST(32, Load , LoadInst ) // Memory manipulation instrs +HANDLE_MEMORY_INST(33, Store , StoreInst ) +HANDLE_MEMORY_INST(34, GetElementPtr, GetElementPtrInst) +HANDLE_MEMORY_INST(35, Fence , FenceInst ) +HANDLE_MEMORY_INST(36, AtomicCmpXchg , AtomicCmpXchgInst ) +HANDLE_MEMORY_INST(37, AtomicRMW , AtomicRMWInst ) + LAST_MEMORY_INST(37) // Cast operators ... // NOTE: The order matters here because CastInst::isEliminableCastPair // NOTE: (see Instructions.cpp) encodes a table based on this ordering. - FIRST_CAST_INST(37) -HANDLE_CAST_INST(37, Trunc , TruncInst ) // Truncate integers -HANDLE_CAST_INST(38, ZExt , ZExtInst ) // Zero extend integers -HANDLE_CAST_INST(39, SExt , SExtInst ) // Sign extend integers -HANDLE_CAST_INST(40, FPToUI , FPToUIInst ) // floating point -> UInt -HANDLE_CAST_INST(41, FPToSI , FPToSIInst ) // floating point -> SInt -HANDLE_CAST_INST(42, UIToFP , UIToFPInst ) // UInt -> floating point -HANDLE_CAST_INST(43, SIToFP , SIToFPInst ) // SInt -> floating point -HANDLE_CAST_INST(44, FPTrunc , FPTruncInst ) // Truncate floating point -HANDLE_CAST_INST(45, FPExt , FPExtInst ) // Extend floating point -HANDLE_CAST_INST(46, PtrToInt, PtrToIntInst) // Pointer -> Integer -HANDLE_CAST_INST(47, IntToPtr, IntToPtrInst) // Integer -> Pointer -HANDLE_CAST_INST(48, BitCast , BitCastInst ) // Type cast -HANDLE_CAST_INST(49, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast - LAST_CAST_INST(49) - - FIRST_FUNCLETPAD_INST(50) -HANDLE_FUNCLETPAD_INST(50, CleanupPad, CleanupPadInst) -HANDLE_FUNCLETPAD_INST(51, CatchPad , CatchPadInst) - LAST_FUNCLETPAD_INST(51) + FIRST_CAST_INST(38) +HANDLE_CAST_INST(38, Trunc , TruncInst ) // Truncate integers +HANDLE_CAST_INST(39, ZExt , ZExtInst ) // Zero extend integers +HANDLE_CAST_INST(40, SExt , SExtInst ) // Sign extend integers +HANDLE_CAST_INST(41, FPToUI , FPToUIInst ) // floating point -> UInt +HANDLE_CAST_INST(42, FPToSI , FPToSIInst ) // floating point -> SInt +HANDLE_CAST_INST(43, UIToFP , UIToFPInst ) // UInt -> floating point +HANDLE_CAST_INST(44, SIToFP , SIToFPInst ) // SInt -> floating point +HANDLE_CAST_INST(45, FPTrunc , FPTruncInst ) // Truncate floating point +HANDLE_CAST_INST(46, FPExt , FPExtInst ) // Extend floating point +HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst) // Pointer -> Integer +HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst) // Integer -> Pointer +HANDLE_CAST_INST(49, BitCast , BitCastInst ) // Type cast +HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast + LAST_CAST_INST(50) + + FIRST_FUNCLETPAD_INST(51) +HANDLE_FUNCLETPAD_INST(51, CleanupPad, CleanupPadInst) +HANDLE_FUNCLETPAD_INST(52, CatchPad , CatchPadInst) + LAST_FUNCLETPAD_INST(52) // Other operators... - FIRST_OTHER_INST(52) -HANDLE_OTHER_INST(52, ICmp , ICmpInst ) // Integer comparison instruction -HANDLE_OTHER_INST(53, FCmp , FCmpInst ) // Floating point comparison instr. -HANDLE_OTHER_INST(54, PHI , PHINode ) // PHI node instruction -HANDLE_OTHER_INST(55, Call , CallInst ) // Call a function -HANDLE_OTHER_INST(56, Select , SelectInst ) // select instruction -HANDLE_USER_INST (57, UserOp1, Instruction) // May be used internally in a pass -HANDLE_USER_INST (58, UserOp2, Instruction) // Internal to passes only -HANDLE_OTHER_INST(59, VAArg , VAArgInst ) // vaarg instruction -HANDLE_OTHER_INST(60, ExtractElement, ExtractElementInst)// extract from vector -HANDLE_OTHER_INST(61, InsertElement, InsertElementInst) // insert into vector -HANDLE_OTHER_INST(62, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. -HANDLE_OTHER_INST(63, ExtractValue, ExtractValueInst)// extract from aggregate -HANDLE_OTHER_INST(64, InsertValue, InsertValueInst) // insert into aggregate -HANDLE_OTHER_INST(65, LandingPad, LandingPadInst) // Landing pad instruction. - LAST_OTHER_INST(65) + FIRST_OTHER_INST(53) +HANDLE_OTHER_INST(53, ICmp , ICmpInst ) // Integer comparison instruction +HANDLE_OTHER_INST(54, FCmp , FCmpInst ) // Floating point comparison instr. +HANDLE_OTHER_INST(55, PHI , PHINode ) // PHI node instruction +HANDLE_OTHER_INST(56, Call , CallInst ) // Call a function +HANDLE_OTHER_INST(57, Select , SelectInst ) // select instruction +HANDLE_USER_INST (58, UserOp1, Instruction) // May be used internally in a pass +HANDLE_USER_INST (59, UserOp2, Instruction) // Internal to passes only +HANDLE_OTHER_INST(60, VAArg , VAArgInst ) // vaarg instruction +HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector +HANDLE_OTHER_INST(62, InsertElement, InsertElementInst) // insert into vector +HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. +HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate +HANDLE_OTHER_INST(65, InsertValue, InsertValueInst) // insert into aggregate +HANDLE_OTHER_INST(66, LandingPad, LandingPadInst) // Landing pad instruction. + LAST_OTHER_INST(66) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h index dfd6bccdda0..b940d125a27 100644 --- a/include/llvm/IR/Instruction.h +++ b/include/llvm/IR/Instruction.h @@ -135,6 +135,9 @@ public: bool isExceptionalTerminator() const { return isExceptionalTerminator(getOpcode()); } + bool isIndirectTerminator() const { + return isIndirectTerminator(getOpcode()); + } static const char* getOpcodeName(unsigned OpCode); @@ -202,6 +205,17 @@ public: } } + /// Returns true if the OpCode is a terminator with indirect targets. + static inline bool isIndirectTerminator(unsigned OpCode) { + switch (OpCode) { + case Instruction::IndirectBr: + case Instruction::CallBr: + return true; + default: + return false; + } + } + //===--------------------------------------------------------------------===// // Metadata manipulation. //===--------------------------------------------------------------------===// diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index f9f3ff373ca..a82cedab25d 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -3886,6 +3886,249 @@ InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr); } +//===----------------------------------------------------------------------===// +// CallBrInst Class +//===----------------------------------------------------------------------===// + +/// CallBr instruction, tracking function calls that may not return control but +/// instead transfer it to a third location. The SubclassData field is used to +/// hold the calling convention of the call. +/// +class CallBrInst : public CallBase { + + unsigned NumIndirectDests; + + CallBrInst(const CallBrInst &BI); + + /// Construct a CallBrInst given a range of arguments. + /// + /// Construct a CallBrInst from a range of arguments + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore); + + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd); + + void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr); + + /// Compute the number of operands to allocate. + static int ComputeNumOperands(int NumArgs, int NumIndirectDests, + int NumBundleInputs = 0) { + // We need one operand for the called function, plus our extra operands and + // the input operand counts provided. + return 2 + NumIndirectDests + NumArgs + NumBundleInputs; + } + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + + CallBrInst *cloneImpl() const; + +public: + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, Bundles, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionCallee Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, Bundles, NameStr, InsertAtEnd); + } + + /// Create a clone of \p CBI with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned callbr instruction is identical to \p CBI in every way + /// except that the operand bundles for the new instruction are set to the + /// operand bundles in \p Bundles. + static CallBrInst *Create(CallBrInst *CBI, + ArrayRef Bundles, + Instruction *InsertPt = nullptr); + + /// Return the number of callbr indirect dest labels. + /// + unsigned getNumIndirectDests() const { return NumIndirectDests; } + + /// getIndirectDestLabel - Return the i-th indirect dest label. + /// + Value *getIndirectDestLabel(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperand(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + Value *getIndirectDestLabelUse(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperandUse(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + // Return the destination basic blocks... + BasicBlock *getDefaultDest() const { + return cast(*(&Op<-1>() - getNumIndirectDests() - 1)); + } + BasicBlock *getIndirectDest(unsigned i) const { + return cast(*(&Op<-1>() - getNumIndirectDests() + i)); + } + SmallVector getIndirectDests() const { + SmallVector IndirectDests; + for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i) + IndirectDests.push_back(getIndirectDest(i)); + return IndirectDests; + } + void setDefaultDest(BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast(B); + } + void setIndirectDest(unsigned i, BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast(B); + } + + BasicBlock *getSuccessor(unsigned i) const { + assert(i < getNumSuccessors() + 1 && + "Successor # out of range for callbr!"); + return i == 0 ? getDefaultDest() : getIndirectDest(i - 1); + } + + void setSuccessor(unsigned idx, BasicBlock *NewSucc) { + assert(idx < getNumIndirectDests() + 1 && + "Successor # out of range for callbr!"); + *(&Op<-1>() - getNumIndirectDests() -1 + idx) = + reinterpret_cast(NewSucc); + } + + unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { + return (I->getOpcode() == Instruction::CallBr); + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + +private: + + // Shadow Instruction::setInstructionSubclassData with a private forwarding + // method so that subclasses cannot accidentally use it. + void setInstructionSubclassData(unsigned short D) { + Instruction::setInstructionSubclassData(D); + } +}; + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore) + : CallBase(Ty->getReturnType(), Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertBefore) { + init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr); +} + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd) + : CallBase( + cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertAtEnd) { + init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr); +} + //===----------------------------------------------------------------------===// // ResumeInst Class //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/TargetOpcodes.def b/include/llvm/Support/TargetOpcodes.def index 23d008a0147..f44f0f131a9 100644 --- a/include/llvm/Support/TargetOpcodes.def +++ b/include/llvm/Support/TargetOpcodes.def @@ -28,6 +28,7 @@ /// HANDLE_TARGET_OPCODE(PHI) HANDLE_TARGET_OPCODE(INLINEASM) +HANDLE_TARGET_OPCODE(INLINEASM_BR) HANDLE_TARGET_OPCODE(CFI_INSTRUCTION) HANDLE_TARGET_OPCODE(EH_LABEL) HANDLE_TARGET_OPCODE(GC_LABEL) diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index af7c93f6f9b..380e8485816 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -933,6 +933,15 @@ def INLINEASM : StandardPseudoInstruction { let AsmString = ""; let hasSideEffects = 0; // Note side effect is encoded in an operand. } +def INLINEASM_BR : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins variable_ops); + let AsmString = ""; + let hasSideEffects = 0; // Note side effect is encoded in an operand. + let isTerminator = 1; + let isBranch = 1; + let isIndirectBranch = 1; +} def CFI_INSTRUCTION : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp index eb49f904ea4..6f88a6b34ca 100644 --- a/lib/Analysis/ValueTracking.cpp +++ b/lib/Analysis/ValueTracking.cpp @@ -3922,6 +3922,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V, case Instruction::VAArg: case Instruction::Alloca: case Instruction::Invoke: + case Instruction::CallBr: case Instruction::PHI: case Instruction::Store: case Instruction::Ret: diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp index b543115a88e..461117c92b8 100644 --- a/lib/AsmParser/LLLexer.cpp +++ b/lib/AsmParser/LLLexer.cpp @@ -858,6 +858,7 @@ lltok::Kind LLLexer::LexIdentifier() { INSTKEYWORD(invoke, Invoke); INSTKEYWORD(resume, Resume); INSTKEYWORD(unreachable, Unreachable); + INSTKEYWORD(callbr, CallBr); INSTKEYWORD(alloca, Alloca); INSTKEYWORD(load, Load); diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index 855c5d26500..2a36bb18984 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -163,6 +163,14 @@ bool LLParser::ValidateEndOfModule() { AS = AS.addAttributes(Context, AttributeList::FunctionIndex, AttributeSet::get(Context, FnAttrs)); II->setAttributes(AS); + } else if (CallBrInst *CBI = dyn_cast(V)) { + AttributeList AS = CBI->getAttributes(); + AttrBuilder FnAttrs(AS.getFnAttributes()); + AS = AS.removeAttributes(Context, AttributeList::FunctionIndex); + FnAttrs.merge(B); + AS = AS.addAttributes(Context, AttributeList::FunctionIndex, + AttributeSet::get(Context, FnAttrs)); + CBI->setAttributes(AS); } else if (auto *GV = dyn_cast(V)) { AttrBuilder Attrs(GV->getAttributes()); Attrs.merge(B); @@ -5566,6 +5574,7 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_catchswitch: return ParseCatchSwitch(Inst, PFS); case lltok::kw_catchpad: return ParseCatchPad(Inst, PFS); case lltok::kw_cleanuppad: return ParseCleanupPad(Inst, PFS); + case lltok::kw_callbr: return ParseCallBr(Inst, PFS); // Unary Operators. case lltok::kw_fneg: { FastMathFlags FMF = EatFastMathFlagsIfPresent(); @@ -6184,6 +6193,124 @@ bool LLParser::ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, return false; } +/// ParseCallBr +/// ::= 'callbr' OptionalCallingConv OptionalAttrs Type Value ParamList +/// OptionalAttrs OptionalOperandBundles 'to' TypeAndValue +/// '[' LabelList ']' +bool LLParser::ParseCallBr(Instruction *&Inst, PerFunctionState &PFS) { + LocTy CallLoc = Lex.getLoc(); + AttrBuilder RetAttrs, FnAttrs; + std::vector FwdRefAttrGrps; + LocTy NoBuiltinLoc; + unsigned CC; + Type *RetType = nullptr; + LocTy RetTypeLoc; + ValID CalleeID; + SmallVector ArgList; + SmallVector BundleList; + + BasicBlock *DefaultDest; + if (ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) || + ParseType(RetType, RetTypeLoc, true /*void allowed*/) || + ParseValID(CalleeID) || ParseParameterList(ArgList, PFS) || + ParseFnAttributeValuePairs(FnAttrs, FwdRefAttrGrps, false, + NoBuiltinLoc) || + ParseOptionalOperandBundles(BundleList, PFS) || + ParseToken(lltok::kw_to, "expected 'to' in callbr") || + ParseTypeAndBasicBlock(DefaultDest, PFS) || + ParseToken(lltok::lsquare, "expected '[' in callbr")) + return true; + + // Parse the destination list. + SmallVector IndirectDests; + + if (Lex.getKind() != lltok::rsquare) { + BasicBlock *DestBB; + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + + while (EatIfPresent(lltok::comma)) { + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + } + } + + if (ParseToken(lltok::rsquare, "expected ']' at end of block list")) + return true; + + // If RetType is a non-function pointer type, then this is the short syntax + // for the call, which means that RetType is just the return type. Infer the + // rest of the function argument types from the arguments that are present. + FunctionType *Ty = dyn_cast(RetType); + if (!Ty) { + // Pull out the types of all of the arguments... + std::vector ParamTypes; + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) + ParamTypes.push_back(ArgList[i].V->getType()); + + if (!FunctionType::isValidReturnType(RetType)) + return Error(RetTypeLoc, "Invalid result type for LLVM function"); + + Ty = FunctionType::get(RetType, ParamTypes, false); + } + + CalleeID.FTy = Ty; + + // Look up the callee. + Value *Callee; + if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS, + /*IsCall=*/true)) + return true; + + if (isa(Callee) && !Ty->getReturnType()->isVoidTy()) + return Error(RetTypeLoc, "asm-goto outputs not supported"); + + // Set up the Attribute for the function. + SmallVector Args; + SmallVector ArgAttrs; + + // Loop through FunctionType's arguments and ensure they are specified + // correctly. Also, gather any parameter attributes. + FunctionType::param_iterator I = Ty->param_begin(); + FunctionType::param_iterator E = Ty->param_end(); + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) { + Type *ExpectedTy = nullptr; + if (I != E) { + ExpectedTy = *I++; + } else if (!Ty->isVarArg()) { + return Error(ArgList[i].Loc, "too many arguments specified"); + } + + if (ExpectedTy && ExpectedTy != ArgList[i].V->getType()) + return Error(ArgList[i].Loc, "argument is not of expected type '" + + getTypeString(ExpectedTy) + "'"); + Args.push_back(ArgList[i].V); + ArgAttrs.push_back(ArgList[i].Attrs); + } + + if (I != E) + return Error(CallLoc, "not enough parameters specified for call"); + + if (FnAttrs.hasAlignmentAttr()) + return Error(CallLoc, "callbr instructions may not have an alignment"); + + // Finish off the Attribute and check them + AttributeList PAL = + AttributeList::get(Context, AttributeSet::get(Context, FnAttrs), + AttributeSet::get(Context, RetAttrs), ArgAttrs); + + CallBrInst *CBI = + CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, + BundleList); + CBI->setCallingConv(CC); + CBI->setAttributes(PAL); + ForwardRefAttrGroups[CBI] = FwdRefAttrGrps; + Inst = CBI; + return false; +} + //===----------------------------------------------------------------------===// // Binary Operators. //===----------------------------------------------------------------------===// diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h index d8efbb1cf10..95aea0c775a 100644 --- a/lib/AsmParser/LLParser.h +++ b/lib/AsmParser/LLParser.h @@ -570,6 +570,7 @@ namespace llvm { bool ParseCatchSwitch(Instruction *&Inst, PerFunctionState &PFS); bool ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS); bool ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS); + bool ParseCallBr(Instruction *&Inst, PerFunctionState &PFS); bool ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc, unsigned OperandType); diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h index 41899b29ce5..88eeae11a4b 100644 --- a/lib/AsmParser/LLToken.h +++ b/lib/AsmParser/LLToken.h @@ -327,6 +327,7 @@ enum Kind { kw_catchret, kw_catchpad, kw_cleanuppad, + kw_callbr, kw_alloca, kw_load, diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 61bc031026b..b94bb66a0d6 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -4231,6 +4231,74 @@ Error BitcodeReader::parseFunctionBody(Function *F) { InstructionList.push_back(I); break; } + case bitc::FUNC_CODE_INST_CALLBR: { + // CALLBR: [attr, cc, norm, transfs, fty, fnid, args] + unsigned OpNum = 0; + AttributeList PAL = getAttributes(Record[OpNum++]); + unsigned CCInfo = Record[OpNum++]; + + BasicBlock *DefaultDest = getBasicBlock(Record[OpNum++]); + unsigned NumIndirectDests = Record[OpNum++]; + SmallVector IndirectDests; + for (unsigned i = 0, e = NumIndirectDests; i != e; ++i) + IndirectDests.push_back(getBasicBlock(Record[OpNum++])); + + FunctionType *FTy = nullptr; + if (CCInfo >> bitc::CALL_EXPLICIT_TYPE & 1 && + !(FTy = dyn_cast(getTypeByID(Record[OpNum++])))) + return error("Explicit call type is not a function type"); + + Value *Callee; + if (getValueTypePair(Record, OpNum, NextValueNo, Callee)) + return error("Invalid record"); + + PointerType *OpTy = dyn_cast(Callee->getType()); + if (!OpTy) + return error("Callee is not a pointer type"); + if (!FTy) { + FTy = dyn_cast(OpTy->getElementType()); + if (!FTy) + return error("Callee is not of pointer to function type"); + } else if (OpTy->getElementType() != FTy) + return error("Explicit call type does not match pointee type of " + "callee operand"); + if (Record.size() < FTy->getNumParams() + OpNum) + return error("Insufficient operands to call"); + + SmallVector Args; + // Read the fixed params. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) { + if (FTy->getParamType(i)->isLabelTy()) + Args.push_back(getBasicBlock(Record[OpNum])); + else + Args.push_back(getValue(Record, OpNum, NextValueNo, + FTy->getParamType(i))); + if (!Args.back()) + return error("Invalid record"); + } + + // Read type/value pairs for varargs params. + if (!FTy->isVarArg()) { + if (OpNum != Record.size()) + return error("Invalid record"); + } else { + while (OpNum != Record.size()) { + Value *Op; + if (getValueTypePair(Record, OpNum, NextValueNo, Op)) + return error("Invalid record"); + Args.push_back(Op); + } + } + + I = CallBrInst::Create(FTy, Callee, DefaultDest, IndirectDests, Args, + OperandBundles); + OperandBundles.clear(); + InstructionList.push_back(I); + cast(I)->setCallingConv( + static_cast((0x7ff & CCInfo) >> bitc::CALL_CCONV)); + cast(I)->setAttributes(PAL); + break; + } case bitc::FUNC_CODE_INST_UNREACHABLE: // UNREACHABLE I = new UnreachableInst(Context); InstructionList.push_back(I); diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index f4a539e51f7..a15ad55b0a3 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -2777,6 +2777,41 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest())); break; } + case Instruction::CallBr: { + const CallBrInst *CBI = cast(&I); + const Value *Callee = CBI->getCalledValue(); + FunctionType *FTy = CBI->getFunctionType(); + + if (CBI->hasOperandBundles()) + writeOperandBundles(CBI, InstID); + + Code = bitc::FUNC_CODE_INST_CALLBR; + + Vals.push_back(VE.getAttributeListID(CBI->getAttributes())); + + Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV | + 1 << bitc::CALL_EXPLICIT_TYPE); + + Vals.push_back(VE.getValueID(CBI->getDefaultDest())); + Vals.push_back(CBI->getNumIndirectDests()); + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) + Vals.push_back(VE.getValueID(CBI->getIndirectDest(i))); + + Vals.push_back(VE.getTypeID(FTy)); + pushValueAndType(Callee, InstID, Vals); + + // Emit value #'s for the fixed parameters. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) + pushValue(I.getOperand(i), InstID, Vals); // fixed param. + + // Emit type/value pairs for varargs params. + if (FTy->isVarArg()) { + for (unsigned i = FTy->getNumParams(), e = CBI->getNumArgOperands(); + i != e; ++i) + pushValueAndType(I.getOperand(i), InstID, Vals); // vararg + } + break; + } case Instruction::Unreachable: Code = bitc::FUNC_CODE_INST_UNREACHABLE; AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV; diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index f2f11c19251..c735efab9c1 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -414,10 +414,8 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateMetadata(&F, MD->getMetadata()); } EnumerateType(I.getType()); - if (const CallInst *CI = dyn_cast(&I)) - EnumerateAttributes(CI->getAttributes()); - else if (const InvokeInst *II = dyn_cast(&I)) - EnumerateAttributes(II->getAttributes()); + if (const auto *Call = dyn_cast(&I)) + EnumerateAttributes(Call->getAttributes()); // Enumerate metadata attached with this instruction. MDs.clear(); diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 7b4679e1f17..5319519ddc0 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1067,6 +1067,7 @@ void AsmPrinter::EmitFunctionBody() { OutStreamer->EmitLabel(MI.getOperand(0).getMCSymbol()); break; case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: EmitInlineAsm(&MI); break; case TargetOpcode::DBG_VALUE: diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index 9e6d35c5e9a..52acabec06d 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -433,9 +433,16 @@ static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI, ++OpNo; // Skip over the ID number. if (Modifier[0] == 'l') { // Labels are target independent. - // FIXME: What if the operand isn't an MBB, report error? - const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); - Sym->print(OS, AP->MAI); + if (MI->getOperand(OpNo).isBlockAddress()) { + const BlockAddress *BA = MI->getOperand(OpNo).getBlockAddress(); + MCSymbol *Sym = AP->GetBlockAddressSymbol(BA); + Sym->print(OS, AP->MAI); + } else if (MI->getOperand(OpNo).isMBB()) { + const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); + Sym->print(OS, AP->MAI); + } else { + Error = true; + } } else { if (InlineAsm::isMemKind(OpFlags)) { Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant, diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index 1e04f7918ee..14f56279e85 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -655,6 +655,16 @@ bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB, BB->getSinglePredecessor()->getSingleSuccessor())) return false; + // Skip merging if the block's successor is also a successor to any callbr + // that leads to this block. + // FIXME: Is this really needed? Is this a correctness issue? + for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { + if (auto *CBI = dyn_cast((*PI)->getTerminator())) + for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i) + if (DestBB == CBI->getSuccessor(i)) + return false; + } + // Try to skip merging if the unique predecessor of BB is terminated by a // switch or indirect branch instruction, and BB is used as an incoming block // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index c6cb17c2b0d..ab9d980ec64 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1259,6 +1259,12 @@ bool IRTranslator::translateInvoke(const User &U, return true; } +bool IRTranslator::translateCallBr(const User &U, + MachineIRBuilder &MIRBuilder) { + // FIXME: Implement this. + return false; +} + bool IRTranslator::translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder) { const LandingPadInst &LP = cast(U); diff --git a/lib/CodeGen/IndirectBrExpandPass.cpp b/lib/CodeGen/IndirectBrExpandPass.cpp index 9a96abfbf8c..7ac093ba4a7 100644 --- a/lib/CodeGen/IndirectBrExpandPass.cpp +++ b/lib/CodeGen/IndirectBrExpandPass.cpp @@ -148,11 +148,9 @@ bool IndirectBrExpandPass::runOnFunction(Function &F) { ConstantInt *BBIndexC = ConstantInt::get(ITy, BBIndex); // Now rewrite the blockaddress to an integer constant based on the index. - // FIXME: We could potentially preserve the uses as arguments to inline asm. - // This would allow some uses such as diagnostic information in crashes to - // have higher quality even when this transform is enabled, but would break - // users that round-trip blockaddresses through inline assembly and then - // back into an indirectbr. + // FIXME: This part doesn't properly recognize other uses of blockaddress + // expressions, for instance, where they are used to pass labels to + // asm-goto. This part of the pass needs a rework. BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(BBIndexC, BA->getType())); } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 618a0b0f7ef..5cbc9e2a88b 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -1048,14 +1048,18 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, break; } - case ISD::INLINEASM: { + case ISD::INLINEASM: + case ISD::INLINEASM_BR: { unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) --NumOps; // Ignore the glue operand. // Create the inline asm machine instruction. - MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), - TII->get(TargetOpcode::INLINEASM)); + unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR + ? TargetOpcode::INLINEASM_BR + : TargetOpcode::INLINEASM; + MachineInstrBuilder MIB = + BuildMI(*MF, Node->getDebugLoc(), TII->get(TgtOpc)); // Add the asm string as an external symbol operand. SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString); diff --git a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp index e85e29d0b7f..34660e3a48e 100644 --- a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp +++ b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp @@ -84,6 +84,7 @@ ResourcePriorityQueue::numberRCValPredInSU(SUnit *SU, unsigned RCId) { case ISD::CopyFromReg: NumberDeps++; break; case ISD::CopyToReg: break; case ISD::INLINEASM: break; + case ISD::INLINEASM_BR: break; } if (!ScegN->isMachineOpcode()) continue; @@ -120,6 +121,7 @@ unsigned ResourcePriorityQueue::numberRCValSuccInSU(SUnit *SU, case ISD::CopyFromReg: break; case ISD::CopyToReg: NumberDeps++; break; case ISD::INLINEASM: break; + case ISD::INLINEASM_BR: break; } if (!ScegN->isMachineOpcode()) continue; @@ -445,6 +447,7 @@ int ResourcePriorityQueue::SUSchedulingCost(SUnit *SU) { break; case ISD::INLINEASM: + case ISD::INLINEASM_BR: ResCount += PriorityThree; break; } @@ -547,6 +550,7 @@ void ResourcePriorityQueue::initNumRegDefsLeft(SUnit *SU) { NodeNumDefs++; break; case ISD::INLINEASM: + case ISD::INLINEASM_BR: NodeNumDefs++; break; } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index d2e97ce4c77..2cb850fa1a3 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -479,7 +479,8 @@ bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU, } for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { - if (Node->getOpcode() == ISD::INLINEASM) { + if (Node->getOpcode() == ISD::INLINEASM || + Node->getOpcode() == ISD::INLINEASM_BR) { // Inline asm can clobber physical defs. unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 7616d11e050..75d9eb265ae 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -708,6 +708,7 @@ void ScheduleDAGRRList::EmitNode(SUnit *SU) { // removed. return; case ISD::INLINEASM: + case ISD::INLINEASM_BR: // For inline asm, clear the pipeline state. HazardRec->Reset(); return; @@ -1347,7 +1348,8 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl &LRegs) { } for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { - if (Node->getOpcode() == ISD::INLINEASM) { + if (Node->getOpcode() == ISD::INLINEASM || + Node->getOpcode() == ISD::INLINEASM_BR) { // Inline asm can clobber physical defs. unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index b205e97fa39..d41158134b2 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2548,6 +2548,35 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { InvokeMBB->normalizeSuccProbs(); // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); +} + +void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { + MachineBasicBlock *CallBrMBB = FuncInfo.MBB; + + // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't + // have to do anything here to lower funclet bundles. + assert(!I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && + "Cannot lower callbrs with arbitrary operand bundles yet!"); + + assert(isa(I.getCalledValue()) && + "Only know how to handle inlineasm callbr"); + visitInlineAsm(&I); + + // Retrieve successors. + MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; + + // Update successor info. + addSuccessorWithProb(CallBrMBB, Return); + for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { + MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)]; + addSuccessorWithProb(CallBrMBB, Target); + } + CallBrMBB->normalizeSuccProbs(); + + // Drop into default successor. DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Return))); @@ -7584,7 +7613,14 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // Process the call argument. BasicBlocks are labels, currently appearing // only in asm's. - if (const BasicBlock *BB = dyn_cast(OpInfo.CallOperandVal)) { + const Instruction *I = CS.getInstruction(); + if (isa(I) && + (ArgNo - 1) >= (cast(I)->getNumArgOperands() - + cast(I)->getNumIndirectDests())) { + const auto *BA = cast(OpInfo.CallOperandVal); + EVT VT = TLI.getValueType(DAG.getDataLayout(), BA->getType(), true); + OpInfo.CallOperand = DAG.getTargetBlockAddress(BA, VT); + } else if (const auto *BB = dyn_cast(OpInfo.CallOperandVal)) { OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); } else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); @@ -7883,7 +7919,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; if (Flag.getNode()) AsmNodeOperands.push_back(Flag); - Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(), + unsigned ISDOpc = isa(CS.getInstruction()) ? ISD::INLINEASM_BR : ISD::INLINEASM; + Chain = DAG.getNode(ISDOpc, getCurSDLoc(), DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); Flag = Chain.getValue(1); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 81941d55007..37b04e99ccf 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -46,6 +46,7 @@ class AtomicRMWInst; class BasicBlock; class BranchInst; class CallInst; +class CallBrInst; class CatchPadInst; class CatchReturnInst; class CatchSwitchInst; @@ -851,6 +852,7 @@ public: private: // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); + void visitCallBr(const CallBrInst &I); void visitResume(const ResumeInst &I); void visitUnary(const User &I, unsigned Opcode); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index c14b94ebd25..490b2f9957e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -172,6 +172,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::UNDEF: return "undef"; case ISD::MERGE_VALUES: return "merge_values"; case ISD::INLINEASM: return "inlineasm"; + case ISD::INLINEASM_BR: return "inlineasm_br"; case ISD::EH_LABEL: return "eh_label"; case ISD::HANDLENODE: return "handlenode"; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 9c69e7eee14..4eabab45e28 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2441,14 +2441,14 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, return !findNonImmUse(Root, N.getNode(), U, IgnoreChains); } -void SelectionDAGISel::Select_INLINEASM(SDNode *N) { +void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) { SDLoc DL(N); std::vector Ops(N->op_begin(), N->op_end()); SelectInlineAsmMemoryOperands(Ops, DL); const EVT VTs[] = {MVT::Other, MVT::Glue}; - SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops); + SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops); New->setNodeId(-1); ReplaceUses(N, New.getNode()); CurDAG->RemoveDeadNode(N); @@ -2998,7 +2998,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, CurDAG->RemoveDeadNode(NodeToMatch); return; case ISD::INLINEASM: - Select_INLINEASM(NodeToMatch); + case ISD::INLINEASM_BR: + Select_INLINEASM(NodeToMatch, + NodeToMatch->getOpcode() == ISD::INLINEASM_BR); return; case ISD::READ_REGISTER: Select_READ_REGISTER(NodeToMatch); diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 0f343f5989f..484dbffefbb 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3289,7 +3289,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, switch (ConstraintLetter) { default: break; case 'X': // Allows any operand; labels (basic block) use this. - if (Op.getOpcode() == ISD::BasicBlock) { + if (Op.getOpcode() == ISD::BasicBlock || + Op.getOpcode() == ISD::TargetBlockAddress) { Ops.push_back(Op); return; } @@ -3776,6 +3777,9 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, return; } + if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress) + return; + // Otherwise, try to resolve it to something we know about by looking at // the actual operand type. if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) { diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 280305d516c..7eeea33a842 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -1455,6 +1455,7 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { case Switch: return 0; case IndirectBr: return 0; case Invoke: return 0; + case CallBr: return 0; case Resume: return 0; case Unreachable: return 0; case CleanupRet: return 0; diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index 980de9ab1ee..af3db5186a2 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -3836,6 +3836,51 @@ void AssemblyWriter::printInstruction(const Instruction &I) { writeOperand(II->getNormalDest(), true); Out << " unwind "; writeOperand(II->getUnwindDest(), true); + } else if (const CallBrInst *CBI = dyn_cast(&I)) { + Operand = CBI->getCalledValue(); + FunctionType *FTy = CBI->getFunctionType(); + Type *RetTy = FTy->getReturnType(); + const AttributeList &PAL = CBI->getAttributes(); + + // Print the calling convention being used. + if (CBI->getCallingConv() != CallingConv::C) { + Out << " "; + PrintCallingConv(CBI->getCallingConv(), Out); + } + + if (PAL.hasAttributes(AttributeList::ReturnIndex)) + Out << ' ' << PAL.getAsString(AttributeList::ReturnIndex); + + // If possible, print out the short form of the callbr instruction. We can + // only do this if the first argument is a pointer to a nonvararg function, + // and if the return type is not a pointer to a function. + // + Out << ' '; + TypePrinter.print(FTy->isVarArg() ? FTy : RetTy, Out); + Out << ' '; + writeOperand(Operand, false); + Out << '('; + for (unsigned op = 0, Eop = CBI->getNumArgOperands(); op < Eop; ++op) { + if (op) + Out << ", "; + writeParamOperand(CBI->getArgOperand(op), PAL.getParamAttributes(op)); + } + + Out << ')'; + if (PAL.hasAttributes(AttributeList::FunctionIndex)) + Out << " #" << Machine.getAttributeGroupSlot(PAL.getFnAttributes()); + + writeOperandBundles(CBI); + + Out << "\n to "; + writeOperand(CBI->getDefaultDest(), true); + Out << " ["; + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) { + if (i != 0) + Out << ", "; + writeOperand(CBI->getIndirectDest(i), true); + } + Out << ']'; } else if (const AllocaInst *AI = dyn_cast(&I)) { Out << ' '; if (AI->isUsedWithInAlloca()) diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index e031886bf15..32f3bfa66b1 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -301,6 +301,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) { case CatchRet: return "catchret"; case CatchPad: return "catchpad"; case CatchSwitch: return "catchswitch"; + case CallBr: return "callbr"; // Standard unary operators... case FNeg: return "fneg"; @@ -405,6 +406,10 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, return CI->getCallingConv() == cast(I2)->getCallingConv() && CI->getAttributes() == cast(I2)->getAttributes() && CI->hasIdenticalOperandBundleSchema(*cast(I2)); + if (const CallBrInst *CI = dyn_cast(I1)) + return CI->getCallingConv() == cast(I2)->getCallingConv() && + CI->getAttributes() == cast(I2)->getAttributes() && + CI->hasIdenticalOperandBundleSchema(*cast(I2)); if (const InsertValueInst *IVI = dyn_cast(I1)) return IVI->getIndices() == cast(I2)->getIndices(); if (const ExtractValueInst *EVI = dyn_cast(I1)) @@ -516,6 +521,7 @@ bool Instruction::mayReadFromMemory() const { return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: return !cast(this)->doesNotAccessMemory(); case Instruction::Store: return !cast(this)->isUnordered(); @@ -535,6 +541,7 @@ bool Instruction::mayWriteToMemory() const { return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: return !cast(this)->onlyReadsMemory(); case Instruction::Load: return !cast(this)->isUnordered(); @@ -772,8 +779,8 @@ void Instruction::updateProfWeight(uint64_t S, uint64_t T) { } void Instruction::setProfWeight(uint64_t W) { - assert((isa(this) || isa(this)) && - "Can only set weights for call and invoke instrucitons"); + assert(isa(this) && + "Can only set weights for call like instructions"); SmallVector Weights; Weights.push_back(W); MDBuilder MDB(getContext()); diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index b9e6cd653ea..766c41188ff 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -256,6 +256,11 @@ void LandingPadInst::addClause(Constant *Val) { Function *CallBase::getCaller() { return getParent()->getParent(); } +unsigned CallBase::getNumSubclassExtraOperandsDynamic() const { + assert(getOpcode() == Instruction::CallBr && "Unexpected opcode!"); + return cast(this)->getNumIndirectDests() + 1; +} + bool CallBase::isIndirectCall() const { const Value *V = getCalledValue(); if (isa(V) || isa(V)) @@ -726,6 +731,76 @@ LandingPadInst *InvokeInst::getLandingPadInst() const { return cast(getUnwindDest()->getFirstNonPHI()); } +//===----------------------------------------------------------------------===// +// CallBrInst Implementation +//===----------------------------------------------------------------------===// + +void CallBrInst::init(FunctionType *FTy, Value *Fn, BasicBlock *Fallthrough, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr) { + this->FTy = FTy; + + assert((int)getNumOperands() == + ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)) && + "NumOperands not set up?"); + NumIndirectDests = IndirectDests.size(); + setDefaultDest(Fallthrough); + for (unsigned i = 0; i != NumIndirectDests; ++i) + setIndirectDest(i, IndirectDests[i]); + setCalledOperand(Fn); + +#ifndef NDEBUG + assert(((Args.size() == FTy->getNumParams()) || + (FTy->isVarArg() && Args.size() > FTy->getNumParams())) && + "Calling a function with bad signature"); + + for (unsigned i = 0, e = Args.size(); i != e; i++) + assert((i >= FTy->getNumParams() || + FTy->getParamType(i) == Args[i]->getType()) && + "Calling a function with a bad signature!"); +#endif + + std::copy(Args.begin(), Args.end(), op_begin()); + + auto It = populateBundleOperandInfos(Bundles, Args.size()); + (void)It; + assert(It + 2 + IndirectDests.size() == op_end() && "Should add up!"); + + setName(NameStr); +} + +CallBrInst::CallBrInst(const CallBrInst &CBI) + : CallBase(CBI.Attrs, CBI.FTy, CBI.getType(), Instruction::CallBr, + OperandTraits::op_end(this) - CBI.getNumOperands(), + CBI.getNumOperands()) { + setCallingConv(CBI.getCallingConv()); + std::copy(CBI.op_begin(), CBI.op_end(), op_begin()); + std::copy(CBI.bundle_op_info_begin(), CBI.bundle_op_info_end(), + bundle_op_info_begin()); + SubclassOptionalData = CBI.SubclassOptionalData; + NumIndirectDests = CBI.NumIndirectDests; +} + +CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef OpB, + Instruction *InsertPt) { + std::vector Args(CBI->arg_begin(), CBI->arg_end()); + + auto *NewCBI = CallBrInst::Create(CBI->getFunctionType(), + CBI->getCalledValue(), + CBI->getDefaultDest(), + CBI->getIndirectDests(), + Args, OpB, CBI->getName(), InsertPt); + NewCBI->setCallingConv(CBI->getCallingConv()); + NewCBI->SubclassOptionalData = CBI->SubclassOptionalData; + NewCBI->setAttributes(CBI->getAttributes()); + NewCBI->setDebugLoc(CBI->getDebugLoc()); + NewCBI->NumIndirectDests = CBI->NumIndirectDests; + return NewCBI; +} + //===----------------------------------------------------------------------===// // ReturnInst Implementation //===----------------------------------------------------------------------===// @@ -3996,6 +4071,14 @@ InvokeInst *InvokeInst::cloneImpl() const { return new(getNumOperands()) InvokeInst(*this); } +CallBrInst *CallBrInst::cloneImpl() const { + if (hasOperandBundles()) { + unsigned DescriptorBytes = getNumOperandBundles() * sizeof(BundleOpInfo); + return new (getNumOperands(), DescriptorBytes) CallBrInst(*this); + } + return new (getNumOperands()) CallBrInst(*this); +} + ResumeInst *ResumeInst::cloneImpl() const { return new (1) ResumeInst(*this); } CleanupReturnInst *CleanupReturnInst::cloneImpl() const { diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp index 2951cf1ed64..38eed76fe45 100644 --- a/lib/IR/Value.cpp +++ b/lib/IR/Value.cpp @@ -57,7 +57,8 @@ Value::Value(Type *ty, unsigned scid) // FIXME: Why isn't this in the subclass gunk?? // Note, we cannot call isa before the CallInst has been // constructed. - if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke) + if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke || + SubclassID == Instruction::CallBr) assert((VTy->isFirstClassType() || VTy->isVoidTy() || VTy->isStructTy()) && "invalid CallInst type!"); else if (SubclassID != BasicBlockVal && diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index 404749b2d8e..4a39ff40fd1 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -466,6 +466,7 @@ private: void visitReturnInst(ReturnInst &RI); void visitSwitchInst(SwitchInst &SI); void visitIndirectBrInst(IndirectBrInst &BI); + void visitCallBrInst(CallBrInst &CBI); void visitSelectInst(SelectInst &SI); void visitUserOp1(Instruction &I); void visitUserOp2(Instruction &I) { visitUserOp1(I); } @@ -2450,6 +2451,26 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { visitTerminator(BI); } +void Verifier::visitCallBrInst(CallBrInst &CBI) { + Assert(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", + &CBI); + Assert(CBI.getType()->isVoidTy(), "Callbr return value is not supported!", + &CBI); + for (unsigned i = 0, e = CBI.getNumSuccessors(); i != e; ++i) + Assert(CBI.getSuccessor(i)->getType()->isLabelTy(), + "Callbr successors must all have pointer type!", &CBI); + for (unsigned i = 0, e = CBI.getNumOperands(); i != e; ++i) { + Assert(i >= CBI.getNumArgOperands() || !isa(CBI.getOperand(i)), + "Using an unescaped label as a callbr argument!", &CBI); + if (isa(CBI.getOperand(i))) + for (unsigned j = i + 1; j != e; ++j) + Assert(CBI.getOperand(i) != CBI.getOperand(j), + "Duplicate callbr destination!", &CBI); + } + + visitTerminator(CBI); +} + void Verifier::visitSelectInst(SelectInst &SI) { Assert(!SelectInst::areInvalidOperands(SI.getOperand(0), SI.getOperand(1), SI.getOperand(2)), diff --git a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 0f0d877685d..1fc44b67136 100644 --- a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -590,6 +590,7 @@ static bool hasSourceMods(const SDNode *N) { case ISD::FDIV: case ISD::FREM: case ISD::INLINEASM: + case ISD::INLINEASM_BR: case AMDGPUISD::INTERP_P1: case AMDGPUISD::INTERP_P2: case AMDGPUISD::DIV_SCALE: diff --git a/lib/Target/AMDGPU/SIISelLowering.cpp b/lib/Target/AMDGPU/SIISelLowering.cpp index 5fa4e3765b4..d6abd183105 100644 --- a/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/lib/Target/AMDGPU/SIISelLowering.cpp @@ -9697,7 +9697,8 @@ static bool isCopyFromRegOfInlineAsm(const SDNode *N) { do { // Follow the chain until we find an INLINEASM node. N = N->getOperand(0).getNode(); - if (N->getOpcode() == ISD::INLINEASM) + if (N->getOpcode() == ISD::INLINEASM || + N->getOpcode() == ISD::INLINEASM_BR) return true; } while (N->getOpcode() == ISD::CopyFromReg); return false; diff --git a/lib/Target/AMDGPU/SIInstrInfo.cpp b/lib/Target/AMDGPU/SIInstrInfo.cpp index c6cd7a1a4a6..0e3048792aa 100644 --- a/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -5313,7 +5313,8 @@ unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { return 0; case TargetOpcode::BUNDLE: return getInstBundleSize(MI); - case TargetOpcode::INLINEASM: { + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { const MachineFunction *MF = MI.getParent()->getParent(); const char *AsmStr = MI.getOperand(0).getSymbolName(); return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp index 648435a3ed1..f765334577d 100644 --- a/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -2615,6 +2615,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) { return; break; case ISD::INLINEASM: + case ISD::INLINEASM_BR: if (tryInlineAsm(N)) return; break; @@ -4319,7 +4320,7 @@ bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){ if (!Changed) return false; - SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), + SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N), CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); New->setNodeId(-1); ReplaceNode(N, New.getNode()); diff --git a/lib/Target/AVR/AVRInstrInfo.cpp b/lib/Target/AVR/AVRInstrInfo.cpp index 4ea30a0fc39..ba7a95e92c5 100644 --- a/lib/Target/AVR/AVRInstrInfo.cpp +++ b/lib/Target/AVR/AVRInstrInfo.cpp @@ -487,7 +487,8 @@ unsigned AVRInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { case TargetOpcode::KILL: case TargetOpcode::DBG_VALUE: return 0; - case TargetOpcode::INLINEASM: { + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { const MachineFunction &MF = *MI.getParent()->getParent(); const AVRTargetMachine &TM = static_cast(MF.getTarget()); const AVRSubtarget &STI = MF.getSubtarget(); diff --git a/lib/Target/Hexagon/HexagonISelLowering.cpp b/lib/Target/Hexagon/HexagonISelLowering.cpp index 0b9f424822c..c9ee83a249f 100644 --- a/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -578,7 +578,8 @@ HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); unsigned LR = HRI.getRARegister(); - if (Op.getOpcode() != ISD::INLINEASM || HMFI.hasClobberLR()) + if ((Op.getOpcode() != ISD::INLINEASM && + Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR()) return Op; unsigned NumOps = Op.getNumOperands(); @@ -1291,6 +1292,7 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); setOperationAction(ISD::INLINEASM, MVT::Other, Custom); + setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); setOperationAction(ISD::PREFETCH, MVT::Other, Custom); setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); @@ -2740,7 +2742,7 @@ HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { unsigned Opc = Op.getOpcode(); // Handle INLINEASM first. - if (Opc == ISD::INLINEASM) + if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR) return LowerINLINEASM(Op, DAG); if (isHvxOperation(Op)) { diff --git a/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/lib/Target/Hexagon/HexagonMachineScheduler.cpp index 3a291bd1d80..0e655502430 100644 --- a/lib/Target/Hexagon/HexagonMachineScheduler.cpp +++ b/lib/Target/Hexagon/HexagonMachineScheduler.cpp @@ -112,6 +112,7 @@ bool VLIWResourceModel::isResourceAvailable(SUnit *SU, bool IsTop) { case TargetOpcode::IMPLICIT_DEF: case TargetOpcode::COPY: case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: break; } @@ -167,6 +168,7 @@ bool VLIWResourceModel::reserveResources(SUnit *SU, bool IsTop) { case TargetOpcode::EH_LABEL: case TargetOpcode::COPY: case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: break; } Packet.push_back(SU); diff --git a/lib/Target/MSP430/MSP430InstrInfo.cpp b/lib/Target/MSP430/MSP430InstrInfo.cpp index de5c243bb2f..5c3a3fc6926 100644 --- a/lib/Target/MSP430/MSP430InstrInfo.cpp +++ b/lib/Target/MSP430/MSP430InstrInfo.cpp @@ -307,7 +307,8 @@ unsigned MSP430InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { case TargetOpcode::KILL: case TargetOpcode::DBG_VALUE: return 0; - case TargetOpcode::INLINEASM: { + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { const MachineFunction *MF = MI.getParent()->getParent(); const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); return TII.getInlineAsmLength(MI.getOperand(0).getSymbolName(), diff --git a/lib/Target/Mips/MipsInstrInfo.cpp b/lib/Target/Mips/MipsInstrInfo.cpp index 610c4079a67..fbd56206b24 100644 --- a/lib/Target/Mips/MipsInstrInfo.cpp +++ b/lib/Target/Mips/MipsInstrInfo.cpp @@ -577,7 +577,8 @@ unsigned MipsInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { switch (MI.getOpcode()) { default: return MI.getDesc().getSize(); - case TargetOpcode::INLINEASM: { // Inline Asm: Variable size. + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { // Inline Asm: Variable size. const MachineFunction *MF = MI.getParent()->getParent(); const char *AsmStr = MI.getOperand(0).getSymbolName(); return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); diff --git a/lib/Target/PowerPC/PPCRegisterInfo.cpp b/lib/Target/PowerPC/PPCRegisterInfo.cpp index 12ebdb4b1d1..86062dde9fa 100644 --- a/lib/Target/PowerPC/PPCRegisterInfo.cpp +++ b/lib/Target/PowerPC/PPCRegisterInfo.cpp @@ -1000,7 +1000,8 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, if (noImmForm) OperandBase = 1; - else if (OpC != TargetOpcode::INLINEASM) { + else if (OpC != TargetOpcode::INLINEASM && + OpC != TargetOpcode::INLINEASM_BR) { assert(ImmToIdxMap.count(OpC) && "No indexed form of load or store available!"); unsigned NewOpcode = ImmToIdxMap.find(OpC)->second; diff --git a/lib/Target/RISCV/RISCVInstrInfo.cpp b/lib/Target/RISCV/RISCVInstrInfo.cpp index afa872b4a5c..ddb976b47fb 100644 --- a/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -439,7 +439,8 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { case RISCV::PseudoCALL: case RISCV::PseudoTAIL: return 8; - case TargetOpcode::INLINEASM: { + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { const MachineFunction &MF = *MI.getParent()->getParent(); const auto &TM = static_cast(MF.getTarget()); return getInlineAsmLength(MI.getOperand(0).getSymbolName(), diff --git a/lib/Target/Sparc/SparcISelDAGToDAG.cpp b/lib/Target/Sparc/SparcISelDAGToDAG.cpp index 17ad34663da..8cff50d19ed 100644 --- a/lib/Target/Sparc/SparcISelDAGToDAG.cpp +++ b/lib/Target/Sparc/SparcISelDAGToDAG.cpp @@ -312,7 +312,7 @@ bool SparcDAGToDAGISel::tryInlineAsm(SDNode *N){ SelectInlineAsmMemoryOperands(AsmNodeOperands, SDLoc(N)); - SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), + SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N), CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); New->setNodeId(-1); ReplaceNode(N, New.getNode()); @@ -328,7 +328,8 @@ void SparcDAGToDAGISel::Select(SDNode *N) { switch (N->getOpcode()) { default: break; - case ISD::INLINEASM: { + case ISD::INLINEASM: + case ISD::INLINEASM_BR: { if (tryInlineAsm(N)) return; break; diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp index 8b5ac8fd066..cd2ae824982 100644 --- a/lib/Target/X86/X86AsmPrinter.cpp +++ b/lib/Target/X86/X86AsmPrinter.cpp @@ -253,6 +253,11 @@ static void printOperand(X86AsmPrinter &P, const MachineInstr *MI, printSymbolOperand(P, MO, O); break; } + case MachineOperand::MO_BlockAddress: { + MCSymbol *Sym = P.GetBlockAddressSymbol(MO.getBlockAddress()); + Sym->print(O, P.MAI); + break; + } } } diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp index bd2e8a2c063..3ef2c1b5171 100644 --- a/lib/Target/X86/X86FloatingPoint.cpp +++ b/lib/Target/X86/X86FloatingPoint.cpp @@ -1476,7 +1476,8 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { break; } - case TargetOpcode::INLINEASM: { + case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: { // The inline asm MachineInstr currently only *uses* FP registers for the // 'f' constraint. These should be turned into the current ST(x) register // in the machine instr. diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp index b34b3fd1619..ab7c3a5e294 100644 --- a/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file implements the visitCall and visitInvoke functions. +// This file implements the visitCall, visitInvoke, and visitCallBr functions. // //===----------------------------------------------------------------------===// @@ -1834,8 +1834,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { IntrinsicInst *II = dyn_cast(&CI); if (!II) return visitCallBase(CI); - // Intrinsics cannot occur in an invoke, so handle them here instead of in - // visitCallBase. + // Intrinsics cannot occur in an invoke or a callbr, so handle them here + // instead of in visitCallBase. if (auto *MI = dyn_cast(II)) { bool Changed = false; @@ -4017,6 +4017,11 @@ Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) { return visitCallBase(II); } +// CallBrInst simplification +Instruction *InstCombiner::visitCallBrInst(CallBrInst &CBI) { + return visitCallBase(CBI); +} + /// If this cast does not affect the value passed through the varargs area, we /// can eliminate the use of the cast. static bool isSafeToEliminateVarargsCast(const CallBase &Call, @@ -4145,7 +4150,7 @@ static IntrinsicInst *findInitTrampoline(Value *Callee) { return nullptr; } -/// Improvements for call and invoke instructions. +/// Improvements for call, callbr and invoke instructions. Instruction *InstCombiner::visitCallBase(CallBase &Call) { if (isAllocLikeFn(&Call, &TLI)) return visitAllocSite(Call); @@ -4178,7 +4183,7 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) { } // If the callee is a pointer to a function, attempt to move any casts to the - // arguments of the call/invoke. + // arguments of the call/callbr/invoke. Value *Callee = Call.getCalledValue(); if (!isa(Callee) && transformConstExprCastCall(Call)) return nullptr; @@ -4211,9 +4216,9 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) { if (isa(OldCall)) return eraseInstFromFunction(*OldCall); - // We cannot remove an invoke, because it would change the CFG, just - // change the callee to a null pointer. - cast(OldCall)->setCalledFunction( + // We cannot remove an invoke or a callbr, because it would change thexi + // CFG, just change the callee to a null pointer. + cast(OldCall)->setCalledFunction( CalleeF->getFunctionType(), Constant::getNullValue(CalleeF->getType())); return nullptr; @@ -4228,8 +4233,8 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) { if (!Call.getType()->isVoidTy()) replaceInstUsesWith(Call, UndefValue::get(Call.getType())); - if (isa(Call)) { - // Can't remove an invoke because we cannot change the CFG. + if (Call.isTerminator()) { + // Can't remove an invoke or callbr because we cannot change the CFG. return nullptr; } @@ -4282,7 +4287,7 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) { } /// If the callee is a constexpr cast of a function, attempt to move the cast to -/// the arguments of the call/invoke. +/// the arguments of the call/callbr/invoke. bool InstCombiner::transformConstExprCastCall(CallBase &Call) { auto *Callee = dyn_cast(Call.getCalledValue()->stripPointerCasts()); if (!Callee) @@ -4333,17 +4338,21 @@ bool InstCombiner::transformConstExprCastCall(CallBase &Call) { return false; // Attribute not compatible with transformed value. } - // If the callbase is an invoke instruction, and the return value is used by - // a PHI node in a successor, we cannot change the return type of the call - // because there is no place to put the cast instruction (without breaking - // the critical edge). Bail out in this case. - if (!Caller->use_empty()) + // If the callbase is an invoke/callbr instruction, and the return value is + // used by a PHI node in a successor, we cannot change the return type of + // the call because there is no place to put the cast instruction (without + // breaking the critical edge). Bail out in this case. + if (!Caller->use_empty()) { if (InvokeInst *II = dyn_cast(Caller)) for (User *U : II->users()) if (PHINode *PN = dyn_cast(U)) if (PN->getParent() == II->getNormalDest() || PN->getParent() == II->getUnwindDest()) return false; + // FIXME: Be conservative for callbr to avoid a quadratic search. + if (CallBrInst *CBI = dyn_cast(Caller)) + return false; + } } unsigned NumActualArgs = Call.arg_size(); @@ -4497,6 +4506,9 @@ bool InstCombiner::transformConstExprCastCall(CallBase &Call) { if (InvokeInst *II = dyn_cast(Caller)) { NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(), II->getUnwindDest(), Args, OpBundles); + } else if (CallBrInst *CBI = dyn_cast(Caller)) { + NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(), + CBI->getIndirectDests(), Args, OpBundles); } else { NewCall = Builder.CreateCall(Callee, Args, OpBundles); cast(NewCall)->setTailCallKind( @@ -4520,11 +4532,14 @@ bool InstCombiner::transformConstExprCastCall(CallBase &Call) { NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy); NC->setDebugLoc(Caller->getDebugLoc()); - // If this is an invoke instruction, we should insert it after the first - // non-phi, instruction in the normal successor block. + // If this is an invoke/callbr instruction, we should insert it after the + // first non-phi instruction in the normal successor block. if (InvokeInst *II = dyn_cast(Caller)) { BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt(); InsertNewInstBefore(NC, *I); + } else if (CallBrInst *CBI = dyn_cast(Caller)) { + BasicBlock::iterator I = CBI->getDefaultDest()->getFirstInsertionPt(); + InsertNewInstBefore(NC, *I); } else { // Otherwise, it's a call, just insert cast right after the call. InsertNewInstBefore(NC, *Caller); @@ -4673,6 +4688,12 @@ InstCombiner::transformCallThroughTrampoline(CallBase &Call, NewArgs, OpBundles); cast(NewCaller)->setCallingConv(II->getCallingConv()); cast(NewCaller)->setAttributes(NewPAL); + } else if (CallBrInst *CBI = dyn_cast(&Call)) { + NewCaller = + CallBrInst::Create(NewFTy, NewCallee, CBI->getDefaultDest(), + CBI->getIndirectDests(), NewArgs, OpBundles); + cast(NewCaller)->setCallingConv(CBI->getCallingConv()); + cast(NewCaller)->setAttributes(NewPAL); } else { NewCaller = CallInst::Create(NewFTy, NewCallee, NewArgs, OpBundles); cast(NewCaller)->setTailCallKind( diff --git a/lib/Transforms/InstCombine/InstCombineInternal.h b/lib/Transforms/InstCombine/InstCombineInternal.h index 951e0e72e9e..5b0c7fce0d1 100644 --- a/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/lib/Transforms/InstCombine/InstCombineInternal.h @@ -392,6 +392,7 @@ public: Instruction *visitSelectInst(SelectInst &SI); Instruction *visitCallInst(CallInst &CI); Instruction *visitInvokeInst(InvokeInst &II); + Instruction *visitCallBrInst(CallBrInst &CBI); Instruction *SliceUpIllegalIntegerPHI(PHINode &PN); Instruction *visitPHINode(PHINode &PN); diff --git a/lib/Transforms/InstCombine/InstructionCombining.cpp b/lib/Transforms/InstCombine/InstructionCombining.cpp index 1f40c8f5a4a..4d04e3ff99e 100644 --- a/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -921,8 +921,8 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) { // If the InVal is an invoke at the end of the pred block, then we can't // insert a computation after it without breaking the edge. - if (InvokeInst *II = dyn_cast(InVal)) - if (II->getParent() == NonConstBB) + if (isa(InVal)) + if (cast(InVal)->getParent() == NonConstBB) return nullptr; // If the incoming non-constant value is in I's block, we will remove one diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp index 7595ae05787..a02f32f5643 100644 --- a/lib/Transforms/Scalar/GVN.cpp +++ b/lib/Transforms/Scalar/GVN.cpp @@ -1131,6 +1131,14 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock, return false; } + // FIXME: Can we support the fallthrough edge? + if (isa(Pred->getTerminator())) { + LLVM_DEBUG( + dbgs() << "COULD NOT PRE LOAD BECAUSE OF CALLBR CRITICAL EDGE '" + << Pred->getName() << "': " << *LI << '\n'); + return false; + } + if (LoadBB->isEHPad()) { LLVM_DEBUG( dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '" @@ -2167,8 +2175,8 @@ bool GVN::performScalarPRE(Instruction *CurInst) { return false; // We don't currently value number ANY inline asm calls. - if (CallInst *CallI = dyn_cast(CurInst)) - if (CallI->isInlineAsm()) + if (auto *CallB = dyn_cast(CurInst)) + if (CallB->isInlineAsm()) return false; uint32_t ValNo = VN.lookup(CurInst); @@ -2251,6 +2259,11 @@ bool GVN::performScalarPRE(Instruction *CurInst) { if (isa(PREPred->getTerminator())) return false; + // Don't do PRE across callbr. + // FIXME: Can we do this across the fallthrough edge? + if (isa(PREPred->getTerminator())) + return false; + // We can't do PRE safely on a critical edge, so instead we schedule // the edge to be split and perform the PRE the next time we iterate // on the function. diff --git a/lib/Transforms/Scalar/JumpThreading.cpp b/lib/Transforms/Scalar/JumpThreading.cpp index 7738a79425b..f74f7e28e52 100644 --- a/lib/Transforms/Scalar/JumpThreading.cpp +++ b/lib/Transforms/Scalar/JumpThreading.cpp @@ -1055,7 +1055,7 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) { Condition = IB->getAddress()->stripPointerCasts(); Preference = WantBlockAddress; } else { - return false; // Must be an invoke. + return false; // Must be an invoke or callbr. } // Run constant folding to see if we can reduce the condition to a simple @@ -1428,7 +1428,9 @@ bool JumpThreadingPass::SimplifyPartiallyRedundantLoad(LoadInst *LoadI) { // Add all the unavailable predecessors to the PredsToSplit list. for (BasicBlock *P : predecessors(LoadBB)) { // If the predecessor is an indirect goto, we can't split the edge. - if (isa(P->getTerminator())) + // Same for CallBr. + if (isa(P->getTerminator()) || + isa(P->getTerminator())) return false; if (!AvailablePredSet.count(P)) @@ -1641,8 +1643,9 @@ bool JumpThreadingPass::ProcessThreadableEdges(Value *Cond, BasicBlock *BB, ++PredWithKnownDest; // If the predecessor ends with an indirect goto, we can't change its - // destination. - if (isa(Pred->getTerminator())) + // destination. Same for CallBr. + if (isa(Pred->getTerminator()) || + isa(Pred->getTerminator())) continue; PredToDestList.push_back(std::make_pair(Pred, DestBB)); diff --git a/lib/Transforms/Scalar/SCCP.cpp b/lib/Transforms/Scalar/SCCP.cpp index 5dd7f43e662..39d294f8602 100644 --- a/lib/Transforms/Scalar/SCCP.cpp +++ b/lib/Transforms/Scalar/SCCP.cpp @@ -638,6 +638,11 @@ private: visitTerminator(II); } + void visitCallBrInst (CallBrInst &CBI) { + visitCallSite(&CBI); + visitTerminator(CBI); + } + void visitCallSite (CallSite CS); void visitResumeInst (ResumeInst &I) { /*returns void*/ } void visitUnreachableInst(UnreachableInst &I) { /*returns void*/ } @@ -733,6 +738,13 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI, return; } + // In case of callbr, we pessimistically assume that all successors are + // feasible. + if (isa(&TI)) { + Succs.assign(TI.getNumSuccessors(), true); + return; + } + LLVM_DEBUG(dbgs() << "Unknown terminator instruction: " << TI << '\n'); llvm_unreachable("SCCP: Don't know how to handle this terminator!"); } @@ -1597,6 +1609,7 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) { return true; case Instruction::Call: case Instruction::Invoke: + case Instruction::CallBr: // There are two reasons a call can have an undef result // 1. It could be tracked. // 2. It could be constant-foldable. diff --git a/lib/Transforms/Utils/BasicBlockUtils.cpp b/lib/Transforms/Utils/BasicBlockUtils.cpp index 41ad4fefe1f..2410f652fd7 100644 --- a/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -549,6 +549,8 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB, // all BlockAddress uses would need to be updated. assert(!isa(Preds[i]->getTerminator()) && "Cannot split an edge from an IndirectBrInst"); + assert(!isa(Preds[i]->getTerminator()) && + "Cannot split an edge from a CallBrInst"); Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB); } diff --git a/lib/Transforms/Utils/BreakCriticalEdges.cpp b/lib/Transforms/Utils/BreakCriticalEdges.cpp index ab604a6c57c..d73fefdf9c9 100644 --- a/lib/Transforms/Utils/BreakCriticalEdges.cpp +++ b/lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -144,6 +144,10 @@ llvm::SplitCriticalEdge(Instruction *TI, unsigned SuccNum, // it in this generic function. if (DestBB->isEHPad()) return nullptr; + // Don't split the non-fallthrough edge from a callbr. + if (isa(TI) && SuccNum > 0) + return nullptr; + // Create a new basic block, linking it into the CFG. BasicBlock *NewBB = BasicBlock::Create(TI->getContext(), TIBB->getName() + "." + DestBB->getName() + "_crit_edge"); diff --git a/lib/Transforms/Utils/InlineFunction.cpp b/lib/Transforms/Utils/InlineFunction.cpp index 9015b36fa35..7443a7f9c5e 100644 --- a/lib/Transforms/Utils/InlineFunction.cpp +++ b/lib/Transforms/Utils/InlineFunction.cpp @@ -1504,6 +1504,10 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, assert(TheCall->getParent() && TheCall->getFunction() && "Instruction not in function!"); + // FIXME: we don't inline callbr yet. + if (isa(TheCall)) + return false; + // If IFI has any state in it, zap it before we fill it in. IFI.reset(); @@ -1729,6 +1733,8 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, Instruction *NewI = nullptr; if (isa(I)) NewI = CallInst::Create(cast(I), OpDefs, I); + else if (isa(I)) + NewI = CallBrInst::Create(cast(I), OpDefs, I); else NewI = InvokeInst::Create(cast(I), OpDefs, I); @@ -2031,6 +2037,8 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, Instruction *NewInst; if (CS.isCall()) NewInst = CallInst::Create(cast(I), OpBundles, I); + else if (CS.isCallBr()) + NewInst = CallBrInst::Create(cast(I), OpBundles, I); else NewInst = InvokeInst::Create(cast(I), OpBundles, I); NewInst->takeName(I); diff --git a/lib/Transforms/Utils/Local.cpp b/lib/Transforms/Utils/Local.cpp index 70812dc2de8..062bbcdae2c 100644 --- a/lib/Transforms/Utils/Local.cpp +++ b/lib/Transforms/Utils/Local.cpp @@ -996,6 +996,18 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB, } } + // We cannot fold the block if it's a branch to an already present callbr + // successor because that creates duplicate successors. + for (auto I = pred_begin(BB), E = pred_end(BB); I != E; ++I) { + if (auto *CBI = dyn_cast((*I)->getTerminator())) { + if (Succ == CBI->getDefaultDest()) + return false; + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) + if (Succ == CBI->getIndirectDest(i)) + return false; + } + } + LLVM_DEBUG(dbgs() << "Killing Trivial BB: \n" << *BB); SmallVector Updates; diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp index b2aa20bc0f8..954e8038dfb 100644 --- a/lib/Transforms/Utils/LoopSimplify.cpp +++ b/lib/Transforms/Utils/LoopSimplify.cpp @@ -27,6 +27,9 @@ // to transform the loop and make these guarantees. Client code should check // that these conditions are true before relying on them. // +// Similar complications arise from callbr instructions, particularly in +// asm-goto where blockaddress expressions are used. +// // Note that the simplifycfg pass will clean up blocks which are split out but // end up being unnecessary, so usage of this pass should not pessimize // generated code. @@ -123,10 +126,11 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT, PI != PE; ++PI) { BasicBlock *P = *PI; if (!L->contains(P)) { // Coming in from outside the loop? - // If the loop is branched to from an indirect branch, we won't + // If the loop is branched to from an indirect terminator, we won't // be able to fully transform the loop, because it prohibits // edge splitting. - if (isa(P->getTerminator())) return nullptr; + if (P->getTerminator()->isIndirectTerminator()) + return nullptr; // Keep track of it. OutsideBlocks.push_back(P); @@ -235,8 +239,8 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader, for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { if (PN->getIncomingValue(i) != PN || !L->contains(PN->getIncomingBlock(i))) { - // We can't split indirectbr edges. - if (isa(PN->getIncomingBlock(i)->getTerminator())) + // We can't split indirect control flow edges. + if (PN->getIncomingBlock(i)->getTerminator()->isIndirectTerminator()) return nullptr; OuterLoopPreds.push_back(PN->getIncomingBlock(i)); } @@ -357,8 +361,8 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader, for (pred_iterator I = pred_begin(Header), E = pred_end(Header); I != E; ++I){ BasicBlock *P = *I; - // Indirectbr edges cannot be split, so we must fail if we find one. - if (isa(P->getTerminator())) + // Indirect edges cannot be split, so we must fail if we find one. + if (P->getTerminator()->isIndirectTerminator()) return nullptr; if (P != Preheader) BackedgeBlocks.push_back(P); diff --git a/lib/Transforms/Utils/LoopUtils.cpp b/lib/Transforms/Utils/LoopUtils.cpp index 5e661ae8c21..5539ff12e4a 100644 --- a/lib/Transforms/Utils/LoopUtils.cpp +++ b/lib/Transforms/Utils/LoopUtils.cpp @@ -65,6 +65,9 @@ bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, if (isa(PredBB->getTerminator())) // We cannot rewrite exiting edges from an indirectbr. return false; + if (isa(PredBB->getTerminator())) + // We cannot rewrite exiting edges from a callbr. + return false; InLoopPredecessors.push_back(PredBB); } else { diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp index 3fec17ac8cc..00bcb8479c3 100644 --- a/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1265,8 +1265,10 @@ static bool HoistThenElseCodeToIf(BranchInst *BI, while (isa(I2)) I2 = &*BB2_Itr++; } + // FIXME: Can we define a safety predicate for CallBr? if (isa(I1) || !I1->isIdenticalToWhenDefined(I2) || - (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))) + (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) || + isa(I1)) return false; BasicBlock *BIParent = BI->getParent(); @@ -1349,9 +1351,14 @@ static bool HoistThenElseCodeToIf(BranchInst *BI, HoistTerminator: // It may not be possible to hoist an invoke. + // FIXME: Can we define a safety predicate for CallBr? if (isa(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) return Changed; + // TODO: callbr hoisting currently disabled pending further study. + if (isa(I1)) + return Changed; + for (BasicBlock *Succ : successors(BB1)) { for (PHINode &PN : Succ->phis()) { Value *BB1V = PN.getIncomingValueForBlock(BB1); @@ -1443,7 +1450,7 @@ static bool canSinkInstructions( // Conservatively return false if I is an inline-asm instruction. Sinking // and merging inline-asm instructions can potentially create arguments // that cannot satisfy the inline-asm constraints. - if (const auto *C = dyn_cast(I)) + if (const auto *C = dyn_cast(I)) if (C->isInlineAsm()) return false; @@ -1506,7 +1513,7 @@ static bool canSinkInstructions( // We can't create a PHI from this GEP. return false; // Don't create indirect calls! The called value is the final operand. - if ((isa(I0) || isa(I0)) && OI == OE - 1) { + if (isa(I0) && OI == OE - 1) { // FIXME: if the call was *already* indirect, we should do this. return false; } diff --git a/test/Bitcode/callbr.ll b/test/Bitcode/callbr.ll new file mode 100644 index 00000000000..ecc397ac754 --- /dev/null +++ b/test/Bitcode/callbr.ll @@ -0,0 +1,14 @@ +; RUN: llvm-dis < %s.bc | FileCheck %s + +; callbr.ll.bc was generated by passing this file to llvm-as. + +define i32 @test_asm_goto(i32 %x){ +entry: +; CHECK: callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail)) +; CHECK-NEXT: to label %normal [label %fail] + callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail)) to label %normal [label %fail] +normal: + ret i32 1 +fail: + ret i32 0 +} diff --git a/test/Bitcode/callbr.ll.bc b/test/Bitcode/callbr.ll.bc new file mode 100644 index 0000000000000000000000000000000000000000..d07cba433c191c375f19113644f3532b87db033b GIT binary patch literal 1036 zcmX|AU2NM_6uya*I!#i?Yr~dO+p^nDil+XU6hw>@w8jht5lxvkq#_kH&f2U+VRe3# zrX4Ct+_P|X*9LeX{ z=brQVJKw$6BW&!wWCcJE00^4d)4sRA4E?tA>d;%aaeWo$88ZL}aLuG;fELD+@Ili9 zcgCu^H#%?!x1y2FXY__%N_Q)m!q00xD<+SOJFHHF?MRoC}B%U(lUYT6dP%Xj?*%Ui2-^e&?Dqj-ZP>^Svu7<#8;isDW$_OV_ z<{=*gx@bqEh4jQejva1bKl`yi*@5z-F{)8N^|(&%K?2@hzLasVXZmE-nG?90>bhU% zsw$k(Rr9V#3@Bt3fdY=B;1e4>({TSy#;6_YiY|^z1;GvFDGA&|b@_n!y4dw}thMb2 z4EZl#@Bu6r5&E))@QuVq|N(7dXFd65R-IT(9%!XPYd->J)&y0o-^dm3!e zmT=0XJHu+hM`$Nl4&XYi!&))+$w;fWjqAt1w~8n6iBo|WFI2#Howzja&lB*i8(E!iCF0Yu_@o$nZ}RF~e2yDthuK)-8hv#(J|wbSXo?H9 Z3PwD$@hfvv(<4{Ja5xwp4u{)V{{XIEQT+e_ literal 0 HcmV?d00001 diff --git a/test/CodeGen/X86/callbr-asm-blockplacement.ll b/test/CodeGen/X86/callbr-asm-blockplacement.ll new file mode 100644 index 00000000000..d0af12fa20e --- /dev/null +++ b/test/CodeGen/X86/callbr-asm-blockplacement.ll @@ -0,0 +1,106 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s + +; This test asserted in MachineBlockPlacement during asm-goto bring up. + +%struct.wibble = type { %struct.pluto, i32, i8* } +%struct.pluto = type { i32, i32, i32 } + +@global = external global [0 x %struct.wibble] + +define i32 @foo(i32 %arg, i32 (i8*)* %arg3) nounwind { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %bb +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %r13 +; CHECK-NEXT: pushq %r12 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: movabsq $-2305847407260205056, %rbx # imm = 0xDFFFFC0000000000 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: testb %al, %al +; CHECK-NEXT: jne .LBB0_5 +; CHECK-NEXT: # %bb.1: # %bb5 +; CHECK-NEXT: movq %rsi, %r14 +; CHECK-NEXT: movslq %edi, %rbp +; CHECK-NEXT: leaq (,%rbp,8), %rax +; CHECK-NEXT: leaq global(%rax,%rax,2), %r15 +; CHECK-NEXT: leaq global+4(%rax,%rax,2), %r12 +; CHECK-NEXT: xorl %r13d, %r13d +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .LBB0_2: # %bb8 +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: callq bar +; CHECK-NEXT: movq %rax, %rbx +; CHECK-NEXT: movq %rax, %rdi +; CHECK-NEXT: callq *%r14 +; CHECK-NEXT: movq %r15, %rdi +; CHECK-NEXT: callq hoge +; CHECK-NEXT: movq %r12, %rdi +; CHECK-NEXT: callq hoge +; CHECK-NEXT: testb %r13b, %r13b +; CHECK-NEXT: jne .LBB0_2 +; CHECK-NEXT: # %bb.3: # %bb15 +; CHECK-NEXT: leaq (%rbp,%rbp,2), %rax +; CHECK-NEXT: movq %rbx, global+16(,%rax,8) +; CHECK-NEXT: movabsq $-2305847407260205056, %rbx # imm = 0xDFFFFC0000000000 +; CHECK-NEXT: #APP +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: .LBB0_4: # %bb17 +; CHECK-NEXT: callq widget +; CHECK-NEXT: .Ltmp0: # Block address taken +; CHECK-NEXT: .LBB0_5: # %bb18 +; CHECK-NEXT: movw $0, 14(%rbx) +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r13 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq +bb: + %tmp = add i64 0, -2305847407260205056 + %tmp4 = sext i32 %arg to i64 + br i1 undef, label %bb18, label %bb5 + +bb5: ; preds = %bb + %tmp6 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 0, i32 0 + %tmp7 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 0, i32 1 + br label %bb8 + +bb8: ; preds = %bb8, %bb5 + %tmp9 = call i8* @bar(i64 undef) + %tmp10 = call i32 %arg3(i8* nonnull %tmp9) + %tmp11 = ptrtoint i32* %tmp6 to i64 + call void @hoge(i64 %tmp11) + %tmp12 = ptrtoint i32* %tmp7 to i64 + %tmp13 = add i64 undef, -2305847407260205056 + call void @hoge(i64 %tmp12) + %tmp14 = icmp eq i32 0, 0 + br i1 %tmp14, label %bb15, label %bb8 + +bb15: ; preds = %bb8 + %tmp16 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 2 + store i8* %tmp9, i8** %tmp16 + callbr void asm sideeffect "", "X"(i8* blockaddress(@foo, %bb18)) + to label %bb17 [label %bb18] + +bb17: ; preds = %bb15 + call void @widget() + br label %bb18 + +bb18: ; preds = %bb17, %bb15, %bb + %tmp19 = add i64 %tmp, 14 + %tmp20 = inttoptr i64 %tmp19 to i16* + store i16 0, i16* %tmp20 + ret i32 undef +} + +declare i8* @bar(i64) + +declare void @widget() + +declare void @hoge(i64) diff --git a/test/CodeGen/X86/callbr-asm-branch-folding.ll b/test/CodeGen/X86/callbr-asm-branch-folding.ll new file mode 100644 index 00000000000..bdbff4ac89b --- /dev/null +++ b/test/CodeGen/X86/callbr-asm-branch-folding.ll @@ -0,0 +1,151 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s + +; This test hung in the BranchFolding pass during asm-goto bring up + +@e = global i32 0 +@j = global i32 0 + +define void @n(i32* %o, i32 %p, i32 %u) nounwind { +; CHECK-LABEL: n: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %r13 +; CHECK-NEXT: pushq %r12 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: movl %edx, %ebx +; CHECK-NEXT: movl %esi, %r12d +; CHECK-NEXT: movq %rdi, %r15 +; CHECK-NEXT: callq c +; CHECK-NEXT: movl %eax, %r13d +; CHECK-NEXT: movq %r15, %rdi +; CHECK-NEXT: callq l +; CHECK-NEXT: testl %eax, %eax +; CHECK-NEXT: je .LBB0_1 +; CHECK-NEXT: .LBB0_10: # %cleanup +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r13 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB0_1: # %if.end +; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-NEXT: cmpl $0, {{.*}}(%rip) +; CHECK-NEXT: # implicit-def: $ebx +; CHECK-NEXT: # implicit-def: $r14d +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: # %bb.2: # %if.then4 +; CHECK-NEXT: movslq %r12d, %rdi +; CHECK-NEXT: callq m +; CHECK-NEXT: # implicit-def: $ebx +; CHECK-NEXT: # implicit-def: $ebp +; CHECK-NEXT: .LBB0_3: # %r +; CHECK-NEXT: callq c +; CHECK-NEXT: movl %ebp, %r14d +; CHECK-NEXT: .LBB0_4: # %if.end8 +; CHECK-NEXT: movl %ebx, %edi +; CHECK-NEXT: callq i +; CHECK-NEXT: movl %eax, %ebp +; CHECK-NEXT: orl %r14d, %ebp +; CHECK-NEXT: testl %r13d, %r13d +; CHECK-NEXT: je .LBB0_6 +; CHECK-NEXT: # %bb.5: +; CHECK-NEXT: andl $4, %ebx +; CHECK-NEXT: jmp .LBB0_3 +; CHECK-NEXT: .LBB0_6: # %if.end12 +; CHECK-NEXT: testl %ebp, %ebp +; CHECK-NEXT: je .LBB0_9 +; CHECK-NEXT: # %bb.7: # %if.then14 +; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload +; CHECK-NEXT: #APP +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: jmp .LBB0_10 +; CHECK-NEXT: .Ltmp0: # Block address taken +; CHECK-NEXT: .LBB0_8: # %if.then20.critedge +; CHECK-NEXT: movl {{.*}}(%rip), %edi +; CHECK-NEXT: movslq %eax, %rcx +; CHECK-NEXT: movl $1, %esi +; CHECK-NEXT: movq %r15, %rdx +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r13 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: jmp k # TAILCALL +; CHECK-NEXT: .LBB0_9: # %if.else +; CHECK-NEXT: incq 0 +; CHECK-NEXT: jmp .LBB0_10 +entry: + %call = tail call i32 @c() + %call1 = tail call i32 @l(i32* %o) + %tobool = icmp eq i32 %call1, 0 + br i1 %tobool, label %if.end, label %cleanup + +if.end: ; preds = %entry + %0 = load i32, i32* @e + %tobool3 = icmp eq i32 %0, 0 + br i1 %tobool3, label %if.end8, label %if.then4, !prof !0 + +if.then4: ; preds = %if.end + %conv5 = sext i32 %p to i64 + %call6 = tail call i32 @m(i64 %conv5) + br label %r + +r: ; preds = %if.end8, %if.then4 + %flags.0 = phi i32 [ undef, %if.then4 ], [ %and, %if.end8 ] + %major.0 = phi i32 [ undef, %if.then4 ], [ %or, %if.end8 ] + %call7 = tail call i32 @c() + br label %if.end8 + +if.end8: ; preds = %r, %if.end + %flags.1 = phi i32 [ %flags.0, %r ], [ undef, %if.end ] + %major.1 = phi i32 [ %major.0, %r ], [ undef, %if.end ] + %call9 = tail call i32 @i(i32 %flags.1) + %or = or i32 %call9, %major.1 + %and = and i32 %flags.1, 4 + %tobool10 = icmp eq i32 %call, 0 + br i1 %tobool10, label %if.end12, label %r + +if.end12: ; preds = %if.end8 + %tobool13 = icmp eq i32 %or, 0 + br i1 %tobool13, label %if.else, label %if.then14 + +if.then14: ; preds = %if.end12 + callbr void asm sideeffect "", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@n, %if.then20.critedge)) + to label %cleanup [label %if.then20.critedge] + +if.then20.critedge: ; preds = %if.then14 + %1 = load i32, i32* @j + %conv21 = sext i32 %u to i64 + %call22 = tail call i32 @k(i32 %1, i64 1, i32* %o, i64 %conv21) + br label %cleanup + +if.else: ; preds = %if.end12 + %2 = load i64, i64* null + %inc = add i64 %2, 1 + store i64 %inc, i64* null + br label %cleanup + +cleanup: ; preds = %if.else, %if.then20.critedge, %if.then14, %entry + ret void +} + +declare i32 @c() + +declare i32 @l(i32*) + +declare i32 @m(i64) + +declare i32 @i(i32) + +declare i32 @k(i32, i64, i32*, i64) + +!0 = !{!"branch_weights", i32 2000, i32 1} diff --git a/test/CodeGen/X86/callbr-asm-destinations.ll b/test/CodeGen/X86/callbr-asm-destinations.ll new file mode 100644 index 00000000000..91f0f2d1d65 --- /dev/null +++ b/test/CodeGen/X86/callbr-asm-destinations.ll @@ -0,0 +1,15 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: Duplicate callbr destination + +; A test for asm-goto duplicate labels limitation + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %fail [label %fail] + +fail: + ret i32 1 +} diff --git a/test/CodeGen/X86/callbr-asm-errors.ll b/test/CodeGen/X86/callbr-asm-errors.ll new file mode 100644 index 00000000000..5569e3c15ac --- /dev/null +++ b/test/CodeGen/X86/callbr-asm-errors.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: Duplicate callbr destination + +; A test for asm-goto duplicate labels limitation + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail), i8* blockaddress(@test, %fail)) to label %normal [label %fail, label %fail] + +normal: + ret i32 %0 + +fail: + ret i32 1 +} diff --git a/test/CodeGen/X86/callbr-asm-outputs.ll b/test/CodeGen/X86/callbr-asm-outputs.ll new file mode 100644 index 00000000000..7595deba336 --- /dev/null +++ b/test/CodeGen/X86/callbr-asm-outputs.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=i686-- < %s 2> %t +; RUN: FileCheck %s < %t + +; CHECK: error: asm-goto outputs not supported + +; A test for asm-goto output prohibition + +define i32 @test(i32 %a) { +entry: + %0 = add i32 %a, 4 + %1 = callbr i32 asm "xorl $1, $1; jmp ${1:l}", "=&r,r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %normal [label %fail] + +normal: + ret i32 %1 + +fail: + ret i32 1 +} diff --git a/test/CodeGen/X86/callbr-asm.ll b/test/CodeGen/X86/callbr-asm.ll new file mode 100644 index 00000000000..48a80ae167b --- /dev/null +++ b/test/CodeGen/X86/callbr-asm.ll @@ -0,0 +1,133 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=i686-- -O3 | FileCheck %s + +; Tests for using callbr as an asm-goto wrapper + +; Test 1 - fallthrough label gets removed, but the fallthrough code that is +; unreachable due to asm ending on a jmp is still left in. +define i32 @test1(i32 %a) { +; CHECK-LABEL: test1: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: addl $4, %eax +; CHECK-NEXT: #APP +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: jmp .Ltmp00 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: .LBB0_1: # %normal +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .Ltmp0: # Block address taken +; CHECK-NEXT: .LBB0_2: # %fail +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl +entry: + %0 = add i32 %a, 4 + callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test1, %fail)) to label %normal [label %fail] + +normal: + ret i32 0 + +fail: + ret i32 1 +} + +; Test 2 - callbr terminates an unreachable block, function gets simplified +; to a trivial zero return. +define i32 @test2(i32 %a) { +; CHECK-LABEL: test2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +entry: + br label %normal + +unreachableasm: + %0 = add i32 %a, 4 + callbr void asm sideeffect "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test2, %fail)) to label %normal [label %fail] + +normal: + ret i32 0 + +fail: + ret i32 1 +} + + +; Test 3 - asm-goto implements a loop. The loop gets recognized, but many loop +; transforms fail due to canonicalization having callbr exceptions. Trivial +; blocks at labels 1 and 3 also don't get simplified due to callbr. +define dso_local i32 @test3(i32 %a) { +; CHECK-LABEL: test3: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: .Ltmp1: # Block address taken +; CHECK-NEXT: .LBB2_1: # %label01 +; CHECK-NEXT: # =>This Loop Header: Depth=1 +; CHECK-NEXT: # Child Loop BB2_2 Depth 2 +; CHECK-NEXT: # Child Loop BB2_3 Depth 3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: .Ltmp2: # Block address taken +; CHECK-NEXT: .LBB2_2: # %label02 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # => This Loop Header: Depth=2 +; CHECK-NEXT: # Child Loop BB2_3 Depth 3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: addl $4, {{[0-9]+}}(%esp) +; CHECK-NEXT: .Ltmp3: # Block address taken +; CHECK-NEXT: .LBB2_3: # %label03 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # Parent Loop BB2_2 Depth=2 +; CHECK-NEXT: # => This Loop Header: Depth=3 +; CHECK-NEXT: # Child Loop BB2_4 Depth 4 +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .Ltmp4: # Block address taken +; CHECK-NEXT: .LBB2_4: # %label04 +; CHECK-NEXT: # Parent Loop BB2_1 Depth=1 +; CHECK-NEXT: # Parent Loop BB2_2 Depth=2 +; CHECK-NEXT: # Parent Loop BB2_3 Depth=3 +; CHECK-NEXT: # => This Inner Loop Header: Depth=4 +; CHECK-NEXT: #APP +; CHECK-NEXT: jmp .Ltmp10 +; CHECK-NEXT: jmp .Ltmp20 +; CHECK-NEXT: jmp .Ltmp30 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: .LBB2_5: # %normal0 +; CHECK-NEXT: # in Loop: Header=BB2_4 Depth=4 +; CHECK-NEXT: #APP +; CHECK-NEXT: jmp .Ltmp10 +; CHECK-NEXT: jmp .Ltmp20 +; CHECK-NEXT: jmp .Ltmp30 +; CHECK-NEXT: jmp .Ltmp40 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: .LBB2_6: # %normal1 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: retl +entry: + %a.addr = alloca i32, align 4 + store i32 %a, i32* %a.addr, align 4 + br label %label01 + +label01: ; preds = %normal0, %label04, %entry + br label %label02 + +label02: ; preds = %normal0, %label04, %label01 + %0 = load i32, i32* %a.addr, align 4 + %add = add nsw i32 %0, 4 + store i32 %add, i32* %a.addr, align 4 + br label %label03 + +label03: ; preds = %normal0, %label04, %label02 + br label %label04 + +label04: ; preds = %normal0, %label03 + callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}", "X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03)) + to label %normal0 [label %label01, label %label02, label %label03] + +normal0: ; preds = %label04 + callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}; jmp ${3:l}", "X,X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03), i8* blockaddress(@test3, %label04)) + to label %normal1 [label %label01, label %label02, label %label03, label %label04] + +normal1: ; preds = %normal0 + %1 = load i32, i32* %a.addr, align 4 + ret i32 %1 +} diff --git a/test/Transforms/GVN/callbr-loadpre-critedge.ll b/test/Transforms/GVN/callbr-loadpre-critedge.ll new file mode 100644 index 00000000000..2a6a0aa778a --- /dev/null +++ b/test/Transforms/GVN/callbr-loadpre-critedge.ll @@ -0,0 +1,49 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -gvn -S | FileCheck %s + +; This test checks that we don't hang trying to split a critical edge in loadpre +; when the control flow uses a callbr instruction. + +%struct.pluto = type <{ i8, i8 }> + +define void @widget(%struct.pluto** %tmp1) { +; CHECK-LABEL: @widget( +; CHECK-NEXT: bb: +; CHECK-NEXT: callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, [[BB5:%.*]]), i8* blockaddress(@widget, [[BB8:%.*]])) +; CHECK-NEXT: to label [[BB4:%.*]] [label [[BB5]], label %bb8] +; CHECK: bb4: +; CHECK-NEXT: br label [[BB5]] +; CHECK: bb5: +; CHECK-NEXT: [[TMP6:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1:%.*]] +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_PLUTO:%.*]], %struct.pluto* [[TMP6]], i64 0, i32 1 +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb8: +; CHECK-NEXT: [[TMP9:%.*]] = phi i8* [ [[TMP7]], [[BB5]] ], [ null, [[BB:%.*]] ] +; CHECK-NEXT: [[TMP10:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_PLUTO]], %struct.pluto* [[TMP10]], i64 0, i32 0 +; CHECK-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]] +; CHECK-NEXT: tail call void @spam(i8* [[TMP9]], i8 [[TMP12]]) +; CHECK-NEXT: ret void +; +bb: + callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, %bb5), i8* blockaddress(@widget, %bb8)) + to label %bb4 [label %bb5, label %bb8] + +bb4: ; preds = %bb + br label %bb5 + +bb5: ; preds = %bb4, %bb + %tmp6 = load %struct.pluto*, %struct.pluto** %tmp1 + %tmp7 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp6, i64 0, i32 1 + br label %bb8 + +bb8: ; preds = %bb5, %bb + %tmp9 = phi i8* [ %tmp7, %bb5 ], [ null, %bb ] + %tmp10 = load %struct.pluto*, %struct.pluto** %tmp1 + %tmp11 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp10, i64 0, i32 0 + %tmp12 = load i8, i8* %tmp11 + tail call void @spam(i8* %tmp9, i8 %tmp12) + ret void +} + +declare void @spam(i8*, i8) diff --git a/test/Transforms/GVN/callbr-scalarpre-critedge.ll b/test/Transforms/GVN/callbr-scalarpre-critedge.ll new file mode 100644 index 00000000000..733ba4a0cc8 --- /dev/null +++ b/test/Transforms/GVN/callbr-scalarpre-critedge.ll @@ -0,0 +1,43 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -gvn -S | FileCheck %s + +; This test checks that we don't hang trying to split a critical edge in scalar +; PRE when the control flow uses a callbr instruction. + +define void @wombat(i64 %arg, i64* %arg1, i64 %arg2, i32* %arg3) { +; CHECK-LABEL: @wombat( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[ARG2:%.*]], [[ARG:%.*]] +; CHECK-NEXT: callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, [[BB7:%.*]]), i8* blockaddress(@wombat, [[BB9:%.*]])) +; CHECK-NEXT: to label [[BB6:%.*]] [label [[BB7]], label %bb9] +; CHECK: bb6: +; CHECK-NEXT: br label [[BB7]] +; CHECK: bb7: +; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP5]] to i32 +; CHECK-NEXT: tail call void @barney(i32 [[TMP8]]) +; CHECK-NEXT: br label [[BB9]] +; CHECK: bb9: +; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32 +; CHECK-NEXT: store i32 [[TMP10]], i32* [[ARG3:%.*]] +; CHECK-NEXT: ret void +; +bb: + %tmp5 = or i64 %arg2, %arg + callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, %bb7), i8* blockaddress(@wombat, %bb9)) + to label %bb6 [label %bb7, label %bb9] + +bb6: ; preds = %bb + br label %bb7 + +bb7: ; preds = %bb6, %bb + %tmp8 = trunc i64 %tmp5 to i32 + tail call void @barney(i32 %tmp8) + br label %bb9 + +bb9: ; preds = %bb7, %bb + %tmp10 = trunc i64 %tmp5 to i32 + store i32 %tmp10, i32* %arg3 + ret void +} + +declare void @barney(i32) diff --git a/test/Transforms/JumpThreading/callbr-edge-split.ll b/test/Transforms/JumpThreading/callbr-edge-split.ll new file mode 100644 index 00000000000..a341f73dee5 --- /dev/null +++ b/test/Transforms/JumpThreading/callbr-edge-split.ll @@ -0,0 +1,58 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -jump-threading | FileCheck %s + +; This test used to cause jump threading to try to split an edge of a callbr. + +@a = global i32 0 + +define i32 @c() { +; CHECK-LABEL: @c( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* @a +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @b() +; CHECK-NEXT: [[PHITMP:%.*]] = icmp ne i32 [[CALL]], 0 +; CHECK-NEXT: br i1 [[PHITMP]], label [[IF_THEN2:%.*]], label [[IF_END4:%.*]] +; CHECK: if.else: +; CHECK-NEXT: callbr void asm sideeffect "", "X"(i8* blockaddress(@c, [[IF_THEN2]])) +; CHECK-NEXT: to label [[IF_END_THREAD:%.*]] [label %if.then2] +; CHECK: if.end.thread: +; CHECK-NEXT: br label [[IF_THEN2]] +; CHECK: if.then2: +; CHECK-NEXT: [[CALL3:%.*]] = call i32 @b() +; CHECK-NEXT: br label [[IF_END4]] +; CHECK: if.end4: +; CHECK-NEXT: ret i32 undef +; +entry: + %0 = load i32, i32* @a + %tobool = icmp eq i32 %0, 0 + br i1 %tobool, label %if.else, label %if.then + +if.then: ; preds = %entry + %call = call i32 @b() #2 + %phitmp = icmp ne i32 %call, 0 + br label %if.end + +if.else: ; preds = %entry + callbr void asm sideeffect "", "X"(i8* blockaddress(@c, %if.end)) #2 + to label %normal [label %if.end] + +normal: ; preds = %if.else + br label %if.end + +if.end: ; preds = %if.else, %normal, %if.then + %d.0 = phi i1 [ %phitmp, %if.then ], [ undef, %normal ], [ undef, %if.else ] + br i1 %d.0, label %if.then2, label %if.end4 + +if.then2: ; preds = %if.end + %call3 = call i32 @b() + br label %if.end4 + +if.end4: ; preds = %if.then2, %if.end + ret i32 undef +} + +declare i32 @b() diff --git a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll index f138ac42914..4b8d6bb53e4 100644 --- a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll +++ b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll @@ -63,14 +63,6 @@ lpad: resume { i8*, i32 } zeroinitializer } -define i8 @call_with_same_range() { -; CHECK-LABEL: @call_with_same_range -; CHECK: tail call i8 @call_with_range - bitcast i8 0 to i8 - %out = call i8 @dummy(), !range !0 - ret i8 %out -} - define i8 @invoke_with_same_range() personality i8* undef { ; CHECK-LABEL: @invoke_with_same_range() ; CHECK: tail call i8 @invoke_with_range() @@ -84,6 +76,13 @@ lpad: resume { i8*, i32 } zeroinitializer } +define i8 @call_with_same_range() { +; CHECK-LABEL: @call_with_same_range +; CHECK: tail call i8 @call_with_range + bitcast i8 0 to i8 + %out = call i8 @dummy(), !range !0 + ret i8 %out +} declare i8 @dummy(); diff --git a/test/Transforms/MergeFunc/inline-asm.ll b/test/Transforms/MergeFunc/inline-asm.ll index 370d3c56f06..15760242cf6 100644 --- a/test/Transforms/MergeFunc/inline-asm.ll +++ b/test/Transforms/MergeFunc/inline-asm.ll @@ -3,13 +3,13 @@ ; CHECK-LABEL: @int_ptr_arg_different ; CHECK-NEXT: call void asm +; CHECK-LABEL: @int_ptr_null +; CHECK-NEXT: tail call void @float_ptr_null() + ; CHECK-LABEL: @int_ptr_arg_same ; CHECK-NEXT: %2 = bitcast i32* %0 to float* ; CHECK-NEXT: tail call void @float_ptr_arg_same(float* %2) -; CHECK-LABEL: @int_ptr_null -; CHECK-NEXT: tail call void @float_ptr_null() - ; Used to satisfy minimum size limit declare void @stuff() diff --git a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp index 6f702b6e545..092dbd13675 100644 --- a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp +++ b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp @@ -291,6 +291,7 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(FUNC_CODE, INST_LOADATOMIC) STRINGIFY_CODE(FUNC_CODE, INST_STOREATOMIC) STRINGIFY_CODE(FUNC_CODE, INST_CMPXCHG) + STRINGIFY_CODE(FUNC_CODE, INST_CALLBR) } case bitc::VALUE_SYMTAB_BLOCK_ID: switch (CodeID) { diff --git a/utils/vim/syntax/llvm.vim b/utils/vim/syntax/llvm.vim index 9345a9c0901..56eb1b19826 100644 --- a/utils/vim/syntax/llvm.vim +++ b/utils/vim/syntax/llvm.vim @@ -23,7 +23,7 @@ syn match llvmType /\/ " The true and false tokens can be used for comparison opcodes, but it's " much more common for these tokens to be used for boolean constants. syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw -syn keyword llvmStatement bitcast br catchpad catchswitch catchret call +syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr syn keyword llvmStatement cleanuppad cleanupret cmpxchg eq exact extractelement syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fpext syn keyword llvmStatement fptosi fptoui fptrunc free frem fsub fneg getelementptr -- 2.40.0