static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
// We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
// set by the CCReg instruction using the CCValid / CCMask masks,
- // If the CCReg instruction is itself a (ICMP (SELECT_CCMASK)) testing
- // the condition code set by some other instruction, see whether we
- // can directly use that condition code.
- bool Invert = false;
+ // If the CCReg instruction is itself a ICMP testing the condition
+ // code set by some other instruction, see whether we can directly
+ // use that condition code.
- // Verify that we have an appropriate mask for a EQ or NE comparison.
+ // Verify that we have an ICMP against some constant.
if (CCValid != SystemZ::CCMASK_ICMP)
return false;
- if (CCMask == SystemZ::CCMASK_CMP_NE)
- Invert = !Invert;
- else if (CCMask != SystemZ::CCMASK_CMP_EQ)
- return false;
-
- // Verify that we have an ICMP that is the user of a SELECT_CCMASK.
- SDNode *ICmp = CCReg.getNode();
+ auto *ICmp = CCReg.getNode();
if (ICmp->getOpcode() != SystemZISD::ICMP)
return false;
- SDNode *Select = ICmp->getOperand(0).getNode();
- if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
+ auto *CompareLHS = ICmp->getOperand(0).getNode();
+ auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
+ if (!CompareRHS)
return false;
- // Verify that the ICMP compares against one of select values.
- auto *CompareVal = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
- if (!CompareVal)
- return false;
- auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
- if (!TrueVal)
- return false;
- auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
- if (!FalseVal)
- return false;
- if (CompareVal->getZExtValue() == FalseVal->getZExtValue())
- Invert = !Invert;
- else if (CompareVal->getZExtValue() != TrueVal->getZExtValue())
- return false;
+ // Optimize the case where CompareLHS is a SELECT_CCMASK.
+ if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {
+ // Verify that we have an appropriate mask for a EQ or NE comparison.
+ bool Invert = false;
+ if (CCMask == SystemZ::CCMASK_CMP_NE)
+ Invert = !Invert;
+ else if (CCMask != SystemZ::CCMASK_CMP_EQ)
+ return false;
- // Compute the effective CC mask for the new branch or select.
- auto *NewCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
- auto *NewCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
- if (!NewCCValid || !NewCCMask)
- return false;
- CCValid = NewCCValid->getZExtValue();
- CCMask = NewCCMask->getZExtValue();
- if (Invert)
- CCMask ^= CCValid;
+ // Verify that the ICMP compares against one of select values.
+ auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));
+ if (!TrueVal)
+ return false;
+ auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
+ if (!FalseVal)
+ return false;
+ if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())
+ Invert = !Invert;
+ else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())
+ return false;
- // Return the updated CCReg link.
- CCReg = Select->getOperand(4);
- return true;
+ // Compute the effective CC mask for the new branch or select.
+ auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));
+ auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));
+ if (!NewCCValid || !NewCCMask)
+ return false;
+ CCValid = NewCCValid->getZExtValue();
+ CCMask = NewCCMask->getZExtValue();
+ if (Invert)
+ CCMask ^= CCValid;
+
+ // Return the updated CCReg link.
+ CCReg = CompareLHS->getOperand(4);
+ return true;
+ }
+
+ // Optimize the case where CompareRHS is (SRA (SHL (IPM))).
+ if (CompareLHS->getOpcode() == ISD::SRA) {
+ auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
+ if (!SRACount || SRACount->getZExtValue() != 30)
+ return false;
+ auto *SHL = CompareLHS->getOperand(0).getNode();
+ if (SHL->getOpcode() != ISD::SHL)
+ return false;
+ auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));
+ if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)
+ return false;
+ auto *IPM = SHL->getOperand(0).getNode();
+ if (IPM->getOpcode() != SystemZISD::IPM)
+ return false;
+
+ // Avoid introducing CC spills (because SRA would clobber CC).
+ if (!CompareLHS->hasOneUse())
+ return false;
+ // Verify that the ICMP compares against zero.
+ if (CompareRHS->getZExtValue() != 0)
+ return false;
+
+ // Compute the effective CC mask for the new branch or select.
+ switch (CCMask) {
+ case SystemZ::CCMASK_CMP_EQ: break;
+ case SystemZ::CCMASK_CMP_NE: break;
+ case SystemZ::CCMASK_CMP_LT: CCMask = SystemZ::CCMASK_CMP_GT; break;
+ case SystemZ::CCMASK_CMP_GT: CCMask = SystemZ::CCMASK_CMP_LT; break;
+ case SystemZ::CCMASK_CMP_LE: CCMask = SystemZ::CCMASK_CMP_GE; break;
+ case SystemZ::CCMASK_CMP_GE: CCMask = SystemZ::CCMASK_CMP_LE; break;
+ default: return false;
+ }
+
+ // Return the updated CCReg link.
+ CCReg = IPM->getOperand(0);
+ return true;
+ }
+
+ return false;
}
SDValue SystemZTargetLowering::combineBR_CCMASK(
return false;
}
-// If Reg is a virtual register, return its definition, otherwise return null.
-static MachineInstr *getDef(unsigned Reg,
- const MachineRegisterInfo *MRI) {
- if (TargetRegisterInfo::isPhysicalRegister(Reg))
- return nullptr;
- return MRI->getUniqueVRegDef(Reg);
-}
-
-// Return true if MI is a shift of type Opcode by Imm bits.
-static bool isShift(MachineInstr *MI, unsigned Opcode, int64_t Imm) {
- return (MI->getOpcode() == Opcode &&
- !MI->getOperand(2).getReg() &&
- MI->getOperand(3).getImm() == Imm);
-}
-
-// If the destination of MI has no uses, delete it as dead.
-static void eraseIfDead(MachineInstr *MI, const MachineRegisterInfo *MRI) {
- if (MRI->use_nodbg_empty(MI->getOperand(0).getReg()))
- MI->eraseFromParent();
-}
-
-// Compare compares SrcReg against zero. Check whether SrcReg contains
-// the result of an IPM sequence whose input CC survives until Compare,
-// and whether Compare is therefore redundant. Delete it and return
-// true if so.
-static bool removeIPMBasedCompare(MachineInstr &Compare, unsigned SrcReg,
- const MachineRegisterInfo *MRI,
- const TargetRegisterInfo *TRI) {
- MachineInstr *LGFR = nullptr;
- MachineInstr *RLL = getDef(SrcReg, MRI);
- if (RLL && RLL->getOpcode() == SystemZ::LGFR) {
- LGFR = RLL;
- RLL = getDef(LGFR->getOperand(1).getReg(), MRI);
- }
- if (!RLL || !isShift(RLL, SystemZ::RLL, 31))
- return false;
-
- MachineInstr *SRL = getDef(RLL->getOperand(1).getReg(), MRI);
- if (!SRL || !isShift(SRL, SystemZ::SRL, SystemZ::IPM_CC))
- return false;
-
- MachineInstr *IPM = getDef(SRL->getOperand(1).getReg(), MRI);
- if (!IPM || IPM->getOpcode() != SystemZ::IPM)
- return false;
-
- // Check that there are no assignments to CC between the IPM and Compare,
- if (IPM->getParent() != Compare.getParent())
- return false;
- MachineBasicBlock::iterator MBBI = IPM, MBBE = Compare.getIterator();
- for (++MBBI; MBBI != MBBE; ++MBBI) {
- MachineInstr &MI = *MBBI;
- if (MI.modifiesRegister(SystemZ::CC, TRI))
- return false;
- }
-
- Compare.eraseFromParent();
- if (LGFR)
- eraseIfDead(LGFR, MRI);
- eraseIfDead(RLL, MRI);
- eraseIfDead(SRL, MRI);
- eraseIfDead(IPM, MRI);
-
- return true;
-}
-
-bool SystemZInstrInfo::optimizeCompareInstr(
- MachineInstr &Compare, unsigned SrcReg, unsigned SrcReg2, int Mask,
- int Value, const MachineRegisterInfo *MRI) const {
- assert(!SrcReg2 && "Only optimizing constant comparisons so far");
- bool IsLogical = (Compare.getDesc().TSFlags & SystemZII::IsLogical) != 0;
- return Value == 0 && !IsLogical &&
- removeIPMBasedCompare(Compare, SrcReg, MRI, &RI);
-}
-
bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Pred,
unsigned TrueReg, unsigned FalseReg,
int *BytesAdded = nullptr) const override;
bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
unsigned &SrcReg2, int &Mask, int &Value) const override;
- bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
- unsigned SrcReg2, int Mask, int Value,
- const MachineRegisterInfo *MRI) const override;
bool canInsertSelect(const MachineBasicBlock&, ArrayRef<MachineOperand> Cond,
unsigned, unsigned, int&, int&, int&) const override;
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
}
// Convert the current CC value into an integer that is 0 if CC == 0,
-// less than zero if CC == 1 and greater than zero if CC >= 2.
+// greater than zero if CC == 1 and less than zero if CC >= 2.
// The sequence starts with IPM, which puts CC into bits 29 and 28
// of an integer and clears bits 30 and 31.
static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
SelectionDAG &DAG) {
SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
- SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
- DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
- SDValue ROTL = DAG.getNode(ISD::ROTL, DL, MVT::i32, SRL,
- DAG.getConstant(31, DL, MVT::i32));
- return ROTL;
+ SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM,
+ DAG.getConstant(30 - SystemZ::IPM_CC, DL, MVT::i32));
+ SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
+ DAG.getConstant(30, DL, MVT::i32));
+ return SRA;
}
std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
uint64_t Bytes = CSize->getZExtValue();
assert(Bytes > 0 && "Caller should have handled 0-size case");
- SDValue CCReg = emitCLC(DAG, DL, Chain, Src1, Src2, Bytes);
+ // Swap operands to invert CC == 1 vs. CC == 2 cases.
+ SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
Chain = CCReg.getValue(1);
return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
}
SDValue Src2, MachinePointerInfo Op1PtrInfo,
MachinePointerInfo Op2PtrInfo) const {
SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::i32, MVT::Other);
- SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src1, Src2,
+ // Swap operands to invert CC == 1 vs. CC == 2 cases.
+ SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1,
DAG.getConstant(0, DL, MVT::i32));
SDValue CCReg = Unused.getValue(1);
Chain = Unused.getValue(2);
; Check a case where the result is used as an integer.
define i32 @f2(i8 *%src1, i8 *%src2) {
; CHECK-LABEL: f2:
-; CHECK: clc 0(2,%r2), 0(%r3)
-; CHECK: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK: clc 0(2,%r3), 0(%r2)
+; CHECK: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
; CHECK: br %r14
%res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 2)
ret i32 %res
; Check a case where the result is tested for equality.
define void @f3(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f3:
-; CHECK: clc 0(3,%r2), 0(%r3)
+; CHECK: clc 0(3,%r3), 0(%r2)
; CHECK-NEXT: ber %r14
; CHECK: br %r14
%res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 3)
; Check a case where the result is tested for inequality.
define void @f4(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f4:
-; CHECK: clc 0(4,%r2), 0(%r3)
+; CHECK: clc 0(4,%r3), 0(%r2)
; CHECK-NEXT: blhr %r14
; CHECK: br %r14
entry:
; Check a case where the result is tested via slt.
define void @f5(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f5:
-; CHECK: clc 0(5,%r2), 0(%r3)
-; CHECK-NEXT: blr %r14
+; CHECK: clc 0(5,%r3), 0(%r2)
+; CHECK-NEXT: bhr %r14
; CHECK: br %r14
entry:
%res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 5)
; Check a case where the result is tested for sgt.
define void @f6(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f6:
-; CHECK: clc 0(6,%r2), 0(%r3)
-; CHECK-NEXT: bhr %r14
+; CHECK: clc 0(6,%r3), 0(%r2)
+; CHECK-NEXT: blr %r14
; CHECK: br %r14
entry:
%res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 6)
; an integer and for branching.
define i32 @f7(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f7:
-; CHECK: clc 0(256,%r2), 0(%r3)
-; CHECK: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK: clc 0(256,%r3), 0(%r2)
+; CHECK: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
; CHECK: blr %r14
; CHECK: br %r14
entry:
; 257 bytes needs two CLCs.
define i32 @f8(i8 *%src1, i8 *%src2) {
; CHECK-LABEL: f8:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(1,%r2), 256(%r3)
+; CHECK: clc 256(1,%r3), 256(%r2)
; CHECK: [[LABEL]]:
; CHECK: ipm [[REG:%r[0-5]]]
; CHECK: br %r14
; Test a comparison of 258 bytes in which the CC result can be used directly.
define void @f9(i8 *%src1, i8 *%src2, i32 *%dest) {
; CHECK-LABEL: f9:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(1,%r2), 256(%r3)
+; CHECK: clc 256(1,%r3), 256(%r2)
; CHECK: [[LABEL]]:
-; CHECK-NEXT: blr %r14
+; CHECK-NEXT: bhr %r14
; CHECK: br %r14
entry:
%res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 257)
; Test the largest size that can use two CLCs.
define i32 @f10(i8 *%src1, i8 *%src2) {
; CHECK-LABEL: f10:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
; CHECK: [[LABEL]]:
; CHECK: ipm [[REG:%r[0-5]]]
; CHECK: br %r14
; Test the smallest size that needs 3 CLCs.
define i32 @f11(i8 *%src1, i8 *%src2) {
; CHECK-LABEL: f11:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
; CHECK: jlh [[LABEL]]
-; CHECK: clc 512(1,%r2), 512(%r3)
+; CHECK: clc 512(1,%r3), 512(%r2)
; CHECK: [[LABEL]]:
; CHECK: ipm [[REG:%r[0-5]]]
; CHECK: br %r14
; Test the largest size than can use 3 CLCs.
define i32 @f12(i8 *%src1, i8 *%src2) {
; CHECK-LABEL: f12:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
; CHECK: jlh [[LABEL]]
-; CHECK: clc 512(256,%r2), 512(%r3)
+; CHECK: clc 512(256,%r3), 512(%r2)
; CHECK: [[LABEL]]:
; CHECK: ipm [[REG:%r[0-5]]]
; CHECK: br %r14
; CHECK-LABEL: f13:
; CHECK: lghi [[COUNT:%r[0-5]]], 3
; CHECK: [[LOOP:.L[^:]*]]:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
; CHECK: jlh [[LABEL:\..*]]
; CHECK-DAG: la %r2, 256(%r2)
; CHECK-DAG: la %r3, 256(%r3)
; CHECK: brctg [[COUNT]], [[LOOP]]
-; CHECK: clc 0(1,%r2), 0(%r3)
+; CHECK: clc 0(1,%r3), 0(%r2)
; CHECK: [[LABEL]]:
; CHECK: ipm [[REG:%r[0-5]]]
; CHECK: br %r14
; CHECK-LABEL: f1:
; CHECK: lhi %r0, 0
; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
; CHECK-NEXT: jo [[LABEL]]
; CHECK-NEXT: %bb.{{[0-9]+}}
-; CHECK-NEXT: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK-NEXT: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
; CHECK: br %r14
%res = call i32 @strcmp(i8 *%src1, i8 *%src2)
ret i32 %res
; CHECK-LABEL: f2:
; CHECK: lhi %r0, 0
; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
; CHECK-NEXT: jo [[LABEL]]
; CHECK-NEXT: %bb.{{[0-9]+}}
; CHECK-NEXT: ber %r14
; CHECK-LABEL: f3:
; CHECK: lhi %r0, 0
; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
; CHECK-NEXT: jo [[LABEL]]
; CHECK-NEXT: %bb.{{[0-9]+}}
-; CHECK-NEXT: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK-NEXT: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
; CHECK: blr %r14
; CHECK: br %r14
entry:
projects / llvm / commitdiff