From: Craig Topper Date: Mon, 11 Mar 2019 04:36:47 +0000 (+0000) Subject: [X86] Split the VFIXUPIMM/VFIXUPIMMS nodes into a current rounding mode and SAE ISD... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b0f6bf123b74cf21d5d0eac237a96b44c7c4fa8f;p=llvm [X86] Split the VFIXUPIMM/VFIXUPIMMS nodes into a current rounding mode and SAE ISD opcode. Remove matching of FROUND_CURRENT and FROUND_NO_EXC for these nodes from isel table. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@355800 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 4328d0b3c9a..273f9159ab3 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -22346,34 +22346,32 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, Mask); } - case FIXUPIMMS: - case FIXUPIMMS_MASKZ: case FIXUPIMM: - case FIXUPIMM_MASKZ:{ + case FIXUPIMM_MASKZ: { SDValue Src1 = Op.getOperand(1); SDValue Src2 = Op.getOperand(2); SDValue Src3 = Op.getOperand(3); SDValue Imm = Op.getOperand(4); SDValue Mask = Op.getOperand(5); - SDValue Passthru = (IntrData->Type == FIXUPIMM || IntrData->Type == FIXUPIMMS ) ? - Src1 : getZeroVector(VT, Subtarget, DAG, dl); - // We specify 2 possible modes for intrinsics, with/without rounding - // modes. - // First, we check if the intrinsic have rounding mode (7 operands), - // if not, we set rounding mode to "current". - SDValue Rnd; - if (Op.getNumOperands() == 7) - Rnd = Op.getOperand(6); - else - Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); - if (IntrData->Type == FIXUPIMM || IntrData->Type == FIXUPIMM_MASKZ) - return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, - Src1, Src2, Src3, Imm, Rnd), - Mask, Passthru, Subtarget, DAG); - else // Scalar - FIXUPIMMS, FIXUPIMMS_MASKZ - return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, - Src1, Src2, Src3, Imm, Rnd), - Mask, Passthru, Subtarget, DAG); + SDValue Passthru = (IntrData->Type == FIXUPIMM) + ? Src1 + : getZeroVector(VT, Subtarget, DAG, dl); + + unsigned Opc = IntrData->Opc0; + if (IntrData->Opc1 != 0) { + SDValue Sae = Op.getOperand(6); + if (isRoundModeSAE(Sae)) + Opc = IntrData->Opc1; + else if (!isRoundModeCurDirection(Sae)) + return SDValue(); + } + + SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm); + + if (Opc == X86ISD::VFIXUPIMM || Opc == X86ISD::VFIXUPIMM_SAE) + return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); + + return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); } case ROUNDP: { assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode"); @@ -27672,7 +27670,9 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::VPERMI: return "X86ISD::VPERMI"; case X86ISD::VPTERNLOG: return "X86ISD::VPTERNLOG"; case X86ISD::VFIXUPIMM: return "X86ISD::VFIXUPIMM"; + case X86ISD::VFIXUPIMM_SAE: return "X86ISD::VFIXUPIMM_SAE"; case X86ISD::VFIXUPIMMS: return "X86ISD::VFIXUPIMMS"; + case X86ISD::VFIXUPIMMS_SAE: return "X86ISD::VFIXUPIMMS_SAE"; case X86ISD::VRANGE: return "X86ISD::VRANGE"; case X86ISD::VRANGE_RND: return "X86ISD::VRANGE_RND"; case X86ISD::VRANGES: return "X86ISD::VRANGES"; diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index 30de65b6bfd..fbc88ac7716 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -417,8 +417,8 @@ namespace llvm { // Bitwise ternary logic. VPTERNLOG, // Fix Up Special Packed Float32/64 values. - VFIXUPIMM, - VFIXUPIMMS, + VFIXUPIMM, VFIXUPIMM_SAE, + VFIXUPIMMS, VFIXUPIMMS_SAE, // Range Restriction Calculation For Packed Pairs of Float32/64 values. VRANGE, VRANGE_RND, VRANGES, VRANGES_RND, // Reduce - Perform Reduction Transformation on scalar\packed FP. diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td index 3b85b396dd5..e813878d317 100644 --- a/lib/Target/X86/X86InstrAVX512.td +++ b/lib/Target/X86/X86InstrAVX512.td @@ -12174,58 +12174,55 @@ let Predicates = [HasVLX] in { // AVX-512 - FixupImm //===----------------------------------------------------------------------===// -multiclass avx512_fixupimm_packed opc, string OpcodeStr, SDNode OpNode, +multiclass avx512_fixupimm_packed opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _, X86VectorVTInfo TblVT>{ let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in { defm rri : AVX512_maskable_3src, Sched<[sched]>; + (X86VFixupimm (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (TblVT.VT _.RC:$src3), + (i32 imm:$src4))>, Sched<[sched]>; defm rmi : AVX512_maskable_3src, + (X86VFixupimm (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (TblVT.VT (bitconvert (TblVT.LdFrag addr:$src3))), + (i32 imm:$src4))>, Sched<[sched.Folded, sched.ReadAfterFold]>; defm rmbi : AVX512_maskable_3src, + (X86VFixupimm (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (TblVT.VT (X86VBroadcast(TblVT.ScalarLdFrag addr:$src3))), + (i32 imm:$src4))>, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>; } // Constraints = "$src1 = $dst" } multiclass avx512_fixupimm_packed_sae opc, string OpcodeStr, - SDNode OpNode, X86FoldableSchedWrite sched, - X86VectorVTInfo _, X86VectorVTInfo TblVT>{ + X86FoldableSchedWrite sched, + X86VectorVTInfo _, X86VectorVTInfo TblVT> + : avx512_fixupimm_packed { let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in { defm rrib : AVX512_maskable_3src, + (X86VFixupimmSAE (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (TblVT.VT _.RC:$src3), + (i32 imm:$src4))>, EVEX_B, Sched<[sched]>; } } -multiclass avx512_fixupimm_scalar opc, string OpcodeStr, SDNode OpNode, +multiclass avx512_fixupimm_scalar opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _, X86VectorVTInfo _src3VT> { let Constraints = "$src1 = $dst" , Predicates = [HasAVX512], @@ -12233,30 +12230,27 @@ multiclass avx512_fixupimm_scalar opc, string OpcodeStr, SDNode OpNode, defm rri : AVX512_maskable_3src_scalar, Sched<[sched]>; + (X86VFixupimms (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (_src3VT.VT _src3VT.RC:$src3), + (i32 imm:$src4))>, Sched<[sched]>; defm rrib : AVX512_maskable_3src_scalar, + (X86VFixupimmSAEs (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (_src3VT.VT _src3VT.RC:$src3), + (i32 imm:$src4))>, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>; defm rmi : AVX512_maskable_3src_scalar, + (X86VFixupimms (_.VT _.RC:$src1), + (_.VT _.RC:$src2), + (_src3VT.VT (scalar_to_vector + (_src3VT.ScalarLdFrag addr:$src3))), + (i32 imm:$src4))>, Sched<[sched.Folded, sched.ReadAfterFold]>; } } @@ -12265,25 +12259,23 @@ multiclass avx512_fixupimm_packed_all { let Predicates = [HasAVX512] in - defm Z : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.ZMM, - _Vec.info512, _Tbl.info512>, - avx512_fixupimm_packed_sae<0x54, "vfixupimm", X86VFixupimm, sched.ZMM, + defm Z : avx512_fixupimm_packed_sae<0x54, "vfixupimm", sched.ZMM, _Vec.info512, _Tbl.info512>, AVX512AIi8Base, EVEX_4V, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { - defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.XMM, + defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", sched.XMM, _Vec.info128, _Tbl.info128>, AVX512AIi8Base, EVEX_4V, EVEX_V128; - defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.YMM, + defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", sched.YMM, _Vec.info256, _Tbl.info256>, AVX512AIi8Base, EVEX_4V, EVEX_V256; } } -defm VFIXUPIMMSSZ : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar, +defm VFIXUPIMMSSZ : avx512_fixupimm_scalar<0x55, "vfixupimm", SchedWriteFAdd.Scl, f32x_info, v4i32x_info>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; -defm VFIXUPIMMSDZ : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar, +defm VFIXUPIMMSDZ : avx512_fixupimm_scalar<0x55, "vfixupimm", SchedWriteFAdd.Scl, f64x_info, v2i64x_info>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; defm VFIXUPIMMPS : avx512_fixupimm_packed_all, SDTCisVec<0>, SDTCisSameAs<0,2>, SDTCisVT<3, i32>, SDTCisVT<4, i32>]>; -def SDTFPTernaryOpImmRound: SDTypeProfile<1, 5, [SDTCisFP<0>, SDTCisSameAs<0,1>, - SDTCisSameAs<0,2>, - SDTCisInt<3>, - SDTCisSameSizeAs<0, 3>, - SDTCisSameNumEltsAs<0, 3>, - SDTCisVT<4, i32>, - SDTCisVT<5, i32>]>; +def SDTFPTernaryOpImm: SDTypeProfile<1, 4, [SDTCisFP<0>, SDTCisSameAs<0,1>, + SDTCisSameAs<0,2>, + SDTCisInt<3>, + SDTCisSameSizeAs<0, 3>, + SDTCisSameNumEltsAs<0, 3>, + SDTCisVT<4, i32>]>; def SDTFPUnaryOpImm: SDTypeProfile<1, 2, [SDTCisFP<0>, SDTCisVec<0>, SDTCisSameAs<0,1>, SDTCisVT<2, i32>]>; @@ -422,8 +421,10 @@ def X86vpternlog : SDNode<"X86ISD::VPTERNLOG", SDTTernlog>; def X86VPerm2x128 : SDNode<"X86ISD::VPERM2X128", SDTShuff3OpI>; -def X86VFixupimm : SDNode<"X86ISD::VFIXUPIMM", SDTFPTernaryOpImmRound>; -def X86VFixupimmScalar : SDNode<"X86ISD::VFIXUPIMMS", SDTFPTernaryOpImmRound>; +def X86VFixupimm : SDNode<"X86ISD::VFIXUPIMM", SDTFPTernaryOpImm>; +def X86VFixupimmSAE : SDNode<"X86ISD::VFIXUPIMM_SAE", SDTFPTernaryOpImm>; +def X86VFixupimms : SDNode<"X86ISD::VFIXUPIMMS", SDTFPTernaryOpImm>; +def X86VFixupimmSAEs : SDNode<"X86ISD::VFIXUPIMMS_SAE", SDTFPTernaryOpImm>; def X86VRange : SDNode<"X86ISD::VRANGE", SDTFPBinOpImm>; def X86VRangeRnd : SDNode<"X86ISD::VRANGE_RND", SDTFPBinOpImmRound>; def X86VReduce : SDNode<"X86ISD::VREDUCE", SDTFPUnaryOpImm>; diff --git a/lib/Target/X86/X86IntrinsicsInfo.h b/lib/Target/X86/X86IntrinsicsInfo.h index cd85fd26d2c..d2f65047f7b 100644 --- a/lib/Target/X86/X86IntrinsicsInfo.h +++ b/lib/Target/X86/X86IntrinsicsInfo.h @@ -33,8 +33,7 @@ enum IntrinsicType : uint16_t { COMPRESS_EXPAND_IN_REG, TRUNCATE_TO_REG, CVTPS2PH_MASK, CVTPD2DQ_MASK, CVTQQ2PS_MASK, TRUNCATE_TO_MEM_VI8, TRUNCATE_TO_MEM_VI16, TRUNCATE_TO_MEM_VI32, - FIXUPIMM, FIXUPIMM_MASKZ, FIXUPIMMS, - FIXUPIMMS_MASKZ, GATHER_AVX2, + FIXUPIMM, FIXUPIMM_MASKZ, GATHER_AVX2, ROUNDP, ROUNDS }; @@ -578,12 +577,12 @@ static const IntrinsicData IntrinsicsWithoutChain[] = { X86ISD::EXPAND, 0), X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_128, FIXUPIMM, X86ISD::VFIXUPIMM, 0), X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_256, FIXUPIMM, X86ISD::VFIXUPIMM, 0), - X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_512, FIXUPIMM, X86ISD::VFIXUPIMM, 0), + X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_512, FIXUPIMM, X86ISD::VFIXUPIMM, X86ISD::VFIXUPIMM_SAE), X86_INTRINSIC_DATA(avx512_mask_fixupimm_ps_128, FIXUPIMM, X86ISD::VFIXUPIMM, 0), X86_INTRINSIC_DATA(avx512_mask_fixupimm_ps_256, FIXUPIMM, X86ISD::VFIXUPIMM, 0), - X86_INTRINSIC_DATA(avx512_mask_fixupimm_ps_512, FIXUPIMM, X86ISD::VFIXUPIMM, 0), - X86_INTRINSIC_DATA(avx512_mask_fixupimm_sd, FIXUPIMMS, X86ISD::VFIXUPIMMS, 0), - X86_INTRINSIC_DATA(avx512_mask_fixupimm_ss, FIXUPIMMS, X86ISD::VFIXUPIMMS, 0), + X86_INTRINSIC_DATA(avx512_mask_fixupimm_ps_512, FIXUPIMM, X86ISD::VFIXUPIMM, X86ISD::VFIXUPIMM_SAE), + X86_INTRINSIC_DATA(avx512_mask_fixupimm_sd, FIXUPIMM, X86ISD::VFIXUPIMMS, X86ISD::VFIXUPIMMS_SAE), + X86_INTRINSIC_DATA(avx512_mask_fixupimm_ss, FIXUPIMM, X86ISD::VFIXUPIMMS, X86ISD::VFIXUPIMMS_SAE), X86_INTRINSIC_DATA(avx512_mask_fpclass_sd, FPCLASSS, X86ISD::VFPCLASSS, 0), X86_INTRINSIC_DATA(avx512_mask_fpclass_ss, FPCLASSS, X86ISD::VFPCLASSS, 0), X86_INTRINSIC_DATA(avx512_mask_getexp_pd_128, INTR_TYPE_1OP_MASK_RM, @@ -798,17 +797,17 @@ static const IntrinsicData IntrinsicsWithoutChain[] = { X86_INTRINSIC_DATA(avx512_maskz_fixupimm_pd_256, FIXUPIMM_MASKZ, X86ISD::VFIXUPIMM, 0), X86_INTRINSIC_DATA(avx512_maskz_fixupimm_pd_512, FIXUPIMM_MASKZ, - X86ISD::VFIXUPIMM, 0), + X86ISD::VFIXUPIMM, X86ISD::VFIXUPIMM_SAE), X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ps_128, FIXUPIMM_MASKZ, X86ISD::VFIXUPIMM, 0), X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ps_256, FIXUPIMM_MASKZ, X86ISD::VFIXUPIMM, 0), X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ps_512, FIXUPIMM_MASKZ, - X86ISD::VFIXUPIMM, 0), - X86_INTRINSIC_DATA(avx512_maskz_fixupimm_sd, FIXUPIMMS_MASKZ, - X86ISD::VFIXUPIMMS, 0), - X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ss, FIXUPIMMS_MASKZ, - X86ISD::VFIXUPIMMS, 0), + X86ISD::VFIXUPIMM, X86ISD::VFIXUPIMM_SAE), + X86_INTRINSIC_DATA(avx512_maskz_fixupimm_sd, FIXUPIMM_MASKZ, + X86ISD::VFIXUPIMMS, X86ISD::VFIXUPIMMS_SAE), + X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ss, FIXUPIMM_MASKZ, + X86ISD::VFIXUPIMMS, X86ISD::VFIXUPIMMS_SAE), X86_INTRINSIC_DATA(avx512_max_pd_512, INTR_TYPE_2OP_SAE, X86ISD::FMAX, X86ISD::FMAX_SAE), X86_INTRINSIC_DATA(avx512_max_ps_512, INTR_TYPE_2OP_SAE, X86ISD::FMAX, X86ISD::FMAX_SAE),