From 90ccb6a749e634a7cd6b14d3b65835b62a07d42f Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Thu, 7 Feb 2019 10:43:09 +0000 Subject: [PATCH] [BPF] add code-gen support for JMP32 instructions JMP32 instructions has been added to eBPF ISA. They are 32-bit variants of existing BPF conditional jump instructions, but the comparison happens on low 32-bit sub-register only, therefore some unnecessary extensions could be saved. JMP32 instructions will only be available for -mcpu=v3. Host probe hook has been updated accordingly. JMP32 instructions will only be enabled in code-gen when -mattr=+alu32 enabled, meaning compiling the program using sub-register mode. For JMP32 encoding, it is a new instruction class, and is using the reserved eBPF class number 0x6. This patch has been tested by compiling and running kernel bpf selftests with JMP32 enabled. Acked-by: Yonghong Song Signed-off-by: Jiong Wang git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@353384 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Support/Host.cpp | 31 +++++++- lib/Target/BPF/BPF.td | 1 + lib/Target/BPF/BPFISelLowering.cpp | 57 ++++++-------- lib/Target/BPF/BPFISelLowering.h | 2 + lib/Target/BPF/BPFInstrFormats.td | 1 + lib/Target/BPF/BPFInstrInfo.td | 76 ++++++++++++++++--- lib/Target/BPF/BPFSubtarget.cpp | 6 ++ lib/Target/BPF/BPFSubtarget.h | 5 ++ .../BPF/Disassembler/BPFDisassembler.cpp | 2 +- test/MC/BPF/insn-unit-32.s | 47 ++++++++++++ 10 files changed, 179 insertions(+), 49 deletions(-) diff --git a/lib/Support/Host.cpp b/lib/Support/Host.cpp index c81b1f14647..1b907bc6284 100644 --- a/lib/Support/Host.cpp +++ b/lib/Support/Host.cpp @@ -330,7 +330,19 @@ StringRef sys::detail::getHostCPUNameForBPF() { #if !defined(__linux__) || !defined(__x86_64__) return "generic"; #else - uint8_t insns[40] __attribute__ ((aligned (8))) = + uint8_t v3_insns[40] __attribute__ ((aligned (8))) = + /* BPF_MOV64_IMM(BPF_REG_0, 0) */ + { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + /* BPF_MOV64_IMM(BPF_REG_2, 1) */ + 0xb7, 0x2, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, + /* BPF_JMP32_REG(BPF_JLT, BPF_REG_0, BPF_REG_2, 1) */ + 0xae, 0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, + /* BPF_MOV64_IMM(BPF_REG_0, 1) */ + 0xb7, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, + /* BPF_EXIT_INSN() */ + 0x95, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; + + uint8_t v2_insns[40] __attribute__ ((aligned (8))) = /* BPF_MOV64_IMM(BPF_REG_0, 0) */ { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, /* BPF_MOV64_IMM(BPF_REG_2, 1) */ @@ -355,10 +367,23 @@ StringRef sys::detail::getHostCPUNameForBPF() { } attr = {}; attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */ attr.insn_cnt = 5; - attr.insns = (uint64_t)insns; + attr.insns = (uint64_t)v3_insns; attr.license = (uint64_t)"DUMMY"; - int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr)); + int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, + sizeof(attr)); + if (fd >= 0) { + close(fd); + return "v3"; + } + + /* Clear the whole attr in case its content changed by syscall. */ + memset(&attr, 0, sizeof(attr)); + attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */ + attr.insn_cnt = 5; + attr.insns = (uint64_t)v2_insns; + attr.license = (uint64_t)"DUMMY"; + fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr)); if (fd >= 0) { close(fd); return "v2"; diff --git a/lib/Target/BPF/BPF.td b/lib/Target/BPF/BPF.td index dd97698d401..fad966ff5a1 100644 --- a/lib/Target/BPF/BPF.td +++ b/lib/Target/BPF/BPF.td @@ -20,6 +20,7 @@ class Proc Features> def : Proc<"generic", []>; def : Proc<"v1", []>; def : Proc<"v2", []>; +def : Proc<"v3", []>; def : Proc<"probe", []>; def DummyFeature : SubtargetFeature<"dummy", "isDummyMode", diff --git a/lib/Target/BPF/BPFISelLowering.cpp b/lib/Target/BPF/BPFISelLowering.cpp index e37df6e08b8..ff69941d26f 100644 --- a/lib/Target/BPF/BPFISelLowering.cpp +++ b/lib/Target/BPF/BPFISelLowering.cpp @@ -105,7 +105,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, if (STI.getHasAlu32()) { setOperationAction(ISD::BSWAP, MVT::i32, Promote); - setOperationAction(ISD::BR_CC, MVT::i32, Promote); + setOperationAction(ISD::BR_CC, MVT::i32, + STI.getHasJmp32() ? Custom : Promote); } setOperationAction(ISD::CTTZ, MVT::i64, Custom); @@ -162,6 +163,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, // CPU/Feature control HasAlu32 = STI.getHasAlu32(); + HasJmp32 = STI.getHasJmp32(); HasJmpExt = STI.getHasJmpExt(); } @@ -506,7 +508,7 @@ SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { NegateCC(LHS, RHS, CC); return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS, - DAG.getConstant(CC, DL, MVT::i64), Dest); + DAG.getConstant(CC, DL, LHS.getValueType()), Dest); } SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { @@ -676,36 +678,23 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, int CC = MI.getOperand(3).getImm(); int NewCC; switch (CC) { - case ISD::SETGT: - NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri; - break; - case ISD::SETUGT: - NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri; - break; - case ISD::SETGE: - NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri; - break; - case ISD::SETUGE: - NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri; - break; - case ISD::SETEQ: - NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri; - break; - case ISD::SETNE: - NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri; - break; - case ISD::SETLT: - NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri; - break; - case ISD::SETULT: - NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri; - break; - case ISD::SETLE: - NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri; - break; - case ISD::SETULE: - NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri; - break; +#define SET_NEWCC(X, Y) \ + case ISD::X: \ + if (is32BitCmp && HasJmp32) \ + NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \ + else \ + NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \ + break + SET_NEWCC(SETGT, JSGT); + SET_NEWCC(SETUGT, JUGT); + SET_NEWCC(SETGE, JSGE); + SET_NEWCC(SETUGE, JUGE); + SET_NEWCC(SETEQ, JEQ); + SET_NEWCC(SETNE, JNE); + SET_NEWCC(SETLT, JSLT); + SET_NEWCC(SETULT, JULT); + SET_NEWCC(SETLE, JSLE); + SET_NEWCC(SETULE, JULE); default: report_fatal_error("unimplemented select CondCode " + Twine(CC)); } @@ -723,13 +712,13 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, // // We simply do extension for all situations in this method, but we will // try to remove those unnecessary in BPFMIPeephole pass. - if (is32BitCmp) + if (is32BitCmp && !HasJmp32) LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp); if (isSelectRROp) { unsigned RHS = MI.getOperand(2).getReg(); - if (is32BitCmp) + if (is32BitCmp && !HasJmp32) RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp); BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB); diff --git a/lib/Target/BPF/BPFISelLowering.h b/lib/Target/BPF/BPFISelLowering.h index cafc6d4a5bd..2a44165de69 100644 --- a/lib/Target/BPF/BPFISelLowering.h +++ b/lib/Target/BPF/BPFISelLowering.h @@ -55,6 +55,7 @@ public: MachineBasicBlock *BB) const override; bool getHasAlu32() const { return HasAlu32; } + bool getHasJmp32() const { return HasJmp32; } bool getHasJmpExt() const { return HasJmpExt; } EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, @@ -65,6 +66,7 @@ public: private: // Control Instruction Selection Features bool HasAlu32; + bool HasJmp32; bool HasJmpExt; SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; diff --git a/lib/Target/BPF/BPFInstrFormats.td b/lib/Target/BPF/BPFInstrFormats.td index c5e712f4bab..9f00dc85d78 100644 --- a/lib/Target/BPF/BPFInstrFormats.td +++ b/lib/Target/BPF/BPFInstrFormats.td @@ -16,6 +16,7 @@ def BPF_ST : BPFOpClass<0x2>; def BPF_STX : BPFOpClass<0x3>; def BPF_ALU : BPFOpClass<0x4>; def BPF_JMP : BPFOpClass<0x5>; +def BPF_JMP32 : BPFOpClass<0x6>; def BPF_ALU64 : BPFOpClass<0x7>; class BPFSrcType val> { diff --git a/lib/Target/BPF/BPFInstrInfo.td b/lib/Target/BPF/BPFInstrInfo.td index 4c4b2146bfd..36724b81fc6 100644 --- a/lib/Target/BPF/BPFInstrInfo.td +++ b/lib/Target/BPF/BPFInstrInfo.td @@ -101,6 +101,26 @@ def BPF_CC_LTU : PatLeaf<(i64 imm), [{return (N->getZExtValue() == ISD::SETULT);}]>; def BPF_CC_LEU : PatLeaf<(i64 imm), [{return (N->getZExtValue() == ISD::SETULE);}]>; +def BPF_CC_EQ_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETEQ);}]>; +def BPF_CC_NE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETNE);}]>; +def BPF_CC_GE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETGE);}]>; +def BPF_CC_GT_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETGT);}]>; +def BPF_CC_GTU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETUGT);}]>; +def BPF_CC_GEU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETUGE);}]>; +def BPF_CC_LE_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETLE);}]>; +def BPF_CC_LT_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETLT);}]>; +def BPF_CC_LTU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETULT);}]>; +def BPF_CC_LEU_32 : PatLeaf<(i32 imm), + [{return (N->getZExtValue() == ISD::SETULE);}]>; // For arithmetic and jump instructions the 8-bit 'code' // field is divided into three parts: @@ -166,23 +186,57 @@ class JMP_RI let BPFClass = BPF_JMP; } -multiclass J { +class JMP_RR_32 + : TYPE_ALU_JMP { + bits<4> dst; + bits<4> src; + bits<16> BrDst; + + let Inst{55-52} = src; + let Inst{51-48} = dst; + let Inst{47-32} = BrDst; + let BPFClass = BPF_JMP32; +} + +class JMP_RI_32 + : TYPE_ALU_JMP { + bits<4> dst; + bits<16> BrDst; + bits<32> imm; + + let Inst{51-48} = dst; + let Inst{47-32} = BrDst; + let Inst{31-0} = imm; + let BPFClass = BPF_JMP32; +} + +multiclass J { def _rr : JMP_RR; def _ri : JMP_RI; + def _rr_32 : JMP_RR_32; + def _ri_32 : JMP_RI_32; } let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { // cmp+goto instructions -defm JEQ : J; -defm JUGT : J", BPF_CC_GTU>; -defm JUGE : J=", BPF_CC_GEU>; -defm JNE : J", BPF_CC_GT>; -defm JSGE : J=", BPF_CC_GE>; -defm JULT : J; -defm JULE : J; -defm JSLE : J", BPF_CC_GTU, BPF_CC_GTU_32>; +defm JUGE : J=", BPF_CC_GEU, BPF_CC_GEU_32>; +defm JNE : J", BPF_CC_GT, BPF_CC_GT_32>; +defm JSGE : J=", BPF_CC_GE, BPF_CC_GE_32>; +defm JULT : J; +defm JULE : J; +defm JSLE : J>= 64 + + if w0 == w1 goto Llabel0 // BPF_JEQ | BPF_X + if w3 != w4 goto Llabel0 // BPF_JNE | BPF_X +// CHECK: 1e 10 0b 00 00 00 00 00 if w0 == w1 goto +11 +// CHECK: 5e 43 0a 00 00 00 00 00 if w3 != w4 goto +10 + + if w1 > w2 goto Llabel0 // BPF_JGT | BPF_X + if w2 >= w3 goto Llabel0 // BPF_JGE | BPF_X + if w4 s> w5 goto Llabel0 // BPF_JSGT | BPF_X + if w5 s>= w6 goto Llabel0 // BPF_JSGE | BPF_X +// CHECK: 2e 21 09 00 00 00 00 00 if w1 > w2 goto +9 +// CHECK: 3e 32 08 00 00 00 00 00 if w2 >= w3 goto +8 +// CHECK: 6e 54 07 00 00 00 00 00 if w4 s> w5 goto +7 +// CHECK: 7e 65 06 00 00 00 00 00 if w5 s>= w6 goto +6 + + if w6 < w7 goto Llabel0 // BPF_JLT | BPF_X + if w7 <= w8 goto Llabel0 // BPF_JLE | BPF_X + if w8 s< w9 goto Llabel0 // BPF_JSLT | BPF_X + if w9 s<= w10 goto Llabel0 // BPF_JSLE | BPF_X +// CHECK: ae 76 05 00 00 00 00 00 if w6 < w7 goto +5 +// CHECK: be 87 04 00 00 00 00 00 if w7 <= w8 goto +4 +// CHECK: ce 98 03 00 00 00 00 00 if w8 s< w9 goto +3 +// CHECK: de a9 02 00 00 00 00 00 if w9 s<= w10 goto +2 + + if w0 == 0 goto Llabel0 // BPF_JEQ | BPF_K + if w3 != -1 goto Llabel0 // BPF_JNE | BPF_K +// CHECK: 16 00 01 00 00 00 00 00 if w0 == 0 goto +1 +// CHECK: 56 03 00 00 ff ff ff ff if w3 != -1 goto +0 + +Llabel0 : + if w1 > 64 goto Llabel0 // BPF_JGT | BPF_K + if w2 >= 0xffffffff goto Llabel0 // BPF_JGE | BPF_K + if w4 s> 0xffffffff goto Llabel0 // BPF_JSGT | BPF_K + if w5 s>= 0x7fffffff goto Llabel0 // BPF_JSGE | BPF_K +// CHECK: 26 01 ff ff 40 00 00 00 if w1 > 64 goto -1 +// CHECK: 36 02 fe ff ff ff ff ff if w2 >= -1 goto -2 +// CHECK: 66 04 fd ff ff ff ff ff if w4 s> -1 goto -3 +// CHECK: 76 05 fc ff ff ff ff 7f if w5 s>= 2147483647 goto -4 + + if w6 < 0xff goto Llabel0 // BPF_JLT | BPF_K + if w7 <= 0xffff goto Llabel0 // BPF_JLE | BPF_K + if w8 s< 0 goto Llabel0 // BPF_JSLT | BPF_K + if w9 s<= -1 goto Llabel0 // BPF_JSLE | BPF_K +// CHECK: a6 06 fb ff ff 00 00 00 if w6 < 255 goto -5 +// CHECK: b6 07 fa ff ff ff 00 00 if w7 <= 65535 goto -6 +// CHECK: c6 08 f9 ff 00 00 00 00 if w8 s< 0 goto -7 +// CHECK: d6 09 f8 ff ff ff ff ff if w9 s<= -1 goto -8 -- 2.40.0