[ARM] Generate 8.1-m CSINC, CSNEG and CSINV instructions.

Arm 8.1-M adds a number of related CSEL instructions, including CSINC, CSNEG and CSINV. These choose between two values given the content in CPSR and a condition, performing an increment, negation or inverse of the false value.

This adds some selection for them, either from constant values or patterns. It does not include CSEL directly, which is currently not always making code better. It is still useful, but we will have to check more carefully where it should and shouldn't be used.

Code by Ranjeet Singh and Simon Tatham, with some modifications from me.

Differential revision: https://reviews.llvm.org/D66483

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@370739 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 680729dd29b9cac714ad3f3da62686cabc8b7562..54ddb946a2d2aee0a4a893dc71565a7016e2ed13 100644 (file)
--- a/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3040,18 +3040,22 @@ bool ARMBaseInstrInfo::optimizeCompareInstr(
         break;
       case ARM::VSELEQD:
       case ARM::VSELEQS:
+      case ARM::VSELEQH:
         CC = ARMCC::EQ;
         break;
       case ARM::VSELGTD:
       case ARM::VSELGTS:
+      case ARM::VSELGTH:
         CC = ARMCC::GT;
         break;
       case ARM::VSELGED:
       case ARM::VSELGES:
+      case ARM::VSELGEH:
         CC = ARMCC::GE;
         break;
-      case ARM::VSELVSS:
       case ARM::VSELVSD:
+      case ARM::VSELVSS:
+      case ARM::VSELVSH:
         CC = ARMCC::VS;
         break;
       }

diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 8a4de79a21872c9943cbaf4386f1487a12da0dbc..0239d6af8ad04a3ddee8f73bd93d399e25a3952f 100644 (file)
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -1634,6 +1634,9 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::WLS:           return "ARMISD::WLS";
   case ARMISD::LE:            return "ARMISD::LE";
   case ARMISD::LOOP_DEC:      return "ARMISD::LOOP_DEC";
+  case ARMISD::CSINV:         return "ARMISD::CSINV";
+  case ARMISD::CSNEG:         return "ARMISD::CSNEG";
+  case ARMISD::CSINC:         return "ARMISD::CSINC";
   }
   return nullptr;
 }
@@ -4815,6 +4818,49 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDValue TrueVal = Op.getOperand(2);
   SDValue FalseVal = Op.getOperand(3);
+  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
+  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
+
+  if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
+      LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
+    unsigned TVal = CTVal->getZExtValue();
+    unsigned FVal = CFVal->getZExtValue();
+    unsigned Opcode = 0;
+
+    if (TVal == ~FVal) {
+      Opcode = ARMISD::CSINV;
+    } else if (TVal == ~FVal + 1) {
+      Opcode = ARMISD::CSNEG;
+    } else if (TVal + 1 == FVal) {
+      Opcode = ARMISD::CSINC;
+    } else if (TVal == FVal + 1) {
+      Opcode = ARMISD::CSINC;
+      std::swap(TrueVal, FalseVal);
+      std::swap(TVal, FVal);
+      CC = ISD::getSetCCInverse(CC, true);
+    }
+
+    if (Opcode) {
+      // Attempt to use ZR checking TVal is 0, possibly inverting the condition
+      // to get there. CSINC not is invertable like the other two (~(~a) == a,
+      // -(-a) == a, but (a+1)+1 != a).
+      if (FVal == 0 && Opcode != ARMISD::CSINC) {
+        std::swap(TrueVal, FalseVal);
+        std::swap(TVal, FVal);
+        CC = ISD::getSetCCInverse(CC, true);
+      }
+      if (TVal == 0)
+        TrueVal = DAG.getRegister(ARM::ZR, MVT::i32);
+
+      // Drops F's value because we can get it by inverting/negating TVal.
+      FalseVal = TrueVal;
+
+      SDValue ARMcc;
+      SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
+      EVT VT = TrueVal.getValueType();
+      return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
+    }
+  }
 
   if (isUnsupportedFloatingType(LHS.getValueType())) {
     DAG.getTargetLoweringInfo().softenSetCCOperands(

diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index d84a235b8b2a1d49cb1a7396cfb98b6709e77b11..d7586bca2ba00ec65db32b79485a223a43960929 100644 (file)
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -240,6 +240,11 @@ class VectorType;
       // instructions.
       MEMCPY,
 
+      // V8.1MMainline condition select
+      CSINV, // Conditional select invert.
+      CSNEG, // Conditional select negate.
+      CSINC, // Conditional select increment.
+
       // Vector load N-element structure to all lanes:
       VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
       VLD2DUP,

diff --git a/lib/Target/ARM/ARMInstrFormats.td b/lib/Target/ARM/ARMInstrFormats.td
index 8885b73174909cc0e58746e12386944d46f31cd2..043b54a7deb5dd4b87d535eaa80a48b6b1c97504 100644 (file)
--- a/lib/Target/ARM/ARMInstrFormats.td
+++ b/lib/Target/ARM/ARMInstrFormats.td
@@ -188,6 +188,13 @@ def s_cc_out : OptionalDefOperand<OtherVT, (ops CCR), (ops (i32 CPSR))> {
   let DecoderMethod = "DecodeCCOutOperand";
 }
 
+// Transform to generate the inverse of a condition code during ISel
+def inv_cond_XFORM : SDNodeXForm<imm, [{
+  ARMCC::CondCodes CC = static_cast<ARMCC::CondCodes>(N->getZExtValue());
+  return CurDAG->getTargetConstant(ARMCC::getOppositeCondition(CC), SDLoc(N),
+                                   MVT::i32);
+}]>;
+
 // VPT predicate
 
 def VPTPredNOperand : AsmOperandClass {

diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td
index ac1be46447f3401aab20e778fa8d7d18069774a4..5396ffb7af5ca067aad31b87a16f637764a9998c 100644 (file)
--- a/lib/Target/ARM/ARMInstrInfo.td
+++ b/lib/Target/ARM/ARMInstrInfo.td
@@ -116,6 +116,16 @@ def ARMSmlaldx       : SDNode<"ARMISD::SMLALDX", SDT_LongMac>;
 def ARMSmlsld        : SDNode<"ARMISD::SMLSLD", SDT_LongMac>;
 def ARMSmlsldx       : SDNode<"ARMISD::SMLSLDX", SDT_LongMac>;
 
+def SDT_ARMCSel      : SDTypeProfile<1, 3,
+                                   [SDTCisSameAs<0, 1>,
+                                    SDTCisSameAs<0, 2>,
+                                    SDTCisInt<3>,
+                                    SDTCisVT<3, i32>]>;
+
+def ARMcsinv         : SDNode<"ARMISD::CSINV", SDT_ARMCSel, [SDNPOptInGlue]>;
+def ARMcsneg         : SDNode<"ARMISD::CSNEG", SDT_ARMCSel, [SDNPOptInGlue]>;
+def ARMcsinc         : SDNode<"ARMISD::CSINC", SDT_ARMCSel, [SDNPOptInGlue]>;
+
 def SDT_MulHSR       : SDTypeProfile<1, 3, [SDTCisVT<0,i32>,
                                             SDTCisSameAs<0, 1>,
                                             SDTCisSameAs<0, 2>,

diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td
index 7e74a0a384545d1c92d45281a8a6ded3755836e3..f8cea50408cf86543e543395f477dd264b7de2c4 100644 (file)
--- a/lib/Target/ARM/ARMInstrThumb2.td
+++ b/lib/Target/ARM/ARMInstrThumb2.td
@@ -5257,6 +5257,25 @@ def t2CSINC : CS<"csinc", 0b1001>;
 def t2CSINV : CS<"csinv", 0b1010>;
 def t2CSNEG : CS<"csneg", 0b1011>;
 
+let Predicates = [HasV8_1MMainline] in {
+  def : T2Pat<(ARMcsinc GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSINC GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+  def : T2Pat<(ARMcsinv GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSINV GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+  def : T2Pat<(ARMcsneg GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSNEG GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+
+  multiclass ModifiedV8_1CSEL<Instruction Insn, dag modvalue> {
+    def : T2Pat<(ARMcmov modvalue, GPRwithZR:$tval, cmovpred:$imm),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+    def : T2Pat<(ARMcmov GPRwithZR:$tval, modvalue, cmovpred:$imm),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval,
+                         (i32 (inv_cond_XFORM imm:$imm)))>;
+  }
+  defm : ModifiedV8_1CSEL<t2CSINC, (add rGPR:$fval, 1)>;
+  defm : ModifiedV8_1CSEL<t2CSINV, (xor rGPR:$fval, -1)>;
+  defm : ModifiedV8_1CSEL<t2CSNEG, (sub 0, rGPR:$fval)>;
+}
 
 // CS aliases.
 let Predicates = [HasV8_1MMainline] in {

diff --git a/test/CodeGen/Thumb2/csel.ll b/test/CodeGen/Thumb2/csel.ll
index 256dd18e279e1bdd891964308fbfaeb019620597..17a111278b65ab6fa6235e1721549ead40d41489 100644 (file)
--- a/test/CodeGen/Thumb2/csel.ll
+++ b/test/CodeGen/Thumb2/csel.ll
@@ -6,9 +6,7 @@ define i32 @csinc_const_65(i32 %a) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #6
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -19,11 +17,9 @@ entry:
 define i32 @csinc_const_56(i32 %a) {
 ; CHECK-LABEL: csinc_const_56:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #6
+; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -34,11 +30,8 @@ entry:
 define i32 @csinc_const_zext(i32 %a) {
 ; CHECK-LABEL: csinc_const_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, zr, zr, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -49,11 +42,9 @@ entry:
 define i32 @csinv_const_56(i32 %a) {
 ; CHECK-LABEL: csinv_const_56:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    mvn r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -64,11 +55,9 @@ entry:
 define i32 @csinv_const_65(i32 %a) {
 ; CHECK-LABEL: csinv_const_65:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mvn r1, #5
+; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -79,11 +68,8 @@ entry:
 define i32 @csinv_const_sext(i32 %a) {
 ; CHECK-LABEL: csinv_const_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, zr, zr, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -94,11 +80,9 @@ entry:
 define i32 @csneg_const(i32 %a) {
 ; CHECK-LABEL: csneg_const:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mov.w r1, #-1
+; CHECK-NEXT:    movs r1, #1
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -109,11 +93,9 @@ entry:
 define i32 @csneg_const_r(i32 %a) {
 ; CHECK-LABEL: csneg_const_r:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    mov.w r1, #-1
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -139,9 +121,7 @@ define i32 @csinc_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinc_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    addle r1, r2, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -154,9 +134,7 @@ define i32 @csinc_swap_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinc_swap_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    addgt r2, r1, #1
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csinc r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -169,9 +147,7 @@ define i32 @csinv_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinv_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    mvnle r1, r2
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -184,9 +160,7 @@ define i32 @csinv_swap_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinv_swap_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r2, r1
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csinv r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -199,9 +173,7 @@ define i32 @csneg_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    rsble r1, r2, #0
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -214,9 +186,7 @@ define i32 @csneg_swap_var_sgt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sgt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    rsbgt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -229,9 +199,7 @@ define i32 @csneg_swap_var_sge(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sge:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #44
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    rsbgt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sge i32 %a, 45
@@ -244,9 +212,7 @@ define i32 @csneg_swap_var_sle(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sle:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #46
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    rsblt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ge
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sle i32 %a, 45
@@ -259,9 +225,7 @@ define i32 @csneg_swap_var_slt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_slt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    rsblt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ge
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 45
@@ -274,9 +238,7 @@ define i32 @csneg_swap_var_ugt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ugt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    rsbhi r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ls
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ugt i32 %a, 45
@@ -289,9 +251,7 @@ define i32 @csneg_swap_var_uge(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_uge:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #44
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    rsbhi r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ls
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp uge i32 %a, 45
@@ -304,9 +264,7 @@ define i32 @csneg_swap_var_ule(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ule:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #46
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    rsblo r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, hs
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ule i32 %a, 45
@@ -319,9 +277,7 @@ define i32 @csneg_swap_var_ult(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ult:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    rsblo r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, hs
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ult i32 %a, 45
@@ -334,9 +290,7 @@ define i32 @csneg_swap_var_ne(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ne:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    rsbeq r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ne
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp eq i32 %a, 45
@@ -349,9 +303,7 @@ define i32 @csneg_swap_var_eq(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_eq:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    rsbne r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, eq
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ne i32 %a, 45
@@ -364,8 +316,7 @@ define i32 @csinc_inplace(i32 %a, i32 %b) {
 ; CHECK-LABEL: csinc_inplace:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r1, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    addgt r0, #1
+; CHECK-NEXT:    csinc r0, r0, r0, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %b, 45
@@ -378,8 +329,7 @@ define i32 @csinv_inplace(i32 %a, i32 %b) {
 ; CHECK-LABEL: csinv_inplace:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r1, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r0, r0
+; CHECK-NEXT:    csinv r0, r0, r0, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %b, 45

diff --git a/test/CodeGen/Thumb2/mve-abs.ll b/test/CodeGen/Thumb2/mve-abs.ll
index 081157b070429d2ac1e84317abfc224c8c6a79d4..df89d3e902d653611aef9d690e71ead51e8bf909 100644 (file)
--- a/test/CodeGen/Thumb2/mve-abs.ll
+++ b/test/CodeGen/Thumb2/mve-abs.ll
@@ -40,39 +40,36 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @abs_v2i64(<2 x i64> %s1) {
 ; CHECK-LABEL: abs_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    vmov r12, s2
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    rsbs.w r3, r12, #0
-; CHECK-NEXT:    sbc.w lr, r2, r0
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    vmov lr, s4
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    rsbs.w r3, lr, #0
+; CHECK-NEXT:    sbc.w r2, r12, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq lr, r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    rsbs r5, r4, #0
-; CHECK-NEXT:    sbc.w r6, r2, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r2, #1
-; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csinc r1, zr, zr, pl
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    moveq r6, r0
-; CHECK-NEXT:    moveq r5, r4
-; CHECK-NEXT:    vmov.32 q0[0], r5
-; CHECK-NEXT:    vmov.32 q0[1], r6
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r2, r0
+; CHECK-NEXT:    moveq r3, lr
+; CHECK-NEXT:    vmov lr, s6
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    rsbs.w r2, lr, #0
+; CHECK-NEXT:    sbc.w r3, r12, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r1, zr, zr, pl
+; CHECK-NEXT:    ands r1, r1, #1
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq r2, lr
+; CHECK-NEXT:    vmov.32 q0[2], r2
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq r3, r12
-; CHECK-NEXT:    vmov.32 q0[2], r3
-; CHECK-NEXT:    vmov.32 q0[3], lr
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    moveq r3, r0
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = icmp slt <2 x i64> %s1, zeroinitializer
   %1 = sub nsw <2 x i64> zeroinitializer, %s1

diff --git a/test/CodeGen/Thumb2/mve-fmath.ll b/test/CodeGen/Thumb2/mve-fmath.ll
index dc60f71d235643d313f93fa309db4e5a8a075a91..d275b5dd200d87af5ff85e735ddee4ef482fa732 100644 (file)
--- a/test/CodeGen/Thumb2/mve-fmath.ll
+++ b/test/CodeGen/Thumb2/mve-fmath.ll
@@ -1326,54 +1326,49 @@ define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x hal
 ; CHECK-NEXT:    ldrb.w r0, [sp, #29]
 ; CHECK-NEXT:    vabs.f16 s4, s0
 ; CHECK-NEXT:    vneg.f16 s6, s4
+; CHECK-NEXT:    ldrb.w r1, [sp, #25]
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vabs.f16 s8, s1
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
-; CHECK-NEXT:    ldrb.w r1, [sp, #25]
+; CHECK-NEXT:    tst.w r1, #128
 ; CHECK-NEXT:    vmov r0, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    ands r1, r1, #128
+; CHECK-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-NEXT:    vabs.f16 s4, s4
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
 ; CHECK-NEXT:    vneg.f16 s6, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    lsls r1, r1, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
-; CHECK-NEXT:    vabs.f16 s0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s3
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    vmov.16 q1[0], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #21]
 ; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vabs.f16 s0, s0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[2], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #17]
 ; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vabs.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[3], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #13]
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmovx.f16 s8, s2
@@ -1381,29 +1376,26 @@ define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x hal
 ; CHECK-NEXT:    ldrb.w r0, [sp, #9]
 ; CHECK-NEXT:    vabs.f16 s8, s8
 ; CHECK-NEXT:    vneg.f16 s2, s0
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vabs.f16 s8, s3
 ; CHECK-NEXT:    vmov.16 q1[5], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #5]
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #1]
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0

diff --git a/test/CodeGen/Thumb2/mve-minmax.ll b/test/CodeGen/Thumb2/mve-minmax.ll
index 3a79cdf40330c37c905d5cda4597f1b833008aae..35e9f120cb20e553e9e0063de5d679d134dad743 100644 (file)
--- a/test/CodeGen/Thumb2/mve-minmax.ll
+++ b/test/CodeGen/Thumb2/mve-minmax.ll
@@ -55,15 +55,13 @@ define arm_aapcs_vfpcc <2 x i64> @smin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -131,15 +129,13 @@ define arm_aapcs_vfpcc <2 x i64> @umin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -208,15 +204,13 @@ define arm_aapcs_vfpcc <2 x i64> @smax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -284,15 +278,13 @@ define arm_aapcs_vfpcc <2 x i64> @umax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -388,22 +380,20 @@ define arm_aapcs_vfpcc <2 x double> @maxnm_float64_t(<2 x double> %src1, <2 x do
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    vmov r2, r3, d11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov r12, r1, d8
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    vmov r2, r3, d10
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinv r4, zr, zr, eq
+; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q0[0], r0
 ; CHECK-NEXT:    vmov.32 q0[1], r0
 ; CHECK-NEXT:    vmov.32 q0[2], r4

diff --git a/test/CodeGen/Thumb2/mve-pred-and.ll b/test/CodeGen/Thumb2/mve-pred-and.ll
index 7a20141502c921e421b0e54477594bd580753081..a9b31c7ef2bc08fbaaeb8a4d9147889d0140b503 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-and.ll
+++ b/test/CodeGen/Thumb2/mve-pred-and.ll
@@ -612,18 +612,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s8
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s10
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q1, q1, q3
@@ -650,10 +648,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -662,27 +659,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vand q2, q2, q3
@@ -706,10 +700,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r2
 ; CHECK-NEXT:    vmov.32 q2[1], r2
 ; CHECK-NEXT:    vmov r2, s7
@@ -718,27 +711,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vand q2, q3, q2

diff --git a/test/CodeGen/Thumb2/mve-pred-bitcast.ll b/test/CodeGen/Thumb2/mve-pred-bitcast.ll
index 03aef305ee9cddabd2c7c3f809fc40da55682d0a..af8cdacfbec8b92c16de2635dca3b8266d0bd074 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-bitcast.ll
+++ b/test/CodeGen/Thumb2/mve-pred-bitcast.ll
@@ -155,11 +155,10 @@ define arm_aapcs_vfpcc i2 @bitcast_from_v2i1(<2 x i64> %a) {
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    csinc r0, zr, zr, ne
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    clz r1, r1
-; CHECK-NEXT:    lsrs r1, r1, #5
+; CHECK-NEXT:    csinc r1, zr, zr, ne
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r1, #1
 ; CHECK-NEXT:    bfi r1, r0, #0, #1

diff --git a/test/CodeGen/Thumb2/mve-pred-build-var.ll b/test/CodeGen/Thumb2/mve-pred-build-var.ll
index b8b518c471e922ccc57757ee81a66bb634926c2c..5be074d298ff11a377768d4b5666e894acf8588a 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-build-var.ll
+++ b/test/CodeGen/Thumb2/mve-pred-build-var.ll
@@ -6,13 +6,12 @@ define arm_aapcs_vfpcc <4 x i32> @build_var0_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_var0_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -26,13 +25,12 @@ define arm_aapcs_vfpcc <4 x i32> @build_var3_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_var3_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #12, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #12, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -46,16 +44,15 @@ define arm_aapcs_vfpcc <4 x i32> @build_varN_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_varN_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #4
-; CHECK-NEXT:    bfi r2, r0, #4, #4
-; CHECK-NEXT:    bfi r2, r0, #8, #4
-; CHECK-NEXT:    bfi r2, r0, #12, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #4
+; CHECK-NEXT:    bfi r1, r0, #4, #4
+; CHECK-NEXT:    bfi r1, r0, #8, #4
+; CHECK-NEXT:    bfi r1, r0, #12, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -71,13 +68,12 @@ define arm_aapcs_vfpcc <8 x i16> @build_var0_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_var0_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -91,13 +87,12 @@ define arm_aapcs_vfpcc <8 x i16> @build_var3_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_var3_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #6, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #6, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -111,20 +106,19 @@ define arm_aapcs_vfpcc <8 x i16> @build_varN_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_varN_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #2
-; CHECK-NEXT:    bfi r2, r0, #2, #2
-; CHECK-NEXT:    bfi r2, r0, #4, #2
-; CHECK-NEXT:    bfi r2, r0, #6, #2
-; CHECK-NEXT:    bfi r2, r0, #8, #2
-; CHECK-NEXT:    bfi r2, r0, #10, #2
-; CHECK-NEXT:    bfi r2, r0, #12, #2
-; CHECK-NEXT:    bfi r2, r0, #14, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #2
+; CHECK-NEXT:    bfi r1, r0, #2, #2
+; CHECK-NEXT:    bfi r1, r0, #4, #2
+; CHECK-NEXT:    bfi r1, r0, #6, #2
+; CHECK-NEXT:    bfi r1, r0, #8, #2
+; CHECK-NEXT:    bfi r1, r0, #10, #2
+; CHECK-NEXT:    bfi r1, r0, #12, #2
+; CHECK-NEXT:    bfi r1, r0, #14, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -140,13 +134,12 @@ define arm_aapcs_vfpcc <16 x i8> @build_var0_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_var0_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -160,13 +153,12 @@ define arm_aapcs_vfpcc <16 x i8> @build_var3_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_var3_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #3, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #3, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -180,28 +172,27 @@ define arm_aapcs_vfpcc <16 x i8> @build_varN_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_varN_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #1
-; CHECK-NEXT:    bfi r2, r0, #1, #1
-; CHECK-NEXT:    bfi r2, r0, #2, #1
-; CHECK-NEXT:    bfi r2, r0, #3, #1
-; CHECK-NEXT:    bfi r2, r0, #4, #1
-; CHECK-NEXT:    bfi r2, r0, #5, #1
-; CHECK-NEXT:    bfi r2, r0, #6, #1
-; CHECK-NEXT:    bfi r2, r0, #7, #1
-; CHECK-NEXT:    bfi r2, r0, #8, #1
-; CHECK-NEXT:    bfi r2, r0, #9, #1
-; CHECK-NEXT:    bfi r2, r0, #10, #1
-; CHECK-NEXT:    bfi r2, r0, #11, #1
-; CHECK-NEXT:    bfi r2, r0, #12, #1
-; CHECK-NEXT:    bfi r2, r0, #13, #1
-; CHECK-NEXT:    bfi r2, r0, #14, #1
-; CHECK-NEXT:    bfi r2, r0, #15, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #1
+; CHECK-NEXT:    bfi r1, r0, #1, #1
+; CHECK-NEXT:    bfi r1, r0, #2, #1
+; CHECK-NEXT:    bfi r1, r0, #3, #1
+; CHECK-NEXT:    bfi r1, r0, #4, #1
+; CHECK-NEXT:    bfi r1, r0, #5, #1
+; CHECK-NEXT:    bfi r1, r0, #6, #1
+; CHECK-NEXT:    bfi r1, r0, #7, #1
+; CHECK-NEXT:    bfi r1, r0, #8, #1
+; CHECK-NEXT:    bfi r1, r0, #9, #1
+; CHECK-NEXT:    bfi r1, r0, #10, #1
+; CHECK-NEXT:    bfi r1, r0, #11, #1
+; CHECK-NEXT:    bfi r1, r0, #12, #1
+; CHECK-NEXT:    bfi r1, r0, #13, #1
+; CHECK-NEXT:    bfi r1, r0, #14, #1
+; CHECK-NEXT:    bfi r1, r0, #15, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -216,11 +207,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_var0_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_var0_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vldr s10, .LCPI9_0
 ; CHECK-NEXT:    vmov.f32 s9, s8
@@ -243,11 +233,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_var1_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_var1_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vmov s10, r0
 ; CHECK-NEXT:    vldr s8, .LCPI10_0
 ; CHECK-NEXT:    vmov.f32 s9, s8
@@ -270,11 +259,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_varN_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_varN_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vdup.32 q2, r0
 ; CHECK-NEXT:    vbic q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2

diff --git a/test/CodeGen/Thumb2/mve-pred-ext.ll b/test/CodeGen/Thumb2/mve-pred-ext.ll
index 387c84204a2af8e75554f0a5b06e7b891c5db844..bf3f64bf64bfcad741708d4eef3195625287d171 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-ext.ll
+++ b/test/CodeGen/Thumb2/mve-pred-ext.ll
@@ -57,17 +57,15 @@ define arm_aapcs_vfpcc <2 x i64> @sext_v2i1_v2i64(<2 x i64> %src) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    sbcs.w r1, r2, r1
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vmov.32 q0[0], r2
-; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    csinv r1, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r1
 ; CHECK-NEXT:    vmov.32 q0[2], r0
 ; CHECK-NEXT:    vmov.32 q0[3], r0
 ; CHECK-NEXT:    bx lr
@@ -136,15 +134,13 @@ define arm_aapcs_vfpcc <2 x i64> @zext_v2i1_v2i64(<2 x i64> %src) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    sbcs.w r2, r0, r2
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q0[0], r0
 ; CHECK-NEXT:    vmov.32 q0[2], r1
 ; CHECK-NEXT:    vand q0, q0, q1

diff --git a/test/CodeGen/Thumb2/mve-pred-loadstore.ll b/test/CodeGen/Thumb2/mve-pred-loadstore.ll
index 036c7c0f9cd4370621972b829a4e85b5c38e4964..27d8d077afbaa00dc3183072a1f439dd80825998 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-loadstore.ll
+++ b/test/CodeGen/Thumb2/mve-pred-loadstore.ll
@@ -167,11 +167,10 @@ define arm_aapcs_vfpcc void @store_v2i1(<2 x i1> *%dst, <2 x i64> %a) {
 ; CHECK-LE-NEXT:    vmov r3, s2
 ; CHECK-LE-NEXT:    orrs r1, r2
 ; CHECK-LE-NEXT:    vmov r2, s3
-; CHECK-LE-NEXT:    clz r1, r1
-; CHECK-LE-NEXT:    lsrs r1, r1, #5
+; CHECK-LE-NEXT:    csinc r1, zr, zr, ne
 ; CHECK-LE-NEXT:    orrs r2, r3
-; CHECK-LE-NEXT:    clz r2, r2
-; CHECK-LE-NEXT:    lsrs r2, r2, #5
+; CHECK-LE-NEXT:    csinc r2, zr, zr, ne
+; CHECK-LE-NEXT:    ands r2, r2, #1
 ; CHECK-LE-NEXT:    it ne
 ; CHECK-LE-NEXT:    mvnne r2, #1
 ; CHECK-LE-NEXT:    bfi r2, r1, #0, #1
@@ -187,11 +186,10 @@ define arm_aapcs_vfpcc void @store_v2i1(<2 x i1> *%dst, <2 x i64> %a) {
 ; CHECK-BE-NEXT:    vmov r3, s5
 ; CHECK-BE-NEXT:    orrs r1, r2
 ; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    clz r1, r1
-; CHECK-BE-NEXT:    lsrs r1, r1, #5
+; CHECK-BE-NEXT:    csinc r1, zr, zr, ne
 ; CHECK-BE-NEXT:    orrs r2, r3
-; CHECK-BE-NEXT:    clz r2, r2
-; CHECK-BE-NEXT:    lsrs r2, r2, #5
+; CHECK-BE-NEXT:    csinc r2, zr, zr, ne
+; CHECK-BE-NEXT:    ands r2, r2, #1
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1

diff --git a/test/CodeGen/Thumb2/mve-pred-not.ll b/test/CodeGen/Thumb2/mve-pred-not.ll
index 0b950ad62df6254990f79ea28e080bafa4ae802e..32b3b807e4513c26d9153988381c3f1ad180effc 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-not.ll
+++ b/test/CodeGen/Thumb2/mve-pred-not.ll
@@ -327,18 +327,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vbic q0, q0, q2
@@ -359,18 +357,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vbic q0, q0, q2

diff --git a/test/CodeGen/Thumb2/mve-pred-or.ll b/test/CodeGen/Thumb2/mve-pred-or.ll
index 2b4aaaf827d4cd3cd65905ab253ed02c4cea5747..df228a9d567f8d80a1f054594eb77e27a05b53ff 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-or.ll
+++ b/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -425,36 +425,32 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s7
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vorr q2, q3, q2
@@ -482,10 +478,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -494,27 +489,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vorr q2, q2, q3

diff --git a/test/CodeGen/Thumb2/mve-pred-xor.ll b/test/CodeGen/Thumb2/mve-pred-xor.ll
index 924d88d6026126df8be4818798bd1c8f88e0300e..5f93ecb3b37a1a29c06c0b9139e9b36a80b54f39 100644 (file)
--- a/test/CodeGen/Thumb2/mve-pred-xor.ll
+++ b/test/CodeGen/Thumb2/mve-pred-xor.ll
@@ -461,36 +461,32 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s7
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    veor q2, q3, q2
@@ -518,10 +514,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -530,27 +525,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    veor q2, q2, q3

diff --git a/test/CodeGen/Thumb2/mve-vcmp.ll b/test/CodeGen/Thumb2/mve-vcmp.ll
index a704b4eee67d9a74be5f2b809967c2321302b357..fadcb6432100a7a458f735360b4f03fd35bebc2e 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmp.ll
+++ b/test/CodeGen/Thumb2/mve-vcmp.ll
@@ -378,10 +378,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
 ; CHECK-NEXT:    vmov r0, s7
@@ -389,10 +388,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r1, s6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[2], r0
 ; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vbic q0, q3, q4
@@ -420,10 +418,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
 ; CHECK-NEXT:    vmov r0, s7
@@ -431,10 +428,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r1, s6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[2], r0
 ; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vbic q0, q3, q4
@@ -459,79 +455,67 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov lr, s10
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    subs r1, r0, r2
-; CHECK-NEXT:    asr.w r12, r0, #31
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
 ; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
 ; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r1
 ; CHECK-NEXT:    vmov.32 q3[1], r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    subs.w r2, r1, lr
+; CHECK-NEXT:    subs r0, r1, r2
 ; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    sbcs.w r2, r12, lr, asr #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    vmov.32 q4[2], r1
-; CHECK-NEXT:    vmov.32 q3[2], r3
-; CHECK-NEXT:    vmov.32 q4[3], r1
-; CHECK-NEXT:    vmov.32 q3[3], r3
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[0], r0
 ; CHECK-NEXT:    vmov.32 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[2], r0
 ; CHECK-NEXT:    vmov.32 q5[3], r0
 ; CHECK-NEXT:    vand q1, q5, q4

diff --git a/test/CodeGen/Thumb2/mve-vcmpf.ll b/test/CodeGen/Thumb2/mve-vcmpf.ll
index 224c6f5fe462835309d3f2101fbfb7f0c5b6fa72..b8235d6d38822a6e8d63e20dac1beeba92819003 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmpf.ll
+++ b/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -61,30 +57,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -92,24 +86,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -130,45 +122,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -186,45 +174,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -242,45 +226,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -298,45 +278,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -354,30 +330,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -385,24 +359,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -422,45 +394,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -478,45 +446,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -535,45 +499,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -592,45 +552,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -649,45 +605,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -706,45 +658,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -765,45 +713,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -835,12 +779,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -848,25 +791,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -876,11 +819,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -890,11 +832,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -904,11 +846,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -918,25 +859,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -969,12 +905,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -984,28 +919,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1018,10 +952,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1031,15 +964,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1052,10 +983,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1069,9 +999,8 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
@@ -1080,11 +1009,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1118,12 +1046,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1131,25 +1058,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1159,11 +1086,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1173,11 +1099,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1187,11 +1113,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1201,25 +1126,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1250,12 +1170,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1263,25 +1182,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1291,11 +1210,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1305,11 +1223,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1319,11 +1237,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1333,25 +1250,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1382,12 +1294,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1395,25 +1306,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1423,11 +1334,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1437,11 +1347,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1451,11 +1361,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1465,25 +1374,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1514,12 +1418,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1527,25 +1430,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1555,11 +1458,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1569,11 +1471,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1583,11 +1485,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1597,25 +1498,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1648,12 +1544,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1663,28 +1558,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1697,10 +1591,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1710,15 +1603,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1731,10 +1622,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1748,9 +1638,8 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
@@ -1759,11 +1648,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1796,12 +1684,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1809,25 +1696,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1837,11 +1724,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1851,11 +1737,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1865,11 +1751,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1879,25 +1764,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1928,12 +1808,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1941,25 +1820,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1969,11 +1848,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1983,11 +1861,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1997,11 +1875,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2011,25 +1888,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2061,12 +1933,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2074,25 +1945,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2102,11 +1973,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2116,11 +1986,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2130,11 +2000,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2144,25 +2013,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2194,12 +2058,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2207,25 +2070,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2235,11 +2098,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2249,11 +2111,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2263,11 +2125,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2277,25 +2138,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2327,12 +2183,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2340,25 +2195,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2368,11 +2223,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2382,11 +2236,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2396,11 +2250,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2410,25 +2263,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2460,12 +2308,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2473,25 +2320,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2501,11 +2348,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2515,11 +2361,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2529,11 +2375,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2543,25 +2388,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2595,12 +2435,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2608,25 +2447,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2636,11 +2475,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2650,11 +2488,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2664,11 +2502,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2678,25 +2515,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4

diff --git a/test/CodeGen/Thumb2/mve-vcmpfr.ll b/test/CodeGen/Thumb2/mve-vcmpfr.ll
index 8901ba493387cc151d48fb10aa583cd70e7f8a3b..73dc8856088feef4d6a923818d5d2cd79aceb407 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmpfr.ll
+++ b/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -64,30 +60,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -95,24 +89,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -137,45 +129,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -196,45 +184,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -255,45 +239,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -315,45 +295,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -375,30 +351,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -406,24 +380,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -447,45 +419,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -506,45 +474,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -567,45 +531,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -628,45 +588,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -688,45 +644,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -748,45 +700,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -811,45 +759,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -887,36 +831,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -926,10 +868,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -940,11 +881,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -954,10 +893,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -968,22 +906,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1023,10 +959,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1036,28 +971,26 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1070,24 +1003,22 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
@@ -1101,39 +1032,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1175,36 +1103,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1214,10 +1140,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1228,11 +1153,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1242,10 +1165,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1256,22 +1178,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1309,36 +1229,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1348,10 +1266,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1362,11 +1279,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1376,10 +1291,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1390,22 +1304,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1443,36 +1355,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1482,10 +1392,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1496,11 +1405,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1510,10 +1417,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1524,22 +1430,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1578,36 +1482,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1617,10 +1519,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1631,11 +1532,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1645,10 +1544,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1659,22 +1557,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1715,10 +1611,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1728,28 +1623,26 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1762,24 +1655,22 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
@@ -1793,39 +1684,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1866,36 +1754,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1905,10 +1791,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1919,11 +1804,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1933,10 +1816,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1947,22 +1829,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2000,36 +1880,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2039,10 +1917,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2053,11 +1930,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2067,10 +1942,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2081,22 +1955,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2136,36 +2008,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2175,10 +2045,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2189,11 +2058,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2203,10 +2070,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2217,22 +2083,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2272,36 +2136,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2311,10 +2173,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2325,11 +2186,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2339,10 +2198,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2353,22 +2211,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2407,36 +2263,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2446,10 +2300,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2460,11 +2313,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2474,10 +2325,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2488,22 +2338,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2542,36 +2390,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2581,10 +2427,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2595,11 +2440,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2609,10 +2452,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2623,22 +2465,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2680,36 +2520,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2719,10 +2557,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2733,11 +2570,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2747,10 +2582,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2761,22 +2595,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3

diff --git a/test/CodeGen/Thumb2/mve-vcmpfz.ll b/test/CodeGen/Thumb2/mve-vcmpfz.ll
index a16492f6fda2b734feacea7e21415cd012f5a59f..16ba72d88b9003f5d7e26b23aaaecf2ae1977c49 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -61,30 +57,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -92,24 +86,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -131,45 +123,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -187,45 +175,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -243,45 +227,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -299,45 +279,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -355,30 +331,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -386,24 +360,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -424,45 +396,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -480,45 +448,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -537,45 +501,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -594,45 +554,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -651,45 +607,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -708,45 +660,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -768,45 +716,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -839,36 +783,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -878,10 +821,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -892,11 +834,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -906,10 +846,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -920,23 +859,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -969,11 +905,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -982,28 +917,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1016,26 +950,23 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1048,39 +979,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
-; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1115,36 +1043,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1154,10 +1081,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1168,11 +1094,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1182,10 +1106,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1196,23 +1119,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1243,36 +1163,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1282,10 +1201,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1296,11 +1214,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1310,10 +1226,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1324,23 +1239,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1371,36 +1283,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1410,10 +1321,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1424,11 +1334,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1438,10 +1346,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1452,23 +1359,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1499,36 +1403,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1538,10 +1441,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1552,11 +1454,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1566,10 +1466,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1580,23 +1479,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1629,11 +1525,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1642,28 +1537,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1676,26 +1570,23 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1708,39 +1599,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1774,36 +1662,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1813,10 +1700,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1827,11 +1713,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1841,10 +1725,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1855,23 +1738,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1902,36 +1782,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1941,10 +1820,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1955,11 +1833,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1969,10 +1845,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1983,23 +1858,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2031,36 +1903,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2070,10 +1941,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2084,11 +1954,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2098,10 +1966,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2112,23 +1979,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2160,36 +2024,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2199,10 +2062,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2213,11 +2075,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2227,10 +2087,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2241,23 +2100,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2289,36 +2145,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2328,10 +2183,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2342,11 +2196,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2356,10 +2208,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2370,23 +2221,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2418,36 +2266,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2457,10 +2304,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s2
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2471,11 +2317,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2485,10 +2329,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s3
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2499,23 +2342,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2550,36 +2390,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2589,10 +2428,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s2
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2603,11 +2441,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2617,10 +2453,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s3
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2631,23 +2466,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3

diff --git a/test/CodeGen/Thumb2/mve-vcmpr.ll b/test/CodeGen/Thumb2/mve-vcmpr.ll
index 3adc4cb55b2a8bf5e16bc4f4e83df1c1e7094669..38ffd2981508ee94cddbe077ba70db58e184dcba 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmpr.ll
+++ b/test/CodeGen/Thumb2/mve-vcmpr.ll
@@ -444,10 +444,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r2
 ; CHECK-NEXT:    vmov.32 q3[1], r2
 ; CHECK-NEXT:    vmov r2, s3
@@ -455,10 +454,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -481,10 +479,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r2
 ; CHECK-NEXT:    vmov.32 q3[1], r2
 ; CHECK-NEXT:    vmov r2, s3
@@ -492,10 +489,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -521,79 +517,67 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov lr, s10
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    subs r1, r0, r2
-; CHECK-NEXT:    asr.w r12, r0, #31
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
 ; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
 ; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r1
 ; CHECK-NEXT:    vmov.32 q3[1], r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    subs.w r2, r1, lr
+; CHECK-NEXT:    subs r0, r1, r2
 ; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    sbcs.w r2, r12, lr, asr #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    vmov.32 q4[2], r1
-; CHECK-NEXT:    vmov.32 q3[2], r3
-; CHECK-NEXT:    vmov.32 q4[3], r1
-; CHECK-NEXT:    vmov.32 q3[3], r3
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[0], r0
 ; CHECK-NEXT:    vmov.32 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[2], r0
 ; CHECK-NEXT:    vmov.32 q5[3], r0
 ; CHECK-NEXT:    vand q1, q5, q4

diff --git a/test/CodeGen/Thumb2/mve-vcmpz.ll b/test/CodeGen/Thumb2/mve-vcmpz.ll
index 173c0335deeed2d3dc6bd1967f78344ade5b95ac..5e9fbf344a3d6dfc130e9482e138304bf17cefae 100644 (file)
--- a/test/CodeGen/Thumb2/mve-vcmpz.ll
+++ b/test/CodeGen/Thumb2/mve-vcmpz.ll
@@ -365,18 +365,16 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eqz_v2i64(<2 x i64> %src, <2 x i64> %a, <
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -396,18 +394,16 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eqz_v2i32(<2 x i64> %src, <2 x i32> %a, <
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3