AARCH64_ARCH("armv8.2-a", ARMV8_2A, "8.2-A", "v8.2a",
ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
(AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
- AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE))
+ AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
+ AArch64::AEK_DOTPROD))
#undef AARCH64_ARCH
#ifndef AARCH64_ARCH_EXT_NAME
AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc")
AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse")
AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto")
+AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod")
AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8")
AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon")
AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16")
AEK_PROFILE = 0x40,
AEK_RAS = 0x80,
AEK_LSE = 0x100,
- AEK_SVE = 0x200
+ AEK_SVE = 0x200,
+ AEK_DOTPROD = 0x400
};
StringRef getCanonicalArchName(StringRef Arch);
"use-reciprocal-square-root", "UseRSqrt", "true",
"Use the reciprocal square root approximation">;
+def FeatureDotProd : SubtargetFeature<
+ "dotprod", "HasDotProd", "true",
+ "Enable dot product support">;
+
def FeatureNoNegativeImmediates : SubtargetFeature<"no-neg-immediates",
"NegativeImmediates", "false",
"Convert immediates and instructions "
let Inst{4-0} = Rd;
}
+class BaseSIMDThreeSameVectorDot<bit Q, bit U, string asm, string kind1,
+ string kind2> :
+ BaseSIMDThreeSameVector<Q, U, 0b100, 0b10010, V128, asm, kind1, [] > {
+ let AsmString = !strconcat(asm, "{\t$Rd" # kind1 # ", $Rn" # kind2 # ", $Rm" # kind2 # "}");
+}
+
// All operand sizes distinguished in the encoding.
multiclass SIMDThreeSameVector<bit U, bits<5> opc, string asm,
SDPatternOperator OpNode> {
let Inst{4-0} = Rd;
}
+// ARMv8.2 Index Dot product instructions
+class BaseSIMDThreeSameVectorDotIndex<bit Q, bit U, string asm, string dst_kind,
+ string lhs_kind, string rhs_kind> :
+ BaseSIMDIndexedTied<Q, U, 0b0, 0b10, 0b1110, V128, V128, V128, VectorIndexS,
+ asm, "", dst_kind, lhs_kind, rhs_kind, []> {
+ bits<2> idx;
+ let Inst{21} = idx{0}; // L
+ let Inst{11} = idx{1}; // H
+}
+
multiclass SIMDFPIndexed<bit U, bits<4> opc, string asm,
SDPatternOperator OpNode> {
let Predicates = [HasNEON, HasFullFP16] in {
//----------------------------------------------------------------------------
// Allow the size specifier tokens to be upper case, not just lower.
+def : TokenAlias<".4B", ".4b">; // Add dot product
def : TokenAlias<".8B", ".8b">;
def : TokenAlias<".4H", ".4h">;
def : TokenAlias<".2S", ".2s">;
AssemblerPredicate<"FeatureNEON", "neon">;
def HasCrypto : Predicate<"Subtarget->hasCrypto()">,
AssemblerPredicate<"FeatureCrypto", "crypto">;
+def HasDotProd : Predicate<"Subtarget->hasDotProd()">,
+ AssemblerPredicate<"FeatureDotProd", "dotprod">;
def HasCRC : Predicate<"Subtarget->hasCRC()">,
AssemblerPredicate<"FeatureCRC", "crc">;
def HasLSE : Predicate<"Subtarget->hasLSE()">,
[(int_aarch64_isb (i32 imm32_0_15:$CRm))]>;
}
+// ARMv8.2 Dot Product
+let Predicates = [HasDotProd] in {
+def UDOT2S : BaseSIMDThreeSameVectorDot<0, 1, "udot", ".2s", ".8b">;
+def SDOT2S : BaseSIMDThreeSameVectorDot<0, 0, "sdot", ".2s", ".8b">;
+def UDOT4S : BaseSIMDThreeSameVectorDot<1, 1, "udot", ".4s", ".16b">;
+def SDOT4S : BaseSIMDThreeSameVectorDot<1, 0, "sdot", ".4s", ".16b">;
+def UDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 1, "udot", ".2s", ".8b", ".4b">;
+def SDOTIDX2S : BaseSIMDThreeSameVectorDotIndex<0, 0, "sdot", ".2s", ".8b", ".4b">;
+def UDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 1, "udot", ".4s", ".16b", ".4b">;
+def SDOTIDX4S : BaseSIMDThreeSameVectorDotIndex<1, 0, "sdot", ".4s", ".16b", ".4b">;
+}
+
def : InstAlias<"clrex", (CLREX 0xf)>;
def : InstAlias<"isb", (ISB 0xf)>;
bool HasFPARMv8 = false;
bool HasNEON = false;
bool HasCrypto = false;
+ bool HasDotProd = false;
bool HasCRC = false;
bool HasLSE = false;
bool HasRAS = false;
bool hasFPARMv8() const { return HasFPARMv8; }
bool hasNEON() const { return HasNEON; }
bool hasCrypto() const { return HasCrypto; }
+ bool hasDotProd() const { return HasDotProd; }
bool hasCRC() const { return HasCRC; }
bool hasLSE() const { return HasLSE; }
bool hasRAS() const { return HasRAS; }
.Case(".d", true)
// Needed for fp16 scalar pairwise reductions
.Case(".2h", true)
+ // another special case for the ARMv8.2a dot product operand
+ .Case(".4b", true)
.Default(false);
}
--- /dev/null
+// RUN: not llvm-mc -triple aarch64 -mattr=+dotprod -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-ERROR < %t %s
+
+udot v0.2s, v1.8b, v2.4b[4]
+sdot v0.2s, v1.8b, v2.4b[4]
+udot v0.4s, v1.16b, v2.4b[4]
+sdot v0.4s, v1.16b, v2.4b[4]
+
+// CHECK-ERROR: vector lane must be an integer in range [0, 3]
+// CHECK-ERROR: vector lane must be an integer in range [0, 3]
+// CHECK-ERROR: vector lane must be an integer in range [0, 3]
+// CHECK-ERROR: vector lane must be an integer in range [0, 3]
--- /dev/null
+// RUN: llvm-mc -triple aarch64 -mattr=+dotprod -show-encoding < %s | FileCheck %s --check-prefix=CHECK-DOTPROD
+// RUN: not llvm-mc -triple aarch64 -mattr=+v8.2a -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-NO-DOTPROD < %t %s
+
+udot v0.2s, v1.8b, v2.8b
+sdot v0.2s, v1.8b, v2.8b
+udot v0.4s, v1.16b, v2.16b
+sdot v0.4s, v1.16b, v2.16b
+udot v0.2s, v1.8b, v2.4b[0]
+sdot v0.2s, v1.8b, v2.4b[1]
+udot v0.4s, v1.16b, v2.4b[2]
+sdot v0.4s, v1.16b, v2.4b[3]
+
+// Check that the upper case types are aliases
+udot v0.2S, v1.8B, v2.4B[0]
+udot v0.4S, v1.16B, v2.4B[2]
+
+// CHECK-DOTPROD: udot v0.2s, v1.8b, v2.8b // encoding: [0x20,0x94,0x82,0x2e]
+// CHECK-DOTPROD: sdot v0.2s, v1.8b, v2.8b // encoding: [0x20,0x94,0x82,0x0e]
+// CHECK-DOTPROD: udot v0.4s, v1.16b, v2.16b // encoding: [0x20,0x94,0x82,0x6e]
+// CHECK-DOTPROD: sdot v0.4s, v1.16b, v2.16b // encoding: [0x20,0x94,0x82,0x4e]
+// CHECK-DOTPROD: udot v0.2s, v1.8b, v2.4b[0] // encoding: [0x20,0xe0,0x82,0x2f]
+// CHECK-DOTPROD: sdot v0.2s, v1.8b, v2.4b[1] // encoding: [0x20,0xe0,0xa2,0x0f]
+// CHECK-DOTPROD: udot v0.4s, v1.16b, v2.4b[2] // encoding: [0x20,0xe8,0x82,0x6f]
+// CHECK-DOTPROD: sdot v0.4s, v1.16b, v2.4b[3] // encoding: [0x20,0xe8,0xa2,0x4f]
+
+// CHECK-DOTPROD: udot v0.2s, v1.8b, v2.4b[0] // encoding: [0x20,0xe0,0x82,0x2f]
+// CHECK-DOTPROD: udot v0.4s, v1.16b, v2.4b[2] // encoding: [0x20,0xe8,0x82,0x6f]
+
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.2s, v1.8b, v2.8b
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: sdot v0.2s, v1.8b, v2.8b
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.4s, v1.16b, v2.16b
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: sdot v0.4s, v1.16b, v2.16b
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.2s, v1.8b, v2.4b[0]
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: sdot v0.2s, v1.8b, v2.4b[1]
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.4s, v1.16b, v2.4b[2]
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: sdot v0.4s, v1.16b, v2.4b[3]
+// CHECK-NO-DOTPROD: ^
+
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.2S, v1.8B, v2.4B[0]
+// CHECK-NO-DOTPROD: ^
+// CHECK-NO-DOTPROD: error: instruction requires: dotprod
+// CHECK-NO-DOTPROD: udot v0.4S, v1.16B, v2.4B[2]
+// CHECK-NO-DOTPROD: ^
uzp1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
uzp1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp1 v0.4h, v1.2h, v2.2h
uzp2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
uzp2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp2 v0.4h, v1.2h, v2.2h
zip1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
zip1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip1 v0.4h, v1.2h, v2.2h
// CHECK-ERROR: [[@LINE-1]]:14: error: invalid operand for instruction
-\
+
zip2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
zip2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip2 v0.4h, v1.2h, v2.2h
trn1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
trn1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn1 v0.4h, v1.2h, v2.2h
trn2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
trn2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn2 v0.4h, v1.2h, v2.2h
uzp1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
uzp1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp1 v0.4h, v1.2h, v2.2h
uzp2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
uzp2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
uzp2 v0.4h, v1.2h, v2.2h
zip1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
zip1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip1 v0.4h, v1.2h, v2.2h
zip2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
zip2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
zip2 v0.4h, v1.2h, v2.2h
trn1 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
trn1 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn1 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn1 v0.4h, v1.2h, v2.2h
trn2 v0.16b, v1.8b, v2.8b
// CHECK-ERROR: [[@LINE-1]]:22: error: invalid operand for instruction
trn2 v0.8b, v1.4b, v2.4b
-// CHECK-ERROR: [[@LINE-1]]:21: error: invalid vector kind qualifier
-// CHECK-ERROR: [[@LINE-2]]:28: error: invalid vector kind qualifier
+// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn2 v0.8h, v1.4h, v2.4h
// CHECK-ERROR: [[@LINE-1]]:21: error: invalid operand for instruction
trn2 v0.4h, v1.2h, v2.2h
--- /dev/null
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+dotprod --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=-dotprod --disassemble < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+0x20,0x94,0x82,0x2e
+0x20,0x94,0x82,0x0e
+0x20,0x94,0x82,0x6e
+0x20,0x94,0x82,0x4e
+0x20,0xe0,0x82,0x2f
+0x20,0xe0,0xa2,0x0f
+0x20,0xe8,0x82,0x6f
+0x20,0xe8,0xa2,0x4f
+
+#CHECK: udot v0.2s, v1.8b, v2.8b
+#CHECK: sdot v0.2s, v1.8b, v2.8b
+#CHECK: udot v0.4s, v1.16b, v2.16b
+#CHECK: sdot v0.4s, v1.16b, v2.16b
+#CHECK: udot v0.2s, v1.8b, v2.4b[0]
+#CHECK: sdot v0.2s, v1.8b, v2.4b[1]
+#CHECK: udot v0.4s, v1.16b, v2.4b[2]
+#CHECK: sdot v0.4s, v1.16b, v2.4b[3]
+
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding
+# CHECK-ERROR: invalid instruction encoding