Fix AVX-512 capability detection

When Checking for AVX Support, only the CPU's Capabilities and YMM
Register support by the OS were queried. In case of AVX-512, that is
insufficient, and ZMM Register support by the OS needs querying,
otherwise the OS will raise an Illegal Operation Exception if the CPU
is capable of AVX-512 but the OS is not.

Change-Id: I3444b19156d5743841de96cecbdaac19cc3f2b3f

diff --git a/vpx_ports/x86.h b/vpx_ports/x86.h
index 970ac2518f26d0ac7d409916d125b7df78cea164..ed26b16715a7790892b903e46b26c18142f4e27a 100644 (file)
--- a/vpx_ports/x86.h
+++ b/vpx_ports/x86.h
@@ -202,6 +202,7 @@ static INLINE int x86_simd_caps(void) {
 
   // bits 27 (OSXSAVE) & 28 (256-bit AVX)
   if ((reg_ecx & (BIT(27) | BIT(28))) == (BIT(27) | BIT(28))) {
+    // Check for OS-support of YMM state. Necessary for AVX and AVX2.
     if ((xgetbv() & 0x6) == 0x6) {
       flags |= HAS_AVX;
 
@@ -214,8 +215,10 @@ static INLINE int x86_simd_caps(void) {
         // bits 16 (AVX-512F) & 17 (AVX-512DQ) & 28 (AVX-512CD) &
         // 30 (AVX-512BW) & 32 (AVX-512VL)
         if ((reg_ebx & (BIT(16) | BIT(17) | BIT(28) | BIT(30) | BIT(31))) ==
-            (BIT(16) | BIT(17) | BIT(28) | BIT(30) | BIT(31)))
-          flags |= HAS_AVX512;
+            (BIT(16) | BIT(17) | BIT(28) | BIT(30) | BIT(31))) {
+          // Check for OS-support of ZMM and YMM state. Necessary for AVX-512.
+          if ((xgetbv() & 0xe6) == 0xe6) flags |= HAS_AVX512;
+        }
       }
     }
   }