x86: runtime AVX512-VNNI tier for the int8 DNN GEMV (+ multi-accumulator cgemv)

CMakeLists.txt

@@ -207,6 +207,14 @@ if(OPUS_CPU_X86 OR OPUS_CPU_X64)
                          OFF)
   add_feature_info(OPUS_X86_MAY_HAVE_AVX2 OPUS_X86_MAY_HAVE_AVX2 ${OPUS_X86_MAY_HAVE_AVX2_HELP_STR})
 
+  set(OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR "does runtime check for AVX512-VNNI support.")
+  cmake_dependent_option(OPUS_X86_MAY_HAVE_AVX512VNNI
+                         ${OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR}
+                         ON
+                         "AVX512VNNI_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS"
+                         OFF)
+  add_feature_info(OPUS_X86_MAY_HAVE_AVX512VNNI OPUS_X86_MAY_HAVE_AVX512VNNI ${OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR})
+
   # PRESUME depends on MAY HAVE, but PRESUME will override runtime detection
   set(OPUS_X86_PRESUME_SSE_HELP_STR "assume target CPU has SSE1 support (override runtime check).")
   set(OPUS_X86_PRESUME_SSE2_HELP_STR "assume target CPU has SSE2 support (override runtime check).")
@@ -253,6 +261,14 @@ if(OPUS_CPU_X86 OR OPUS_CPU_X64)
                          "OPUS_X86_MAY_HAVE_AVX2; NOT OPUS_DISABLE_INTRINSICS"
                          OFF)
   add_feature_info(OPUS_X86_PRESUME_AVX2 OPUS_X86_PRESUME_AVX2 ${OPUS_X86_PRESUME_AVX2_HELP_STR})
+
+  set(OPUS_X86_PRESUME_AVX512VNNI_HELP_STR "assume target CPU has AVX512-VNNI support (override runtime check).")
+  cmake_dependent_option(OPUS_X86_PRESUME_AVX512VNNI
+                         ${OPUS_X86_PRESUME_AVX512VNNI_HELP_STR}
+                         OFF
+                         "OPUS_X86_MAY_HAVE_AVX512VNNI; NOT OPUS_DISABLE_INTRINSICS"
+                         OFF)
+  add_feature_info(OPUS_X86_PRESUME_AVX512VNNI OPUS_X86_PRESUME_AVX512VNNI ${OPUS_X86_PRESUME_AVX512VNNI_HELP_STR})
 endif()
 
 feature_summary(WHAT ALL)
@@ -430,7 +446,8 @@ if(NOT OPUS_DISABLE_INTRINSICS)
   if(((OPUS_X86_MAY_HAVE_SSE AND NOT OPUS_X86_PRESUME_SSE) OR
      (OPUS_X86_MAY_HAVE_SSE2 AND NOT OPUS_X86_PRESUME_SSE2) OR
      (OPUS_X86_MAY_HAVE_SSE4_1 AND NOT OPUS_X86_PRESUME_SSE4_1) OR
-     (OPUS_X86_MAY_HAVE_AVX2 AND NOT OPUS_X86_PRESUME_AVX2)) AND
+     (OPUS_X86_MAY_HAVE_AVX2 AND NOT OPUS_X86_PRESUME_AVX2) OR
+     (OPUS_X86_MAY_HAVE_AVX512VNNI AND NOT OPUS_X86_PRESUME_AVX512VNNI)) AND
       RUNTIME_CPU_CAPABILITY_DETECTION)
     target_compile_definitions(opus PRIVATE OPUS_HAVE_RTCD)
     if(NOT MSVC)
@@ -543,6 +560,28 @@ if(NOT OPUS_DISABLE_INTRINSICS)
     endif()
   endif()
 
+  if(AVX512VNNI_SUPPORTED)
+    if(OPUS_X86_MAY_HAVE_AVX512VNNI)
+      if(OPUS_DNN)
+        add_sources_group(opus lpcnet ${dnn_sources_avx512vnni})
+      endif()
+      target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_AVX512VNNI)
+      if(MSVC)
+        set(AVX512VNNI_FLAGS "${AVX512VNNI_FLAGS} /arch:AVX512")
+      else()
+        # Keep vectors 256-bit wide: the win is the int8 dot-product instruction.
+        set(AVX512VNNI_FLAGS "${AVX512VNNI_FLAGS} -mavx -mfma -mavx2 -mavx512f -mavx512vl -mavx512vnni -mprefer-vector-width=256")
+      endif()
+      set_source_files_properties(${dnn_sources_avx512vnni} PROPERTIES COMPILE_FLAGS ${AVX512VNNI_FLAGS})
+    endif()
+    if(OPUS_X86_PRESUME_AVX512VNNI)
+      target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_AVX512VNNI)
+      if(NOT MSVC)
+        target_compile_options(opus PRIVATE -mavx -mfma -mavx2 -mavx512f -mavx512vl -mavx512vnni -mprefer-vector-width=256)
+      endif()
+    endif()
+  endif()
+
   if(MSVC)
     if(AVX2_SUPPORTED AND OPUS_X86_PRESUME_AVX2) # on 64 bit and 32 bits
       add_definitions(/arch:AVX2)

Makefile.am

@@ -82,6 +82,11 @@ if ENABLE_DEEP_PLC
 LPCNET_SOURCES += $(DNN_SOURCES_AVX2)
 endif
 endif
+if HAVE_AVX512VNNI
+if ENABLE_DEEP_PLC
+LPCNET_SOURCES += $(DNN_SOURCES_AVX512VNNI)
+endif
+endif
 endif
 
 if CPU_ARM
@@ -504,6 +509,11 @@ AVX2_OBJ = $(CELT_SOURCES_AVX2:.c=.lo) \
 $(AVX2_OBJ): CFLAGS += $(OPUS_X86_AVX2_CFLAGS)
 endif
 
+if HAVE_AVX512VNNI
+AVX512VNNI_OBJ = $(DNN_SOURCES_AVX512VNNI:.c=.lo)
+$(AVX512VNNI_OBJ): CFLAGS += $(OPUS_X86_AVX512VNNI_CFLAGS)
+endif
+
 if HAVE_ARM_NEON_INTR
 ARM_NEON_INTR_OBJ = $(CELT_SOURCES_ARM_NEON_INTR:.c=.lo) \
                     $(SILK_SOURCES_ARM_NEON_INTR:.c=.lo) \

celt/cpu_support.h

@@ -48,15 +48,17 @@
   ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \
-  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)))
+  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \
+  (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI)))
 
 #include "x86/x86cpu.h"
-/* We currently support 5 x86 variants:
+/* We currently support 6 x86 variants:
  * arch[0] -> non-sse
  * arch[1] -> sse
  * arch[2] -> sse2
  * arch[3] -> sse4.1
  * arch[4] -> avx
+ * arch[5] -> avx512vnni
  */
 #define OPUS_ARCHMASK 7
 int opus_select_arch(void);

celt/x86/x86_celt_map.c

@@ -53,7 +53,8 @@ void (*const CELT_FIR_IMPL[OPUS_ARCHMASK + 1])(
   celt_fir_c,
   celt_fir_c,
   MAY_HAVE_SSE4_1(celt_fir), /* sse4.1  */
-  MAY_HAVE_SSE4_1(celt_fir)  /* avx  */
+  MAY_HAVE_SSE4_1(celt_fir), /* avx  */
+  MAY_HAVE_SSE4_1(celt_fir)  /* avx512vnni  */
 };
 
 void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])(
@@ -66,7 +67,8 @@ void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])(
   xcorr_kernel_c,
   xcorr_kernel_c,
   MAY_HAVE_SSE4_1(xcorr_kernel), /* sse4.1  */
-  MAY_HAVE_SSE4_1(xcorr_kernel)  /* avx  */
+  MAY_HAVE_SSE4_1(xcorr_kernel), /* avx  */
+  MAY_HAVE_SSE4_1(xcorr_kernel)  /* avx512vnni  */
 };
 
 #endif
@@ -83,7 +85,8 @@ opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])(
   celt_inner_prod_c,
   MAY_HAVE_SSE2(celt_inner_prod),
   MAY_HAVE_SSE4_1(celt_inner_prod), /* sse4.1  */
-  MAY_HAVE_SSE4_1(celt_inner_prod)  /* avx  */
+  MAY_HAVE_SSE4_1(celt_inner_prod), /* avx  */
+  MAY_HAVE_SSE4_1(celt_inner_prod)  /* avx512vnni  */
 };
 
 #endif
@@ -104,7 +107,8 @@ void (*const PITCH_XCORR_IMPL[OPUS_ARCHMASK + 1])(
   celt_pitch_xcorr_c,
   celt_pitch_xcorr_c,
   celt_pitch_xcorr_c,
-  MAY_HAVE_AVX2(celt_pitch_xcorr)
+  MAY_HAVE_AVX2(celt_pitch_xcorr), /* avx  */
+  MAY_HAVE_AVX2(celt_pitch_xcorr)  /* avx512vnni  */
 };
 
 #endif
@@ -122,6 +126,7 @@ void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])(
   MAY_HAVE_SSE(xcorr_kernel),
   MAY_HAVE_SSE(xcorr_kernel),
   MAY_HAVE_SSE(xcorr_kernel),
+  MAY_HAVE_SSE(xcorr_kernel),
   MAY_HAVE_SSE(xcorr_kernel)
 };
 
@@ -134,6 +139,7 @@ opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])(
   MAY_HAVE_SSE(celt_inner_prod),
   MAY_HAVE_SSE(celt_inner_prod),
   MAY_HAVE_SSE(celt_inner_prod),
+  MAY_HAVE_SSE(celt_inner_prod),
   MAY_HAVE_SSE(celt_inner_prod)
 };
 
@@ -149,6 +155,7 @@ void (*const DUAL_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])(
   MAY_HAVE_SSE(dual_inner_prod),
   MAY_HAVE_SSE(dual_inner_prod),
   MAY_HAVE_SSE(dual_inner_prod),
+  MAY_HAVE_SSE(dual_inner_prod),
   MAY_HAVE_SSE(dual_inner_prod)
 };
 
@@ -165,6 +172,7 @@ void (*const COMB_FILTER_CONST_IMPL[OPUS_ARCHMASK + 1])(
   MAY_HAVE_SSE(comb_filter_const),
   MAY_HAVE_SSE(comb_filter_const),
   MAY_HAVE_SSE(comb_filter_const),
+  MAY_HAVE_SSE(comb_filter_const),
   MAY_HAVE_SSE(comb_filter_const)
 };
 
@@ -179,6 +187,7 @@ opus_val16 (*const OP_PVQ_SEARCH_IMPL[OPUS_ARCHMASK + 1])(
   op_pvq_search_c,
   MAY_HAVE_SSE2(op_pvq_search),
   MAY_HAVE_SSE2(op_pvq_search),
+  MAY_HAVE_SSE2(op_pvq_search),
   MAY_HAVE_SSE2(op_pvq_search)
 };
 #endif

celt/x86/x86cpu.c

@@ -39,7 +39,8 @@
   ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \
-  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)))
+  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \
+  (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI)))
 
 #if defined(_MSC_VER)
 
@@ -99,13 +100,36 @@ static void cpuid(unsigned int CPUInfo[4], unsigned int InfoType)
 
 #endif
 
+#if defined(OPUS_X86_MAY_HAVE_AVX512VNNI)
+/* Returns the low 32 bits of the extended control register XCR0, which tells
+   us whether the OS has enabled saving/restoring of the relevant vector state.
+   This must only be called once OSXSAVE (CPUID.1:ECX[27]) has been confirmed,
+   otherwise XGETBV is an illegal instruction. */
+static opus_uint32 get_xcr0(void)
+{
+#if defined(_MSC_VER)
+    return (opus_uint32)_xgetbv(0);
+#elif defined(CPU_INFO_BY_ASM) || defined(CPU_INFO_BY_C)
+    opus_uint32 eax, edx;
+    __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
+    (void)edx;
+    return eax;
+#else
+    return 0;
+#endif
+}
+#endif
+
 typedef struct CPU_Feature{
     /*  SIMD: 128-bit */
     int HW_SSE;
     int HW_SSE2;
     int HW_SSE41;
     /*  SIMD: 256-bit */
     int HW_AVX2;
+    /*  256-bit VNNI int8 dot product (EVEX-encoded vpdpbusd), requires
+        AVX512F + AVX512VL + AVX512_VNNI and OS support for AVX-512 state. */
+    int HW_AVX512VNNI;
 } CPU_Feature;
 
 static void opus_cpu_feature_check(CPU_Feature *cpu_feature)
@@ -117,14 +141,35 @@ static void opus_cpu_feature_check(CPU_Feature *cpu_feature)
     nIds = info[0];
 
     if (nIds >= 1){
+        unsigned int leaf1_ecx;
         cpuid(info, 1);
+        leaf1_ecx = info[2];
         cpu_feature->HW_SSE = (info[3] & (1 << 25)) != 0;
         cpu_feature->HW_SSE2 = (info[3] & (1 << 26)) != 0;
         cpu_feature->HW_SSE41 = (info[2] & (1 << 19)) != 0;
         cpu_feature->HW_AVX2 = (info[2] & (1 << 28)) != 0 && (info[2] & (1 << 12)) != 0;
+        cpu_feature->HW_AVX512VNNI = 0;
         if (cpu_feature->HW_AVX2 && nIds >= 7) {
             cpuid(info, 7);
             cpu_feature->HW_AVX2 = cpu_feature->HW_AVX2 && (info[1] & (1 << 5)) != 0;
+#if defined(OPUS_X86_MAY_HAVE_AVX512VNNI)
+            /* leaf 7, sub-leaf 0: EBX[16]=AVX512F, EBX[31]=AVX512VL,
+               ECX[11]=AVX512_VNNI. The 256-bit EVEX vpdpbusd we emit for this
+               tier needs F+VL+VNNI. We also require that the OS has enabled
+               AVX-512 register state (XCR0 bits 5,6,7, on top of the SSE/AVX
+               bits 1,2), otherwise the EVEX-encoded instructions fault. */
+            if (cpu_feature->HW_AVX2
+             && (info[1] & (1u << 16)) != 0   /* AVX512F     */
+             && (info[1] & (1u << 31)) != 0   /* AVX512VL    */
+             && (info[2] & (1u << 11)) != 0   /* AVX512_VNNI */
+             && (leaf1_ecx & (1u << 27)) != 0 /* OSXSAVE     */) {
+                opus_uint32 xcr0 = get_xcr0();
+                unsigned int avx512_state = (1u << 1) | (1u << 2)
+                                          | (1u << 5) | (1u << 6) | (1u << 7);
+                cpu_feature->HW_AVX512VNNI =
+                    (xcr0 & avx512_state) == avx512_state;
+            }
+#endif
         } else {
             cpu_feature->HW_AVX2 = 0;
         }
@@ -134,6 +179,7 @@ static void opus_cpu_feature_check(CPU_Feature *cpu_feature)
         cpu_feature->HW_SSE2 = 0;
         cpu_feature->HW_SSE41 = 0;
         cpu_feature->HW_AVX2 = 0;
+        cpu_feature->HW_AVX512VNNI = 0;
     }
 }
 
@@ -169,6 +215,12 @@ static int opus_select_arch_impl(void)
     }
     arch++;
 
+    if (!cpu_feature.HW_AVX512VNNI)
+    {
+        return arch;
+    }
+    arch++;
+
     return arch;
 }
 

celt/x86/x86cpu.h

@@ -52,11 +52,18 @@
 #  define MAY_HAVE_AVX2(name) name ## _c
 # endif
 
+# if defined(OPUS_X86_MAY_HAVE_AVX512VNNI)
+#  define MAY_HAVE_AVX512VNNI(name) name ## _avx512vnni
+# else
+#  define MAY_HAVE_AVX512VNNI(name) MAY_HAVE_AVX2(name)
+# endif
+
 # if defined(OPUS_HAVE_RTCD) && \
   ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \
   (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \
-  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)))
+  (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \
+  (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI)))
 int opus_select_arch(void);
 # endif
 

cmake/OpusFunctions.cmake

@@ -126,7 +126,20 @@ function(opus_detect_sse COMPILER_SUPPORT_SIMD)
         PARENT_SCOPE)
   endif()
 
-  if(SSE1_SUPPORTED OR SSE2_SUPPORTED OR SSE4_1_SUPPORTED OR AVX2_SUPPORTED)
+  # AVX512-VNNI provides the EVEX-encoded 256-bit vpdpbusd int8 dot product.
+  if(HAVE_IMMINTRIN_H)
+    if(MSVC)
+      check_flag(AVX512VNNI /arch:AVX512)
+    else()
+      check_flag(AVX512VNNI -mavx512vnni)
+    endif()
+  else()
+    set(AVX512VNNI_SUPPORTED
+        0
+        PARENT_SCOPE)
+  endif()
+
+  if(SSE1_SUPPORTED OR SSE2_SUPPORTED OR SSE4_1_SUPPORTED OR AVX2_SUPPORTED OR AVX512VNNI_SUPPORTED)
     set(COMPILER_SUPPORT_SIMD 1 PARENT_SCOPE)
   else()
     message(STATUS "No SIMD support in compiler")

cmake/OpusSources.cmake

@@ -50,6 +50,7 @@ get_opus_sources(DNN_SOURCES_X86_RTCD lpcnet_sources.mk dnn_sources_x86_rtcd)
 get_opus_sources(DNN_SOURCES_SSE2 lpcnet_sources.mk dnn_sources_sse2)
 get_opus_sources(DNN_SOURCES_SSE4_1 lpcnet_sources.mk dnn_sources_sse4_1)
 get_opus_sources(DNN_SOURCES_AVX2 lpcnet_sources.mk dnn_sources_avx2)
+get_opus_sources(DNN_SOURCES_AVX512VNNI lpcnet_sources.mk dnn_sources_avx512vnni)
 get_opus_sources(DNN_SOURCES_NEON lpcnet_sources.mk dnn_sources_arm_neon)
 get_opus_sources(DNN_SOURCES_DOTPROD lpcnet_sources.mk dnn_sources_arm_dotprod)
 get_opus_sources(DNN_SOURCES_ARM_RTCD lpcnet_sources.mk dnn_sources_arm_rtcd)