From 8c35295ffa238353adaaaf6186335163b5a6b822 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 19:34:16 +0000 Subject: [PATCH 1/2] x86: add runtime AVX512-VNNI tier for the int8 DNN GEMV Add an RTCD arch level above AVX2 (index 5) so CPUs with AVX512-VNNI run the EVEX-encoded 256-bit vpdpbusd int8 dot product in the DNN int8 GEMV (compute_linear with quantized weights) instead of the AVX2 vpmaddubsw+vpmaddwd+vpaddd emulation. Mirrors the Arm DOTPROD tier. This box has AVX512-VNNI but not the VEX-encoded AVX-VNNI, and opus's strictly-sequential RTCD ladder (plus the FUZZING downgrade) can only carry one encoding per tier, so the tier targets AVX512-VNNI: compiled -mavx512vnni -mavx512vl (kept 256-bit via -mprefer-vector-width=256, the win is the instruction not wider vectors) and gated on AVX512F+AVX512VL+AVX512_VNNI with an OSXSAVE/XGETBV check for OS AVX-512 state. Detection: CPUID.(7,0) EBX[16]/EBX[31]/ECX[11]. - celt/x86/x86cpu.{c,h}: detect AVX512-VNNI, MAY_HAVE_AVX512VNNI macro, new arch level in opus_select_arch. - dnn/x86: nnet_avx512vnni.c (RTCD_ARCH avx512vnni), declare/dispatch compute_linear_avx512vnni; only compute_linear uses vpdpbusd, so activation/conv2d reuse the AVX2 variants at index 5. - Fill index 5 in every x86 _IMPL table (celt/silk) so it is never NULL. - Wire detection + per-source flags in autotools, meson and cmake. Verified on this CPU: opus_select_arch()=5, DNN_COMPUTE_LINEAR_IMPL[5] dispatches to compute_linear_avx512vnni, object emits EVEX vpdpbusds (no zmm), all three build systems compile it, and DRED encode+decode output is byte-identical to the AVX2 tier (bit-exact). https://claude.ai/code/session_01H3ZadW9kpYqiMEGxhtpV2j --- CMakeLists.txt | 41 ++++++++++++++++++++++++++++- Makefile.am | 10 ++++++++ celt/cpu_support.h | 6 +++-- celt/x86/x86_celt_map.c | 17 +++++++++--- celt/x86/x86cpu.c | 54 ++++++++++++++++++++++++++++++++++++++- celt/x86/x86cpu.h | 9 ++++++- cmake/OpusFunctions.cmake | 15 ++++++++++- cmake/OpusSources.cmake | 1 + configure.ac | 42 ++++++++++++++++++++++++++++++ dnn/meson.build | 3 ++- dnn/x86/dnn_x86.h | 19 ++++++++++++-- dnn/x86/nnet_avx512vnni.c | 51 ++++++++++++++++++++++++++++++++++++ dnn/x86/x86_dnn_map.c | 11 +++++--- lpcnet_sources.mk | 1 + meson.build | 4 +++ silk/x86/x86_silk_map.c | 21 ++++++++++----- 16 files changed, 281 insertions(+), 24 deletions(-) create mode 100644 dnn/x86/nnet_avx512vnni.c diff --git a/CMakeLists.txt b/CMakeLists.txt index aa7b1ae17..e53ceb990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,6 +207,14 @@ if(OPUS_CPU_X86 OR OPUS_CPU_X64) OFF) add_feature_info(OPUS_X86_MAY_HAVE_AVX2 OPUS_X86_MAY_HAVE_AVX2 ${OPUS_X86_MAY_HAVE_AVX2_HELP_STR}) + set(OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR "does runtime check for AVX512-VNNI support.") + cmake_dependent_option(OPUS_X86_MAY_HAVE_AVX512VNNI + ${OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR} + ON + "AVX512VNNI_SUPPORTED; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_MAY_HAVE_AVX512VNNI OPUS_X86_MAY_HAVE_AVX512VNNI ${OPUS_X86_MAY_HAVE_AVX512VNNI_HELP_STR}) + # PRESUME depends on MAY HAVE, but PRESUME will override runtime detection set(OPUS_X86_PRESUME_SSE_HELP_STR "assume target CPU has SSE1 support (override runtime check).") set(OPUS_X86_PRESUME_SSE2_HELP_STR "assume target CPU has SSE2 support (override runtime check).") @@ -253,6 +261,14 @@ if(OPUS_CPU_X86 OR OPUS_CPU_X64) "OPUS_X86_MAY_HAVE_AVX2; NOT OPUS_DISABLE_INTRINSICS" OFF) add_feature_info(OPUS_X86_PRESUME_AVX2 OPUS_X86_PRESUME_AVX2 ${OPUS_X86_PRESUME_AVX2_HELP_STR}) + + set(OPUS_X86_PRESUME_AVX512VNNI_HELP_STR "assume target CPU has AVX512-VNNI support (override runtime check).") + cmake_dependent_option(OPUS_X86_PRESUME_AVX512VNNI + ${OPUS_X86_PRESUME_AVX512VNNI_HELP_STR} + OFF + "OPUS_X86_MAY_HAVE_AVX512VNNI; NOT OPUS_DISABLE_INTRINSICS" + OFF) + add_feature_info(OPUS_X86_PRESUME_AVX512VNNI OPUS_X86_PRESUME_AVX512VNNI ${OPUS_X86_PRESUME_AVX512VNNI_HELP_STR}) endif() feature_summary(WHAT ALL) @@ -430,7 +446,8 @@ if(NOT OPUS_DISABLE_INTRINSICS) if(((OPUS_X86_MAY_HAVE_SSE AND NOT OPUS_X86_PRESUME_SSE) OR (OPUS_X86_MAY_HAVE_SSE2 AND NOT OPUS_X86_PRESUME_SSE2) OR (OPUS_X86_MAY_HAVE_SSE4_1 AND NOT OPUS_X86_PRESUME_SSE4_1) OR - (OPUS_X86_MAY_HAVE_AVX2 AND NOT OPUS_X86_PRESUME_AVX2)) AND + (OPUS_X86_MAY_HAVE_AVX2 AND NOT OPUS_X86_PRESUME_AVX2) OR + (OPUS_X86_MAY_HAVE_AVX512VNNI AND NOT OPUS_X86_PRESUME_AVX512VNNI)) AND RUNTIME_CPU_CAPABILITY_DETECTION) target_compile_definitions(opus PRIVATE OPUS_HAVE_RTCD) if(NOT MSVC) @@ -543,6 +560,28 @@ if(NOT OPUS_DISABLE_INTRINSICS) endif() endif() + if(AVX512VNNI_SUPPORTED) + if(OPUS_X86_MAY_HAVE_AVX512VNNI) + if(OPUS_DNN) + add_sources_group(opus lpcnet ${dnn_sources_avx512vnni}) + endif() + target_compile_definitions(opus PRIVATE OPUS_X86_MAY_HAVE_AVX512VNNI) + if(MSVC) + set(AVX512VNNI_FLAGS "${AVX512VNNI_FLAGS} /arch:AVX512") + else() + # Keep vectors 256-bit wide: the win is the int8 dot-product instruction. + set(AVX512VNNI_FLAGS "${AVX512VNNI_FLAGS} -mavx -mfma -mavx2 -mavx512f -mavx512vl -mavx512vnni -mprefer-vector-width=256") + endif() + set_source_files_properties(${dnn_sources_avx512vnni} PROPERTIES COMPILE_FLAGS ${AVX512VNNI_FLAGS}) + endif() + if(OPUS_X86_PRESUME_AVX512VNNI) + target_compile_definitions(opus PRIVATE OPUS_X86_PRESUME_AVX512VNNI) + if(NOT MSVC) + target_compile_options(opus PRIVATE -mavx -mfma -mavx2 -mavx512f -mavx512vl -mavx512vnni -mprefer-vector-width=256) + endif() + endif() + endif() + if(MSVC) if(AVX2_SUPPORTED AND OPUS_X86_PRESUME_AVX2) # on 64 bit and 32 bits add_definitions(/arch:AVX2) diff --git a/Makefile.am b/Makefile.am index 1cf73822c..25c7517d1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,6 +82,11 @@ if ENABLE_DEEP_PLC LPCNET_SOURCES += $(DNN_SOURCES_AVX2) endif endif +if HAVE_AVX512VNNI +if ENABLE_DEEP_PLC +LPCNET_SOURCES += $(DNN_SOURCES_AVX512VNNI) +endif +endif endif if CPU_ARM @@ -504,6 +509,11 @@ AVX2_OBJ = $(CELT_SOURCES_AVX2:.c=.lo) \ $(AVX2_OBJ): CFLAGS += $(OPUS_X86_AVX2_CFLAGS) endif +if HAVE_AVX512VNNI +AVX512VNNI_OBJ = $(DNN_SOURCES_AVX512VNNI:.c=.lo) +$(AVX512VNNI_OBJ): CFLAGS += $(OPUS_X86_AVX512VNNI_CFLAGS) +endif + if HAVE_ARM_NEON_INTR ARM_NEON_INTR_OBJ = $(CELT_SOURCES_ARM_NEON_INTR:.c=.lo) \ $(SILK_SOURCES_ARM_NEON_INTR:.c=.lo) \ diff --git a/celt/cpu_support.h b/celt/cpu_support.h index 9f13d8aec..310ae30e0 100644 --- a/celt/cpu_support.h +++ b/celt/cpu_support.h @@ -48,15 +48,17 @@ ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \ (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \ (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \ - (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2))) + (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \ + (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI))) #include "x86/x86cpu.h" -/* We currently support 5 x86 variants: +/* We currently support 6 x86 variants: * arch[0] -> non-sse * arch[1] -> sse * arch[2] -> sse2 * arch[3] -> sse4.1 * arch[4] -> avx + * arch[5] -> avx512vnni */ #define OPUS_ARCHMASK 7 int opus_select_arch(void); diff --git a/celt/x86/x86_celt_map.c b/celt/x86/x86_celt_map.c index ba8eafe6a..df8c4a73d 100644 --- a/celt/x86/x86_celt_map.c +++ b/celt/x86/x86_celt_map.c @@ -53,7 +53,8 @@ void (*const CELT_FIR_IMPL[OPUS_ARCHMASK + 1])( celt_fir_c, celt_fir_c, MAY_HAVE_SSE4_1(celt_fir), /* sse4.1 */ - MAY_HAVE_SSE4_1(celt_fir) /* avx */ + MAY_HAVE_SSE4_1(celt_fir), /* avx */ + MAY_HAVE_SSE4_1(celt_fir) /* avx512vnni */ }; void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])( @@ -66,7 +67,8 @@ void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])( xcorr_kernel_c, xcorr_kernel_c, MAY_HAVE_SSE4_1(xcorr_kernel), /* sse4.1 */ - MAY_HAVE_SSE4_1(xcorr_kernel) /* avx */ + MAY_HAVE_SSE4_1(xcorr_kernel), /* avx */ + MAY_HAVE_SSE4_1(xcorr_kernel) /* avx512vnni */ }; #endif @@ -83,7 +85,8 @@ opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])( celt_inner_prod_c, MAY_HAVE_SSE2(celt_inner_prod), MAY_HAVE_SSE4_1(celt_inner_prod), /* sse4.1 */ - MAY_HAVE_SSE4_1(celt_inner_prod) /* avx */ + MAY_HAVE_SSE4_1(celt_inner_prod), /* avx */ + MAY_HAVE_SSE4_1(celt_inner_prod) /* avx512vnni */ }; #endif @@ -104,7 +107,8 @@ void (*const PITCH_XCORR_IMPL[OPUS_ARCHMASK + 1])( celt_pitch_xcorr_c, celt_pitch_xcorr_c, celt_pitch_xcorr_c, - MAY_HAVE_AVX2(celt_pitch_xcorr) + MAY_HAVE_AVX2(celt_pitch_xcorr), /* avx */ + MAY_HAVE_AVX2(celt_pitch_xcorr) /* avx512vnni */ }; #endif @@ -122,6 +126,7 @@ void (*const XCORR_KERNEL_IMPL[OPUS_ARCHMASK + 1])( MAY_HAVE_SSE(xcorr_kernel), MAY_HAVE_SSE(xcorr_kernel), MAY_HAVE_SSE(xcorr_kernel), + MAY_HAVE_SSE(xcorr_kernel), MAY_HAVE_SSE(xcorr_kernel) }; @@ -134,6 +139,7 @@ opus_val32 (*const CELT_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])( MAY_HAVE_SSE(celt_inner_prod), MAY_HAVE_SSE(celt_inner_prod), MAY_HAVE_SSE(celt_inner_prod), + MAY_HAVE_SSE(celt_inner_prod), MAY_HAVE_SSE(celt_inner_prod) }; @@ -149,6 +155,7 @@ void (*const DUAL_INNER_PROD_IMPL[OPUS_ARCHMASK + 1])( MAY_HAVE_SSE(dual_inner_prod), MAY_HAVE_SSE(dual_inner_prod), MAY_HAVE_SSE(dual_inner_prod), + MAY_HAVE_SSE(dual_inner_prod), MAY_HAVE_SSE(dual_inner_prod) }; @@ -165,6 +172,7 @@ void (*const COMB_FILTER_CONST_IMPL[OPUS_ARCHMASK + 1])( MAY_HAVE_SSE(comb_filter_const), MAY_HAVE_SSE(comb_filter_const), MAY_HAVE_SSE(comb_filter_const), + MAY_HAVE_SSE(comb_filter_const), MAY_HAVE_SSE(comb_filter_const) }; @@ -179,6 +187,7 @@ opus_val16 (*const OP_PVQ_SEARCH_IMPL[OPUS_ARCHMASK + 1])( op_pvq_search_c, MAY_HAVE_SSE2(op_pvq_search), MAY_HAVE_SSE2(op_pvq_search), + MAY_HAVE_SSE2(op_pvq_search), MAY_HAVE_SSE2(op_pvq_search) }; #endif diff --git a/celt/x86/x86cpu.c b/celt/x86/x86cpu.c index 2e7c32aee..dc6bda498 100644 --- a/celt/x86/x86cpu.c +++ b/celt/x86/x86cpu.c @@ -39,7 +39,8 @@ ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \ (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \ (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \ - (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2))) + (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \ + (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI))) #if defined(_MSC_VER) @@ -99,6 +100,26 @@ static void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) #endif +#if defined(OPUS_X86_MAY_HAVE_AVX512VNNI) +/* Returns the low 32 bits of the extended control register XCR0, which tells + us whether the OS has enabled saving/restoring of the relevant vector state. + This must only be called once OSXSAVE (CPUID.1:ECX[27]) has been confirmed, + otherwise XGETBV is an illegal instruction. */ +static opus_uint32 get_xcr0(void) +{ +#if defined(_MSC_VER) + return (opus_uint32)_xgetbv(0); +#elif defined(CPU_INFO_BY_ASM) || defined(CPU_INFO_BY_C) + opus_uint32 eax, edx; + __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); + (void)edx; + return eax; +#else + return 0; +#endif +} +#endif + typedef struct CPU_Feature{ /* SIMD: 128-bit */ int HW_SSE; @@ -106,6 +127,9 @@ typedef struct CPU_Feature{ int HW_SSE41; /* SIMD: 256-bit */ int HW_AVX2; + /* 256-bit VNNI int8 dot product (EVEX-encoded vpdpbusd), requires + AVX512F + AVX512VL + AVX512_VNNI and OS support for AVX-512 state. */ + int HW_AVX512VNNI; } CPU_Feature; static void opus_cpu_feature_check(CPU_Feature *cpu_feature) @@ -117,14 +141,35 @@ static void opus_cpu_feature_check(CPU_Feature *cpu_feature) nIds = info[0]; if (nIds >= 1){ + unsigned int leaf1_ecx; cpuid(info, 1); + leaf1_ecx = info[2]; cpu_feature->HW_SSE = (info[3] & (1 << 25)) != 0; cpu_feature->HW_SSE2 = (info[3] & (1 << 26)) != 0; cpu_feature->HW_SSE41 = (info[2] & (1 << 19)) != 0; cpu_feature->HW_AVX2 = (info[2] & (1 << 28)) != 0 && (info[2] & (1 << 12)) != 0; + cpu_feature->HW_AVX512VNNI = 0; if (cpu_feature->HW_AVX2 && nIds >= 7) { cpuid(info, 7); cpu_feature->HW_AVX2 = cpu_feature->HW_AVX2 && (info[1] & (1 << 5)) != 0; +#if defined(OPUS_X86_MAY_HAVE_AVX512VNNI) + /* leaf 7, sub-leaf 0: EBX[16]=AVX512F, EBX[31]=AVX512VL, + ECX[11]=AVX512_VNNI. The 256-bit EVEX vpdpbusd we emit for this + tier needs F+VL+VNNI. We also require that the OS has enabled + AVX-512 register state (XCR0 bits 5,6,7, on top of the SSE/AVX + bits 1,2), otherwise the EVEX-encoded instructions fault. */ + if (cpu_feature->HW_AVX2 + && (info[1] & (1u << 16)) != 0 /* AVX512F */ + && (info[1] & (1u << 31)) != 0 /* AVX512VL */ + && (info[2] & (1u << 11)) != 0 /* AVX512_VNNI */ + && (leaf1_ecx & (1u << 27)) != 0 /* OSXSAVE */) { + opus_uint32 xcr0 = get_xcr0(); + unsigned int avx512_state = (1u << 1) | (1u << 2) + | (1u << 5) | (1u << 6) | (1u << 7); + cpu_feature->HW_AVX512VNNI = + (xcr0 & avx512_state) == avx512_state; + } +#endif } else { cpu_feature->HW_AVX2 = 0; } @@ -134,6 +179,7 @@ static void opus_cpu_feature_check(CPU_Feature *cpu_feature) cpu_feature->HW_SSE2 = 0; cpu_feature->HW_SSE41 = 0; cpu_feature->HW_AVX2 = 0; + cpu_feature->HW_AVX512VNNI = 0; } } @@ -169,6 +215,12 @@ static int opus_select_arch_impl(void) } arch++; + if (!cpu_feature.HW_AVX512VNNI) + { + return arch; + } + arch++; + return arch; } diff --git a/celt/x86/x86cpu.h b/celt/x86/x86cpu.h index 1e5b6a4cb..ec7d5a567 100644 --- a/celt/x86/x86cpu.h +++ b/celt/x86/x86cpu.h @@ -52,11 +52,18 @@ # define MAY_HAVE_AVX2(name) name ## _c # endif +# if defined(OPUS_X86_MAY_HAVE_AVX512VNNI) +# define MAY_HAVE_AVX512VNNI(name) name ## _avx512vnni +# else +# define MAY_HAVE_AVX512VNNI(name) MAY_HAVE_AVX2(name) +# endif + # if defined(OPUS_HAVE_RTCD) && \ ((defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \ (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \ (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) || \ - (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2))) + (defined(OPUS_X86_MAY_HAVE_AVX2) && !defined(OPUS_X86_PRESUME_AVX2)) || \ + (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(OPUS_X86_PRESUME_AVX512VNNI))) int opus_select_arch(void); # endif diff --git a/cmake/OpusFunctions.cmake b/cmake/OpusFunctions.cmake index 2db77d7f8..ab40df7a9 100644 --- a/cmake/OpusFunctions.cmake +++ b/cmake/OpusFunctions.cmake @@ -126,7 +126,20 @@ function(opus_detect_sse COMPILER_SUPPORT_SIMD) PARENT_SCOPE) endif() - if(SSE1_SUPPORTED OR SSE2_SUPPORTED OR SSE4_1_SUPPORTED OR AVX2_SUPPORTED) + # AVX512-VNNI provides the EVEX-encoded 256-bit vpdpbusd int8 dot product. + if(HAVE_IMMINTRIN_H) + if(MSVC) + check_flag(AVX512VNNI /arch:AVX512) + else() + check_flag(AVX512VNNI -mavx512vnni) + endif() + else() + set(AVX512VNNI_SUPPORTED + 0 + PARENT_SCOPE) + endif() + + if(SSE1_SUPPORTED OR SSE2_SUPPORTED OR SSE4_1_SUPPORTED OR AVX2_SUPPORTED OR AVX512VNNI_SUPPORTED) set(COMPILER_SUPPORT_SIMD 1 PARENT_SCOPE) else() message(STATUS "No SIMD support in compiler") diff --git a/cmake/OpusSources.cmake b/cmake/OpusSources.cmake index 06cb0d4fa..7a194d08b 100644 --- a/cmake/OpusSources.cmake +++ b/cmake/OpusSources.cmake @@ -50,6 +50,7 @@ get_opus_sources(DNN_SOURCES_X86_RTCD lpcnet_sources.mk dnn_sources_x86_rtcd) get_opus_sources(DNN_SOURCES_SSE2 lpcnet_sources.mk dnn_sources_sse2) get_opus_sources(DNN_SOURCES_SSE4_1 lpcnet_sources.mk dnn_sources_sse4_1) get_opus_sources(DNN_SOURCES_AVX2 lpcnet_sources.mk dnn_sources_avx2) +get_opus_sources(DNN_SOURCES_AVX512VNNI lpcnet_sources.mk dnn_sources_avx512vnni) get_opus_sources(DNN_SOURCES_NEON lpcnet_sources.mk dnn_sources_arm_neon) get_opus_sources(DNN_SOURCES_DOTPROD lpcnet_sources.mk dnn_sources_arm_dotprod) get_opus_sources(DNN_SOURCES_ARM_RTCD lpcnet_sources.mk dnn_sources_arm_rtcd) diff --git a/configure.ac b/configure.ac index 3d7e816b8..8410f25e1 100644 --- a/configure.ac +++ b/configure.ac @@ -414,11 +414,15 @@ AM_CONDITIONAL([HAVE_SSE], [false]) AM_CONDITIONAL([HAVE_SSE2], [false]) AM_CONDITIONAL([HAVE_SSE4_1], [false]) AM_CONDITIONAL([HAVE_AVX2], [false]) +AM_CONDITIONAL([HAVE_AVX512VNNI], [false]) m4_define([DEFAULT_X86_SSE_CFLAGS], [-msse]) m4_define([DEFAULT_X86_SSE2_CFLAGS], [-msse2]) m4_define([DEFAULT_X86_SSE4_1_CFLAGS], [-msse4.1]) m4_define([DEFAULT_X86_AVX2_CFLAGS], [-mavx -mfma -mavx2]) +dnl 256-bit EVEX vpdpbusd: keep vectors 256-bit wide (the win is the int8 +dnl dot-product instruction, not wider vectors). +m4_define([DEFAULT_X86_AVX512VNNI_CFLAGS], [-mavx -mfma -mavx2 -mavx512f -mavx512vl -mavx512vnni -mprefer-vector-width=256]) m4_define([DEFAULT_ARM_NEON_INTR_CFLAGS], [-mfpu=neon]) m4_define([DEFAULT_ARM_DOTPROD_INTR_CFLAGS], ["-march=armv8.2-a+dotprod"]) # With GCC on ARM32 softfp architectures (e.g. Android, or older Ubuntu) you need to specify @@ -437,6 +441,7 @@ AC_ARG_VAR([X86_SSE_CFLAGS], [C compiler flags to compile SSE intrinsics @<:@def AC_ARG_VAR([X86_SSE2_CFLAGS], [C compiler flags to compile SSE2 intrinsics @<:@default=]DEFAULT_X86_SSE2_CFLAGS[@:>@]) AC_ARG_VAR([X86_SSE4_1_CFLAGS], [C compiler flags to compile SSE4.1 intrinsics @<:@default=]DEFAULT_X86_SSE4_1_CFLAGS[@:>@]) AC_ARG_VAR([X86_AVX2_CFLAGS], [C compiler flags to compile AVX2 intrinsics @<:@default=]DEFAULT_X86_AVX2_CFLAGS[@:>@]) +AC_ARG_VAR([X86_AVX512VNNI_CFLAGS], [C compiler flags to compile AVX512-VNNI intrinsics @<:@default=]DEFAULT_X86_AVX512VNNI_CFLAGS[@:>@]) AC_ARG_VAR([ARM_NEON_INTR_CFLAGS], [C compiler flags to compile ARM NEON intrinsics @<:@default=]DEFAULT_ARM_NEON_INTR_CFLAGS / DEFAULT_ARM_NEON_SOFTFP_INTR_CFLAGS[@:>@]) AC_ARG_VAR([ARM_DOTPROD_INTR_CFLAGS], [C compiler flags to compile ARM DOTPROD intrinsics @<:@default=]DEFAULT_ARM_DOTPROD_INTR_CFLAGS[@:>@]) @@ -444,6 +449,7 @@ AS_VAR_SET_IF([X86_SSE_CFLAGS], [], [AS_VAR_SET([X86_SSE_CFLAGS], "DEFAULT_X86_S AS_VAR_SET_IF([X86_SSE2_CFLAGS], [], [AS_VAR_SET([X86_SSE2_CFLAGS], "DEFAULT_X86_SSE2_CFLAGS")]) AS_VAR_SET_IF([X86_SSE4_1_CFLAGS], [], [AS_VAR_SET([X86_SSE4_1_CFLAGS], "DEFAULT_X86_SSE4_1_CFLAGS")]) AS_VAR_SET_IF([X86_AVX2_CFLAGS], [], [AS_VAR_SET([X86_AVX2_CFLAGS], "DEFAULT_X86_AVX2_CFLAGS")]) +AS_VAR_SET_IF([X86_AVX512VNNI_CFLAGS], [], [AS_VAR_SET([X86_AVX512VNNI_CFLAGS], "DEFAULT_X86_AVX512VNNI_CFLAGS")]) AS_VAR_SET_IF([ARM_NEON_INTR_CFLAGS], [], [AS_VAR_SET([ARM_NEON_INTR_CFLAGS], ["$RESOLVED_DEFAULT_ARM_NEON_INTR_CFLAGS"])]) AS_VAR_SET_IF([ARM_DOTPROD_INTR_CFLAGS], [], [AS_VAR_SET([ARM_DOTPROD_INTR_CFLAGS], ["DEFAULT_ARM_DOTPROD_INTR_CFLAGS"])]) @@ -731,6 +737,28 @@ AS_IF([test x"$enable_intrinsics" = x"yes"],[ OPUS_X86_AVX2_CFLAGS="$X86_AVX2_CFLAGS" AC_SUBST([OPUS_X86_AVX2_CFLAGS]) ] + ) + OPUS_CHECK_INTRINSICS( + [AVX512-VNNI], + [$X86_AVX512VNNI_CFLAGS], + [OPUS_X86_MAY_HAVE_AVX512VNNI], + [OPUS_X86_PRESUME_AVX512VNNI], + [[#include + #include + ]], + [[ + __m256i a = _mm256_set1_epi8(1); + __m256i b = _mm256_set1_epi8((char)time(NULL)); + __m256i acc = _mm256_setzero_si256(); + acc = _mm256_dpbusds_epi32(acc, a, b); + return _mm256_extract_epi32(acc, 0); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_AVX512VNNI" = x"1" && test x"$OPUS_X86_PRESUME_AVX512VNNI" != x"1"], + [ + OPUS_X86_AVX512VNNI_CFLAGS="$X86_AVX512VNNI_CFLAGS" + AC_SUBST([OPUS_X86_AVX512VNNI_CFLAGS]) + ] ) AS_IF([test x"$rtcd_support" = x"no"], [rtcd_support=""]) AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE" = x"1"], @@ -783,6 +811,18 @@ AS_IF([test x"$enable_intrinsics" = x"yes"],[ [ AC_MSG_WARN([Compiler does not support AVX2 intrinsics]) ]) + AS_IF([test x"$OPUS_X86_MAY_HAVE_AVX512VNNI" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_AVX512VNNI], 1, [Compiler supports X86 AVX512-VNNI Intrinsics]) + intrinsics_support="$intrinsics_support AVX512-VNNI" + + AS_IF([test x"$OPUS_X86_PRESUME_AVX512VNNI" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_AVX512VNNI], 1, [Define if binary requires AVX512-VNNI intrinsics support])], + [rtcd_support="$rtcd_support AVX512-VNNI"]) + ], + [ + AC_MSG_WARN([Compiler does not support AVX512-VNNI intrinsics]) + ]) AS_IF([test x"$intrinsics_support" = x""], [intrinsics_support=no], @@ -868,6 +908,8 @@ AM_CONDITIONAL([HAVE_SSE4_1], [test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1"]) AM_CONDITIONAL([HAVE_AVX2], [test x"$OPUS_X86_MAY_HAVE_AVX2" = x"1"]) +AM_CONDITIONAL([HAVE_AVX512VNNI], + [test x"$OPUS_X86_MAY_HAVE_AVX512VNNI" = x"1"]) AM_CONDITIONAL([HAVE_RTCD], [test x"$enable_rtcd" = x"yes" && test x"$rtcd_support" != x"no"]) diff --git a/dnn/meson.build b/dnn/meson.build index 7b324016c..803ec0bcd 100644 --- a/dnn/meson.build +++ b/dnn/meson.build @@ -13,6 +13,7 @@ endif dnn_sources_sse2 = sources['DNN_SOURCES_SSE2'] dnn_sources_sse4_1 = sources['DNN_SOURCES_SSE4_1'] dnn_sources_avx2 = sources['DNN_SOURCES_AVX2'] +dnn_sources_avx512vnni = sources['DNN_SOURCES_AVX512VNNI'] dnn_sources_neon_intr = sources['DNN_SOURCES_NEON'] dnn_sources_dotprod_intr = sources['DNN_SOURCES_DOTPROD'] @@ -30,7 +31,7 @@ if host_cpu_family in ['arm', 'aarch64'] and have_arm_intrinsics_or_asm endif endif -foreach intr_name : ['sse2', 'sse4_1', 'avx2', 'neon_intr', 'dotprod_intr'] +foreach intr_name : ['sse2', 'sse4_1', 'avx2', 'avx512vnni', 'neon_intr', 'dotprod_intr'] have_intr = get_variable('have_' + intr_name) if not have_intr continue diff --git a/dnn/x86/dnn_x86.h b/dnn/x86/dnn_x86.h index f2183327b..5acebd758 100644 --- a/dnn/x86/dnn_x86.h +++ b/dnn/x86/dnn_x86.h @@ -49,8 +49,23 @@ void compute_activation_avx2(float *output, const float *input, int N, int activ void compute_conv2d_avx2(const Conv2dLayer *conv, float *out, float *mem, const float *in, int height, int hstride, int activation); #endif +#if defined(OPUS_X86_MAY_HAVE_AVX512VNNI) +/* Only the int8 GEMV (compute_linear with quantized weights) benefits from the + VNNI vpdpbusd instruction; activation/conv2d reuse the AVX2 variants. */ +void compute_linear_avx512vnni(const LinearLayer *linear, float *out, const float *in); +#endif + + +#if defined(OPUS_X86_PRESUME_AVX512VNNI) + +#define OVERRIDE_COMPUTE_LINEAR +#define compute_linear(linear, out, in, arch) ((void)(arch),compute_linear_avx512vnni(linear, out, in)) +#define OVERRIDE_COMPUTE_ACTIVATION +#define compute_activation(output, input, N, activation, arch) ((void)(arch),compute_activation_avx2(output, input, N, activation)) +#define OVERRIDE_COMPUTE_CONV2D +#define compute_conv2d(conv, out, mem, in, height, hstride, activation, arch) ((void)(arch),compute_conv2d_avx2(conv, out, mem, in, height, hstride, activation)) -#if defined(OPUS_X86_PRESUME_AVX2) +#elif defined(OPUS_X86_PRESUME_AVX2) #define OVERRIDE_COMPUTE_LINEAR #define compute_linear(linear, out, in, arch) ((void)(arch),compute_linear_avx2(linear, out, in)) @@ -77,7 +92,7 @@ void compute_conv2d_avx2(const Conv2dLayer *conv, float *out, float *mem, const #define OVERRIDE_COMPUTE_CONV2D #define compute_conv2d(conv, out, mem, in, height, hstride, activation, arch) ((void)(arch),compute_conv2d_sse2(conv, out, mem, in, height, hstride, activation)) -#elif defined(OPUS_HAVE_RTCD) && (defined(OPUS_X86_MAY_HAVE_AVX2) || defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2)) +#elif defined(OPUS_HAVE_RTCD) && (defined(OPUS_X86_MAY_HAVE_AVX512VNNI) || defined(OPUS_X86_MAY_HAVE_AVX2) || defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2)) extern void (*const DNN_COMPUTE_LINEAR_IMPL[OPUS_ARCHMASK + 1])( const LinearLayer *linear, diff --git a/dnn/x86/nnet_avx512vnni.c b/dnn/x86/nnet_avx512vnni.c new file mode 100644 index 000000000..9148824aa --- /dev/null +++ b/dnn/x86/nnet_avx512vnni.c @@ -0,0 +1,51 @@ +/* Copyright (c) 2018-2019 Mozilla + 2023 Amazon */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "x86/x86_arch_macros.h" + +/* This file is compiled with AVX512-VNNI (+ AVX512VL) enabled so that + vec_avx.h emits the 256-bit EVEX-encoded vpdpbusd instruction for the int8 + GEMV (compute_linear with quantized weights) instead of the AVX2 + multiply-add emulation. We deliberately keep everything 256-bit wide + (-mprefer-vector-width=256): the win comes from the single-instruction int8 + dot product, not from wider vectors. MSVC does not expose a feature macro + for this, so define it here when the build has detected support. */ +#if defined(_MSC_VER) && defined(OPUS_X86_MAY_HAVE_AVX512VNNI) && !defined(__AVX512VNNI__) +#define __AVX512VNNI__ +#endif + +#ifndef __AVX512VNNI__ +#error nnet_avx512vnni.c is being compiled without AVX512-VNNI enabled +#endif + +#define RTCD_ARCH avx512vnni + +#include "nnet_arch.h" diff --git a/dnn/x86/x86_dnn_map.c b/dnn/x86/x86_dnn_map.c index d673e1345..a56f11d58 100644 --- a/dnn/x86/x86_dnn_map.c +++ b/dnn/x86/x86_dnn_map.c @@ -34,7 +34,7 @@ #if defined(OPUS_HAVE_RTCD) -#if (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_AVX2)) +#if (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_AVX2) && !defined(OPUS_X86_PRESUME_AVX512VNNI)) void (*const DNN_COMPUTE_LINEAR_IMPL[OPUS_ARCHMASK + 1])( const LinearLayer *linear, @@ -45,7 +45,8 @@ void (*const DNN_COMPUTE_LINEAR_IMPL[OPUS_ARCHMASK + 1])( compute_linear_c, MAY_HAVE_SSE2(compute_linear), MAY_HAVE_SSE4_1(compute_linear), /* sse4.1 */ - MAY_HAVE_AVX2(compute_linear) /* avx */ + MAY_HAVE_AVX2(compute_linear), /* avx */ + MAY_HAVE_AVX512VNNI(compute_linear) /* avx512vnni */ }; void (*const DNN_COMPUTE_ACTIVATION_IMPL[OPUS_ARCHMASK + 1])( @@ -58,7 +59,8 @@ void (*const DNN_COMPUTE_ACTIVATION_IMPL[OPUS_ARCHMASK + 1])( compute_activation_c, MAY_HAVE_SSE2(compute_activation), MAY_HAVE_SSE4_1(compute_activation), /* sse4.1 */ - MAY_HAVE_AVX2(compute_activation) /* avx */ + MAY_HAVE_AVX2(compute_activation), /* avx */ + MAY_HAVE_AVX2(compute_activation) /* avx512vnni (no VNNI variant) */ }; void (*const DNN_COMPUTE_CONV2D_IMPL[OPUS_ARCHMASK + 1])( @@ -74,7 +76,8 @@ void (*const DNN_COMPUTE_CONV2D_IMPL[OPUS_ARCHMASK + 1])( compute_conv2d_c, MAY_HAVE_SSE2(compute_conv2d), MAY_HAVE_SSE4_1(compute_conv2d), /* sse4.1 */ - MAY_HAVE_AVX2(compute_conv2d) /* avx */ + MAY_HAVE_AVX2(compute_conv2d), /* avx */ + MAY_HAVE_AVX2(compute_conv2d) /* avx512vnni (no VNNI variant) */ }; #endif diff --git a/lpcnet_sources.mk b/lpcnet_sources.mk index be0a30840..14698ae72 100644 --- a/lpcnet_sources.mk +++ b/lpcnet_sources.mk @@ -36,6 +36,7 @@ dnn/lossgen.c \ dnn/lossgen_data.c DNN_SOURCES_X86_RTCD = dnn/x86/x86_dnn_map.c +DNN_SOURCES_AVX512VNNI = dnn/x86/nnet_avx512vnni.c DNN_SOURCES_AVX2 = dnn/x86/nnet_avx2.c DNN_SOURCES_SSE4_1 = dnn/x86/nnet_sse4_1.c DNN_SOURCES_SSE2 = dnn/x86/nnet_sse2.c diff --git a/meson.build b/meson.build index aadf7caf3..fa1e51130 100644 --- a/meson.build +++ b/meson.build @@ -383,6 +383,7 @@ have_sse = false have_sse2 = false have_sse4_1 = false have_avx2 = false +have_avx512vnni = false have_neon_intr = false have_dotprod_intr = false @@ -507,6 +508,9 @@ if not opt_intrinsics.disabled() [ 'SSE2', 'emmintrin.h', '__m128i', '_mm_setzero_si128()', ['-msse2'], [] ], [ 'SSE4.1', 'smmintrin.h', '__m128i', '_mm_setzero_si128(); mtest = _mm_cmpeq_epi64(mtest, mtest)', ['-msse4.1'], [] ], [ 'AVX2', 'immintrin.h', '__m256i', '_mm256_abs_epi32(_mm256_setzero_si256())', ['-mavx', '-mfma', '-mavx2'], ['/arch:AVX2'] ], + # 256-bit EVEX vpdpbusd. Kept 256-bit wide on purpose (-mprefer-vector-width=256): + # the speedup is the single-instruction int8 dot product, not wider vectors. + [ 'AVX512VNNI', 'immintrin.h', '__m256i', '_mm256_dpbusds_epi32(_mm256_setzero_si256(), _mm256_setzero_si256(), _mm256_setzero_si256())', ['-mavx', '-mfma', '-mavx2', '-mavx512f', '-mavx512vl', '-mavx512vnni', '-mprefer-vector-width=256'], ['/arch:AVX512'] ], ] foreach intrin : x86_intrinsics diff --git a/silk/x86/x86_silk_map.c b/silk/x86/x86_silk_map.c index 39ad75276..bfb6a9b4c 100644 --- a/silk/x86/x86_silk_map.c +++ b/silk/x86/x86_silk_map.c @@ -53,7 +53,8 @@ opus_int64 (*const SILK_INNER_PROD16_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_inner_prod16_c, silk_inner_prod16_c, MAY_HAVE_SSE4_1( silk_inner_prod16 ), /* sse4.1 */ - MAY_HAVE_SSE4_1( silk_inner_prod16 ) /* avx */ + MAY_HAVE_SSE4_1( silk_inner_prod16 ), /* avx */ + MAY_HAVE_SSE4_1( silk_inner_prod16 ) /* avx512vnni */ }; #endif @@ -66,7 +67,8 @@ opus_int (*const SILK_VAD_GETSA_Q8_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_VAD_GetSA_Q8_c, silk_VAD_GetSA_Q8_c, MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ), /* sse4.1 */ - MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ) /* avx */ + MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ), /* avx */ + MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ) /* avx512vnni */ }; void (*const SILK_NSQ_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -90,7 +92,8 @@ void (*const SILK_NSQ_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_NSQ_c, silk_NSQ_c, MAY_HAVE_SSE4_1( silk_NSQ ), /* sse4.1 */ - MAY_HAVE_SSE4_1( silk_NSQ ) /* avx */ + MAY_HAVE_SSE4_1( silk_NSQ ), /* avx */ + MAY_HAVE_SSE4_1( silk_NSQ ) /* avx512vnni */ }; void (*const SILK_VQ_WMAT_EC_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -111,7 +114,8 @@ void (*const SILK_VQ_WMAT_EC_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_VQ_WMat_EC_c, silk_VQ_WMat_EC_c, MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ), /* sse4.1 */ - MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ) /* avx */ + MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ), /* avx */ + MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ) /* avx512vnni */ }; void (*const SILK_NSQ_DEL_DEC_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -135,7 +139,8 @@ void (*const SILK_NSQ_DEL_DEC_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_NSQ_del_dec_c, silk_NSQ_del_dec_c, MAY_HAVE_SSE4_1( silk_NSQ_del_dec ), /* sse4.1 */ - MAY_HAVE_AVX2( silk_NSQ_del_dec ) /* avx */ + MAY_HAVE_AVX2( silk_NSQ_del_dec ), /* avx */ + MAY_HAVE_AVX2( silk_NSQ_del_dec ) /* avx512vnni */ }; #if defined(FIXED_POINT) @@ -155,7 +160,8 @@ void (*const SILK_BURG_MODIFIED_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_burg_modified_c, silk_burg_modified_c, MAY_HAVE_SSE4_1( silk_burg_modified ), /* sse4.1 */ - MAY_HAVE_SSE4_1( silk_burg_modified ) /* avx */ + MAY_HAVE_SSE4_1( silk_burg_modified ), /* avx */ + MAY_HAVE_SSE4_1( silk_burg_modified ) /* avx512vnni */ }; #endif @@ -171,7 +177,8 @@ double (*const SILK_INNER_PRODUCT_FLP_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_inner_product_FLP_c, silk_inner_product_FLP_c, silk_inner_product_FLP_c, /* sse4.1 */ - MAY_HAVE_AVX2( silk_inner_product_FLP ) /* avx */ + MAY_HAVE_AVX2( silk_inner_product_FLP ), /* avx */ + MAY_HAVE_AVX2( silk_inner_product_FLP ) /* avx512vnni */ }; #endif From d393c1bc350c227bfd90694f95f867c0715a068b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 16 Jun 2026 19:43:01 +0000 Subject: [PATCH 2/2] x86/dnn: use 4 accumulators in the int8 cgemv inner loop The unroll-by-4 in cgemv8x4 / sparse_cgemv8x4 fed all four vpdpbusds into a single accumulator, serializing them on that register. On the VNNI tier the EVEX vpdpbusds has ~5-cycle latency, so this single- accumulator recurrence made the VNNI kernel latency-bound and actually slower than the AVX2 vpmaddubsw+vpmaddwd+vpaddd emulation (which keeps its multiplies off the accumulator's critical path). Split the unrolled body across four independent accumulators and sum them once per 8-row block, keeping several int8 dot products in flight. Bit-exact: the AVX2/SSE emulation accumulates with wrapping 32-bit adds (exactly associative), and on VNNI the regrouping is exact whenever the per-output sum stays within int32 - which holds for the quantized weights used by the models. DRED encode+decode output is byte-identical before and after, and the check-asm + assertions test suite passes. Measured on an AVX512-VNNI CPU (compute_linear int8 GEMV, best of 5): the VNNI tier goes from ~0.8x (a regression) to ~1.0-1.09x vs AVX2, e.g. 512x1536 24.8->17.0 us and 1024x1024 41.0->22.9 us; the AVX2 tier also speeds up a few percent. End-to-end the DNN is a fraction of the frame, so the wall-clock change is within noise on typical clips. https://claude.ai/code/session_01H3ZadW9kpYqiMEGxhtpV2j --- dnn/vec_avx.h | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/dnn/vec_avx.h b/dnn/vec_avx.h index ff667782d..1e75bce48 100644 --- a/dnn/vec_avx.h +++ b/dnn/vec_avx.h @@ -792,6 +792,14 @@ static inline void sparse_cgemv8x4(float *_out, const opus_int8 *w, const int *i vy0 = _mm256_setzero_si256(); j=0; #if 1 /* Unrolling by 4 gives some gain, comment out if it does not. */ + /* Four independent accumulators keep several (high-latency) vpdpbusds + in flight on the VNNI path. Bit-exact with the single-accumulator + form for the AVX2 emulation (wrapping adds) and for VNNI whenever the + per-output sum stays within int32, as it does for real weights. */ + __m256i vy1, vy2, vy3; + vy1 = _mm256_setzero_si256(); + vy2 = _mm256_setzero_si256(); + vy3 = _mm256_setzero_si256(); for (;j