diff --git a/.github/workflows/linux_win_cross_build.yml b/.github/workflows/linux_win_cross_build.yml new file mode 100644 index 0000000000000..c38d91f7d7a33 --- /dev/null +++ b/.github/workflows/linux_win_cross_build.yml @@ -0,0 +1,192 @@ +name: ONNX Runtime Windows-x64 Cross-Build (Linux) + +# Cross-compiles the full onnxruntime.dll for Windows x64 from a Linux runner, +# using clang-cl + lld-link for C/C++, llvm-ml for the MLAS amd64 MASM assembly +# kernels, llvm-rc for the version resource, and a Windows SDK/CRT staged by +# `xwin`. +# +# Why: downstream consumers such as the Edge browser build their Windows releases +# on Linux machines (faster/cheaper than native Windows builders). Historically +# ONNX Runtime could not cross-build for Windows because the MLAS x64 assembly +# assembled only under ml64.exe (a Windows-only PE tool). The amd64/*.asm sources +# are now dual-assembler compatible (ml64 + llvm-ml), and the remaining C/C++ +# portability gaps for clang-cl have been addressed, so the whole DLL links on +# Linux. This job is the authoritative guard that the property keeps holding. +# +# The complementary native path (cl.exe + ml64.exe, warnings-as-errors on) is +# exercised by the regular Windows build/test pipelines, so both toolchains +# remain covered. +# +# clang-cl surfaces many warnings MSVC never emits; treating them as errors would +# make the cross-build unbuildable for no portability benefit, so this job sets +# -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF (the native pipelines keep -WX on). +# +# The MSVC runtime library is pinned to the dynamic CRT (MultiThreadedDLL) for +# every target, including the FetchContent'd protobuf, so the shared DLL links +# without a /failifmismatch RuntimeLibrary conflict (protobuf otherwise defaults +# to the static CRT under MSVC-like toolchains). +# +# Tests are intentionally NOT run here: the artifacts are Windows-x64 binaries, +# which cannot execute natively on a Linux runner (Wine is unreliable for the +# AVX-512/AMX kernels MLAS ships). Runtime correctness is covered by the existing +# native Windows test pipelines. + +on: + push: + branches: [main, 'rel-*'] + paths: + - 'onnxruntime/core/**' + - 'include/onnxruntime/**' + - 'cmake/**' + - '.github/workflows/linux_win_cross_build.yml' + pull_request: + branches: [main] + paths: + - 'onnxruntime/core/**' + - 'include/onnxruntime/**' + - 'cmake/**' + - '.github/workflows/linux_win_cross_build.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # The MSVC CRT headers staged by xwin enforce a minimum Clang version via a + # static_assert (STL1000); keep this at or above what that CRT requires. + LLVM_VERSION: '19' + XWIN_VERSION: '0.9.0' + # SHA256 of xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz. GitHub release + # assets are mutable, so pin the digest and verify it rather than trusting the + # release (or its co-located .sha256 sidecar, which an attacker could swap too). + XWIN_SHA256: '31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd' + # SHA256 of apt.llvm.org/llvm.sh, which is fetched and run as root. The script + # is maintained upstream (opencollab/llvm-jenkins.debian.net) and changes when + # new LLVM releases are added, so this digest must be re-verified and updated + # when it legitimately changes; a mismatch fails the job rather than silently + # executing tampered/rotated content as root. + LLVM_SH_SHA256: '9474ecd78b52aba6e923976b1e9773f5613027cc7e237b9956986cb536e02a36' + +jobs: + win-cross-build: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: false + + - name: Install build tools (cmake, ninja, LLVM/clang cross toolchain) + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends cmake ninja-build curl ca-certificates + + # LLVM (clang-cl, lld-link, llvm-ml, llvm-lib, llvm-mt, llvm-rc) from apt.llvm.org. + curl -fsSL https://apt.llvm.org/llvm.sh -o /tmp/llvm.sh + echo "${LLVM_SH_SHA256} /tmp/llvm.sh" | sha256sum -c - + chmod +x /tmp/llvm.sh + sudo /tmp/llvm.sh "${LLVM_VERSION}" + sudo apt-get install -y --no-install-recommends "llvm-${LLVM_VERSION}" + + # Expose the version-suffixed tools under stable, unsuffixed names. + BIN="${HOME}/llvmbin" + mkdir -p "${BIN}" + link() { # link + local dest="$1"; shift + local src + for src in "$@"; do + if command -v "${src}" >/dev/null 2>&1; then + ln -sf "$(command -v "${src}")" "${BIN}/${dest}" + return 0 + fi + done + echo "::error::required LLVM tool not found (tried: $*)" + return 1 + } + link clang-cl "clang-cl-${LLVM_VERSION}" + link lld-link "lld-link-${LLVM_VERSION}" + link llvm-ml "llvm-ml-${LLVM_VERSION}" + link llvm-mt "llvm-mt-${LLVM_VERSION}" + link llvm-rc "llvm-rc-${LLVM_VERSION}" + # llvm-lib is llvm-ar invoked under the lib.exe-compatible name. + link llvm-lib "llvm-lib-${LLVM_VERSION}" "llvm-ar-${LLVM_VERSION}" + # llvm-ar (Unix archive tool) is used to inspect the built .lib in the + # verify step; it lists members that the lib.exe-mode driver does not. + link llvm-ar "llvm-ar-${LLVM_VERSION}" + + echo "ORT_LLVM_BIN=${BIN}" >> "${GITHUB_ENV}" + echo "${BIN}" >> "${GITHUB_PATH}" + echo "== resolved LLVM tools ==" + ls -l "${BIN}" + + - name: Stage Windows SDK / CRT with xwin + shell: bash + run: | + set -euo pipefail + url="https://github.com/Jake-Shadle/xwin/releases/download/${XWIN_VERSION}/xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz" + curl -fsSL "${url}" -o /tmp/xwin.tar.gz + echo "${XWIN_SHA256} /tmp/xwin.tar.gz" | sha256sum -c - + mkdir -p /tmp/xwin-dist + tar -xzf /tmp/xwin.tar.gz -C /tmp/xwin-dist --strip-components=1 + XWIN_BIN="$(find /tmp/xwin-dist -name xwin -type f | head -1)" + chmod +x "${XWIN_BIN}" + + XWIN_ROOT="${HOME}/xwinroot" + "${XWIN_BIN}" --accept-license --arch x86_64 splat --output "${XWIN_ROOT}" + echo "ORT_WIN_XWIN_ROOT=${XWIN_ROOT}" >> "${GITHUB_ENV}" + echo "== xwin layout ==" + ls "${XWIN_ROOT}" && ls "${XWIN_ROOT}/crt" && ls "${XWIN_ROOT}/sdk" + + - name: Configure (Windows x64 cross toolchain) + shell: bash + run: | + set -euo pipefail + cmake -S cmake -B build_win_cross -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="${GITHUB_WORKSPACE}/cmake/win_x64_crosscompile_toolchain.cmake" \ + -DORT_WIN_XWIN_ROOT="${ORT_WIN_XWIN_ROOT}" \ + -DORT_LLVM_BIN="${ORT_LLVM_BIN}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL \ + -Dprotobuf_MSVC_STATIC_RUNTIME=OFF \ + -Donnxruntime_BUILD_UNIT_TESTS=OFF \ + -Donnxruntime_BUILD_SHARED_LIB=ON \ + -Donnxruntime_ENABLE_PYTHON=OFF \ + -Donnxruntime_USE_FULL_PROTOBUF=OFF + + - name: Build onnxruntime.dll + shell: bash + run: cmake --build build_win_cross --target onnxruntime -j"$(nproc)" + + - name: Verify cross-built artifacts + shell: bash + run: | + set -euo pipefail + dll="$(find build_win_cross -name 'onnxruntime.dll' | head -1)" + if [ -z "${dll}" ]; then + echo "::error::onnxruntime.dll was not produced" + exit 1 + fi + echo "Built: ${dll} ($(stat -c%s "${dll}") bytes)" + # A PE32+ (x86-64) DLL starts with the 'MZ' DOS stub. + if [ "$(head -c2 "${dll}")" != "MZ" ]; then + echo "::error::onnxruntime.dll is not a PE image" + exit 1 + fi + + # The DLL must embed the MLAS amd64 MASM kernels assembled by llvm-ml. + mlas_lib="$(find build_win_cross -name 'onnxruntime_mlas.lib' | head -1)" + asm_objs="$( { llvm-ar t "${mlas_lib}" 2>/dev/null || true; } | grep -c '\.asm\.obj$' || true)" + echo "amd64 .asm objects in onnxruntime_mlas.lib: ${asm_objs}" + if [ "${asm_objs}" -lt 1 ]; then + echo "::error::no amd64 .asm objects found in onnxruntime_mlas.lib" + exit 1 + fi diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 28d6aa83c5343..e10bffd24e09c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1115,7 +1115,12 @@ function(onnxruntime_set_compile_flags target_name) target_compile_definitions(${target_name} PRIVATE USE_KLEIDIAI) endif() - set_target_properties(${target_name} PROPERTIES COMPILE_WARNING_AS_ERROR ON) + # Honor an explicit -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF (default remains ON). + # Useful for clang-cl cross-compilation, which surfaces warnings MSVC never + # emits; consumers that treat those as errors would otherwise be unbuildable. + if (NOT DEFINED CMAKE_COMPILE_WARNING_AS_ERROR OR CMAKE_COMPILE_WARNING_AS_ERROR) + set_target_properties(${target_name} PROPERTIES COMPILE_WARNING_AS_ERROR ON) + endif() if (onnxruntime_USE_CUDA) # Suppress a "conversion_function_not_usable" warning in gsl/span target_compile_options(${target_name} PRIVATE "$<$:SHELL:-Xcudafe \"--diag_suppress=conversion_function_not_usable\">") diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index b081e22e8b3f4..25c59cd2c686d 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -138,6 +138,16 @@ if(NOT WIN32 AND NOT APPLE AND NOT ANDROID AND CMAKE_SYSTEM_PROCESSOR MATCHES "x ) endif() +if(WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|AMD64)") + # clang-cl (used when cross-compiling ORT for Windows) enforces the waitpkg + # target feature for the _tpause intrinsic, which SpinPause uses behind a + # runtime HasTPAUSE() guard. MSVC allows the intrinsic unconditionally. + set_source_files_properties( + ${ONNXRUNTIME_ROOT}/core/common/spin_pause.cc + PROPERTIES COMPILE_FLAGS "-mwaitpkg" + ) +endif() + if (onnxruntime_USE_TELEMETRY) set_target_properties(onnxruntime_common PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h") endif() diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index d55a4d49fb455..2aa8858669337 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -303,6 +303,39 @@ function(setup_mlas_source_for_windows) ${MLAS_SRC_DIR}/q4gemm_avx512.cpp ) endif() + + # clang-cl (used when cross-compiling ONNX Runtime for Windows on a non-Windows + # host) enforces per-function target features: an intrinsic may only be used in + # a translation unit that was compiled with that feature enabled. MSVC has no + # such requirement, so several MLAS x64 kernels use AVX2/AVX-512/AMX/SSE4.1 + # intrinsics without a TU-level arch flag. Supply the required features here, + # gated on Clang so the MSVC (cl.exe/ml64.exe) build stays byte-for-byte the + # same. Files already carrying an /arch flag above are not repeated. + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set_source_files_properties( + ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX2") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX2 -mavxvnni") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx512vnni.cpp + ${MLAS_SRC_DIR}/q4gemm_avx512.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mavx512vnni") + set_source_files_properties( + ${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp + PROPERTIES COMPILE_FLAGS "-msse4.1") + set_source_files_properties( + ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mamx-tile -mamx-int8 -mamx-bf16") + endif() else() target_sources(onnxruntime_mlas PRIVATE ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp diff --git a/cmake/patches/onnx/onnx.patch b/cmake/patches/onnx/onnx.patch index acd5505beaff6..aac34acd1c128 100644 --- a/cmake/patches/onnx/onnx.patch +++ b/cmake/patches/onnx/onnx.patch @@ -46,3 +46,50 @@ index 3887169..8432613 100644 .SetDoc(GroupNormalization_ver18_doc) .Attr("epsilon", "The epsilon value to use to avoid division by zero.", AttributeProto::FLOAT, 1e-5f) .Attr( +diff --git a/onnx/common/scoped_resource.h b/onnx/common/scoped_resource.h +index 525e4d6..2c41f0b 100644 +--- a/onnx/common/scoped_resource.h ++++ b/onnx/common/scoped_resource.h +@@ -56,7 +56,41 @@ class ScopedResource { + inline void close_handle(HANDLE h) { + CloseHandle(h); + } +-using ScopedHandle = ScopedResource; ++// INVALID_HANDLE_VALUE is ((HANDLE)(LONG_PTR)-1); the reinterpret_cast it ++// performs is not a valid constant expression for a non-type template parameter ++// under Clang (e.g. clang-cl cross-compilation), so it cannot be used as a ++// ScopedResource template argument. Hold the sentinel as a runtime value. ++class ScopedHandle { ++ HANDLE val_; ++ ++ public: ++ explicit ScopedHandle(HANDLE v) : val_(v) {} ++ ~ScopedHandle() { ++ if (val_ != INVALID_HANDLE_VALUE) { ++ close_handle(val_); ++ } ++ } ++ HANDLE get() const { ++ return val_; ++ } ++ HANDLE release() { ++ HANDLE tmp = val_; ++ val_ = INVALID_HANDLE_VALUE; ++ return tmp; ++ } ++ ScopedHandle(ScopedHandle&& other) : val_(other.release()) {} ++ ScopedHandle& operator=(ScopedHandle&& other) { ++ if (this != &other) { ++ if (val_ != INVALID_HANDLE_VALUE) { ++ close_handle(val_); ++ } ++ val_ = other.release(); ++ } ++ return *this; ++ } ++ ScopedHandle(const ScopedHandle&) = delete; ++ ScopedHandle& operator=(const ScopedHandle&) = delete; ++}; + #endif + + inline void close_fd(int fd) { diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake new file mode 100644 index 0000000000000..6bc9e193e1879 --- /dev/null +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -0,0 +1,115 @@ +# Cross-compile ONNX Runtime for Windows x64 from a non-Windows host (e.g. Linux) +# using the LLVM/clang toolchain. This is what lets downstream consumers such as +# the Edge browser build Windows-x64 ONNX Runtime on faster/cheaper Linux +# builders. Today it is exercised to build the onnxruntime_mlas static library +# (see .github/workflows/linux_mlas_win_cross.yml), which contains the MLAS x64 +# MASM assembly kernels. +# +# Toolchain pieces: +# * clang-cl / lld-link - C/C++ compile + link, targeting x86_64-pc-windows-msvc +# * llvm-ml - the LLVM MASM assembler for onnxruntime/core/mlas/lib/amd64/*.asm +# * Windows SDK + MSVC CRT headers/import libs staged by `xwin` +# (https://github.com/Jake-Shadle/xwin): run `xwin splat --output ` and +# pass that directory below. +# +# Required (cache var or environment variable): +# ORT_WIN_XWIN_ROOT - the `xwin splat` output directory (contains crt/ and sdk/) +# Optional: +# ORT_LLVM_BIN - directory holding clang-cl/lld-link/llvm-ml/... (else PATH) + +if(NOT ORT_WIN_XWIN_ROOT) + set(ORT_WIN_XWIN_ROOT "$ENV{ORT_WIN_XWIN_ROOT}") +endif() +if(NOT ORT_WIN_XWIN_ROOT) + message(FATAL_ERROR + "win x64 cross toolchain: set ORT_WIN_XWIN_ROOT to the `xwin splat` output directory") +endif() +if(NOT ORT_LLVM_BIN) + set(ORT_LLVM_BIN "$ENV{ORT_LLVM_BIN}") +endif() + +# CMake re-parses this toolchain file inside its try_compile subprojects, where +# the outer -D cache entries are not visible. Forward the settings this file needs. +list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES ORT_WIN_XWIN_ROOT ORT_LLVM_BIN) + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR AMD64) + +macro(_ort_find_llvm_tool _var _name _required) + if(ORT_LLVM_BIN) + find_program(${_var} NAMES ${_name} PATHS "${ORT_LLVM_BIN}" NO_DEFAULT_PATH) + endif() + if(NOT ${_var}) + find_program(${_var} NAMES ${_name}) + endif() + if(${_required} AND NOT ${_var}) + message(FATAL_ERROR "win x64 cross toolchain: could not find '${_name}'") + endif() +endmacro() + +_ort_find_llvm_tool(_ort_clang_cl clang-cl TRUE) +_ort_find_llvm_tool(_ort_lld_link lld-link TRUE) +_ort_find_llvm_tool(_ort_llvm_ml llvm-ml TRUE) +_ort_find_llvm_tool(_ort_llvm_lib llvm-lib TRUE) +# mt (manifest) and rc (resource compiler) are only needed for DLL/exe links, not +# for the onnxruntime_mlas static library, so they are optional. +_ort_find_llvm_tool(_ort_llvm_mt llvm-mt FALSE) +_ort_find_llvm_tool(_ort_llvm_rc llvm-rc FALSE) + +set(CMAKE_C_COMPILER "${_ort_clang_cl}") +set(CMAKE_CXX_COMPILER "${_ort_clang_cl}") +set(CMAKE_ASM_MASM_COMPILER "${_ort_llvm_ml}") +set(CMAKE_LINKER "${_ort_lld_link}") +set(CMAKE_AR "${_ort_llvm_lib}") +if(_ort_llvm_mt) + set(CMAKE_MT "${_ort_llvm_mt}") +endif() +if(_ort_llvm_rc) + set(CMAKE_RC_COMPILER "${_ort_llvm_rc}") +endif() + +set(CMAKE_C_COMPILER_TARGET x86_64-pc-windows-msvc) +set(CMAKE_CXX_COMPILER_TARGET x86_64-pc-windows-msvc) + +set(_ort_crt "${ORT_WIN_XWIN_ROOT}/crt") +set(_ort_sdk "${ORT_WIN_XWIN_ROOT}/sdk") + +# Windows SDK / CRT header search. Passed via /imsvc (system-include semantics) so +# ORT's -WX and warning flags do not apply to SDK headers. C/C++ only; the .asm +# files must not receive these. -Wno-unused-command-line-argument silences the +# MSVC-only flags ORT passes that clang-cl accepts-but-ignores (e.g. +# /experimental:external), which -WX would otherwise turn into errors. +string(JOIN " " _ort_win_include_flags + "/imsvc${_ort_crt}/include" + "/imsvc${_ort_sdk}/include/ucrt" + "/imsvc${_ort_sdk}/include/um" + "/imsvc${_ort_sdk}/include/shared" + "/imsvc${_ort_sdk}/include/winrt") +add_compile_options("$<$:SHELL:${_ort_win_include_flags}>") +add_compile_options("$<$:-Wno-unused-command-line-argument>") + +add_link_options( + "/libpath:${_ort_crt}/lib/x86_64" + "/libpath:${_ort_sdk}/lib/ucrt/x86_64" + "/libpath:${_ort_sdk}/lib/um/x86_64") + +# llvm-rc (resource compiler, used for the DLL's version resource) does not honor +# /imsvc, so give it the Windows SDK headers on its own include path. +set(CMAKE_RC_FLAGS_INIT + "/I \"${_ort_crt}/include\" /I \"${_ort_sdk}/include/um\" /I \"${_ort_sdk}/include/shared\" /I \"${_ort_sdk}/include/ucrt\"") + +# MLAS amd64 .asm via llvm-ml: unlike ml64.exe, llvm-ml is not implicitly 64-bit, +# does not search the source file's own directory for includes, and needs the +# LLVM_ML define that selects the dual-assembler-compatible spellings. ml64.exe +# needs none of these. CMAKE_CURRENT_LIST_DIR is this file's directory (cmake/). +set(CMAKE_ASM_MASM_FLAGS_INIT + "-m64 /DLLVM_ML=1 -I${CMAKE_CURRENT_LIST_DIR}/../onnxruntime/core/mlas/lib/amd64") + +# onnxruntime_mlas is a static library, so skip CMake's link-based compiler check +# (manifest/rc/link), which is irrelevant when cross-building a static archive. +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) diff --git a/include/onnxruntime/core/common/logging/capture.h b/include/onnxruntime/core/common/logging/capture.h index 13d3a3ad17aff..b9aaacf23cbca 100644 --- a/include/onnxruntime/core/common/logging/capture.h +++ b/include/onnxruntime/core/common/logging/capture.h @@ -44,7 +44,13 @@ class Capture { #ifdef _MSC_VER // add SAL annotation for printf format string. requires Code Analysis to run to validate usage. #define msvc_printf_check _Printf_format_string_ -#define __attribute__(x) // Disable for MSVC. Supported by GCC and CLang. +#if !defined(__clang__) +// Disable __attribute__ only for real MSVC (cl.exe). clang-cl defines _MSC_VER but +// supports __attribute__, and its own intrinsic headers (e.g. mmintrin.h reached via +// ) rely on __attribute__((__vector_size__)); poisoning it here would strip +// those vector typedefs and break the headers. +#define __attribute__(x) // Supported by GCC and Clang (including clang-cl). +#endif #else #define msvc_printf_check #endif diff --git a/onnxruntime/core/dll/onnxruntime.rc b/onnxruntime/core/dll/onnxruntime.rc index 2e7eb7842f935..710c74171c74e 100644 --- a/onnxruntime/core/dll/onnxruntime.rc +++ b/onnxruntime/core/dll/onnxruntime.rc @@ -4,7 +4,7 @@ // This file REQUIRES the following external definitions: // FILE_NAME, VER_MAJOR, VER_MINOR, VER_BUILD, VER_PRIVATE, and VER_STRING -#include +#include #if defined(DEBUG) || defined(_DEBUG) #define VER_DEBUG VS_FF_DEBUG @@ -32,9 +32,9 @@ BEGIN VALUE "FileDescription", "ONNX Runtime" VALUE "FileVersion", VER_STRING VALUE "InternalName", "ONNX Runtime" - VALUE "LegalCopyright", "\251 Microsoft Corporation. All rights reserved." + VALUE "LegalCopyright", L"\x00A9 Microsoft Corporation. All rights reserved." VALUE "OriginalFilename", FILE_NAME - VALUE "ProductName", "Microsoft\256 Windows\256 Operating System" + VALUE "ProductName", L"Microsoft\x00AE Windows\x00AE Operating System" VALUE "ProductVersion", VER_STRING END END diff --git a/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc b/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc index ed885dcb7b781..cea7982a058a6 100644 --- a/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc +++ b/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc @@ -87,7 +87,6 @@ GprIndex_r15 EQU 15 ; VnniZmmZmmZmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, Payload2, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -139,7 +138,7 @@ VpdpwssdsZmmZmmZmm MACRO DestReg, Src1Reg, Src2Reg ; ; This macro builds a VNNI instruction of the form: ; -; instr zmm1,zmm2,DWORD BCST [BaseReg+IndexReg*Scale+ByteOffset] +; instr zmm1,zmm2,DWORD PTR [BaseReg+IndexReg*Scale+ByteOffset]{1to16} ; ; Arguments: ; @@ -160,7 +159,6 @@ VpdpwssdsZmmZmmZmm MACRO DestReg, Src1Reg, Src2Reg ; VnniZmmZmmBroadcast MACRO Opcode, DestReg, Src1Reg, BaseReg, ByteOffset, IndexReg, Scale - LOCAL Payload0, Payload1, Payload2, ModRMByte, SibByte .errnz (ByteOffset AND 3) diff --git a/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc b/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc index d5867e8884c44..9ae9b9b8b55b0 100644 --- a/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc +++ b/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc @@ -72,7 +72,6 @@ XmmIndex_xmm15 EQU 15 ; VnniYmmYmmYmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -134,7 +133,6 @@ VpdpwssdsYmmYmmYmm MACRO DestReg, Src1Reg, Src2Reg ; VnniXmmXmmXmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -198,7 +196,6 @@ VpdpwssdsXmmXmmXmm MACRO DestReg, Src1Reg, Src2Reg ; Avx2VnniYmmYmmYmm MACRO Opcode, Prefix, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -274,7 +271,6 @@ VpdpbuudsYmmYmmYmm MACRO DestReg, Src1Reg, Src2Reg ; Avx2VnniXmmXmmXmm MACRO Opcode, Prefix, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm index 9c334bea2f468..b1ee8df3bec59 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE ConvSymKernelCommon.inc INCLUDE AssembleAvxVnni.inc - .list extern CheckSaturationForVPMADDUBSW:proc @@ -236,11 +234,7 @@ ComputeBlock MACRO Isa, RowCount, VectorOffset, BroadcastOffset ; ComputeBlockLoop MACRO Isa, RowCount, UnrollLoop - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit IFNB sub rax,4*4 @@ -357,6 +351,8 @@ ConvertFloatToIntegerRegList MACRO RegList ; ConvSymKernelFunction MACRO Isa + LOCAL Pnib, InDir, InIndir, CblStart, CblDone, Bsv, Claf, Chaf, Cfi, Sqo16, Sqo6By16, Sqo5By16, Sqo4By16, Sqo3By16, Sqo2By16, Sqo1By16, ExitK, Sqo8, \ + Sqo6By8, Sqo5By8, Sqo4By8, Sqo3By8, Sqo2By8, Sqo1By8, Cbl4, Cbl2 ;++ ; @@ -468,16 +464,16 @@ ENDIF ; Process an input block of length InputChannels for each element of the kernel. ; -ProcessNextInputBlock: +Pnib: test bpl,MLAS_CONV_SYM_FLAG_INPUT_DIRECT - jz InputIndirection + jz InIndir ; ; The input buffer points directly at the input data and this is effectively a ; GEMM operation (such as a pointwise convolution or an Im2Col transform). ; -InputDirect: +InDir: xor r10,r10 mov r11,rsi lea r12,[r11+r11] @@ -497,9 +493,9 @@ IFIDNI , cmovb r15,r10 ENDIF mov r10,rcx - jmp ComputeBlockLoopStart + jmp CblStart -InputIndirection: +InIndir: lea r11,[rcx+rdi] lea r12,[rcx+rdi*2] lea r13,[r11+rdi*2] @@ -534,21 +530,21 @@ IFIDNI , sub r15,r10 ENDIF -ComputeBlockLoopStart: +CblStart: mov rax,rsi ; reload input channels cmp ebx,2 ; output count <= 2? - jbe ComputeBlockLoopBy2 + jbe Cbl2 IFIDNI , cmp ebx,4 ; output count <= 4? - jbe ComputeBlockLoopBy4 + jbe Cbl4 ComputeBlockLoop Isa,6,UnrollLoop ELSE ComputeBlockLoop Isa,4,UnrollLoop ENDIF -ComputeBlockLoopDone: +CblDone: dec r9 ; decrement input blocks remaining - jnz ProcessNextInputBlock + jnz Pnib ; ; Apply the bias and convert the block accumulators to intermediate float values. @@ -563,27 +559,27 @@ ComputeBlockLoopDone: add r10,r8 vmovdqu ymm0,YMMWORD PTR [rcx] ; load low bias vector test bpl,MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - jz BroadcastScaleValue + jz Bsv vmovups ymm1,YMMWORD PTR [r9] ; load low scale vector - jmp ConvertLowAccumulatorsToFloat + jmp Claf -BroadcastScaleValue: +Bsv: vbroadcastss ymm1,DWORD PTR [r9] -ConvertLowAccumulatorsToFloat: +Claf: IFIDNI , ConvertAccumulatorToFloatRegList ELSE ConvertAccumulatorToFloatRegList ENDIF cmp r11d,8 ; output single vector? - jbe ConvertFloatsToIntegers + jbe Cfi vmovdqu ymm0,YMMWORD PTR [rcx+8*4] ; load high bias vector test bpl,MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - jz ConvertHighAccumulatorsToFloat + jz Chaf vmovups ymm1,YMMWORD PTR [r9+8*4] ; load high scale vector -ConvertHighAccumulatorsToFloat: +Chaf: IFIDNI , ConvertAccumulatorToFloatRegList ELSE @@ -594,7 +590,7 @@ ENDIF ; Convert the intermediate float values to 32-bit integers in the range 0 to 255. ; -ConvertFloatsToIntegers: +Cfi: vbroadcastss ymm0,DWORD PTR ConvSymPostProcessParams.MinimumValue[rdx] vbroadcastss ymm1,DWORD PTR ConvSymPostProcessParams.MaximumValue[rdx] vpbroadcastd ymm2,DWORD PTR ConvSymPostProcessParams.OutputZeroPoint[rdx] @@ -604,7 +600,7 @@ ELSE ConvertFloatToIntegerRegList ENDIF cmp r11d,8 ; output single vector? - jbe StoreQuantizedOutputBy8 + jbe Sqo8 IFIDNI , ConvertFloatToIntegerRegList ELSE @@ -615,21 +611,21 @@ ENDIF ; Pack with saturation and store 16 bytes to the output buffer. ; -StoreQuantizedOutputBy16: +Sqo16: IFIDNI , cmp ebx,5 - ja StoreQuantizedOutput6By16 - je StoreQuantizedOutput5By16 + ja Sqo6By16 + je Sqo5By16 ENDIF cmp ebx,3 - ja StoreQuantizedOutput4By16 - je StoreQuantizedOutput3By16 + ja Sqo4By16 + je Sqo3By16 cmp ebx,1 - ja StoreQuantizedOutput2By16 - jmp StoreQuantizedOutput1By16 + ja Sqo2By16 + jmp Sqo1By16 IFIDNI , -StoreQuantizedOutput6By16: +Sqo6By16: vextracti128 xmm0,ymm14,1 vpackusdw xmm14,xmm14,xmm0 vextracti128 xmm1,ymm15,1 @@ -637,7 +633,7 @@ StoreQuantizedOutput6By16: vpackuswb xmm14,xmm14,xmm15 vmovdqu XMMWORD PTR [r10+rsi*2],xmm14 -StoreQuantizedOutput5By16: +Sqo5By16: vextracti128 xmm0,ymm12,1 vpackusdw xmm12,xmm12,xmm0 vextracti128 xmm1,ymm13,1 @@ -646,7 +642,7 @@ StoreQuantizedOutput5By16: vmovdqu XMMWORD PTR [r10+rsi],xmm12 ENDIF -StoreQuantizedOutput4By16: +Sqo4By16: vextracti128 xmm0,ymm10,1 vpackusdw xmm10,xmm10,xmm0 vextracti128 xmm1,ymm11,1 @@ -654,7 +650,7 @@ StoreQuantizedOutput4By16: vpackuswb xmm10,xmm10,xmm11 vmovdqu XMMWORD PTR [r10],xmm10 -StoreQuantizedOutput3By16: +Sqo3By16: vextracti128 xmm0,ymm8,1 vpackusdw xmm8,xmm8,xmm0 vextracti128 xmm1,ymm9,1 @@ -662,7 +658,7 @@ StoreQuantizedOutput3By16: vpackuswb xmm8,xmm8,xmm9 vmovdqu XMMWORD PTR [r8+rsi*2],xmm8 -StoreQuantizedOutput2By16: +Sqo2By16: vextracti128 xmm0,ymm6,1 vpackusdw xmm6,xmm6,xmm0 vextracti128 xmm1,ymm7,1 @@ -670,7 +666,7 @@ StoreQuantizedOutput2By16: vpackuswb xmm6,xmm6,xmm7 vmovdqu XMMWORD PTR [r8+rsi],xmm6 -StoreQuantizedOutput1By16: +Sqo1By16: vextracti128 xmm0,ymm4,1 vpackusdw xmm4,xmm4,xmm0 vextracti128 xmm1,ymm5,1 @@ -682,19 +678,19 @@ StoreQuantizedOutput1By16: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitK: vzeroupper - movaps xmm6,ConvSymKernelFrame.SavedXmm6[rsp] - movaps xmm7,ConvSymKernelFrame.SavedXmm7[rsp] - movaps xmm8,ConvSymKernelFrame.SavedXmm8[rsp] - movaps xmm9,ConvSymKernelFrame.SavedXmm9[rsp] - movaps xmm10,ConvSymKernelFrame.SavedXmm10[rsp] - movaps xmm11,ConvSymKernelFrame.SavedXmm11[rsp] - movaps xmm12,ConvSymKernelFrame.SavedXmm12[rsp] + movaps xmm6,XMMWORD PTR ConvSymKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR ConvSymKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR ConvSymKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR ConvSymKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR ConvSymKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR ConvSymKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR ConvSymKernelFrame.SavedXmm12[rsp] IFIDNI , - movaps xmm13,ConvSymKernelFrame.SavedXmm13[rsp] - movaps xmm14,ConvSymKernelFrame.SavedXmm14[rsp] - movaps xmm15,ConvSymKernelFrame.SavedXmm15[rsp] + movaps xmm13,XMMWORD PTR ConvSymKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR ConvSymKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR ConvSymKernelFrame.SavedXmm15[rsp] mov r14,ConvSymKernelFrame.SavedR14[rsp] mov r15,ConvSymKernelFrame.SavedR15[rsp] ENDIF @@ -714,71 +710,71 @@ ENDIF ; Pack with saturation and store 8 bytes to the output buffer. ; -StoreQuantizedOutputBy8: +Sqo8: IFIDNI , cmp ebx,5 - ja StoreQuantizedOutput6By8 - je StoreQuantizedOutput5By8 + ja Sqo6By8 + je Sqo5By8 ENDIF cmp ebx,3 - ja StoreQuantizedOutput4By8 - je StoreQuantizedOutput3By8 + ja Sqo4By8 + je Sqo3By8 cmp ebx,1 - ja StoreQuantizedOutput2By8 - jmp StoreQuantizedOutput1By8 + ja Sqo2By8 + jmp Sqo1By8 IFIDNI , -StoreQuantizedOutput6By8: +Sqo6By8: vextracti128 xmm0,ymm14,1 vpackusdw xmm14,xmm14,xmm0 vpackuswb xmm14,xmm14,xmm14 vmovq QWORD PTR [r10+rsi*2],xmm14 -StoreQuantizedOutput5By8: +Sqo5By8: vextracti128 xmm0,ymm12,1 vpackusdw xmm12,xmm12,xmm0 vpackuswb xmm12,xmm12,xmm12 vmovq QWORD PTR [r10+rsi],xmm12 ENDIF -StoreQuantizedOutput4By8: +Sqo4By8: vextracti128 xmm0,ymm10,1 vpackusdw xmm10,xmm10,xmm0 vpackuswb xmm10,xmm10,xmm10 vmovq QWORD PTR [r10],xmm10 -StoreQuantizedOutput3By8: +Sqo3By8: vextracti128 xmm0,ymm8,1 vpackusdw xmm8,xmm8,xmm0 vpackuswb xmm8,xmm8,xmm8 vmovq QWORD PTR [r8+rsi*2],xmm8 -StoreQuantizedOutput2By8: +Sqo2By8: vextracti128 xmm0,ymm6,1 vpackusdw xmm6,xmm6,xmm0 vpackuswb xmm6,xmm6,xmm6 vmovq QWORD PTR [r8+rsi],xmm6 -StoreQuantizedOutput1By8: +Sqo1By8: vextracti128 xmm0,ymm4,1 vpackusdw xmm4,xmm4,xmm0 vpackuswb xmm4,xmm4,xmm4 vmovq QWORD PTR [r8],xmm4 - jmp ExitKernel + jmp ExitK ; ; Process the tail output counts out of line with a reduced block size. ; IFIDNI , -ComputeBlockLoopBy4: +Cbl4: ComputeBlockLoop Isa,4 - jmp ComputeBlockLoopDone + jmp CblDone ENDIF -ComputeBlockLoopBy2: +Cbl2: ComputeBlockLoop Isa,2 - jmp ComputeBlockLoopDone + jmp CblDone NESTED_END MlasConvSymKernel&Isa&, _TEXT @@ -825,6 +821,7 @@ DepthwiseMultiplyAccumulateCellAvxVnni MACRO AccumReg, Mult1Reg, Mult2Reg ; ConvSymDepthwiseKernelFunction MACRO Isa + LOCAL Pnib, Bsv, Claf, Chaf, Cfi, Sqo16, Sqo4By16, Sqo3By16, Sqo2By16, Sqo1By16, ExitK ;++ ; @@ -902,7 +899,7 @@ ConvSymDepthwiseKernelFunction MACRO Isa ; Process an input block of length Channels for each element of the kernel. ; -ProcessNextInputBlock: +Pnib: vpmovsxbd ymm0,QWORD PTR [rdx] vpmovsxbd ymm1,QWORD PTR [rdx+8] lea r11,[rcx+rdi] @@ -936,7 +933,7 @@ ProcessNextInputBlock: add rdx,rsi ; advance filter to next kernel DepthwiseMultiplyAccumulateCell&Isa& ymm11,ymm3,ymm1 dec r9 ; decrement input blocks remaining - jnz ProcessNextInputBlock + jnz Pnib ; ; Apply the bias and convert the block accumulators to intermediate float values. @@ -947,28 +944,28 @@ ProcessNextInputBlock: mov r9,ConvSymPostProcessParams.Scale[rdx] vmovdqu ymm0,YMMWORD PTR [rcx] ; load low bias vector test bpl,MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - jz BroadcastScaleValue + jz Bsv vmovups ymm1,YMMWORD PTR [r9] ; load low scale vector - jmp ConvertLowAccumulatorsToFloat + jmp Claf -BroadcastScaleValue: +Bsv: vbroadcastss ymm1,DWORD PTR [r9] -ConvertLowAccumulatorsToFloat: +Claf: ConvertAccumulatorToFloatRegList vmovdqu ymm0,YMMWORD PTR [rcx+8*4] ; load high bias vector test bpl,MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - jz ConvertHighAccumulatorsToFloat + jz Chaf vmovups ymm1,YMMWORD PTR [r9+8*4] ; load high scale vector -ConvertHighAccumulatorsToFloat: +Chaf: ConvertAccumulatorToFloatRegList ; ; Convert the intermediate float values to 32-bit integers in the range 0 to 255. ; -ConvertFloatsToIntegers: +Cfi: vbroadcastss ymm0,DWORD PTR ConvSymPostProcessParams.MinimumValue[rdx] vbroadcastss ymm1,DWORD PTR ConvSymPostProcessParams.MaximumValue[rdx] vpbroadcastd ymm2,DWORD PTR ConvSymPostProcessParams.OutputZeroPoint[rdx] @@ -979,16 +976,16 @@ ConvertFloatsToIntegers: ; Pack with saturation and store 16 bytes to the output buffer. ; -StoreQuantizedOutputBy16: +Sqo16: lea r10,[rsi*2+rsi] cmp ebx,3 - ja StoreQuantizedOutput4By16 - je StoreQuantizedOutput3By16 + ja Sqo4By16 + je Sqo3By16 cmp ebx,1 - ja StoreQuantizedOutput2By16 - jmp StoreQuantizedOutput1By16 + ja Sqo2By16 + jmp Sqo1By16 -StoreQuantizedOutput4By16: +Sqo4By16: vextracti128 xmm0,ymm10,1 vpackusdw xmm10,xmm10,xmm0 vextracti128 xmm1,ymm11,1 @@ -996,7 +993,7 @@ StoreQuantizedOutput4By16: vpackuswb xmm10,xmm10,xmm11 vmovdqu XMMWORD PTR [r8+r10],xmm10 -StoreQuantizedOutput3By16: +Sqo3By16: vextracti128 xmm0,ymm8,1 vpackusdw xmm8,xmm8,xmm0 vextracti128 xmm1,ymm9,1 @@ -1004,7 +1001,7 @@ StoreQuantizedOutput3By16: vpackuswb xmm8,xmm8,xmm9 vmovdqu XMMWORD PTR [r8+rsi*2],xmm8 -StoreQuantizedOutput2By16: +Sqo2By16: vextracti128 xmm0,ymm6,1 vpackusdw xmm6,xmm6,xmm0 vextracti128 xmm1,ymm7,1 @@ -1012,7 +1009,7 @@ StoreQuantizedOutput2By16: vpackuswb xmm6,xmm6,xmm7 vmovdqu XMMWORD PTR [r8+rsi],xmm6 -StoreQuantizedOutput1By16: +Sqo1By16: vextracti128 xmm0,ymm4,1 vpackusdw xmm4,xmm4,xmm0 vextracti128 xmm1,ymm5,1 @@ -1024,14 +1021,14 @@ StoreQuantizedOutput1By16: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitK: vzeroupper - movaps xmm6,ConvSymDepthwiseKernelFrame.SavedXmm6[rsp] - movaps xmm7,ConvSymDepthwiseKernelFrame.SavedXmm7[rsp] - movaps xmm8,ConvSymDepthwiseKernelFrame.SavedXmm8[rsp] - movaps xmm9,ConvSymDepthwiseKernelFrame.SavedXmm9[rsp] - movaps xmm10,ConvSymDepthwiseKernelFrame.SavedXmm10[rsp] - movaps xmm11,ConvSymDepthwiseKernelFrame.SavedXmm11[rsp] + movaps xmm6,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm11[rsp] add rsp,(ConvSymDepthwiseKernelFrame.SavedR13) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm index cd1ddb55e8b74..ba15b427bc28f 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE ConvSymKernelCommon.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Macro Description: @@ -226,7 +224,6 @@ ComputeBlock MACRO Isa, ColumnCount, VectorOffset, BroadcastOffset ; ComputeBlockLoop MACRO Isa, ColumnCount - LOCAL ComputeBlockBy1Loop ComputeBlockBy1Loop: @@ -250,6 +247,7 @@ ComputeBlockBy1Loop: ; ConvSymKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, InputDirect, InputIndirection, ComputeBlockLoopStart, ComputeBlockLoopBy64, ComputeBlockLoopBy48, ComputeBlockLoopBy32, ComputeBlockLoopBy16, ComputeBlockLoopDone, ExitKernel ;++ ; @@ -442,16 +440,16 @@ ComputeBlockLoopDone: ExitKernel: vzeroupper - movaps xmm6,ConvSymKernelFrame.SavedXmm6[rsp] - movaps xmm7,ConvSymKernelFrame.SavedXmm7[rsp] - movaps xmm8,ConvSymKernelFrame.SavedXmm8[rsp] - movaps xmm9,ConvSymKernelFrame.SavedXmm9[rsp] - movaps xmm10,ConvSymKernelFrame.SavedXmm10[rsp] - movaps xmm11,ConvSymKernelFrame.SavedXmm11[rsp] - movaps xmm12,ConvSymKernelFrame.SavedXmm12[rsp] - movaps xmm13,ConvSymKernelFrame.SavedXmm13[rsp] - movaps xmm14,ConvSymKernelFrame.SavedXmm14[rsp] - movaps xmm15,ConvSymKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR ConvSymKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR ConvSymKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR ConvSymKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR ConvSymKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR ConvSymKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR ConvSymKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR ConvSymKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR ConvSymKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR ConvSymKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR ConvSymKernelFrame.SavedXmm15[rsp] add rsp,(ConvSymKernelFrame.SavedR15) BEGIN_EPILOGUE @@ -482,6 +480,7 @@ ExitKernel: ; ConvSymDepthwiseKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, ComputeDepthwiseBlockBy64, ComputeDepthwiseBlockBy48, ComputeDepthwiseBlockBy32, ComputeDepthwiseBlockBy16, ExitKernel ;++ ; @@ -668,16 +667,16 @@ ComputeDepthwiseBlockBy16: ExitKernel: vzeroupper - movaps xmm6,ConvSymDepthwiseKernelFrame.SavedXmm6[rsp] - movaps xmm7,ConvSymDepthwiseKernelFrame.SavedXmm7[rsp] - movaps xmm8,ConvSymDepthwiseKernelFrame.SavedXmm8[rsp] - movaps xmm9,ConvSymDepthwiseKernelFrame.SavedXmm9[rsp] - movaps xmm10,ConvSymDepthwiseKernelFrame.SavedXmm10[rsp] - movaps xmm11,ConvSymDepthwiseKernelFrame.SavedXmm11[rsp] - movaps xmm12,ConvSymDepthwiseKernelFrame.SavedXmm12[rsp] - movaps xmm13,ConvSymDepthwiseKernelFrame.SavedXmm13[rsp] - movaps xmm14,ConvSymDepthwiseKernelFrame.SavedXmm14[rsp] - movaps xmm15,ConvSymDepthwiseKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR ConvSymDepthwiseKernelFrame.SavedXmm15[rsp] add rsp,(ConvSymDepthwiseKernelFrame.SavedR15) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc index 29a09395a696e..4cc6ed1faea69 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc @@ -42,16 +42,16 @@ ConvSymPostProcessParams ENDS ConvSymKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR15 QWORD ? SavedR14 QWORD ? @@ -77,16 +77,16 @@ ConvSymKernelFrame ENDS ConvSymDepthwiseKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR15 QWORD ? SavedR14 QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm index a7f9f6a206766..68f35d706e7d6 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelAvxCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm index 87d1a2aec82e1..6850a43ee2644 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelAvx512FCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc index 52ee7156437af..74b4c2c7d8c66 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc @@ -21,8 +21,14 @@ FgemmElementShift EQU 3 FgemmElementSize EQU (1 SHL FgemmElementShift) -FgemmElementPtr EQU QWORD PTR -FgemmElementBcst EQU QWORD BCST +FgemmElementPtr TEXTEQU +IFDEF LLVM_ML +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <{1to8}> +ELSE +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <> +ENDIF ; ; Define the typed instructions for double precision. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm index 5cddec31ddbec..458dcfb248894 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelFma3Common.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm index 6ac3bed97a641..0178b41b6f6e5 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelSse2Common.inc - .list ; ; Macro Description: @@ -116,15 +114,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop8xN - LOCAL Compute8xNBlockBy1Loop - LOCAL Output8xNBlock - LOCAL OutputPartial8xNBlock - LOCAL OutputPartialLessThan6xNBlock - LOCAL OutputPartialLessThan4xNBlock - LOCAL OutputPartial1xNBlock - LOCAL SkipAccumulateOutput1xN + LOCAL ProcessNextColumnLoop8xN, Compute8xNBlockBy1Loop, Output8xNBlock, OutputPartial8xNBlock, OutputPartialLessThan6xNBlock, OutputPartialLessThan4xNBlock, OutputPartial1xNBlock, SkipAccumulateOutput1xN ProcessNextColumnLoop8xN: EmitIfCountGE RowCount, 1, diff --git a/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm index e4c0859f02594..0f6843ce1b337 100644 --- a/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm @@ -17,12 +17,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasErfConstants:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasErfConstants:PROC ; ; Structure layout for the erf constants block. @@ -72,18 +70,18 @@ ErfConstants ENDS ErfKernelFrame STRUCT - ErfBuffer0 OWORD 8 DUP(?) - ErfBuffer1 OWORD 8 DUP(?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + ErfBuffer0 QWORD 16 DUP (?) + ErfBuffer1 QWORD 16 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding0 QWORD ? Padding1 QWORD ? CountN QWORD ? @@ -548,16 +546,16 @@ LBiggerNumbersRemaining: LErfBatchExp: vzeroupper - movaps xmm6,ErfKernelFrame.SavedXmm6[rsp] - movaps xmm7,ErfKernelFrame.SavedXmm7[rsp] - movaps xmm8,ErfKernelFrame.SavedXmm8[rsp] - movaps xmm9,ErfKernelFrame.SavedXmm9[rsp] - movaps xmm10,ErfKernelFrame.SavedXmm10[rsp] - movaps xmm11,ErfKernelFrame.SavedXmm11[rsp] - movaps xmm12,ErfKernelFrame.SavedXmm12[rsp] - movaps xmm13,ErfKernelFrame.SavedXmm13[rsp] - movaps xmm14,ErfKernelFrame.SavedXmm14[rsp] - movaps xmm15,ErfKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR ErfKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR ErfKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR ErfKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR ErfKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR ErfKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR ErfKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR ErfKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR ErfKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR ErfKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR ErfKernelFrame.SavedXmm15[rsp] add rsp,(ErfKernelFrame.ReturnAddress) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc index 10d3b5ff21be2..09a4c424463f6 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc @@ -144,18 +144,18 @@ IFNB prefetcht0 [rdx+VectorOffset+PrefetchOffset] ENDIF vmovapf zmm0,ZMMWORD PTR [rdx+VectorOffset] - EmitIfCountGE RowCount, 1, - EmitIfCountGE RowCount, 2, - EmitIfCountGE RowCount, 3, - EmitIfCountGE RowCount, 4, - EmitIfCountGE RowCount, 5, - EmitIfCountGE RowCount, 6, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 1, + EmitIfCountGE RowCount, 2, + EmitIfCountGE RowCount, 3, + EmitIfCountGE RowCount, 4, + EmitIfCountGE RowCount, 5, + EmitIfCountGE RowCount, 6, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, ENDM @@ -240,15 +240,7 @@ ENDIF ; ProcessCountM MACRO RowCount - - LOCAL ProcessNextColumnLoop2xN - LOCAL MultiplyAlpha2xNBlock - LOCAL Store2xNBlock - LOCAL Output1xNBlock - LOCAL Output1xNBlockWithMask - LOCAL MultiplyAlpha1xNBlockWithMask - LOCAL Store1xNBlockWithMask - LOCAL ProcessRemainingCountN + LOCAL ProcessNextColumnLoop2xN, MultiplyAlpha2xNBlock, Store2xNBlock, Output1xNBlock, Output1xNBlockWithMask, MultiplyAlpha1xNBlockWithMask, Store1xNBlockWithMask, ProcessRemainingCountN cmp rbp,FgemmZmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc index 5b65c37bb957b..1d93953b612da 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc @@ -17,7 +17,7 @@ ; ;-- - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Macro Description: @@ -213,15 +213,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop2xN - LOCAL Store2xNBlock - LOCAL ProcessRemainingCountN - LOCAL Store1xNBlock - LOCAL OutputMasked2xNBlock - LOCAL StoreMasked2xNBlock - LOCAL OutputMasked1xNBlock - LOCAL StoreMasked1xNBlock + LOCAL ProcessNextColumnLoop2xN, Store2xNBlock, ProcessRemainingCountN, Store1xNBlock, OutputMasked2xNBlock, StoreMasked2xNBlock, OutputMasked1xNBlock, StoreMasked1xNBlock cmp rbp,FgemmYmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc index d80feac1e5ed4..ad71b4aeeee80 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc @@ -21,16 +21,16 @@ FgemmKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -206,11 +206,7 @@ ENDIF ; ComputeBlockLoop MACRO ComputeBlock, RowCount, AdvanceMatrixAPlusRows - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL OutputBlock + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, OutputBlock mov rdi,r9 ; reload CountK sub rdi,4 diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc index 298b9dca81ee2..90b05e7fba6a8 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc @@ -17,7 +17,7 @@ ; ;-- - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Macro Description: @@ -217,19 +217,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop2xN - LOCAL MultiplyAlpha2xNBlock - LOCAL Store2xNBlock - LOCAL ProcessRemainingCountN - LOCAL MultiplyAlpha1xNBlock - LOCAL Store1xNBlock - LOCAL OutputMasked2xNBlock - LOCAL MultiplyAlphaMasked2xNBlock - LOCAL StoreMasked2xNBlock - LOCAL OutputMasked1xNBlock - LOCAL MultiplyAlphaMasked1xNBlock - LOCAL StoreMasked1xNBlock + LOCAL ProcessNextColumnLoop2xN, MultiplyAlpha2xNBlock, Store2xNBlock, ProcessRemainingCountN, MultiplyAlpha1xNBlock, Store1xNBlock, OutputMasked2xNBlock, MultiplyAlphaMasked2xNBlock, StoreMasked2xNBlock, OutputMasked1xNBlock, MultiplyAlphaMasked1xNBlock, StoreMasked1xNBlock cmp rbp,FgemmYmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc index 960263a642839..b71afa6aa4e99 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc @@ -41,7 +41,6 @@ ; AccumulateAndStoreBlock MACRO RowCount, VectorCount - LOCAL SkipAccumulateOutput test r15b,r15b ; ZeroMode? diff --git a/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm index e50a99baf12fd..a8d905650946b 100644 --- a/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasLogisticConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasLogisticConstants:PROC ;++ ; @@ -136,16 +134,16 @@ ProcessRemainingCount: ExitKernel: vzeroupper - movaps xmm6,TransKernelFrame.SavedXmm6[rsp] - movaps xmm7,TransKernelFrame.SavedXmm7[rsp] - movaps xmm8,TransKernelFrame.SavedXmm8[rsp] - movaps xmm9,TransKernelFrame.SavedXmm9[rsp] - movaps xmm10,TransKernelFrame.SavedXmm10[rsp] - movaps xmm11,TransKernelFrame.SavedXmm11[rsp] - movaps xmm12,TransKernelFrame.SavedXmm12[rsp] - movaps xmm13,TransKernelFrame.SavedXmm13[rsp] - movaps xmm14,TransKernelFrame.SavedXmm14[rsp] - movaps xmm15,TransKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR TransKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR TransKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR TransKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR TransKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR TransKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR TransKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR TransKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR TransKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR TransKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR TransKernelFrame.SavedXmm15[rsp] add rsp,(TransKernelFrame.ReturnAddress) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm index 4dd292ff1a21c..9cc44e6e0e63c 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm @@ -18,9 +18,7 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list ; diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm index bc5ac69e730d1..78994749cbe49 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm @@ -18,12 +18,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the Int8 CopyPackA routine. @@ -31,12 +29,12 @@ INCLUDE AssembleAvxVnni.inc GemmInt8CopyPackAFrame STRUCT - PaddedMatrixAData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? + PaddedMatrixAData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) Padding QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -60,11 +58,11 @@ GemmInt8CopyPackAFrame ENDS GemmInt8CopyPackBFrame STRUCT - PaddedMatrixBData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? + PaddedMatrixBData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? @@ -110,6 +108,8 @@ GemmInt8CopyPackBFrame ENDS ;-- MlasGemmCopyPackAAvx2 MACRO ASigned + LOCAL PnrM4, PnclM4, PrcM4, Crk16M4, Crk8M4, Crk4M4, Crk2M4, PpmaM4, RrsbM4, Prr, PnrM1, PnclM1, PrcM1, Crk16M1, Crk8M1, Crk4M1, \ + Crk2M1, PpmaM1, RrsbM1, ExitRt rex_push_reg rbp push_reg rbx @@ -162,9 +162,9 @@ MlasGemmCopyPackAAvx2 MACRO ASigned ; sub r9,4 - jb ProcessRemainingRows + jb Prr -ProcessNextRowM4: +PnrM4: vpxor xmm0,xmm0,xmm0 ; clear row accumulators vpxor xmm1,xmm1,xmm1 vpxor xmm2,xmm2,xmm2 @@ -177,9 +177,9 @@ ProcessNextRowM4: lea rdi,[rdi+r11*4] ; advance next matrix D by 4 rows mov rbx,r10 ; reload columns remaining sub rbx,32 - jb ProcessRemainingColumnsM4 + jb PrcM4 -ProcessNextColumnLoopM4: +PnclM4: vmovdqu ymm4,YMMWORD PTR [rdx] vmovdqu ymm5,YMMWORD PTR [rdx+r8] vmovdqu ymm6,YMMWORD PTR [rdx+r8*2] @@ -206,13 +206,13 @@ ENDIF add rdx,32 ; advance matrix A by 32 bytes add rcx,32 ; advance matrix D by 32 bytes sub rbx,32 ; subtract columns remaining - jae ProcessNextColumnLoopM4 + jae PnclM4 -ProcessRemainingColumnsM4: +PrcM4: add rbx,32 ; correct for over-subtract above - jz ReduceRowSumBufferM4 + jz RrsbM4 test bl,16 ; (CountK & 16) != 0? - jz CopyRemainingCountKLessThan16M4 + jz Crk16M4 vmovdqu xmm4,XMMWORD PTR [rdx] vmovdqu xmm5,XMMWORD PTR [rdx+r8] vmovdqu xmm6,XMMWORD PTR [rdx+r8*2] @@ -239,17 +239,17 @@ ENDIF add rdx,16 ; advance matrix A by 16 bytes add rcx,16 ; advance matrix D by 16 bytes test bl,15 ; test for unaligned columns - jz ReduceRowSumBufferM4 + jz RrsbM4 ; ; Copy the unaligned CountK columns to a zero padded stack buffer. ; -CopyRemainingCountKLessThan16M4: +Crk16M4: .errnz GemmInt8CopyPackAFrame.PaddedMatrixAData mov rbp,rsp ; GemmInt8CopyPackAFrame.PaddedMatrixAData test bl,8 ; (CountK & 8) != 0? - jz CopyRemainingCountKLessThan8M4 + jz Crk8M4 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] @@ -261,9 +261,9 @@ CopyRemainingCountKLessThan16M4: add rdx,8 add rbp,8 ; advance padded buffer destination -CopyRemainingCountKLessThan8M4: +Crk8M4: test bl,4 ; (CountK & 4) != 0? - jz CopyRemainingCountKLessThan4M4 + jz Crk4M4 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] @@ -275,9 +275,9 @@ CopyRemainingCountKLessThan8M4: add rdx,4 add rbp,4 ; advance padded buffer destination -CopyRemainingCountKLessThan4M4: +Crk4M4: test bl,2 ; (CountK & 2) != 0? - jz CopyRemainingCountKLessThan2M4 + jz Crk2M4 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] @@ -289,9 +289,9 @@ CopyRemainingCountKLessThan4M4: add rdx,2 add rbp,2 ; advance padded buffer destination -CopyRemainingCountKLessThan2M4: +Crk2M4: test bl,1 ; (CountK & 1) != 0? - jz ProcessPaddedMatrixADataM4 + jz PpmaM4 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] @@ -305,7 +305,7 @@ CopyRemainingCountKLessThan2M4: ; Process the remaining CountK columns using the zero padded stack buffer. ; -ProcessPaddedMatrixADataM4: +PpmaM4: vmovdqu xmm4,XMMWORD PTR GemmInt8CopyPackAFrame.PaddedMatrixAData[rsp] vmovdqu xmm5,XMMWORD PTR GemmInt8CopyPackAFrame.PaddedMatrixAData[rsp+16] vmovdqu xmm6,XMMWORD PTR GemmInt8CopyPackAFrame.PaddedMatrixAData[rsp+32] @@ -335,7 +335,7 @@ ENDIF ; Reduce the sums for the four rows of output. ; -ReduceRowSumBufferM4: +RrsbM4: IF ASigned EQ 1 vphaddd ymm0,ymm0,ymm1 ; reduce and interleave Sum1/Sum0 ELSE @@ -352,17 +352,17 @@ ENDIF vmovdqu XMMWORD PTR [r12],xmm0 add r12,4*4 ; advance row sum buffer by 4 dwords sub r9,4 ; subtract rows remaining - jae ProcessNextRowM4 + jae PnrM4 -ProcessRemainingRows: +Prr: add r9,4 ; correct for over-subtract above - jz ExitRoutine + jz ExitRt ; ; Process a single row of matrix A in a loop. ; -ProcessNextRowM1: +PnrM1: vpxor xmm0,xmm0,xmm0 ; clear row accumulator mov rdx,rsi mov rcx,rdi @@ -370,9 +370,9 @@ ProcessNextRowM1: add rdi,r11 mov rbx,r10 ; reload columns remaining sub rbx,32 - jb ProcessRemainingColumnsM1 + jb PrcM1 -ProcessNextColumnLoopM1: +PnclM1: vmovdqu ymm4,YMMWORD PTR [rdx] vmovdqu YMMWORD PTR [rcx],ymm4 IF ASigned EQ 1 @@ -384,13 +384,13 @@ ENDIF add rdx,32 ; advance matrix A by 32 bytes add rcx,32 ; advance matrix D by 32 bytes sub rbx,32 ; subtract columns remaining - jae ProcessNextColumnLoopM1 + jae PnclM1 -ProcessRemainingColumnsM1: +PrcM1: add rbx,32 ; correct for over-subtract above - jz ReduceRowSumBufferM1 + jz RrsbM1 test bl,16 ; (CountK & 16) != 0? - jz CopyRemainingCountKLessThan16M1 + jz Crk16M1 vmovdqu xmm4,XMMWORD PTR [rdx] vmovdqu XMMWORD PTR [rcx],xmm4 IF ASigned EQ 1 @@ -402,41 +402,41 @@ ENDIF add rdx,16 ; advance matrix A by 16 bytes add rcx,16 ; advance matrix D by 16 bytes test bl,15 ; test for unaligned columns - jz ReduceRowSumBufferM1 + jz RrsbM1 ; ; Copy the unaligned CountK columns to a zero padded stack buffer. ; -CopyRemainingCountKLessThan16M1: +Crk16M1: .errnz GemmInt8CopyPackAFrame.PaddedMatrixAData mov rbp,rsp ; GemmInt8CopyPackAFrame.PaddedMatrixAData test bl,8 ; (CountK & 8) != 0? - jz CopyRemainingCountKLessThan8M1 + jz Crk8M1 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 add rbp,8 ; advance padded buffer destination -CopyRemainingCountKLessThan8M1: +Crk8M1: test bl,4 ; (CountK & 4) != 0? - jz CopyRemainingCountKLessThan4M1 + jz Crk4M1 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 add rbp,4 ; advance padded buffer destination -CopyRemainingCountKLessThan4M1: +Crk4M1: test bl,2 ; (CountK & 2) != 0? - jz CopyRemainingCountKLessThan2M1 + jz Crk2M1 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 add rbp,2 ; advance padded buffer destination -CopyRemainingCountKLessThan2M1: +Crk2M1: test bl,1 ; (CountK & 1) != 0? - jz ProcessPaddedMatrixADataM1 + jz PpmaM1 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al @@ -444,7 +444,7 @@ CopyRemainingCountKLessThan2M1: ; Process the remaining CountK columns using the zero padded stack buffer. ; -ProcessPaddedMatrixADataM1: +PpmaM1: vmovdqu xmm4,XMMWORD PTR GemmInt8CopyPackAFrame.PaddedMatrixAData[rsp] vpmaskmovd XMMWORD PTR [rcx],xmm10,xmm4 IF ASigned EQ 1 @@ -458,7 +458,7 @@ ENDIF ; Reduce the sum for the single row of output. ; -ReduceRowSumBufferM1: +RrsbM1: IF ASigned EQ 0 vpmaddwd ymm0,ymm0,ymm8 ; horizontal word+word=dword per row ENDIF @@ -469,19 +469,19 @@ ENDIF vmovd DWORD PTR [r12],xmm0 add r12,4 ; advance row sum buffer by 1 dword dec r9 ; decrement rows remaining - jnz ProcessNextRowM1 + jnz PnrM1 ; ; Restore non-volatile registers and return. ; -ExitRoutine: +ExitRt: vzeroupper - movaps xmm6,GemmInt8CopyPackAFrame.SavedXmm6[rsp] - movaps xmm7,GemmInt8CopyPackAFrame.SavedXmm7[rsp] - movaps xmm8,GemmInt8CopyPackAFrame.SavedXmm8[rsp] - movaps xmm9,GemmInt8CopyPackAFrame.SavedXmm9[rsp] - movaps xmm10,GemmInt8CopyPackAFrame.SavedXmm10[rsp] + movaps xmm6,XMMWORD PTR GemmInt8CopyPackAFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR GemmInt8CopyPackAFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR GemmInt8CopyPackAFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR GemmInt8CopyPackAFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR GemmInt8CopyPackAFrame.SavedXmm10[rsp] add rsp,(GemmInt8CopyPackAFrame.SavedR13) BEGIN_EPILOGUE @@ -536,6 +536,8 @@ ExitRoutine: ;-- MlasGemmCopyPackBAvx2 MACRO IsVnni, BSigned + LOCAL Subfv, PncN16, PnrlN16, IrdN16, PrrN16, ScsbN16, PrCol, ExitRt, PcNu, PnrlNu, Crn8K4, Crn4K4, Crn2K4, Ppmb, PrrNu, Curl, \ + Crn8Ks, Crn4Ks, Crn2Ks, DoneCrnKs, ScsbNu rex_push_reg rbp push_reg rbx @@ -565,35 +567,35 @@ MlasGemmCopyPackBAvx2 MACRO IsVnni, BSigned vpxor xmm9,xmm9,xmm9 ; generate word vector [0x0000] IF IsVnni EQ 0 cmp BYTE PTR GemmInt8CopyPackBFrame.BIsSigned[rsp],0 - jnz SkipUnsignedBitFlipVector + jnz Subfv vpsllw ymm9,ymm8,7 ; generate word vector [0x8080] ENDIF -SkipUnsignedBitFlipVector: +Subfv: ; ; Process 16 columns of matrix B in a loop. ; sub r9,16 - jb ProcessRemainingColumns + jb PrCol -ProcessNextColumnN16: +PncN16: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 mov rdx,rsi add rsi,16 ; advance next matrix B by 16 columns mov rbx,r10 ; reload rows remaining sub rbx,4 - jb ProcessRemainingRowsN16 + jb PrrN16 -ProcessNextRowLoopN16: +PnrlN16: vmovdqu xmm2,XMMWORD PTR [rdx] ; load 4 rows vmovdqu xmm3,XMMWORD PTR [rdx+r8] vmovdqu xmm4,XMMWORD PTR [rdx+r8*2] vmovdqu xmm5,XMMWORD PTR [rdx+rdi] lea rdx,[rdx+r8*4] ; advance matrix B by 4 rows -InterleaveRowDataN16: +IrdN16: vpunpcklbw xmm6,xmm2,xmm3 ; interleave row data vpunpckhbw xmm3,xmm2,xmm3 vpunpcklbw xmm2,xmm4,xmm5 @@ -628,49 +630,49 @@ ELSE ENDIF add rcx,64 ; advance matrix D by 64 bytes sub rbx,4 ; subtract rows remaining - jae ProcessNextRowLoopN16 + jae PnrlN16 ; ; Process the less than 4 remaining rows where the row has 16 columns. ; -ProcessRemainingRowsN16: +PrrN16: add rbx,4 ; correct for over-subtract above - jz StoreColumnSumBufferN16 + jz ScsbN16 vmovdqu xmm2,XMMWORD PTR [rdx] vmovaps xmm3,xmm9 vmovaps xmm4,xmm9 vmovaps xmm5,xmm9 xor ebx,ebx ; no more rows remaining test r10b,2 ; (CountK & 2) != 0? - jz InterleaveRowDataN16 + jz IrdN16 vmovdqu xmm3,XMMWORD PTR [rdx+r8] test r10b,1 ; (CountK & 1) != 0? - jz InterleaveRowDataN16 + jz IrdN16 vmovdqu xmm4,XMMWORD PTR [rdx+r8*2] - jmp InterleaveRowDataN16 + jmp IrdN16 -StoreColumnSumBufferN16: +ScsbN16: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 add r11,16*4 ; advance column sum buffer by 16 dwords sub r9,16 ; subtract columns remaining - jae ProcessNextColumnN16 + jae PncN16 -ProcessRemainingColumns: +PrCol: add r9,16 ; correct for over-subtract above - jnz ProcessColumnNUnaligned + jnz PcNu ; ; Restore non-volatile registers and return. ; -ExitRoutine: +ExitRt: vzeroupper - movaps xmm6,GemmInt8CopyPackBFrame.SavedXmm6[rsp] - movaps xmm7,GemmInt8CopyPackBFrame.SavedXmm7[rsp] - movaps xmm8,GemmInt8CopyPackBFrame.SavedXmm8[rsp] - movaps xmm9,GemmInt8CopyPackBFrame.SavedXmm9[rsp] + movaps xmm6,XMMWORD PTR GemmInt8CopyPackBFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR GemmInt8CopyPackBFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR GemmInt8CopyPackBFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR GemmInt8CopyPackBFrame.SavedXmm9[rsp] add rsp,(GemmInt8CopyPackBFrame.SavedRdi) BEGIN_EPILOGUE @@ -685,20 +687,20 @@ ExitRoutine: ; Process the remaining columns of matrix B. ; -ProcessColumnNUnaligned: +PcNu: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 vmovdqu YMMWORD PTR GemmInt8CopyPackBFrame.PaddedMatrixBData[rsp],ymm9 vmovdqu YMMWORD PTR GemmInt8CopyPackBFrame.PaddedMatrixBData[rsp+32],ymm9 sub r10,4 - jb ProcessRemainingRowsNUnaligned + jb PrrNu -ProcessNextRowLoopNUnaligned: +PnrlNu: mov rdx,rsi .errnz GemmInt8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmInt8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8K4 + jz Crn8K4 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] @@ -710,9 +712,9 @@ ProcessNextRowLoopNUnaligned: add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8K4: +Crn8K4: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4K4 + jz Crn4K4 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] @@ -724,9 +726,9 @@ CopyRemainingCountNLessThan8K4: add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4K4: +Crn4K4: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2K4 + jz Crn2K4 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] @@ -738,9 +740,9 @@ CopyRemainingCountNLessThan4K4: add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2K4: +Crn2K4: test r9b,1 ; (CountN & 1) != 0? - jz ProcessPaddedMatrixBData + jz Ppmb movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] @@ -750,7 +752,7 @@ CopyRemainingCountNLessThan2K4: movzx eax,BYTE PTR [rdx+rdi] mov BYTE PTR [rbp+48],al -ProcessPaddedMatrixBData: +Ppmb: vmovdqu xmm2,XMMWORD PTR GemmInt8CopyPackBFrame.PaddedMatrixBData[rsp] vmovdqu xmm3,XMMWORD PTR GemmInt8CopyPackBFrame.PaddedMatrixBData[rsp+16] vmovdqu xmm4,XMMWORD PTR GemmInt8CopyPackBFrame.PaddedMatrixBData[rsp+32] @@ -790,11 +792,11 @@ ENDIF lea rsi,[rsi+r8*4] ; advance next matrix B by 4 rows add rcx,64 ; advance matrix D by 64 bytes sub r10,4 ; subtract rows remaining - jae ProcessNextRowLoopNUnaligned + jae PnrlNu -ProcessRemainingRowsNUnaligned: +PrrNu: add r10,4 - jz StoreColumnSumBufferNUnaligned + jz ScsbNu ; ; Process the less than 4 remaining rows where the row has less than 16 columns. @@ -805,49 +807,49 @@ ProcessRemainingRowsNUnaligned: vmovdqu YMMWORD PTR [rbp],ymm9 vmovdqu YMMWORD PTR [rbp+32],ymm9 -CopyUnalignedRowLoop: +Curl: lea rdi,[rbp+16] ; advance next padded buffer by 16 bytes mov rdx,rsi test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8KSmall + jz Crn8Ks mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8KSmall: +Crn8Ks: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4KSmall + jz Crn4Ks mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4KSmall: +Crn4Ks: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2KSmall + jz Crn2Ks movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2KSmall: +Crn2Ks: test r9b,1 ; (CountN & 1) != 0? - jz DoneCopyRemainingCountNKSmall + jz DoneCrnKs movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al -DoneCopyRemainingCountNKSmall: +DoneCrnKs: dec r10 - jz ProcessPaddedMatrixBData + jz Ppmb add rsi,r8 ; advance next matrix B by 1 row mov rbp,rdi - jmp CopyUnalignedRowLoop + jmp Curl -StoreColumnSumBufferNUnaligned: +ScsbNu: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 - jmp ExitRoutine + jmp ExitRt ENDM diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm index 30c97fd36fa17..cbd7fe3634683 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the U8U8 CopyPackA routine. @@ -29,11 +27,11 @@ INCLUDE mlasi.inc GemmU8U8CopyPackAFrame STRUCT - PaddedMatrixAData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? + PaddedMatrixAData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) Padding QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -57,7 +55,7 @@ GemmU8U8CopyPackAFrame ENDS GemmU8U8CopyPackBFrame STRUCT - PaddedMatrixBData OWORD 2 DUP (?) + PaddedMatrixBData QWORD 4 DUP (?) SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? @@ -400,10 +398,10 @@ ReduceRowSumBufferM1: ExitRoutine: vzeroupper - movaps xmm6,GemmU8U8CopyPackAFrame.SavedXmm6[rsp] - movaps xmm7,GemmU8U8CopyPackAFrame.SavedXmm7[rsp] - movaps xmm8,GemmU8U8CopyPackAFrame.SavedXmm8[rsp] - movaps xmm9,GemmU8U8CopyPackAFrame.SavedXmm9[rsp] + movaps xmm6,XMMWORD PTR GemmU8U8CopyPackAFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR GemmU8U8CopyPackAFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR GemmU8U8CopyPackAFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR GemmU8U8CopyPackAFrame.SavedXmm9[rsp] add rsp,(GemmU8U8CopyPackAFrame.SavedR13) BEGIN_EPILOGUE @@ -473,18 +471,18 @@ ExitRoutine: ; sub r9,16 - jb ProcessRemainingColumns + jb ProcessRemainingColumns_2 -ProcessNextColumnN16: +ProcessNextColumnN16_2: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 mov rdx,rsi add rsi,16 ; advance next matrix B by 16 columns mov rbx,r10 ; reload rows remaining sub rbx,2 - jb ProcessRemainingRowsN16 + jb ProcessRemainingRowsN16_2 -ProcessNextRowLoopN16: +ProcessNextRowLoopN16_2: vmovdqu xmm2,XMMWORD PTR [rdx] ; load 2 rows vmovdqu xmm3,XMMWORD PTR [rdx+r8] lea rdx,[rdx+r8*2] ; advance matrix B by 2 rows @@ -500,11 +498,11 @@ ProcessNextRowLoopN16: vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 sub rbx,2 ; subtract rows remaining - jae ProcessNextRowLoopN16 + jae ProcessNextRowLoopN16_2 -ProcessRemainingRowsN16: +ProcessRemainingRowsN16_2: add rbx,2 ; correct for over-subtract above - jz StoreColumnSumBufferN16 + jz StoreColumnSumBufferN16_2 vpmovzxbw ymm4,XMMWORD PTR [rdx] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 @@ -516,22 +514,22 @@ ProcessRemainingRowsN16: vpaddd ymm1,ymm1,ymm3 add rcx,32 ; advance matrix D by 32 bytes -StoreColumnSumBufferN16: +StoreColumnSumBufferN16_2: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 add r11,16*4 ; advance column sum buffer by 16 dwords sub r9,16 ; subtract columns remaining - jae ProcessNextColumnN16 + jae ProcessNextColumnN16_2 -ProcessRemainingColumns: +ProcessRemainingColumns_2: add r9,16 ; correct for over-subtract above - jnz ProcessColumnNUnaligned + jnz ProcessColumnNUnaligned_2 ; ; Restore non-volatile registers and return. ; -ExitRoutine: +ExitRoutine_2: vzeroupper add rsp,(GemmU8U8CopyPackBFrame.SavedRsi) @@ -546,18 +544,18 @@ ExitRoutine: ; Process the remaining columns of matrix B. ; -ProcessColumnNUnaligned: +ProcessColumnNUnaligned_2: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 sub r10,2 - jb ProcessRemainingRowsNUnaligned + jb ProcessRemainingRowsNUnaligned_2 -ProcessNextRowLoopNUnaligned: +ProcessNextRowLoopNUnaligned_2: mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8K2 + jz CopyRemainingCountNLessThan8K2_2 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] @@ -565,9 +563,9 @@ ProcessNextRowLoopNUnaligned: add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8K2: +CopyRemainingCountNLessThan8K2_2: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4K2 + jz CopyRemainingCountNLessThan4K2_2 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] @@ -575,9 +573,9 @@ CopyRemainingCountNLessThan8K2: add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4K2: +CopyRemainingCountNLessThan4K2_2: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2K2 + jz CopyRemainingCountNLessThan2K2_2 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] @@ -585,17 +583,17 @@ CopyRemainingCountNLessThan4K2: add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2K2: +CopyRemainingCountNLessThan2K2_2: test r9b,1 ; (CountN & 1) != 0? - jz ProcessPaddedMatrixBDataK2 + jz ProcessPaddedMatrixBDataK2_2 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al -ProcessPaddedMatrixBDataK2: - vmovdqu xmm2,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] - vmovdqu xmm3,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp+16] +ProcessPaddedMatrixBDataK2_2: + vmovdqu xmm2,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] + vmovdqu xmm3,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp+16] vpunpcklbw xmm4,xmm2,xmm3 ; interleave row data vpunpckhbw xmm3,xmm2,xmm3 vmovdqu XMMWORD PTR [rcx],xmm4 ; store interleaved rows @@ -609,44 +607,44 @@ ProcessPaddedMatrixBDataK2: lea rsi,[rsi+r8*2] ; advance next matrix B by 2 rows add rcx,32 ; advance matrix D by 32 bytes sub r10,2 ; subtract columns remaining - jae ProcessNextRowLoopNUnaligned + jae ProcessNextRowLoopNUnaligned_2 -ProcessRemainingRowsNUnaligned: +ProcessRemainingRowsNUnaligned_2: add r10,2 - jz StoreColumnSumBufferNUnaligned + jz StoreColumnSumBufferNUnaligned_2 mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8K1 + jz CopyRemainingCountNLessThan8K1_2 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8K1: +CopyRemainingCountNLessThan8K1_2: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4K1 + jz CopyRemainingCountNLessThan4K1_2 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4K1: +CopyRemainingCountNLessThan4K1_2: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2K1 + jz CopyRemainingCountNLessThan2K1_2 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2K1: +CopyRemainingCountNLessThan2K1_2: test r9b,1 ; (CountN & 1) != 0? - jz ProcessPaddedMatrixBDataK1 + jz ProcessPaddedMatrixBDataK1_2 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al -ProcessPaddedMatrixBDataK1: +ProcessPaddedMatrixBDataK1_2: vpmovzxbw ymm4,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 @@ -657,10 +655,10 @@ ProcessPaddedMatrixBDataK1: vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 -StoreColumnSumBufferNUnaligned: +StoreColumnSumBufferNUnaligned_2: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 - jmp ExitRoutine + jmp ExitRoutine_2 NESTED_END MlasGemmU8U8CopyPackBAvx2, _TEXT diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm index e65e43d93e671..b84c9f87862cf 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm @@ -18,12 +18,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the Int8 kernel. @@ -31,16 +29,16 @@ INCLUDE AssembleAvxVnni.inc GemmInt8KernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) SavedR14 QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -322,11 +320,7 @@ ComputeBlockAvxVnni MACRO ColumnCount, RowCount, VectorOffset, BroadcastOffset, ; ComputeBlockLoop MACRO Isa, ColumnCount, RowCount, ASigned, BSigned - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit mov rsi,r9 ; reload row length remaining @@ -479,11 +473,7 @@ ComputeBlockU8U8Avx2 MACRO ColumnCount, RowCount, VectorOffset, BroadcastOffset ; ComputeBlockLoopU8U8 MACRO Isa, ColumnCount, RowCount - - LOCAL ComputeBlockBy2Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ExitComputeBlockLoop + LOCAL ComputeBlockBy2Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ExitComputeBlockLoop mov rsi,r9 ; reload row length remaining @@ -555,12 +545,7 @@ ExitComputeBlockLoop: ; ProduceOutputBlock MACRO ColumnCount, RowCount, ASigned, BSigned - - LOCAL SkipScaleByZeroPointB - LOCAL AccumulatorsInitialized - LOCAL ProduceWithInt8AvxVnni - LOCAL ProduceWithU8U8Avx2 - LOCAL ExitProduceOutputBlock + LOCAL SkipScaleByZeroPointB, AccumulatorsInitialized, ProduceWithInt8AvxVnni, ProduceWithU8U8Avx2, ExitProduceOutputBlock ; ; Initialize the accumulators with the row and column sums. @@ -719,17 +704,8 @@ ENDIF ; r13 - Optionally supplies the address of the matrix B zero point buffer. ; -ProcessCountM MACRO RowCount, ASigned, BSigned, Fallthrough - - LOCAL ProcessNextColumnLoop16xN - LOCAL SkipAccumulateOutput16xNBlock - LOCAL OutputMasked16xNBlock - LOCAL ExitProcessCountM - LOCAL ProcessRemainingCountN - LOCAL SkipAccumulateOutput8xNBlock - LOCAL SkipAccumulateOutputMasked16xNBlock - LOCAL OutputMasked8xNBlock - LOCAL SkipAccumulateOutputMasked8xNBlock +ProcessCountM MACRO RowCount, ASigned, BSigned, ExitLabel + LOCAL ProcessNextColumnLoop16xN, SkipAccumulateOutput16xNBlock, OutputMasked16xNBlock, ExitProcessCountM, ProcessRemainingCountN, SkipAccumulateOutput8xNBlock, SkipAccumulateOutputMasked16xNBlock, OutputMasked8xNBlock, SkipAccumulateOutputMasked8xNBlock cmp rbp,8 jbe ProcessRemainingCountN @@ -775,7 +751,7 @@ SkipAccumulateOutput16xNBlock: ExitProcessCountM: mov eax,RowCount - jmp ExitKernel + jmp ExitLabel ProcessRemainingCountN: ProduceOutputBlock 8, RowCount, ASigned, BSigned @@ -866,23 +842,8 @@ SkipAccumulateOutputMasked8xNBlock: ; ; -ProcessCount1AvxVnni MACRO RowCount, ASigned, BSigned, Fallthrough - - LOCAL LProcessNextColumnLoop32xN1 - LOCAL LSkipAccumulateOutputMasked32xNBlock1 - LOCAL LProcessNextColumnLoop16xN1 - LOCAL LSkipAccumulateOutput16xNBlock1 - LOCAL LProcessRemainingCountN1 - LOCAL LSkipAccumulateOutput8xNBlock1 - LOCAL LExitProcessCountM1 - LOCAL LOutputMasked32xNBlock1 - LOCAL LSkipAccumulateOutputMasked32xNBlock1 - LOCAL LOutputMasked24xNBlock1 - LOCAL LSkipAccumulateOutputMasked24xNBlock1 - LOCAL LOutputMasked16xNBlock1 - LOCAL LSkipAccumulateOutputMasked16xNBlock1 - LOCAL LOutputMasked8xNBlock1 - LOCAL LSkipAccumulateOutputMasked8xNBlock1 +ProcessCount1AvxVnni MACRO RowCount, ASigned, BSigned, ExitLabel + LOCAL LProcessNextColumnLoop32xN1, LSkipAccumulateOutputMasked32xNBlock1, LProcessNextColumnLoop16xN1, LSkipAccumulateOutput16xNBlock1, LProcessRemainingCountN1, LSkipAccumulateOutput8xNBlock1, LExitProcessCountM1, LOutputMasked32xNBlock1, LSkipAccumulateOutput32xNBlock1, LOutputMasked24xNBlock1, LSkipAccumulateOutputMasked24xNBlock1, LOutputMasked16xNBlock1, LSkipAccumulateOutputMasked16xNBlock1, LOutputMasked8xNBlock1, LSkipAccumulateOutputMasked8xNBlock1 cmp rbp,8 jbe LProcessRemainingCountN1 ; num of cols <= 8?: process the tail @@ -948,7 +909,7 @@ LSkipAccumulateOutput8xNBlock1: LExitProcessCountM1: ; num of cols = 0, we are done mov eax, 1 - jmp ExitKernel + jmp ExitLabel ;; -- Section to write final tail of C matrix and exit -- ;; ;; write <= 32 elements ;; @@ -1076,6 +1037,7 @@ LSkipAccumulateOutputMasked8xNBlock1: ;-- MlasGemmInt8KernelAvx2 MACRO ASigned, BSigned + LOCAL CheckCountM6OrMore, CheckCountM4OrMore, ProcessCountM2, ProcessCountM4, ProcessCountM6, ExitKernel, ProcessCountM1, ProcessCountM1AvxVnni, ProcessCountM3, ProcessCountM5 rex_push_reg rbp push_reg rbx @@ -1133,13 +1095,13 @@ CheckCountM4OrMore: je ProcessCountM1 ProcessCountM2: - ProcessCountM 2, ASigned, BSigned + ProcessCountM 2, ASigned, BSigned, ExitKernel ProcessCountM4: - ProcessCountM 4, ASigned, BSigned + ProcessCountM 4, ASigned, BSigned, ExitKernel ProcessCountM6: - ProcessCountM 6, ASigned, BSigned + ProcessCountM 6, ASigned, BSigned, ExitKernel ; ; Restore non-volatile registers and return. @@ -1147,16 +1109,16 @@ ProcessCountM6: ExitKernel: vzeroupper - movaps xmm6,GemmInt8KernelFrame.SavedXmm6[rsp] - movaps xmm7,GemmInt8KernelFrame.SavedXmm7[rsp] - movaps xmm8,GemmInt8KernelFrame.SavedXmm8[rsp] - movaps xmm9,GemmInt8KernelFrame.SavedXmm9[rsp] - movaps xmm10,GemmInt8KernelFrame.SavedXmm10[rsp] - movaps xmm11,GemmInt8KernelFrame.SavedXmm11[rsp] - movaps xmm12,GemmInt8KernelFrame.SavedXmm12[rsp] - movaps xmm13,GemmInt8KernelFrame.SavedXmm13[rsp] - movaps xmm14,GemmInt8KernelFrame.SavedXmm14[rsp] - movaps xmm15,GemmInt8KernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR GemmInt8KernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR GemmInt8KernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR GemmInt8KernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR GemmInt8KernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR GemmInt8KernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR GemmInt8KernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR GemmInt8KernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR GemmInt8KernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR GemmInt8KernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR GemmInt8KernelFrame.SavedXmm15[rsp] add rsp,(GemmInt8KernelFrame.SavedR14) BEGIN_EPILOGUE @@ -1173,16 +1135,16 @@ ExitKernel: ProcessCountM1: cmp DWORD PTR GemmInt8KernelFrame.PreviousP1Home[rsp],-1 je ProcessCountM1AvxVnni - ProcessCountM 1, ASigned, BSigned + ProcessCountM 1, ASigned, BSigned, ExitKernel ProcessCountM1AvxVnni: - ProcessCount1AvxVnni 1, ASigned, BSigned + ProcessCount1AvxVnni 1, ASigned, BSigned, ExitKernel ProcessCountM3: - ProcessCountM 3, ASigned, BSigned + ProcessCountM 3, ASigned, BSigned, ExitKernel ProcessCountM5: - ProcessCountM 5, ASigned, BSigned + ProcessCountM 5, ASigned, BSigned, ExitKernel ENDM diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm index 9606c3ddb006f..9c47d853b8ba5 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Stack frame layout for the U8X8 kernel. @@ -28,9 +26,9 @@ INCLUDE AssembleAvx512Vnni.inc GemmU8X8KernelFrame STRUCT - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) SavedR14 QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -225,11 +223,7 @@ ENDIF ; ComputeBlockLoopU8S8 MACRO Isa, ColumnCount, RowCount - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit mov rsi,r9 ; reload row length remaining @@ -270,7 +264,6 @@ ComputeBlockLoopExit: ENDM ComputeBlockLoopU8U8 MACRO Isa, ColumnCount, RowCount - LOCAL ComputeBlockBy1Loop mov rsi,r9 ; reload row length remaining @@ -315,12 +308,7 @@ ENDIF ; ProduceOutputBlock MACRO ColumnCount, RowCount - - LOCAL SkipScaleByZeroPointB - LOCAL AccumulatorsInitialized - LOCAL ProduceWithU8S8Avx512Core - LOCAL ProduceWithU8U8Avx512Core - LOCAL ExitProduceOutputBlock + LOCAL SkipScaleByZeroPointB, AccumulatorsInitialized, ProduceWithU8S8Avx512Core, ProduceWithU8U8Avx512Core, ExitProduceOutputBlock ; ; Initialize the accumulators with the row and column sums. @@ -354,63 +342,63 @@ ENDIF ELSE vmovdqu32 zmm3,ZMMWORD PTR [r13] ENDIF - EmitIfCount2GE RowCount, 1, ColumnCount, 16, - EmitIfCount2GE RowCount, 1, ColumnCount, 32, - EmitIfCount2GE RowCount, 1, ColumnCount, 48, + EmitIfCount2GE RowCount, 1, ColumnCount, 16, + EmitIfCount2GE RowCount, 1, ColumnCount, 32, + EmitIfCount2GE RowCount, 1, ColumnCount, 48, EmitIfCount2GE RowCount, 1, ColumnCount, 16, EmitIfCount2GE RowCount, 1, ColumnCount, 32, EmitIfCount2GE RowCount, 1, ColumnCount, 48, - EmitIfCount2GE RowCount, 2, ColumnCount, 16, - EmitIfCount2GE RowCount, 2, ColumnCount, 32, - EmitIfCount2GE RowCount, 2, ColumnCount, 48, + EmitIfCount2GE RowCount, 2, ColumnCount, 16, + EmitIfCount2GE RowCount, 2, ColumnCount, 32, + EmitIfCount2GE RowCount, 2, ColumnCount, 48, EmitIfCount2GE RowCount, 2, ColumnCount, 16, EmitIfCount2GE RowCount, 2, ColumnCount, 32, EmitIfCount2GE RowCount, 2, ColumnCount, 48, - EmitIfCount2GE RowCount, 3, ColumnCount, 16, - EmitIfCount2GE RowCount, 3, ColumnCount, 32, - EmitIfCount2GE RowCount, 3, ColumnCount, 48, + EmitIfCount2GE RowCount, 3, ColumnCount, 16, + EmitIfCount2GE RowCount, 3, ColumnCount, 32, + EmitIfCount2GE RowCount, 3, ColumnCount, 48, EmitIfCount2GE RowCount, 3, ColumnCount, 16, EmitIfCount2GE RowCount, 3, ColumnCount, 32, EmitIfCount2GE RowCount, 3, ColumnCount, 48, - EmitIfCount2GE RowCount, 4, ColumnCount, 16, - EmitIfCount2GE RowCount, 4, ColumnCount, 32, - EmitIfCount2GE RowCount, 4, ColumnCount, 48, + EmitIfCount2GE RowCount, 4, ColumnCount, 16, + EmitIfCount2GE RowCount, 4, ColumnCount, 32, + EmitIfCount2GE RowCount, 4, ColumnCount, 48, EmitIfCount2GE RowCount, 4, ColumnCount, 16, EmitIfCount2GE RowCount, 4, ColumnCount, 32, EmitIfCount2GE RowCount, 4, ColumnCount, 48, - EmitIfCount2GE RowCount, 5, ColumnCount, 16, - EmitIfCount2GE RowCount, 5, ColumnCount, 32, - EmitIfCount2GE RowCount, 5, ColumnCount, 48, + EmitIfCount2GE RowCount, 5, ColumnCount, 16, + EmitIfCount2GE RowCount, 5, ColumnCount, 32, + EmitIfCount2GE RowCount, 5, ColumnCount, 48, EmitIfCount2GE RowCount, 5, ColumnCount, 16, EmitIfCount2GE RowCount, 5, ColumnCount, 32, EmitIfCount2GE RowCount, 5, ColumnCount, 48, - EmitIfCount2GE RowCount, 6, ColumnCount, 16, - EmitIfCount2GE RowCount, 6, ColumnCount, 32, - EmitIfCount2GE RowCount, 6, ColumnCount, 48, + EmitIfCount2GE RowCount, 6, ColumnCount, 16, + EmitIfCount2GE RowCount, 6, ColumnCount, 32, + EmitIfCount2GE RowCount, 6, ColumnCount, 48, EmitIfCount2GE RowCount, 6, ColumnCount, 16, EmitIfCount2GE RowCount, 6, ColumnCount, 32, EmitIfCount2GE RowCount, 6, ColumnCount, 48, jmp AccumulatorsInitialized SkipScaleByZeroPointB: - EmitIfCount2GE RowCount, 1, ColumnCount, 16, - EmitIfCount2GE RowCount, 1, ColumnCount, 32, - EmitIfCount2GE RowCount, 1, ColumnCount, 48, - EmitIfCount2GE RowCount, 2, ColumnCount, 16, - EmitIfCount2GE RowCount, 2, ColumnCount, 32, - EmitIfCount2GE RowCount, 2, ColumnCount, 48, - EmitIfCount2GE RowCount, 3, ColumnCount, 16, - EmitIfCount2GE RowCount, 3, ColumnCount, 32, - EmitIfCount2GE RowCount, 3, ColumnCount, 48, - EmitIfCount2GE RowCount, 4, ColumnCount, 16, - EmitIfCount2GE RowCount, 4, ColumnCount, 32, - EmitIfCount2GE RowCount, 4, ColumnCount, 48, - EmitIfCount2GE RowCount, 5, ColumnCount, 16, - EmitIfCount2GE RowCount, 5, ColumnCount, 32, - EmitIfCount2GE RowCount, 5, ColumnCount, 48, - EmitIfCount2GE RowCount, 6, ColumnCount, 16, - EmitIfCount2GE RowCount, 6, ColumnCount, 32, - EmitIfCount2GE RowCount, 6, ColumnCount, 48, + EmitIfCount2GE RowCount, 1, ColumnCount, 16, + EmitIfCount2GE RowCount, 1, ColumnCount, 32, + EmitIfCount2GE RowCount, 1, ColumnCount, 48, + EmitIfCount2GE RowCount, 2, ColumnCount, 16, + EmitIfCount2GE RowCount, 2, ColumnCount, 32, + EmitIfCount2GE RowCount, 2, ColumnCount, 48, + EmitIfCount2GE RowCount, 3, ColumnCount, 16, + EmitIfCount2GE RowCount, 3, ColumnCount, 32, + EmitIfCount2GE RowCount, 3, ColumnCount, 48, + EmitIfCount2GE RowCount, 4, ColumnCount, 16, + EmitIfCount2GE RowCount, 4, ColumnCount, 32, + EmitIfCount2GE RowCount, 4, ColumnCount, 48, + EmitIfCount2GE RowCount, 5, ColumnCount, 16, + EmitIfCount2GE RowCount, 5, ColumnCount, 32, + EmitIfCount2GE RowCount, 5, ColumnCount, 48, + EmitIfCount2GE RowCount, 6, ColumnCount, 16, + EmitIfCount2GE RowCount, 6, ColumnCount, 32, + EmitIfCount2GE RowCount, 6, ColumnCount, 48, AccumulatorsInitialized: @@ -480,16 +468,7 @@ ENDIF ; ProcessCountM MACRO RowCount - - LOCAL ProcessNextColumnLoop32xN - LOCAL Output32xNBlock - LOCAL SkipAccumulateOutput32xNBlock - LOCAL Output16xNBlock - LOCAL Output16xNBlockWithMask - LOCAL SkipAccumulateOutput16xNBlockWithMask - LOCAL ProcessRemainingCountN - LOCAL ProcessNextColumnLoop48xN - LOCAL SkipAccumulateOutput48xNBlock + LOCAL ProcessNextColumnLoop32xN, Output32xNBlock, SkipAccumulateOutput32xNBlock, Output16xNBlock, Output16xNBlockWithMask, SkipAccumulateOutput16xNBlockWithMask, ProcessRemainingCountN, ProcessNextColumnLoop48xN, SkipAccumulateOutput48xNBlock cmp rbp,32 ja ProcessNextColumnLoop48xN @@ -743,9 +722,9 @@ ProcessCountM6: ExitKernel: vzeroupper - movaps xmm13,GemmU8X8KernelFrame.SavedXmm13[rsp] - movaps xmm14,GemmU8X8KernelFrame.SavedXmm14[rsp] - movaps xmm15,GemmU8X8KernelFrame.SavedXmm15[rsp] + movaps xmm13,XMMWORD PTR GemmU8X8KernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR GemmU8X8KernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR GemmU8X8KernelFrame.SavedXmm15[rsp] add rsp,(GemmU8X8KernelFrame.SavedR14) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm index 3a6f83d2c2ada..d218bc676509b 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm @@ -17,12 +17,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasTranspose4x4BytesAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasTranspose4x4BytesAvx:PROC ; ; Stack frame layout for the U8S8 kernel. @@ -30,7 +28,7 @@ INCLUDE mlasi.inc GemvU8S8KernelFrame STRUCT - SavedXmm6 OWORD ? + SavedXmm6 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? @@ -179,7 +177,7 @@ ProcessRemainingRows: ExitKernel: vzeroupper - movaps xmm6,GemvU8S8KernelFrame.SavedXmm6[rsp] + movaps xmm6,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm6[rsp] add rsp,(GemvU8S8KernelFrame.SavedRdi) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm index c1727b3e34d25..831f01c4b3363 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE QgemvU8S8KernelAvx512Common.inc - .list ; ; Generate the GEMV kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm index cb175881f556e..fbfab44aebeb4 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE QgemvU8S8KernelAvx512Common.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Generate the GEMV kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm index be13dcebbc580..0064a9861c993 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasTranspose4x4BytesAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasTranspose4x4BytesAvx:PROC ; ; Stack frame layout for the U8S8 kernel. @@ -31,11 +29,11 @@ INCLUDE AssembleAvxVnni.inc GemvU8S8KernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? @@ -186,11 +184,11 @@ ProcessRemainingRows: ExitKernel: vzeroupper - movaps xmm6,GemvU8S8KernelFrame.SavedXmm6[rsp] - movaps xmm7,GemvU8S8KernelFrame.SavedXmm7[rsp] - movaps xmm8,GemvU8S8KernelFrame.SavedXmm8[rsp] - movaps xmm9,GemvU8S8KernelFrame.SavedXmm9[rsp] - movaps xmm10,GemvU8S8KernelFrame.SavedXmm10[rsp] + movaps xmm6,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR GemvU8S8KernelFrame.SavedXmm10[rsp] add rsp,(GemvU8S8KernelFrame.SavedRdi) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm index dd6c12e2caecf..17cd63dd4c46a 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelAvxCommon.inc - .list ; ; Macro Description: @@ -136,11 +134,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy2 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCount, ProcessNextOutputCountBy2, ProcessRemainingOutputCount, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -176,7 +170,7 @@ ProcessRemainingOutputCount: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleAvxFilter&FilterCount ENDM @@ -209,9 +203,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy2 - LOCAL ProcessRemainingOutputCount + LOCAL ProcessNextOutputCountBy2, ProcessRemainingOutputCount sub r10,2 jb ProcessRemainingOutputCount @@ -224,7 +216,7 @@ ProcessNextOutputCountBy2: ProcessRemainingOutputCount: add r10,2 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise ProcessPointwiseOutputCountN Avx, 8, FilterCount, 1 ENDM @@ -258,7 +250,19 @@ SconvKernelPointwiseFunction Avx, BiasFilter LEAF_ENTRY MlasConvPostProcessFloatAvxFilter&FilterCount&Output&OutputCount, _TEXT PUBLIC MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount +; +; This alternate entry point shares the AVX post-processing code with the FMA3 +; kernel. ml64 requires the double-colon form (a module-visible label defined +; inside a PROC under the default OPTION SCOPED); llvm-ml does not parse "::", +; but treats every label as global, so a single-colon label plus PUBLIC is +; equivalent. LLVM_ML is defined on the llvm-ml command line (see the MLAS +; cross-assembler CI workflow). +; +IFDEF LLVM_ML +MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount: +ELSE MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:: +ENDIF IF FilterCount GT 2 lea rbx,[r8+rax*2] ; compute output plus 2 rows @@ -270,7 +274,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, EmitIfCount2GE FilterCount, 1, OutputCount, 3, @@ -284,14 +288,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 2, EmitIfCount2GE FilterCount, 4, OutputCount, 3, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount IF OutputCount EQ 1 EmitIfCountGE FilterCount, 1, EmitIfCountGE FilterCount, 2, @@ -316,14 +320,14 @@ ELSE EmitIfCount2GE FilterCount, 4, OutputCount, 3, ENDIF -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount vxorps xmm15,xmm15,xmm15 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, @@ -338,7 +342,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 4, OutputCount, 2, EmitIfCount2GE FilterCount, 4, OutputCount, 3, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm index 43eaa88e49946..f958e3ac975ce 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelCommon.inc - .list ; ; Macro Description: @@ -117,12 +115,12 @@ IFIDNI , ELSE IF FilterCount EQ 1 vmovups zmm24,ZMMWORD PTR [rdx+VectorOffset] - EmitIfCountGE OutputCount, 1, - EmitIfCountGE OutputCount, 2, - EmitIfCountGE OutputCount, 3, - EmitIfCountGE OutputCount, 4, - EmitIfCountGE OutputCount, 5, - EmitIfCountGE OutputCount, 6, + EmitIfCountGE OutputCount, 1, + EmitIfCountGE OutputCount, 2, + EmitIfCountGE OutputCount, 3, + EmitIfCountGE OutputCount, 4, + EmitIfCountGE OutputCount, 5, + EmitIfCountGE OutputCount, 6, ELSE EmitIfCountGE OutputCount, 1, EmitIfCountGE OutputCount, 2, @@ -203,13 +201,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy6 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCountLessThan3 - LOCAL ProcessRemainingOutputCount1 - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCount, ProcessNextOutputCountBy6, ProcessRemainingOutputCount, ProcessRemainingOutputCountLessThan3, ProcessRemainingOutputCount1, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -261,7 +253,7 @@ ProcessRemainingOutputCountLessThan3: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleAvx512FFilter&FilterCount ENDM @@ -294,11 +286,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy6 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCountLessThan3 - LOCAL ProcessRemainingOutputCount1 + LOCAL ProcessNextOutputCountBy6, ProcessRemainingOutputCount, ProcessRemainingOutputCountLessThan3, ProcessRemainingOutputCount1 sub r10,6 jb ProcessRemainingOutputCount @@ -312,20 +300,20 @@ ProcessNextOutputCountBy6: ProcessRemainingOutputCount: add r10,6 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise cmp r10,3 jb ProcessRemainingOutputCountLessThan3 ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 3 lea rax,[r9*2+r9] add rdi,rax ; advance input by 3 elements sub r10,3 - jz ExitKernel + jz ExitKernelPointwise ProcessRemainingOutputCountLessThan3: cmp r10,2 jb ProcessRemainingOutputCount1 ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessRemainingOutputCount1: ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 1 @@ -373,7 +361,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, EmitIfCount2GE FilterCount, 1, OutputCount, 3, @@ -399,14 +387,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 5, EmitIfCount2GE FilterCount, 4, OutputCount, 6, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount IF OutputCount EQ 1 EmitIfCountGE FilterCount, 1, EmitIfCountGE FilterCount, 2, @@ -443,14 +431,14 @@ ELSE EmitIfCount2GE FilterCount, 4, OutputCount, 6, ENDIF -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount vpxord zmm24,zmm24,zmm24 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, @@ -477,7 +465,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 2, OutputCount, 5, EmitIfCount2GE FilterCount, 2, OutputCount, 6, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc index 2f5f997c6f12c..acbe2f5f575d2 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc @@ -30,16 +30,16 @@ MLAS_CONV_KERNEL_FLAG_OTHER_ACTIVATION EQU 00000008h SconvKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -75,22 +75,22 @@ SconvKernelFrame ENDS SconvKernelSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SconvKernelFrame <> + KernelFrame SconvKernelFrame {} SconvKernelSingleFrame ENDS SconvKernelDepthwiseFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -123,22 +123,22 @@ SconvKernelDepthwiseFrame ENDS SconvKernelDepthwiseSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SconvKernelDepthwiseFrame <> + KernelFrame SconvKernelDepthwiseFrame {} SconvKernelDepthwiseSingleFrame ENDS SconvKernelPointwiseFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR14 QWORD ? @@ -204,11 +204,7 @@ SconvKernelPointwiseFrame ENDS ; ProcessOutputCountN MACRO Isa, KernelFrame, KernelType, BlockSize, FilterCount, OutputCount - - LOCAL ProcessNextRow - LOCAL ProcessNextColumn - LOCAL HandlePostProcessing - LOCAL SkipOverPadding + LOCAL ProcessNextRow, ProcessNextColumn, HandlePostProcessing, SkipOverPadding mov rcx,rdi IFIDNI , @@ -307,6 +303,7 @@ ENDIF ; SconvKernelFunction MACRO KernelType, BlockSize, Isa, BiasFilter + LOCAL ProcessFilterCount3, ProcessFilterCountLessThan3, ProcessFilterCount1 ;++ ; @@ -424,17 +421,17 @@ ENDIF je ProcessFilterCount3 jb ProcessFilterCountLessThan3 ProcessFilterCountN SconvKernelFrame, KernelType, 4 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCount3: ProcessFilterCountN SconvKernelFrame, KernelType, 3 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCountLessThan3: cmp r11,2 jb ProcessFilterCount1 ProcessFilterCountN SconvKernelFrame, KernelType, 2 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCount1: ProcessFilterCountN SconvKernelFrame, KernelType, 1 @@ -443,7 +440,7 @@ ProcessFilterCount1: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernel&KernelType&: IFDIFI , vzeroupper ENDIF @@ -483,11 +480,11 @@ IFDIFI , LEAF_ENTRY MlasConv&KernelType&FloatSingle&Isa&Filter&FilterCount, _TEXT -ProcessNextOutputCount: +@@: ProcessOutputCountN Isa, SconvKernelSingleFrame.KernelFrame, KernelType, BlockSize, FilterCount, 1 add rdi,r9 ; advance input by 1 element dec r10 ; decrement output count remaining - jnz ProcessNextOutputCount + jnz @B ret LEAF_END MlasConv&KernelType&FloatSingle&Isa&Filter&FilterCount, _TEXT @@ -621,7 +618,7 @@ SconvKernelDepthwiseFunction MACRO BlockSize, Isa ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernelDepthwise: IFDIFI , vzeroupper ENDIF @@ -659,11 +656,11 @@ IFDIFI , LEAF_ENTRY MlasConvDepthwiseFloatSingle&Isa&Filter1, _TEXT -ProcessNextOutputCount: +@@: ProcessOutputCountN Isa, SconvKernelDepthwiseSingleFrame.KernelFrame, Depthwise, BlockSize, 1, 1 add rdi,r9 ; advance input by 1 element dec r10 ; decrement output count remaining - jnz ProcessNextOutputCount + jnz @B ret LEAF_END MlasConvDepthwiseFloatSingle&Isa&Filter1, _TEXT @@ -705,11 +702,7 @@ ENDIF ; ProcessPointwiseOutputCountN MACRO Isa, BlockSize, FilterCount, OutputCount - - LOCAL ProcessNextInputBlock - LOCAL SkipAccumulateOutput - LOCAL SkipBiasAddition - LOCAL SkipReluActivation + LOCAL ProcessNextInputBlock, SkipAccumulateOutput, SkipBiasAddition, SkipReluActivation mov rcx,rdi mov rdx,r12 @@ -769,6 +762,7 @@ ENDIF ;-- SconvKernelPointwiseFunction MACRO Isa, BiasFilter + LOCAL ProcessFilterCount3, ProcessFilterCountLessThan3, ProcessFilterCount1 ;++ ; @@ -859,17 +853,17 @@ ENDIF je ProcessFilterCount3 jb ProcessFilterCountLessThan3 ProcessPointwiseFilterCountN 4 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCount3: ProcessPointwiseFilterCountN 3 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCountLessThan3: cmp r11,2 jb ProcessFilterCount1 ProcessPointwiseFilterCountN 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCount1: ProcessPointwiseFilterCountN 1 @@ -878,7 +872,7 @@ ProcessFilterCount1: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernelPointwise: IFDIFI , vzeroupper ENDIF diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm index 38b260b4df93d..714aa3b62787d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelAvxCommon.inc - .list ; ; Share the post process functions with the AVX implementation. @@ -29,7 +27,7 @@ INCLUDE SconvKernelAvxCommon.inc IRP FilterCount, <1, 2, 3, 4> IRP OutputCount, <1, 2, 3> - EXTERN MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:NEAR + EXTERN MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:PROC ENDM ENDM @@ -73,9 +71,9 @@ ComputeBlock MACRO KernelType, FilterCount, OutputCount, VectorOffset, Broadcast IFIDNI , vmovups ymm12,YMMWORD PTR [rdx] - EmitIfCountGE OutputCount, 1, - EmitIfCountGE OutputCount, 2, - EmitIfCountGE OutputCount, 3, + EmitIfCountGE OutputCount, 1, + EmitIfCountGE OutputCount, 2, + EmitIfCountGE OutputCount, 3, ELSE EmitIfCountGE OutputCount, 1, EmitIfCountGE OutputCount, 2, @@ -140,13 +138,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCountLeftPad - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy3 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCount1 - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCountLeftPad, ProcessOutputCount, ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessRemainingOutputCount1, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -189,7 +181,7 @@ ProcessRemainingOutputCount: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleFma3Filter&FilterCount ENDM @@ -222,10 +214,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy3 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCount1 + LOCAL ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessRemainingOutputCount1 sub r10,3 jb ProcessRemainingOutputCount @@ -239,11 +228,11 @@ ProcessNextOutputCountBy3: ProcessRemainingOutputCount: add r10,3 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise cmp r10,2 jb ProcessRemainingOutputCount1 ProcessPointwiseOutputCountN Fma3, 8, FilterCount, 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessRemainingOutputCount1: ProcessPointwiseOutputCountN Fma3, 8, FilterCount, 1 diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm index 57eb402ff0c16..3fcdbc5b727f0 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelCommon.inc - .list ; ; Macro Description: @@ -159,7 +157,6 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - LOCAL ProcessNextOutputCount mov r10,KernelFrame.OutputCountLeftPad[rsp] @@ -202,7 +199,6 @@ ProcessNextOutputCount: ; ProcessPointwiseFilterCountN MACRO FilterCount - LOCAL ProcessNextOutputCount ProcessNextOutputCount: @@ -251,7 +247,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 2, OutputCount, 1, @@ -269,14 +265,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 2, OutputCount, 1, @@ -294,14 +290,14 @@ SkipAccumulateOutput: EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount xorps xmm15,xmm15 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, @@ -312,7 +308,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm index 9ba503df16805..bc186f1f9845d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelAvxCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm index d59880a1f552a..7021248cf646a 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelAvx512FCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc index f7bc7870bb3ae..d7d929f219287 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc @@ -21,8 +21,14 @@ FgemmElementShift EQU 2 FgemmElementSize EQU (1 SHL FgemmElementShift) -FgemmElementPtr EQU DWORD PTR -FgemmElementBcst EQU DWORD BCST +FgemmElementPtr TEXTEQU +IFDEF LLVM_ML +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <{1to16}> +ELSE +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <> +ENDIF ; ; Define the typed instructions for single precision. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm index 3651ad18f5333..b106ccc8a1c20 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelFma3Common.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm index 418c8e332eb46..57f1354924e69 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC ; ; Stack frame layout for the SGEMM M=1 kernels. @@ -29,9 +27,9 @@ INCLUDE mlasi.inc SgemmKernelM1Frame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? @@ -217,9 +215,9 @@ ProcessRemainingCountK: ExitKernel: vzeroupper - movaps xmm6,SgemmKernelM1Frame.SavedXmm6[rsp] - movaps xmm7,SgemmKernelM1Frame.SavedXmm7[rsp] - movaps xmm8,SgemmKernelM1Frame.SavedXmm8[rsp] + movaps xmm6,XMMWORD PTR SgemmKernelM1Frame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR SgemmKernelM1Frame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR SgemmKernelM1Frame.SavedXmm8[rsp] add rsp,(SgemmKernelM1Frame.SavedRsi) BEGIN_EPILOGUE @@ -384,9 +382,9 @@ ProcessRemainingCountN1: ; sub rbp,4 - jb ProcessRemainingCountN + jb ProcessRemainingCountN_2 -ProcessRowLoop4: +ProcessRowLoop4_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators vxorps xmm3,xmm3,xmm3 vxorps xmm4,xmm4,xmm4 @@ -396,9 +394,9 @@ ProcessRowLoop4: mov rax,r9 ; reload CountK lea r11,[rdx+rbx*4] ; advance matrix B by 4 rows sub rax,8 - jb ProcessRemainingCountK4 + jb ProcessRemainingCountK4_2 -ProcessColumnLoop4: +ProcessColumnLoop4_2: lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] @@ -412,11 +410,11 @@ ProcessColumnLoop4: add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop4 + jae ProcessColumnLoop4_2 -ProcessRemainingCountK4: +ProcessRemainingCountK4_2: test al,7 ; test for unaligned columns - jz Output4x1Block + jz Output4x1Block_2 lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] @@ -436,7 +434,7 @@ ProcessRemainingCountK4: ; Reduce and output the row accumulators. ; -Output4x1Block: +Output4x1Block_2: vunpcklps ymm6,ymm2,ymm3 ; transpose row accumulators vunpckhps ymm1,ymm2,ymm3 vunpcklps ymm2,ymm4,ymm5 @@ -455,18 +453,18 @@ Output4x1Block: vmovups XMMWORD PTR [r8],xmm4 add r8,4*4 ; advance matrix C by 4 columns sub rbp,4 - jae ProcessRowLoop4 + jae ProcessRowLoop4_2 -ProcessRemainingCountN: +ProcessRemainingCountN_2: test ebp,2 - jnz ProcessRowLoop2 + jnz ProcessRowLoop2_2 test ebp,1 - jnz ProcessRowLoop1 + jnz ProcessRowLoop1_2 -ExitKernel: +ExitKernel_2: vzeroupper - movaps xmm6,SgemmKernelM1Frame.SavedXmm6[rsp] - movaps xmm7,SgemmKernelM1Frame.SavedXmm7[rsp] + movaps xmm6,XMMWORD PTR SgemmKernelM1Frame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR SgemmKernelM1Frame.SavedXmm7[rsp] add rsp,(SgemmKernelM1Frame.SavedRsi) BEGIN_EPILOGUE @@ -480,7 +478,7 @@ ExitKernel: ; Process 2 rows of the matrices. ; -ProcessRowLoop2: +ProcessRowLoop2_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators vxorps xmm3,xmm3,xmm3 mov rcx,r10 ; reload matrix A @@ -488,9 +486,9 @@ ProcessRowLoop2: mov rax,r9 ; reload CountK lea r11,[rdx+rbx*2] ; advance matrix B by 2 rows sub rax,8 - jb ProcessRemainingCountK2 + jb ProcessRemainingCountK2_2 -ProcessColumnLoop2: +ProcessColumnLoop2_2: vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] vaddps ymm2,ymm2,ymm6 @@ -499,11 +497,11 @@ ProcessColumnLoop2: add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop2 + jae ProcessColumnLoop2_2 -ProcessRemainingCountK2: +ProcessRemainingCountK2_2: test al,7 ; test for unaligned columns - jz Output2x1Block + jz Output2x1Block_2 vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] vmulps ymm6,ymm1,ymm6 @@ -516,7 +514,7 @@ ProcessRemainingCountK2: ; Reduce and output the row accumulators. ; -Output2x1Block: +Output2x1Block_2: vunpcklps ymm4,ymm2,ymm3 ; reduce row accumulators vunpckhps ymm2,ymm2,ymm3 vaddps ymm2,ymm2,ymm4 @@ -530,32 +528,32 @@ Output2x1Block: vmovsd QWORD PTR [r8],xmm2 add r8,2*4 ; advance matrix C by 2 columns test ebp,1 - jz ExitKernel + jz ExitKernel_2 ; ; Process 1 row of the matrices. ; -ProcessRowLoop1: +ProcessRowLoop1_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators mov rcx,r10 ; reload matrix A mov rdx,r11 ; reload matrix B mov rax,r9 ; reload CountK sub rax,8 - jb ProcessRemainingCountK1 + jb ProcessRemainingCountK1_2 -ProcessColumnLoop1: +ProcessColumnLoop1_2: vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] vaddps ymm2,ymm2,ymm6 add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop1 + jae ProcessColumnLoop1_2 -ProcessRemainingCountK1: +ProcessRemainingCountK1_2: test al,7 ; test for unaligned columns - jz Output1x1Block + jz Output1x1Block_2 vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] vmulps ymm6,ymm1,ymm6 @@ -565,7 +563,7 @@ ProcessRemainingCountK1: ; Reduce and output the row accumulators. ; -Output1x1Block: +Output1x1Block_2: vhaddps ymm2,ymm2,ymm2 ; reduce row accumulators vhaddps ymm2,ymm2,ymm2 vextractf128 xmm4,ymm2,1 @@ -574,7 +572,7 @@ Output1x1Block: vandnps xmm3,xmm0,xmm3 vaddss xmm2,xmm2,xmm3 vmovss DWORD PTR [r8],xmm2 - jmp ExitKernel + jmp ExitKernel_2 NESTED_END MlasSgemmKernelM1TransposeBAvx, _TEXT diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm index 4146a0e37646e..4e6c50180b9c8 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelSse2Common.inc - .list ; ; Macro Description: @@ -122,19 +120,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop16xN - LOCAL Compute16xNBlockBy4Loop - LOCAL ProcessRemaining16xNBlocks - LOCAL Compute16xNBlockBy1Loop - LOCAL Output16xNBlock - LOCAL OutputPartial16xNBlock - LOCAL OutputPartialLessThan12xNBlock - LOCAL OutputPartialLessThan8xNBlock - LOCAL OutputPartialLessThan4xNBlock - LOCAL SkipAccumulateOutput2xN - LOCAL OutputPartial1xNBlock - LOCAL SkipAccumulateOutput1xN + LOCAL ProcessNextColumnLoop16xN, Compute16xNBlockBy4Loop, ProcessRemaining16xNBlocks, Compute16xNBlockBy1Loop, Output16xNBlock, OutputPartial16xNBlock, OutputPartialLessThan12xNBlock, OutputPartialLessThan8xNBlock, OutputPartialLessThan4xNBlock, SkipAccumulateOutput2xN, OutputPartial1xNBlock, SkipAccumulateOutput1xN ProcessNextColumnLoop16xN: EmitIfCountGE RowCount, 1, diff --git a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm index bfdff7009191e..ca6edd8f56ff9 100644 --- a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMinimumF32Value:NEAR + EXTERN MlasMinimumF32Value:PROC ;++ ; @@ -123,9 +121,9 @@ ExitKernel: vbroadcastss ymm4,DWORD PTR [r8] ; broadcast scale value cmp rdx,32 - jb ProcessRemainingCountBy8 + jb ProcessRemainingCountBy8_2 -ProcessRemainingCountBy32: +ProcessRemainingCountBy32_2: vmulps ymm0,ymm4,YMMWORD PTR [rcx] vmulps ymm1,ymm4,YMMWORD PTR [rcx+8*4] sub rdx,32 @@ -137,29 +135,29 @@ ProcessRemainingCountBy32: vmovups YMMWORD PTR [rcx+24*4],ymm3 add rcx,32*4 ; advance output by 32 elements cmp rdx,32 - jae ProcessRemainingCountBy32 + jae ProcessRemainingCountBy32_2 -ProcessRemainingCountBy8: +ProcessRemainingCountBy8_2: cmp rdx,8 - jb ProcessRemainingCountLessThan8 + jb ProcessRemainingCountLessThan8_2 vmulps ymm0,ymm4,YMMWORD PTR [rcx] sub rdx,8 vmovups YMMWORD PTR [rcx],ymm0 add rcx,8*4 ; advance output by 8 elements - jmp ProcessRemainingCountBy8 + jmp ProcessRemainingCountBy8_2 -ProcessRemainingCountLessThan8: +ProcessRemainingCountLessThan8_2: test rdx,rdx - jz ExitKernel + jz ExitKernel_2 -ProcessRemainingCountBy1: +ProcessRemainingCountBy1_2: vmulss xmm0,xmm4,DWORD PTR [rcx] vmovss DWORD PTR [rcx],xmm0 add rcx,4 ; advance output by 1 element dec edx - jnz ProcessRemainingCountBy1 + jnz ProcessRemainingCountBy1_2 -ExitKernel: +ExitKernel_2: vzeroupper ret @@ -194,9 +192,9 @@ ExitKernel: vbroadcastss ymm4,DWORD PTR [r9] ; broadcast negative minimum value vbroadcastss ymm5,DWORD PTR [r9+4] ; broadcast log(SumExp) cmp r8,32 - jb ProcessRemainingCountBy8 + jb ProcessRemainingCountBy8_3 -ProcessRemainingCountBy32: +ProcessRemainingCountBy32_3: vaddps ymm0,ymm4,YMMWORD PTR [rcx] vaddps ymm1,ymm4,YMMWORD PTR [rcx+8*4] sub r8,32 @@ -213,33 +211,33 @@ ProcessRemainingCountBy32: vmovups YMMWORD PTR [rdx+24*4],ymm3 add rdx,32*4 ; advance output by 32 elements cmp r8,32 - jae ProcessRemainingCountBy32 + jae ProcessRemainingCountBy32_3 -ProcessRemainingCountBy8: +ProcessRemainingCountBy8_3: cmp r8,8 - jb ProcessRemainingCountLessThan8 + jb ProcessRemainingCountLessThan8_3 vaddps ymm0,ymm4,YMMWORD PTR [rcx] add rcx,8*4 ; advance input by 8 elements vsubps ymm0,ymm0,ymm5 ; do as two steps for numeric stability sub r8,8 vmovups YMMWORD PTR [rdx],ymm0 add rdx,8*4 ; advance output by 8 elements - jmp ProcessRemainingCountBy8 + jmp ProcessRemainingCountBy8_3 -ProcessRemainingCountLessThan8: +ProcessRemainingCountLessThan8_3: test r8,r8 - jz ExitKernel + jz ExitKernel_3 -ProcessRemainingCountBy1: +ProcessRemainingCountBy1_3: vaddss xmm0,xmm4,DWORD PTR [rcx] add rcx,4 ; advance input by 1 element vsubss xmm0,xmm0,xmm5 vmovss DWORD PTR [rdx],xmm0 add rdx,4 ; advance output by 1 element dec r8d - jnz ProcessRemainingCountBy1 + jnz ProcessRemainingCountBy1_3 -ExitKernel: +ExitKernel_3: vzeroupper ret diff --git a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm index 3e83bc852f558..a65e3b872d2a0 100644 --- a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMinimumF32Value:NEAR + EXTERN MlasMinimumF32Value:PROC ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm index 3d0f64f447f8e..7a60386e7301b 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelAvxCommon.inc - .list ; ; Macro Description: @@ -44,7 +42,7 @@ IFIDNI , imul rax,SpoolKernelFrame.KernelWidth[rsp] vcvtsi2ss xmm5,xmm5,rax ELSE - vcvtsi2ss xmm5,xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + vcvtsi2ss xmm5,xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] ENDIF vshufps xmm5,xmm5,xmm5,0 vinsertf128 ymm5,ymm5,xmm5,1 ; AVX lacks "vbroadcastss ymm5,xmm5" diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm index ca1fb690194d3..5159c8685d00d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelAvxCommon.inc - .list ; ; Macro Description: @@ -44,7 +42,7 @@ IFIDNI , imul rax,SpoolKernelFrame.KernelWidth[rsp] vcvtsi2ss xmm5,xmm5,rax ELSE - vcvtsi2ss xmm5,xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + vcvtsi2ss xmm5,xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] ENDIF vbroadcastss zmm5,xmm5 ENDIF diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc index fd082af2bf3fe..ac7ffa04e4256 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc @@ -30,6 +30,7 @@ INCLUDE SpoolKernelCommon.inc ; SpoolKernelFunction MACRO PoolingType, Isa + LOCAL ProcessOutputCountLeftPad, ProcessOutputCount, ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessOutputCountRightPad, ExitKernel, ProcessNextOutputCount ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc index 3f735bb27cde2..51f77defabc22 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc @@ -49,7 +49,7 @@ SpoolKernelFrame ENDS SpoolKernelSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SpoolKernelFrame <> + KernelFrame SpoolKernelFrame {} SpoolKernelSingleFrame ENDS @@ -136,11 +136,7 @@ SpoolKernelExit MACRO ; ProcessOutputCountN MACRO KernelFrame, PoolingType, OutputCount - - LOCAL ProcessNextRow - LOCAL ProcessNextColumn - LOCAL SkipOverPadding - LOCAL HandlePostProcessing + LOCAL ProcessNextRow, ProcessNextColumn, SkipOverPadding, HandlePostProcessing mov rcx,rdi mov r11,KernelFrame.KernelHeight[rsp] diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm index d5c0ef8c9af42..3364d202cd3ff 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelCommon.inc - .list ; ; Macro Description: @@ -41,7 +39,7 @@ IFIDNI , ENDIF IFIDNI , - cvtsi2ss xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + cvtsi2ss xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] shufps xmm5,xmm5,0 ENDIF @@ -200,6 +198,7 @@ ENDIF ; SpoolKernelFunction MACRO PoolingType, Isa + LOCAL ProcessNextOutputCount, ExitKernel ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm index 6d94d533d72ad..0b881d39fb98f 100644 --- a/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasTanhConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasTanhConstants:PROC ;++ ; @@ -129,16 +127,16 @@ ProcessRemainingCount: ExitKernel: vzeroupper - movaps xmm6,TransKernelFrame.SavedXmm6[rsp] - movaps xmm7,TransKernelFrame.SavedXmm7[rsp] - movaps xmm8,TransKernelFrame.SavedXmm8[rsp] - movaps xmm9,TransKernelFrame.SavedXmm9[rsp] - movaps xmm10,TransKernelFrame.SavedXmm10[rsp] - movaps xmm11,TransKernelFrame.SavedXmm11[rsp] - movaps xmm12,TransKernelFrame.SavedXmm12[rsp] - movaps xmm13,TransKernelFrame.SavedXmm13[rsp] - movaps xmm14,TransKernelFrame.SavedXmm14[rsp] - movaps xmm15,TransKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR TransKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR TransKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR TransKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR TransKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR TransKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR TransKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR TransKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR TransKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR TransKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR TransKernelFrame.SavedXmm15[rsp] add rsp,(TransKernelFrame.ReturnAddress) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm index 2a34d7884c0c6..ade8cf0ec1fd9 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm @@ -16,13 +16,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasExpConstants:NEAR - EXTERN MlasOpmask16BitTableAvx512:NEAR + EXTERN MlasExpConstants:PROC + EXTERN MlasOpmask16BitTableAvx512:PROC ;++ ; @@ -153,9 +151,9 @@ ExitKernel: vbroadcastss zmm19,DWORD PTR [r9] ; broadcast negative maximum value vpxord zmm20,zmm20,zmm20 ; clear exp() accumulator sub r8,48 - jb ProcessRemainingCount + jb ProcessRemainingCount_2 -ComputeExpBy48Loop: +ComputeExpBy48Loop_2: vaddps zmm0,zmm19,ZMMWORD PTR [rcx] ; bias by negative maximum value vaddps zmm3,zmm19,ZMMWORD PTR [rcx+64] vaddps zmm16,zmm19,ZMMWORD PTR [rcx+128] @@ -206,29 +204,29 @@ ComputeExpBy48Loop: vaddps zmm20,zmm20,zmm17 add rcx,48*4 ; advance input by 48 elements test rdx,rdx - jz SkipStoreResultsBy48 + jz SkipStoreResultsBy48_2 vmovups ZMMWORD PTR [rdx],zmm1 vmovups ZMMWORD PTR [rdx+64],zmm4 vmovups ZMMWORD PTR [rdx+128],zmm17 add rdx,48*4 ; advance output by 48 elements -SkipStoreResultsBy48: +SkipStoreResultsBy48_2: sub r8,48 - jae ComputeExpBy48Loop + jae ComputeExpBy48Loop_2 -ProcessRemainingCount: +ProcessRemainingCount_2: add r8,48 ; correct for over-subtract above - jz ReduceAccumulator + jz ReduceAccumulator_2 mov eax,-1 kmovw k1,eax ; update mask to access all elements -ComputeExpBy16Loop: +ComputeExpBy16Loop_2: cmp r8,16 - jae ProcessSingleVector + jae ProcessSingleVector_2 lea r10,MlasOpmask16BitTableAvx512 kmovw k1,WORD PTR [r10+r8*2] -ProcessSingleVector: +ProcessSingleVector_2: vaddps zmm0{k1}{z},zmm19,ZMMWORD PTR [rcx] ; bias by negative maximum value vmaxps zmm0,zmm21,zmm0 ; clamp lower bound @@ -248,15 +246,15 @@ ProcessSingleVector: vaddps zmm20{k1},zmm20,zmm1 ; accumulate exp() results add rcx,16*4 ; advance input by 16 elements test rdx,rdx - jz SkipStoreResultsBy16 + jz SkipStoreResultsBy16_2 vmovups ZMMWORD PTR [rdx]{k1},zmm1 add rdx,16*4 ; advance output by 16 elements -SkipStoreResultsBy16: +SkipStoreResultsBy16_2: sub r8,16 - ja ComputeExpBy16Loop + ja ComputeExpBy16Loop_2 -ReduceAccumulator: +ReduceAccumulator_2: vextractf64x4 ymm0,zmm20,1 vaddps zmm0,zmm0,zmm20 vhaddps ymm0,ymm0,ymm0 diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc index 96582fe46c851..6634be4276e1a 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc @@ -91,16 +91,16 @@ TanhConstants ENDS TransKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? ReturnAddress QWORD ? PreviousP1Home QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm index 993bb0686c4d7..ba391f701b9bc 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm @@ -16,13 +16,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasExpConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasExpConstants:PROC ;++ ; @@ -142,16 +140,16 @@ ProcessRemainingCount: ExitKernel: vzeroupper - movaps xmm6,TransKernelFrame.SavedXmm6[rsp] - movaps xmm7,TransKernelFrame.SavedXmm7[rsp] - movaps xmm8,TransKernelFrame.SavedXmm8[rsp] - movaps xmm9,TransKernelFrame.SavedXmm9[rsp] - movaps xmm10,TransKernelFrame.SavedXmm10[rsp] - movaps xmm11,TransKernelFrame.SavedXmm11[rsp] - movaps xmm12,TransKernelFrame.SavedXmm12[rsp] - movaps xmm13,TransKernelFrame.SavedXmm13[rsp] - movaps xmm14,TransKernelFrame.SavedXmm14[rsp] - movaps xmm15,TransKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR TransKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR TransKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR TransKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR TransKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR TransKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR TransKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR TransKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR TransKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR TransKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR TransKernelFrame.SavedXmm15[rsp] add rsp,(TransKernelFrame.ReturnAddress) BEGIN_EPILOGUE @@ -207,9 +205,9 @@ ExitKernel: vbroadcastss ymm9,DWORD PTR [r9] ; broadcast negative maximum value vxorps xmm10,xmm10,xmm10 ; clear exp() accumulator sub r8,24 - jb ProcessRemainingCount + jb ProcessRemainingCount_2 -ComputeExpBy24Loop: +ComputeExpBy24Loop_2: vbroadcastss ymm11,ExpConstants.LowerRangeSumExp[rax] vbroadcastss ymm2,ExpConstants.Log2Reciprocal[rax] vaddps ymm0,ymm9,YMMWORD PTR [rcx] ; bias by negative maximum value @@ -276,35 +274,35 @@ ComputeExpBy24Loop: add rcx,24*4 ; advance input by 24 elements vaddps ymm10,ymm10,ymm7 test rdx,rdx - jz SkipStoreResultsBy24 + jz SkipStoreResultsBy24_2 vmovups YMMWORD PTR [rdx],ymm1 vmovups YMMWORD PTR [rdx+32],ymm4 vmovups YMMWORD PTR [rdx+64],ymm7 add rdx,24*4 ; advance output by 24 elements -SkipStoreResultsBy24: +SkipStoreResultsBy24_2: sub r8,24 - jae ComputeExpBy24Loop + jae ComputeExpBy24Loop_2 -ProcessRemainingCount: +ProcessRemainingCount_2: add r8,24 ; correct for over-subtract above - jz ReduceAccumulator + jz ReduceAccumulator_2 vbroadcastss ymm11,ExpConstants.LowerRangeSumExp[rax] -ComputeExpBy8Loop: +ComputeExpBy8Loop_2: cmp r8,8 ; remaining count < 8? - jb LoadPartialVector + jb LoadPartialVector_2 vmovups ymm0,YMMWORD PTR [rcx] - jmp ProcessSingleVector + jmp ProcessSingleVector_2 -LoadPartialVector: +LoadPartialVector_2: lea r10,MlasMaskMoveTableAvx+8*4 neg r8 ; carry flag unchanged vmovups ymm3,YMMWORD PTR [r10+r8*4] vmaskmovps ymm0,ymm3,YMMWORD PTR [rcx] vandps ymm9,ymm9,ymm3 ; mask unused maximum value to 0.0 -ProcessSingleVector: +ProcessSingleVector_2: vbroadcastss ymm2,ExpConstants.Log2Reciprocal[rax] vaddps ymm0,ymm9,ymm0 ; bias by negative maximum value vbroadcastss ymm15,ExpConstants.RoundingBias[rax] @@ -331,43 +329,43 @@ ProcessSingleVector: vfmadd213ps ymm1,ymm0,ymm15 ; p = p * x + poly_5 vfmadd213ps ymm1,ymm0,ymm15 ; p = p * x + poly_6 vmulps ymm1,ymm1,ymm2 - jb StorePartialVector ; remaining count < 8? + jb StorePartialVector_2 ; remaining count < 8? vaddps ymm10,ymm10,ymm1 ; accumulate exp() results test rdx,rdx ; store exp() results? - jz SkipStoreResultsBy8 + jz SkipStoreResultsBy8_2 vmovups YMMWORD PTR [rdx],ymm1 add rdx,8*4 ; advance output by 8 elements -SkipStoreResultsBy8: +SkipStoreResultsBy8_2: add rcx,8*4 ; advance input by 8 elements sub r8,8 - jnz ComputeExpBy8Loop - jmp ReduceAccumulator + jnz ComputeExpBy8Loop_2 + jmp ReduceAccumulator_2 -StorePartialVector: +StorePartialVector_2: vandps ymm1,ymm1,ymm3 ; mask unused exp() results to 0.0 vaddps ymm10,ymm10,ymm1 ; accumulate exp() results test rdx,rdx ; store exp() results? - jz ReduceAccumulator + jz ReduceAccumulator_2 vmaskmovps YMMWORD PTR [rdx],ymm3,ymm1 -ReduceAccumulator: +ReduceAccumulator_2: vhaddps ymm10,ymm10,ymm10 vhaddps ymm10,ymm10,ymm10 vextractf128 xmm0,ymm10,1 vaddss xmm0,xmm0,xmm10 vzeroupper - movaps xmm6,TransKernelFrame.SavedXmm6[rsp] - movaps xmm7,TransKernelFrame.SavedXmm7[rsp] - movaps xmm8,TransKernelFrame.SavedXmm8[rsp] - movaps xmm9,TransKernelFrame.SavedXmm9[rsp] - movaps xmm10,TransKernelFrame.SavedXmm10[rsp] - movaps xmm11,TransKernelFrame.SavedXmm11[rsp] - movaps xmm12,TransKernelFrame.SavedXmm12[rsp] - movaps xmm13,TransKernelFrame.SavedXmm13[rsp] - movaps xmm14,TransKernelFrame.SavedXmm14[rsp] - movaps xmm15,TransKernelFrame.SavedXmm15[rsp] + movaps xmm6,XMMWORD PTR TransKernelFrame.SavedXmm6[rsp] + movaps xmm7,XMMWORD PTR TransKernelFrame.SavedXmm7[rsp] + movaps xmm8,XMMWORD PTR TransKernelFrame.SavedXmm8[rsp] + movaps xmm9,XMMWORD PTR TransKernelFrame.SavedXmm9[rsp] + movaps xmm10,XMMWORD PTR TransKernelFrame.SavedXmm10[rsp] + movaps xmm11,XMMWORD PTR TransKernelFrame.SavedXmm11[rsp] + movaps xmm12,XMMWORD PTR TransKernelFrame.SavedXmm12[rsp] + movaps xmm13,XMMWORD PTR TransKernelFrame.SavedXmm13[rsp] + movaps xmm14,XMMWORD PTR TransKernelFrame.SavedXmm14[rsp] + movaps xmm15,XMMWORD PTR TransKernelFrame.SavedXmm15[rsp] add rsp,(TransKernelFrame.ReturnAddress) BEGIN_EPILOGUE diff --git a/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm b/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm index 800863c77a230..0b5c41dd33c2a 100644 --- a/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm +++ b/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm @@ -14,18 +14,17 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - .const +CONST SEGMENT READONLY ALIGN(16) 'CONST' SINGLE_SIZE equ 4 HALF_SIZE equ 2 LOW_SELECTOR equ 00100000b HIGH_SELECTOR equ 00110001b - SUBTTL "Convert buffer of half-precision floats to single-precision floats" +CONST ENDS + ;++ ; ; Routine Description: diff --git a/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm index 0ad98d3115208..545130fd88c78 100644 --- a/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm +++ b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm @@ -14,20 +14,18 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - .const +CONST SEGMENT READONLY ALIGN(16) 'CONST' - ALIGN 16 MlasFp16MaskSign DD 4 DUP (00007FFFh) MlasFp16CompareInfinity DD 4 DUP (00007C00h) MlasFp16CompareSmallest DD 4 DUP (00000400h) MlasFp16AdjustExponent DD 4 DUP (38000000h) MlasFp16MagicDenormal DD 4 DUP (38800000h) - SUBTTL "Convert buffer of half-precision floats to single-precision floats" +CONST ENDS + ;++ ; ; Routine Description: diff --git a/onnxruntime/core/mlas/lib/amd64/mlasi.inc b/onnxruntime/core/mlas/lib/amd64/mlasi.inc index a4f58c1060a8a..a857c182bcd61 100644 --- a/onnxruntime/core/mlas/lib/amd64/mlasi.inc +++ b/onnxruntime/core/mlas/lib/amd64/mlasi.inc @@ -15,9 +15,127 @@ ; ;-- - .xlist -INCLUDE macamd64.inc - .list +; +; Dual-assembler prologue/epilogue macros. +; +; These replace the Windows SDK macamd64.inc entry and unwind macros so the +; amd64 kernels assemble with BOTH ml64.exe and the LLVM MASM assembler +; (llvm-ml), enabling a Windows x64 cross-build from a non-Windows host. Each +; macro emits the same instructions and the same built-in structured-exception +; unwind directives (.pushreg/.allocstack/.savereg/.savexmm128/.setframe/ +; .endprolog/.beginepilog) that macamd64.inc emits, so the generated code and +; .pdata/.xdata are identical. The only omission is macamd64.inc's +; ifdif
, _CurrentSection_ same-module section check, a diagnostic-only +; construct that llvm-ml cannot parse. +; + +; +; + +; +; AVX-512 embedded-broadcast operand forms. +; +; ml64 uses the MASM 'DWORD/QWORD BCST [mem]' keyword; llvm-ml uses the +; NASM-style 'DWORD/QWORD PTR [mem]{1toN}' decorator. Use these text equates so +; a broadcast operand is written as MlasBcstD [mem]MlasBcstDSuffix and expands +; to the correct form for whichever assembler is building. +; +IFDEF LLVM_ML +MlasBcstD TEXTEQU +MlasBcstDSuffix TEXTEQU <{1to16}> +MlasBcstQ TEXTEQU +MlasBcstQSuffix TEXTEQU <{1to8}> +ELSE +MlasBcstD TEXTEQU +MlasBcstDSuffix TEXTEQU <> +MlasBcstQ TEXTEQU +MlasBcstQSuffix TEXTEQU <> +ENDIF + +push_reg MACRO Reg + push Reg + .pushreg Reg + ENDM + +rex_push_reg MACRO Reg + db 048h + push Reg + .pushreg Reg + ENDM + +alloc_stack MACRO Size + sub rsp, Size + .allocstack Size + ENDM + +save_reg MACRO Reg, Offset + mov Offset[rsp], Reg + .savereg Reg, Offset + ENDM + +save_xmm128 MACRO Reg, Offset + movaps XMMWORD PTR Offset[rsp], Reg + .savexmm128 Reg, Offset + ENDM + +set_frame MACRO Reg, Offset +IF Offset + lea Reg, Offset[rsp] +ELSE + mov Reg, rsp +ENDIF + .setframe Reg, Offset + ENDM + +END_PROLOGUE MACRO + .endprolog + ENDM + +; +; BEGIN_EPILOGUE emits no unwind marker. ml64.exe's macamd64.inc form emits +; .beginepilog, but llvm-ml lowers that to .seh_startepilogue which then requires +; a matching .seh_endepilogue that ml64 does not emit. The amd64 kernels use only +; standard epilogs (stack restore, pops, ret), which the Windows unwinder resolves +; via legacy epilog detection without explicit epilog unwind codes, so omitting the +; marker keeps unwind correct under both assemblers. +; +BEGIN_EPILOGUE MACRO + ENDM + +LEAF_ENTRY MACRO Name, Section, NoPad +Section segment para 'CODE' +IFB + db 6 dup (0cch) +ENDIF + align 16 + public Name +Name proc frame + END_PROLOGUE + ENDM + +LEAF_END MACRO Name, Section +Name endp +Section ends + ENDM + +NESTED_ENTRY MACRO Name, Section, Handler, NoPad +Section segment para 'CODE' +IFB + db 6 dup (0cch) +ENDIF + align 16 + public Name +IFB +Name proc frame +ELSE +Name proc frame:Handler +ENDIF + ENDM + +NESTED_END MACRO Name, Section +Name endp +Section ends + ENDM ; ; Macro Description: diff --git a/onnxruntime/core/mlas/lib/amd64/sgemma.asm b/onnxruntime/core/mlas/lib/amd64/sgemma.asm index 80805d337e500..f8d1d466cd2d4 100644 --- a/onnxruntime/core/mlas/lib/amd64/sgemma.asm +++ b/onnxruntime/core/mlas/lib/amd64/sgemma.asm @@ -15,9 +15,7 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list ;++ ; diff --git a/onnxruntime/core/platform/path_lib.cc b/onnxruntime/core/platform/path_lib.cc index 4f7cc52639a8a..9468b1628b29a 100644 --- a/onnxruntime/core/platform/path_lib.cc +++ b/onnxruntime/core/platform/path_lib.cc @@ -19,10 +19,10 @@ #define WINAPI_FAMILY WINAPI_FAMILY_DESKTOP_APP #include #pragma pop_macro("WINAPI_FAMILY") -#pragma comment(lib, "PathCch.lib") +#pragma comment(lib, "pathcch.lib") #else #include -#pragma comment(lib, "PathCch.lib") +#pragma comment(lib, "pathcch.lib") #endif #else #include diff --git a/onnxruntime/core/platform/windows/debug_alloc.cc b/onnxruntime/core/platform/windows/debug_alloc.cc index 14eb06a9cfa39..5bf7f417662d7 100644 --- a/onnxruntime/core/platform/windows/debug_alloc.cc +++ b/onnxruntime/core/platform/windows/debug_alloc.cc @@ -33,7 +33,7 @@ constexpr int c_callstack_limit = 32; // Maximum depth of callstack in leak tra #include #include "debug_alloc.h" #include -#pragma comment(lib, "Dbghelp.lib") +#pragma comment(lib, "dbghelp.lib") // If you are seeing errors of // "Error LNK2005: "void __cdecl operator delete(void *)" (??3@YAXPEAX@Z) already defined in LIBCMTD.lib(delete_scalar.obj)" diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 07d5dfc9c0b22..3a953dd97504f 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -40,7 +40,7 @@ limitations under the License. #include "core/platform/windows/hardware_core_enumerator.h" #endif #include -#include +#include #include "core/platform/path_lib.h" // for LoopDir() #include "core/platform/windows/dll_load_error.h" diff --git a/onnxruntime/test/shared_lib/test_model_loading.cc b/onnxruntime/test/shared_lib/test_model_loading.cc index 93a81f2960a52..8b955d4358f36 100644 --- a/onnxruntime/test/shared_lib/test_model_loading.cc +++ b/onnxruntime/test/shared_lib/test_model_loading.cc @@ -13,7 +13,7 @@ #include "gmock/gmock.h" #ifdef _WIN32 -#include +#include #else #include #include