From c1f056be43a1c8e3acf6e081472b9347f990a773 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Tue, 7 Jul 2026 12:59:08 -0700 Subject: [PATCH 01/14] Make MLAS x64 (amd64) assembly buildable by both ml64.exe and llvm-ml The MLAS x64 assembly kernels in onnxruntime/core/mlas/lib/amd64 previously only assembled with the Microsoft Macro Assembler (ml64.exe), because they relied on the Windows SDK's macamd64.inc and on several MASM constructs that the LLVM MASM-compatible assembler (llvm-ml) does not accept. This blocked cross-compiling ONNX Runtime for Windows-x64 from a non-Windows host (e.g. building on Linux, as the Edge browser does). This change makes all 37 amd64/*.asm files assemble under BOTH ml64.exe and llvm-ml, and adds CI to keep them that way: - Replace the Windows SDK macamd64.inc dependency with an ORT-owned, dual-compatible prologue/entry/unwind macro layer in mlasi.inc (built on MASM `proc frame` + SEH directives). The generated code, .pdata and .xdata are unchanged for ml64. - Fix every remaining MASM-vs-llvm-ml incompatibility class, including: nested-struct default init (`<>` -> `{}`, which also unblocked llvm-ml's GT parser), label collisions from llvm-ml lacking MASM's default OPTION SCOPED (via LOCAL / anonymous @@ labels / token-pasted unique suffixes), the single `::` public-in-proc label in SconvKernelAvx.asm (guarded by IFDEF LLVM_ML), and an invalid VEX FMA size hint (DWORD PTR -> YMMWORD PTR) in SconvKernelFma3.asm. See individual files for details. - Add .github/workflows/windows_mlas_masm_cross_assembler.yml: on a Windows runner it assembles every amd64/*.asm with both ml64.exe (no define) and llvm-ml (/DLLVM_ML=1), failing if either assembler rejects any file. llvm-ml must be invoked with /DLLVM_ML=1 so the alternate-entry public label uses the llvm-ml-compatible single-colon + PUBLIC form; ml64 gets no define and is byte-for-byte unchanged. Downstream cross-compile consumers (Edge) must add that define to their llvm-ml invocation. Verified locally: 37/37 amd64/*.asm assemble under llvm-ml. ml64 assembly and runtime SEH-unwind correctness are validated by the new Windows CI job. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../windows_mlas_masm_cross_assembler.yml | 135 ++++++++++++++++++ .../mlas/lib/amd64/AssembleAvx512Vnni.inc | 4 +- .../core/mlas/lib/amd64/AssembleAvxVnni.inc | 4 - .../core/mlas/lib/amd64/ConvSymKernelAvx2.asm | 11 +- .../lib/amd64/ConvSymKernelAvx512Core.asm | 5 +- .../mlas/lib/amd64/ConvSymKernelCommon.inc | 40 +++--- .../core/mlas/lib/amd64/DgemmKernelAvx.asm | 2 - .../mlas/lib/amd64/DgemmKernelAvx512F.asm | 2 - .../core/mlas/lib/amd64/DgemmKernelCommon.inc | 5 +- .../core/mlas/lib/amd64/DgemmKernelFma3.asm | 2 - .../core/mlas/lib/amd64/DgemmKernelSse2.asm | 12 +- .../core/mlas/lib/amd64/ErfKernelFma3.asm | 30 ++-- .../lib/amd64/FgemmKernelAvx512FCommon.inc | 34 ++--- .../mlas/lib/amd64/FgemmKernelAvxCommon.inc | 12 +- .../core/mlas/lib/amd64/FgemmKernelCommon.inc | 26 ++-- .../mlas/lib/amd64/FgemmKernelFma3Common.inc | 16 +-- .../mlas/lib/amd64/FgemmKernelSse2Common.inc | 1 - .../mlas/lib/amd64/LogisticKernelFma3.asm | 6 +- .../mlas/lib/amd64/QgemmU8S8KernelAmx.asm | 2 - .../mlas/lib/amd64/QgemmU8S8KernelAvx2.asm | 30 ++-- .../mlas/lib/amd64/QgemmU8U8KernelAvx2.asm | 92 ++++++------ .../mlas/lib/amd64/QgemmU8X8KernelAvx2.asm | 72 +++------- .../lib/amd64/QgemmU8X8KernelAvx512Core.asm | 105 ++++++-------- .../mlas/lib/amd64/QgemvU8S8KernelAvx2.asm | 8 +- .../lib/amd64/QgemvU8S8KernelAvx512Core.asm | 2 - .../lib/amd64/QgemvU8S8KernelAvx512Vnni.asm | 2 - .../mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm | 16 +-- .../core/mlas/lib/amd64/SconvKernelAvx.asm | 40 +++--- .../mlas/lib/amd64/SconvKernelAvx512F.asm | 48 +++---- .../core/mlas/lib/amd64/SconvKernelCommon.inc | 104 +++++++------- .../core/mlas/lib/amd64/SconvKernelFma3.asm | 29 ++-- .../core/mlas/lib/amd64/SconvKernelSse2.asm | 16 +-- .../core/mlas/lib/amd64/SgemmKernelAvx.asm | 2 - .../mlas/lib/amd64/SgemmKernelAvx512F.asm | 2 - .../core/mlas/lib/amd64/SgemmKernelCommon.inc | 5 +- .../core/mlas/lib/amd64/SgemmKernelFma3.asm | 2 - .../core/mlas/lib/amd64/SgemmKernelM1Avx.asm | 68 +++++---- .../core/mlas/lib/amd64/SgemmKernelSse2.asm | 16 +-- .../core/mlas/lib/amd64/SoftmaxKernelAvx.asm | 48 +++---- .../mlas/lib/amd64/SoftmaxKernelAvx512F.asm | 4 +- .../core/mlas/lib/amd64/SpoolKernelAvx.asm | 4 +- .../mlas/lib/amd64/SpoolKernelAvx512F.asm | 4 +- .../mlas/lib/amd64/SpoolKernelAvxCommon.inc | 1 + .../core/mlas/lib/amd64/SpoolKernelCommon.inc | 8 +- .../core/mlas/lib/amd64/SpoolKernelSse2.asm | 5 +- .../core/mlas/lib/amd64/TanhKernelFma3.asm | 6 +- .../mlas/lib/amd64/TransKernelAvx512F.asm | 34 +++-- .../core/mlas/lib/amd64/TransKernelCommon.inc | 20 +-- .../core/mlas/lib/amd64/TransKernelFma3.asm | 46 +++--- .../core/mlas/lib/amd64/cvtfp16Avx.asm | 7 +- onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm | 8 +- onnxruntime/core/mlas/lib/amd64/mlasi.inc | 101 ++++++++++++- onnxruntime/core/mlas/lib/amd64/sgemma.asm | 2 - 53 files changed, 666 insertions(+), 640 deletions(-) create mode 100644 .github/workflows/windows_mlas_masm_cross_assembler.yml diff --git a/.github/workflows/windows_mlas_masm_cross_assembler.yml b/.github/workflows/windows_mlas_masm_cross_assembler.yml new file mode 100644 index 0000000000000..6df32ef74e1a2 --- /dev/null +++ b/.github/workflows/windows_mlas_masm_cross_assembler.yml @@ -0,0 +1,135 @@ +name: MLAS x64 MASM Cross-Assembler Check + +# Validates that the MLAS x64 (amd64) assembly kernels assemble under BOTH: +# * ml64.exe - the Microsoft Macro Assembler (native Windows toolchain), and +# * llvm-ml - the LLVM MASM-compatible assembler (used when cross-compiling +# Windows-x64 binaries from a non-Windows host, e.g. Linux). +# +# Keeping the amd64/*.asm sources buildable by both assemblers lets downstream +# consumers (such as the Edge browser) cross-compile ONNX Runtime for Windows on +# Linux build machines. This job is the authoritative guard for that property: +# a change that breaks either assembler fails here. +# +# llvm-ml is invoked with /DLLVM_ML=1. That define selects the handful of +# constructs where a single MASM spelling cannot satisfy both assemblers (today +# only the alternate-entry public label in SconvKernelAvx.asm, where ml64 needs +# the "::" global-label form and llvm-ml needs a single-colon label + PUBLIC). +# ml64.exe is invoked WITHOUT the define, so its output is completely unchanged. + +on: + push: + branches: [main, 'rel-*'] + paths: + - 'onnxruntime/core/mlas/lib/amd64/**' + - '.github/workflows/windows_mlas_masm_cross_assembler.yml' + pull_request: + branches: [main] + paths: + - 'onnxruntime/core/mlas/lib/amd64/**' + - '.github/workflows/windows_mlas_masm_cross_assembler.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + mlas-masm-dual-assemble: + runs-on: windows-latest + timeout-minutes: 20 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: false + + - name: Install LLVM (provides llvm-ml.exe) + shell: pwsh + run: | + choco install llvm --no-progress -y + if ($LASTEXITCODE -ne 0) { throw "choco install llvm failed" } + + - name: Assemble amd64/*.asm with ml64.exe and llvm-ml + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $asmDir = Join-Path $env:GITHUB_WORKSPACE 'onnxruntime/core/mlas/lib/amd64' + if (-not (Test-Path $asmDir)) { throw "amd64 asm directory not found: $asmDir" } + + # Locate ml64.exe via vswhere (bundled on GitHub-hosted Windows images). + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' + if (-not (Test-Path $vswhere)) { throw "vswhere.exe not found: $vswhere" } + $ml64 = (& $vswhere -latest -products '*' -find '**/x64/ml64.exe' | Select-Object -First 1) + if (-not $ml64) { $ml64 = (& $vswhere -latest -products '*' -find '**/ml64.exe' | Select-Object -First 1) } + if (-not $ml64) { throw "ml64.exe not found via vswhere" } + + # Locate llvm-ml.exe from the LLVM install. + $llvmml = (Get-Command llvm-ml.exe -ErrorAction SilentlyContinue).Source + if (-not $llvmml) { + $candidate = 'C:/Program Files/LLVM/bin/llvm-ml.exe' + if (Test-Path $candidate) { $llvmml = $candidate } + } + if (-not $llvmml) { throw "llvm-ml.exe not found" } + + Write-Host "ml64.exe : $ml64" + Write-Host "llvm-ml : $llvmml" + & $ml64 2>&1 | Select-Object -First 1 | Write-Host + & $llvmml --version 2>&1 | Select-Object -First 3 | Write-Host + + $outDir = Join-Path $env:RUNNER_TEMP 'mlas_asm_out' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + + $asmFiles = Get-ChildItem -Path $asmDir -Filter '*.asm' | Sort-Object Name + if ($asmFiles.Count -eq 0) { throw "no .asm files found in $asmDir" } + Write-Host "Found $($asmFiles.Count) amd64 .asm files." + + $ml64Failures = @() + $llvmFailures = @() + + foreach ($f in $asmFiles) { + $name = $f.BaseName + + # --- ml64.exe (native MASM; no LLVM_ML define) --- + $ml64Obj = Join-Path $outDir "$name.ml64.obj" + $ml64Log = & $ml64 /nologo /c /Fo"$ml64Obj" /I"$asmDir" $f.FullName 2>&1 + if ($LASTEXITCODE -ne 0) { + $ml64Failures += $f.Name + Write-Host "::error::ml64 failed to assemble $($f.Name)" + $ml64Log | ForEach-Object { Write-Host " $_" } + } + + # --- llvm-ml (cross-assembler; LLVM_ML define selects dual-safe spellings) --- + $llvmObj = Join-Path $outDir "$name.llvm.obj" + $llvmLog = & $llvmml -m64 /c /DLLVM_ML=1 /Fo"$llvmObj" /I"$asmDir" $f.FullName 2>&1 + if ($LASTEXITCODE -ne 0) { + $llvmFailures += $f.Name + Write-Host "::error::llvm-ml failed to assemble $($f.Name)" + $llvmLog | ForEach-Object { Write-Host " $_" } + } + } + + Write-Host "" + Write-Host "===================== Summary =====================" + Write-Host ("Total files : {0}" -f $asmFiles.Count) + Write-Host ("ml64 failures : {0}" -f $ml64Failures.Count) + Write-Host ("llvm-ml failures : {0}" -f $llvmFailures.Count) + + if ($ml64Failures.Count -gt 0) { + Write-Host "ml64 could not assemble:" + $ml64Failures | ForEach-Object { Write-Host " - $_" } + } + if ($llvmFailures.Count -gt 0) { + Write-Host "llvm-ml could not assemble (breaks Windows-on-Linux cross-compile):" + $llvmFailures | ForEach-Object { Write-Host " - $_" } + } + + if ($ml64Failures.Count -gt 0 -or $llvmFailures.Count -gt 0) { + throw "One or more amd64 .asm files failed to assemble under ml64 and/or llvm-ml." + } + + Write-Host "All $($asmFiles.Count) amd64 .asm files assembled successfully under both ml64 and llvm-ml." diff --git a/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc b/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc index ed885dcb7b781..cea7982a058a6 100644 --- a/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc +++ b/onnxruntime/core/mlas/lib/amd64/AssembleAvx512Vnni.inc @@ -87,7 +87,6 @@ GprIndex_r15 EQU 15 ; VnniZmmZmmZmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, Payload2, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -139,7 +138,7 @@ VpdpwssdsZmmZmmZmm MACRO DestReg, Src1Reg, Src2Reg ; ; This macro builds a VNNI instruction of the form: ; -; instr zmm1,zmm2,DWORD BCST [BaseReg+IndexReg*Scale+ByteOffset] +; instr zmm1,zmm2,DWORD PTR [BaseReg+IndexReg*Scale+ByteOffset]{1to16} ; ; Arguments: ; @@ -160,7 +159,6 @@ VpdpwssdsZmmZmmZmm MACRO DestReg, Src1Reg, Src2Reg ; VnniZmmZmmBroadcast MACRO Opcode, DestReg, Src1Reg, BaseReg, ByteOffset, IndexReg, Scale - LOCAL Payload0, Payload1, Payload2, ModRMByte, SibByte .errnz (ByteOffset AND 3) diff --git a/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc b/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc index d5867e8884c44..9ae9b9b8b55b0 100644 --- a/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc +++ b/onnxruntime/core/mlas/lib/amd64/AssembleAvxVnni.inc @@ -72,7 +72,6 @@ XmmIndex_xmm15 EQU 15 ; VnniYmmYmmYmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -134,7 +133,6 @@ VpdpwssdsYmmYmmYmm MACRO DestReg, Src1Reg, Src2Reg ; VnniXmmXmmXmm MACRO Opcode, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -198,7 +196,6 @@ VpdpwssdsXmmXmmXmm MACRO DestReg, Src1Reg, Src2Reg ; Avx2VnniYmmYmmYmm MACRO Opcode, Prefix, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix @@ -274,7 +271,6 @@ VpdpbuudsYmmYmmYmm MACRO DestReg, Src1Reg, Src2Reg ; Avx2VnniXmmXmmXmm MACRO Opcode, Prefix, DestReg, Src1Reg, Src2Reg - LOCAL Payload0, Payload1, ModRMByte Payload0 = 002h ; "0F 38" prefix diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm index 9c334bea2f468..0f2a4d9d708f5 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE ConvSymKernelCommon.inc INCLUDE AssembleAvxVnni.inc - .list extern CheckSaturationForVPMADDUBSW:proc @@ -236,11 +234,7 @@ ComputeBlock MACRO Isa, RowCount, VectorOffset, BroadcastOffset ; ComputeBlockLoop MACRO Isa, RowCount, UnrollLoop - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit IFNB sub rax,4*4 @@ -357,6 +351,8 @@ ConvertFloatToIntegerRegList MACRO RegList ; ConvSymKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, InputDirect, InputIndirection, ComputeBlockLoopStart, ComputeBlockLoopDone, BroadcastScaleValue, ConvertLowAccumulatorsToFloat, ConvertHighAccumulatorsToFloat, ConvertFloatsToIntegers, StoreQuantizedOutputBy16, StoreQuantizedOutput6By16, StoreQuantizedOutput5By16, StoreQuantizedOutput4By16, StoreQuantizedOutput3By16, StoreQuantizedOutput2By16, StoreQuantizedOutput1By16, ExitKernel, StoreQuantizedOutputBy8, \ + StoreQuantizedOutput6By8, StoreQuantizedOutput5By8, StoreQuantizedOutput4By8, StoreQuantizedOutput3By8, StoreQuantizedOutput2By8, StoreQuantizedOutput1By8, ComputeBlockLoopBy4, ComputeBlockLoopBy2 ;++ ; @@ -825,6 +821,7 @@ DepthwiseMultiplyAccumulateCellAvxVnni MACRO AccumReg, Mult1Reg, Mult2Reg ; ConvSymDepthwiseKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, BroadcastScaleValue, ConvertLowAccumulatorsToFloat, ConvertHighAccumulatorsToFloat, ConvertFloatsToIntegers, StoreQuantizedOutputBy16, StoreQuantizedOutput4By16, StoreQuantizedOutput3By16, StoreQuantizedOutput2By16, StoreQuantizedOutput1By16, ExitKernel ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm index cd1ddb55e8b74..6d8a33da9217d 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelAvx512Core.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE ConvSymKernelCommon.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Macro Description: @@ -226,7 +224,6 @@ ComputeBlock MACRO Isa, ColumnCount, VectorOffset, BroadcastOffset ; ComputeBlockLoop MACRO Isa, ColumnCount - LOCAL ComputeBlockBy1Loop ComputeBlockBy1Loop: @@ -250,6 +247,7 @@ ComputeBlockBy1Loop: ; ConvSymKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, InputDirect, InputIndirection, ComputeBlockLoopStart, ComputeBlockLoopBy64, ComputeBlockLoopBy48, ComputeBlockLoopBy32, ComputeBlockLoopBy16, ComputeBlockLoopDone, ExitKernel ;++ ; @@ -482,6 +480,7 @@ ExitKernel: ; ConvSymDepthwiseKernelFunction MACRO Isa + LOCAL ProcessNextInputBlock, ComputeDepthwiseBlockBy64, ComputeDepthwiseBlockBy48, ComputeDepthwiseBlockBy32, ComputeDepthwiseBlockBy16, ExitKernel ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc index 29a09395a696e..4cc6ed1faea69 100644 --- a/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/ConvSymKernelCommon.inc @@ -42,16 +42,16 @@ ConvSymPostProcessParams ENDS ConvSymKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR15 QWORD ? SavedR14 QWORD ? @@ -77,16 +77,16 @@ ConvSymKernelFrame ENDS ConvSymDepthwiseKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR15 QWORD ? SavedR14 QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm index a7f9f6a206766..68f35d706e7d6 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelAvxCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm index 87d1a2aec82e1..6850a43ee2644 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelAvx512FCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc index 52ee7156437af..810eaa414a23b 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelCommon.inc @@ -21,8 +21,9 @@ FgemmElementShift EQU 3 FgemmElementSize EQU (1 SHL FgemmElementShift) -FgemmElementPtr EQU QWORD PTR -FgemmElementBcst EQU QWORD BCST +FgemmElementPtr TEXTEQU +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <{1to8}> ; ; Define the typed instructions for double precision. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm index 5cddec31ddbec..458dcfb248894 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelFma3.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelFma3Common.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm index 6ac3bed97a641..0178b41b6f6e5 100644 --- a/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/DgemmKernelSse2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE DgemmKernelCommon.inc INCLUDE FgemmKernelSse2Common.inc - .list ; ; Macro Description: @@ -116,15 +114,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop8xN - LOCAL Compute8xNBlockBy1Loop - LOCAL Output8xNBlock - LOCAL OutputPartial8xNBlock - LOCAL OutputPartialLessThan6xNBlock - LOCAL OutputPartialLessThan4xNBlock - LOCAL OutputPartial1xNBlock - LOCAL SkipAccumulateOutput1xN + LOCAL ProcessNextColumnLoop8xN, Compute8xNBlockBy1Loop, Output8xNBlock, OutputPartial8xNBlock, OutputPartialLessThan6xNBlock, OutputPartialLessThan4xNBlock, OutputPartial1xNBlock, SkipAccumulateOutput1xN ProcessNextColumnLoop8xN: EmitIfCountGE RowCount, 1, diff --git a/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm index e4c0859f02594..742d2151279f9 100644 --- a/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/ErfKernelFma3.asm @@ -17,12 +17,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasErfConstants:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasErfConstants:PROC ; ; Structure layout for the erf constants block. @@ -72,18 +70,18 @@ ErfConstants ENDS ErfKernelFrame STRUCT - ErfBuffer0 OWORD 8 DUP(?) - ErfBuffer1 OWORD 8 DUP(?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + ErfBuffer0 QWORD 16 DUP (?) + ErfBuffer1 QWORD 16 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding0 QWORD ? Padding1 QWORD ? CountN QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc index 10d3b5ff21be2..09a4c424463f6 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvx512FCommon.inc @@ -144,18 +144,18 @@ IFNB prefetcht0 [rdx+VectorOffset+PrefetchOffset] ENDIF vmovapf zmm0,ZMMWORD PTR [rdx+VectorOffset] - EmitIfCountGE RowCount, 1, - EmitIfCountGE RowCount, 2, - EmitIfCountGE RowCount, 3, - EmitIfCountGE RowCount, 4, - EmitIfCountGE RowCount, 5, - EmitIfCountGE RowCount, 6, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, - EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 1, + EmitIfCountGE RowCount, 2, + EmitIfCountGE RowCount, 3, + EmitIfCountGE RowCount, 4, + EmitIfCountGE RowCount, 5, + EmitIfCountGE RowCount, 6, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, + EmitIfCountGE RowCount, 12, ENDM @@ -240,15 +240,7 @@ ENDIF ; ProcessCountM MACRO RowCount - - LOCAL ProcessNextColumnLoop2xN - LOCAL MultiplyAlpha2xNBlock - LOCAL Store2xNBlock - LOCAL Output1xNBlock - LOCAL Output1xNBlockWithMask - LOCAL MultiplyAlpha1xNBlockWithMask - LOCAL Store1xNBlockWithMask - LOCAL ProcessRemainingCountN + LOCAL ProcessNextColumnLoop2xN, MultiplyAlpha2xNBlock, Store2xNBlock, Output1xNBlock, Output1xNBlockWithMask, MultiplyAlpha1xNBlockWithMask, Store1xNBlockWithMask, ProcessRemainingCountN cmp rbp,FgemmZmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc index 5b65c37bb957b..1d93953b612da 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelAvxCommon.inc @@ -17,7 +17,7 @@ ; ;-- - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Macro Description: @@ -213,15 +213,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop2xN - LOCAL Store2xNBlock - LOCAL ProcessRemainingCountN - LOCAL Store1xNBlock - LOCAL OutputMasked2xNBlock - LOCAL StoreMasked2xNBlock - LOCAL OutputMasked1xNBlock - LOCAL StoreMasked1xNBlock + LOCAL ProcessNextColumnLoop2xN, Store2xNBlock, ProcessRemainingCountN, Store1xNBlock, OutputMasked2xNBlock, StoreMasked2xNBlock, OutputMasked1xNBlock, StoreMasked1xNBlock cmp rbp,FgemmYmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc index d80feac1e5ed4..ad71b4aeeee80 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelCommon.inc @@ -21,16 +21,16 @@ FgemmKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -206,11 +206,7 @@ ENDIF ; ComputeBlockLoop MACRO ComputeBlock, RowCount, AdvanceMatrixAPlusRows - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL OutputBlock + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, OutputBlock mov rdi,r9 ; reload CountK sub rdi,4 diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc index 298b9dca81ee2..90b05e7fba6a8 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelFma3Common.inc @@ -17,7 +17,7 @@ ; ;-- - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Macro Description: @@ -217,19 +217,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop2xN - LOCAL MultiplyAlpha2xNBlock - LOCAL Store2xNBlock - LOCAL ProcessRemainingCountN - LOCAL MultiplyAlpha1xNBlock - LOCAL Store1xNBlock - LOCAL OutputMasked2xNBlock - LOCAL MultiplyAlphaMasked2xNBlock - LOCAL StoreMasked2xNBlock - LOCAL OutputMasked1xNBlock - LOCAL MultiplyAlphaMasked1xNBlock - LOCAL StoreMasked1xNBlock + LOCAL ProcessNextColumnLoop2xN, MultiplyAlpha2xNBlock, Store2xNBlock, ProcessRemainingCountN, MultiplyAlpha1xNBlock, Store1xNBlock, OutputMasked2xNBlock, MultiplyAlphaMasked2xNBlock, StoreMasked2xNBlock, OutputMasked1xNBlock, MultiplyAlphaMasked1xNBlock, StoreMasked1xNBlock cmp rbp,FgemmYmmElementCount jbe ProcessRemainingCountN diff --git a/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc b/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc index 960263a642839..b71afa6aa4e99 100644 --- a/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc +++ b/onnxruntime/core/mlas/lib/amd64/FgemmKernelSse2Common.inc @@ -41,7 +41,6 @@ ; AccumulateAndStoreBlock MACRO RowCount, VectorCount - LOCAL SkipAccumulateOutput test r15b,r15b ; ZeroMode? diff --git a/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm index e50a99baf12fd..2ab137aa5b183 100644 --- a/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/LogisticKernelFma3.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasLogisticConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasLogisticConstants:PROC ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm index 4dd292ff1a21c..9cc44e6e0e63c 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAmx.asm @@ -18,9 +18,7 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list ; diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm index bc5ac69e730d1..cfeef6f70bb13 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8S8KernelAvx2.asm @@ -18,12 +18,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the Int8 CopyPackA routine. @@ -31,12 +29,12 @@ INCLUDE AssembleAvxVnni.inc GemmInt8CopyPackAFrame STRUCT - PaddedMatrixAData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? + PaddedMatrixAData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) Padding QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -60,11 +58,11 @@ GemmInt8CopyPackAFrame ENDS GemmInt8CopyPackBFrame STRUCT - PaddedMatrixBData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? + PaddedMatrixBData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? @@ -110,6 +108,8 @@ GemmInt8CopyPackBFrame ENDS ;-- MlasGemmCopyPackAAvx2 MACRO ASigned + LOCAL ProcessNextRowM4, ProcessNextColumnLoopM4, ProcessRemainingColumnsM4, CopyRemainingCountKLessThan16M4, CopyRemainingCountKLessThan8M4, CopyRemainingCountKLessThan4M4, CopyRemainingCountKLessThan2M4, ProcessPaddedMatrixADataM4, ReduceRowSumBufferM4, ProcessRemainingRows, ProcessNextRowM1, ProcessNextColumnLoopM1, ProcessRemainingColumnsM1, CopyRemainingCountKLessThan16M1, CopyRemainingCountKLessThan8M1, CopyRemainingCountKLessThan4M1, \ + CopyRemainingCountKLessThan2M1, ProcessPaddedMatrixADataM1, ReduceRowSumBufferM1, ExitRoutine rex_push_reg rbp push_reg rbx @@ -536,6 +536,8 @@ ExitRoutine: ;-- MlasGemmCopyPackBAvx2 MACRO IsVnni, BSigned + LOCAL SkipUnsignedBitFlipVector, ProcessNextColumnN16, ProcessNextRowLoopN16, InterleaveRowDataN16, ProcessRemainingRowsN16, StoreColumnSumBufferN16, ProcessRemainingColumns, ExitRoutine, ProcessColumnNUnaligned, ProcessNextRowLoopNUnaligned, CopyRemainingCountNLessThan8K4, CopyRemainingCountNLessThan4K4, CopyRemainingCountNLessThan2K4, ProcessPaddedMatrixBData, ProcessRemainingRowsNUnaligned, CopyUnalignedRowLoop, \ + CopyRemainingCountNLessThan8KSmall, CopyRemainingCountNLessThan4KSmall, CopyRemainingCountNLessThan2KSmall, DoneCopyRemainingCountNKSmall, StoreColumnSumBufferNUnaligned rex_push_reg rbp push_reg rbx diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm index 30c97fd36fa17..f043cecd0af23 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the U8U8 CopyPackA routine. @@ -29,11 +27,11 @@ INCLUDE mlasi.inc GemmU8U8CopyPackAFrame STRUCT - PaddedMatrixAData OWORD 4 DUP (?) - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? + PaddedMatrixAData QWORD 8 DUP (?) + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) Padding QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -57,7 +55,7 @@ GemmU8U8CopyPackAFrame ENDS GemmU8U8CopyPackBFrame STRUCT - PaddedMatrixBData OWORD 2 DUP (?) + PaddedMatrixBData QWORD 4 DUP (?) SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? @@ -473,18 +471,18 @@ ExitRoutine: ; sub r9,16 - jb ProcessRemainingColumns + jb ProcessRemainingColumns_2 -ProcessNextColumnN16: +ProcessNextColumnN16_2: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 mov rdx,rsi add rsi,16 ; advance next matrix B by 16 columns mov rbx,r10 ; reload rows remaining sub rbx,2 - jb ProcessRemainingRowsN16 + jb ProcessRemainingRowsN16_2 -ProcessNextRowLoopN16: +ProcessNextRowLoopN16_2: vmovdqu xmm2,XMMWORD PTR [rdx] ; load 2 rows vmovdqu xmm3,XMMWORD PTR [rdx+r8] lea rdx,[rdx+r8*2] ; advance matrix B by 2 rows @@ -500,11 +498,11 @@ ProcessNextRowLoopN16: vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 sub rbx,2 ; subtract rows remaining - jae ProcessNextRowLoopN16 + jae ProcessNextRowLoopN16_2 -ProcessRemainingRowsN16: +ProcessRemainingRowsN16_2: add rbx,2 ; correct for over-subtract above - jz StoreColumnSumBufferN16 + jz StoreColumnSumBufferN16_2 vpmovzxbw ymm4,XMMWORD PTR [rdx] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 @@ -516,22 +514,22 @@ ProcessRemainingRowsN16: vpaddd ymm1,ymm1,ymm3 add rcx,32 ; advance matrix D by 32 bytes -StoreColumnSumBufferN16: +StoreColumnSumBufferN16_2: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 add r11,16*4 ; advance column sum buffer by 16 dwords sub r9,16 ; subtract columns remaining - jae ProcessNextColumnN16 + jae ProcessNextColumnN16_2 -ProcessRemainingColumns: +ProcessRemainingColumns_2: add r9,16 ; correct for over-subtract above - jnz ProcessColumnNUnaligned + jnz ProcessColumnNUnaligned_2 ; ; Restore non-volatile registers and return. ; -ExitRoutine: +ExitRoutine_2: vzeroupper add rsp,(GemmU8U8CopyPackBFrame.SavedRsi) @@ -546,18 +544,18 @@ ExitRoutine: ; Process the remaining columns of matrix B. ; -ProcessColumnNUnaligned: +ProcessColumnNUnaligned_2: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 sub r10,2 - jb ProcessRemainingRowsNUnaligned + jb ProcessRemainingRowsNUnaligned_2 -ProcessNextRowLoopNUnaligned: +ProcessNextRowLoopNUnaligned_2: mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8K2 + jz CopyRemainingCountNLessThan8K2_2 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] @@ -565,9 +563,9 @@ ProcessNextRowLoopNUnaligned: add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8K2: +CopyRemainingCountNLessThan8K2_2: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4K2 + jz CopyRemainingCountNLessThan4K2_2 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] @@ -575,9 +573,9 @@ CopyRemainingCountNLessThan8K2: add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4K2: +CopyRemainingCountNLessThan4K2_2: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2K2 + jz CopyRemainingCountNLessThan2K2_2 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] @@ -585,17 +583,17 @@ CopyRemainingCountNLessThan4K2: add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2K2: +CopyRemainingCountNLessThan2K2_2: test r9b,1 ; (CountN & 1) != 0? - jz ProcessPaddedMatrixBDataK2 + jz ProcessPaddedMatrixBDataK2_2 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al -ProcessPaddedMatrixBDataK2: - vmovdqu xmm2,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] - vmovdqu xmm3,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp+16] +ProcessPaddedMatrixBDataK2_2: + vmovdqu xmm2,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] + vmovdqu xmm3,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp+16] vpunpcklbw xmm4,xmm2,xmm3 ; interleave row data vpunpckhbw xmm3,xmm2,xmm3 vmovdqu XMMWORD PTR [rcx],xmm4 ; store interleaved rows @@ -609,44 +607,44 @@ ProcessPaddedMatrixBDataK2: lea rsi,[rsi+r8*2] ; advance next matrix B by 2 rows add rcx,32 ; advance matrix D by 32 bytes sub r10,2 ; subtract columns remaining - jae ProcessNextRowLoopNUnaligned + jae ProcessNextRowLoopNUnaligned_2 -ProcessRemainingRowsNUnaligned: +ProcessRemainingRowsNUnaligned_2: add r10,2 - jz StoreColumnSumBufferNUnaligned + jz StoreColumnSumBufferNUnaligned_2 mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? - jz CopyRemainingCountNLessThan8K1 + jz CopyRemainingCountNLessThan8K1_2 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination -CopyRemainingCountNLessThan8K1: +CopyRemainingCountNLessThan8K1_2: test r9b,4 ; (CountN & 4) != 0? - jz CopyRemainingCountNLessThan4K1 + jz CopyRemainingCountNLessThan4K1_2 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination -CopyRemainingCountNLessThan4K1: +CopyRemainingCountNLessThan4K1_2: test r9b,2 ; (CountN & 2) != 0? - jz CopyRemainingCountNLessThan2K1 + jz CopyRemainingCountNLessThan2K1_2 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination -CopyRemainingCountNLessThan2K1: +CopyRemainingCountNLessThan2K1_2: test r9b,1 ; (CountN & 1) != 0? - jz ProcessPaddedMatrixBDataK1 + jz ProcessPaddedMatrixBDataK1_2 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al -ProcessPaddedMatrixBDataK1: +ProcessPaddedMatrixBDataK1_2: vpmovzxbw ymm4,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 @@ -657,10 +655,10 @@ ProcessPaddedMatrixBDataK1: vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 -StoreColumnSumBufferNUnaligned: +StoreColumnSumBufferNUnaligned_2: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 - jmp ExitRoutine + jmp ExitRoutine_2 NESTED_END MlasGemmU8U8CopyPackBAvx2, _TEXT diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm index e65e43d93e671..e14e7da79afb4 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm @@ -18,12 +18,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR + EXTERN MlasMaskMoveTableAvx:PROC ; ; Stack frame layout for the Int8 kernel. @@ -31,16 +29,16 @@ INCLUDE AssembleAvxVnni.inc GemmInt8KernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) SavedR14 QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -322,11 +320,7 @@ ComputeBlockAvxVnni MACRO ColumnCount, RowCount, VectorOffset, BroadcastOffset, ; ComputeBlockLoop MACRO Isa, ColumnCount, RowCount, ASigned, BSigned - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit mov rsi,r9 ; reload row length remaining @@ -479,11 +473,7 @@ ComputeBlockU8U8Avx2 MACRO ColumnCount, RowCount, VectorOffset, BroadcastOffset ; ComputeBlockLoopU8U8 MACRO Isa, ColumnCount, RowCount - - LOCAL ComputeBlockBy2Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ExitComputeBlockLoop + LOCAL ComputeBlockBy2Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ExitComputeBlockLoop mov rsi,r9 ; reload row length remaining @@ -555,12 +545,7 @@ ExitComputeBlockLoop: ; ProduceOutputBlock MACRO ColumnCount, RowCount, ASigned, BSigned - - LOCAL SkipScaleByZeroPointB - LOCAL AccumulatorsInitialized - LOCAL ProduceWithInt8AvxVnni - LOCAL ProduceWithU8U8Avx2 - LOCAL ExitProduceOutputBlock + LOCAL SkipScaleByZeroPointB, AccumulatorsInitialized, ProduceWithInt8AvxVnni, ProduceWithU8U8Avx2, ExitProduceOutputBlock ; ; Initialize the accumulators with the row and column sums. @@ -720,16 +705,7 @@ ENDIF ; ProcessCountM MACRO RowCount, ASigned, BSigned, Fallthrough - - LOCAL ProcessNextColumnLoop16xN - LOCAL SkipAccumulateOutput16xNBlock - LOCAL OutputMasked16xNBlock - LOCAL ExitProcessCountM - LOCAL ProcessRemainingCountN - LOCAL SkipAccumulateOutput8xNBlock - LOCAL SkipAccumulateOutputMasked16xNBlock - LOCAL OutputMasked8xNBlock - LOCAL SkipAccumulateOutputMasked8xNBlock + LOCAL ProcessNextColumnLoop16xN, SkipAccumulateOutput16xNBlock, OutputMasked16xNBlock, ExitProcessCountM, ProcessRemainingCountN, SkipAccumulateOutput8xNBlock, SkipAccumulateOutputMasked16xNBlock, OutputMasked8xNBlock, SkipAccumulateOutputMasked8xNBlock cmp rbp,8 jbe ProcessRemainingCountN @@ -867,22 +843,7 @@ SkipAccumulateOutputMasked8xNBlock: ; ProcessCount1AvxVnni MACRO RowCount, ASigned, BSigned, Fallthrough - - LOCAL LProcessNextColumnLoop32xN1 - LOCAL LSkipAccumulateOutputMasked32xNBlock1 - LOCAL LProcessNextColumnLoop16xN1 - LOCAL LSkipAccumulateOutput16xNBlock1 - LOCAL LProcessRemainingCountN1 - LOCAL LSkipAccumulateOutput8xNBlock1 - LOCAL LExitProcessCountM1 - LOCAL LOutputMasked32xNBlock1 - LOCAL LSkipAccumulateOutputMasked32xNBlock1 - LOCAL LOutputMasked24xNBlock1 - LOCAL LSkipAccumulateOutputMasked24xNBlock1 - LOCAL LOutputMasked16xNBlock1 - LOCAL LSkipAccumulateOutputMasked16xNBlock1 - LOCAL LOutputMasked8xNBlock1 - LOCAL LSkipAccumulateOutputMasked8xNBlock1 + LOCAL LProcessNextColumnLoop32xN1, LSkipAccumulateOutputMasked32xNBlock1, LProcessNextColumnLoop16xN1, LSkipAccumulateOutput16xNBlock1, LProcessRemainingCountN1, LSkipAccumulateOutput8xNBlock1, LExitProcessCountM1, LOutputMasked32xNBlock1, LSkipAccumulateOutput32xNBlock1, LOutputMasked24xNBlock1, LSkipAccumulateOutputMasked24xNBlock1, LOutputMasked16xNBlock1, LSkipAccumulateOutputMasked16xNBlock1, LOutputMasked8xNBlock1, LSkipAccumulateOutputMasked8xNBlock1 cmp rbp,8 jbe LProcessRemainingCountN1 ; num of cols <= 8?: process the tail @@ -1076,6 +1037,7 @@ LSkipAccumulateOutputMasked8xNBlock1: ;-- MlasGemmInt8KernelAvx2 MACRO ASigned, BSigned + LOCAL CheckCountM6OrMore, CheckCountM4OrMore, ProcessCountM2, ProcessCountM4, ProcessCountM6, ExitKernel, ProcessCountM1, ProcessCountM1AvxVnni, ProcessCountM3, ProcessCountM5 rex_push_reg rbp push_reg rbx diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm index 9606c3ddb006f..4403bbd736169 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx512Core.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Stack frame layout for the U8X8 kernel. @@ -28,9 +26,9 @@ INCLUDE AssembleAvx512Vnni.inc GemmU8X8KernelFrame STRUCT - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) SavedR14 QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? @@ -225,11 +223,7 @@ ENDIF ; ComputeBlockLoopU8S8 MACRO Isa, ColumnCount, RowCount - - LOCAL ComputeBlockBy4Loop - LOCAL ProcessRemainingBlocks - LOCAL ComputeBlockBy1Loop - LOCAL ComputeBlockLoopExit + LOCAL ComputeBlockBy4Loop, ProcessRemainingBlocks, ComputeBlockBy1Loop, ComputeBlockLoopExit mov rsi,r9 ; reload row length remaining @@ -270,7 +264,6 @@ ComputeBlockLoopExit: ENDM ComputeBlockLoopU8U8 MACRO Isa, ColumnCount, RowCount - LOCAL ComputeBlockBy1Loop mov rsi,r9 ; reload row length remaining @@ -315,12 +308,7 @@ ENDIF ; ProduceOutputBlock MACRO ColumnCount, RowCount - - LOCAL SkipScaleByZeroPointB - LOCAL AccumulatorsInitialized - LOCAL ProduceWithU8S8Avx512Core - LOCAL ProduceWithU8U8Avx512Core - LOCAL ExitProduceOutputBlock + LOCAL SkipScaleByZeroPointB, AccumulatorsInitialized, ProduceWithU8S8Avx512Core, ProduceWithU8U8Avx512Core, ExitProduceOutputBlock ; ; Initialize the accumulators with the row and column sums. @@ -354,63 +342,63 @@ ENDIF ELSE vmovdqu32 zmm3,ZMMWORD PTR [r13] ENDIF - EmitIfCount2GE RowCount, 1, ColumnCount, 16, - EmitIfCount2GE RowCount, 1, ColumnCount, 32, - EmitIfCount2GE RowCount, 1, ColumnCount, 48, + EmitIfCount2GE RowCount, 1, ColumnCount, 16, + EmitIfCount2GE RowCount, 1, ColumnCount, 32, + EmitIfCount2GE RowCount, 1, ColumnCount, 48, EmitIfCount2GE RowCount, 1, ColumnCount, 16, EmitIfCount2GE RowCount, 1, ColumnCount, 32, EmitIfCount2GE RowCount, 1, ColumnCount, 48, - EmitIfCount2GE RowCount, 2, ColumnCount, 16, - EmitIfCount2GE RowCount, 2, ColumnCount, 32, - EmitIfCount2GE RowCount, 2, ColumnCount, 48, + EmitIfCount2GE RowCount, 2, ColumnCount, 16, + EmitIfCount2GE RowCount, 2, ColumnCount, 32, + EmitIfCount2GE RowCount, 2, ColumnCount, 48, EmitIfCount2GE RowCount, 2, ColumnCount, 16, EmitIfCount2GE RowCount, 2, ColumnCount, 32, EmitIfCount2GE RowCount, 2, ColumnCount, 48, - EmitIfCount2GE RowCount, 3, ColumnCount, 16, - EmitIfCount2GE RowCount, 3, ColumnCount, 32, - EmitIfCount2GE RowCount, 3, ColumnCount, 48, + EmitIfCount2GE RowCount, 3, ColumnCount, 16, + EmitIfCount2GE RowCount, 3, ColumnCount, 32, + EmitIfCount2GE RowCount, 3, ColumnCount, 48, EmitIfCount2GE RowCount, 3, ColumnCount, 16, EmitIfCount2GE RowCount, 3, ColumnCount, 32, EmitIfCount2GE RowCount, 3, ColumnCount, 48, - EmitIfCount2GE RowCount, 4, ColumnCount, 16, - EmitIfCount2GE RowCount, 4, ColumnCount, 32, - EmitIfCount2GE RowCount, 4, ColumnCount, 48, + EmitIfCount2GE RowCount, 4, ColumnCount, 16, + EmitIfCount2GE RowCount, 4, ColumnCount, 32, + EmitIfCount2GE RowCount, 4, ColumnCount, 48, EmitIfCount2GE RowCount, 4, ColumnCount, 16, EmitIfCount2GE RowCount, 4, ColumnCount, 32, EmitIfCount2GE RowCount, 4, ColumnCount, 48, - EmitIfCount2GE RowCount, 5, ColumnCount, 16, - EmitIfCount2GE RowCount, 5, ColumnCount, 32, - EmitIfCount2GE RowCount, 5, ColumnCount, 48, + EmitIfCount2GE RowCount, 5, ColumnCount, 16, + EmitIfCount2GE RowCount, 5, ColumnCount, 32, + EmitIfCount2GE RowCount, 5, ColumnCount, 48, EmitIfCount2GE RowCount, 5, ColumnCount, 16, EmitIfCount2GE RowCount, 5, ColumnCount, 32, EmitIfCount2GE RowCount, 5, ColumnCount, 48, - EmitIfCount2GE RowCount, 6, ColumnCount, 16, - EmitIfCount2GE RowCount, 6, ColumnCount, 32, - EmitIfCount2GE RowCount, 6, ColumnCount, 48, + EmitIfCount2GE RowCount, 6, ColumnCount, 16, + EmitIfCount2GE RowCount, 6, ColumnCount, 32, + EmitIfCount2GE RowCount, 6, ColumnCount, 48, EmitIfCount2GE RowCount, 6, ColumnCount, 16, EmitIfCount2GE RowCount, 6, ColumnCount, 32, EmitIfCount2GE RowCount, 6, ColumnCount, 48, jmp AccumulatorsInitialized SkipScaleByZeroPointB: - EmitIfCount2GE RowCount, 1, ColumnCount, 16, - EmitIfCount2GE RowCount, 1, ColumnCount, 32, - EmitIfCount2GE RowCount, 1, ColumnCount, 48, - EmitIfCount2GE RowCount, 2, ColumnCount, 16, - EmitIfCount2GE RowCount, 2, ColumnCount, 32, - EmitIfCount2GE RowCount, 2, ColumnCount, 48, - EmitIfCount2GE RowCount, 3, ColumnCount, 16, - EmitIfCount2GE RowCount, 3, ColumnCount, 32, - EmitIfCount2GE RowCount, 3, ColumnCount, 48, - EmitIfCount2GE RowCount, 4, ColumnCount, 16, - EmitIfCount2GE RowCount, 4, ColumnCount, 32, - EmitIfCount2GE RowCount, 4, ColumnCount, 48, - EmitIfCount2GE RowCount, 5, ColumnCount, 16, - EmitIfCount2GE RowCount, 5, ColumnCount, 32, - EmitIfCount2GE RowCount, 5, ColumnCount, 48, - EmitIfCount2GE RowCount, 6, ColumnCount, 16, - EmitIfCount2GE RowCount, 6, ColumnCount, 32, - EmitIfCount2GE RowCount, 6, ColumnCount, 48, + EmitIfCount2GE RowCount, 1, ColumnCount, 16, + EmitIfCount2GE RowCount, 1, ColumnCount, 32, + EmitIfCount2GE RowCount, 1, ColumnCount, 48, + EmitIfCount2GE RowCount, 2, ColumnCount, 16, + EmitIfCount2GE RowCount, 2, ColumnCount, 32, + EmitIfCount2GE RowCount, 2, ColumnCount, 48, + EmitIfCount2GE RowCount, 3, ColumnCount, 16, + EmitIfCount2GE RowCount, 3, ColumnCount, 32, + EmitIfCount2GE RowCount, 3, ColumnCount, 48, + EmitIfCount2GE RowCount, 4, ColumnCount, 16, + EmitIfCount2GE RowCount, 4, ColumnCount, 32, + EmitIfCount2GE RowCount, 4, ColumnCount, 48, + EmitIfCount2GE RowCount, 5, ColumnCount, 16, + EmitIfCount2GE RowCount, 5, ColumnCount, 32, + EmitIfCount2GE RowCount, 5, ColumnCount, 48, + EmitIfCount2GE RowCount, 6, ColumnCount, 16, + EmitIfCount2GE RowCount, 6, ColumnCount, 32, + EmitIfCount2GE RowCount, 6, ColumnCount, 48, AccumulatorsInitialized: @@ -480,16 +468,7 @@ ENDIF ; ProcessCountM MACRO RowCount - - LOCAL ProcessNextColumnLoop32xN - LOCAL Output32xNBlock - LOCAL SkipAccumulateOutput32xNBlock - LOCAL Output16xNBlock - LOCAL Output16xNBlockWithMask - LOCAL SkipAccumulateOutput16xNBlockWithMask - LOCAL ProcessRemainingCountN - LOCAL ProcessNextColumnLoop48xN - LOCAL SkipAccumulateOutput48xNBlock + LOCAL ProcessNextColumnLoop32xN, Output32xNBlock, SkipAccumulateOutput32xNBlock, Output16xNBlock, Output16xNBlockWithMask, SkipAccumulateOutput16xNBlockWithMask, ProcessRemainingCountN, ProcessNextColumnLoop48xN, SkipAccumulateOutput48xNBlock cmp rbp,32 ja ProcessNextColumnLoop48xN diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm index 3a6f83d2c2ada..a33f76980d14f 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx2.asm @@ -17,12 +17,10 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasTranspose4x4BytesAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasTranspose4x4BytesAvx:PROC ; ; Stack frame layout for the U8S8 kernel. @@ -30,7 +28,7 @@ INCLUDE mlasi.inc GemvU8S8KernelFrame STRUCT - SavedXmm6 OWORD ? + SavedXmm6 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm index c1727b3e34d25..831f01c4b3363 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Core.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE QgemvU8S8KernelAvx512Common.inc - .list ; ; Generate the GEMV kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm index cb175881f556e..fbfab44aebeb4 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvx512Vnni.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE QgemvU8S8KernelAvx512Common.inc INCLUDE AssembleAvx512Vnni.inc - .list ; ; Generate the GEMV kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm index be13dcebbc580..329c3d7330908 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemvU8S8KernelAvxVnni.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE AssembleAvxVnni.inc - .list - EXTERN MlasMaskMoveAvx:NEAR - EXTERN MlasTranspose4x4BytesAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC + EXTERN MlasTranspose4x4BytesAvx:PROC ; ; Stack frame layout for the U8S8 kernel. @@ -31,11 +29,11 @@ INCLUDE AssembleAvxVnni.inc GemvU8S8KernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) Padding QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm index dd6c12e2caecf..17cd63dd4c46a 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelAvxCommon.inc - .list ; ; Macro Description: @@ -136,11 +134,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy2 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCount, ProcessNextOutputCountBy2, ProcessRemainingOutputCount, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -176,7 +170,7 @@ ProcessRemainingOutputCount: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleAvxFilter&FilterCount ENDM @@ -209,9 +203,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy2 - LOCAL ProcessRemainingOutputCount + LOCAL ProcessNextOutputCountBy2, ProcessRemainingOutputCount sub r10,2 jb ProcessRemainingOutputCount @@ -224,7 +216,7 @@ ProcessNextOutputCountBy2: ProcessRemainingOutputCount: add r10,2 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise ProcessPointwiseOutputCountN Avx, 8, FilterCount, 1 ENDM @@ -258,7 +250,19 @@ SconvKernelPointwiseFunction Avx, BiasFilter LEAF_ENTRY MlasConvPostProcessFloatAvxFilter&FilterCount&Output&OutputCount, _TEXT PUBLIC MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount +; +; This alternate entry point shares the AVX post-processing code with the FMA3 +; kernel. ml64 requires the double-colon form (a module-visible label defined +; inside a PROC under the default OPTION SCOPED); llvm-ml does not parse "::", +; but treats every label as global, so a single-colon label plus PUBLIC is +; equivalent. LLVM_ML is defined on the llvm-ml command line (see the MLAS +; cross-assembler CI workflow). +; +IFDEF LLVM_ML +MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount: +ELSE MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:: +ENDIF IF FilterCount GT 2 lea rbx,[r8+rax*2] ; compute output plus 2 rows @@ -270,7 +274,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, EmitIfCount2GE FilterCount, 1, OutputCount, 3, @@ -284,14 +288,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 2, EmitIfCount2GE FilterCount, 4, OutputCount, 3, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount IF OutputCount EQ 1 EmitIfCountGE FilterCount, 1, EmitIfCountGE FilterCount, 2, @@ -316,14 +320,14 @@ ELSE EmitIfCount2GE FilterCount, 4, OutputCount, 3, ENDIF -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount vxorps xmm15,xmm15,xmm15 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, @@ -338,7 +342,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 4, OutputCount, 2, EmitIfCount2GE FilterCount, 4, OutputCount, 3, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm index 43eaa88e49946..38d8c933c01da 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelAvx512F.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelCommon.inc - .list ; ; Macro Description: @@ -117,12 +115,12 @@ IFIDNI , ELSE IF FilterCount EQ 1 vmovups zmm24,ZMMWORD PTR [rdx+VectorOffset] - EmitIfCountGE OutputCount, 1, - EmitIfCountGE OutputCount, 2, - EmitIfCountGE OutputCount, 3, - EmitIfCountGE OutputCount, 4, - EmitIfCountGE OutputCount, 5, - EmitIfCountGE OutputCount, 6, + EmitIfCountGE OutputCount, 1, + EmitIfCountGE OutputCount, 2, + EmitIfCountGE OutputCount, 3, + EmitIfCountGE OutputCount, 4, + EmitIfCountGE OutputCount, 5, + EmitIfCountGE OutputCount, 6, ELSE EmitIfCountGE OutputCount, 1, EmitIfCountGE OutputCount, 2, @@ -203,13 +201,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy6 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCountLessThan3 - LOCAL ProcessRemainingOutputCount1 - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCount, ProcessNextOutputCountBy6, ProcessRemainingOutputCount, ProcessRemainingOutputCountLessThan3, ProcessRemainingOutputCount1, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -261,7 +253,7 @@ ProcessRemainingOutputCountLessThan3: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleAvx512FFilter&FilterCount ENDM @@ -294,11 +286,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy6 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCountLessThan3 - LOCAL ProcessRemainingOutputCount1 + LOCAL ProcessNextOutputCountBy6, ProcessRemainingOutputCount, ProcessRemainingOutputCountLessThan3, ProcessRemainingOutputCount1 sub r10,6 jb ProcessRemainingOutputCount @@ -312,20 +300,20 @@ ProcessNextOutputCountBy6: ProcessRemainingOutputCount: add r10,6 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise cmp r10,3 jb ProcessRemainingOutputCountLessThan3 ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 3 lea rax,[r9*2+r9] add rdi,rax ; advance input by 3 elements sub r10,3 - jz ExitKernel + jz ExitKernelPointwise ProcessRemainingOutputCountLessThan3: cmp r10,2 jb ProcessRemainingOutputCount1 ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessRemainingOutputCount1: ProcessPointwiseOutputCountN Avx512F, 16, FilterCount, 1 @@ -373,7 +361,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, EmitIfCount2GE FilterCount, 1, OutputCount, 3, @@ -399,14 +387,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 5, EmitIfCount2GE FilterCount, 4, OutputCount, 6, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount IF OutputCount EQ 1 EmitIfCountGE FilterCount, 1, EmitIfCountGE FilterCount, 2, @@ -443,14 +431,14 @@ ELSE EmitIfCount2GE FilterCount, 4, OutputCount, 6, ENDIF -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount vpxord zmm24,zmm24,zmm24 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 2, @@ -477,7 +465,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 2, OutputCount, 5, EmitIfCount2GE FilterCount, 2, OutputCount, 6, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc index 2f5f997c6f12c..acbe2f5f575d2 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelCommon.inc @@ -30,16 +30,16 @@ MLAS_CONV_KERNEL_FLAG_OTHER_ACTIVATION EQU 00000008h SconvKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -75,22 +75,22 @@ SconvKernelFrame ENDS SconvKernelSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SconvKernelFrame <> + KernelFrame SconvKernelFrame {} SconvKernelSingleFrame ENDS SconvKernelDepthwiseFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR13 QWORD ? @@ -123,22 +123,22 @@ SconvKernelDepthwiseFrame ENDS SconvKernelDepthwiseSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SconvKernelDepthwiseFrame <> + KernelFrame SconvKernelDepthwiseFrame {} SconvKernelDepthwiseSingleFrame ENDS SconvKernelPointwiseFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? SavedR12 QWORD ? SavedR14 QWORD ? @@ -204,11 +204,7 @@ SconvKernelPointwiseFrame ENDS ; ProcessOutputCountN MACRO Isa, KernelFrame, KernelType, BlockSize, FilterCount, OutputCount - - LOCAL ProcessNextRow - LOCAL ProcessNextColumn - LOCAL HandlePostProcessing - LOCAL SkipOverPadding + LOCAL ProcessNextRow, ProcessNextColumn, HandlePostProcessing, SkipOverPadding mov rcx,rdi IFIDNI , @@ -307,6 +303,7 @@ ENDIF ; SconvKernelFunction MACRO KernelType, BlockSize, Isa, BiasFilter + LOCAL ProcessFilterCount3, ProcessFilterCountLessThan3, ProcessFilterCount1 ;++ ; @@ -424,17 +421,17 @@ ENDIF je ProcessFilterCount3 jb ProcessFilterCountLessThan3 ProcessFilterCountN SconvKernelFrame, KernelType, 4 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCount3: ProcessFilterCountN SconvKernelFrame, KernelType, 3 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCountLessThan3: cmp r11,2 jb ProcessFilterCount1 ProcessFilterCountN SconvKernelFrame, KernelType, 2 - jmp ExitKernel + jmp ExitKernel&KernelType& ProcessFilterCount1: ProcessFilterCountN SconvKernelFrame, KernelType, 1 @@ -443,7 +440,7 @@ ProcessFilterCount1: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernel&KernelType&: IFDIFI , vzeroupper ENDIF @@ -483,11 +480,11 @@ IFDIFI , LEAF_ENTRY MlasConv&KernelType&FloatSingle&Isa&Filter&FilterCount, _TEXT -ProcessNextOutputCount: +@@: ProcessOutputCountN Isa, SconvKernelSingleFrame.KernelFrame, KernelType, BlockSize, FilterCount, 1 add rdi,r9 ; advance input by 1 element dec r10 ; decrement output count remaining - jnz ProcessNextOutputCount + jnz @B ret LEAF_END MlasConv&KernelType&FloatSingle&Isa&Filter&FilterCount, _TEXT @@ -621,7 +618,7 @@ SconvKernelDepthwiseFunction MACRO BlockSize, Isa ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernelDepthwise: IFDIFI , vzeroupper ENDIF @@ -659,11 +656,11 @@ IFDIFI , LEAF_ENTRY MlasConvDepthwiseFloatSingle&Isa&Filter1, _TEXT -ProcessNextOutputCount: +@@: ProcessOutputCountN Isa, SconvKernelDepthwiseSingleFrame.KernelFrame, Depthwise, BlockSize, 1, 1 add rdi,r9 ; advance input by 1 element dec r10 ; decrement output count remaining - jnz ProcessNextOutputCount + jnz @B ret LEAF_END MlasConvDepthwiseFloatSingle&Isa&Filter1, _TEXT @@ -705,11 +702,7 @@ ENDIF ; ProcessPointwiseOutputCountN MACRO Isa, BlockSize, FilterCount, OutputCount - - LOCAL ProcessNextInputBlock - LOCAL SkipAccumulateOutput - LOCAL SkipBiasAddition - LOCAL SkipReluActivation + LOCAL ProcessNextInputBlock, SkipAccumulateOutput, SkipBiasAddition, SkipReluActivation mov rcx,rdi mov rdx,r12 @@ -769,6 +762,7 @@ ENDIF ;-- SconvKernelPointwiseFunction MACRO Isa, BiasFilter + LOCAL ProcessFilterCount3, ProcessFilterCountLessThan3, ProcessFilterCount1 ;++ ; @@ -859,17 +853,17 @@ ENDIF je ProcessFilterCount3 jb ProcessFilterCountLessThan3 ProcessPointwiseFilterCountN 4 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCount3: ProcessPointwiseFilterCountN 3 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCountLessThan3: cmp r11,2 jb ProcessFilterCount1 ProcessPointwiseFilterCountN 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessFilterCount1: ProcessPointwiseFilterCountN 1 @@ -878,7 +872,7 @@ ProcessFilterCount1: ; Restore non-volatile registers and return. ; -ExitKernel: +ExitKernelPointwise: IFDIFI , vzeroupper ENDIF diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm index 38b260b4df93d..714aa3b62787d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelFma3.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelAvxCommon.inc - .list ; ; Share the post process functions with the AVX implementation. @@ -29,7 +27,7 @@ INCLUDE SconvKernelAvxCommon.inc IRP FilterCount, <1, 2, 3, 4> IRP OutputCount, <1, 2, 3> - EXTERN MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:NEAR + EXTERN MlasConvPostProcessFloatFma3Filter&FilterCount&Output&OutputCount:PROC ENDM ENDM @@ -73,9 +71,9 @@ ComputeBlock MACRO KernelType, FilterCount, OutputCount, VectorOffset, Broadcast IFIDNI , vmovups ymm12,YMMWORD PTR [rdx] - EmitIfCountGE OutputCount, 1, - EmitIfCountGE OutputCount, 2, - EmitIfCountGE OutputCount, 3, + EmitIfCountGE OutputCount, 1, + EmitIfCountGE OutputCount, 2, + EmitIfCountGE OutputCount, 3, ELSE EmitIfCountGE OutputCount, 1, EmitIfCountGE OutputCount, 2, @@ -140,13 +138,7 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - - LOCAL ProcessOutputCountLeftPad - LOCAL ProcessOutputCount - LOCAL ProcessNextOutputCountBy3 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCount1 - LOCAL ProcessOutputCountRightPadAndRemaining + LOCAL ProcessOutputCountLeftPad, ProcessOutputCount, ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessRemainingOutputCount1, ProcessOutputCountRightPadAndRemaining ; ; Process the output blocks that include left padding. @@ -189,7 +181,7 @@ ProcessRemainingOutputCount: ProcessOutputCountRightPadAndRemaining: add r10,KernelFrame.OutputCountRightPad[rsp] - jz ExitKernel + jz ExitKernel&KernelType& call MlasConv&KernelType&FloatSingleFma3Filter&FilterCount ENDM @@ -222,10 +214,7 @@ ProcessOutputCountRightPadAndRemaining: ; ProcessPointwiseFilterCountN MACRO FilterCount - - LOCAL ProcessNextOutputCountBy3 - LOCAL ProcessRemainingOutputCount - LOCAL ProcessRemainingOutputCount1 + LOCAL ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessRemainingOutputCount1 sub r10,3 jb ProcessRemainingOutputCount @@ -239,11 +228,11 @@ ProcessNextOutputCountBy3: ProcessRemainingOutputCount: add r10,3 ; correct for over-subtract above - jz ExitKernel + jz ExitKernelPointwise cmp r10,2 jb ProcessRemainingOutputCount1 ProcessPointwiseOutputCountN Fma3, 8, FilterCount, 2 - jmp ExitKernel + jmp ExitKernelPointwise ProcessRemainingOutputCount1: ProcessPointwiseOutputCountN Fma3, 8, FilterCount, 1 diff --git a/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm index 57eb402ff0c16..3fcdbc5b727f0 100644 --- a/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SconvKernelSse2.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SconvKernelCommon.inc - .list ; ; Macro Description: @@ -159,7 +157,6 @@ ENDIF ; ProcessFilterCountN MACRO KernelFrame, KernelType, FilterCount - LOCAL ProcessNextOutputCount mov r10,KernelFrame.OutputCountLeftPad[rsp] @@ -202,7 +199,6 @@ ProcessNextOutputCount: ; ProcessPointwiseFilterCountN MACRO FilterCount - LOCAL ProcessNextOutputCount ProcessNextOutputCount: @@ -251,7 +247,7 @@ ENDIF ; test dl,MLAS_CONV_KERNEL_FLAG_ACCUMULATE_OUTPUT - jz SkipAccumulateOutput + jz SkipAccumulateOutput&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 2, OutputCount, 1, @@ -269,14 +265,14 @@ ENDIF EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipAccumulateOutput: +SkipAccumulateOutput&FilterCount&Output&OutputCount: ; ; Test if the bias buffer should be accumulated with the output block. ; test dl,MLAS_CONV_KERNEL_FLAG_BIAS_ADDITION - jz SkipBiasAddition + jz SkipBiasAddition&FilterCount&Output&OutputCount EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 2, OutputCount, 1, @@ -294,14 +290,14 @@ SkipAccumulateOutput: EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipBiasAddition: +SkipBiasAddition&FilterCount&Output&OutputCount: ; ; Test for fused ReLU activation. ; test dl,MLAS_CONV_KERNEL_FLAG_RELU_ACTIVATION - jz SkipReluActivation + jz SkipReluActivation&FilterCount&Output&OutputCount xorps xmm15,xmm15 EmitIfCount2GE FilterCount, 1, OutputCount, 1, EmitIfCount2GE FilterCount, 1, OutputCount, 1, @@ -312,7 +308,7 @@ SkipBiasAddition: EmitIfCount2GE FilterCount, 4, OutputCount, 1, EmitIfCount2GE FilterCount, 4, OutputCount, 1, -SkipReluActivation: +SkipReluActivation&FilterCount&Output&OutputCount: ; ; Store the output block in the output buffer. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm index 9ba503df16805..bc186f1f9845d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelAvxCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm index d59880a1f552a..7021248cf646a 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelAvx512FCommon.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc index f7bc7870bb3ae..ea09545329c43 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelCommon.inc @@ -21,8 +21,9 @@ FgemmElementShift EQU 2 FgemmElementSize EQU (1 SHL FgemmElementShift) -FgemmElementPtr EQU DWORD PTR -FgemmElementBcst EQU DWORD BCST +FgemmElementPtr TEXTEQU +FgemmElementBcst TEXTEQU +FgemmElementBcstSuffix TEXTEQU <{1to16}> ; ; Define the typed instructions for single precision. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm index 3651ad18f5333..b106ccc8a1c20 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelFma3.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelFma3Common.inc - .list ; ; Generate the GEMM kernel. diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm index 418c8e332eb46..a10f9fce7ba86 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelM1Avx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMaskMoveAvx:NEAR + EXTERN MlasMaskMoveAvx:PROC ; ; Stack frame layout for the SGEMM M=1 kernels. @@ -29,9 +27,9 @@ INCLUDE mlasi.inc SgemmKernelM1Frame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? @@ -384,9 +382,9 @@ ProcessRemainingCountN1: ; sub rbp,4 - jb ProcessRemainingCountN + jb ProcessRemainingCountN_2 -ProcessRowLoop4: +ProcessRowLoop4_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators vxorps xmm3,xmm3,xmm3 vxorps xmm4,xmm4,xmm4 @@ -396,9 +394,9 @@ ProcessRowLoop4: mov rax,r9 ; reload CountK lea r11,[rdx+rbx*4] ; advance matrix B by 4 rows sub rax,8 - jb ProcessRemainingCountK4 + jb ProcessRemainingCountK4_2 -ProcessColumnLoop4: +ProcessColumnLoop4_2: lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] @@ -412,11 +410,11 @@ ProcessColumnLoop4: add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop4 + jae ProcessColumnLoop4_2 -ProcessRemainingCountK4: +ProcessRemainingCountK4_2: test al,7 ; test for unaligned columns - jz Output4x1Block + jz Output4x1Block_2 lea rsi,[rdx+rbx*2] ; compute matrix B plus 2 rows vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] @@ -436,7 +434,7 @@ ProcessRemainingCountK4: ; Reduce and output the row accumulators. ; -Output4x1Block: +Output4x1Block_2: vunpcklps ymm6,ymm2,ymm3 ; transpose row accumulators vunpckhps ymm1,ymm2,ymm3 vunpcklps ymm2,ymm4,ymm5 @@ -455,15 +453,15 @@ Output4x1Block: vmovups XMMWORD PTR [r8],xmm4 add r8,4*4 ; advance matrix C by 4 columns sub rbp,4 - jae ProcessRowLoop4 + jae ProcessRowLoop4_2 -ProcessRemainingCountN: +ProcessRemainingCountN_2: test ebp,2 - jnz ProcessRowLoop2 + jnz ProcessRowLoop2_2 test ebp,1 - jnz ProcessRowLoop1 + jnz ProcessRowLoop1_2 -ExitKernel: +ExitKernel_2: vzeroupper movaps xmm6,SgemmKernelM1Frame.SavedXmm6[rsp] movaps xmm7,SgemmKernelM1Frame.SavedXmm7[rsp] @@ -480,7 +478,7 @@ ExitKernel: ; Process 2 rows of the matrices. ; -ProcessRowLoop2: +ProcessRowLoop2_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators vxorps xmm3,xmm3,xmm3 mov rcx,r10 ; reload matrix A @@ -488,9 +486,9 @@ ProcessRowLoop2: mov rax,r9 ; reload CountK lea r11,[rdx+rbx*2] ; advance matrix B by 2 rows sub rax,8 - jb ProcessRemainingCountK2 + jb ProcessRemainingCountK2_2 -ProcessColumnLoop2: +ProcessColumnLoop2_2: vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] vaddps ymm2,ymm2,ymm6 @@ -499,11 +497,11 @@ ProcessColumnLoop2: add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop2 + jae ProcessColumnLoop2_2 -ProcessRemainingCountK2: +ProcessRemainingCountK2_2: test al,7 ; test for unaligned columns - jz Output2x1Block + jz Output2x1Block_2 vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] vmulps ymm6,ymm1,ymm6 @@ -516,7 +514,7 @@ ProcessRemainingCountK2: ; Reduce and output the row accumulators. ; -Output2x1Block: +Output2x1Block_2: vunpcklps ymm4,ymm2,ymm3 ; reduce row accumulators vunpckhps ymm2,ymm2,ymm3 vaddps ymm2,ymm2,ymm4 @@ -530,32 +528,32 @@ Output2x1Block: vmovsd QWORD PTR [r8],xmm2 add r8,2*4 ; advance matrix C by 2 columns test ebp,1 - jz ExitKernel + jz ExitKernel_2 ; ; Process 1 row of the matrices. ; -ProcessRowLoop1: +ProcessRowLoop1_2: vxorps xmm2,xmm2,xmm2 ; clear row accumulators mov rcx,r10 ; reload matrix A mov rdx,r11 ; reload matrix B mov rax,r9 ; reload CountK sub rax,8 - jb ProcessRemainingCountK1 + jb ProcessRemainingCountK1_2 -ProcessColumnLoop1: +ProcessColumnLoop1_2: vmovups ymm1,YMMWORD PTR [rcx] vmulps ymm6,ymm1,YMMWORD PTR [rdx] vaddps ymm2,ymm2,ymm6 add rcx,8*4 ; advance matrix A by 8 columns add rdx,8*4 ; advance matrix B by 8 columns sub rax,8 - jae ProcessColumnLoop1 + jae ProcessColumnLoop1_2 -ProcessRemainingCountK1: +ProcessRemainingCountK1_2: test al,7 ; test for unaligned columns - jz Output1x1Block + jz Output1x1Block_2 vmaskmovps ymm1,ymm7,YMMWORD PTR [rcx] vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx] vmulps ymm6,ymm1,ymm6 @@ -565,7 +563,7 @@ ProcessRemainingCountK1: ; Reduce and output the row accumulators. ; -Output1x1Block: +Output1x1Block_2: vhaddps ymm2,ymm2,ymm2 ; reduce row accumulators vhaddps ymm2,ymm2,ymm2 vextractf128 xmm4,ymm2,1 @@ -574,7 +572,7 @@ Output1x1Block: vandnps xmm3,xmm0,xmm3 vaddss xmm2,xmm2,xmm3 vmovss DWORD PTR [r8],xmm2 - jmp ExitKernel + jmp ExitKernel_2 NESTED_END MlasSgemmKernelM1TransposeBAvx, _TEXT diff --git a/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm index 4146a0e37646e..4e6c50180b9c8 100644 --- a/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SgemmKernelSse2.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SgemmKernelCommon.inc INCLUDE FgemmKernelSse2Common.inc - .list ; ; Macro Description: @@ -122,19 +120,7 @@ ENDIF ; ProcessCountM MACRO RowCount, Fallthrough - - LOCAL ProcessNextColumnLoop16xN - LOCAL Compute16xNBlockBy4Loop - LOCAL ProcessRemaining16xNBlocks - LOCAL Compute16xNBlockBy1Loop - LOCAL Output16xNBlock - LOCAL OutputPartial16xNBlock - LOCAL OutputPartialLessThan12xNBlock - LOCAL OutputPartialLessThan8xNBlock - LOCAL OutputPartialLessThan4xNBlock - LOCAL SkipAccumulateOutput2xN - LOCAL OutputPartial1xNBlock - LOCAL SkipAccumulateOutput1xN + LOCAL ProcessNextColumnLoop16xN, Compute16xNBlockBy4Loop, ProcessRemaining16xNBlocks, Compute16xNBlockBy1Loop, Output16xNBlock, OutputPartial16xNBlock, OutputPartialLessThan12xNBlock, OutputPartialLessThan8xNBlock, OutputPartialLessThan4xNBlock, SkipAccumulateOutput2xN, OutputPartial1xNBlock, SkipAccumulateOutput1xN ProcessNextColumnLoop16xN: EmitIfCountGE RowCount, 1, diff --git a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm index bfdff7009191e..ca6edd8f56ff9 100644 --- a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMinimumF32Value:NEAR + EXTERN MlasMinimumF32Value:PROC ;++ ; @@ -123,9 +121,9 @@ ExitKernel: vbroadcastss ymm4,DWORD PTR [r8] ; broadcast scale value cmp rdx,32 - jb ProcessRemainingCountBy8 + jb ProcessRemainingCountBy8_2 -ProcessRemainingCountBy32: +ProcessRemainingCountBy32_2: vmulps ymm0,ymm4,YMMWORD PTR [rcx] vmulps ymm1,ymm4,YMMWORD PTR [rcx+8*4] sub rdx,32 @@ -137,29 +135,29 @@ ProcessRemainingCountBy32: vmovups YMMWORD PTR [rcx+24*4],ymm3 add rcx,32*4 ; advance output by 32 elements cmp rdx,32 - jae ProcessRemainingCountBy32 + jae ProcessRemainingCountBy32_2 -ProcessRemainingCountBy8: +ProcessRemainingCountBy8_2: cmp rdx,8 - jb ProcessRemainingCountLessThan8 + jb ProcessRemainingCountLessThan8_2 vmulps ymm0,ymm4,YMMWORD PTR [rcx] sub rdx,8 vmovups YMMWORD PTR [rcx],ymm0 add rcx,8*4 ; advance output by 8 elements - jmp ProcessRemainingCountBy8 + jmp ProcessRemainingCountBy8_2 -ProcessRemainingCountLessThan8: +ProcessRemainingCountLessThan8_2: test rdx,rdx - jz ExitKernel + jz ExitKernel_2 -ProcessRemainingCountBy1: +ProcessRemainingCountBy1_2: vmulss xmm0,xmm4,DWORD PTR [rcx] vmovss DWORD PTR [rcx],xmm0 add rcx,4 ; advance output by 1 element dec edx - jnz ProcessRemainingCountBy1 + jnz ProcessRemainingCountBy1_2 -ExitKernel: +ExitKernel_2: vzeroupper ret @@ -194,9 +192,9 @@ ExitKernel: vbroadcastss ymm4,DWORD PTR [r9] ; broadcast negative minimum value vbroadcastss ymm5,DWORD PTR [r9+4] ; broadcast log(SumExp) cmp r8,32 - jb ProcessRemainingCountBy8 + jb ProcessRemainingCountBy8_3 -ProcessRemainingCountBy32: +ProcessRemainingCountBy32_3: vaddps ymm0,ymm4,YMMWORD PTR [rcx] vaddps ymm1,ymm4,YMMWORD PTR [rcx+8*4] sub r8,32 @@ -213,33 +211,33 @@ ProcessRemainingCountBy32: vmovups YMMWORD PTR [rdx+24*4],ymm3 add rdx,32*4 ; advance output by 32 elements cmp r8,32 - jae ProcessRemainingCountBy32 + jae ProcessRemainingCountBy32_3 -ProcessRemainingCountBy8: +ProcessRemainingCountBy8_3: cmp r8,8 - jb ProcessRemainingCountLessThan8 + jb ProcessRemainingCountLessThan8_3 vaddps ymm0,ymm4,YMMWORD PTR [rcx] add rcx,8*4 ; advance input by 8 elements vsubps ymm0,ymm0,ymm5 ; do as two steps for numeric stability sub r8,8 vmovups YMMWORD PTR [rdx],ymm0 add rdx,8*4 ; advance output by 8 elements - jmp ProcessRemainingCountBy8 + jmp ProcessRemainingCountBy8_3 -ProcessRemainingCountLessThan8: +ProcessRemainingCountLessThan8_3: test r8,r8 - jz ExitKernel + jz ExitKernel_3 -ProcessRemainingCountBy1: +ProcessRemainingCountBy1_3: vaddss xmm0,xmm4,DWORD PTR [rcx] add rcx,4 ; advance input by 1 element vsubss xmm0,xmm0,xmm5 vmovss DWORD PTR [rdx],xmm0 add rdx,4 ; advance output by 1 element dec r8d - jnz ProcessRemainingCountBy1 + jnz ProcessRemainingCountBy1_3 -ExitKernel: +ExitKernel_3: vzeroupper ret diff --git a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm index 3e83bc852f558..a65e3b872d2a0 100644 --- a/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SoftmaxKernelAvx512F.asm @@ -17,11 +17,9 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - EXTERN MlasMinimumF32Value:NEAR + EXTERN MlasMinimumF32Value:PROC ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm index 3d0f64f447f8e..7a60386e7301b 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelAvxCommon.inc - .list ; ; Macro Description: @@ -44,7 +42,7 @@ IFIDNI , imul rax,SpoolKernelFrame.KernelWidth[rsp] vcvtsi2ss xmm5,xmm5,rax ELSE - vcvtsi2ss xmm5,xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + vcvtsi2ss xmm5,xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] ENDIF vshufps xmm5,xmm5,xmm5,0 vinsertf128 ymm5,ymm5,xmm5,1 ; AVX lacks "vbroadcastss ymm5,xmm5" diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm index ca1fb690194d3..5159c8685d00d 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvx512F.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelAvxCommon.inc - .list ; ; Macro Description: @@ -44,7 +42,7 @@ IFIDNI , imul rax,SpoolKernelFrame.KernelWidth[rsp] vcvtsi2ss xmm5,xmm5,rax ELSE - vcvtsi2ss xmm5,xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + vcvtsi2ss xmm5,xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] ENDIF vbroadcastss zmm5,xmm5 ENDIF diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc index fd082af2bf3fe..ac7ffa04e4256 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelAvxCommon.inc @@ -30,6 +30,7 @@ INCLUDE SpoolKernelCommon.inc ; SpoolKernelFunction MACRO PoolingType, Isa + LOCAL ProcessOutputCountLeftPad, ProcessOutputCount, ProcessNextOutputCountBy3, ProcessRemainingOutputCount, ProcessOutputCountRightPad, ExitKernel, ProcessNextOutputCount ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc index 3f735bb27cde2..51f77defabc22 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelCommon.inc @@ -49,7 +49,7 @@ SpoolKernelFrame ENDS SpoolKernelSingleFrame STRUCT ReturnAddress QWORD ? - KernelFrame SpoolKernelFrame <> + KernelFrame SpoolKernelFrame {} SpoolKernelSingleFrame ENDS @@ -136,11 +136,7 @@ SpoolKernelExit MACRO ; ProcessOutputCountN MACRO KernelFrame, PoolingType, OutputCount - - LOCAL ProcessNextRow - LOCAL ProcessNextColumn - LOCAL SkipOverPadding - LOCAL HandlePostProcessing + LOCAL ProcessNextRow, ProcessNextColumn, SkipOverPadding, HandlePostProcessing mov rcx,rdi mov r11,KernelFrame.KernelHeight[rsp] diff --git a/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm b/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm index d5c0ef8c9af42..3364d202cd3ff 100644 --- a/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm +++ b/onnxruntime/core/mlas/lib/amd64/SpoolKernelSse2.asm @@ -17,10 +17,8 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE SpoolKernelCommon.inc - .list ; ; Macro Description: @@ -41,7 +39,7 @@ IFIDNI , ENDIF IFIDNI , - cvtsi2ss xmm5,SpoolKernelFrame.ActualKernelSize[rsp] + cvtsi2ss xmm5,QWORD PTR SpoolKernelFrame.ActualKernelSize[rsp] shufps xmm5,xmm5,0 ENDIF @@ -200,6 +198,7 @@ ENDIF ; SpoolKernelFunction MACRO PoolingType, Isa + LOCAL ProcessNextOutputCount, ExitKernel ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm index 6d94d533d72ad..f48bc84216d68 100644 --- a/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/TanhKernelFma3.asm @@ -17,13 +17,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasTanhConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasTanhConstants:PROC ;++ ; diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm b/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm index 2a34d7884c0c6..ade8cf0ec1fd9 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelAvx512F.asm @@ -16,13 +16,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasExpConstants:NEAR - EXTERN MlasOpmask16BitTableAvx512:NEAR + EXTERN MlasExpConstants:PROC + EXTERN MlasOpmask16BitTableAvx512:PROC ;++ ; @@ -153,9 +151,9 @@ ExitKernel: vbroadcastss zmm19,DWORD PTR [r9] ; broadcast negative maximum value vpxord zmm20,zmm20,zmm20 ; clear exp() accumulator sub r8,48 - jb ProcessRemainingCount + jb ProcessRemainingCount_2 -ComputeExpBy48Loop: +ComputeExpBy48Loop_2: vaddps zmm0,zmm19,ZMMWORD PTR [rcx] ; bias by negative maximum value vaddps zmm3,zmm19,ZMMWORD PTR [rcx+64] vaddps zmm16,zmm19,ZMMWORD PTR [rcx+128] @@ -206,29 +204,29 @@ ComputeExpBy48Loop: vaddps zmm20,zmm20,zmm17 add rcx,48*4 ; advance input by 48 elements test rdx,rdx - jz SkipStoreResultsBy48 + jz SkipStoreResultsBy48_2 vmovups ZMMWORD PTR [rdx],zmm1 vmovups ZMMWORD PTR [rdx+64],zmm4 vmovups ZMMWORD PTR [rdx+128],zmm17 add rdx,48*4 ; advance output by 48 elements -SkipStoreResultsBy48: +SkipStoreResultsBy48_2: sub r8,48 - jae ComputeExpBy48Loop + jae ComputeExpBy48Loop_2 -ProcessRemainingCount: +ProcessRemainingCount_2: add r8,48 ; correct for over-subtract above - jz ReduceAccumulator + jz ReduceAccumulator_2 mov eax,-1 kmovw k1,eax ; update mask to access all elements -ComputeExpBy16Loop: +ComputeExpBy16Loop_2: cmp r8,16 - jae ProcessSingleVector + jae ProcessSingleVector_2 lea r10,MlasOpmask16BitTableAvx512 kmovw k1,WORD PTR [r10+r8*2] -ProcessSingleVector: +ProcessSingleVector_2: vaddps zmm0{k1}{z},zmm19,ZMMWORD PTR [rcx] ; bias by negative maximum value vmaxps zmm0,zmm21,zmm0 ; clamp lower bound @@ -248,15 +246,15 @@ ProcessSingleVector: vaddps zmm20{k1},zmm20,zmm1 ; accumulate exp() results add rcx,16*4 ; advance input by 16 elements test rdx,rdx - jz SkipStoreResultsBy16 + jz SkipStoreResultsBy16_2 vmovups ZMMWORD PTR [rdx]{k1},zmm1 add rdx,16*4 ; advance output by 16 elements -SkipStoreResultsBy16: +SkipStoreResultsBy16_2: sub r8,16 - ja ComputeExpBy16Loop + ja ComputeExpBy16Loop_2 -ReduceAccumulator: +ReduceAccumulator_2: vextractf64x4 ymm0,zmm20,1 vaddps zmm0,zmm0,zmm20 vhaddps ymm0,ymm0,ymm0 diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc b/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc index 96582fe46c851..6634be4276e1a 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelCommon.inc @@ -91,16 +91,16 @@ TanhConstants ENDS TransKernelFrame STRUCT - SavedXmm6 OWORD ? - SavedXmm7 OWORD ? - SavedXmm8 OWORD ? - SavedXmm9 OWORD ? - SavedXmm10 OWORD ? - SavedXmm11 OWORD ? - SavedXmm12 OWORD ? - SavedXmm13 OWORD ? - SavedXmm14 OWORD ? - SavedXmm15 OWORD ? + SavedXmm6 QWORD 2 DUP (?) + SavedXmm7 QWORD 2 DUP (?) + SavedXmm8 QWORD 2 DUP (?) + SavedXmm9 QWORD 2 DUP (?) + SavedXmm10 QWORD 2 DUP (?) + SavedXmm11 QWORD 2 DUP (?) + SavedXmm12 QWORD 2 DUP (?) + SavedXmm13 QWORD 2 DUP (?) + SavedXmm14 QWORD 2 DUP (?) + SavedXmm15 QWORD 2 DUP (?) Padding QWORD ? ReturnAddress QWORD ? PreviousP1Home QWORD ? diff --git a/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm b/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm index 993bb0686c4d7..ce0b18ac4e56d 100644 --- a/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm +++ b/onnxruntime/core/mlas/lib/amd64/TransKernelFma3.asm @@ -16,13 +16,11 @@ ; ;-- - .xlist INCLUDE mlasi.inc INCLUDE TransKernelCommon.inc - .list - EXTERN MlasMaskMoveTableAvx:NEAR - EXTERN MlasExpConstants:NEAR + EXTERN MlasMaskMoveTableAvx:PROC + EXTERN MlasExpConstants:PROC ;++ ; @@ -207,9 +205,9 @@ ExitKernel: vbroadcastss ymm9,DWORD PTR [r9] ; broadcast negative maximum value vxorps xmm10,xmm10,xmm10 ; clear exp() accumulator sub r8,24 - jb ProcessRemainingCount + jb ProcessRemainingCount_2 -ComputeExpBy24Loop: +ComputeExpBy24Loop_2: vbroadcastss ymm11,ExpConstants.LowerRangeSumExp[rax] vbroadcastss ymm2,ExpConstants.Log2Reciprocal[rax] vaddps ymm0,ymm9,YMMWORD PTR [rcx] ; bias by negative maximum value @@ -276,35 +274,35 @@ ComputeExpBy24Loop: add rcx,24*4 ; advance input by 24 elements vaddps ymm10,ymm10,ymm7 test rdx,rdx - jz SkipStoreResultsBy24 + jz SkipStoreResultsBy24_2 vmovups YMMWORD PTR [rdx],ymm1 vmovups YMMWORD PTR [rdx+32],ymm4 vmovups YMMWORD PTR [rdx+64],ymm7 add rdx,24*4 ; advance output by 24 elements -SkipStoreResultsBy24: +SkipStoreResultsBy24_2: sub r8,24 - jae ComputeExpBy24Loop + jae ComputeExpBy24Loop_2 -ProcessRemainingCount: +ProcessRemainingCount_2: add r8,24 ; correct for over-subtract above - jz ReduceAccumulator + jz ReduceAccumulator_2 vbroadcastss ymm11,ExpConstants.LowerRangeSumExp[rax] -ComputeExpBy8Loop: +ComputeExpBy8Loop_2: cmp r8,8 ; remaining count < 8? - jb LoadPartialVector + jb LoadPartialVector_2 vmovups ymm0,YMMWORD PTR [rcx] - jmp ProcessSingleVector + jmp ProcessSingleVector_2 -LoadPartialVector: +LoadPartialVector_2: lea r10,MlasMaskMoveTableAvx+8*4 neg r8 ; carry flag unchanged vmovups ymm3,YMMWORD PTR [r10+r8*4] vmaskmovps ymm0,ymm3,YMMWORD PTR [rcx] vandps ymm9,ymm9,ymm3 ; mask unused maximum value to 0.0 -ProcessSingleVector: +ProcessSingleVector_2: vbroadcastss ymm2,ExpConstants.Log2Reciprocal[rax] vaddps ymm0,ymm9,ymm0 ; bias by negative maximum value vbroadcastss ymm15,ExpConstants.RoundingBias[rax] @@ -331,27 +329,27 @@ ProcessSingleVector: vfmadd213ps ymm1,ymm0,ymm15 ; p = p * x + poly_5 vfmadd213ps ymm1,ymm0,ymm15 ; p = p * x + poly_6 vmulps ymm1,ymm1,ymm2 - jb StorePartialVector ; remaining count < 8? + jb StorePartialVector_2 ; remaining count < 8? vaddps ymm10,ymm10,ymm1 ; accumulate exp() results test rdx,rdx ; store exp() results? - jz SkipStoreResultsBy8 + jz SkipStoreResultsBy8_2 vmovups YMMWORD PTR [rdx],ymm1 add rdx,8*4 ; advance output by 8 elements -SkipStoreResultsBy8: +SkipStoreResultsBy8_2: add rcx,8*4 ; advance input by 8 elements sub r8,8 - jnz ComputeExpBy8Loop - jmp ReduceAccumulator + jnz ComputeExpBy8Loop_2 + jmp ReduceAccumulator_2 -StorePartialVector: +StorePartialVector_2: vandps ymm1,ymm1,ymm3 ; mask unused exp() results to 0.0 vaddps ymm10,ymm10,ymm1 ; accumulate exp() results test rdx,rdx ; store exp() results? - jz ReduceAccumulator + jz ReduceAccumulator_2 vmaskmovps YMMWORD PTR [rdx],ymm3,ymm1 -ReduceAccumulator: +ReduceAccumulator_2: vhaddps ymm10,ymm10,ymm10 vhaddps ymm10,ymm10,ymm10 vextractf128 xmm0,ymm10,1 diff --git a/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm b/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm index 800863c77a230..0b5c41dd33c2a 100644 --- a/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm +++ b/onnxruntime/core/mlas/lib/amd64/cvtfp16Avx.asm @@ -14,18 +14,17 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - .const +CONST SEGMENT READONLY ALIGN(16) 'CONST' SINGLE_SIZE equ 4 HALF_SIZE equ 2 LOW_SELECTOR equ 00100000b HIGH_SELECTOR equ 00110001b - SUBTTL "Convert buffer of half-precision floats to single-precision floats" +CONST ENDS + ;++ ; ; Routine Description: diff --git a/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm index 0ad98d3115208..545130fd88c78 100644 --- a/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm +++ b/onnxruntime/core/mlas/lib/amd64/cvtfp16a.asm @@ -14,20 +14,18 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list - .const +CONST SEGMENT READONLY ALIGN(16) 'CONST' - ALIGN 16 MlasFp16MaskSign DD 4 DUP (00007FFFh) MlasFp16CompareInfinity DD 4 DUP (00007C00h) MlasFp16CompareSmallest DD 4 DUP (00000400h) MlasFp16AdjustExponent DD 4 DUP (38000000h) MlasFp16MagicDenormal DD 4 DUP (38800000h) - SUBTTL "Convert buffer of half-precision floats to single-precision floats" +CONST ENDS + ;++ ; ; Routine Description: diff --git a/onnxruntime/core/mlas/lib/amd64/mlasi.inc b/onnxruntime/core/mlas/lib/amd64/mlasi.inc index a4f58c1060a8a..4809aa75bba37 100644 --- a/onnxruntime/core/mlas/lib/amd64/mlasi.inc +++ b/onnxruntime/core/mlas/lib/amd64/mlasi.inc @@ -15,9 +15,104 @@ ; ;-- - .xlist -INCLUDE macamd64.inc - .list +; +; Dual-assembler prologue/epilogue macros. +; +; These replace the Windows SDK macamd64.inc entry and unwind macros so the +; amd64 kernels assemble with BOTH ml64.exe and the LLVM MASM assembler +; (llvm-ml), enabling a Windows x64 cross-build from a non-Windows host. Each +; macro emits the same instructions and the same built-in structured-exception +; unwind directives (.pushreg/.allocstack/.savereg/.savexmm128/.setframe/ +; .endprolog/.beginepilog) that macamd64.inc emits, so the generated code and +; .pdata/.xdata are identical. The only omission is macamd64.inc's +; ifdif
, _CurrentSection_ same-module section check, a diagnostic-only +; construct that llvm-ml cannot parse. +; + +push_reg MACRO Reg + push Reg + .pushreg Reg + ENDM + +rex_push_reg MACRO Reg + db 048h + push Reg + .pushreg Reg + ENDM + +alloc_stack MACRO Size + sub rsp, Size + .allocstack Size + ENDM + +save_reg MACRO Reg, Offset + mov Offset[rsp], Reg + .savereg Reg, Offset + ENDM + +save_xmm128 MACRO Reg, Offset + movaps Offset[rsp], Reg + .savexmm128 Reg, Offset + ENDM + +set_frame MACRO Reg, Offset +IF Offset + lea Reg, Offset[rsp] +ELSE + mov Reg, rsp +ENDIF + .setframe Reg, Offset + ENDM + +END_PROLOGUE MACRO + .endprolog + ENDM + +; +; BEGIN_EPILOGUE emits no unwind marker. ml64.exe's macamd64.inc form emits +; .beginepilog, but llvm-ml lowers that to .seh_startepilogue which then requires +; a matching .seh_endepilogue that ml64 does not emit. The amd64 kernels use only +; standard epilogs (stack restore, pops, ret), which the Windows unwinder resolves +; via legacy epilog detection without explicit epilog unwind codes, so omitting the +; marker keeps unwind correct under both assemblers. +; +BEGIN_EPILOGUE MACRO + ENDM + +LEAF_ENTRY MACRO Name, Section, NoPad +Section segment para 'CODE' +IFB + db 6 dup (0cch) +ENDIF + align 16 + public Name +Name proc frame + END_PROLOGUE + ENDM + +LEAF_END MACRO Name, Section +Name endp +Section ends + ENDM + +NESTED_ENTRY MACRO Name, Section, Handler, NoPad +Section segment para 'CODE' +IFB + db 6 dup (0cch) +ENDIF + align 16 + public Name +IFB +Name proc frame +ELSE +Name proc frame:Handler +ENDIF + ENDM + +NESTED_END MACRO Name, Section +Name endp +Section ends + ENDM ; ; Macro Description: diff --git a/onnxruntime/core/mlas/lib/amd64/sgemma.asm b/onnxruntime/core/mlas/lib/amd64/sgemma.asm index 80805d337e500..f8d1d466cd2d4 100644 --- a/onnxruntime/core/mlas/lib/amd64/sgemma.asm +++ b/onnxruntime/core/mlas/lib/amd64/sgemma.asm @@ -15,9 +15,7 @@ ; ;-- - .xlist INCLUDE mlasi.inc - .list ;++ ; From 65bb90259defbc65bf884b80c0154067a8384302 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Tue, 7 Jul 2026 16:45:47 -0700 Subject: [PATCH 02/14] Cross-build onnxruntime_mlas for Windows x64 on Linux (clang-cl + llvm-ml) Extend the MLAS dual-assembler work into a real Linux cross-build of the whole onnxruntime_mlas library, and move CI off windows-latest onto a Linux runner. - cmake/win_x64_crosscompile_toolchain.cmake (new): a Windows-x64 cross toolchain using clang-cl + lld-link for C/C++, llvm-ml for the amd64 MASM kernels, and a Windows SDK/CRT staged by `xwin` (consumed via /imsvc + /libpath:). Targets x86_64-pc-windows-msvc, feeds llvm-ml -m64 /DLLVM_ML=1 and the amd64 include dir, and skips CMake's link-based compiler check (static-lib only). - cmake/onnxruntime_mlas.cmake: add clang-gated per-file arch flags (AVX2/AVX512/AVXVNNI/AVX512VNNI/AMX/SSE4.1) for the x64 intrinsic kernels. clang-cl enforces per-function target features where MSVC does not; the block is gated on Clang so the MSVC/ml64 build is byte-for-byte unchanged. - .github/workflows/linux_mlas_win_cross.yml (new): ubuntu-latest job that installs LLVM + xwin, cross-configures with the toolchain, builds onnxruntime_mlas, and asserts the library plus its amd64 .asm objects. This is a strict superset of the old assemble-only check and exercises the llvm-ml (/DLLVM_ML) branch end to end. - Remove .github/workflows/windows_mlas_masm_cross_assembler.yml: superseded by the Linux cross-build. The complementary ml64.exe path (no /DLLVM_ML) is still covered by the native Windows build/test pipelines. Tests are not run on Linux: the artifacts are Windows-x64 binaries and Wine is unreliable for the AVX-512/AMX kernels; runtime correctness stays with the native Windows pipelines. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/linux_mlas_win_cross.yml | 160 ++++++++++++++++++ .../windows_mlas_masm_cross_assembler.yml | 135 --------------- cmake/onnxruntime_mlas.cmake | 33 ++++ cmake/win_x64_crosscompile_toolchain.cmake | 110 ++++++++++++ 4 files changed, 303 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/linux_mlas_win_cross.yml delete mode 100644 .github/workflows/windows_mlas_masm_cross_assembler.yml create mode 100644 cmake/win_x64_crosscompile_toolchain.cmake diff --git a/.github/workflows/linux_mlas_win_cross.yml b/.github/workflows/linux_mlas_win_cross.yml new file mode 100644 index 0000000000000..637e555b5bcde --- /dev/null +++ b/.github/workflows/linux_mlas_win_cross.yml @@ -0,0 +1,160 @@ +name: MLAS Windows-x64 Cross-Build (Linux) + +# Cross-compiles the MLAS static library (onnxruntime_mlas) for Windows x64 from a +# Linux runner, using clang-cl + lld-link for C/C++, llvm-ml for the amd64 MASM +# assembly kernels, and a Windows SDK/CRT staged by `xwin`. +# +# Why: downstream consumers such as the Edge browser build their Windows releases +# on Linux machines (faster/cheaper than native Windows builders). Historically +# ONNX Runtime's MLAS x64 assembly assembled only under ml64.exe, which is not +# available on Linux. The amd64/*.asm sources are now dual-assembler compatible +# (ml64 + llvm-ml), and this job is the authoritative guard that the whole MLAS +# library keeps cross-building on Linux - a strict superset of merely assembling +# the .asm files. It exercises the llvm-ml (/DLLVM_ML) branch of the sources plus +# the clang-cl per-file arch-flag gating in cmake/onnxruntime_mlas.cmake. +# +# The complementary ml64.exe branch (no /DLLVM_ML define) is exercised by the +# regular Windows build/test pipelines, so both assemblers remain covered. +# +# Tests are intentionally NOT run here: the artifacts are Windows-x64 binaries, +# which cannot execute natively on a Linux runner (Wine is unreliable for the +# AVX-512/AMX kernels MLAS ships). Runtime correctness of these kernels is covered +# by the existing native Windows test pipelines. + +on: + push: + branches: [main, 'rel-*'] + paths: + - 'onnxruntime/core/mlas/**' + - 'cmake/onnxruntime_mlas.cmake' + - 'cmake/win_x64_crosscompile_toolchain.cmake' + - '.github/workflows/linux_mlas_win_cross.yml' + pull_request: + branches: [main] + paths: + - 'onnxruntime/core/mlas/**' + - 'cmake/onnxruntime_mlas.cmake' + - 'cmake/win_x64_crosscompile_toolchain.cmake' + - '.github/workflows/linux_mlas_win_cross.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: true + +permissions: + contents: read + +env: + LLVM_VERSION: '18' + XWIN_VERSION: '0.9.0' + +jobs: + mlas-win-cross-build: + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: false + + - name: Install build tools (cmake, ninja, LLVM/clang cross toolchain) + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends cmake ninja-build curl ca-certificates + + # LLVM (clang-cl, lld-link, llvm-ml, llvm-lib, llvm-mt, llvm-rc) from apt.llvm.org. + curl -fsSL https://apt.llvm.org/llvm.sh -o /tmp/llvm.sh + chmod +x /tmp/llvm.sh + sudo /tmp/llvm.sh "${LLVM_VERSION}" + sudo apt-get install -y --no-install-recommends "llvm-${LLVM_VERSION}" + + # Expose the version-suffixed tools under stable, unsuffixed names. + BIN="${HOME}/llvmbin" + mkdir -p "${BIN}" + link() { # link + local dest="$1"; shift + local src + for src in "$@"; do + if command -v "${src}" >/dev/null 2>&1; then + ln -sf "$(command -v "${src}")" "${BIN}/${dest}" + return 0 + fi + done + echo "::error::required LLVM tool not found (tried: $*)" + return 1 + } + link clang-cl "clang-cl-${LLVM_VERSION}" + link lld-link "lld-link-${LLVM_VERSION}" + link llvm-ml "llvm-ml-${LLVM_VERSION}" + link llvm-mt "llvm-mt-${LLVM_VERSION}" + link llvm-rc "llvm-rc-${LLVM_VERSION}" + # llvm-lib is llvm-ar invoked under the lib.exe-compatible name. + link llvm-lib "llvm-lib-${LLVM_VERSION}" "llvm-ar-${LLVM_VERSION}" + + echo "ORT_LLVM_BIN=${BIN}" >> "${GITHUB_ENV}" + echo "${BIN}" >> "${GITHUB_PATH}" + echo "== resolved LLVM tools ==" + ls -l "${BIN}" + + - name: Stage Windows SDK / CRT with xwin + shell: bash + run: | + set -euo pipefail + url="https://github.com/Jake-Shadle/xwin/releases/download/${XWIN_VERSION}/xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz" + curl -fsSL "${url}" -o /tmp/xwin.tar.gz + mkdir -p /tmp/xwin-dist + tar -xzf /tmp/xwin.tar.gz -C /tmp/xwin-dist --strip-components=1 + XWIN_BIN="$(find /tmp/xwin-dist -name xwin -type f | head -1)" + chmod +x "${XWIN_BIN}" + + XWIN_ROOT="${HOME}/xwinroot" + "${XWIN_BIN}" --accept-license --arch x86_64 splat --output "${XWIN_ROOT}" + echo "ORT_WIN_XWIN_ROOT=${XWIN_ROOT}" >> "${GITHUB_ENV}" + echo "== xwin layout ==" + ls "${XWIN_ROOT}" && ls "${XWIN_ROOT}/crt" && ls "${XWIN_ROOT}/sdk" + + - name: Configure (Windows x64 cross toolchain) + shell: bash + run: | + set -euo pipefail + cmake -S cmake -B build_win_cross -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="${GITHUB_WORKSPACE}/cmake/win_x64_crosscompile_toolchain.cmake" \ + -DORT_WIN_XWIN_ROOT="${ORT_WIN_XWIN_ROOT}" \ + -DORT_LLVM_BIN="${ORT_LLVM_BIN}" \ + -DCMAKE_BUILD_TYPE=Release \ + -Donnxruntime_BUILD_UNIT_TESTS=OFF \ + -Donnxruntime_BUILD_SHARED_LIB=OFF \ + -Donnxruntime_ENABLE_PYTHON=OFF \ + -Donnxruntime_USE_FULL_PROTOBUF=OFF + + - name: Build onnxruntime_mlas + shell: bash + run: cmake --build build_win_cross --target onnxruntime_mlas -j"$(nproc)" + + - name: Verify cross-built library + shell: bash + run: | + set -euo pipefail + lib="$(find build_win_cross -name 'onnxruntime_mlas.lib' | head -1)" + if [ -z "${lib}" ]; then + echo "::error::onnxruntime_mlas.lib was not produced" + exit 1 + fi + echo "Built: ${lib} ($(stat -c%s "${lib}") bytes)" + + # The library must contain the amd64 MASM kernels assembled by llvm-ml. + asm_objs="$(llvm-lib /list "${lib}" 2>/dev/null | grep -c '\.asm\.obj$' || true)" + if [ "${asm_objs}" = "0" ]; then + # llvm-lib may not support /list on this platform; fall back to llvm-ar. + asm_objs="$( { llvm-ar t "${lib}" 2>/dev/null || true; } | grep -c '\.asm\.obj$' || true)" + fi + echo "amd64 .asm objects archived: ${asm_objs}" + if [ "${asm_objs}" -lt 1 ]; then + echo "::error::no amd64 .asm objects found in onnxruntime_mlas.lib" + exit 1 + fi diff --git a/.github/workflows/windows_mlas_masm_cross_assembler.yml b/.github/workflows/windows_mlas_masm_cross_assembler.yml deleted file mode 100644 index 6df32ef74e1a2..0000000000000 --- a/.github/workflows/windows_mlas_masm_cross_assembler.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: MLAS x64 MASM Cross-Assembler Check - -# Validates that the MLAS x64 (amd64) assembly kernels assemble under BOTH: -# * ml64.exe - the Microsoft Macro Assembler (native Windows toolchain), and -# * llvm-ml - the LLVM MASM-compatible assembler (used when cross-compiling -# Windows-x64 binaries from a non-Windows host, e.g. Linux). -# -# Keeping the amd64/*.asm sources buildable by both assemblers lets downstream -# consumers (such as the Edge browser) cross-compile ONNX Runtime for Windows on -# Linux build machines. This job is the authoritative guard for that property: -# a change that breaks either assembler fails here. -# -# llvm-ml is invoked with /DLLVM_ML=1. That define selects the handful of -# constructs where a single MASM spelling cannot satisfy both assemblers (today -# only the alternate-entry public label in SconvKernelAvx.asm, where ml64 needs -# the "::" global-label form and llvm-ml needs a single-colon label + PUBLIC). -# ml64.exe is invoked WITHOUT the define, so its output is completely unchanged. - -on: - push: - branches: [main, 'rel-*'] - paths: - - 'onnxruntime/core/mlas/lib/amd64/**' - - '.github/workflows/windows_mlas_masm_cross_assembler.yml' - pull_request: - branches: [main] - paths: - - 'onnxruntime/core/mlas/lib/amd64/**' - - '.github/workflows/windows_mlas_masm_cross_assembler.yml' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - mlas-masm-dual-assemble: - runs-on: windows-latest - timeout-minutes: 20 - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: false - - - name: Install LLVM (provides llvm-ml.exe) - shell: pwsh - run: | - choco install llvm --no-progress -y - if ($LASTEXITCODE -ne 0) { throw "choco install llvm failed" } - - - name: Assemble amd64/*.asm with ml64.exe and llvm-ml - shell: pwsh - run: | - $ErrorActionPreference = 'Stop' - - $asmDir = Join-Path $env:GITHUB_WORKSPACE 'onnxruntime/core/mlas/lib/amd64' - if (-not (Test-Path $asmDir)) { throw "amd64 asm directory not found: $asmDir" } - - # Locate ml64.exe via vswhere (bundled on GitHub-hosted Windows images). - $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' - if (-not (Test-Path $vswhere)) { throw "vswhere.exe not found: $vswhere" } - $ml64 = (& $vswhere -latest -products '*' -find '**/x64/ml64.exe' | Select-Object -First 1) - if (-not $ml64) { $ml64 = (& $vswhere -latest -products '*' -find '**/ml64.exe' | Select-Object -First 1) } - if (-not $ml64) { throw "ml64.exe not found via vswhere" } - - # Locate llvm-ml.exe from the LLVM install. - $llvmml = (Get-Command llvm-ml.exe -ErrorAction SilentlyContinue).Source - if (-not $llvmml) { - $candidate = 'C:/Program Files/LLVM/bin/llvm-ml.exe' - if (Test-Path $candidate) { $llvmml = $candidate } - } - if (-not $llvmml) { throw "llvm-ml.exe not found" } - - Write-Host "ml64.exe : $ml64" - Write-Host "llvm-ml : $llvmml" - & $ml64 2>&1 | Select-Object -First 1 | Write-Host - & $llvmml --version 2>&1 | Select-Object -First 3 | Write-Host - - $outDir = Join-Path $env:RUNNER_TEMP 'mlas_asm_out' - New-Item -ItemType Directory -Force -Path $outDir | Out-Null - - $asmFiles = Get-ChildItem -Path $asmDir -Filter '*.asm' | Sort-Object Name - if ($asmFiles.Count -eq 0) { throw "no .asm files found in $asmDir" } - Write-Host "Found $($asmFiles.Count) amd64 .asm files." - - $ml64Failures = @() - $llvmFailures = @() - - foreach ($f in $asmFiles) { - $name = $f.BaseName - - # --- ml64.exe (native MASM; no LLVM_ML define) --- - $ml64Obj = Join-Path $outDir "$name.ml64.obj" - $ml64Log = & $ml64 /nologo /c /Fo"$ml64Obj" /I"$asmDir" $f.FullName 2>&1 - if ($LASTEXITCODE -ne 0) { - $ml64Failures += $f.Name - Write-Host "::error::ml64 failed to assemble $($f.Name)" - $ml64Log | ForEach-Object { Write-Host " $_" } - } - - # --- llvm-ml (cross-assembler; LLVM_ML define selects dual-safe spellings) --- - $llvmObj = Join-Path $outDir "$name.llvm.obj" - $llvmLog = & $llvmml -m64 /c /DLLVM_ML=1 /Fo"$llvmObj" /I"$asmDir" $f.FullName 2>&1 - if ($LASTEXITCODE -ne 0) { - $llvmFailures += $f.Name - Write-Host "::error::llvm-ml failed to assemble $($f.Name)" - $llvmLog | ForEach-Object { Write-Host " $_" } - } - } - - Write-Host "" - Write-Host "===================== Summary =====================" - Write-Host ("Total files : {0}" -f $asmFiles.Count) - Write-Host ("ml64 failures : {0}" -f $ml64Failures.Count) - Write-Host ("llvm-ml failures : {0}" -f $llvmFailures.Count) - - if ($ml64Failures.Count -gt 0) { - Write-Host "ml64 could not assemble:" - $ml64Failures | ForEach-Object { Write-Host " - $_" } - } - if ($llvmFailures.Count -gt 0) { - Write-Host "llvm-ml could not assemble (breaks Windows-on-Linux cross-compile):" - $llvmFailures | ForEach-Object { Write-Host " - $_" } - } - - if ($ml64Failures.Count -gt 0 -or $llvmFailures.Count -gt 0) { - throw "One or more amd64 .asm files failed to assemble under ml64 and/or llvm-ml." - } - - Write-Host "All $($asmFiles.Count) amd64 .asm files assembled successfully under both ml64 and llvm-ml." diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index d55a4d49fb455..2aa8858669337 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -303,6 +303,39 @@ function(setup_mlas_source_for_windows) ${MLAS_SRC_DIR}/q4gemm_avx512.cpp ) endif() + + # clang-cl (used when cross-compiling ONNX Runtime for Windows on a non-Windows + # host) enforces per-function target features: an intrinsic may only be used in + # a translation unit that was compiled with that feature enabled. MSVC has no + # such requirement, so several MLAS x64 kernels use AVX2/AVX-512/AMX/SSE4.1 + # intrinsics without a TU-level arch flag. Supply the required features here, + # gated on Clang so the MSVC (cl.exe/ml64.exe) build stays byte-for-byte the + # same. Files already carrying an /arch flag above are not repeated. + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set_source_files_properties( + ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX2") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX2 -mavxvnni") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512") + set_source_files_properties( + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx512vnni.cpp + ${MLAS_SRC_DIR}/q4gemm_avx512.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mavx512vnni") + set_source_files_properties( + ${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp + PROPERTIES COMPILE_FLAGS "-msse4.1") + set_source_files_properties( + ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp + PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mamx-tile -mamx-int8 -mamx-bf16") + endif() else() target_sources(onnxruntime_mlas PRIVATE ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake new file mode 100644 index 0000000000000..a94128eace2d1 --- /dev/null +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -0,0 +1,110 @@ +# Cross-compile ONNX Runtime for Windows x64 from a non-Windows host (e.g. Linux) +# using the LLVM/clang toolchain. This is what lets downstream consumers such as +# the Edge browser build Windows-x64 ONNX Runtime on faster/cheaper Linux +# builders. Today it is exercised to build the onnxruntime_mlas static library +# (see .github/workflows/linux_mlas_win_cross.yml), which contains the MLAS x64 +# MASM assembly kernels. +# +# Toolchain pieces: +# * clang-cl / lld-link - C/C++ compile + link, targeting x86_64-pc-windows-msvc +# * llvm-ml - the LLVM MASM assembler for onnxruntime/core/mlas/lib/amd64/*.asm +# * Windows SDK + MSVC CRT headers/import libs staged by `xwin` +# (https://github.com/Jake-Shadle/xwin): run `xwin splat --output ` and +# pass that directory below. +# +# Required (cache var or environment variable): +# ORT_WIN_XWIN_ROOT - the `xwin splat` output directory (contains crt/ and sdk/) +# Optional: +# ORT_LLVM_BIN - directory holding clang-cl/lld-link/llvm-ml/... (else PATH) + +if(NOT ORT_WIN_XWIN_ROOT) + set(ORT_WIN_XWIN_ROOT "$ENV{ORT_WIN_XWIN_ROOT}") +endif() +if(NOT ORT_WIN_XWIN_ROOT) + message(FATAL_ERROR + "win x64 cross toolchain: set ORT_WIN_XWIN_ROOT to the `xwin splat` output directory") +endif() +if(NOT ORT_LLVM_BIN) + set(ORT_LLVM_BIN "$ENV{ORT_LLVM_BIN}") +endif() + +# CMake re-parses this toolchain file inside its try_compile subprojects, where +# the outer -D cache entries are not visible. Forward the settings this file needs. +list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES ORT_WIN_XWIN_ROOT ORT_LLVM_BIN) + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR AMD64) + +macro(_ort_find_llvm_tool _var _name _required) + if(ORT_LLVM_BIN) + find_program(${_var} NAMES ${_name} PATHS "${ORT_LLVM_BIN}" NO_DEFAULT_PATH) + endif() + if(NOT ${_var}) + find_program(${_var} NAMES ${_name}) + endif() + if(${_required} AND NOT ${_var}) + message(FATAL_ERROR "win x64 cross toolchain: could not find '${_name}'") + endif() +endmacro() + +_ort_find_llvm_tool(_ort_clang_cl clang-cl TRUE) +_ort_find_llvm_tool(_ort_lld_link lld-link TRUE) +_ort_find_llvm_tool(_ort_llvm_ml llvm-ml TRUE) +_ort_find_llvm_tool(_ort_llvm_lib llvm-lib TRUE) +# mt (manifest) and rc (resource compiler) are only needed for DLL/exe links, not +# for the onnxruntime_mlas static library, so they are optional. +_ort_find_llvm_tool(_ort_llvm_mt llvm-mt FALSE) +_ort_find_llvm_tool(_ort_llvm_rc llvm-rc FALSE) + +set(CMAKE_C_COMPILER "${_ort_clang_cl}") +set(CMAKE_CXX_COMPILER "${_ort_clang_cl}") +set(CMAKE_ASM_MASM_COMPILER "${_ort_llvm_ml}") +set(CMAKE_LINKER "${_ort_lld_link}") +set(CMAKE_AR "${_ort_llvm_lib}") +if(_ort_llvm_mt) + set(CMAKE_MT "${_ort_llvm_mt}") +endif() +if(_ort_llvm_rc) + set(CMAKE_RC_COMPILER "${_ort_llvm_rc}") +endif() + +set(CMAKE_C_COMPILER_TARGET x86_64-pc-windows-msvc) +set(CMAKE_CXX_COMPILER_TARGET x86_64-pc-windows-msvc) + +set(_ort_crt "${ORT_WIN_XWIN_ROOT}/crt") +set(_ort_sdk "${ORT_WIN_XWIN_ROOT}/sdk") + +# Windows SDK / CRT header search. Passed via /imsvc (system-include semantics) so +# ORT's -WX and warning flags do not apply to SDK headers. C/C++ only; the .asm +# files must not receive these. -Wno-unused-command-line-argument silences the +# MSVC-only flags ORT passes that clang-cl accepts-but-ignores (e.g. +# /experimental:external), which -WX would otherwise turn into errors. +string(JOIN " " _ort_win_include_flags + "/imsvc${_ort_crt}/include" + "/imsvc${_ort_sdk}/include/ucrt" + "/imsvc${_ort_sdk}/include/um" + "/imsvc${_ort_sdk}/include/shared" + "/imsvc${_ort_sdk}/include/winrt") +add_compile_options("$<$:SHELL:${_ort_win_include_flags}>") +add_compile_options("$<$:-Wno-unused-command-line-argument>") + +add_link_options( + "/libpath:${_ort_crt}/lib/x86_64" + "/libpath:${_ort_sdk}/lib/ucrt/x86_64" + "/libpath:${_ort_sdk}/lib/um/x86_64") + +# MLAS amd64 .asm via llvm-ml: unlike ml64.exe, llvm-ml is not implicitly 64-bit, +# does not search the source file's own directory for includes, and needs the +# LLVM_ML define that selects the dual-assembler-compatible spellings. ml64.exe +# needs none of these. CMAKE_CURRENT_LIST_DIR is this file's directory (cmake/). +set(CMAKE_ASM_MASM_FLAGS_INIT + "-m64 /DLLVM_ML=1 -I${CMAKE_CURRENT_LIST_DIR}/../onnxruntime/core/mlas/lib/amd64") + +# onnxruntime_mlas is a static library, so skip CMake's link-based compiler check +# (manifest/rc/link), which is irrelevant when cross-building a static archive. +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) From 8be93e4b0baab8828a31e638381df423b9f47068 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Tue, 7 Jul 2026 17:00:51 -0700 Subject: [PATCH 03/14] Pin and verify xwin release SHA256 in the MLAS cross-build workflow GitHub release assets (and their co-located .sha256 sidecars) are mutable, so a compromised release could silently swap the xwin binary. Pin the known-good SHA256 of the xwin tarball and verify the download against it before use. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/linux_mlas_win_cross.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/linux_mlas_win_cross.yml b/.github/workflows/linux_mlas_win_cross.yml index 637e555b5bcde..19179817a8a1a 100644 --- a/.github/workflows/linux_mlas_win_cross.yml +++ b/.github/workflows/linux_mlas_win_cross.yml @@ -48,6 +48,10 @@ permissions: env: LLVM_VERSION: '18' XWIN_VERSION: '0.9.0' + # SHA256 of xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz. GitHub release + # assets are mutable, so pin the digest and verify it rather than trusting the + # release (or its co-located .sha256 sidecar, which an attacker could swap too). + XWIN_SHA256: '31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd' jobs: mlas-win-cross-build: @@ -107,6 +111,7 @@ jobs: set -euo pipefail url="https://github.com/Jake-Shadle/xwin/releases/download/${XWIN_VERSION}/xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz" curl -fsSL "${url}" -o /tmp/xwin.tar.gz + echo "${XWIN_SHA256} /tmp/xwin.tar.gz" | sha256sum -c - mkdir -p /tmp/xwin-dist tar -xzf /tmp/xwin.tar.gz -C /tmp/xwin-dist --strip-components=1 XWIN_BIN="$(find /tmp/xwin-dist -name xwin -type f | head -1)" From 9693790eef005eab26011cd781db27cca151c640 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Tue, 7 Jul 2026 17:03:07 -0700 Subject: [PATCH 04/14] Bump cross-build LLVM to 19 for xwin CRT compatibility The MSVC CRT headers staged by xwin enforce a minimum Clang version via a static_assert (STL1000: "expected Clang 19.0.0 or newer"). LLVM 18 fails that check; use LLVM 19. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/linux_mlas_win_cross.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_mlas_win_cross.yml b/.github/workflows/linux_mlas_win_cross.yml index 19179817a8a1a..7100bc56c8803 100644 --- a/.github/workflows/linux_mlas_win_cross.yml +++ b/.github/workflows/linux_mlas_win_cross.yml @@ -46,7 +46,9 @@ permissions: contents: read env: - LLVM_VERSION: '18' + # The MSVC CRT headers staged by xwin enforce a minimum Clang version via a + # static_assert (STL1000); keep this at or above what that CRT requires. + LLVM_VERSION: '19' XWIN_VERSION: '0.9.0' # SHA256 of xwin-${XWIN_VERSION}-x86_64-unknown-linux-musl.tar.gz. GitHub release # assets are mutable, so pin the digest and verify it rather than trusting the From d42825fa764e71aa2d6b8f40507aae7df901c32e Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Wed, 8 Jul 2026 16:23:48 -0700 Subject: [PATCH 05/14] Cross-build the full onnxruntime.dll for Windows x64 on Linux Extend the Linux->Windows-x64 cross-compile support beyond the MLAS static library to the entire onnxruntime.dll, using clang-cl + lld-link for C/C++, llvm-ml for the MLAS amd64 MASM kernels, llvm-rc for the version resource, and a Windows SDK/CRT staged by xwin. This is what downstream consumers such as the Edge browser actually build (they compile ORT from source, not just MLAS). Fixes the following real cross-portability issues that only surface under a case-sensitive filesystem and/or clang's stricter front end; the native MSVC build is unaffected: - ONNX ScopedHandle: INVALID_HANDLE_VALUE performs a reinterpret_cast that is not a valid non-type template parameter under Clang. Replace the ScopedResource alias with a concrete class holding the sentinel at runtime (onnx.patch). - / : the real headers are lowercase; fix the include spellings so they resolve on case-sensitive filesystems. - spin_pause.cc: clang-cl requires the waitpkg target feature for the _tpause intrinsic (used behind a runtime HasTPAUSE() guard); add -mwaitpkg for the Windows/Clang/x86_64 cross build. - onnxruntime.rc VERSIONINFO: the octal-escaped copyright/registered glyphs in narrow strings are rejected by llvm-rc (no code page); use wide L"\x00A9" / L"\x00AE" escapes, which both rc.exe and llvm-rc accept with identical output. Also pass the Windows SDK include paths to llvm-rc, which does not honor /imsvc. - MLAS QgemmU8X8KernelAvx2.asm: ExitKernel is declared LOCAL in an outer macro but referenced from nested macros. ml64 resolves the nested reference to the outer LOCAL; llvm-ml emits it as an undefined external, breaking the DLL link. Thread the exit label through as an explicit macro parameter so the substitution happens in the outer macro body (where the LOCAL is visible) under both assemblers. Gate the per-target COMPILE_WARNING_AS_ERROR so an explicit -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF is honored (default stays ON). clang-cl emits many warnings MSVC never does; treating them as errors would make the cross-build unbuildable for no portability benefit. The native Windows pipelines keep -WX on. Replace the MLAS-only cross-build workflow with linux_win_cross_build.yml, which cross-builds and link-checks the whole onnxruntime.dll on a Linux runner (LLVM 19 + SHA-pinned xwin, dynamic MSVC runtime pinned so protobuf and ORT agree), and verifies the output is a PE image embedding the llvm-ml-assembled MLAS kernels. Runtime correctness and the ml64 path remain covered by the native Windows build/test pipelines. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...in_cross.yml => linux_win_cross_build.yml} | 96 +++++++++++-------- cmake/CMakeLists.txt | 7 +- cmake/onnxruntime_common.cmake | 10 ++ cmake/patches/onnx/onnx.patch | 37 +++++++ cmake/win_x64_crosscompile_toolchain.cmake | 5 + onnxruntime/core/dll/onnxruntime.rc | 6 +- .../mlas/lib/amd64/QgemmU8X8KernelAvx2.asm | 22 ++--- onnxruntime/core/platform/windows/env.cc | 2 +- .../test/shared_lib/test_model_loading.cc | 2 +- 9 files changed, 131 insertions(+), 56 deletions(-) rename .github/workflows/{linux_mlas_win_cross.yml => linux_win_cross_build.yml} (60%) diff --git a/.github/workflows/linux_mlas_win_cross.yml b/.github/workflows/linux_win_cross_build.yml similarity index 60% rename from .github/workflows/linux_mlas_win_cross.yml rename to .github/workflows/linux_win_cross_build.yml index 7100bc56c8803..f6f437f46e909 100644 --- a/.github/workflows/linux_mlas_win_cross.yml +++ b/.github/workflows/linux_win_cross_build.yml @@ -1,41 +1,51 @@ -name: MLAS Windows-x64 Cross-Build (Linux) +name: ONNX Runtime Windows-x64 Cross-Build (Linux) -# Cross-compiles the MLAS static library (onnxruntime_mlas) for Windows x64 from a -# Linux runner, using clang-cl + lld-link for C/C++, llvm-ml for the amd64 MASM -# assembly kernels, and a Windows SDK/CRT staged by `xwin`. +# Cross-compiles the full onnxruntime.dll for Windows x64 from a Linux runner, +# using clang-cl + lld-link for C/C++, llvm-ml for the MLAS amd64 MASM assembly +# kernels, llvm-rc for the version resource, and a Windows SDK/CRT staged by +# `xwin`. # # Why: downstream consumers such as the Edge browser build their Windows releases # on Linux machines (faster/cheaper than native Windows builders). Historically -# ONNX Runtime's MLAS x64 assembly assembled only under ml64.exe, which is not -# available on Linux. The amd64/*.asm sources are now dual-assembler compatible -# (ml64 + llvm-ml), and this job is the authoritative guard that the whole MLAS -# library keeps cross-building on Linux - a strict superset of merely assembling -# the .asm files. It exercises the llvm-ml (/DLLVM_ML) branch of the sources plus -# the clang-cl per-file arch-flag gating in cmake/onnxruntime_mlas.cmake. +# ONNX Runtime could not cross-build for Windows because the MLAS x64 assembly +# assembled only under ml64.exe (a Windows-only PE tool). The amd64/*.asm sources +# are now dual-assembler compatible (ml64 + llvm-ml), and the remaining C/C++ +# portability gaps for clang-cl have been addressed, so the whole DLL links on +# Linux. This job is the authoritative guard that the property keeps holding. # -# The complementary ml64.exe branch (no /DLLVM_ML define) is exercised by the -# regular Windows build/test pipelines, so both assemblers remain covered. +# The complementary native path (cl.exe + ml64.exe, warnings-as-errors on) is +# exercised by the regular Windows build/test pipelines, so both toolchains +# remain covered. +# +# clang-cl surfaces many warnings MSVC never emits; treating them as errors would +# make the cross-build unbuildable for no portability benefit, so this job sets +# -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF (the native pipelines keep -WX on). +# +# The MSVC runtime library is pinned to the dynamic CRT (MultiThreadedDLL) for +# every target, including the FetchContent'd protobuf, so the shared DLL links +# without a /failifmismatch RuntimeLibrary conflict (protobuf otherwise defaults +# to the static CRT under MSVC-like toolchains). # # Tests are intentionally NOT run here: the artifacts are Windows-x64 binaries, # which cannot execute natively on a Linux runner (Wine is unreliable for the -# AVX-512/AMX kernels MLAS ships). Runtime correctness of these kernels is covered -# by the existing native Windows test pipelines. +# AVX-512/AMX kernels MLAS ships). Runtime correctness is covered by the existing +# native Windows test pipelines. on: push: branches: [main, 'rel-*'] paths: - - 'onnxruntime/core/mlas/**' - - 'cmake/onnxruntime_mlas.cmake' - - 'cmake/win_x64_crosscompile_toolchain.cmake' - - '.github/workflows/linux_mlas_win_cross.yml' + - 'onnxruntime/core/**' + - 'include/onnxruntime/**' + - 'cmake/**' + - '.github/workflows/linux_win_cross_build.yml' pull_request: branches: [main] paths: - - 'onnxruntime/core/mlas/**' - - 'cmake/onnxruntime_mlas.cmake' - - 'cmake/win_x64_crosscompile_toolchain.cmake' - - '.github/workflows/linux_mlas_win_cross.yml' + - 'onnxruntime/core/**' + - 'include/onnxruntime/**' + - 'cmake/**' + - '.github/workflows/linux_win_cross_build.yml' workflow_dispatch: concurrency: @@ -56,9 +66,9 @@ env: XWIN_SHA256: '31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd' jobs: - mlas-win-cross-build: + win-cross-build: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 60 steps: - name: Checkout repository @@ -101,6 +111,9 @@ jobs: link llvm-rc "llvm-rc-${LLVM_VERSION}" # llvm-lib is llvm-ar invoked under the lib.exe-compatible name. link llvm-lib "llvm-lib-${LLVM_VERSION}" "llvm-ar-${LLVM_VERSION}" + # llvm-ar (Unix archive tool) is used to inspect the built .lib in the + # verify step; it lists members that the lib.exe-mode driver does not. + link llvm-ar "llvm-ar-${LLVM_VERSION}" echo "ORT_LLVM_BIN=${BIN}" >> "${GITHUB_ENV}" echo "${BIN}" >> "${GITHUB_PATH}" @@ -134,33 +147,38 @@ jobs: -DORT_WIN_XWIN_ROOT="${ORT_WIN_XWIN_ROOT}" \ -DORT_LLVM_BIN="${ORT_LLVM_BIN}" \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL \ + -Dprotobuf_MSVC_STATIC_RUNTIME=OFF \ -Donnxruntime_BUILD_UNIT_TESTS=OFF \ - -Donnxruntime_BUILD_SHARED_LIB=OFF \ + -Donnxruntime_BUILD_SHARED_LIB=ON \ -Donnxruntime_ENABLE_PYTHON=OFF \ -Donnxruntime_USE_FULL_PROTOBUF=OFF - - name: Build onnxruntime_mlas + - name: Build onnxruntime.dll shell: bash - run: cmake --build build_win_cross --target onnxruntime_mlas -j"$(nproc)" + run: cmake --build build_win_cross --target onnxruntime -j"$(nproc)" - - name: Verify cross-built library + - name: Verify cross-built artifacts shell: bash run: | set -euo pipefail - lib="$(find build_win_cross -name 'onnxruntime_mlas.lib' | head -1)" - if [ -z "${lib}" ]; then - echo "::error::onnxruntime_mlas.lib was not produced" + dll="$(find build_win_cross -name 'onnxruntime.dll' | head -1)" + if [ -z "${dll}" ]; then + echo "::error::onnxruntime.dll was not produced" exit 1 fi - echo "Built: ${lib} ($(stat -c%s "${lib}") bytes)" - - # The library must contain the amd64 MASM kernels assembled by llvm-ml. - asm_objs="$(llvm-lib /list "${lib}" 2>/dev/null | grep -c '\.asm\.obj$' || true)" - if [ "${asm_objs}" = "0" ]; then - # llvm-lib may not support /list on this platform; fall back to llvm-ar. - asm_objs="$( { llvm-ar t "${lib}" 2>/dev/null || true; } | grep -c '\.asm\.obj$' || true)" + echo "Built: ${dll} ($(stat -c%s "${dll}") bytes)" + # A PE32+ (x86-64) DLL starts with the 'MZ' DOS stub. + if [ "$(head -c2 "${dll}")" != "MZ" ]; then + echo "::error::onnxruntime.dll is not a PE image" + exit 1 fi - echo "amd64 .asm objects archived: ${asm_objs}" + + # The DLL must embed the MLAS amd64 MASM kernels assembled by llvm-ml. + mlas_lib="$(find build_win_cross -name 'onnxruntime_mlas.lib' | head -1)" + asm_objs="$( { llvm-ar t "${mlas_lib}" 2>/dev/null || true; } | grep -c '\.asm\.obj$' || true)" + echo "amd64 .asm objects in onnxruntime_mlas.lib: ${asm_objs}" if [ "${asm_objs}" -lt 1 ]; then echo "::error::no amd64 .asm objects found in onnxruntime_mlas.lib" exit 1 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 28d6aa83c5343..e10bffd24e09c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1115,7 +1115,12 @@ function(onnxruntime_set_compile_flags target_name) target_compile_definitions(${target_name} PRIVATE USE_KLEIDIAI) endif() - set_target_properties(${target_name} PROPERTIES COMPILE_WARNING_AS_ERROR ON) + # Honor an explicit -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF (default remains ON). + # Useful for clang-cl cross-compilation, which surfaces warnings MSVC never + # emits; consumers that treat those as errors would otherwise be unbuildable. + if (NOT DEFINED CMAKE_COMPILE_WARNING_AS_ERROR OR CMAKE_COMPILE_WARNING_AS_ERROR) + set_target_properties(${target_name} PROPERTIES COMPILE_WARNING_AS_ERROR ON) + endif() if (onnxruntime_USE_CUDA) # Suppress a "conversion_function_not_usable" warning in gsl/span target_compile_options(${target_name} PRIVATE "$<$:SHELL:-Xcudafe \"--diag_suppress=conversion_function_not_usable\">") diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index b081e22e8b3f4..25c59cd2c686d 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -138,6 +138,16 @@ if(NOT WIN32 AND NOT APPLE AND NOT ANDROID AND CMAKE_SYSTEM_PROCESSOR MATCHES "x ) endif() +if(WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|AMD64)") + # clang-cl (used when cross-compiling ORT for Windows) enforces the waitpkg + # target feature for the _tpause intrinsic, which SpinPause uses behind a + # runtime HasTPAUSE() guard. MSVC allows the intrinsic unconditionally. + set_source_files_properties( + ${ONNXRUNTIME_ROOT}/core/common/spin_pause.cc + PROPERTIES COMPILE_FLAGS "-mwaitpkg" + ) +endif() + if (onnxruntime_USE_TELEMETRY) set_target_properties(onnxruntime_common PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h") endif() diff --git a/cmake/patches/onnx/onnx.patch b/cmake/patches/onnx/onnx.patch index acd5505beaff6..2a483fb799108 100644 --- a/cmake/patches/onnx/onnx.patch +++ b/cmake/patches/onnx/onnx.patch @@ -46,3 +46,40 @@ index 3887169..8432613 100644 .SetDoc(GroupNormalization_ver18_doc) .Attr("epsilon", "The epsilon value to use to avoid division by zero.", AttributeProto::FLOAT, 1e-5f) .Attr( +diff --git a/onnx/common/scoped_resource.h b/onnx/common/scoped_resource.h +index 525e4d6..2c41f0b 100644 +--- a/onnx/common/scoped_resource.h ++++ b/onnx/common/scoped_resource.h +@@ -56,7 +56,31 @@ class ScopedResource { + inline void close_handle(HANDLE h) { + CloseHandle(h); + } +-using ScopedHandle = ScopedResource; ++// INVALID_HANDLE_VALUE is ((HANDLE)(LONG_PTR)-1); the reinterpret_cast it ++// performs is not a valid constant expression for a non-type template parameter ++// under Clang (e.g. clang-cl cross-compilation), so it cannot be used as a ++// ScopedResource template argument. Hold the sentinel as a runtime value. ++class ScopedHandle { ++ HANDLE val_; ++ ++ public: ++ explicit ScopedHandle(HANDLE v) : val_(v) {} ++ ~ScopedHandle() { ++ if (val_ != INVALID_HANDLE_VALUE) { ++ close_handle(val_); ++ } ++ } ++ HANDLE get() const { ++ return val_; ++ } ++ HANDLE release() { ++ HANDLE tmp = val_; ++ val_ = INVALID_HANDLE_VALUE; ++ return tmp; ++ } ++ ScopedHandle(const ScopedHandle&) = delete; ++ ScopedHandle& operator=(const ScopedHandle&) = delete; ++}; + #endif + + inline void close_fd(int fd) { diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake index a94128eace2d1..83a1de168be1f 100644 --- a/cmake/win_x64_crosscompile_toolchain.cmake +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -93,6 +93,11 @@ add_link_options( "/libpath:${_ort_sdk}/lib/ucrt/x86_64" "/libpath:${_ort_sdk}/lib/um/x86_64") +# llvm-rc (resource compiler, used for the DLL's version resource) does not honor +# /imsvc, so give it the Windows SDK headers on its own include path. +set(CMAKE_RC_FLAGS_INIT + "/I ${_ort_crt}/include /I ${_ort_sdk}/include/um /I ${_ort_sdk}/include/shared /I ${_ort_sdk}/include/ucrt") + # MLAS amd64 .asm via llvm-ml: unlike ml64.exe, llvm-ml is not implicitly 64-bit, # does not search the source file's own directory for includes, and needs the # LLVM_ML define that selects the dual-assembler-compatible spellings. ml64.exe diff --git a/onnxruntime/core/dll/onnxruntime.rc b/onnxruntime/core/dll/onnxruntime.rc index 2e7eb7842f935..710c74171c74e 100644 --- a/onnxruntime/core/dll/onnxruntime.rc +++ b/onnxruntime/core/dll/onnxruntime.rc @@ -4,7 +4,7 @@ // This file REQUIRES the following external definitions: // FILE_NAME, VER_MAJOR, VER_MINOR, VER_BUILD, VER_PRIVATE, and VER_STRING -#include +#include #if defined(DEBUG) || defined(_DEBUG) #define VER_DEBUG VS_FF_DEBUG @@ -32,9 +32,9 @@ BEGIN VALUE "FileDescription", "ONNX Runtime" VALUE "FileVersion", VER_STRING VALUE "InternalName", "ONNX Runtime" - VALUE "LegalCopyright", "\251 Microsoft Corporation. All rights reserved." + VALUE "LegalCopyright", L"\x00A9 Microsoft Corporation. All rights reserved." VALUE "OriginalFilename", FILE_NAME - VALUE "ProductName", "Microsoft\256 Windows\256 Operating System" + VALUE "ProductName", L"Microsoft\x00AE Windows\x00AE Operating System" VALUE "ProductVersion", VER_STRING END END diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm index e14e7da79afb4..fa674c3979e01 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm @@ -704,7 +704,7 @@ ENDIF ; r13 - Optionally supplies the address of the matrix B zero point buffer. ; -ProcessCountM MACRO RowCount, ASigned, BSigned, Fallthrough +ProcessCountM MACRO RowCount, ASigned, BSigned, ExitLabel LOCAL ProcessNextColumnLoop16xN, SkipAccumulateOutput16xNBlock, OutputMasked16xNBlock, ExitProcessCountM, ProcessRemainingCountN, SkipAccumulateOutput8xNBlock, SkipAccumulateOutputMasked16xNBlock, OutputMasked8xNBlock, SkipAccumulateOutputMasked8xNBlock cmp rbp,8 @@ -751,7 +751,7 @@ SkipAccumulateOutput16xNBlock: ExitProcessCountM: mov eax,RowCount - jmp ExitKernel + jmp ExitLabel ProcessRemainingCountN: ProduceOutputBlock 8, RowCount, ASigned, BSigned @@ -842,7 +842,7 @@ SkipAccumulateOutputMasked8xNBlock: ; ; -ProcessCount1AvxVnni MACRO RowCount, ASigned, BSigned, Fallthrough +ProcessCount1AvxVnni MACRO RowCount, ASigned, BSigned, ExitLabel LOCAL LProcessNextColumnLoop32xN1, LSkipAccumulateOutputMasked32xNBlock1, LProcessNextColumnLoop16xN1, LSkipAccumulateOutput16xNBlock1, LProcessRemainingCountN1, LSkipAccumulateOutput8xNBlock1, LExitProcessCountM1, LOutputMasked32xNBlock1, LSkipAccumulateOutput32xNBlock1, LOutputMasked24xNBlock1, LSkipAccumulateOutputMasked24xNBlock1, LOutputMasked16xNBlock1, LSkipAccumulateOutputMasked16xNBlock1, LOutputMasked8xNBlock1, LSkipAccumulateOutputMasked8xNBlock1 cmp rbp,8 @@ -909,7 +909,7 @@ LSkipAccumulateOutput8xNBlock1: LExitProcessCountM1: ; num of cols = 0, we are done mov eax, 1 - jmp ExitKernel + jmp ExitLabel ;; -- Section to write final tail of C matrix and exit -- ;; ;; write <= 32 elements ;; @@ -1095,13 +1095,13 @@ CheckCountM4OrMore: je ProcessCountM1 ProcessCountM2: - ProcessCountM 2, ASigned, BSigned + ProcessCountM 2, ASigned, BSigned, ExitKernel ProcessCountM4: - ProcessCountM 4, ASigned, BSigned + ProcessCountM 4, ASigned, BSigned, ExitKernel ProcessCountM6: - ProcessCountM 6, ASigned, BSigned + ProcessCountM 6, ASigned, BSigned, ExitKernel ; ; Restore non-volatile registers and return. @@ -1135,16 +1135,16 @@ ExitKernel: ProcessCountM1: cmp DWORD PTR GemmInt8KernelFrame.PreviousP1Home[rsp],-1 je ProcessCountM1AvxVnni - ProcessCountM 1, ASigned, BSigned + ProcessCountM 1, ASigned, BSigned, ExitKernel ProcessCountM1AvxVnni: - ProcessCount1AvxVnni 1, ASigned, BSigned + ProcessCount1AvxVnni 1, ASigned, BSigned, ExitKernel ProcessCountM3: - ProcessCountM 3, ASigned, BSigned + ProcessCountM 3, ASigned, BSigned, ExitKernel ProcessCountM5: - ProcessCountM 5, ASigned, BSigned + ProcessCountM 5, ASigned, BSigned, ExitKernel ENDM diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 07d5dfc9c0b22..3a953dd97504f 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -40,7 +40,7 @@ limitations under the License. #include "core/platform/windows/hardware_core_enumerator.h" #endif #include -#include +#include #include "core/platform/path_lib.h" // for LoopDir() #include "core/platform/windows/dll_load_error.h" diff --git a/onnxruntime/test/shared_lib/test_model_loading.cc b/onnxruntime/test/shared_lib/test_model_loading.cc index 93a81f2960a52..8b955d4358f36 100644 --- a/onnxruntime/test/shared_lib/test_model_loading.cc +++ b/onnxruntime/test/shared_lib/test_model_loading.cc @@ -13,7 +13,7 @@ #include "gmock/gmock.h" #ifdef _WIN32 -#include +#include #else #include #include From 1871d419e180cd7ebe004ce6c05bc566f60c1830 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Wed, 8 Jul 2026 16:47:59 -0700 Subject: [PATCH 06/14] Address cross-build review: pin llvm.sh, quote RC includes, movable ScopedHandle Follow-up to the Windows-x64 Linux cross-build, addressing review findings: - Pin and verify the SHA256 of apt.llvm.org/llvm.sh before running it as root, matching the existing xwin pin. The script is fetched over the network and executed with sudo; verifying its digest prevents a compromised or rotated script from running as root on the runner. Documented that the digest must be re-verified when the upstream script legitimately changes. - Quote the Windows SDK/CRT include paths in CMAKE_RC_FLAGS_INIT so resource compilation still works if the xwin root ever resolves to a path with spaces. - Give the ONNX ScopedHandle replacement a move constructor and move assignment (using release()), restoring the movability the original ScopedResource-based alias effectively lacked but that future ONNX code may expect. Copy remains deleted. Verified the patch still applies cleanly to onnx v1.22.0 and that the full onnxruntime.dll relinks with the updated header. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/linux_win_cross_build.yml | 7 +++++++ cmake/patches/onnx/onnx.patch | 12 +++++++++++- cmake/win_x64_crosscompile_toolchain.cmake | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_win_cross_build.yml b/.github/workflows/linux_win_cross_build.yml index f6f437f46e909..c38d91f7d7a33 100644 --- a/.github/workflows/linux_win_cross_build.yml +++ b/.github/workflows/linux_win_cross_build.yml @@ -64,6 +64,12 @@ env: # assets are mutable, so pin the digest and verify it rather than trusting the # release (or its co-located .sha256 sidecar, which an attacker could swap too). XWIN_SHA256: '31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd' + # SHA256 of apt.llvm.org/llvm.sh, which is fetched and run as root. The script + # is maintained upstream (opencollab/llvm-jenkins.debian.net) and changes when + # new LLVM releases are added, so this digest must be re-verified and updated + # when it legitimately changes; a mismatch fails the job rather than silently + # executing tampered/rotated content as root. + LLVM_SH_SHA256: '9474ecd78b52aba6e923976b1e9773f5613027cc7e237b9956986cb536e02a36' jobs: win-cross-build: @@ -85,6 +91,7 @@ jobs: # LLVM (clang-cl, lld-link, llvm-ml, llvm-lib, llvm-mt, llvm-rc) from apt.llvm.org. curl -fsSL https://apt.llvm.org/llvm.sh -o /tmp/llvm.sh + echo "${LLVM_SH_SHA256} /tmp/llvm.sh" | sha256sum -c - chmod +x /tmp/llvm.sh sudo /tmp/llvm.sh "${LLVM_VERSION}" sudo apt-get install -y --no-install-recommends "llvm-${LLVM_VERSION}" diff --git a/cmake/patches/onnx/onnx.patch b/cmake/patches/onnx/onnx.patch index 2a483fb799108..aac34acd1c128 100644 --- a/cmake/patches/onnx/onnx.patch +++ b/cmake/patches/onnx/onnx.patch @@ -50,7 +50,7 @@ diff --git a/onnx/common/scoped_resource.h b/onnx/common/scoped_resource.h index 525e4d6..2c41f0b 100644 --- a/onnx/common/scoped_resource.h +++ b/onnx/common/scoped_resource.h -@@ -56,7 +56,31 @@ class ScopedResource { +@@ -56,7 +56,41 @@ class ScopedResource { inline void close_handle(HANDLE h) { CloseHandle(h); } @@ -77,6 +77,16 @@ index 525e4d6..2c41f0b 100644 + val_ = INVALID_HANDLE_VALUE; + return tmp; + } ++ ScopedHandle(ScopedHandle&& other) : val_(other.release()) {} ++ ScopedHandle& operator=(ScopedHandle&& other) { ++ if (this != &other) { ++ if (val_ != INVALID_HANDLE_VALUE) { ++ close_handle(val_); ++ } ++ val_ = other.release(); ++ } ++ return *this; ++ } + ScopedHandle(const ScopedHandle&) = delete; + ScopedHandle& operator=(const ScopedHandle&) = delete; +}; diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake index 83a1de168be1f..6bc9e193e1879 100644 --- a/cmake/win_x64_crosscompile_toolchain.cmake +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -96,7 +96,7 @@ add_link_options( # llvm-rc (resource compiler, used for the DLL's version resource) does not honor # /imsvc, so give it the Windows SDK headers on its own include path. set(CMAKE_RC_FLAGS_INIT - "/I ${_ort_crt}/include /I ${_ort_sdk}/include/um /I ${_ort_sdk}/include/shared /I ${_ort_sdk}/include/ucrt") + "/I \"${_ort_crt}/include\" /I \"${_ort_sdk}/include/um\" /I \"${_ort_sdk}/include/shared\" /I \"${_ort_sdk}/include/ucrt\"") # MLAS amd64 .asm via llvm-ml: unlike ml64.exe, llvm-ml is not implicitly 64-bit, # does not search the source file's own directory for includes, and needs the From f3416559b0743993accd9d91a31aeb703d1abf9b Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Wed, 8 Jul 2026 17:14:01 -0700 Subject: [PATCH 07/14] Cross-build: force -mmmx so xwin intrin.h pulls a valid clang mmintrin.h The CI full-DLL cross-build failed compiling ordinary TUs (capture.cc, ostream_sink.cc): the xwin CRT transitively includes clang's , and on the apt.llvm.org clang-19 snapshot the 64-bit MMX vector typedefs (e.g. __v4hi) are only treated as vectors when 'mmx' is in the global target features. Without it the vector_size attribute is dropped and the header fails ("__v4hi (aka 'short')"). Enabling -mmmx globally in the cross toolchain keeps those typedefs vectors. Cross-only (toolchain file); no effect on the native MSVC build or ORT's own SIMD codegen (ORT uses no MMX). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmake/win_x64_crosscompile_toolchain.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake index 6bc9e193e1879..9d3d1e72375c6 100644 --- a/cmake/win_x64_crosscompile_toolchain.cmake +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -88,6 +88,16 @@ string(JOIN " " _ort_win_include_flags add_compile_options("$<$:SHELL:${_ort_win_include_flags}>") add_compile_options("$<$:-Wno-unused-command-line-argument>") +# The xwin CRT's transitively pulls clang's , whose legacy +# MMX intrinsics use the 64-bit integer vector typedefs (e.g. __v4hi). Some clang +# x86_64 builds (notably the apt.llvm.org snapshot used in CI) only treat those as +# real vectors when 'mmx' is in the *global* target features; without it the +# vector_size attribute is dropped and the header fails to compile +# ("__v4hi (aka 'short')"). Enable mmx globally so the header's typedefs stay +# vectors. ORT itself uses no MMX; this only affects the transitively-included +# header, not codegen for ORT's own SIMD kernels. +add_compile_options("$<$:-mmmx>") + add_link_options( "/libpath:${_ort_crt}/lib/x86_64" "/libpath:${_ort_sdk}/lib/ucrt/x86_64" From 982967e10ab1a2100df13651f00582b16d157f1c Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Thu, 9 Jul 2026 12:22:25 -0700 Subject: [PATCH 08/14] Cross-build: don't poison __attribute__ under clang-cl (fixes mmintrin.h) capture.h defined `__attribute__(x)` to nothing under _MSC_VER to disable it "for MSVC". But clang-cl also defines _MSC_VER, and clang's own intrinsic headers rely on __attribute__((__vector_size__)) for the 64-bit MMX vector typedefs (__v2si/__v4hi/__v8qi). Because capture.cc includes capture.h before logging.h, the poison strips those attributes from the later absl -> -> chain, collapsing e.g. __v2si to scalar int and failing to compile with 20 "__v4hi (aka 'short')" errors. Real cl.exe is unaffected (its uses no __attribute__). Narrow the poison to real MSVC only (`#if !defined(__clang__)`); clang-cl keeps real __attribute__, which it supports. Also revert the earlier inert `-mmmx` cross-toolchain option (no-op on the MSVC target; did not fix this). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmake/win_x64_crosscompile_toolchain.cmake | 10 ---------- include/onnxruntime/core/common/logging/capture.h | 8 +++++++- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/cmake/win_x64_crosscompile_toolchain.cmake b/cmake/win_x64_crosscompile_toolchain.cmake index 9d3d1e72375c6..6bc9e193e1879 100644 --- a/cmake/win_x64_crosscompile_toolchain.cmake +++ b/cmake/win_x64_crosscompile_toolchain.cmake @@ -88,16 +88,6 @@ string(JOIN " " _ort_win_include_flags add_compile_options("$<$:SHELL:${_ort_win_include_flags}>") add_compile_options("$<$:-Wno-unused-command-line-argument>") -# The xwin CRT's transitively pulls clang's , whose legacy -# MMX intrinsics use the 64-bit integer vector typedefs (e.g. __v4hi). Some clang -# x86_64 builds (notably the apt.llvm.org snapshot used in CI) only treat those as -# real vectors when 'mmx' is in the *global* target features; without it the -# vector_size attribute is dropped and the header fails to compile -# ("__v4hi (aka 'short')"). Enable mmx globally so the header's typedefs stay -# vectors. ORT itself uses no MMX; this only affects the transitively-included -# header, not codegen for ORT's own SIMD kernels. -add_compile_options("$<$:-mmmx>") - add_link_options( "/libpath:${_ort_crt}/lib/x86_64" "/libpath:${_ort_sdk}/lib/ucrt/x86_64" diff --git a/include/onnxruntime/core/common/logging/capture.h b/include/onnxruntime/core/common/logging/capture.h index 13d3a3ad17aff..b9aaacf23cbca 100644 --- a/include/onnxruntime/core/common/logging/capture.h +++ b/include/onnxruntime/core/common/logging/capture.h @@ -44,7 +44,13 @@ class Capture { #ifdef _MSC_VER // add SAL annotation for printf format string. requires Code Analysis to run to validate usage. #define msvc_printf_check _Printf_format_string_ -#define __attribute__(x) // Disable for MSVC. Supported by GCC and CLang. +#if !defined(__clang__) +// Disable __attribute__ only for real MSVC (cl.exe). clang-cl defines _MSC_VER but +// supports __attribute__, and its own intrinsic headers (e.g. mmintrin.h reached via +// ) rely on __attribute__((__vector_size__)); poisoning it here would strip +// those vector typedefs and break the headers. +#define __attribute__(x) // Supported by GCC and Clang (including clang-cl). +#endif #else #define msvc_printf_check #endif From 824fd8498b2c7f1ed6fe848030d5d58363e7e820 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Thu, 9 Jul 2026 13:09:07 -0700 Subject: [PATCH 09/14] Cross-build: lowercase #pragma comment(lib) names for xwin/lld-link lld-link on a case-sensitive filesystem could not open 'PathCch.lib' at the final onnxruntime.dll link: xwin splats case-variant symlinks (pathcch.lib, PATHCCH.lib) but not the mixed-case "PathCch.lib" that path_lib.cc requested via `#pragma comment(lib, "PathCch.lib")`. Native link.exe is case-insensitive, so lowercase "pathcch.lib" resolves on both Windows and the cross toolchain. Lowercase both PathCch.lib pragmas in path_lib.cc, and proactively lowercase Dbghelp.lib -> dbghelp.lib in debug_alloc.cc (a debug-only TU with the same latent hazard). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- onnxruntime/core/platform/path_lib.cc | 4 ++-- onnxruntime/core/platform/windows/debug_alloc.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/platform/path_lib.cc b/onnxruntime/core/platform/path_lib.cc index 4f7cc52639a8a..9468b1628b29a 100644 --- a/onnxruntime/core/platform/path_lib.cc +++ b/onnxruntime/core/platform/path_lib.cc @@ -19,10 +19,10 @@ #define WINAPI_FAMILY WINAPI_FAMILY_DESKTOP_APP #include #pragma pop_macro("WINAPI_FAMILY") -#pragma comment(lib, "PathCch.lib") +#pragma comment(lib, "pathcch.lib") #else #include -#pragma comment(lib, "PathCch.lib") +#pragma comment(lib, "pathcch.lib") #endif #else #include diff --git a/onnxruntime/core/platform/windows/debug_alloc.cc b/onnxruntime/core/platform/windows/debug_alloc.cc index 14eb06a9cfa39..5bf7f417662d7 100644 --- a/onnxruntime/core/platform/windows/debug_alloc.cc +++ b/onnxruntime/core/platform/windows/debug_alloc.cc @@ -33,7 +33,7 @@ constexpr int c_callstack_limit = 32; // Maximum depth of callstack in leak tra #include #include "debug_alloc.h" #include -#pragma comment(lib, "Dbghelp.lib") +#pragma comment(lib, "dbghelp.lib") // If you are seeing errors of // "Error LNK2005: "void __cdecl operator delete(void *)" (??3@YAXPEAX@Z) already defined in LIBCMTD.lib(delete_scalar.obj)" From 42e071c579d9486c82ef8736edb5e4507a3868db Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Thu, 9 Jul 2026 16:24:29 -0700 Subject: [PATCH 10/14] MLAS amd64: fix ml64 A2039 (line too long) on over-long LOCAL lists The dual-assembler port added multi-label LOCAL directives to macros invoked multiple times, because llvm-ml lacks ml64's default OPTION SCOPED per-PROC label scoping and would otherwise see duplicate global labels. Three of those LOCAL logical lines exceed ml64's ~512-char line limit, breaking the native Windows (MASM) build with A2039 plus a cascade of A2070 errors: QgemmU8S8KernelAvx2.asm MlasGemmCopyPackAAvx2 / MlasGemmCopyPackBAvx2 ConvSymKernelAvx2.asm ConvSymKernelFunction Replace LOCAL with token-paste unique-label suffixes (matching the port's existing ExitKernel&KernelType& pattern). A per-macro literal discriminator plus the macro parameter guarantees globally-unique labels across every invocation: MlasGemmCopyPackAAvx2(ASigned) ->