diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 34a15076c8a7f..f7bc4977bb0bd 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -7517,6 +7517,45 @@ struct OrtApi { */ ORT_API2_STATUS(KernelContext_GetSyncStream, _In_ const OrtKernelContext* context, _Outptr_result_maybenull_ OrtSyncStream** out); + + /** \brief Set the file path of the original (source) ONNX model for weightless EPContext sessions. + * + * When creating a session from a weightless EPContext model, the EP may need access to the source model's + * initializer data. This function sets the file path to the source model, overriding the + * "onnx_model_filename" attribute in the EPContext node if one is present. + * + * This is useful when the source model is deployed to a different location than where it was at compile time. + * + * The file must remain accessible for the lifetime of the session. + * + * \param[in] options The OrtSessionOptions instance. + * \param[in] source_model_path Null terminated string of the file path (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.29. + */ + ORT_API2_STATUS(SessionOptionsSetWeightlessSourceModelPath, _Inout_ OrtSessionOptions* options, + _In_ const ORTCHAR_T* source_model_path); + + /** \brief Set the source ONNX model as a byte buffer for weightless EPContext sessions. + * + * When creating a session from a weightless EPContext model, the EP may need access to the source model's + * initializer data. This function provides the source model as an in-memory byte buffer, for scenarios + * where the source model is not available as a file on disk (e.g., loaded from a package or downloaded). + * + * The caller retains ownership of the buffer and must ensure it remains valid for the lifetime of the session. + * + * \param[in] options The OrtSessionOptions instance. + * \param[in] source_model_data Pointer to the source model byte buffer. + * \param[in] source_model_data_length Size of the byte buffer in bytes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.29. + */ + ORT_API2_STATUS(SessionOptionsSetWeightlessSourceModelFromBuffer, _Inout_ OrtSessionOptions* options, + _In_ const void* source_model_data, _In_ size_t source_model_data_length); }; /* @@ -8362,6 +8401,30 @@ struct OrtCompileApi { ORT_API2_STATUS(ModelCompilationOptions_SetInputModel, _In_ OrtModelCompilationOptions* model_compile_options, _In_ const OrtModel* model); + + /** \brief Enable weightless mode for model compilation. + * + * When enabled, the compiled EPContext model will not embed constant initializer data in the EP's + * compiled binary. Instead, the initializer data must be provided when creating a session from the + * compiled model, either from the source model (via the "onnx_model_filename" EPContext node attribute + * or the "ep.context_source_model_path" session option) or from externalized weights. + * + * This enables smaller compiled models and allows sharing initializer data across multiple compiled + * model variants (e.g., multi-platform caches for different hardware generations). + * + * ORT verifies that the target EP supports weightless mode by calling OrtEpApi::GetWeightlessSupport(). + * If the EP does not support weightless mode, an error is returned. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] use_weightless If true, enable weightless mode for the compiled model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.29. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetWeightlessCache, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ bool use_weightless); }; /** diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 683528304f7a0..1fa9b07822afc 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2111,6 +2111,26 @@ typedef enum OrtGraphCaptureNodeAssignmentPolicy { OrtGraphCaptureNodeAssignmentPolicy_ALLOW_CPU_FOR_SHAPES = 1, } OrtGraphCaptureNodeAssignmentPolicy; +/** + * \brief Describes the scope of an EP's weightless mode support. + * + * Returned by OrtEp::GetWeightlessSupport() to indicate which types of initializers + * the EP can operate on without copying. + * + * \since Version 1.29. + */ +typedef enum OrtWeightlessSupport { + /** EP does not support weightless mode. */ + OrtWeightlessSupport_NONE = 0, + + /** EP supports weightless mode for external initializers only. + * Internal initializers are still copied by the EP during compilation. */ + OrtWeightlessSupport_EXTERNAL_ONLY = 1, + + /** EP supports weightless mode for all initializers (internal and external). */ + OrtWeightlessSupport_ALL = 2, +} OrtWeightlessSupport; + /** * \brief The OrtEp struct provides functions to implement for an execution provider. * \since Version 1.22. @@ -2630,6 +2650,38 @@ struct OrtEp { * \since Version 1.27. */ ORT_API2_STATUS(ReleaseCapturedGraph, _In_ OrtEp* this_ptr, _In_ int graph_annotation_id); + + /** \brief Query the execution provider's weightless mode support. + * + * When weightless mode is enabled (via the "ep.enable_weightless" session option), ORT calls this function + * to determine the scope of the EP's weightless support. The EP returns an OrtWeightlessSupport value + * indicating whether it supports weightless mode for all initializers, external initializers only, or not + * at all. + * + * The EP's response may depend on the underlying hardware or driver capabilities. For example, an EP may + * support weightless mode for all initializers on newer hardware but only for external initializers on + * older hardware that requires weight transformation. + * + * EPs that support weightless mode should set drop_constant_initializers to false in OrtNodeFusionOptions + * so that ORT provides the initializer data as inputs to the compiled/fused node. The EP can then access + * these initializers at Compute() time via KernelContext_GetInput(). + * + * \note Extending the lifetime of initializer data obtained via ValueInfo_GetInitializerValue() during + * Compile() so that the EP can cache and reuse data pointers directly (without going through + * KernelContext) is planned but not yet implemented. Until then, KernelContext_GetInput() is the + * only supported way to access initializer data at Compute() time. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] support Output parameter set to the EP's weightless support scope. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes the EP does not + * support weightless mode (equivalent to OrtWeightlessSupport_NONE). + * + * \since Version 1.29. + */ + ORT_API2_STATUS(GetWeightlessSupport, _In_ const OrtEp* this_ptr, _Out_ OrtWeightlessSupport* support); }; /** \brief The function signature that ORT will call to create OrtEpFactory instances. diff --git a/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h b/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h index cc83b7bca50c5..027ae172a2bbb 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h @@ -30,3 +30,18 @@ static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path" // if this metadata key is not present. // - "1": OrtHardwareDevice is virtual. static const char* const kOrtHardwareDevice_MetadataKey_IsVirtual = "is_virtual"; + +// Key for the execution provider's weightless mode support on a specific device. +// Set by the EP during GetSupportedDevices() via CreateEpDevice() metadata. +// The app can read it via EpDevice_EpMetadata() to check device-specific weightless capability +// before calling ModelCompilationOptions_SetWeightlessCache(). +// +// Possible values: +// - "none": EP does not support weightless mode on this device. This is the assumed default value +// if this metadata key is not present. +// - "external_only": EP supports weightless mode for external initializers only (e.g., older +// hardware/driver that must transform internal constants). +// - "all": EP supports weightless mode for all initializers (internal and external). +// +// \since Version 1.29. +static const char* const kOrtEpDevice_EpMetadataKey_WeightlessSupport = "weightless_support"; diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index c29f43b5e0d30..b52d3b7f72e77 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -583,8 +583,50 @@ static const char* const kOrtSessionOptionsRecordEpGraphAssignmentInfo = "sessio // Option values: // - "0": disable. (default) // - "1": enable. +// +// \deprecated Since version 1.29. Use "ep.enable_weightless" instead, which covers all initializers +// (internal and external) and works in both JIT and AOT flows. static const char* const kOrtSessionOptionEpEnableWeightlessEpContextNodes = "ep.enable_weightless_ep_context_nodes"; +// Enable weightless mode for all initializers (internal and external). +// +// When enabled, ONNX Runtime requests that the execution provider operate without embedding or copying +// constant initializers. +// +// This option works in both JIT (non-cached) and AOT (EPContext model) flows: +// - JIT: The EP should set drop_constant_initializers to false in OrtNodeFusionOptions so that ORT +// provides the initializer data as inputs to the compiled/fused node. The EP can then access these +// initializers at Compute() time via KernelContext_GetInput(). +// NOTE: Extending the lifetime of initializer data obtained via ValueInfo_GetInitializerValue() during +// Compile() so that the EP can cache and reuse data pointers directly is planned but not yet implemented. +// - AOT: ORT generates EPContext models with weightless EPContext nodes. The EP should use the +// "onnx_model_filename" EPContext node attribute or the "ep.context_source_model_path" session option +// to locate the source model's initializer data when creating a session from the compiled model. +// +// ORT checks that the EP supports weightless mode by calling OrtEpApi::GetWeightlessSupport(). +// If the EP does not support it, ORT returns an error. +// +// Option values: +// - "0": disable. (default) +// - "1": enable. +// +// \since Version 1.29. +static const char* const kOrtSessionOptionEpEnableWeightless = "ep.enable_weightless"; + +// Specifies the file path to the original (source) ONNX model when creating a session with a weightless +// EPContext model. +// +// When an EPContext model is generated with weightless mode ("ep.enable_weightless" = "1"), the compiled +// model may not contain the original initializer data. When creating a session from the compiled model, +// the EP needs to load the initializer data from the source model. This session option provides the +// runtime location of the source model, which may differ from the path used at compile time (stored in +// the EPContext node's "onnx_model_filename" attribute). +// +// If not set, the EP falls back to the "onnx_model_filename" attribute in the EPContext node. +// +// \since Version 1.29. +static const char* const kOrtSessionOptionEpContextSourceModelPath = "ep.context_source_model_path"; + // Controls the intra-op thread pool size for a session. // Value should be a base-10 int32 string. // Equivalent to OrtApi::SetIntraOpNumThreads. diff --git a/onnxruntime/core/session/abi_session_options_impl.h b/onnxruntime/core/session/abi_session_options_impl.h index c2a5d4d9985f8..5d3ddf89f4469 100644 --- a/onnxruntime/core/session/abi_session_options_impl.h +++ b/onnxruntime/core/session/abi_session_options_impl.h @@ -39,4 +39,10 @@ struct OrtSessionOptions { // with GetProviderOptionPrefix returning 'ep.myep.' // CUDAExecutionProvider uses the stable short prefix 'ep.cuda.'. static std::string GetProviderOptionPrefix(const char* provider_name); + + // Weightless source model for EPContext sessions. + // Set via SessionOptionsSetWeightlessSourceModelPath or SessionOptionsSetWeightlessSourceModelFromBuffer. + onnxruntime::PathString weightless_source_model_path; + const void* weightless_source_model_data = nullptr; + size_t weightless_source_model_data_size = 0; }; diff --git a/onnxruntime/core/session/compile_api.cc b/onnxruntime/core/session/compile_api.cc index 54d26021d8c99..db479ec75ea9a 100644 --- a/onnxruntime/core/session/compile_api.cc +++ b/onnxruntime/core/session/compile_api.cc @@ -327,6 +327,22 @@ ORT_API_STATUS_IMPL(OrtCompileAPI::ModelCompilationOptions_SetInputModel, API_IMPL_END } +ORT_API_STATUS_IMPL(OrtCompileAPI::ModelCompilationOptions_SetWeightlessCache, + _In_ OrtModelCompilationOptions* ort_model_compile_options, + _In_ bool use_weightless) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + auto model_compile_options = reinterpret_cast(ort_model_compile_options); + ORT_API_RETURN_IF_STATUS_NOT_OK(model_compile_options->SetWeightlessCache(use_weightless)); + return nullptr; +#else + ORT_UNUSED_PARAMETER(ort_model_compile_options); + ORT_UNUSED_PARAMETER(use_weightless); + return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "Compile API is not supported in this build"); +#endif // !defined(ORT_MINIMAL_BUILD) + API_IMPL_END +} + ORT_API_STATUS_IMPL(OrtCompileAPI::CompileModel, _In_ const OrtEnv* env, _In_ const OrtModelCompilationOptions* ort_model_compile_options) { API_IMPL_BEGIN @@ -367,6 +383,8 @@ static constexpr OrtCompileApi ort_compile_api = { &OrtCompileAPI::ModelCompilationOptions_SetInputModel, // End of Version 24 - DO NOT MODIFY ABOVE + + &OrtCompileAPI::ModelCompilationOptions_SetWeightlessCache, }; // checks that we don't violate the rule that the functions must remain in the slots they were originally assigned diff --git a/onnxruntime/core/session/compile_api.h b/onnxruntime/core/session/compile_api.h index e8f171ee24295..81b81c0378bdf 100644 --- a/onnxruntime/core/session/compile_api.h +++ b/onnxruntime/core/session/compile_api.h @@ -45,4 +45,8 @@ ORT_API_STATUS_IMPL(ModelCompilationOptions_SetInputModel, _In_ OrtModelCompilationOptions* model_compile_options, _In_ const OrtModel* model); +ORT_API_STATUS_IMPL(ModelCompilationOptions_SetWeightlessCache, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ bool use_weightless); + } // namespace OrtCompileAPI diff --git a/onnxruntime/core/session/model_compilation_options.cc b/onnxruntime/core/session/model_compilation_options.cc index 9f6d1f9f1a9bc..6c5f26715be7b 100644 --- a/onnxruntime/core/session/model_compilation_options.cc +++ b/onnxruntime/core/session/model_compilation_options.cc @@ -251,6 +251,15 @@ Status ModelCompilationOptions::SetGraphOptimizationLevel(GraphOptimizationLevel return Status::OK(); } +Status ModelCompilationOptions::SetWeightlessCache(bool use_weightless) { + use_weightless_cache_ = use_weightless; + if (use_weightless) { + ORT_RETURN_IF_ERROR( + session_options_.value.config_options.AddConfigEntry(kOrtSessionOptionEpEnableWeightless, "1")); + } + return Status::OK(); +} + Status ModelCompilationOptions::Check() const { const ConfigOptions& config_options = session_options_.value.config_options; diff --git a/onnxruntime/core/session/model_compilation_options.h b/onnxruntime/core/session/model_compilation_options.h index a15af565c4d54..d7f4ddc691b83 100644 --- a/onnxruntime/core/session/model_compilation_options.h +++ b/onnxruntime/core/session/model_compilation_options.h @@ -182,6 +182,14 @@ class ModelCompilationOptions { /// Status SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); + /// + /// Enable weightless mode for model compilation. + /// When enabled, the compiled EPContext model will not embed constant initializer data. + /// + /// True to enable weightless mode + /// Status indicating potential error + Status SetWeightlessCache(bool use_weightless); + /// /// Checks if the compilation options described by this object are valid. /// @@ -235,6 +243,7 @@ class ModelCompilationOptions { const void* input_model_data_ = nullptr; size_t input_model_data_size_ = 0; const OrtModel* input_model_ = nullptr; // Borrowed pointer + bool use_weightless_cache_ = false; }; } // namespace onnxruntime #endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/core/session/model_package/model_package_context.cc b/onnxruntime/core/session/model_package/model_package_context.cc index 351ef2498dcd5..6a3eecd514fae 100644 --- a/onnxruntime/core/session/model_package/model_package_context.cc +++ b/onnxruntime/core/session/model_package/model_package_context.cc @@ -33,7 +33,8 @@ bool IsModelPackagePathSessionOption(std::string_view key) { // Session-option config keys whose values are path references (sha256:, relative, or // absolute) that must be resolved against the model package. Add new path-valued keys here. return key == kOrtSessionOptionsModelExternalInitializersFileFolderPath || - key == kOrtSessionOptionEpContextFilePath; + key == kOrtSessionOptionEpContextFilePath || + key == kOrtSessionOptionEpContextSourceModelPath; } namespace { diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index e55af70915a39..27a356d49c87c 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -2758,6 +2759,42 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSes API_IMPL_END } +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsSetWeightlessSourceModelPath, _Inout_ OrtSessionOptions* options, + _In_ const ORTCHAR_T* source_model_path) { + API_IMPL_BEGIN + if (source_model_path == nullptr || source_model_path[0] == '\0') { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Invalid source model path: path is null or empty"); + } + + if (!std::filesystem::exists(source_model_path)) { + return OrtApis::CreateStatus(ORT_NO_SUCHFILE, "Weightless source model path does not exist"); + } + + options->weightless_source_model_path = source_model_path; + options->weightless_source_model_data = nullptr; + options->weightless_source_model_data_size = 0; + return nullptr; + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsSetWeightlessSourceModelFromBuffer, _Inout_ OrtSessionOptions* options, + _In_ const void* source_model_data, _In_ size_t source_model_data_length) { + API_IMPL_BEGIN + if (source_model_data == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Invalid source model: data pointer is null"); + } + + if (source_model_data_length == 0) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Invalid source model: data size is 0"); + } + + options->weightless_source_model_data = source_model_data; + options->weightless_source_model_data_size = source_model_data_length; + options->weightless_source_model_path.clear(); + return nullptr; + API_IMPL_END +} + ORT_API(void, OrtApis::ReleaseValueInfo, _Frees_ptr_opt_ OrtValueInfo* value_info) { delete value_info; } @@ -4918,6 +4955,9 @@ static constexpr OrtApi ort_api_1_to_29 = { &OrtApis::GetExperimentalFunction, &OrtApis::KernelContext_GetSyncStream, // End of Version 28 - DO NOT MODIFY ABOVE (see above text for more information) + + &OrtApis::SessionOptionsSetWeightlessSourceModelPath, + &OrtApis::SessionOptionsSetWeightlessSourceModelFromBuffer, }; // OrtApiBase can never change as there is no way to know what version of OrtApiBase is returned by OrtGetApiBase. diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index e747d0d0ab2d8..bec15a247d99f 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -829,4 +829,10 @@ ORT_API_STATUS_IMPL(GetTensorElementTypeAndShapeDataReference, _In_ const OrtVal // Experimental API ORT_API(OrtExperimentalFnPtr, GetExperimentalFunction, _In_ const char* name); +// Weightless source model APIs +ORT_API_STATUS_IMPL(SessionOptionsSetWeightlessSourceModelPath, _Inout_ OrtSessionOptions* options, + _In_ const ORTCHAR_T* source_model_path); +ORT_API_STATUS_IMPL(SessionOptionsSetWeightlessSourceModelFromBuffer, _Inout_ OrtSessionOptions* options, + _In_ const void* source_model_data, _In_ size_t source_model_data_length); + } // namespace OrtApis diff --git a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc index 3710901c42a5e..d0c4c9fe89759 100644 --- a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc +++ b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc @@ -29,6 +29,7 @@ #include "core/session/plugin_ep/ep_event_profiling.h" #include "core/session/ort_apis.h" #include "core/providers/partitioning_utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" namespace onnxruntime { @@ -185,6 +186,20 @@ PluginExecutionProvider::PluginExecutionProvider(UniqueOrtEp ep, const OrtSessio kernel_registry_(std::move(kernel_registry)) { generate_ep_ctx_model_ = session_options.value.GetEpContextGenerationOptions().enable; + // Check if the app requested weightless mode and the EP supports it. + { + const auto weightless_requested = + session_options.value.config_options.GetConfigOrDefault(kOrtSessionOptionEpEnableWeightless, "0") != "0"; + if (weightless_requested && ort_ep_->ort_version_supported >= 29 && ort_ep_->GetWeightlessSupport != nullptr) { + OrtWeightlessSupport support = OrtWeightlessSupport_NONE; + auto* ort_status = ort_ep_->GetWeightlessSupport(ort_ep_.get(), &support); + if (ort_status == nullptr && support != OrtWeightlessSupport_NONE) { + weightless_enabled_ = true; + } + OrtApis::ReleaseStatus(ort_status); + } + } + // Extract EP-scoped session config entries. // Arena options go to session_arena_options_; the rest go to provider_options_. { diff --git a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.h b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.h index 610f0b449c119..480d84f3a6894 100644 --- a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.h +++ b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.h @@ -174,6 +174,7 @@ class PluginExecutionProvider : public IExecutionProvider { std::vector ep_devices_; std::vector allocator_mem_infos_; bool generate_ep_ctx_model_ = false; + bool weightless_enabled_ = false; // Set if app requested weightless AND EP supports it // Provider options extracted from session-level config (excluding arena.*). // Exposed through GetProviderOptions() so the framework reports the effective EP configuration.