Skip to content

Commit

Permalink
[CPU] Add interface to release compiled model internal memory (#26390)
Browse files Browse the repository at this point in the history
### Details:
This PR introduces an `ov::CompiledModel` level interface that allows to
release memory allocated by the compiled model. In this PR the interface
is only supported by the CPU plugin.

### Tickets:
 - CVS-145873
  • Loading branch information
maxnick authored Sep 5, 2024
1 parent ad3f51b commit 8c9d4be
Show file tree
Hide file tree
Showing 75 changed files with 1,429 additions and 825 deletions.
6 changes: 6 additions & 0 deletions src/inference/dev_api/openvino/runtime/icompiled_model.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@ class OPENVINO_RUNTIME_API ICompiledModel : public std::enable_shared_from_this<
*/
ov::SoPtr<ov::IRemoteContext> get_context() const;

/**
* @brief Release intermediate memory
*
*/
virtual void release_memory();

virtual ~ICompiledModel() = default;

private:
Expand Down
9 changes: 9 additions & 0 deletions src/inference/include/openvino/runtime/compiled_model.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,15 @@ class OPENVINO_RUNTIME_API CompiledModel {
return get_property(property.name()).template as<T>();
}

/**
* @brief Release intermediate memory.
*
* This method forces the Compiled model to release memory allocated for intermediate structures, e.g. caches,
* tensors, temporal buffers etc., when possible
*
*/
void release_memory();

/**
* @brief Returns pointer to device-specific shared context
* on a remote accelerator device that was used to create this CompiledModel.
Expand Down
4 changes: 4 additions & 0 deletions src/inference/src/cpp/compiled_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ Any CompiledModel::get_property(const std::string& name) const {
});
}

void CompiledModel::release_memory() {
OV_COMPILED_MODEL_CALL_STATEMENT(_impl->release_memory());
}

RemoteContext CompiledModel::get_context() const {
OV_COMPILED_MODEL_CALL_STATEMENT({
auto ctx = _impl->get_context();
Expand Down
4 changes: 4 additions & 0 deletions src/inference/src/dev/icompiled_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,7 @@ ov::SoPtr<ov::IRemoteContext> ov::ICompiledModel::get_context() const {
void ov::ICompiledModel::set_model_shared_object(ov::Model& model, const std::shared_ptr<void>& shared_object) {
model.m_shared_object = shared_object;
}

void ov::ICompiledModel::release_memory() {
// nothing to do
}
8 changes: 8 additions & 0 deletions src/plugins/intel_cpu/src/compiled_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -342,5 +342,13 @@ void CompiledModel::export_model(std::ostream& modelStream) const {
serializer << m_model;
}

void CompiledModel::release_memory() {
for (auto&& graph : m_graphs) {
GraphGuard::Lock graph_lock{graph};
auto ctx = graph_lock._graph.getGraphContext();
ctx->getNetworkMemoryControl()->releaseMemory();
}
}

} // namespace intel_cpu
} // namespace ov
2 changes: 2 additions & 0 deletions src/plugins/intel_cpu/src/compiled_model.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ class CompiledModel : public ov::ICompiledModel {
"Set property to Core::compile_model during compilation");
};

void release_memory() override;

private:
std::shared_ptr<ov::ISyncInferRequest> create_sync_infer_request() const override;
friend class SyncInferRequest;
Expand Down
Loading

0 comments on commit 8c9d4be

Please sign in to comment.