[CPU] Change kvcache type if user sets INFERENCE_PRECISION_HINT (#690)

When user sets `INFERENCE_PRECISION_HINT` change the kvcache type. Ticket: [145861](https://jira.devtools.intel.com/browse/CVS-145861) --------- Co-authored-by: Dariusz Trawinski <[email protected]>
openvinotoolkit · Aug 5, 2024 · f50aaf7 · f50aaf7
1 parent eeabbad
commit f50aaf7
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 2 deletions.
diff --git a/src/cpp/src/continuous_batching_pipeline.cpp b/src/cpp/src/continuous_batching_pipeline.cpp
@@ -84,7 +84,7 @@ class ContinuousBatchingPipeline::Impl {
         // The model can be compiled for GPU as well
         std::shared_ptr<ov::Model> model = core.read_model(models_path + "/openvino_model.xml");
 
-        DeviceConfig device_config(core, scheduler_config, device);
+        DeviceConfig device_config(core, scheduler_config, device, plugin_config);
 
         apply_paged_attention_transformations(model, device_config);
 

diff --git a/src/cpp/src/device_config.hpp b/src/cpp/src/device_config.hpp
@@ -20,7 +20,7 @@ class DeviceConfig {
     std::string m_device;
 
 public:
-    DeviceConfig(ov::Core& core, const SchedulerConfig& scheduling_config, const std::string& device) {
+    DeviceConfig(ov::Core& core, const SchedulerConfig& scheduling_config, const std::string& device, const ov::AnyMap& plugin_config = {}) {
         m_device = device;
 
         // keep information about blocsk
@@ -29,6 +29,20 @@ class DeviceConfig {
         if (m_device == "CPU") {
             auto inference_precision = core.get_property(device, ov::hint::inference_precision);
             m_kv_cache_type = inference_precision == ov::element::bf16 ? ov::element::bf16 : ov::element::f16;
+            // if user sets precision hint, kv cache type should be changed
+            if (plugin_config.find(ov::hint::inference_precision.name()) != plugin_config.end()) {
+                const auto precision = plugin_config.at(ov::hint::inference_precision.name()).as<ov::element::Type>();
+                if (precision == ov::element::f32) {
+                    m_kv_cache_type = ov::element::f32;
+                } else if (precision == ov::element::f16) {
+                    m_kv_cache_type = ov::element::f16;
+                } else if (precision == ov::element::bf16) {
+                    m_kv_cache_type = ov::element::bf16;
+                } else {
+                    // use default f32
+                    m_kv_cache_type = ov::element::f32;
+                }
+            }
         } else if (m_device == "GPU") {
             OPENVINO_ASSERT("GPU is not currently supported. Please, remove this assert and fill configuration");
         } else {