Propagate stop token ids from default generation config in CB

openvinotoolkit · Jan 28, 2025 · fedb16f · fedb16f
1 parent 715cdbd
commit fedb16f
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/cpp/src/continuous_batching_impl.cpp b/src/cpp/src/continuous_batching_impl.cpp
@@ -149,6 +149,9 @@ GenerationHandle
 ContinuousBatchingPipeline::ContinuousBatchingImpl::add_request(uint64_t request_id,
                                                                const ov::Tensor& input_ids,
                                                                ov::genai::GenerationConfig sampling_params) {
+    // If stop_token_ids were not provided, take value from default m_generation_config
+    if (sampling_params.stop_token_ids.empty())
+        sampling_params.stop_token_ids = m_generation_config.stop_token_ids;
     // If eos_token_id was not provided, take value from default m_generation_config
     if (sampling_params.eos_token_id == -1)
         sampling_params.set_eos_token_id(m_generation_config.eos_token_id);