diff --git a/src/BUILD b/src/BUILD index dba7366261..82033516d4 100644 --- a/src/BUILD +++ b/src/BUILD @@ -78,7 +78,6 @@ cc_shared_library( "@fmtlib//:__subpackages__", "@com_github_jarro2783_cxxopts//:__subpackages__", "@aws-sdk-cpp//:__subpackages__", - "@drogon//:__subpackages__", "@boost//:__subpackages__", "@com_github_googleapis_google_cloud_cpp//:__subpackages__", "@curl//:__subpackages__", @@ -259,8 +258,6 @@ cc_library( "grpcservermodule.cpp", "grpcservermodule.hpp", "http_rest_api_handler.cpp", - "drogon_endpoints.cpp", - "drogon_endpoints.hpp", "http_server.cpp", "httpservermodule.cpp", "http_rest_api_handler.hpp", @@ -342,8 +339,6 @@ cc_library( "kfs_frontend/kfs_graph_executor_impl.hpp", "http_frontend/http_graph_executor_impl.cpp", "http_frontend/http_graph_executor_impl.hpp", - "http_frontend/drogon_graph_executor_impl.cpp", - "http_frontend/drogon_graph_executor_impl.hpp", ], "//:disable_mediapipe" : [], }), diff --git a/src/drogon_endpoints.cpp b/src/drogon_endpoints.cpp deleted file mode 100644 index fae6615049..0000000000 --- a/src/drogon_endpoints.cpp +++ /dev/null @@ -1,132 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "drogon_endpoints.hpp" - -#include -#include -#include -#include - -#include -#include - -#include "logging.hpp" -#include "modelmanager.hpp" -#include "status.hpp" - -#if (MEDIAPIPE_DISABLE == 0) -#include "http_payload.hpp" -#include "http_frontend/drogon_graph_executor_impl.hpp" -#include "http_frontend/http_client_connection.hpp" -#include "mediapipe_internal/mediapipegraphexecutor.hpp" -#endif - -using rapidjson::Document; -using rapidjson::SizeType; -using rapidjson::Value; - -namespace ovms { - -Status processDrogonV3(ModelManager* mm, const drogon::HttpRequestPtr& req, std::function&& callback) { - SPDLOG_ERROR("Received request for server side event"); - -#if (MEDIAPIPE_DISABLE == 0) - std::string request_body = std::string(req->getBody()); - std::string uri = req->getPath(); - HttpPayload request; - std::shared_ptr doc = std::make_shared(); - std::shared_ptr executor; - bool streamFieldVal = false; - { - doc->Parse(request_body.c_str()); - } - { - if (doc->HasParseError()) { - return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); - } - - if (!doc->IsObject()) { - return Status(StatusCode::JSON_INVALID, "JSON body must be an object"); - } - - auto modelNameIt = doc->FindMember("model"); - if (modelNameIt == doc->MemberEnd()) { - return Status(StatusCode::JSON_INVALID, "model field is missing in JSON body"); - } - - if (!modelNameIt->value.IsString()) { - return Status(StatusCode::JSON_INVALID, "model field is not a string"); - } - - const std::string model_name = modelNameIt->value.GetString(); - - bool isTextGenerationEndpoint = uri.find("completions") != std::string_view::npos; - if (isTextGenerationEndpoint) { - auto streamIt = doc->FindMember("stream"); - if (streamIt != doc->MemberEnd()) { - if (!streamIt->value.IsBool()) { - return Status(StatusCode::JSON_INVALID, "stream field is not a boolean"); - } - streamFieldVal = streamIt->value.GetBool(); - } - } - - auto status = mm->createPipeline(executor, model_name); - if (!status.ok()) { - return status; - } - // request.headers = request_components.headers; // TODO: Not needed - request.body = request_body; - request.parsedJson = doc; - request.uri = std::string(uri); - request.client = std::make_shared(); // TODO: Not needed - } - if (streamFieldVal == false) { - std::string response; - auto resp = drogon::HttpResponse::newHttpResponse(); - ExecutionContext executionContext{ExecutionContext::Interface::REST, ExecutionContext::Method::V3Unary}; - auto status = executor->infer(&request, &response, executionContext); - resp->setBody(response); - callback(resp); - return status; - } else { - auto resp = drogon::HttpResponse::newAsyncStreamResponse( - [executor, request](drogon::ResponseStreamPtr stream) { - std::thread([executor, - request, - stream = std::shared_ptr{std::move(stream)}]() mutable { - ExecutionContext executionContext{ExecutionContext::Interface::REST, ExecutionContext::Method::V3Stream}; - auto status = executor->inferStream(request, stream, executionContext); - stream->close(); - }) - .detach(); - }); - - resp->addHeader("Content-Type", "text/event-stream"); - resp->addHeader("Cache-Control", "no-cache"); - resp->addHeader("Connection", "keep-alive"); - callback(resp); - return StatusCode::OK; - } - -#else - SPDLOG_DEBUG("Mediapipe support was disabled during build process..."); - return StatusCode::NOT_IMPLEMENTED; -#endif - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/drogon_endpoints.hpp b/src/drogon_endpoints.hpp deleted file mode 100644 index eeef2ce090..0000000000 --- a/src/drogon_endpoints.hpp +++ /dev/null @@ -1,29 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include - -#include - -#include "status.hpp" - -namespace ovms { - -class ModelManager; - -Status processDrogonV3(ModelManager* mm, const drogon::HttpRequestPtr& req, std::function&& callback); - -} // namespace ovms diff --git a/src/http_frontend/drogon_graph_executor_impl.cpp b/src/http_frontend/drogon_graph_executor_impl.cpp deleted file mode 100644 index 70faa4c486..0000000000 --- a/src/http_frontend/drogon_graph_executor_impl.cpp +++ /dev/null @@ -1,82 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "drogon_graph_executor_impl.hpp" - -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow_serving/util/net_http/server/public/server_request_interface.h" -#pragma GCC diagnostic pop -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#pragma GCC diagnostic pop -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/status.h" -#pragma GCC diagnostic pop - -#include "../mediapipe_internal/mediapipe_utils.hpp" - -namespace ovms { - -Status onPacketReadySerializeAndSendImpl( - const std::string& requestId, - const std::string& endpointName, - const std::string& endpointVersion, - const std::string& packetName, - const mediapipe_packet_type_enum packetType, - const ::mediapipe::Packet& packet, - HttpReaderWriter& serverReaderWriter) { - std::string out; - OVMS_RETURN_ON_FAIL( - onPacketReadySerializeImpl( - requestId, - endpointName, - endpointVersion, - packetName, - packetType, - packet, - out)); - if (serverReaderWriter->send(out)) { - return StatusCode::OK; - } else { - throw std::logic_error("disconnected"); - return StatusCode::INTERNAL_ERROR; - } -} - -Status sendErrorImpl( - const std::string& message, - HttpReaderWriter& serverReaderWriter) { - serverReaderWriter->send("{\"error\": \"" + message + "\"}"); - return StatusCode::OK; -} - -bool waitForNewRequest( - HttpReaderWriter& serverReaderWriter, - HttpPayload& newRequest) { - return false; -} - -} // namespace ovms diff --git a/src/http_frontend/drogon_graph_executor_impl.hpp b/src/http_frontend/drogon_graph_executor_impl.hpp deleted file mode 100644 index c4e4a27d27..0000000000 --- a/src/http_frontend/drogon_graph_executor_impl.hpp +++ /dev/null @@ -1,133 +0,0 @@ -//***************************************************************************** -// Copyright 2024 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "../http_payload.hpp" -#include "../mediapipe_internal/packettypes.hpp" -#include "../status.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/packet.h" -#pragma GCC diagnostic pop - -#if (PYTHON_DISABLE == 0) -#include "../python/python_backend.hpp" -#endif - -#include - -namespace ovms { - -class PythonBackend; - -using HttpReaderWriter = std::shared_ptr; - -// Deserialization of parameters inside KServe gRPC request -// into mediapipe Packets. -// To be used by both - infer & inferStream. -Status deserializeInputSidePacketsFromFirstRequestImpl( - std::map& inputSidePackets, // out - const HttpPayload& request); // in - -// For unary graph execution request ID is forwarded to serialization function. -const std::string& getRequestId( - const HttpPayload& request); - -// Used by inferStream only. -// Whenever MediaPipe graph produces some packet, this function is triggered. -// Implementation should transform packet into KServe gRPC response and send it. -// Data race safety: -// MediaPipe packet available callbacks can be triggered simultaneously, from different threads. -// However, the graph executor synchronizes it with locking mechanism. -Status onPacketReadySerializeAndSendImpl( - const std::string& requestId, - const std::string& endpointName, - const std::string& endpointVersion, - const std::string& packetName, - const mediapipe_packet_type_enum packetType, - const ::mediapipe::Packet& packet, - HttpReaderWriter& serverReaderWriter); - -// Used by infer only. -// infer produces single response and lets the caller send the response back on its own. -// This function is triggered when output poller has packet ready for serialization. -// Implementation should transform packet into KServe gRPC response and send it. -// Data race safety: -// This is always triggered on the same thread. -Status onPacketReadySerializeImpl( - const std::string& requestId, - const std::string& endpointName, - const std::string& endpointVersion, - const std::string& packetName, - const mediapipe_packet_type_enum packetType, - const ::mediapipe::Packet& packet, - std::string& response); - -// This is called whenever new request is received. -// It is responsible for creating packet(s) out of the request. -// It is also responsive of pushing the packet inside the graph. -// To be used by both - infer & inferStream. -Status createAndPushPacketsImpl( - // The request wrapped in shared pointer. - std::shared_ptr request, - // Graph input name => type mapping. - // Request can contain multiple packets. - // Implementation should validate for existence of such packet type. - stream_types_mapping_t& inputTypes, - // Context for creating Python bufferprotocol packets. - PythonBackend* pythonBackend, - // The graph instance. - // Implementation is required to push the packet down the graph. - ::mediapipe::CalculatorGraph& graph, - // Timestamp to be used if request specified no manual timestamp. - // Implementation is also expected to leave the timestamp in next - // available state for usage in subsequent requests. - ::mediapipe::Timestamp& currentTimestamp, - // Unary (non-streaming) execution requires information about number of packets - // in order to validate if all inputs were fed into the graph. - size_t& numberOfPacketsCreated); - -// This is called before subsequent createAndPushPacketsImpl in inferStream scenario. -// At this point we may reject requests with invalid data. -Status validateSubsequentRequestImpl( - const HttpPayload& request, - const std::string& endpointName, - const std::string& endpointVersion, - stream_types_mapping_t& inputTypes); - -// Data race safety: -// This may be called from different threads. -// However, the caller implements synchronization mechanism -// which prevents writes at the same time. -Status sendErrorImpl( - const std::string& message, - HttpReaderWriter& serverReaderWriter); - -// Imitation of stream.Read(...) in gRPC stream API -// Required for inferStream only. -bool waitForNewRequest( - HttpReaderWriter& serverReaderWriter, - HttpPayload& newRequest); - -} // namespace ovms diff --git a/src/http_frontend/http_client_connection.hpp b/src/http_frontend/http_client_connection.hpp index c60630def0..b361f0f09d 100644 --- a/src/http_frontend/http_client_connection.hpp +++ b/src/http_frontend/http_client_connection.hpp @@ -28,24 +28,17 @@ namespace ovms { class HttpClientConnection : public ClientConnection { - tensorflow::serving::net_http::ServerRequestInterface* serverReaderWriter{nullptr}; + tensorflow::serving::net_http::ServerRequestInterface* serverReaderWriter; public: - HttpClientConnection(tensorflow::serving::net_http::ServerRequestInterface* serverReaderWriter = nullptr) : - // TODO: Remove + HttpClientConnection(tensorflow::serving::net_http::ServerRequestInterface* serverReaderWriter) : serverReaderWriter(serverReaderWriter) {} bool isDisconnected() const override { - if (!serverReaderWriter) { - return false; - } return this->serverReaderWriter->IsDisconnected(); } void registerDisconnectionCallback(std::function fn) override { - if (!serverReaderWriter) { - return; - } serverReaderWriter->RegisterDisconnectionCallback(std::move(fn)); } }; diff --git a/src/http_payload.hpp b/src/http_payload.hpp index 957955d7ee..0195b97c22 100644 --- a/src/http_payload.hpp +++ b/src/http_payload.hpp @@ -29,8 +29,8 @@ namespace ovms { struct HttpPayload { std::string uri; std::vector> headers; - std::string body; // always - std::shared_ptr parsedJson; // pre-parsed body = null + std::string body; // always + rapidjson::Document* parsedJson; // pre-parsed body = null std::shared_ptr client; }; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index af2fd765f1..1680d5cd52 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -446,25 +446,25 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque #if (MEDIAPIPE_DISABLE == 0) OVMS_PROFILE_FUNCTION(); HttpPayload request; - std::shared_ptr doc = std::make_shared(); + Document doc; std::shared_ptr executor; bool streamFieldVal = false; { OVMS_PROFILE_SCOPE("rapidjson parse body"); - doc->Parse(request_body.c_str()); + doc.Parse(request_body.c_str()); } { OVMS_PROFILE_SCOPE("rapidjson validate"); - if (doc->HasParseError()) { + if (doc.HasParseError()) { return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); } - if (!doc->IsObject()) { + if (!doc.IsObject()) { return Status(StatusCode::JSON_INVALID, "JSON body must be an object"); } - auto modelNameIt = doc->FindMember("model"); - if (modelNameIt == doc->MemberEnd()) { + auto modelNameIt = doc.FindMember("model"); + if (modelNameIt == doc.MemberEnd()) { return Status(StatusCode::JSON_INVALID, "model field is missing in JSON body"); } @@ -476,8 +476,8 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque bool isTextGenerationEndpoint = uri.find("completions") != std::string_view::npos; if (isTextGenerationEndpoint) { - auto streamIt = doc->FindMember("stream"); - if (streamIt != doc->MemberEnd()) { + auto streamIt = doc.FindMember("stream"); + if (streamIt != doc.MemberEnd()) { if (!streamIt->value.IsBool()) { return Status(StatusCode::JSON_INVALID, "stream field is not a boolean"); } @@ -492,7 +492,7 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque // TODO: Possibly avoid making copy request.headers = request_components.headers; request.body = request_body; - request.parsedJson = doc; + request.parsedJson = &doc; request.uri = std::string(uri); request.client = std::make_shared(serverReaderWriter); } diff --git a/src/http_server.cpp b/src/http_server.cpp index d3da7ee226..1283a70d5a 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -15,7 +15,6 @@ //***************************************************************************** #include "http_server.hpp" -#include #include #include #include @@ -34,15 +33,17 @@ #include "tensorflow_serving/util/threadpool_executor.h" #pragma GCC diagnostic pop -#include - -#include "drogon_endpoints.hpp" #include "http_rest_api_handler.hpp" -#include "module_names.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" #include "status.hpp" -using namespace drogon; + +// TODO: Use the example when switching from net_http +// #define DROGON +#ifdef DROGON +#include +#include + +#include +#endif namespace ovms { @@ -252,10 +253,39 @@ class RestApiRequestDispatcher { }; std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms) { +#ifdef DROGON + drogon::app().registerHandler("/stream", [](const drogon::HttpRequestPtr&, std::function&& callback) { + SPDLOG_DEBUG("Received request for server side event"); + auto resp = drogon::HttpResponse::newAsyncStreamResponse([](drogon::ResponseStreamPtr stream) { + std::thread([stream = std::shared_ptr{std::move(stream)}]() mutable { + for (int i = 0; i < 3; i++) { + if (!stream->send("data: [hello] \n\n")) { + break; + } + std::this_thread::sleep_for(std::chrono::seconds(2)); + } + stream->close(); + }) + .detach(); // TODO: ? + }); + resp->addHeader("Content-Type", "text/event-stream"); + resp->addHeader("Cache-Control", "no-cache"); + resp->addHeader("Connection", "keep-alive"); + callback(resp); + }); + std::thread([address, port, num_threads]() { + drogon::app() + .setThreadNum(num_threads) + .setIdleConnectionTimeout(0) + .addListener(address, port + 1) // TODO: replace net_http with drogon + .run(); + }) + .detach(); // TODO: ? +#endif + auto options = std::make_unique(); options->AddPort(static_cast(port)); options->SetAddress(address); - num_threads = 1; // save resources for drogon options->SetExecutor(std::make_unique(num_threads)); auto server = net_http::CreateEvHTTPServer(std::move(options)); @@ -281,72 +311,4 @@ std::unique_ptr createAndStartHttpServer(const std::string& address return nullptr; } - -void createAndStartDrogonServer(ovms::Server& ovmsServer, int workers) { - // `registerHandler()` adds a handler to the desired path. The handler is - // responsible for generating a HTTP response upon an HTTP request being - // sent to Drogon - - app().registerHandler( - "/", - [](const HttpRequestPtr&, - std::function&& callback) { - LOG_INFO << "Received request"; - auto resp = HttpResponse::newHttpResponse(); - resp->setBody("Hello, World!"); - callback(resp); - }, - {Get}); - - app().registerHandler( - "/stream", - [](const HttpRequestPtr&, - std::function&& callback) { - LOG_INFO << "Received request for server side event"; - auto resp = drogon::HttpResponse::newAsyncStreamResponse( - [](drogon::ResponseStreamPtr stream) { - std::thread([stream = - std::shared_ptr{ - std::move(stream)}]() mutable { - for (int i = 0; i < 100; i++) { - std::cout << std::boolalpha << stream->send("data: [hello] \n\n") - << std::endl; - std::this_thread::sleep_for(std::chrono::seconds(2)); - } - stream->close(); - }) - .detach(); - }); - resp->setContentTypeCodeAndCustomString( - ContentType::CT_NONE, "text/event-stream"); - callback(resp); - }); - - ovms::ModelManager* mm = &(dynamic_cast(ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME))->getServableManager()); - app().registerHandler( - "/v3/completions", - [mm](const drogon::HttpRequestPtr& req, - std::function&& callback) { - processDrogonV3(mm, req, std::move(callback)); // Call your handler directly - }); - - app().registerHandler( - "/v3/chat/completions", - [mm](const drogon::HttpRequestPtr& req, - std::function&& callback) { - processDrogonV3(mm, req, std::move(callback)); // Call your handler directly - }); - - LOG_INFO << "Server running on 0.0.0.0:11339"; - - std::thread serverThread([workers]() { - app() - .setThreadNum(workers) - .setIdleConnectionTimeout(0) - .addListener("0.0.0.0", 11339) - .run(); - }); - serverThread.detach(); -} - } // namespace ovms diff --git a/src/http_server.hpp b/src/http_server.hpp index ec8c5e6708..88117e3048 100644 --- a/src/http_server.hpp +++ b/src/http_server.hpp @@ -38,5 +38,4 @@ using http_server = tensorflow::serving::net_http::HTTPServerInterface; * @return std::unique_ptr */ std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms = -1); -void createAndStartDrogonServer(ovms::Server& ovmsServer, int workers); } // namespace ovms diff --git a/src/httpservermodule.cpp b/src/httpservermodule.cpp index b426d19a09..2d8840cecc 100644 --- a/src/httpservermodule.cpp +++ b/src/httpservermodule.cpp @@ -46,7 +46,6 @@ Status HTTPServerModule::start(const ovms::Config& config) { state = ModuleState::INITIALIZED; SPDLOG_INFO("{} started", HTTP_SERVER_MODULE_NAME); SPDLOG_INFO("Started REST server at {}", server_address); - ovms::createAndStartDrogonServer(this->ovmsServer, workers); return StatusCode::OK; } void HTTPServerModule::shutdown() { diff --git a/src/llm/http_llm_calculator.cc b/src/llm/http_llm_calculator.cc index 895e167dda..f5d60b0ba0 100644 --- a/src/llm/http_llm_calculator.cc +++ b/src/llm/http_llm_calculator.cc @@ -55,7 +55,6 @@ class HttpLLMCalculator : public CalculatorBase { ov::genai::GenerationHandle generationHandle; std::shared_ptr apiHandler; std::shared_ptr client; - int reqId = -1; // TODO: To be moved to CB library std::shared_ptr streamer; @@ -106,10 +105,6 @@ class HttpLLMCalculator : public CalculatorBase { try { // First iteration of Process() if (!cc->Inputs().Tag(INPUT_TAG_NAME).IsEmpty()) { - static std::atomic gReqId = 0; - RET_CHECK(this->reqId == -1); - reqId = gReqId++; - SPDLOG_INFO("REQUEST ID: {}", reqId); OVMS_PROFILE_SCOPE("Deserialization of first request"); // Check if we did not receive the payload twice RET_CHECK(this->apiHandler == nullptr); @@ -182,7 +177,6 @@ class HttpLLMCalculator : public CalculatorBase { RET_CHECK(this->apiHandler != nullptr); RET_CHECK(this->streamer != nullptr); RET_CHECK(this->client != nullptr); - RET_CHECK(this->reqId >= 0); // Unary scenario if (!this->apiHandler->isStream()) { @@ -202,7 +196,6 @@ class HttpLLMCalculator : public CalculatorBase { cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new OutputDataType{std::move(response)}, timestamp); } else { OVMS_PROFILE_SCOPE("Stream generation cycle"); - SPDLOG_INFO("Cycle for {}", this->reqId); // Streaming scenario // Each iteration is single execution of Process() method