Merge pull request #392 from janhq/391-bug-stream-toggle-off-takes-forever-to-respond

391 bug stream toggle off takes forever to respond

Author: tikikun

controllers/llamaCPP.cc

@@ -22,8 +22,9 @@ std::shared_ptr<inferenceState> create_inference_state(llamaCPP *instance) {
 // --------------------------------------------
 
 // Function to check if the model is loaded
-void check_model_loaded(llama_server_context &llama, const HttpRequestPtr &req,
-                      std::function<void(const HttpResponsePtr &)> &callback) {
+void check_model_loaded(
+    llama_server_context &llama, const HttpRequestPtr &req,
+    std::function<void(const HttpResponsePtr &)> &callback) {
   if (!llama.model_loaded_external) {
     Json::Value jsonResp;
     jsonResp["message"] =
@@ -299,13 +300,9 @@ void llamaCPP::chatCompletion(
   LOG_INFO << "Current completion text";
   LOG_INFO << formatted_output;
 #endif
-  int task_id;
-
-  LOG_INFO << "Resolved request for task_id:" << task_id;
 
   if (is_streamed) {
     auto state = create_inference_state(this);
-    state->task_id = task_id;
     auto chunked_content_provider =
         [state, data](char *pBuffer, std::size_t nBuffSize) -> std::size_t {
       if (!state->is_streaming) {
@@ -386,9 +383,12 @@ void llamaCPP::chatCompletion(
   } else {
     Json::Value respData;
     auto resp = nitro_utils::nitroHttpResponse();
+    int task_id = llama.request_completion(data, false, false, -1);
+    LOG_INFO << "sent the non stream, waiting for respone";
     if (!json_value(data, "stream", false)) {
       std::string completion_text;
       task_result result = llama.next_result(task_id);
+      LOG_INFO << "Here is the result:" << result.error;
       if (!result.error && result.stop) {
         int prompt_tokens = result.result_json["tokens_evaluated"];
         int predicted_tokens = result.result_json["tokens_predicted"];