fix: load engine linux (#1790)

* fix: load engine linux

* fix linux

---------

Co-authored-by: vansangpfiev <[email protected]>

Author: namchuai

engine/CMakeLists.txt

@@ -142,6 +142,7 @@ file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/cortex_openapi.h"
 add_executable(${TARGET_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/file_logger.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/utils/dylib_path_manager.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/remote_engine.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/openai_engine.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/anthropic_engine.cc

engine/cli/CMakeLists.txt

@@ -75,6 +75,7 @@ find_package(lfreist-hwinfo CONFIG REQUIRED)
 add_executable(${TARGET_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/../utils/dylib_path_manager.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc

engine/cli/command_line_parser.cc

@@ -48,8 +48,11 @@ constexpr const auto kSubcommands = "Subcommands";
 CommandLineParser::CommandLineParser()
     : app_("\nCortex.cpp CLI\n"),
       download_service_{std::make_shared<DownloadService>()},
-      model_service_{ModelService(download_service_)},
-      engine_service_{EngineService(download_service_)} {}
+      dylib_path_manager_{std::make_shared<cortex::DylibPathManager>()},
+      engine_service_{std::make_shared<EngineService>(download_service_,
+                                                      dylib_path_manager_)} {
+  supported_engines_ = engine_service_->GetSupportedEngineNames().value();
+}
 
 bool CommandLineParser::SetupCommand(int argc, char** argv) {
   app_.usage("Usage:\n" + commands::GetCortexBinary() +
@@ -60,8 +63,6 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
 
   SetupCommonCommands();
 
-  SetupInferenceCommands();
-
   SetupModelCommands();
 
   SetupEngineCommands();
@@ -176,17 +177,11 @@ void CommandLineParser::SetupCommonCommands() {
       return;
     commands::RunCmd rc(cml_data_.config.apiServerHost,
                         std::stoi(cml_data_.config.apiServerPort),
-                        cml_data_.model_id, download_service_);
+                        cml_data_.model_id, engine_service_);
     rc.Exec(cml_data_.run_detach, run_settings_);
   });
 }
 
-void CommandLineParser::SetupInferenceCommands() {
-  // auto embeddings_cmd = app_.add_subcommand(
-  //     "embeddings", "Creates an embedding vector representing the input text");
-  // embeddings_cmd->group(kInferenceGroup);
-}
-
 void CommandLineParser::SetupModelCommands() {
   // Models group commands
   auto models_cmd =
@@ -476,7 +471,7 @@ void CommandLineParser::SetupEngineCommands() {
   list_engines_cmd->callback([this]() {
     if (std::exchange(executed_, true))
       return;
-    commands::EngineListCmd command;
+    auto command = commands::EngineListCmd(engine_service_);
     command.Exec(cml_data_.config.apiServerHost,
                  std::stoi(cml_data_.config.apiServerPort));
   });
@@ -493,9 +488,9 @@ void CommandLineParser::SetupEngineCommands() {
       CLI_LOG(install_cmd->help());
     }
   });
-  for (const auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineInstall(install_cmd, engine_name, cml_data_.engine_version,
+
+  for (const auto& engine : supported_engines_) {
+    EngineInstall(install_cmd, engine, cml_data_.engine_version,
                   cml_data_.engine_src);
   }
 
@@ -512,9 +507,8 @@ void CommandLineParser::SetupEngineCommands() {
     }
   });
   uninstall_cmd->group(kSubcommands);
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineUninstall(uninstall_cmd, engine_name);
+  for (const auto& engine : supported_engines_) {
+    EngineUninstall(uninstall_cmd, engine);
   }
 
   auto engine_upd_cmd = engines_cmd->add_subcommand("update", "Update engine");
@@ -529,9 +523,8 @@ void CommandLineParser::SetupEngineCommands() {
     }
   });
   engine_upd_cmd->group(kSubcommands);
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineUpdate(engine_upd_cmd, engine_name);
+  for (const auto& engine : supported_engines_) {
+    EngineUpdate(engine_upd_cmd, engine);
   }
 
   auto engine_use_cmd =
@@ -547,9 +540,8 @@ void CommandLineParser::SetupEngineCommands() {
     }
   });
   engine_use_cmd->group(kSubcommands);
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineUse(engine_use_cmd, engine_name);
+  for (const auto& engine : supported_engines_) {
+    EngineUse(engine_use_cmd, engine);
   }
 
   auto engine_load_cmd = engines_cmd->add_subcommand("load", "Load engine");
@@ -564,9 +556,8 @@ void CommandLineParser::SetupEngineCommands() {
     }
   });
   engine_load_cmd->group(kSubcommands);
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineLoad(engine_load_cmd, engine_name);
+  for (const auto& engine : supported_engines_) {
+    EngineLoad(engine_load_cmd, engine);
   }
 
   auto engine_unload_cmd =
@@ -582,9 +573,8 @@ void CommandLineParser::SetupEngineCommands() {
     }
   });
   engine_unload_cmd->group(kSubcommands);
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    EngineUnload(engine_unload_cmd, engine_name);
+  for (const auto& engine : supported_engines_) {
+    EngineUnload(engine_unload_cmd, engine);
   }
 
   EngineGet(engines_cmd);
@@ -756,7 +746,7 @@ void CommandLineParser::EngineInstall(CLI::App* parent,
       return;
     try {
       commands::EngineInstallCmd(
-          download_service_, cml_data_.config.apiServerHost,
+          engine_service_, cml_data_.config.apiServerHost,
           std::stoi(cml_data_.config.apiServerPort), cml_data_.show_menu)
           .Exec(engine_name, version, src);
     } catch (const std::exception& e) {
@@ -878,20 +868,19 @@ void CommandLineParser::EngineGet(CLI::App* parent) {
     }
   });
 
-  for (auto& engine : engine_service_.kSupportEngines) {
-    std::string engine_name{engine};
-    std::string desc = "Get " + engine_name + " status";
+  for (const auto& engine : supported_engines_) {
+    std::string desc = "Get " + engine + " status";
 
-    auto engine_get_cmd = get_cmd->add_subcommand(engine_name, desc);
+    auto engine_get_cmd = get_cmd->add_subcommand(engine, desc);
     engine_get_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
-                          " engines get " + engine_name + " [options]");
+                          " engines get " + engine + " [options]");
     engine_get_cmd->group(kEngineGroup);
-    engine_get_cmd->callback([this, engine_name] {
+    engine_get_cmd->callback([this, engine] {
       if (std::exchange(executed_, true))
         return;
       commands::EngineGetCmd().Exec(cml_data_.config.apiServerHost,
                                     std::stoi(cml_data_.config.apiServerPort),
-                                    engine_name);
+                                    engine);
     });
   }
 }

engine/cli/command_line_parser.h

@@ -5,7 +5,6 @@
 #include "CLI/CLI.hpp"
 #include "commands/hardware_list_cmd.h"
 #include "services/engine_service.h"
-#include "services/model_service.h"
 #include "utils/config_yaml_utils.h"
 
 class CommandLineParser {
@@ -16,8 +15,6 @@ class CommandLineParser {
  private:
   void SetupCommonCommands();
 
-  void SetupInferenceCommands();
-
   void SetupModelCommands();
 
   void SetupEngineCommands();
@@ -47,8 +44,9 @@ class CommandLineParser {
 
   CLI::App app_;
   std::shared_ptr<DownloadService> download_service_;
-  EngineService engine_service_;
-  ModelService model_service_;
+  std::shared_ptr<cortex::DylibPathManager> dylib_path_manager_;
+  std::shared_ptr<EngineService> engine_service_;
+  std::vector<std::string> supported_engines_;
 
   struct CmlData {
     std::string model_id;

engine/cli/commands/engine_install_cmd.cc

@@ -12,7 +12,7 @@ bool EngineInstallCmd::Exec(const std::string& engine,
                             const std::string& src) {
   // Handle local install, if fails, fallback to remote install
   if (!src.empty()) {
-    auto res = engine_service_.UnzipEngine(engine, version, src);
+    auto res = engine_service_->UnzipEngine(engine, version, src);
     if (res.has_error()) {
       CLI_LOG(res.error());
       return false;

engine/cli/commands/engine_install_cmd.h

@@ -7,9 +7,9 @@ namespace commands {
 
 class EngineInstallCmd {
  public:
-  explicit EngineInstallCmd(std::shared_ptr<DownloadService> download_service,
+  explicit EngineInstallCmd(std::shared_ptr<EngineService> engine_service,
                             const std::string& host, int port, bool show_menu)
-      : engine_service_{EngineService(download_service)},
+      : engine_service_{engine_service},
         host_(host),
         port_(port),
         show_menu_(show_menu),
@@ -21,7 +21,7 @@ class EngineInstallCmd {
             const std::string& src = "");
 
  private:
-  EngineService engine_service_;
+  std::shared_ptr<EngineService> engine_service_;
   std::string host_;
   int port_;
   bool show_menu_;

engine/cli/commands/engine_list_cmd.cc

@@ -13,7 +13,6 @@
 // clang-format on
 
 namespace commands {
-
 bool EngineListCmd::Exec(const std::string& host, int port) {
   // Start server if server is not started yet
   if (!commands::IsServerAlive(host, port)) {
@@ -38,15 +37,10 @@ bool EngineListCmd::Exec(const std::string& host, int port) {
     return false;
   }
 
-  std::vector<std::string> engines = {
-      kLlamaEngine,
-      kOnnxEngine,
-      kTrtLlmEngine,
-  };
-
   std::unordered_map<std::string, std::vector<EngineVariantResponse>>
       engine_map;
 
+  auto engines = engine_service_->GetSupportedEngineNames().value();
   for (const auto& engine : engines) {
     auto installed_variants = result.value()[engine];
     for (const auto& variant : installed_variants) {

engine/cli/commands/engine_list_cmd.h

@@ -1,11 +1,18 @@
 #pragma once
 
 #include <string>
+#include "services/engine_service.h"
 
 namespace commands {
 class EngineListCmd {
  public:
+  explicit EngineListCmd(std::shared_ptr<EngineService> engine_service)
+      : engine_service_{engine_service} {}
+
   bool Exec(const std::string& host, int port);
+
+ private:
+  std::shared_ptr<EngineService> engine_service_;
 };
 
 }  // namespace commands

engine/cli/commands/run_cmd.cc

@@ -94,15 +94,15 @@ void RunCmd::Exec(bool run_detach,
 
     // Check if engine existed. If not, download it
     {
-      auto is_engine_ready = engine_service_.IsEngineReady(mc.engine);
+      auto is_engine_ready = engine_service_->IsEngineReady(mc.engine);
       if (is_engine_ready.has_error()) {
         throw std::runtime_error(is_engine_ready.error());
       }
 
       if (!is_engine_ready.value()) {
         CTL_INF("Engine " << mc.engine
                           << " is not ready. Proceed to install..");
-        if (!EngineInstallCmd(download_service_, host_, port_, false)
+        if (!EngineInstallCmd(engine_service_, host_, port_, false)
                  .Exec(mc.engine)) {
           return;
         } else {

engine/cli/commands/run_cmd.h

@@ -12,12 +12,11 @@ std::optional<std::string> SelectLocalModel(std::string host, int port,
 class RunCmd {
  public:
   explicit RunCmd(std::string host, int port, std::string model_handle,
-                  std::shared_ptr<DownloadService> download_service)
+                  std::shared_ptr<EngineService> engine_service)
       : host_{std::move(host)},
         port_{port},
         model_handle_{std::move(model_handle)},
-        download_service_(download_service),
-        engine_service_{EngineService(download_service)} {};
+        engine_service_{engine_service} {};
 
   void Exec(bool chat_flag,
             const std::unordered_map<std::string, std::string>& options);
@@ -26,8 +25,6 @@ class RunCmd {
   std::string host_;
   int port_;
   std::string model_handle_;
-
-  std::shared_ptr<DownloadService> download_service_;
-  EngineService engine_service_;
+  std::shared_ptr<EngineService> engine_service_;
 };
 }  // namespace commands

engine/cli/commands/server_start_cmd.cc

@@ -112,7 +112,9 @@ bool ServerStartCmd::Exec(const std::string& host, int port,
     return false;
   } else if (pid == 0) {
     // Some engines requires to add lib search path before process being created
-    EngineService().RegisterEngineLibPath();
+    auto download_srv = std::make_shared<DownloadService>();
+    auto dylib_path_mng = std::make_shared<cortex::DylibPathManager>();
+    EngineService(download_srv, dylib_path_mng).RegisterEngineLibPath();
 
     std::string p = cortex_utils::GetCurrentPath() + "/" + exe;
     execl(p.c_str(), exe.c_str(), "--start-server", "--config_file_path",
@@ -131,5 +133,4 @@ bool ServerStartCmd::Exec(const std::string& host, int port,
 #endif
   return true;
 }
-
 };  // namespace commands

engine/controllers/engines.cc

@@ -3,9 +3,9 @@
 #include "utils/archive_utils.h"
 #include "utils/cortex_utils.h"
 #include "utils/engine_constants.h"
-#include "utils/http_util.h"
 #include "utils/logging_utils.h"
 #include "utils/string_utils.h"
+
 namespace {
 // Need to change this after we rename repositories
 std::string NormalizeEngine(const std::string& engine) {
@@ -24,8 +24,8 @@ void Engines::ListEngine(
     const HttpRequestPtr& req,
     std::function<void(const HttpResponsePtr&)>&& callback) const {
   Json::Value ret;
-  auto engine_names = engine_service_->GetSupportedEngineNames().value();
-  for (const auto& engine : engine_names) {
+  auto engines = engine_service_->GetSupportedEngineNames().value();
+  for (const auto& engine : engines) {
     auto installed_engines =
         engine_service_->GetInstalledEngineVariants(engine);
     if (installed_engines.has_error()) {
@@ -37,6 +37,7 @@ void Engines::ListEngine(
     }
     ret[engine] = variants;
   }
+
   // Add remote engine
   auto remote_engines = engine_service_->GetEngines();
   if (remote_engines.has_value()) {
@@ -49,7 +50,6 @@ void Engines::ListEngine(
       }
     }
   }
-
   auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
   resp->setStatusCode(k200OK);
   callback(resp);