triton-inference-server · fpetrini15 · Jul 29, 2024 · May 21, 2024 · May 21, 2024 · May 24, 2024
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,17 @@
+#VSCode
+/.vscode
+
+#C++
+/build
+/builddir
+*.so
+
+#Python
+__pycache__/
+*.pyc
+
+#Other
+node_modules
+
+#GenAI-Perf Artifacts
+artifacts/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,239 +24,151 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cmake_minimum_required (VERSION 3.18)
+cmake_minimum_required(VERSION 3.18)
 
-if(WIN32)
-  message("perf_analyzer is not currently supported on Windows because "
-          "is requires functionalities that are UNIX specific.")
-else()
-
-add_subdirectory(client_backend)
+project(perf_analyzer LANGUAGES C CXX)
 
-find_package(Git REQUIRED)
+# Use C++17 standard as Triton's minimum required.
+set(TRITON_MIN_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard which features are requested to build this target.")
 
-execute_process(WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-  COMMAND "${GIT_EXECUTABLE}" log -n 1 --abbrev-commit --format=format:%h
-  RESULT_VARIABLE RETURN_CODE
-  OUTPUT_VARIABLE GIT_SHA)
-if(NOT RETURN_CODE EQUAL "0")
-  set(GIT_SHA "unknown")
-endif()
-
-set(
-  PERF_ANALYZER_SRCS
-  command_line_parser.cc
-  perf_analyzer.cc
-  model_parser.cc
-  perf_utils.cc
-  load_manager.cc
-  data_loader.cc
-  concurrency_manager.cc
-  request_rate_manager.cc
-  load_worker.cc
-  concurrency_worker.cc
-  request_rate_worker.cc
-  custom_load_manager.cc
-  infer_context.cc
-  inference_profiler.cc
-  report_writer.cc
-  mpi_utils.cc
-  metrics_manager.cc
-  infer_data_manager_base.cc
-  infer_data_manager.cc
-  infer_data_manager_shm.cc
-  sequence_manager.cc
-  profile_data_collector.cc
-  profile_data_exporter.cc
-  periodic_concurrency_manager.cc
-  periodic_concurrency_worker.cc
-)
-
-set(
-  PERF_ANALYZER_HDRS
-  command_line_parser.h
-  perf_analyzer.h
-  model_parser.h
-  perf_utils.h
-  load_manager.h
-  data_loader.h
-  concurrency_manager.h
-  request_rate_manager.h
-  custom_load_manager.h
-  iworker.h
-  load_worker.h
-  request_rate_worker.h
-  concurrency_worker.h
-  infer_context.h
-  inference_profiler.h
-  report_writer.h
-  mpi_utils.h
-  doctest.h
-  constants.h
-  metrics.h
-  metrics_manager.h
-  infer_data_manager_factory.h
-  iinfer_data_manager.h
-  infer_data_manager.h
-  infer_data_manager_shm.h
-  infer_data_manager_base.h
-  infer_data.h
-  sequence_manager.h
-  sequence_status.h
-  ictx_id_tracker.h
-  concurrency_ctx_id_tracker.h
-  fifo_ctx_id_tracker.h
-  rand_ctx_id_tracker.h
-  request_record.h
-  profile_data_collector.h
-  profile_data_exporter.h
-  periodic_concurrency_manager.h
-  periodic_concurrency_worker.h
-  thread_config.h
-)
-
-add_executable(
-  perf_analyzer
-  main.cc
-  ${PERF_ANALYZER_SRCS}
-  ${PERF_ANALYZER_HDRS}
-  $<TARGET_OBJECTS:json-utils-library>
-)
-target_link_libraries(
-  perf_analyzer
-  PRIVATE
-    client-backend-library
-    -lb64
-    ${CMAKE_DL_LIBS}
-)
+set(TRITON_VERSION "0.0.0" CACHE STRING "Version for the clients")
+set(PERF_ANALYZER_VERSION ${TRITON_VERSION} CACHE STRING "Build Version for Perf Analyzer")
 
-target_compile_definitions(
-  perf_analyzer
-  PRIVATE
-    PERF_ANALYZER_VERSION=${PERF_ANALYZER_VERSION}
-    GIT_SHA=${GIT_SHA}
-)
+#
+# Perf Analyzer Options
+#
+option(TRITON_PACKAGE_PERF_ANALYZER "Include Perf Analyzer in python client pip wheel" ON)
+option(TRITON_ENABLE_PERF_ANALYZER_C_API "Enable Performance Analyzer C API" OFF)
+option(TRITON_ENABLE_PERF_ANALYZER_TFS "Enable TensorFlow Serving support for Performance Analyzer" OFF)
+option(TRITON_ENABLE_PERF_ANALYZER_TS "Enable TorchServe support for Performance Analyzer" OFF)
+option(TRITON_ENABLE_PERF_ANALYZER_OPENAI "Enable OpenAI support for Performance Analyzer" OFF)
 
-# If gpu is enabled then compile with CUDA dependencies
-if(TRITON_ENABLE_GPU)
-  target_compile_definitions(
-    perf_analyzer
-    PUBLIC TRITON_ENABLE_GPU=1
-  )
+#
+# Client Options
+#
+option(TRITON_ENABLE_CC_HTTP "Build C++ HTTP client libraries" ON)
+option(TRITON_ENABLE_CC_GRPC "Build C++ GRPC client libraries" ON)
+option(TRITON_ENABLE_PYTHON_HTTP "Build Python HTTP client libraries" OFF)
+option(TRITON_ENABLE_PYTHON_GRPC "Build Python GRPC client libraries" OFF)
+option(TRITON_ENABLE_GPU "Enable GPU support in libraries" ON)
+option(TRITON_ENABLE_ZLIB "Include ZLIB library in build" ON)
 
-  target_link_libraries(
-    perf_analyzer
-    PRIVATE CUDA::cudart
-  )
-endif()
+#
+# Github branch options
+#
+set(TRITON_REPO_ORGANIZATION "https://github.com/triton-inference-server" CACHE STRING "Git repository to pull from")
+set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo")
+set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo")
+set(TRITON_CLIENT_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo")
 
-if(TRITON_ENABLE_PERF_ANALYZER_C_API)
-  target_compile_definitions(
-    client-backend-library
-    PUBLIC TRITON_ENABLE_PERF_ANALYZER_C_API=1
-  )
-endif()
+#
+# Install locations
+#
+set(TRITON_THIRD_PARTY_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/cc_clients/src/cc-clients-build/third-party/" CACHE STRING "Location of third-party build")
 
-if(TRITON_ENABLE_PERF_ANALYZER_TFS)
-  target_compile_definitions(
-    client-backend-library
-    PUBLIC TRITON_ENABLE_PERF_ANALYZER_TFS=1
-  )
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
 endif()
 
-if(TRITON_ENABLE_PERF_ANALYZER_TS)
-  target_compile_definitions(
-    client-backend-library
-    PUBLIC TRITON_ENABLE_PERF_ANALYZER_TS=1
-  )
+if(WIN32)
+  message(FATAL_ERROR "perf_analyzer is not currently supported on Windows because "
+          "it requires functionalities that are UNIX specific.")
 endif()
 
-if(TRITON_ENABLE_PERF_ANALYZER_OPENAI)
-  target_compile_definitions(
-    client-backend-library
-    PUBLIC TRITON_ENABLE_PERF_ANALYZER_OPENAI=1
-  )
+if(NOT(${TRITON_ENABLE_CC_HTTP} AND ${TRITON_ENABLE_CC_GRPC}))
+  message(FATAL_ERROR "perf_analyzer requires both http and grpc client libraries.")
 endif()
 
-install(
-  TARGETS perf_analyzer
-  RUNTIME DESTINATION bin
-)
-
-target_compile_definitions(perf_analyzer PUBLIC DOCTEST_CONFIG_DISABLE)
-
-# Creating perf_client link to perf_analyzer binary for backwards compatibility.
-install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ./perf_analyzer perf_client
-  WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin/)")
-install(CODE "message(\"-- Created symlink: perf_client -> ./perf_analyzer\")")
-
-
-
-set(PERF_ANALYZER_UNIT_TESTS_SRCS ${PERF_ANALYZER_SRCS})
-list(PREPEND PERF_ANALYZER_UNIT_TESTS_SRCS perf_analyzer_unit_tests.cc)
-set(PERF_ANALYZER_UNIT_TESTS_HDRS ${PERF_ANALYZER_HDRS})
-
-add_executable(
-  perf_analyzer_unit_tests
-  ${PERF_ANALYZER_UNIT_TESTS_SRCS}
-  ${PERF_ANALYZER_UNIT_TESTS_HDRS}
-  mock_inference_profiler.h
-  mock_model_parser.h
-  test_utils.h
-  client_backend/mock_client_backend.h
-  mock_concurrency_worker.h
-  mock_data_loader.h
-  mock_infer_context.h
-  mock_infer_data_manager.h
-  mock_request_rate_worker.h
-  mock_sequence_manager.h
-  mock_profile_data_collector.h
-  mock_profile_data_exporter.h
-  test_dataloader.cc
-  test_inference_profiler.cc
-  test_command_line_parser.cc
-  test_idle_timer.cc
-  test_load_manager_base.h
-  test_load_manager.cc
-  test_model_parser.cc
-  test_metrics_manager.cc
-  test_perf_utils.cc
-  test_report_writer.cc
-  client_backend/triton/test_triton_client_backend.cc
-  test_request_rate_manager.cc
-  test_concurrency_manager.cc
-  test_custom_load_manager.cc
-  test_sequence_manager.cc
-  test_infer_context.cc
-  test_ctx_id_tracker.cc
-  test_profile_data_collector.cc
-  test_profile_data_exporter.cc
-  $<TARGET_OBJECTS:json-utils-library>
-)
-
-# -Wno-write-strings is needed for the unit tests in order to statically create
-# input argv cases in the CommandLineParser unit test
 #
-set_target_properties(perf_analyzer_unit_tests
-  PROPERTIES COMPILE_FLAGS "-Wno-write-strings")
-
-target_link_libraries(
-  perf_analyzer_unit_tests
-  PRIVATE
-    gmock
-    client-backend-library
-    -lb64
+# Dependencies
+#
+include(ExternalProject)
+ExternalProject_Add(
+  cc-clients
+  PREFIX ${CMAKE_BINARY_DIR}/cc_clients
+  GIT_REPOSITORY ${TRITON_REPO_ORGANIZATION}/client.git
+  GIT_TAG ${TRITON_CLIENT_REPO_TAG}
+  GIT_SHALLOW ON
+  CMAKE_CACHE_ARGS
+      ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}
+      ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE}
+      ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET}
+      -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION}
+      -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG}
+      -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG}
+      -DTRITON_ENABLE_CC_HTTP:BOOL=${TRITON_ENABLE_CC_HTTP}
+      -DTRITON_ENABLE_CC_GRPC:BOOL=${TRITON_ENABLE_CC_GRPC}
+      -DTRITON_ENABLE_PYTHON_HTTP:BOOL=OFF
+      -DTRITON_ENABLE_PYTHON_HTTP:BOOL=OFF
+      -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU}
+      -DTRITON_ENABLE_ZLIB:BOOL=${TRITON_ENABLE_ZLIB}
+      -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
+      -DTRITON_THIRD_PARTY_INSTALL_PREFIX:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}
+      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+      -DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}
+  INSTALL_COMMAND ""
 )
 
-target_include_directories(
-  perf_analyzer_unit_tests
-  PRIVATE
-    client_backend
+ExternalProject_Add(
+  perf-analyzer
+  PREFIX ${CMAKE_BINARY_DIR}/perf_analyzer
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src
+  CMAKE_CACHE_ARGS
+      ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}
+      ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE}
+      ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET}
+      -DTRITON_ENABLE_CC_HTTP:BOOL=${TRITON_ENABLE_CC_HTTP}
+      -DTRITON_ENABLE_CC_GRPC:BOOL=${TRITON_ENABLE_CC_GRPC}
+      -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG}
+      -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG}
+      -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU}
+      -DTRITON_ENABLE_ZLIB:BOOL=${TRITON_ENABLE_ZLIB}
+      -DTRITON_ENABLE_PERF_ANALYZER_C_API:BOOL=${TRITON_ENABLE_PERF_ANALYZER_C_API}
+      -DTRITON_ENABLE_PERF_ANALYZER_TFS:BOOL=${TRITON_ENABLE_PERF_ANALYZER_TFS}
+      -DTRITON_ENABLE_PERF_ANALYZER_TS:BOOL=${TRITON_ENABLE_PERF_ANALYZER_TS}
+      -DTRITON_ENABLE_PERF_ANALYZER_OPENAI:BOOL=${TRITON_ENABLE_PERF_ANALYZER_OPENAI}
+      -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
+      -DTRITON_THIRD_PARTY_INSTALL_PREFIX:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}
+      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+      -DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}
+  DEPENDS cc-clients
 )
 
-install(
-  TARGETS perf_analyzer_unit_tests
-  RUNTIME DESTINATION bin
+# FIXME: [TPRD-310] This is a WAR for now. We shouldn't have to re-clone the client repo
+# Everything to build the wheel should be present in the cc_clients folder.
+# PA needs to have its own build_wheel.py script.
+
+# Build python client after perf_analyzer has been installed to a known location
+# so that it can be packaged in the python client pip wheel.
+if(TRITON_ENABLE_PYTHON_HTTP OR TRITON_ENABLE_PYTHON_GRPC)
+  ExternalProject_Add(
+    python-clients
+    PREFIX ${CMAKE_BINARY_DIR}/python_clients
+    GIT_REPOSITORY ${TRITON_REPO_ORGANIZATION}/client.git
+    GIT_TAG ${TRITON_CLIENT_REPO_TAG}
+    GIT_SHALLOW ON
+    CMAKE_CACHE_ARGS
+        ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}
+        ${_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE}
+        ${_CMAKE_ARGS_VCPKG_TARGET_TRIPLET}
+        -DTRITON_VERSION:STRING=${TRITON_VERSION}
+        -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION}
+        -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG}
+        -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG}
+        -DTRITON_ENABLE_CC_HTTP:BOOL=OFF
+        -DTRITON_ENABLE_CC_GRPC:BOOL=OFF
+	      -DTRITON_ENABLE_EXAMPLES:BOOL=ON
+        -DTRITON_ENABLE_PYTHON_HTTP:BOOL=${TRITON_ENABLE_PYTHON_HTTP}
+        -DTRITON_ENABLE_PYTHON_GRPC:BOOL=${TRITON_ENABLE_PYTHON_GRPC}
+        -DTRITON_PACKAGE_PERF_ANALYZER:BOOL=${TRITON_PACKAGE_PERF_ANALYZER}
+        -DTRITON_ENABLE_GPU:BOOL=${TRITON_ENABLE_GPU}
+        -DTRITON_ENABLE_ZLIB:BOOL=${TRITON_ENABLE_ZLIB}
+        -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
+        -DTRITON_THIRD_PARTY_INSTALL_PREFIX:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}
+        -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+        -DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}
+    INSTALL_COMMAND ""
+    DEPENDS perf-analyzer
 )
-
 endif()
+