From b6f59914a2ab8d5f07961d397d1ba7328e8678db Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 22 Jun 2023 17:54:27 -0700 Subject: [PATCH 1/2] Add test for Python BLS model loading API --- qa/L0_backend_python/bls/test.sh | 92 ++++++++++++++ .../bls_model_loading/config.pbtxt | 43 +++++++ qa/python_models/bls_model_loading/model.py | 119 ++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 qa/python_models/bls_model_loading/config.pbtxt create mode 100644 qa/python_models/bls_model_loading/model.py diff --git a/qa/L0_backend_python/bls/test.sh b/qa/L0_backend_python/bls/test.sh index bd07a384ca..3f88df01f3 100644 --- a/qa/L0_backend_python/bls/test.sh +++ b/qa/L0_backend_python/bls/test.sh @@ -227,6 +227,98 @@ else RET=1 fi +# Test model loading API with BLS +rm -fr ./models +mkdir -p models/bls_model_loading/1/ +cp ../../python_models/bls_model_loading/model.py models/bls_model_loading/1/ +cp ../../python_models/bls_model_loading/config.pbtxt models/bls_model_loading/ +cp -fr ${DATADIR}/qa_model_repository/onnx_int32_int32_int32 models/. +# Make only version 2, 3 is valid version directory +rm -rf models/onnx_int32_int32_int32/1 + +SERVER_LOG="./bls_model_loading_server.log" +SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --model-control-mode=explicit --log-verbose=1" + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +export MODEL_NAME='bls_model_loading' + +set +e +code=`curl -s -w %{http_code} -X POST localhost:8000/v2/repository/models/${MODEL_NAME}/load` +set -e +if [ "$code" == "400" ]; then + echo -e "\n***\n*** Failed to load model '${MODEL_NAME}'\n***" + RET=1 +fi + +set +e + +python3 $CLIENT_PY >> $CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + echo -e "\n***\n*** 'bls_model_loading' test FAILED. \n***" + cat $CLIENT_LOG + RET=1 +else + check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Result Verification Failed\n***" + RET=1 + fi +fi + +set -e + +kill $SERVER_PID +wait $SERVER_PID + +# Test model loading API with BLS warmup +(cd models/bls_model_loading && \ + echo "model_warmup [{" >> config.pbtxt && \ + echo " name : \"regular sample\"" >> config.pbtxt && \ + echo " batch_size: 1" >> config.pbtxt && \ + echo " inputs {" >> config.pbtxt && \ + echo " key: \"INPUT0\"" >> config.pbtxt && \ + echo " value: {" >> config.pbtxt && \ + echo " data_type: TYPE_FP32" >> config.pbtxt && \ + echo " dims: 4" >> config.pbtxt && \ + echo " zero_data: false" >> config.pbtxt && \ + echo " }" >> config.pbtxt && \ + echo " }" >> config.pbtxt && \ + echo " inputs {" >> config.pbtxt && \ + echo " key: \"INPUT1\"" >> config.pbtxt && \ + echo " value: {" >> config.pbtxt && \ + echo " data_type: TYPE_FP32" >> config.pbtxt && \ + echo " dims: 4" >> config.pbtxt && \ + echo " zero_data: false" >> config.pbtxt && \ + echo " }" >> config.pbtxt && \ + echo " }" >> config.pbtxt && \ + echo "}]" >> config.pbtxt ) + +SERVER_LOG="./bls_model_loading_server_warmup.log" +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +set +e +code=`curl -s -w %{http_code} -X POST localhost:8000/v2/repository/models/${MODEL_NAME}/load` +set -e +if [ "$code" == "400" ]; then + echo -e "\n***\n*** Failed to load model '${MODEL_NAME}'\n***" + RET=1 +fi + +kill $SERVER_PID +wait $SERVER_PID + if [ $RET -eq 1 ]; then cat $CLIENT_LOG cat $SERVER_LOG diff --git a/qa/python_models/bls_model_loading/config.pbtxt b/qa/python_models/bls_model_loading/config.pbtxt new file mode 100644 index 0000000000..8b11d06201 --- /dev/null +++ b/qa/python_models/bls_model_loading/config.pbtxt @@ -0,0 +1,43 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +name: "bls_model_loading" +backend: "python" + +output [ + { + name: "OUTPUT0" + data_type: TYPE_BOOL + dims: [ 1 ] + } +] + +instance_group [ + { + count: 3 + kind: KIND_CPU + } +] diff --git a/qa/python_models/bls_model_loading/model.py b/qa/python_models/bls_model_loading/model.py new file mode 100644 index 0000000000..4a06634ef2 --- /dev/null +++ b/qa/python_models/bls_model_loading/model.py @@ -0,0 +1,119 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import numpy as np +import unittest +import triton_python_backend_utils as pb_utils + + +class PBBLSModelLoadingTest(unittest.TestCase): + + def setUp(self): + self.model_name = "onnx_int32_int32_int32" + + def tearDown(self): + pb_utils.unload_model(self.model_name) + + def test_load_unload_model(self): + self.assertFalse(pb_utils.is_model_ready(self.model_name)) + pb_utils.load_model(self.model_name) + self.assertTrue(pb_utils.is_model_ready(self.model_name)) + pb_utils.unload_model(self.model_name) + self.assertFalse(pb_utils.is_model_ready(self.model_name)) + + def test_load_with_config_override(self): + self.assertFalse(pb_utils.is_model_ready(self.model_name)) + pb_utils.load_model(self.model_name) + self.assertTrue(pb_utils.is_model_ready(self.model_name)) + + # Send the config with the wrong format + wrong_config = "\"parameters\": {\"config\": {{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[2]}}}}}" + with self.assertRaises(pb_utils.TritonModelException): + pb_utils.load_model(self.model_name, wrong_config) + # The model should not be changed after a failed load model request + for version in ["2", "3"]: + self.assertTrue(pb_utils.is_model_ready(self.model_name, version)) + + # Send the config with the correct format + config = config = "{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[2]}}}" + pb_utils.load_model(self.model_name, config=config) + # The model should be changed after a successful load model request + self.assertTrue(pb_utils.is_model_ready(self.model_name, "2")) + self.assertFalse(pb_utils.is_model_ready(self.model_name, "3")) + + def test_load_with_file_override(self): + self.assertFalse(pb_utils.is_model_ready(self.model_name)) + pb_utils.load_model(self.model_name) + self.assertTrue(pb_utils.is_model_ready(self.model_name)) + + override_name = "override_model" + config = "{\"backend\":\"onnxruntime\"}" + with open('models/onnx_int32_int32_int32/3/model.onnx', 'rb') as file: + data = file.read() + files = {"file:1/model.onnx": data} + + # Request to load the model with override file, should fail without + # providing override config. + with self.assertRaises(pb_utils.TritonModelException): + pb_utils.load_model(self.model_name, "", files) + + # Request to load the model with override file and config in a different name + pb_utils.load_model(override_name, config, files) + # Sanity check that the model with original name is unchanged + self.assertFalse(pb_utils.is_model_ready(self.model_name, "1")) + self.assertTrue(pb_utils.is_model_ready(self.model_name, "3")) + + # Check the override model readiness + self.assertTrue(pb_utils.is_model_ready(override_name, "1")) + self.assertFalse(pb_utils.is_model_ready(override_name, "3")) + + # Request to load the model with override file and config in original name + pb_utils.load_model(self.model_name, config, files) + # Check that the model with original name is changed + self.assertTrue(pb_utils.is_model_ready(self.model_name, "1")) + self.assertFalse(pb_utils.is_model_ready(self.model_name, "3")) + + # Sanity check readiness of the different named model + self.assertTrue(pb_utils.is_model_ready(override_name, "1")) + self.assertFalse(pb_utils.is_model_ready(override_name, "3")) + + +class TritonPythonModel: + + def initialize(self, args): + # Run the unittest during initialization + test = unittest.main('model', exit=False) + self.result = test.result.wasSuccessful() + + def execute(self, requests): + responses = [] + for _ in requests: + responses.append( + pb_utils.InferenceResponse([ + pb_utils.Tensor('OUTPUT0', + np.array([self.result], dtype=np.float16)) + ])) + return responses From f43e45253feecd2175cef64f9e21a4d6678d9464 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 22 Jun 2023 18:17:59 -0700 Subject: [PATCH 2/2] Fix up --- qa/python_models/bls_model_loading/model.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qa/python_models/bls_model_loading/model.py b/qa/python_models/bls_model_loading/model.py index 4a06634ef2..aadeb8f855 100644 --- a/qa/python_models/bls_model_loading/model.py +++ b/qa/python_models/bls_model_loading/model.py @@ -38,8 +38,8 @@ def tearDown(self): pb_utils.unload_model(self.model_name) def test_load_unload_model(self): - self.assertFalse(pb_utils.is_model_ready(self.model_name)) - pb_utils.load_model(self.model_name) + self.assertFalse(pb_utils.is_model_ready(model_name=self.model_name)) + pb_utils.load_model(model_name=self.model_name) self.assertTrue(pb_utils.is_model_ready(self.model_name)) pb_utils.unload_model(self.model_name) self.assertFalse(pb_utils.is_model_ready(self.model_name)) @@ -52,13 +52,13 @@ def test_load_with_config_override(self): # Send the config with the wrong format wrong_config = "\"parameters\": {\"config\": {{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[2]}}}}}" with self.assertRaises(pb_utils.TritonModelException): - pb_utils.load_model(self.model_name, wrong_config) + pb_utils.load_model(model_name=self.model_name, config=wrong_config) # The model should not be changed after a failed load model request for version in ["2", "3"]: - self.assertTrue(pb_utils.is_model_ready(self.model_name, version)) + self.assertTrue(pb_utils.is_model_ready(model_name=self.model_name, model_version=version)) # Send the config with the correct format - config = config = "{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[2]}}}" + config = "{\"backend\":\"onnxruntime\", \"version_policy\":{\"specific\":{\"versions\":[2]}}}" pb_utils.load_model(self.model_name, config=config) # The model should be changed after a successful load model request self.assertTrue(pb_utils.is_model_ready(self.model_name, "2")) @@ -81,7 +81,7 @@ def test_load_with_file_override(self): pb_utils.load_model(self.model_name, "", files) # Request to load the model with override file and config in a different name - pb_utils.load_model(override_name, config, files) + pb_utils.load_model(model_name=override_name, config=config, files=files) # Sanity check that the model with original name is unchanged self.assertFalse(pb_utils.is_model_ready(self.model_name, "1")) self.assertTrue(pb_utils.is_model_ready(self.model_name, "3"))