Skip to content

Commit 734363f

Browse files
krishung5krishung5
authored
Add testing for Python custom metrics API (#5669)
* Add testing for python custom metrics API * Add custom metrics example to the test * Fix for CodeQL report * Fix test name * Address comment * Add logger and change the enum usage
1 parent 0a51f7e commit 734363f

File tree

5 files changed

+394
-0
lines changed

5 files changed

+394
-0
lines changed

‎qa/L0_backend_python/custom_metrics/test.sh

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/bash
2+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
CLIENT_PY=../python_unittest.py
29+
CLIENT_LOG="./client.log"
30+
EXPECTED_NUM_TESTS="1"
31+
TEST_RESULT_FILE='test_results.txt'
32+
source ../../common/util.sh
33+
34+
TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
35+
SERVER=${TRITON_DIR}/bin/tritonserver
36+
BACKEND_DIR=${TRITON_DIR}/backends
37+
SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1"
38+
SERVER_LOG="./inference_server.log"
39+
40+
RET=0
41+
rm -fr *.log ./models *.txt
42+
43+
mkdir -p models/custom_metrics/1/
44+
cp ../../python_models/custom_metrics/model.py models/custom_metrics/1/
45+
cp ../../python_models/custom_metrics/config.pbtxt models/custom_metrics
46+
47+
run_server
48+
if [ "$SERVER_PID" == "0" ]; then
49+
echo -e "\n***\n*** Failed to start $SERVER\n***"
50+
cat $SERVER_LOG
51+
exit 1
52+
fi
53+
54+
set +e
55+
56+
export MODEL_NAME='custom_metrics'
57+
python3 $CLIENT_PY >> $CLIENT_LOG 2>&1
58+
if [ $? -ne 0 ]; then
59+
echo -e "\n***\n*** 'Custom Metrics' test FAILED. \n***"
60+
cat $CLIENT_LOG
61+
RET=1
62+
else
63+
check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
64+
if [ $? -ne 0 ]; then
65+
cat $CLIENT_LOG
66+
echo -e "\n***\n*** Test Result Verification Failed\n***"
67+
RET=1
68+
fi
69+
fi
70+
71+
set -e
72+
73+
kill $SERVER_PID
74+
wait $SERVER_PID
75+
76+
77+
if [ $RET -eq 1 ]; then
78+
cat $CLIENT_LOG
79+
cat $SERVER_LOG
80+
echo -e "\n***\n*** Custom Metrics test FAILED. \n***"
81+
else
82+
echo -e "\n***\n*** Custom Metrics test PASSED. \n***"
83+
fi
84+
85+
exit $RET

‎qa/L0_backend_python/examples/test.sh

+30
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,36 @@ set -e
413413
kill $SERVER_PID
414414
wait $SERVER_PID
415415

416+
# Custom Metrics
417+
CLIENT_LOG="./custom_metrics_client.log"
418+
mkdir -p models/custom_metrics/1
419+
cp examples/custom_metrics/model.py models/custom_metrics/1/model.py
420+
cp examples/custom_metrics/config.pbtxt models/custom_metrics/config.pbtxt
421+
run_server
422+
if [ "$SERVER_PID" == "0" ]; then
423+
echo -e "\n***\n*** Failed to start $SERVER\n***"
424+
cat $SERVER_LOG
425+
RET=1
426+
fi
427+
428+
set +e
429+
python3 examples/custom_metrics/client.py > $CLIENT_LOG
430+
if [ $? -ne 0 ]; then
431+
echo -e "\n***\n*** Failed to verify Custom Metrics example. \n***"
432+
RET=1
433+
fi
434+
435+
grep "PASS" $CLIENT_LOG
436+
if [ $? -ne 0 ]; then
437+
echo -e "\n***\n*** Failed to verify Custom Metrics example. \n***"
438+
cat $CLIENT_LOG
439+
RET=1
440+
fi
441+
set -e
442+
443+
kill $SERVER_PID
444+
wait $SERVER_PID
445+
416446

417447
if [ $RET -eq 0 ]; then
418448
echo -e "\n***\n*** Example verification test PASSED.\n***"

‎qa/L0_backend_python/test.sh

+5
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,11 @@ if [ "$TEST_JETSON" == "0" ]; then
442442
fi
443443
fi
444444

445+
(cd custom_metrics && bash -ex test.sh)
446+
if [ $? -ne 0 ]; then
447+
RET=1
448+
fi
449+
445450
if [ $RET -eq 0 ]; then
446451
echo -e "\n***\n*** Test Passed\n***"
447452
else

‎qa/python_models/custom_metrics/config.pbtxt

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
name: "custom_metrics"
28+
backend: "python"
29+
30+
output [
31+
{
32+
name: "OUTPUT0"
33+
data_type: TYPE_FP32
34+
dims: [ 16 ]
35+
}
36+
]
37+
38+
instance_group [
39+
{
40+
count: 3
41+
kind: KIND_CPU
42+
}
43+
]

0 commit comments

Comments
 (0)