Skip to content

Commit 02857f2

Browse files
GuanLuoGuanLuo
authored
Add test for optional internal tensor within an ensemble (#6663) (#6676)
* Add test for optional internal tensor within an ensemble * Fix up
1 parent 3ad3244 commit 02857f2

File tree

5 files changed

+236
-10
lines changed

5 files changed

+236
-10
lines changed

‎qa/L0_optional_input/models/optional_connecting_tensor/config.pbtxt

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
platform: "ensemble"
28+
max_batch_size: 4
29+
input [
30+
{
31+
name: "INPUT0"
32+
data_type: TYPE_FP32
33+
dims: [ 1 ]
34+
optional: true
35+
},
36+
{
37+
name: "INPUT1"
38+
data_type: TYPE_FP32
39+
dims: [ 1 ]
40+
optional: true
41+
}
42+
]
43+
output [
44+
{
45+
name: "OUTPUT0"
46+
data_type: TYPE_FP32
47+
dims: [ 1 ]
48+
},
49+
{
50+
name: "OUTPUT1"
51+
data_type: TYPE_FP32
52+
dims: [ 1 ]
53+
}
54+
]
55+
ensemble_scheduling {
56+
step [
57+
{
58+
model_name: "optional_identity"
59+
model_version: -1
60+
input_map {
61+
key: "INPUT0"
62+
value: "INPUT0"
63+
}
64+
input_map {
65+
key: "INPUT1"
66+
value: "INPUT1"
67+
}
68+
output_map {
69+
key: "OUTPUT0"
70+
value: "internal_output0"
71+
}
72+
output_map {
73+
key: "OUTPUT1"
74+
value: "internal_output1"
75+
}
76+
},
77+
{
78+
model_name: "optional_identity"
79+
model_version: -1
80+
input_map {
81+
key: "INPUT0"
82+
value: "internal_output0"
83+
}
84+
input_map {
85+
key: "INPUT1"
86+
value: "internal_output1"
87+
}
88+
output_map {
89+
key: "OUTPUT0"
90+
value: "OUTPUT0"
91+
}
92+
output_map {
93+
key: "OUTPUT1"
94+
value: "OUTPUT1"
95+
}
96+
}
97+
]
98+
}

‎qa/L0_optional_input/models/optional_identity/1/model.py

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
import triton_python_backend_utils as pb_utils
28+
29+
30+
class TritonPythonModel:
31+
def execute(self, requests):
32+
"""
33+
Identity model in Python backend.
34+
"""
35+
responses = []
36+
for request in requests:
37+
for tidx in ("0", "1"):
38+
input_tensor = pb_utils.get_input_tensor_by_name(
39+
request, "INPUT" + tidx
40+
)
41+
if input_tensor is not None:
42+
out_tensor = pb_utils.Tensor(
43+
"OUTPUT" + tidx, input_tensor.as_numpy()
44+
)
45+
responses.append(pb_utils.InferenceResponse([out_tensor]))
46+
return responses

‎qa/L0_optional_input/models/optional_identity/config.pbtxt

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
backend: "python"
27+
max_batch_size: 4
28+
input [
29+
{
30+
name: "INPUT0"
31+
data_type: TYPE_FP32
32+
dims: [ 1 ]
33+
optional: true
34+
},
35+
{
36+
name: "INPUT1"
37+
data_type: TYPE_FP32
38+
dims: [ 1 ]
39+
optional: true
40+
}
41+
]
42+
output [
43+
{
44+
name: "OUTPUT0"
45+
data_type: TYPE_FP32
46+
dims: [ 1 ]
47+
},
48+
{
49+
name: "OUTPUT1"
50+
data_type: TYPE_FP32
51+
dims: [ 1 ]
52+
}
53+
]

‎qa/L0_optional_input/optional_input_test.py

+35-6
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,8 @@ def check_status(self, model_name, batch_exec, request_cnt, infer_cnt):
142142
stats = self.triton_client_.get_inference_statistics(model_name, "1")
143143
self.assertEqual(len(stats.model_stats), 1, "expect 1 model stats")
144144
actual_exec_cnt = stats.model_stats[0].execution_count
145-
if actual_exec_cnt == exec_cnt:
145+
if stats.model_stats[0].execution_count > 0:
146146
break
147-
print(
148-
"WARNING: expect {} executions, got {} (attempt {})".format(
149-
exec_cnt, actual_exec_cnt, i
150-
)
151-
)
152147
time.sleep(1)
153148

154149
self.assertEqual(
@@ -411,6 +406,40 @@ def test_ensemble_optional_pipeline(self):
411406
except Exception as ex:
412407
self.assertTrue(False, "unexpected error {}".format(ex))
413408

409+
def test_ensemble_optional_connecting_tensor(self):
410+
# The ensemble is a special case of pipelining models with optional
411+
# inputs, where the request will only produce a subset of inputs
412+
# for the second model while the ensemble graph connects all inputs of
413+
# the second model (which is valid because the not-provided inputs
414+
# are marked optional). See 'config.pbtxt' for detail.
415+
self.model_name_ = "optional_connecting_tensor"
416+
417+
# Provide all inputs, send requests that don't form preferred batch
418+
# so all requests should be returned after the queue delay
419+
try:
420+
provided_inputs = ("INPUT0",)
421+
inputs = []
422+
outputs = []
423+
for provided_input in provided_inputs:
424+
inputs.append(self.inputs_[provided_input])
425+
outputs.append(self.outputs_[provided_input])
426+
427+
triton_client = grpcclient.InferenceServerClient("localhost:8001")
428+
results = triton_client.infer(
429+
model_name=self.model_name_, inputs=inputs, outputs=outputs
430+
)
431+
432+
expected = self.input_data_["INPUT0"]
433+
output_data = results.as_numpy("OUTPUT0")
434+
self.assertTrue(
435+
np.array_equal(output_data, expected),
436+
"{}, {}, expected: {}, got {}".format(
437+
self.model_name_, "OUTPUT0", expected, output_data
438+
),
439+
)
440+
except Exception as ex:
441+
self.assertTrue(False, "unexpected error {}".format(ex))
442+
414443

415444
if __name__ == "__main__":
416445
unittest.main()

‎qa/L0_optional_input/test.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ rm -fr *.log
4141
mkdir -p ./models/identity_2_float32/1
4242
mkdir -p ./models/ensemble_identity_2_float32/1
4343
mkdir -p ./models/pipeline_identity_2_float32/1
44+
mkdir -p ./models/optional_connecting_tensor/1
4445

4546
# Basic test cases
4647
TEST_CASES=${TEST_CASES:="test_all_inputs \
@@ -51,8 +52,9 @@ TEST_CASES=${TEST_CASES:="test_all_inputs \
5152
test_ensemble_optional_same_input \
5253
test_ensemble_optional_mix_inputs \
5354
test_ensemble_optional_mix_inputs_2 \
54-
test_ensemble_optional_pipeline"}
55-
55+
test_ensemble_optional_pipeline \
56+
test_ensemble_optional_connecting_tensor"}
57+
RET=0
5658
for i in $TEST_CASES ; do
5759
# Restart server for every test to clear model stats
5860
run_server
@@ -62,8 +64,6 @@ for i in $TEST_CASES ; do
6264
exit 1
6365
fi
6466

65-
RET=0
66-
6767
echo "Test: $i" >>$TEST_LOG
6868

6969
set +e

0 commit comments

Comments
 (0)