From c48ca81fde52753189d6e1488a23a987d8ce9da7 Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Wed, 22 May 2024 13:31:05 +0900 Subject: [PATCH 1/7] add safe_concatenate --- tritony/tools.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tritony/tools.py b/tritony/tools.py index 3dd43bb..d50c46b 100644 --- a/tritony/tools.py +++ b/tritony/tools.py @@ -410,10 +410,15 @@ async def _call_async_item( ret = sorted(itertools.chain(*ret[:ASYNC_TASKS])) result_by_req_id = [output_result_list for req_id, output_result_list in ret] + def safe_concatenate(arrays, axis=0): + arrays = [np.expand_dims(arr, axis=0) if arr.ndim == 0 else arr for arr in arrays] + return np.concatenate(arrays, axis=axis) + + zipped_result = list(zip(*result_by_req_id)) if model_spec.max_batch_size == 0: - result_by_output_name = list(zip(*result_by_req_id)) + result_by_output_name = zipped_result else: - result_by_output_name = list(map(lambda ll: np.concatenate(ll, axis=0), zip(*result_by_req_id))) + result_by_output_name = list(map(lambda ll: safe_concatenate(ll, axis=0), zipped_result)) if len(result_by_output_name) == 1: result_by_output_name = result_by_output_name[0] From 412e70c7f305be724c449fa6db2910aac8f2e7b5 Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Thu, 23 May 2024 18:04:29 +0900 Subject: [PATCH 2/7] move `safe_concatenate` to global, apply to `InferenceClient._call_request` --- tritony/tools.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tritony/tools.py b/tritony/tools.py index d50c46b..dc8efc9 100644 --- a/tritony/tools.py +++ b/tritony/tools.py @@ -66,6 +66,18 @@ async def data_generator(data: list[np.ndarray], batch_size: int, queue: asyncio stop.set() +def safe_concatenate(arrays, axis=0): + arrays = [ + ( + np.expand_dims(arr, axis=0) + if arr is not None and arr.ndim == 0 + else (arr if arr is not None else np.array([])) + ) + for arr in arrays + ] + return np.concatenate(arrays, axis=axis) if arrays else np.array([]) + + async def send_request_async( inference_client: InferenceClient, data_queue, @@ -410,10 +422,6 @@ async def _call_async_item( ret = sorted(itertools.chain(*ret[:ASYNC_TASKS])) result_by_req_id = [output_result_list for req_id, output_result_list in ret] - def safe_concatenate(arrays, axis=0): - arrays = [np.expand_dims(arr, axis=0) if arr.ndim == 0 else arr for arr in arrays] - return np.concatenate(arrays, axis=axis) - zipped_result = list(zip(*result_by_req_id)) if model_spec.max_batch_size == 0: result_by_output_name = zipped_result @@ -481,10 +489,13 @@ def _call_request( ) ) + zipped_result = list(zip(*result_by_req_id)) if model_spec.max_batch_size == 0: - result_by_output_name = list(zip(*result_by_req_id)) + result_by_output_name = zipped_result else: - result_by_output_name = list(map(lambda ll: np.concatenate(ll, axis=0), zip(*result_by_req_id))) + result_by_output_name = list( + map(lambda ll: safe_concatenate(ll, axis=0), zipped_result) + ) if len(result_by_output_name) == 1: result_by_output_name = result_by_output_name[0] From 331d14648e3e009440089bf474a464f8292fcfd6 Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Thu, 23 May 2024 18:05:46 +0900 Subject: [PATCH 3/7] add a test case with the model of `sample_multiple_hybrid_dims` and modify `sample` model --- model_repository/sample/1/model.py | 2 +- .../sample_multiple_hybrid_dims/1/model.py | 45 +++++++++++++ .../sample_multiple_hybrid_dims/config.pbtxt | 63 +++++++++++++++++++ tests/test_model_call.py | 19 ++++++ 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 model_repository/sample_multiple_hybrid_dims/1/model.py create mode 100644 model_repository/sample_multiple_hybrid_dims/config.pbtxt diff --git a/model_repository/sample/1/model.py b/model_repository/sample/1/model.py index 60af9b5..96c3115 100644 --- a/model_repository/sample/1/model.py +++ b/model_repository/sample/1/model.py @@ -17,7 +17,7 @@ def execute(self, requests): responses = [None for _ in requests] for idx, request in enumerate(requests): current_add_value = int(json.loads(request.parameters()).get("add", 0)) - in_tensor = [item.as_numpy() + current_add_value for item in request.inputs() if item.name() == "model_in"] + in_tensor = [item.as_numpy() + current_add_value for item in request.inputs() if "model_in" in item.name()] out_tensor = [ pb_utils.Tensor(output_name, x.astype(output_dtype)) for x, output_name, output_dtype in zip(in_tensor, self.output_name_list, self.output_dtype_list) diff --git a/model_repository/sample_multiple_hybrid_dims/1/model.py b/model_repository/sample_multiple_hybrid_dims/1/model.py new file mode 100644 index 0000000..43561a3 --- /dev/null +++ b/model_repository/sample_multiple_hybrid_dims/1/model.py @@ -0,0 +1,45 @@ +import json + +import numpy as np +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def initialize(self, args): + self.model_config = model_config = json.loads(args["model_config"]) + output_configs = model_config["output"] + + self.output_name_list = [ + output_config["name"] for output_config in output_configs + ] + self.output_dtype_list = [ + pb_utils.triton_string_to_numpy(output_config["data_type"]) + for output_config in output_configs + ] + + def execute(self, requests): + responses = [None for _ in requests] + for idx, request in enumerate(requests): + current_add_value = int(json.loads(request.parameters()).get("add", 0)) + in_tensor = [ + item.as_numpy() + current_add_value + for item in request.inputs() + if "model_in" in item.name() + ] + + out_tensor = [ + pb_utils.Tensor(output_name, x.astype(output_dtype)) + for x, output_name, output_dtype in zip( + in_tensor, self.output_name_list, self.output_dtype_list + ) + ] + inference_response = pb_utils.InferenceResponse(output_tensors=out_tensor) + out_tensor.append( + pb_utils.Tensor( + "model_out2", + np.array([current_add_value], dtype=self.output_dtype_list[1]), + ) + ) + + responses[idx] = inference_response + return responses diff --git a/model_repository/sample_multiple_hybrid_dims/config.pbtxt b/model_repository/sample_multiple_hybrid_dims/config.pbtxt new file mode 100644 index 0000000..6c29b4e --- /dev/null +++ b/model_repository/sample_multiple_hybrid_dims/config.pbtxt @@ -0,0 +1,63 @@ +name: "sample_multiple_hybrid_dims" +backend: "python" +max_batch_size: 2 + +parameters [ + { + key: "add", + value: { string_value: "0" } + } +] + +input [ +{ + name: "model_in0" + data_type: TYPE_FP32 + dims: [ -1 ] +}, +{ + name: "model_in1" + data_type: TYPE_FP32 + dims: [ -1 ] +} +] + +output [ +{ + name: "model_out0" + data_type: TYPE_FP32 + dims: [ -1 ] +}, +{ + name: "model_out1" + data_type: TYPE_FP32 + dims: [ -1 ] +}, +{ + name: "model_out2" + data_type: TYPE_FP32 + dims: [ 1 ] +} +] + +instance_group [{ kind: KIND_CPU, count: 1 }] + +model_warmup { + name: "RandomSampleInput" + batch_size: 1 + inputs [{ + key: "model_in0" + value: { + data_type: TYPE_FP32 + dims: [ 10 ] + random_data: true + } + }, { + key: "model_in1" + value: { + data_type: TYPE_FP32 + dims: [ 10 ] + zero_data: true + } + }] +} \ No newline at end of file diff --git a/tests/test_model_call.py b/tests/test_model_call.py index 4077e3d..2306fe0 100644 --- a/tests/test_model_call.py +++ b/tests/test_model_call.py @@ -69,3 +69,22 @@ def test_reload_model_spec(config): sample = np.random.rand(8, 100).astype(np.float32) result = client(sample) assert np.isclose(result, sample).all() + + +def test_with_multiple_hybrid_dims(config): + client = get_client(*config, model_name="sample_multiple_hybrid_dims") + + samples = [np.random.rand(1, 100).astype(np.float32) for _ in range(2)] + ADD_VALUE = 1 + result = client( + { + client.default_model_spec.model_input[0].name: samples[0], + client.default_model_spec.model_input[1].name: samples[1], + }, + parameters={"add": f"{ADD_VALUE}"}, + ) + + assert np.isclose(result[0], samples[0][0] + ADD_VALUE).all() + assert np.isclose(result[1], samples[1][0] + ADD_VALUE).all() + + assert np.isclose(result[2], ADD_VALUE).all() From d255b64651b2d0d7fa82205bfb9bdbcec8a27cd8 Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Thu, 23 May 2024 18:09:24 +0900 Subject: [PATCH 4/7] apply black lint --- .../sample_multiple_hybrid_dims/1/model.py | 17 ++++------------- tritony/tools.py | 4 +--- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/model_repository/sample_multiple_hybrid_dims/1/model.py b/model_repository/sample_multiple_hybrid_dims/1/model.py index 43561a3..328a011 100644 --- a/model_repository/sample_multiple_hybrid_dims/1/model.py +++ b/model_repository/sample_multiple_hybrid_dims/1/model.py @@ -9,29 +9,20 @@ def initialize(self, args): self.model_config = model_config = json.loads(args["model_config"]) output_configs = model_config["output"] - self.output_name_list = [ - output_config["name"] for output_config in output_configs - ] + self.output_name_list = [output_config["name"] for output_config in output_configs] self.output_dtype_list = [ - pb_utils.triton_string_to_numpy(output_config["data_type"]) - for output_config in output_configs + pb_utils.triton_string_to_numpy(output_config["data_type"]) for output_config in output_configs ] def execute(self, requests): responses = [None for _ in requests] for idx, request in enumerate(requests): current_add_value = int(json.loads(request.parameters()).get("add", 0)) - in_tensor = [ - item.as_numpy() + current_add_value - for item in request.inputs() - if "model_in" in item.name() - ] + in_tensor = [item.as_numpy() + current_add_value for item in request.inputs() if "model_in" in item.name()] out_tensor = [ pb_utils.Tensor(output_name, x.astype(output_dtype)) - for x, output_name, output_dtype in zip( - in_tensor, self.output_name_list, self.output_dtype_list - ) + for x, output_name, output_dtype in zip(in_tensor, self.output_name_list, self.output_dtype_list) ] inference_response = pb_utils.InferenceResponse(output_tensors=out_tensor) out_tensor.append( diff --git a/tritony/tools.py b/tritony/tools.py index dc8efc9..7f8ff95 100644 --- a/tritony/tools.py +++ b/tritony/tools.py @@ -493,9 +493,7 @@ def _call_request( if model_spec.max_batch_size == 0: result_by_output_name = zipped_result else: - result_by_output_name = list( - map(lambda ll: safe_concatenate(ll, axis=0), zipped_result) - ) + result_by_output_name = list(map(lambda ll: safe_concatenate(ll, axis=0), zipped_result)) if len(result_by_output_name) == 1: result_by_output_name = result_by_output_name[0] From 69fd4a868abf294fa199d4561994121b2405245f Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Thu, 23 May 2024 18:33:49 +0900 Subject: [PATCH 5/7] remove unnecessary code --- tritony/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tritony/tools.py b/tritony/tools.py index 7f8ff95..28e02a6 100644 --- a/tritony/tools.py +++ b/tritony/tools.py @@ -75,7 +75,7 @@ def safe_concatenate(arrays, axis=0): ) for arr in arrays ] - return np.concatenate(arrays, axis=axis) if arrays else np.array([]) + return np.concatenate(arrays, axis=axis) async def send_request_async( From 0aa3efbb89a1e4e07663973bd0c75d04bf6eb349 Mon Sep 17 00:00:00 2001 From: Dongwoo Arthur Kim Date: Fri, 24 May 2024 12:03:32 +0900 Subject: [PATCH 6/7] Fix wrong test --- tests/test_model_call.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_model_call.py b/tests/test_model_call.py index 2306fe0..82ba627 100644 --- a/tests/test_model_call.py +++ b/tests/test_model_call.py @@ -81,10 +81,10 @@ def test_with_multiple_hybrid_dims(config): client.default_model_spec.model_input[0].name: samples[0], client.default_model_spec.model_input[1].name: samples[1], }, - parameters={"add": f"{ADD_VALUE}"}, + parameters={"add": ADD_VALUE}, ) assert np.isclose(result[0], samples[0][0] + ADD_VALUE).all() assert np.isclose(result[1], samples[1][0] + ADD_VALUE).all() - assert np.isclose(result[2], ADD_VALUE).all() + assert result[2][0] == ADD_VALUE From a09c5845db4309856755725e53db45aa8aaddc4e Mon Sep 17 00:00:00 2001 From: Pierce Ahn Date: Fri, 24 May 2024 12:29:45 +0900 Subject: [PATCH 7/7] fix model bug --- model_repository/sample_multiple_hybrid_dims/1/model.py | 6 ++++-- tests/test_model_call.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/model_repository/sample_multiple_hybrid_dims/1/model.py b/model_repository/sample_multiple_hybrid_dims/1/model.py index 328a011..b758cb8 100644 --- a/model_repository/sample_multiple_hybrid_dims/1/model.py +++ b/model_repository/sample_multiple_hybrid_dims/1/model.py @@ -24,13 +24,15 @@ def execute(self, requests): pb_utils.Tensor(output_name, x.astype(output_dtype)) for x, output_name, output_dtype in zip(in_tensor, self.output_name_list, self.output_dtype_list) ] - inference_response = pb_utils.InferenceResponse(output_tensors=out_tensor) + out_tensor.append( pb_utils.Tensor( "model_out2", - np.array([current_add_value], dtype=self.output_dtype_list[1]), + np.array([current_add_value], dtype=self.output_dtype_list[2]), ) ) + inference_response = pb_utils.InferenceResponse(output_tensors=out_tensor) + responses[idx] = inference_response return responses diff --git a/tests/test_model_call.py b/tests/test_model_call.py index 82ba627..a9e122d 100644 --- a/tests/test_model_call.py +++ b/tests/test_model_call.py @@ -87,4 +87,4 @@ def test_with_multiple_hybrid_dims(config): assert np.isclose(result[0], samples[0][0] + ADD_VALUE).all() assert np.isclose(result[1], samples[1][0] + ADD_VALUE).all() - assert result[2][0] == ADD_VALUE + assert result[2] == ADD_VALUE