Add TritonModelInput with optional

.github/workflows/pre-commit_pytest.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: pre-commit
     container:
-      image: nvcr.io/nvidia/tritonserver:23.03-pyt-python-py3
+      image: nvcr.io/nvidia/tritonserver:23.08-pyt-python-py3
       options:
         --shm-size=1g
     steps:

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
     hooks:
       - id: flake8
         types: [python]
-        args: ["--max-line-length", "120", "--ignore", "F811,F841,E203,E402,E712,W503"]
+        args: ["--max-line-length", "120", "--ignore", "F811,F841,E203,E402,E712,W503,E501"]
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.9.0.5
     hooks:

README.md

@@ -55,6 +55,7 @@ if __name__ == "__main__":
 
 ## Release Notes
 
+- 23.08.30 Support `optional` with model input, `parameters` on config.pbtxt
 - 23.06.16 Support tritonclient>=2.34.0
 - Loosely modified the requirements related to tritonclient
 
@@ -79,14 +80,11 @@ if __name__ == "__main__":
 ### With Triton
 
 ```bash
-docker run --rm \
-    -v ${PWD}:/models \
-    nvcr.io/nvidia/tritonserver:22.01-pyt-python-py3 \
-    tritonserver --model-repo=/models
+./bin/run_triton_tritony_sample.sh
 ```
 
 ```bash
-pytest -m -s tests/test_tritony.py
+pytest -s --cov-report term-missing --cov=tritony tests/
 ```
 
 ### Example with image_client.py

bin/run_triton_tritony_sample.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+HERE=$(dirname "$(readlink -f $0)")
+PARENT_DIR=$(dirname "$HERE")
+
+docker run -it --rm --name triton_tritony \
+    -p8100:8000   \
+    -p8101:8001   \
+    -p8102:8002    \
+    -v "${PARENT_DIR}"/model_repository:/models:ro \
+    -e OMP_NUM_THREADS=2 \
+    -e OPENBLAS_NUM_THREADS=2 \
+    --shm-size=1g  \
+    nvcr.io/nvidia/tritonserver:23.08-pyt-python-py3 \
+    tritonserver --model-repository=/models \
+    --exit-timeout-secs 15 \
+    --min-supported-compute-capability 7.0 \
+    --log-verbose 0  # 0-nothing, 1-info, 2-debug, 3-trace

model_repository/sample/1/model.py

@@ -16,7 +16,8 @@ def initialize(self, args):
     def execute(self, requests):
         responses = [None for _ in requests]
         for idx, request in enumerate(requests):
-            in_tensor = [item.as_numpy() for item in request.inputs()]
+            current_add_value = int(json.loads(request.parameters()).get("add", 0))
+            in_tensor = [item.as_numpy() + current_add_value for item in request.inputs() if item.name() == "model_in"]
             out_tensor = [
                 pb_utils.Tensor(output_name, x.astype(output_dtype))
                 for x, output_name, output_dtype in zip(in_tensor, self.output_name_list, self.output_dtype_list)

model_repository/sample/config.pbtxt

@@ -1,6 +1,7 @@
 name: "sample"
 backend: "python"
 max_batch_size: 0
+
 input [
 {
     name: "model_in"

model_repository/sample_autobatching/config.pbtxt

@@ -1,6 +1,14 @@
 name: "sample_autobatching"
 backend: "python"
 max_batch_size: 2
+
+parameters [
+  {
+    key: "add",
+    value: { string_value: "0" }
+  }
+]
+
 input [
 {
     name: "model_in"

model_repository/sample_multiple/config.pbtxt

@@ -1,6 +1,14 @@
 name: "sample_multiple"
 backend: "python"
 max_batch_size: 2
+
+parameters [
+  {
+    key: "add",
+    value: { string_value: "0" }
+  }
+]
+
 input [
 {
     name: "model_in0"

model_repository/sample_optional/1/model.py

@@ -0,0 +1,37 @@
+import json
+
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+    def initialize(self, args):
+        self.model_config = model_config = json.loads(args["model_config"])
+        output_configs = model_config["output"]
+
+        self.output_name_list = [output_config["name"] for output_config in output_configs]
+        self.output_dtype_list = [
+            pb_utils.triton_string_to_numpy(output_config["data_type"]) for output_config in output_configs
+        ]
+
+    def execute(self, requests):
+        responses = [None for _ in requests]
+        for idx, request in enumerate(requests):
+            current_add_value = int(json.loads(request.parameters()).get("add", 0))
+            optional_in_tensor = pb_utils.get_input_tensor_by_name(request, "optional_model_sub")
+            if optional_in_tensor:
+                optional_in_tensor = optional_in_tensor.as_numpy()
+            else:
+                optional_in_tensor = 0
+
+            in_tensor = [
+                item.as_numpy() + current_add_value - optional_in_tensor
+                for item in request.inputs()
+                if item.name() == "model_in"
+            ]
+            out_tensor = [
+                pb_utils.Tensor(output_name, x.astype(output_dtype))
+                for x, output_name, output_dtype in zip(in_tensor, self.output_name_list, self.output_dtype_list)
+            ]
+            inference_response = pb_utils.InferenceResponse(output_tensors=out_tensor)
+            responses[idx] = inference_response
+        return responses

model_repository/sample_optional/config.pbtxt

@@ -0,0 +1,60 @@
+name: "sample_optional"
+backend: "python"
+max_batch_size: 0
+
+parameters [
+  {
+    key: "add",
+    value: { string_value: "0" }
+  }
+]
+
+input [
+{
+    name: "model_in"
+    data_type: TYPE_FP32
+    dims: [ -1 ]
+},
+{
+    name: "optional_model_sub"
+    data_type: TYPE_FP32
+    optional: true
+    dims: [ -1 ]
+},
+{
+    name: "optional_model_string"
+    data_type: TYPE_STRING
+    optional: true
+    dims: [ -1 ]
+}
+]
+
+output [
+{
+    name: "model_out"
+    data_type: TYPE_FP32
+    dims: [ -1 ]
+}
+]
+
+instance_group [{ kind: KIND_CPU, count: 1 }]
+
+model_warmup {
+  name: "RandomSampleInput"
+  batch_size: 1
+  inputs [{
+      key: "model_in"
+      value: {
+        data_type: TYPE_FP32
+        dims: [ 10 ]
+        random_data: true
+      }
+   }, {
+      key: "model_in"
+      value: {
+        data_type: TYPE_FP32
+        dims: [ 10 ]
+        zero_data: true
+      }
+   }]
+}

setup.cfg

@@ -43,6 +43,8 @@ classifiers =
 zip_safe = False
 include_package_data = True
 packages = find:
+package_dir =
+    =.
 install_requires =
     tritonclient[all]>=2.21.0
     protobuf>=3.5.0

tests/test_connect.py

@@ -25,10 +25,10 @@ def test_basics(protocol_and_port):
 
     sample = np.random.rand(1, 100).astype(np.float32)
     result = client(sample)
-    print(f"Result: {np.isclose(result, sample).all()}")
+    assert np.isclose(result, sample).all()
 
     result = client({"model_in": sample})
-    print(f"Dict Result: {np.isclose(result, sample).all()}")
+    assert np.isclose(result, sample).all()
 
 
 def test_batching(protocol_and_port):
@@ -40,7 +40,7 @@ def test_batching(protocol_and_port):
     sample = np.random.rand(100, 100).astype(np.float32)
     # client automatically makes sub batches with (50, 2, 100)
     result = client(sample)
-    print(f"Result: {np.isclose(result, sample).all()}")
+    assert np.isclose(result, sample).all()
 
 
 def test_exception(protocol_and_port):

tests/test_model_call.py

@@ -5,43 +5,72 @@
 
 from tritony import InferenceClient
 
-MODEL_NAME = os.environ.get("MODEL_NAME", "sample")
 TRITON_HOST = os.environ.get("TRITON_HOST", "localhost")
 TRITON_HTTP = os.environ.get("TRITON_HTTP", "8000")
 TRITON_GRPC = os.environ.get("TRITON_GRPC", "8001")
 
 
+EPSILON = 1e-8
+
+
 @pytest.fixture(params=[("http", TRITON_HTTP), ("grpc", TRITON_GRPC)])
 def protocol_and_port(request):
     return request.param
 
 
-def test_swithcing(protocol_and_port):
-    protocol, port = protocol_and_port
-    print(f"Testing {protocol}")
+def get_client(protocol, port, model_name):
+    print(f"Testing {protocol}", flush=True)
+    return InferenceClient.create_with(model_name, f"{TRITON_HOST}:{port}", protocol=protocol)
+
 
-    client = InferenceClient.create_with(MODEL_NAME, f"{TRITON_HOST}:{port}", protocol=protocol)
+def test_swithcing(protocol_and_port):
+    client = get_client(*protocol_and_port, model_name="sample")
 
     sample = np.random.rand(1, 100).astype(np.float32)
     result = client(sample)
-    print(f"Result: {np.isclose(result, sample).all()}")
+    assert {np.isclose(result, sample).all()}
 
     sample_batched = np.random.rand(100, 100).astype(np.float32)
     client(sample_batched, model_name="sample_autobatching")
-    print(f"Result: {np.isclose(result, sample).all()}")
+    assert np.isclose(result, sample).all()
 
 
 def test_with_input_name(protocol_and_port):
-    protocol, port = protocol_and_port
-    print(f"Testing {protocol}")
+    client = get_client(*protocol_and_port, model_name="sample")
+
+    sample = np.random.rand(100, 100).astype(np.float32)
+    result = client({client.default_model_spec.model_input[0].name: sample})
+    assert np.isclose(result, sample).all()
+
 
-    client = InferenceClient.create_with(MODEL_NAME, f"{TRITON_HOST}:{port}", protocol=protocol)
+def test_with_parameters(protocol_and_port):
+    client = get_client(*protocol_and_port, model_name="sample")
 
     sample = np.random.rand(1, 100).astype(np.float32)
-    result = client({client.input_name_list[0]: sample})
-    print(f"Result: {np.isclose(result, sample).all()}")
+    ADD_VALUE = 1
+    result = client({client.default_model_spec.model_input[0].name: sample}, parameters={"add": f"{ADD_VALUE}"})
+
+    assert np.isclose(result[0], sample[0] + ADD_VALUE).all()
+
+
+def test_with_optional(protocol_and_port):
+    client = get_client(*protocol_and_port, model_name="sample_optional")
+
+    sample = np.random.rand(1, 100).astype(np.float32)
+
+    result = client({client.default_model_spec.model_input[0].name: sample})
+    assert np.isclose(result[0], sample[0], rtol=EPSILON).all()
+
+    OPTIONAL_SUB_VALUE = np.zeros_like(sample) + 3
+    result = client(
+        {
+            client.default_model_spec.model_input[0].name: sample,
+            "optional_model_sub": OPTIONAL_SUB_VALUE,
+        }
+    )
+    assert np.isclose(result[0], sample[0] - OPTIONAL_SUB_VALUE, rtol=EPSILON).all()
 
-    sample = np.random.rand(100, 100).astype(np.float32)
-    result = client({client.default_model_spec.input_name[0]: sample})
 
-    print(f"Result: {np.isclose(result, sample).all()}")
+if __name__ == "__main__":
+    test_with_parameters(("grpc", "8101"))
+    test_with_optional(("grpc", "8101"))