From 54dbf4649e1d219dd6e7b48a414ffa975069be13 Mon Sep 17 00:00:00 2001 From: Angel Date: Thu, 26 Sep 2024 16:38:24 -0400 Subject: [PATCH 1/3] custom model dosc --- .../_anthropic.md | 1 + .../llm-provider-integration-guides/_azure.md | 2 ++ .../_cohere.md | 2 ++ .../_llama2.md | 1 + .../_mistral.md | 1 + .../_openai.md | 2 ++ .../_anthropic.md | 1 + .../llm-provider-integration-guides/_azure.md | 1 + .../_cohere.md | 2 ++ .../_llama2.md | 2 ++ .../_mistral.md | 1 + .../_openai.md | 1 + .../plugins-hub/ai-custom-model-advanced.md | 32 +++++++++++++++++++ .../md/plugins-hub/ai-custom-model.md | 32 +++++++++++++++++++ 14 files changed, 81 insertions(+) create mode 100644 app/_includes/md/plugins-hub/ai-custom-model-advanced.md create mode 100644 app/_includes/md/plugins-hub/ai-custom-model.md diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md index 876a0ee830ab..f99b6b054216 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md @@ -66,3 +66,4 @@ curl -X POST http://localhost:8000/anthropic-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md index df5e3d6bf822..e33775c1e536 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md @@ -77,3 +77,5 @@ curl -X POST http://localhost:8000/azure-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md index 4ee33a5b4286..0dcc43eae122 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md @@ -64,3 +64,5 @@ curl -X POST http://localhost:8000/cohere-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md index 3548e41c38af..cb7a5ff1b4d1 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md @@ -112,3 +112,4 @@ curl -X POST http://localhost:8000/llama2-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md index 24743c78d150..ec38ac8bf855 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md @@ -103,3 +103,4 @@ curl -X POST http://localhost:8000/mistral-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md index 43356bbfa2a0..ce12ce54e61d 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md @@ -55,6 +55,7 @@ formats: {% endplugin_example %} + ### Test the configuration Make an `llm/v1/chat` type request to test your new endpoint: @@ -64,3 +65,4 @@ curl -X POST http://localhost:8000/openai-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md index 6df8617d5251..4aba06abaa48 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md @@ -82,3 +82,4 @@ curl -X POST http://localhost:8000/anthropic-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md index 765f6dbf683f..656545a64656 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md @@ -89,3 +89,4 @@ curl -X POST http://localhost:8000/azure-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md index c53952862a36..21f09f5fd9c8 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md @@ -76,3 +76,5 @@ curl -X POST http://localhost:8000/cohere-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md index ae7e344223ea..d7c40f0ef4af 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md @@ -123,3 +123,5 @@ curl -X POST http://localhost:8000/llama2-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md index 05030b206825..25e764f425d9 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md @@ -115,3 +115,4 @@ curl -X POST http://localhost:8000/mistral-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md index 4bde4359d404..e058cf6011ee 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md @@ -76,3 +76,4 @@ curl -X POST http://localhost:8000/openai-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_includes/md/plugins-hub/ai-custom-model-advanced.md b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md new file mode 100644 index 000000000000..dd6fedf84e46 --- /dev/null +++ b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md @@ -0,0 +1,32 @@ + +For all providers, the Kong AI Proxy Advanced plugin attaches to **route** entities. + +### Custom model + +You can configure the AI Proxy Advanced Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + targets: + - route_type: "llm/v1/chat" + auth: + header_name: "Authorization" + header_value: "Bearer " + - model: + name: custom_model_name + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock + options: + upstream_url: http://localhost:8000/vi/chat/completions +targets: + - route +formats: + - curl + - konnect + - yaml + - kubernetes + - terraform +{% endplugin_example %} + diff --git a/app/_includes/md/plugins-hub/ai-custom-model.md b/app/_includes/md/plugins-hub/ai-custom-model.md new file mode 100644 index 000000000000..dd6fedf84e46 --- /dev/null +++ b/app/_includes/md/plugins-hub/ai-custom-model.md @@ -0,0 +1,32 @@ + +For all providers, the Kong AI Proxy Advanced plugin attaches to **route** entities. + +### Custom model + +You can configure the AI Proxy Advanced Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + targets: + - route_type: "llm/v1/chat" + auth: + header_name: "Authorization" + header_value: "Bearer " + - model: + name: custom_model_name + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock + options: + upstream_url: http://localhost:8000/vi/chat/completions +targets: + - route +formats: + - curl + - konnect + - yaml + - kubernetes + - terraform +{% endplugin_example %} + From 08f478b829c1844047653cc415084003af8ac048 Mon Sep 17 00:00:00 2001 From: Angel Date: Tue, 10 Dec 2024 09:27:33 -0500 Subject: [PATCH 2/3] Apply suggestions from code review --- app/_includes/md/plugins-hub/ai-custom-model.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/_includes/md/plugins-hub/ai-custom-model.md b/app/_includes/md/plugins-hub/ai-custom-model.md index dd6fedf84e46..c9be2fe2dde1 100644 --- a/app/_includes/md/plugins-hub/ai-custom-model.md +++ b/app/_includes/md/plugins-hub/ai-custom-model.md @@ -1,14 +1,14 @@ -For all providers, the Kong AI Proxy Advanced plugin attaches to **route** entities. +For all providers, the Kong AI Proxy plugin attaches to **route** entities. ### Custom model -You can configure the AI Proxy Advanced Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. +You can configure the AI Proxy Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. {% plugin_example %} plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced +name: ai-advanced config: targets: - route_type: "llm/v1/chat" From cc08269eaaa5bf0c2111346608d5dee17c179cf1 Mon Sep 17 00:00:00 2001 From: Angel Date: Wed, 18 Dec 2024 16:48:31 -0500 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Lucie Milan <32450552+lmilan@users.noreply.github.com> --- .../plugins-hub/ai-custom-model-advanced.md | 4 ++-- .../md/plugins-hub/ai-custom-model.md | 19 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/app/_includes/md/plugins-hub/ai-custom-model-advanced.md b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md index dd6fedf84e46..70bb33577b5e 100644 --- a/app/_includes/md/plugins-hub/ai-custom-model-advanced.md +++ b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md @@ -15,9 +15,9 @@ config: auth: header_name: "Authorization" header_value: "Bearer " - - model: + model: name: custom_model_name - provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock|huggingface options: upstream_url: http://localhost:8000/vi/chat/completions targets: diff --git a/app/_includes/md/plugins-hub/ai-custom-model.md b/app/_includes/md/plugins-hub/ai-custom-model.md index c9be2fe2dde1..33689a61b4ae 100644 --- a/app/_includes/md/plugins-hub/ai-custom-model.md +++ b/app/_includes/md/plugins-hub/ai-custom-model.md @@ -10,16 +10,15 @@ You can configure the AI Proxy Plugin using a custom model of your choice by set plugin: kong-inc/ai-proxy-advanced name: ai-advanced config: - targets: - - route_type: "llm/v1/chat" - auth: - header_name: "Authorization" - header_value: "Bearer " - - model: - name: custom_model_name - provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock - options: - upstream_url: http://localhost:8000/vi/chat/completions + route_type: "llm/v1/chat" + auth: + header_name: "Authorization" + header_value: "Bearer " + model: + name: custom_model_name + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock|huggingface + options: + upstream_url: http://localhost:8000/vi/chat/completions targets: - route formats: