Merge branch 'master' into master

Author: lordgreg

changelog/unreleased/kong/fix-ai-proxy-anthropic-tool-choice.yml

@@ -0,0 +1,4 @@
+message: >
+  **ai-proxy**: Fixed an issue where OpenAI chat completion's tool_choice was not converted to Anthropic's.
+type: bugfix
+scope: Plugin

kong/llm/drivers/anthropic.lua

@@ -128,6 +128,33 @@ local function to_tools(in_tools)
   return out_tools
 end
 
+local function to_tool_choice(openai_tool_choice)
+  -- See https://docs.anthropic.com/en/api/messages#body-tool-choice and
+  -- https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice
+  if type(openai_tool_choice) == "string" then
+    if openai_tool_choice == "required" then
+      return {type = "any"}
+    elseif openai_tool_choice == "none" or openai_tool_choice == "auto" then
+      return {type = openai_tool_choice}
+    else
+      kong.log.warn("invalid tool choice string: ", openai_tool_choice, ", expected 'required', 'none', or 'auto'")
+      return nil
+    end
+  end
+
+  if type(openai_tool_choice) == "table" then
+    if openai_tool_choice.type == "function" and openai_tool_choice["function"].name then
+      return {type = "tool", name = openai_tool_choice["function"].name}
+    end
+
+    kong.log.warn("invalid tool choice table: ", cjson.encode(openai_tool_choice))
+    return nil
+  end
+
+  kong.log.warn("invalid tool choice type: ", type(openai_tool_choice), ", expected string or table")
+  return nil
+end
+
 local transformers_to = {
   ["llm/v1/chat"] = function(request_table, model)
     local messages = {}
@@ -145,7 +172,7 @@ local transformers_to = {
 
     -- handle function calling translation from OpenAI format
     messages.tools = request_table.tools and to_tools(request_table.tools)
-    messages.tool_choice = request_table.tool_choice
+    messages.tool_choice = request_table.tool_choice and to_tool_choice(request_table.tool_choice)
 
     return messages, "application/json", nil
   end,

spec/03-plugins/38-ai-proxy/03-anthropic_integration_spec.lua

@@ -147,6 +147,27 @@ for _, strategy in helpers.all_strategies() do
               }
             }
 
+            location = "/llm/v1/chat/tool_choice" {
+              content_by_lua_block {
+                local pl_file = require "pl.file"
+                local json = require("cjson.safe")
+
+                ngx.req.read_body()
+                local function assert_ok(ok, err)
+                  if not ok then
+                    ngx.status = 500
+                    ngx.say(err)
+                    ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
+                  end
+                  return ok
+                end
+                local body = assert_ok(ngx.req.get_body_data())
+                body = assert_ok(json.decode(body))
+                local tool_choice = body.tool_choice
+                ngx.header["tool-choice"] = json.encode(tool_choice)
+                ngx.print(pl_file.read("spec/fixtures/ai-proxy/anthropic/llm-v1-chat/responses/good.json"))
+              }
+            }
 
             location = "/llm/v1/completions/good" {
               content_by_lua_block {
@@ -374,6 +395,36 @@ for _, strategy in helpers.all_strategies() do
       }
       --
 
+      -- 200 chat tool_choice response
+      local chat_tool_choice = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "/anthropic/llm/v1/chat/tool_choice" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_tool_choice.id },
+        config = {
+          route_type = "llm/v1/chat",
+          auth = {
+            header_name = "x-api-key",
+            header_value = "anthropic-key",
+          },
+          model = {
+            name = "claude-2.1",
+            provider = "anthropic",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              upstream_url = "http://"..helpers.mock_upstream_host..":"..MOCK_PORT.."/llm/v1/chat/tool_choice",
+              anthropic_version = "2023-06-01",
+            },
+          },
+        },
+      }
+      --
+
       -- 401 unauthorized
       local chat_401 = assert(bp.routes:insert {
         service = empty_service,
@@ -740,6 +791,42 @@ for _, strategy in helpers.all_strategies() do
         local json = cjson.decode(body)
         assert.is_truthy(deepcompare(json.usage, {}))
       end)
+
+      it("tool_choice conversion", function()
+        local function get_converted_tool_choice(input)
+          local body = pl_file.read(input)
+          -- rewrite the model so we can reuse the same test fixture with different models
+          body = cjson.decode(body)
+          body.model = "claude-2.1" -- anthropic model name
+          local r = client:post("/anthropic/llm/v1/chat/tool_choice", {
+            headers = {
+              ["content-type"] = "application/json",
+              ["accept"] = "application/json",
+            },
+            body = cjson.encode(body),
+          })
+          r:read_body()
+          local sent = r.headers["tool-choice"]
+          if not sent then
+            return nil
+          end
+          return cjson.decode(sent)
+        end
+
+        for _, case in ipairs({
+          {input = "spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_auto.json",
+           output = {type = "auto"}},
+          {input = "spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_none.json",
+           output = {type = "none"}},
+          {input = "spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_required.json",
+           output = {type = "any"}},
+          {input = "spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_object_function.json",
+           output = {type = "tool", name = "my_function"}},
+        }) do
+          local r = get_converted_tool_choice(case.input)
+          assert.same(case.output, r)
+        end
+      end)
     end)
 
     describe("anthropic llm/v1/completions", function()

spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_auto.json

@@ -0,0 +1,33 @@
+{
+  "model": "gpt-4.1",
+  "messages": [
+    {
+      "role": "user",
+      "content": "What is the weather like in Boston today?"
+    }
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_current_weather",
+        "description": "Get the current weather in a given location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "The city and state, e.g. San Francisco, CA"
+            },
+            "unit": {
+              "type": "string",
+              "enum": ["celsius", "fahrenheit"]
+            }
+          },
+          "required": ["location"]
+        }
+      }
+    }
+  ],
+  "tool_choice": "auto"
+}

spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_none.json

@@ -0,0 +1,10 @@
+{
+  "model": "gpt-4.1",
+  "messages": [
+    {
+      "role": "user",
+      "content": "What is the weather like in Boston today?"
+    }
+  ],
+  "tool_choice": "none"
+}

spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_object_function.json

@@ -0,0 +1,28 @@
+{
+  "model": "gpt-4-0613",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are an AI assistant."
+    },
+    {
+      "role": "user",
+      "content": "Tell me a joke."
+    }
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "my_function",
+        "parameters": {}
+      }
+    }
+  ],
+  "tool_choice": {
+    "type": "function",
+    "function": {
+      "name": "my_function"
+    }
+  }
+}

spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/tool_choice_required.json

@@ -0,0 +1,21 @@
+{
+  "model": "gpt-4-0613",
+  "messages": [
+    {
+      "role": "user",
+      "content": "What is the weather today in Paris?"
+    }
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_current_weather",
+        "parameters": {
+          "location": "Paris"
+        }
+      }
+    }
+  ],
+  "tool_choice": "required"
+}