Skip to content

Commit 4311248

Browse files
shihaobaigemini-code-assist[bot]hiworldwzj
authored
add chat template (#1111)
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: wangzaijun <[email protected]>
1 parent 1f54ecd commit 4311248

File tree

8 files changed

+407
-7
lines changed

8 files changed

+407
-7
lines changed

‎lightllm/server/api_cli.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,19 @@ def make_argument_parser() -> argparse.ArgumentParser:
132132
default=None,
133133
help="tool call parser type",
134134
)
135+
parser.add_argument(
136+
"--chat_template",
137+
type=str,
138+
default=None,
139+
help=(
140+
"chat template jinja file path. For example:\n"
141+
"- /test/chat_template/tool_chat_template_deepseekv31.jinja\n"
142+
"- /test/chat_template/tool_chat_template_deepseekv32.jinja\n"
143+
"- /test/chat_template/tool_chat_template_qwen.jinja\n"
144+
"- /test/chat_template/tool_chat_template_deepseekr1.jinja"
145+
),
146+
)
147+
135148
parser.add_argument(
136149
"--running_max_req_size", type=int, default=1000, help="the max size for forward requests in the same time"
137150
)

‎lightllm/server/build_prompt.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ def init_tokenizer(args):
66
from lightllm.server.tokenizer import get_tokenizer
77

88
tokenizer = get_tokenizer(args.model_dir, args.tokenizer_mode, trust_remote_code=args.trust_remote_code)
9+
chat_path = args.chat_template
10+
if chat_path is not None:
11+
with open(chat_path, "r", encoding="utf-8") as f:
12+
chat_template_str = f.read()
13+
tokenizer.chat_template = chat_template_str
914

1015

1116
async def build_prompt(request, tools) -> str:

‎lightllm/server/core/objs/start_args_type.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class StartArgs:
3333
tool_call_parser: Optional[str] = field(
3434
default=None, metadata={"choices": ["llama3", "qwen25", "mistral", "deepseekv3", "kimi_k2", "qwen"]}
3535
)
36+
chat_template: Optional[str] = field(default=None)
3637
running_max_req_size: int = field(default=1000)
3738
tp: int = field(default=1)
3839
dp: int = field(default=1)

‎lightllm/server/function_call_parser.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,7 @@
2929

3030
logger = logging.getLogger(__name__)
3131

32-
TOOLS_TAG_LIST = [
33-
"<|plugin|>",
34-
"<function=",
35-
"<tool_call>",
36-
"<|python_tag|>",
37-
"[TOOL_CALLS]",
38-
]
32+
TOOLS_TAG_LIST = ["<|plugin|>", "<function=", "<tool_call>", "<|python_tag|>", "[TOOL_CALLS]", "<｜tool▁calls▁begin｜>"]
3933

4034

4135
class ToolCallItem(BaseModel):

‎test/chat_template/tool_chat_template_deepseekr1.jinja‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{% if not add_generation_prompt is defined %}
2+
{% set add_generation_prompt = false %}
3+
{% endif %}
4+
{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
5+
{%- for message in messages %}
6+
{%- if message['role'] == 'system' %}
7+
{%- if ns.is_first_sp %}
8+
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
9+
{% set ns.is_first_sp = false %}
10+
{%- else %}
11+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
12+
{%- endif %}
13+
{%- endif %}
14+
{%- endfor %}
15+
16+
{# --- Append tool descriptions if tools are defined --- #}
17+
{% if tools is defined and tools is not none %}
18+
{% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. '
19+
'When a tool call is needed, you MUST use the following format to issue the call:\n'
20+
'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n'
21+
'```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n'
22+
'Make sure the JSON is valid.'
23+
'## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %}
24+
{% for tool in tools %}
25+
{% set tool_ns.text = tool_ns.text + '- `' + tool['name'] + '`:\n```json\n' + (tool | tojson) + '\n```\n' %}
26+
{% endfor %}
27+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
28+
{% endif %}
29+
30+
{{ bos_token }}
31+
{{ ns.system_prompt }}
32+
{%- for message in messages %}
33+
{% set content = message['content'] %}
34+
{%- if message['role'] == 'user' %}
35+
{%- set ns.is_tool = false -%}
36+
{%- set ns.is_first = false -%}
37+
{%- set ns.is_last_user = true -%}
38+
{{'<｜User｜>' + content + '<｜Assistant｜>'}}
39+
{%- endif %}
40+
{%- if message['role'] == 'assistant' %}
41+
{% if '</think>' in content %}
42+
{% set content = content.split('</think>')[-1] %}
43+
{% endif %}
44+
{% endif %}
45+
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
46+
{%- set ns.is_last_user = false -%}
47+
{%- if ns.is_tool %}
48+
{{'<｜tool▁outputs▁end｜>'}}
49+
{%- endif %}
50+
{%- set ns.is_first = false %}
51+
{%- set ns.is_tool = false -%}
52+
{%- set ns.is_output_first = true %}
53+
{%- for tool in message['tool_calls'] %}
54+
{%- if not ns.is_first %}
55+
{%- if content is none %}
56+
{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}
57+
{%- else %}
58+
{{content + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}
59+
{%- endif %}
60+
{%- set ns.is_first = true -%}
61+
{%- else %}
62+
{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}
63+
{%- endif %}
64+
{%- endfor %}
65+
{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
66+
{%- endif %}
67+
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
68+
{%- set ns.is_last_user = false -%}
69+
{%- if ns.is_tool %}
70+
{{'<｜tool▁outputs▁end｜>' + content + '<｜end▁of▁sentence｜>'}}
71+
{%- set ns.is_tool = false -%}
72+
{%- else %}
73+
{{content + '<｜end▁of▁sentence｜>'}}
74+
{%- endif %}
75+
{%- endif %}
76+
{%- if message['role'] == 'tool' %}
77+
{%- set ns.is_last_user = false -%}
78+
{%- set ns.is_tool = true -%}
79+
{%- if ns.is_output_first %}
80+
{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}
81+
{%- set ns.is_output_first = false %}
82+
{%- else %}
83+
{{'\n<｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}
84+
{%- endif %}
85+
{%- endif %}
86+
{%- endfor -%}
87+
{% if ns.is_tool %}
88+
{{'<｜tool▁outputs▁end｜>'}}
89+
{% endif %}
90+
{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
91+
{{'<｜Assistant｜>'}}
92+
{% endif %}

‎test/chat_template/tool_chat_template_deepseekv3.jinjia‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
2+
{% if not add_generation_prompt is defined %}
3+
{% set add_generation_prompt = false %}
4+
{% endif %}
5+
6+
{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
7+
{%- for message in messages %}
8+
{%- if message['role'] == 'system' %}
9+
{%- if ns.is_first_sp %}
10+
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
11+
{% set ns.is_first_sp = false %}
12+
{%- else %}
13+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
14+
{%- endif %}
15+
{%- endif %}
16+
{%- endfor -%}
17+
18+
{# --- Append tool descriptions if tools are defined --- #}
19+
{% if tools is defined and tools is not none %}
20+
{% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. '
21+
'When a tool call is needed, you MUST use the following format to issue the call:\n'
22+
'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n'
23+
'```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n'
24+
'Make sure the JSON is valid.'
25+
'## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %}
26+
{% for tool in tools %}
27+
{% set tool_ns.text = tool_ns.text + '\n```json\n' + (tool | tojson) + '\n```\n' %}
28+
{% endfor %}
29+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
30+
{% endif %}
31+
32+
{{- bos_token }}
33+
{{- ns.system_prompt }}
34+
35+
{%- for message in messages %}
36+
{%- if message['role'] == 'user' %}
37+
{%- set ns.is_tool = false -%}
38+
{%- set ns.is_first = false -%}
39+
{%- set ns.is_last_user = true -%}
40+
{{'<｜User｜>' + message['content'] + '<｜Assistant｜>'}}
41+
{%- endif %}
42+
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
43+
{%- set ns.is_last_user = false -%}
44+
{%- if ns.is_tool %}
45+
{{- '<｜tool▁outputs▁end｜>'}}
46+
{%- endif %}
47+
{%- set ns.is_first = false %}
48+
{%- set ns.is_tool = false -%}
49+
{%- set ns.is_output_first = true %}
50+
{%- for tool in message['tool_calls'] %}
51+
{%- set formatted_args = tool['function']['arguments'] if tool['function']['arguments'] is string else tool['function']['arguments']|tojson %}
52+
{%- if not ns.is_first %}
53+
{%- if message['content'] is none %}
54+
{{- '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}}
55+
{%- else %}
56+
{{- message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}}
57+
{%- endif %}
58+
{%- set ns.is_first = true -%}
59+
{%- else %}
60+
{{- '\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}}
61+
{%- endif %}
62+
{%- endfor %}
63+
{{- '<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
64+
{%- endif %}
65+
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
66+
{%- set ns.is_last_user = false -%}
67+
{%- if ns.is_tool %}
68+
{{- '<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}
69+
{%- set ns.is_tool = false -%}
70+
{%- else %}
71+
{% set content = message['content'] %}
72+
{{- content + '<｜end▁of▁sentence｜>'}}
73+
{%- endif %}
74+
{%- endif %}
75+
{%- if message['role'] == 'tool' %}
76+
{%- set ns.is_last_user = false -%}
77+
{%- set ns.is_tool = true -%}
78+
{%- if ns.is_output_first %}
79+
{{- 'Use the results below to formulate an answer to the user question unless additional information is needed.' }}
80+
{{- '<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
81+
{%- set ns.is_output_first = false %}
82+
{%- else %}
83+
{{- '\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
84+
{%- endif %}
85+
{%- endif %}
86+
{%- endfor -%}
87+
88+
{% if ns.is_tool %}
89+
{{- '<｜tool▁outputs▁end｜>'}}
90+
{% endif %}
91+
{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
92+
{{- '<｜Assistant｜>'}}
93+
{% endif %}
94+

‎test/chat_template/tool_chat_template_deepseekv31.jinja‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{% if not add_generation_prompt is defined %}
2+
{% set add_generation_prompt = false %}
3+
{% endif %}
4+
{% if not thinking is defined %}
5+
{% set thinking = false %}
6+
{% endif %}
7+
{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %}
8+
{%- for message in messages %}
9+
{%- if message['role'] == 'system' %}
10+
{%- if ns.is_first_sp %}
11+
{% set ns.system_prompt = ns.system_prompt + message['content'] %}
12+
{% set ns.is_first_sp = false %}
13+
{%- else %}
14+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
15+
{%- endif %}
16+
{%- endif %}
17+
{%- endfor %}
18+
19+
{% if tools is defined and tools is not none %}
20+
{% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %}
21+
{% for tool in tools %}
22+
{% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %}
23+
{% endfor %}
24+
{% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>{{additional_tool_calls}}<｜tool▁calls▁end｜>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %}
25+
{% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
26+
{% endif %}
27+
28+
{{ bos_token }}{{ ns.system_prompt }}
29+
{%- for message in messages %}
30+
{%- if message['role'] == 'user' %}
31+
{%- set ns.is_tool = false -%}
32+
{%- set ns.is_first = false -%}
33+
{%- set ns.is_last_user = true -%}
34+
{% if message['content'] is string %}
35+
{{'<｜User｜>' + message['content']}}
36+
{% else %}
37+
{% for content in message['content'] %}
38+
{% if content['type'] == 'text' %}
39+
{{'<｜User｜>' + content['text']}}
40+
{% endif %}
41+
{% endfor %}
42+
{%- endif %}
43+
{%- endif %}
44+
{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
45+
{%- if ns.is_last_user %}
46+
{{'<｜Assistant｜></think>'}}
47+
{%- endif %}
48+
{%- set ns.is_last_user = false -%}
49+
{%- set ns.is_first = false %}
50+
{%- set ns.is_tool = false -%}
51+
{%- for tool in message['tool_calls'] %}
52+
{%- set formatted_args = tool['function']['arguments'] if tool['function']['arguments'] is string else tool['function']['arguments']|tojson %}
53+
{%- if not ns.is_first %}
54+
{%- if message['content'] is none %}
55+
{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}}
56+
{%- else %}
57+
{{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}}
58+
{%- endif %}
59+
{%- set ns.is_first = true -%}
60+
{%- else %}
61+
{{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}}
62+
{%- endif %}
63+
{%- endfor %}
64+
{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
65+
{%- endif %}
66+
{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %}
67+
{%- if ns.is_last_user %}
68+
{{'<｜Assistant｜>'}}
69+
{%- if message['prefix'] is defined and message['prefix'] and thinking %}
70+
{{'<think>'}}
71+
{%- else %}
72+
{{'</think>'}}
73+
{%- endif %}
74+
{%- endif %}
75+
{%- set ns.is_last_user = false -%}
76+
{%- if ns.is_tool %}
77+
{{message['content'] + '<｜end▁of▁sentence｜>'}}
78+
{%- set ns.is_tool = false -%}
79+
{%- else %}
80+
{%- set content = message['content'] -%}
81+
{%- if '</think>' in content %}
82+
{%- set content = content.split('</think>', 1)[1] -%}
83+
{%- endif %}
84+
{{content + '<｜end▁of▁sentence｜>'}}
85+
{%- endif %}
86+
{%- endif %}
87+
{%- if message['role'] == 'tool' %}
88+
{%- set ns.is_last_user = false -%}
89+
{%- set ns.is_tool = true -%}
90+
{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
91+
{%- endif %}
92+
{%- endfor -%}
93+
{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %}
94+
{{'<｜Assistant｜>'}}
95+
{%- if not thinking %}
96+
{{'</think>'}}
97+
{%- else %}
98+
{{'<think>'}}
99+
{%- endif %}
100+
{% endif %}

0 commit comments

Comments
 (0)