Skip to content

Commit 2bd801f

Browse files
authored
fix: ensure Gemini LLM always includes system_instruction in config (#231)
The Gemini SDK overrides the chat-level system_instruction whenever a config is passed to send_message_stream. This caused agent instructions to be ignored when tools were registered or other config options were set. The fix modifies _build_config() to always include self._instructions as the system_instruction (unless an explicit one is provided), ensuring instructions are preserved in all cases.
1 parent 53341d6 commit 2bd801f

File tree

1 file changed

+16
-7
lines changed

1 file changed

+16
-7
lines changed

plugins/gemini/vision_agents/plugins/gemini/gemini_llm.py

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,8 @@ def _build_config(
106106
Build GenerateContentConfig with Gemini 3 features.
107107
108108
Args:
109-
system_instruction: Optional system instruction to include
109+
system_instruction: Optional system instruction to include. If not provided,
110+
uses self._instructions to ensure instructions are always passed.
110111
base_config: Optional base config to extend (takes precedence over self._base_config)
111112
112113
Returns:
@@ -119,8 +120,13 @@ def _build_config(
119120
else:
120121
config = GenerateContentConfig()
121122

122-
if system_instruction:
123-
config.system_instruction = system_instruction
123+
# Always include system instruction - passing any config to send_message_stream
124+
# overrides the chat-level system instruction, so we must include it every time
125+
effective_instruction = (
126+
system_instruction if system_instruction else self._instructions
127+
)
128+
if effective_instruction:
129+
config.system_instruction = effective_instruction
124130

125131
if self.thinking_level:
126132
from google.genai.types import ThinkingConfig
@@ -176,14 +182,17 @@ async def send_message(self, *args, **kwargs):
176182
cfg = self._build_config(base_config=cfg)
177183
cfg.tools = conv_tools # type: ignore[assignment]
178184
kwargs["config"] = cfg
179-
else:
185+
elif self.thinking_level or self.media_resolution:
186+
# Only pass config if we need to set thinking_level or media_resolution
187+
# Don't pass an empty config as it overrides the system_instruction from chat creation
180188
cfg = kwargs.get("config")
181189
if cfg is None or not isinstance(cfg, GenerateContentConfig):
182190
cfg = self._build_config()
183-
kwargs["config"] = cfg
184-
elif self.thinking_level or self.media_resolution:
191+
else:
185192
cfg = self._build_config(base_config=cfg)
186-
kwargs["config"] = cfg
193+
kwargs["config"] = cfg
194+
# If no tools and no thinking/media config needed, don't pass config
195+
# This preserves the system_instruction set during chat creation
187196

188197
# Generate content using the client
189198
iterator: AsyncIterator[

0 commit comments

Comments
 (0)