feat:adapt new features in engine 2.12

.fern/metadata.json

@@ -14,5 +14,5 @@
     },
     "exclude_types_from_init_exports": true
   },
-  "originGitCommit": "f4b5b1d56fb73c4e26c11486424723c12e6376bc"
+  "originGitCommit": "9afc7ddf3c7c24bf5c80ae7cd4eb83c257d7ceff"
 }

docs/concepts/vendors.md

@@ -110,7 +110,7 @@ Used with `agent.with_stt()`.
 
 Use `turn_detection.language` for Agora interaction language; it defaults to `en-US`. STT vendor `language` options are serialized under `asr.params` using each provider's own format. If `with_stt()` is omitted, AgentKit defaults to `AresSTT` for global clients and `FengmingSTT` for `Area.CN` clients. Ares does not take a provider language option; AgentKit uses `turn_detection.language` for REST `asr.language`.
 
-`AresSTT` and `FengmingSTT` accept an optional `keywords: List[str]` hotword list.
+`AresSTT` and `FengmingSTT` accept an optional `keywords: List[str]` hotword list. For Ares, this is serialized as top-level `asr.keywords`, not inside `asr.params`.
 
 | Class | Provider | Required Parameters |
 |---|---|---|
@@ -119,6 +119,7 @@ Use `turn_detection.language` for Agora interaction language; it defaults to `en
 | `MicrosoftSTT` | Microsoft Azure | `key`, `region`, `language` |
 | `OpenAISTT` | OpenAI | `api_key` |
 | `GoogleSTT` | Google Cloud | `project_id`, `location`, `adc_credentials_string`, `language` |
+| `GeminiSTT` | Google Gemini | `api_key`, `model`; optional `sample_rate`, `language`, `word_timestamp` |
 | `AmazonSTT` | Amazon Transcribe | `access_key`, `secret_key`, `region`, `language` |
 | `AssemblyAISTT` | AssemblyAI | `api_key`, `language` |
 | `AresSTT` | Ares | — (all optional) |
@@ -152,6 +153,7 @@ Used with `agent.with_mllm()` for the [MLLM flow](../guides/mllm-flow.md). These
 | Class | Provider | Area | Required Parameters |
 |---|---|---|---|
 | `OpenAIRealtime` | OpenAI Realtime | Global | `api_key`; optional `turn_detection` |
+| `OpenAIGptLive` | OpenAI GPT Live (`mllm.vendor`: `openai_gpt_live`) | Global | `api_key`; optional `model`, `greeting_message`, `turn_detection` |
 | `AzureOpenAIRealtime` | Azure OpenAI Realtime | Global | `api_key`, `url`, `turn_detection`; optional `max_history` |
 | `GeminiLive` | Google Gemini Live API | Global | `api_key`, `model`; optional `turn_detection` |
 | `VertexAI` | Vertex AI (Gemini Live) | Global | `model`, `project_id`, `location`, `adc_credentials_string`; optional `turn_detection` |

docs/guides/agent-builder-features.md

@@ -26,7 +26,7 @@ Pass `client=client` to every `Agent(...)` builder. `create_session()` and `crea
 |---|---|---|
 | `sal` | `with_sal(config)` | Selective Attention Locking — speaker recognition and noise suppression |
 | `advanced_features` | `with_advanced_features(features)` | Enable MLLM, RTM, SAL, tools |
-| `tools` | `with_tools(enabled=True)` | Enable MCP tool invocation |
+| `tools` | `with_tools(enabled=True)` | Enable MCP and inline LLM tool invocation |
 | `parameters` | `with_parameters(params)` | Silence config, farewell config, data channel |
 | `failure_message` | LLM/MLLM vendor option | Message spoken when LLM fails |
 | `max_history` | LLM vendor option | Max conversation turns in LLM context |
@@ -217,6 +217,8 @@ from agora_agent import (
     FillerWordsTriggerFixedTimeConfig,
     FillerWordsContent,
     FillerWordsContentStaticConfig,
+    FillerWordsContentGeneratedConfig,
+    FillerWordsContentGeneratedConfigLlmProvider,
     FillerWordsSelectionRule,
 )
 
@@ -242,6 +244,37 @@ agent = (
 )
 ```
 
+Generated filler words use an OpenAI-compatible LLM and fall back to the static phrases when generation is unavailable. `static_config.phrases` is required and must be non-empty whenever filler words are enabled.
+
+```python
+from agora_agent import (
+    FillerWordsConfig,
+    FillerWordsContent,
+    FillerWordsContentStaticConfig,
+    FillerWordsContentGeneratedConfig,
+    FillerWordsContentGeneratedConfigLlmProvider,
+)
+
+generated = FillerWordsConfig(
+    enable=True,
+    content=FillerWordsContent(
+        mode='generated',
+        static_config=FillerWordsContentStaticConfig(
+            phrases=['Let me think...', 'One moment...'],
+        ),
+        generated_config=FillerWordsContentGeneratedConfig(
+            llm_provider=FillerWordsContentGeneratedConfigLlmProvider(
+                base_url='https://api.openai.com/v1/chat/completions',
+                api_key='your-filler-llm-key',
+                params={'model': 'gpt-4o-mini'},
+            ),
+            prompt='Generate a short conversational filler phrase; do not answer the user.',
+        ),
+    ),
+)
+agent = Agent(client=client).with_filler_words(generated)
+```
+
 ## Properties (Getters)
 
 Read back configuration via properties:

docs/guides/regional-routing.md

@@ -40,7 +40,7 @@ If you omit `with_stt()`, AgentKit uses `FengmingSTT` by default for `Area.CN` c
 
 | Client area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes |
 |---|---|---|---|---|---|
-| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` |
+| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `GeminiSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `OpenAIGptLive`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` |
 | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` |
 
 Global client example:

docs/reference/agent.md

@@ -53,7 +53,7 @@ Agent(
 | `sal` | `Optional[SalConfig]` | `None` | Speech Activity Level configuration |
 | `advanced_features` | `Optional[Dict[str, Any]]` | `None` | Advanced features dict (e.g., `{'enable_rtm': True}`) |
 | `parameters` | `Optional[SessionParams]` | `None` | Additional session parameters |
-| `greeting` | `Optional[str]` | `None` | Deprecated. Use LLM/MLLM vendor `greeting_message` instead. |
+| `greeting` | `Optional[str]` | `None` | Deprecated. Use the LLM/MLLM vendor `greeting_message` instead. |
 | `failure_message` | `Optional[str]` | `None` | Deprecated. Use LLM/MLLM vendor `failure_message` instead. |
 | `max_history` | `Optional[int]` | `None` | Deprecated. Use LLM vendor `max_history` instead. |
 | `geofence` | `Optional[GeofenceConfig]` | `None` | Regional access restriction |
@@ -173,7 +173,7 @@ When `enable_rtm=True`, AgentKit defaults `parameters.data_channel` to `"rtm"` u
 
 ### `with_tools(enabled: bool = True) -> Agent`
 
-Enable or disable MCP tool invocation by setting `advanced_features.enable_tools`.
+Enable or disable MCP and inline LLM tool invocation by setting `advanced_features.enable_tools`. LLM `tools` definitions require this feature to be enabled, just like `mcp_servers`; call `.with_tools()` explicitly when using either configuration.
 
 ### `with_parameters(parameters: SessionParams) -> Agent`
 
@@ -334,6 +334,6 @@ to_properties(
 
 ## Type aliases
 
-Public aliases over Fern-generated types: `LlmConfig`, `SttConfig`, `AsrConfig` (= `SttConfig`), `MllmConfig`, `AvatarConfig`, session/conversation types, and think types (`ThinkOnListeningAction`, etc.).
+Public aliases over Fern-generated types: `LlmConfig`, `SttConfig`, `AsrConfig` (= `SttConfig`), `MllmConfig`, `AvatarConfig`, session/conversation types, think types (`ThinkOnListeningAction`, etc.), generated filler-word types, and inline REST tool types (`LlmToolConfig`, `LlmToolServerConfig`, and related aliases).
 
-Think value constants: `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, `ThinkOnThinkingActionInterrupt`, `ThinkOnThinkingActionIgnore`, `ThinkOnSpeakingActionInterrupt`, `ThinkOnSpeakingActionIgnore`.
+Think value constants: `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, `ThinkOnListeningActionAppend`, `ThinkOnThinkingActionInterrupt`, `ThinkOnThinkingActionIgnore`, `ThinkOnThinkingActionAppend`, `ThinkOnSpeakingActionInterrupt`, `ThinkOnSpeakingActionIgnore`, `ThinkOnSpeakingActionAppend`.

docs/reference/session.md

@@ -183,7 +183,7 @@ await session.update(properties)
 
 Inject a custom text instruction into the running agent.
 
-In API v2.7, omitting `on_listening_action` uses the server default `interrupt`. Pass `on_listening_action='inject'` explicitly to preserve the pre-v2.7 behavior.
+In API v2.7, omitting `on_listening_action` uses the server default `interrupt`. Supported listening, thinking, and speaking actions include `interrupt`, `ignore`, and `append` where applicable. `append` queues the instruction until the current user/LLM/TTS turn completes without interrupting it. Pass `on_listening_action='inject'` explicitly to preserve the pre-v2.7 behavior.
 
 ```python
 session.think('Summarize the last answer', on_listening_action='inject')

docs/reference/vendors.md

@@ -20,7 +20,7 @@ Construct vendors directly from `agora_agent`, then bind a client with `Agent(cl
 
 | Area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes |
 |---|---|---|---|---|---|
-| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` |
+| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `GeminiSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `OpenAIGptLive`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` |
 | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` |
 
 Global example:
@@ -538,6 +538,18 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For
 | `model` | `str` | No | `None` | Recognition model |
 | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters |
 
+### `GeminiSTT`
+
+Google Gemini transcription vendor. The generated request is `asr.vendor="gemini"` with the provider settings under `asr.params`.
+
+| Parameter | Type | Required | Default | Description |
+|---|---|---|---|---|
+| `api_key` | `str` | Yes | — | Google Gemini API key |
+| `model` | `str` | Yes | — | Gemini transcription model identifier |
+| `sample_rate` | `int` | No | `None` | Audio sample rate in Hz |
+| `language` | `str` | No | `None` | Language code for speech recognition |
+| `word_timestamp` | `bool` | No | `None` | Include word-level timestamps |
+
 ### `AmazonSTT`
 
 | Parameter | Type | Required | Default | Description |
@@ -561,7 +573,7 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For
 
 | Parameter | Type | Required | Default | Description |
 |---|---|---|---|---|
-| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy |
+| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy; serialized as top-level `asr.keywords` |
 | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters |
 
 ### `SarvamSTT`
@@ -803,6 +815,63 @@ All CN TTS vendor classes support `skip_patterns` and `additional_params`.
 | `params` | `Dict[str, Any]` | No | `None` | Additional parameters |
 | `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration; overrides top-level `turn_detection` |
 
+### `OpenAIGptLive`
+
+OpenAI GPT Live is a separate vendor from `OpenAIRealtime` and serializes as `mllm.vendor="openai_gpt_live"`.
+
+It accepts the realtime connection options `api_key`, `model`, `url`, `params`, and `turn_detection`. Its greeting field is `greeting_message`.
+
+| Parameter | Type | Required | Default | Description |
+|---|---|---|---|---|
+| `api_key` | `str` | Yes | — | OpenAI API key |
+| `model` | `str` | No | `None` | GPT Live model name |
+| `voice` | `str` | No | `None` | Voice identifier |
+| `instructions` | `str` | No | `None` | System instructions |
+| `input_audio_transcription` | `Dict[str, Any]` | No | `None` | Audio transcription settings |
+| `url` | `str` | No | `wss://api.openai.com/v1/live` | OpenAI GPT Live WebSocket URL |
+| `greeting_message` | `str` | No | `None` | Greeting message |
+| `input_modalities` | `List[str]` | No | `None` | Input modalities |
+| `output_modalities` | `List[str]` | No | `None` | Output modalities |
+| `messages` | `List[Dict[str, Any]]` | No | `None` | Conversation messages |
+| `params` | `Dict[str, Any]` | No | `None` | Additional model parameters |
+| `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration |
+| `failure_message` | `str` | No | `None` | Message played when the model call fails |
+
+### Inline REST LLM tools
+
+`LlmConfig.tools` accepts a list of dictionaries for synchronous pass-through REST tools, using the same public shape as `mcp_servers`. This is the inline equivalent of `mcp_servers`, and it requires `advanced_features.enable_tools=True` to execute. Enable it explicitly with `Agent.with_tools()`.
+
+Each tool requires `function.name`, an object-shaped `function.parameters`, `server.method` (`GET` or `POST`), and `server.url`.
+
+```python
+from agora_agent import Agent, LlmConfig, OpenAI
+
+llm = OpenAI(
+    api_key='your-openai-key',
+    base_url='https://api.openai.com/v1/chat/completions',
+    model='gpt-4o-mini',
+    tools=[{
+        'type': 'function',
+        'function': {
+            'name': 'lookup_order',
+            'description': 'Look up an order by ID.',
+            'parameters': {
+                'type': 'object',
+                'properties': {'order_id': {'type': 'string'}},
+                'required': ['order_id'],
+            },
+        },
+        'server': {
+            'method': 'GET',
+            'url': 'https://api.example.com/orders/{{args.order_id}}',
+        },
+    }],
+)
+agent = Agent(client=client).with_llm(llm).with_tools()
+```
+
+`server.body` is only valid for `POST`. Template values may use `{{args.name}}` in URLs and bodies, and `{{template_variables.name}}` or `{{tool_call_id}}` in URLs, headers, and bodies. `execution.mode` currently supports only `sync`; `timeout_ms` must be between `1000` and `100000`.
+
 ### `AzureOpenAIRealtime`
 
 Global Azure OpenAI Realtime vendor (`mllm.vendor`: `"azure"`).

src/agora_agent/agentkit/__init__.py

@@ -26,6 +26,8 @@
     FillerWordsTriggerFixedTimeConfig,
     FillerWordsContent,
     FillerWordsContentStaticConfig,
+    FillerWordsContentGeneratedConfig,
+    FillerWordsContentGeneratedConfigLlmProvider,
     FillerWordsContentSelectionRule,
     TurnDetectionConfig,
     TurnDetectionNestedConfig,
@@ -59,6 +61,11 @@
     Labels,
     LlmGreetingConfigs,
     LlmGreetingConfigsMode,
+    LlmToolConfig,
+    LlmToolExecutionConfig,
+    LlmToolFunctionConfig,
+    LlmToolFunctionParametersConfig,
+    LlmToolServerConfig,
     McpServersItem,
     SessionInfo,
     SessionListResponse,
@@ -120,11 +127,14 @@
     SpeakPriorityIgnore,
     SpeakPriorityInterrupt,
     ThinkOnListeningActionIgnore,
+    ThinkOnListeningActionAppend,
     ThinkOnListeningActionInject,
     ThinkOnListeningActionInterrupt,
     ThinkOnSpeakingActionIgnore,
+    ThinkOnSpeakingActionAppend,
     ThinkOnSpeakingActionInterrupt,
     ThinkOnThinkingActionIgnore,
+    ThinkOnThinkingActionAppend,
     ThinkOnThinkingActionInterrupt,
     TurnDetectionTypeValues,
 )
@@ -173,8 +183,10 @@
     FishAudioTTS,
     Gemini,
     GeminiLive,
+    OpenAIGptLive,
     GenericAvatar,
     GoogleSTT,
+    GeminiSTT,
     GoogleTTS,
     GradiumTTS,
     Groq,
@@ -253,6 +265,8 @@
     "FillerWordsTriggerFixedTimeConfig",
     "FillerWordsContent",
     "FillerWordsContentStaticConfig",
+    "FillerWordsContentGeneratedConfig",
+    "FillerWordsContentGeneratedConfigLlmProvider",
     "FillerWordsContentSelectionRule",
     # Turn detection types
     "TurnDetectionConfig",
@@ -316,16 +330,24 @@
     "ThinkOnListeningActionInject",
     "ThinkOnListeningActionInterrupt",
     "ThinkOnListeningActionIgnore",
+    "ThinkOnListeningActionAppend",
     "ThinkOnThinkingActionInterrupt",
     "ThinkOnThinkingActionIgnore",
+    "ThinkOnThinkingActionAppend",
     "ThinkOnSpeakingActionInterrupt",
     "ThinkOnSpeakingActionIgnore",
+    "ThinkOnSpeakingActionAppend",
     "CNAgent",
     "GlobalAgent",
     "RegionalAgent",
     # LLM sub-types
     "LlmGreetingConfigs",
     "LlmGreetingConfigsMode",
+    "LlmToolConfig",
+    "LlmToolExecutionConfig",
+    "LlmToolFunctionConfig",
+    "LlmToolFunctionParametersConfig",
+    "LlmToolServerConfig",
     "McpServersItem",
     "AgentSession",
     "AsyncAgentSession",
@@ -421,6 +443,7 @@
     "MicrosoftCNSTT",
     "OpenAISTT",
     "GoogleSTT",
+    "GeminiSTT",
     "AmazonSTT",
     "AssemblyAISTT",
     "AresSTT",
@@ -434,6 +457,7 @@
     "OpenAIRealtime",
     "AzureOpenAIRealtime",
     "GeminiLive",
+    "OpenAIGptLive",
     "QwenOmni",
     "VertexAI",
     "XaiGrok",