diff --git a/.fern/metadata.json b/.fern/metadata.json index 3e6a357..3a73355 100644 --- a/.fern/metadata.json +++ b/.fern/metadata.json @@ -14,5 +14,5 @@ }, "exclude_types_from_init_exports": true }, - "originGitCommit": "f4b5b1d56fb73c4e26c11486424723c12e6376bc" + "originGitCommit": "9afc7ddf3c7c24bf5c80ae7cd4eb83c257d7ceff" } \ No newline at end of file diff --git a/.fern/replay.lock b/.fern/replay.lock index cea052a..83609b1 100644 --- a/.fern/replay.lock +++ b/.fern/replay.lock @@ -48,14 +48,20 @@ generations: cli_version: unknown generator_versions: fernapi/fern-python-sdk: 4.37.0 -current_generation: c210d15d50ce28b5edf1a02b33fe0f2855b69b8f + - commit_sha: e7ec17df8622ff081a4339a44fa3a4a8e2e6f21e + tree_hash: 300680ed735b4afab3b2fea7600c72b9d4968558 + timestamp: 2026-08-20T08:02:01.021Z + cli_version: unknown + generator_versions: + fernapi/fern-python-sdk: 4.37.0 +current_generation: e7ec17df8622ff081a4339a44fa3a4a8e2e6f21e patches: - id: patch-13d8e068 content_hash: sha256:97d879ab169016b9abaf1866427a8ebc45af608f69e063a07e5126c9613c329b original_commit: 13d8e0683ffa08bcfc9a381eb9604adf5abdbe0b original_message: "chore: ignore local venv and planning docs" original_author: plutoless - base_generation: c210d15d50ce28b5edf1a02b33fe0f2855b69b8f + base_generation: e7ec17df8622ff081a4339a44fa3a4a8e2e6f21e files: - .gitignore patch_content: | diff --git a/docs/concepts/vendors.md b/docs/concepts/vendors.md index 34e2970..16fb81a 100644 --- a/docs/concepts/vendors.md +++ b/docs/concepts/vendors.md @@ -110,7 +110,7 @@ Used with `agent.with_stt()`. Use `turn_detection.language` for Agora interaction language; it defaults to `en-US`. STT vendor `language` options are serialized under `asr.params` using each provider's own format. If `with_stt()` is omitted, AgentKit defaults to `AresSTT` for global clients and `FengmingSTT` for `Area.CN` clients. Ares does not take a provider language option; AgentKit uses `turn_detection.language` for REST `asr.language`. -`AresSTT` and `FengmingSTT` accept an optional `keywords: List[str]` hotword list. +`AresSTT` and `FengmingSTT` accept an optional `keywords: List[str]` hotword list. For Ares, this is serialized as top-level `asr.keywords`, not inside `asr.params`. | Class | Provider | Required Parameters | |---|---|---| @@ -119,6 +119,7 @@ Use `turn_detection.language` for Agora interaction language; it defaults to `en | `MicrosoftSTT` | Microsoft Azure | `key`, `region`, `language` | | `OpenAISTT` | OpenAI | `api_key` | | `GoogleSTT` | Google Cloud | `project_id`, `location`, `adc_credentials_string`, `language` | +| `GeminiSTT` | Google Gemini | `api_key`, `model`; optional `sample_rate`, `language`, `word_timestamp` | | `AmazonSTT` | Amazon Transcribe | `access_key`, `secret_key`, `region`, `language` | | `AssemblyAISTT` | AssemblyAI | `api_key`, `language` | | `AresSTT` | Ares | — (all optional) | @@ -152,6 +153,7 @@ Used with `agent.with_mllm()` for the [MLLM flow](../guides/mllm-flow.md). These | Class | Provider | Area | Required Parameters | |---|---|---|---| | `OpenAIRealtime` | OpenAI Realtime | Global | `api_key`; optional `turn_detection` | +| `OpenAIGptLive` | OpenAI GPT Live (`mllm.vendor`: `openai_gpt_live`) | Global | `api_key`; optional `model`, `greeting_message`, `turn_detection` | | `AzureOpenAIRealtime` | Azure OpenAI Realtime | Global | `api_key`, `url`, `turn_detection`; optional `max_history` | | `GeminiLive` | Google Gemini Live API | Global | `api_key`, `model`; optional `turn_detection` | | `VertexAI` | Vertex AI (Gemini Live) | Global | `model`, `project_id`, `location`, `adc_credentials_string`; optional `turn_detection` | diff --git a/docs/guides/agent-builder-features.md b/docs/guides/agent-builder-features.md index cb910e0..2d99520 100644 --- a/docs/guides/agent-builder-features.md +++ b/docs/guides/agent-builder-features.md @@ -26,7 +26,7 @@ Pass `client=client` to every `Agent(...)` builder. `create_session()` and `crea |---|---|---| | `sal` | `with_sal(config)` | Selective Attention Locking — speaker recognition and noise suppression | | `advanced_features` | `with_advanced_features(features)` | Enable MLLM, RTM, SAL, tools | -| `tools` | `with_tools(enabled=True)` | Enable MCP tool invocation | +| `tools` | `with_tools(enabled=True)` | Enable MCP and inline LLM tool invocation | | `parameters` | `with_parameters(params)` | Silence config, farewell config, data channel | | `failure_message` | LLM/MLLM vendor option | Message spoken when LLM fails | | `max_history` | LLM vendor option | Max conversation turns in LLM context | @@ -217,6 +217,8 @@ from agora_agent import ( FillerWordsTriggerFixedTimeConfig, FillerWordsContent, FillerWordsContentStaticConfig, + FillerWordsContentGeneratedConfig, + FillerWordsContentGeneratedConfigLlmProvider, FillerWordsSelectionRule, ) @@ -242,6 +244,37 @@ agent = ( ) ``` +Generated filler words use an OpenAI-compatible LLM and fall back to the static phrases when generation is unavailable. `static_config.phrases` is required and must be non-empty whenever filler words are enabled. + +```python +from agora_agent import ( + FillerWordsConfig, + FillerWordsContent, + FillerWordsContentStaticConfig, + FillerWordsContentGeneratedConfig, + FillerWordsContentGeneratedConfigLlmProvider, +) + +generated = FillerWordsConfig( + enable=True, + content=FillerWordsContent( + mode='generated', + static_config=FillerWordsContentStaticConfig( + phrases=['Let me think...', 'One moment...'], + ), + generated_config=FillerWordsContentGeneratedConfig( + llm_provider=FillerWordsContentGeneratedConfigLlmProvider( + base_url='https://api.openai.com/v1/chat/completions', + api_key='your-filler-llm-key', + params={'model': 'gpt-4o-mini'}, + ), + prompt='Generate a short conversational filler phrase; do not answer the user.', + ), + ), +) +agent = Agent(client=client).with_filler_words(generated) +``` + ## Properties (Getters) Read back configuration via properties: diff --git a/docs/guides/regional-routing.md b/docs/guides/regional-routing.md index a6051bd..4354bef 100644 --- a/docs/guides/regional-routing.md +++ b/docs/guides/regional-routing.md @@ -40,7 +40,7 @@ If you omit `with_stt()`, AgentKit uses `FengmingSTT` by default for `Area.CN` c | Client area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `GeminiSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `OpenAIGptLive`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global client example: diff --git a/docs/reference/agent.md b/docs/reference/agent.md index f040652..048f408 100644 --- a/docs/reference/agent.md +++ b/docs/reference/agent.md @@ -53,7 +53,7 @@ Agent( | `sal` | `Optional[SalConfig]` | `None` | Speech Activity Level configuration | | `advanced_features` | `Optional[Dict[str, Any]]` | `None` | Advanced features dict (e.g., `{'enable_rtm': True}`) | | `parameters` | `Optional[SessionParams]` | `None` | Additional session parameters | -| `greeting` | `Optional[str]` | `None` | Deprecated. Use LLM/MLLM vendor `greeting_message` instead. | +| `greeting` | `Optional[str]` | `None` | Deprecated. Use the LLM/MLLM vendor `greeting_message` instead. | | `failure_message` | `Optional[str]` | `None` | Deprecated. Use LLM/MLLM vendor `failure_message` instead. | | `max_history` | `Optional[int]` | `None` | Deprecated. Use LLM vendor `max_history` instead. | | `geofence` | `Optional[GeofenceConfig]` | `None` | Regional access restriction | @@ -173,7 +173,7 @@ When `enable_rtm=True`, AgentKit defaults `parameters.data_channel` to `"rtm"` u ### `with_tools(enabled: bool = True) -> Agent` -Enable or disable MCP tool invocation by setting `advanced_features.enable_tools`. +Enable or disable MCP and inline LLM tool invocation by setting `advanced_features.enable_tools`. LLM `tools` definitions require this feature to be enabled, just like `mcp_servers`; call `.with_tools()` explicitly when using either configuration. ### `with_parameters(parameters: SessionParams) -> Agent` @@ -334,6 +334,6 @@ to_properties( ## Type aliases -Public aliases over Fern-generated types: `LlmConfig`, `SttConfig`, `AsrConfig` (= `SttConfig`), `MllmConfig`, `AvatarConfig`, session/conversation types, and think types (`ThinkOnListeningAction`, etc.). +Public aliases over Fern-generated types: `LlmConfig`, `SttConfig`, `AsrConfig` (= `SttConfig`), `MllmConfig`, `AvatarConfig`, session/conversation types, think types (`ThinkOnListeningAction`, etc.), generated filler-word types, and inline REST tool types (`LlmToolConfig`, `LlmToolServerConfig`, and related aliases). -Think value constants: `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, `ThinkOnThinkingActionInterrupt`, `ThinkOnThinkingActionIgnore`, `ThinkOnSpeakingActionInterrupt`, `ThinkOnSpeakingActionIgnore`. +Think value constants: `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, `ThinkOnListeningActionAppend`, `ThinkOnThinkingActionInterrupt`, `ThinkOnThinkingActionIgnore`, `ThinkOnThinkingActionAppend`, `ThinkOnSpeakingActionInterrupt`, `ThinkOnSpeakingActionIgnore`, `ThinkOnSpeakingActionAppend`. diff --git a/docs/reference/session.md b/docs/reference/session.md index 8ff50e4..afbfb88 100644 --- a/docs/reference/session.md +++ b/docs/reference/session.md @@ -183,7 +183,7 @@ await session.update(properties) Inject a custom text instruction into the running agent. -In API v2.7, omitting `on_listening_action` uses the server default `interrupt`. Pass `on_listening_action='inject'` explicitly to preserve the pre-v2.7 behavior. +In API v2.7, omitting `on_listening_action` uses the server default `interrupt`. Supported listening, thinking, and speaking actions include `interrupt`, `ignore`, and `append` where applicable. `append` queues the instruction until the current user/LLM/TTS turn completes without interrupting it. Pass `on_listening_action='inject'` explicitly to preserve the pre-v2.7 behavior. ```python session.think('Summarize the last answer', on_listening_action='inject') diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index 073c0b6..0029292 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -20,7 +20,7 @@ Construct vendors directly from `agora_agent`, then bind a client with `Agent(cl | Area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `GeminiSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `OpenAIGptLive`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global example: @@ -538,6 +538,18 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For | `model` | `str` | No | `None` | Recognition model | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters | +### `GeminiSTT` + +Google Gemini transcription vendor. The generated request is `asr.vendor="gemini"` with the provider settings under `asr.params`. + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Google Gemini API key | +| `model` | `str` | Yes | — | Gemini transcription model identifier | +| `sample_rate` | `int` | No | `None` | Audio sample rate in Hz | +| `language` | `str` | No | `None` | Language code for speech recognition | +| `word_timestamp` | `bool` | No | `None` | Include word-level timestamps | + ### `AmazonSTT` | Parameter | Type | Required | Default | Description | @@ -561,7 +573,7 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For | Parameter | Type | Required | Default | Description | |---|---|---|---|---| -| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy | +| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy; serialized as top-level `asr.keywords` | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters | ### `SarvamSTT` @@ -803,6 +815,63 @@ All CN TTS vendor classes support `skip_patterns` and `additional_params`. | `params` | `Dict[str, Any]` | No | `None` | Additional parameters | | `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration; overrides top-level `turn_detection` | +### `OpenAIGptLive` + +OpenAI GPT Live is a separate vendor from `OpenAIRealtime` and serializes as `mllm.vendor="openai_gpt_live"`. + +It accepts the realtime connection options `api_key`, `model`, `url`, `params`, and `turn_detection`. Its greeting field is `greeting_message`. + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | OpenAI API key | +| `model` | `str` | No | `None` | GPT Live model name | +| `voice` | `str` | No | `None` | Voice identifier | +| `instructions` | `str` | No | `None` | System instructions | +| `input_audio_transcription` | `Dict[str, Any]` | No | `None` | Audio transcription settings | +| `url` | `str` | No | `wss://api.openai.com/v1/live` | OpenAI GPT Live WebSocket URL | +| `greeting_message` | `str` | No | `None` | Greeting message | +| `input_modalities` | `List[str]` | No | `None` | Input modalities | +| `output_modalities` | `List[str]` | No | `None` | Output modalities | +| `messages` | `List[Dict[str, Any]]` | No | `None` | Conversation messages | +| `params` | `Dict[str, Any]` | No | `None` | Additional model parameters | +| `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration | +| `failure_message` | `str` | No | `None` | Message played when the model call fails | + +### Inline REST LLM tools + +`LlmConfig.tools` accepts a list of dictionaries for synchronous pass-through REST tools, using the same public shape as `mcp_servers`. This is the inline equivalent of `mcp_servers`, and it requires `advanced_features.enable_tools=True` to execute. Enable it explicitly with `Agent.with_tools()`. + +Each tool requires `function.name`, an object-shaped `function.parameters`, `server.method` (`GET` or `POST`), and `server.url`. + +```python +from agora_agent import Agent, LlmConfig, OpenAI + +llm = OpenAI( + api_key='your-openai-key', + base_url='https://api.openai.com/v1/chat/completions', + model='gpt-4o-mini', + tools=[{ + 'type': 'function', + 'function': { + 'name': 'lookup_order', + 'description': 'Look up an order by ID.', + 'parameters': { + 'type': 'object', + 'properties': {'order_id': {'type': 'string'}}, + 'required': ['order_id'], + }, + }, + 'server': { + 'method': 'GET', + 'url': 'https://api.example.com/orders/{{args.order_id}}', + }, + }], +) +agent = Agent(client=client).with_llm(llm).with_tools() +``` + +`server.body` is only valid for `POST`. Template values may use `{{args.name}}` in URLs and bodies, and `{{template_variables.name}}` or `{{tool_call_id}}` in URLs, headers, and bodies. `execution.mode` currently supports only `sync`; `timeout_ms` must be between `1000` and `100000`. + ### `AzureOpenAIRealtime` Global Azure OpenAI Realtime vendor (`mllm.vendor`: `"azure"`). diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index 7afe863..7d81254 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -26,6 +26,8 @@ FillerWordsTriggerFixedTimeConfig, FillerWordsContent, FillerWordsContentStaticConfig, + FillerWordsContentGeneratedConfig, + FillerWordsContentGeneratedConfigLlmProvider, FillerWordsContentSelectionRule, TurnDetectionConfig, TurnDetectionNestedConfig, @@ -59,6 +61,11 @@ Labels, LlmGreetingConfigs, LlmGreetingConfigsMode, + LlmToolConfig, + LlmToolExecutionConfig, + LlmToolFunctionConfig, + LlmToolFunctionParametersConfig, + LlmToolServerConfig, McpServersItem, SessionInfo, SessionListResponse, @@ -120,11 +127,14 @@ SpeakPriorityIgnore, SpeakPriorityInterrupt, ThinkOnListeningActionIgnore, + ThinkOnListeningActionAppend, ThinkOnListeningActionInject, ThinkOnListeningActionInterrupt, ThinkOnSpeakingActionIgnore, + ThinkOnSpeakingActionAppend, ThinkOnSpeakingActionInterrupt, ThinkOnThinkingActionIgnore, + ThinkOnThinkingActionAppend, ThinkOnThinkingActionInterrupt, TurnDetectionTypeValues, ) @@ -173,8 +183,10 @@ FishAudioTTS, Gemini, GeminiLive, + OpenAIGptLive, GenericAvatar, GoogleSTT, + GeminiSTT, GoogleTTS, GradiumTTS, Groq, @@ -253,6 +265,8 @@ "FillerWordsTriggerFixedTimeConfig", "FillerWordsContent", "FillerWordsContentStaticConfig", + "FillerWordsContentGeneratedConfig", + "FillerWordsContentGeneratedConfigLlmProvider", "FillerWordsContentSelectionRule", # Turn detection types "TurnDetectionConfig", @@ -316,16 +330,24 @@ "ThinkOnListeningActionInject", "ThinkOnListeningActionInterrupt", "ThinkOnListeningActionIgnore", + "ThinkOnListeningActionAppend", "ThinkOnThinkingActionInterrupt", "ThinkOnThinkingActionIgnore", + "ThinkOnThinkingActionAppend", "ThinkOnSpeakingActionInterrupt", "ThinkOnSpeakingActionIgnore", + "ThinkOnSpeakingActionAppend", "CNAgent", "GlobalAgent", "RegionalAgent", # LLM sub-types "LlmGreetingConfigs", "LlmGreetingConfigsMode", + "LlmToolConfig", + "LlmToolExecutionConfig", + "LlmToolFunctionConfig", + "LlmToolFunctionParametersConfig", + "LlmToolServerConfig", "McpServersItem", "AgentSession", "AsyncAgentSession", @@ -421,6 +443,7 @@ "MicrosoftCNSTT", "OpenAISTT", "GoogleSTT", + "GeminiSTT", "AmazonSTT", "AssemblyAISTT", "AresSTT", @@ -434,6 +457,7 @@ "OpenAIRealtime", "AzureOpenAIRealtime", "GeminiLive", + "OpenAIGptLive", "QwenOmni", "VertexAI", "XaiGrok", diff --git a/src/agora_agent/agentkit/agent.py b/src/agora_agent/agentkit/agent.py index e0080fd..6f7e170 100644 --- a/src/agora_agent/agentkit/agent.py +++ b/src/agora_agent/agentkit/agent.py @@ -57,6 +57,8 @@ from ..agents.types.start_agents_request_properties_filler_words_trigger_fixed_time_config import StartAgentsRequestPropertiesFillerWordsTriggerFixedTimeConfig from ..agents.types.start_agents_request_properties_filler_words_content import StartAgentsRequestPropertiesFillerWordsContent from ..agents.types.start_agents_request_properties_filler_words_content_static_config import StartAgentsRequestPropertiesFillerWordsContentStaticConfig +from ..agents.types.start_agents_request_properties_filler_words_content_generated_config import StartAgentsRequestPropertiesFillerWordsContentGeneratedConfig +from ..agents.types.start_agents_request_properties_filler_words_content_generated_config_llm_provider import StartAgentsRequestPropertiesFillerWordsContentGeneratedConfigLlmProvider from ..agents.types.start_agents_request_properties_filler_words_content_static_config_selection_rule import StartAgentsRequestPropertiesFillerWordsContentStaticConfigSelectionRule from ..types.tts import Tts from ..types.asr import Asr @@ -66,6 +68,11 @@ from ..types.mllm_turn_detection import MllmTurnDetection from ..types.mllm_turn_detection_mode import MllmTurnDetectionMode as GeneratedMllmTurnDetectionMode from ..types.mllm_vendor import MllmVendor as GeneratedMllmVendor +from ..types.llm_tool import LlmTool +from ..types.llm_tool_execution import LlmToolExecution +from ..types.llm_tool_function import LlmToolFunction +from ..types.llm_tool_function_parameters import LlmToolFunctionParameters +from ..types.llm_tool_server import LlmToolServer from ..agent_management.types.agent_think_agent_management_request_on_listening_action import ( AgentThinkAgentManagementRequestOnListeningAction, ) @@ -214,7 +221,14 @@ def _start_properties_from_mapping( FillerWordsTriggerFixedTimeConfig = StartAgentsRequestPropertiesFillerWordsTriggerFixedTimeConfig FillerWordsContent = StartAgentsRequestPropertiesFillerWordsContent FillerWordsContentStaticConfig = StartAgentsRequestPropertiesFillerWordsContentStaticConfig +FillerWordsContentGeneratedConfig = StartAgentsRequestPropertiesFillerWordsContentGeneratedConfig +FillerWordsContentGeneratedConfigLlmProvider = StartAgentsRequestPropertiesFillerWordsContentGeneratedConfigLlmProvider FillerWordsContentSelectionRule = StartAgentsRequestPropertiesFillerWordsContentStaticConfigSelectionRule +LlmToolConfig = LlmTool +LlmToolExecutionConfig = LlmToolExecution +LlmToolFunctionConfig = LlmToolFunction +LlmToolFunctionParametersConfig = LlmToolFunctionParameters +LlmToolServerConfig = LlmToolServer # Think type aliases and response ThinkOnListeningAction = AgentThinkAgentManagementRequestOnListeningAction diff --git a/src/agora_agent/agentkit/constants.py b/src/agora_agent/agentkit/constants.py index e8aa311..93dc3ce 100644 --- a/src/agora_agent/agentkit/constants.py +++ b/src/agora_agent/agentkit/constants.py @@ -108,7 +108,10 @@ class MllmTurnDetectionMode: ThinkOnListeningActionInject = "inject" ThinkOnListeningActionInterrupt = "interrupt" ThinkOnListeningActionIgnore = "ignore" +ThinkOnListeningActionAppend = "append" ThinkOnThinkingActionInterrupt = "interrupt" ThinkOnThinkingActionIgnore = "ignore" +ThinkOnThinkingActionAppend = "append" ThinkOnSpeakingActionInterrupt = "interrupt" ThinkOnSpeakingActionIgnore = "ignore" +ThinkOnSpeakingActionAppend = "append" diff --git a/src/agora_agent/agentkit/vendors/__init__.py b/src/agora_agent/agentkit/vendors/__init__.py index c69669d..1d6f89e 100644 --- a/src/agora_agent/agentkit/vendors/__init__.py +++ b/src/agora_agent/agentkit/vendors/__init__.py @@ -13,7 +13,7 @@ ) from .avatar import AkoolAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar, LiveAvatarAvatar from .llm import AmazonBedrock, Anthropic, AzureOpenAI, CustomLLM, Dify, Gemini, Groq, OpenAI, VertexAILLM -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGptLive, OpenAIRealtime, VertexAI, XaiGrok from .cn import ( AliyunLLM, BytedanceDuplexTTS, @@ -42,6 +42,7 @@ AssemblyAISTT, DeepgramSTT, GoogleSTT, + GeminiSTT, MicrosoftSTT, OpenAISTT, SarvamSTT, @@ -121,6 +122,7 @@ "MicrosoftCNSTT", "OpenAISTT", "GoogleSTT", + "GeminiSTT", "AmazonSTT", "AssemblyAISTT", "AresSTT", @@ -136,6 +138,7 @@ "DeepSeekLLM", "TencentLLM", "OpenAIRealtime", + "OpenAIGptLive", "AzureOpenAIRealtime", "GeminiLive", "QwenOmni", diff --git a/src/agora_agent/agentkit/vendors/catalog.py b/src/agora_agent/agentkit/vendors/catalog.py index 25c99df..a8e9b44 100644 --- a/src/agora_agent/agentkit/vendors/catalog.py +++ b/src/agora_agent/agentkit/vendors/catalog.py @@ -16,13 +16,14 @@ OpenAI, VertexAILLM, ) -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGptLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, AssemblyAISTT, DeepgramSTT, GoogleSTT, + GeminiSTT, MicrosoftSTT, OpenAISTT, SarvamSTT, @@ -75,6 +76,7 @@ def __init__( "microsoft": MicrosoftSTT, "openai": OpenAISTT, "google": GoogleSTT, + "gemini": GeminiSTT, "amazon": AmazonSTT, "assemblyai": AssemblyAISTT, "speechmatics": SpeechmaticsSTT, @@ -94,6 +96,7 @@ def __init__( }, mllm={ "openai": OpenAIRealtime, + "openai_gpt_live": OpenAIGptLive, "azure": AzureOpenAIRealtime, "gemini": GeminiLive, "vertexai": VertexAI, diff --git a/src/agora_agent/agentkit/vendors/cn.py b/src/agora_agent/agentkit/vendors/cn.py index 0d1c9c7..a229f7a 100644 --- a/src/agora_agent/agentkit/vendors/cn.py +++ b/src/agora_agent/agentkit/vendors/cn.py @@ -512,6 +512,7 @@ class AliyunLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="aliyun") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -569,6 +570,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -596,6 +599,7 @@ class BytedanceLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="bytedance") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -653,6 +657,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -680,6 +686,7 @@ class DeepSeekLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="deepseek") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -737,6 +744,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -764,6 +773,7 @@ class TencentLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="tencent") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -821,6 +831,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history diff --git a/src/agora_agent/agentkit/vendors/llm.py b/src/agora_agent/agentkit/vendors/llm.py index 1ba29f6..24109e3 100644 --- a/src/agora_agent/agentkit/vendors/llm.py +++ b/src/agora_agent/agentkit/vendors/llm.py @@ -47,6 +47,7 @@ class OpenAIOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -107,6 +108,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -136,6 +139,7 @@ class AzureOpenAIOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -183,6 +187,8 @@ def to_config(self) -> Dict[str, Any]: config["template_variables"] = self.template_variables if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -210,6 +216,7 @@ class AnthropicOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -251,6 +258,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -279,6 +288,7 @@ class GeminiOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -325,6 +335,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -352,6 +364,7 @@ class GroqOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -401,6 +414,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -428,6 +443,7 @@ class CustomLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -477,6 +493,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -508,6 +526,7 @@ class VertexAILLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -557,6 +576,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -587,6 +608,7 @@ class AmazonBedrockOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -630,6 +652,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history return config @@ -655,6 +679,7 @@ class DifyOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0) @@ -693,6 +718,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history return config diff --git a/src/agora_agent/agentkit/vendors/mllm.py b/src/agora_agent/agentkit/vendors/mllm.py index bb207b7..0c43361 100644 --- a/src/agora_agent/agentkit/vendors/mllm.py +++ b/src/agora_agent/agentkit/vendors/mllm.py @@ -71,6 +71,63 @@ def to_config(self) -> Dict[str, Any]: return config +class OpenAIGptLiveOptions(BaseModel): + """Options for the distinct OpenAI GPT Live MLLM vendor.""" + + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="OpenAI API key") + model: Optional[str] = Field(default=None, description="Model name") + voice: Optional[str] = Field(default=None, description="Voice identifier") + instructions: Optional[str] = Field(default=None, description="System instructions") + input_audio_transcription: Optional[Dict[str, Any]] = Field(default=None, description="Audio transcription settings") + url: str = Field(default="wss://api.openai.com/v1/live", description="OpenAI GPT Live WebSocket URL") + greeting_message: Optional[str] = Field(default=None, description="Greeting message for the OpenAI GPT Live session") + input_modalities: Optional[List[str]] = Field(default=None, description="Input modalities") + output_modalities: Optional[List[str]] = Field(default=None, description="Output modalities") + messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages") + params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters") + turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration") + failure_message: Optional[str] = Field(default=None, description="Message played on failure") + + +class OpenAIGptLive(OpenAIGptLiveOptions, BaseMLLM): + def to_config(self) -> Dict[str, Any]: + config: Dict[str, Any] = {"vendor": "openai_gpt_live", "api_key": self.api_key, "url": self.url} + if ( + self.model is not None + or self.params is not None + or self.voice is not None + or self.instructions is not None + or self.input_audio_transcription is not None + ): + inner_params: Dict[str, Any] = {} + if self.model is not None: + inner_params["model"] = self.model + if self.params is not None: + inner_params.update(self.params) + if self.voice is not None: + inner_params["voice"] = self.voice + if self.instructions is not None: + inner_params["instructions"] = self.instructions + if self.input_audio_transcription is not None: + inner_params["input_audio_transcription"] = self.input_audio_transcription + config["params"] = inner_params + if self.greeting_message is not None: + config["greeting_message"] = self.greeting_message + if self.input_modalities is not None: + config["input_modalities"] = self.input_modalities + if self.output_modalities is not None: + config["output_modalities"] = self.output_modalities + if self.messages is not None: + config["messages"] = self.messages + if self.failure_message is not None: + config["failure_message"] = self.failure_message + if self.turn_detection is not None: + config["turn_detection"] = self.turn_detection + return config + + class AzureOpenAIRealtimeOptions(BaseModel): """Azure OpenAI Realtime MLLM vendor (`mllm.vendor`: ``azure``).""" diff --git a/src/agora_agent/agentkit/vendors/namespaces.py b/src/agora_agent/agentkit/vendors/namespaces.py index 001b374..71e9a15 100644 --- a/src/agora_agent/agentkit/vendors/namespaces.py +++ b/src/agora_agent/agentkit/vendors/namespaces.py @@ -13,13 +13,14 @@ OpenAI, VertexAILLM, ) -from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIGptLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, AssemblyAISTT, DeepgramSTT, GoogleSTT, + GeminiSTT, MicrosoftSTT, OpenAISTT, SarvamSTT, @@ -54,6 +55,7 @@ class GlobalSTTVendors: microsoft = MicrosoftSTT openai = OpenAISTT google = GoogleSTT + gemini = GeminiSTT amazon = AmazonSTT assemblyai = AssemblyAISTT speechmatics = SpeechmaticsSTT @@ -91,6 +93,7 @@ class CNLLMVendors: class GlobalMLLMVendors: openai = OpenAIRealtime + openai_gpt_live = OpenAIGptLive azure = AzureOpenAIRealtime gemini = GeminiLive vertexai = VertexAI diff --git a/src/agora_agent/agentkit/vendors/region.py b/src/agora_agent/agentkit/vendors/region.py index 721f7e3..6f64891 100644 --- a/src/agora_agent/agentkit/vendors/region.py +++ b/src/agora_agent/agentkit/vendors/region.py @@ -23,6 +23,7 @@ "microsoft", "openai", "google", + "gemini", "amazon", "assemblyai", "speechmatics", @@ -64,6 +65,7 @@ CN_MLLM_VENDORS: typing.Tuple[str, ...] = ("qwen_omni",) GLOBAL_MLLM_VENDORS: typing.Tuple[str, ...] = ( "openai", + "openai_gpt_live", "azure", "gemini", "vertexai", diff --git a/src/agora_agent/agentkit/vendors/stt.py b/src/agora_agent/agentkit/vendors/stt.py index 2dfdd56..6c72d27 100644 --- a/src/agora_agent/agentkit/vendors/stt.py +++ b/src/agora_agent/agentkit/vendors/stt.py @@ -173,6 +173,28 @@ def to_config(self) -> Dict[str, Any]: return config +class GeminiSTTOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="Google Gemini API key") + model: str = Field(..., description="Gemini transcription model identifier") + sample_rate: Optional[int] = Field(default=None, description="Audio sample rate in Hz") + language: Optional[str] = Field(default=None, description="Language code for speech recognition") + word_timestamp: Optional[bool] = Field(default=None, description="Include word-level timestamps") + + +class GeminiSTT(GeminiSTTOptions, BaseSTT): + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = {"api_key": self.api_key, "model": self.model} + if self.sample_rate is not None: + params["sample_rate"] = self.sample_rate + if self.language is not None: + params["language"] = self.language + if self.word_timestamp is not None: + params["word_timestamp"] = self.word_timestamp + return {"vendor": "gemini", "params": params} + + class AmazonSTTOptions(BaseModel): model_config = ConfigDict(extra="forbid") @@ -236,10 +258,9 @@ class AresSTTOptions(BaseModel): class AresSTT(AresSTTOptions, BaseSTT): def to_config(self) -> Dict[str, Any]: params: Dict[str, Any] = dict(self.additional_params or {}) - if self.keywords is not None: - params["keywords"] = self.keywords - config: Dict[str, Any] = {"vendor": "ares"} + if self.keywords is not None: + config["keywords"] = self.keywords if params: config["params"] = params return config diff --git a/src/agora_agent/types/asr.py b/src/agora_agent/types/asr.py index 8e02cde..2f62cc6 100644 --- a/src/agora_agent/types/asr.py +++ b/src/agora_agent/types/asr.py @@ -14,6 +14,7 @@ from .assembly_ai_asr_params import AssemblyAiAsrParams from .deepgram_asr_params import DeepgramAsrParams from .fengming_asr_params import FengmingAsrParams +from .gemini_asr_params import GeminiAsrParams from .google_asr_params import GoogleAsrParams from .microsoft_asr_params import MicrosoftAsrParams from .open_ai_asr_params import OpenAiAsrParams @@ -133,6 +134,21 @@ class Config: extra = pydantic.Extra.allow +class Asr_Gemini(UncheckedBaseModel): + vendor: typing.Literal["gemini"] = "gemini" + language: typing.Optional[AsrLanguage] = None + params: GeminiAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + class Asr_Amazon(UncheckedBaseModel): vendor: typing.Literal["amazon"] = "amazon" language: typing.Optional[AsrLanguage] = None @@ -262,6 +278,7 @@ class Config: Asr_Deepgram, Asr_Openai, Asr_Google, + Asr_Gemini, Asr_Amazon, Asr_Assemblyai, Asr_Speechmatics, diff --git a/src/agora_agent/types/gemini_asr.py b/src/agora_agent/types/gemini_asr.py new file mode 100644 index 0000000..dc0d1a8 --- /dev/null +++ b/src/agora_agent/types/gemini_asr.py @@ -0,0 +1,27 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .asr_language import AsrLanguage +from .gemini_asr_params import GeminiAsrParams + + +class GeminiAsr(UncheckedBaseModel): + """ + Google Gemini ASR configuration. + """ + + language: typing.Optional[AsrLanguage] = None + params: GeminiAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/gemini_asr_params.py b/src/agora_agent/types/gemini_asr_params.py new file mode 100644 index 0000000..aee08f7 --- /dev/null +++ b/src/agora_agent/types/gemini_asr_params.py @@ -0,0 +1,47 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class GeminiAsrParams(UncheckedBaseModel): + """ + Google Gemini ASR configuration parameters. + """ + + api_key: str = pydantic.Field() + """ + Google Gemini API key. + """ + + model: str = pydantic.Field() + """ + Gemini transcription model identifier. + """ + + sample_rate: typing.Optional[int] = pydantic.Field(default=None) + """ + Audio sample rate in Hz. + """ + + language: typing.Optional[str] = pydantic.Field(default=None) + """ + Language code for speech recognition. + """ + + word_timestamp: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to include word-level timestamps in transcription results. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/mllm.py b/src/agora_agent/types/mllm.py index acc7171..5569048 100644 --- a/src/agora_agent/types/mllm.py +++ b/src/agora_agent/types/mllm.py @@ -71,6 +71,11 @@ class Mllm(UncheckedBaseModel): Agent greeting message. """ + greeting: typing.Optional[str] = pydantic.Field(default=None) + """ + Greeting message for the MLLM session. + """ + failure_message: typing.Optional[str] = pydantic.Field(default=None) """ Agent failure message. diff --git a/src/agora_agent/types/mllm_vendor.py b/src/agora_agent/types/mllm_vendor.py index 6da87bb..16e7528 100644 --- a/src/agora_agent/types/mllm_vendor.py +++ b/src/agora_agent/types/mllm_vendor.py @@ -2,4 +2,6 @@ import typing -MllmVendor = typing.Union[typing.Literal["openai", "azure", "gemini", "vertexai", "xai", "qwen_omni"], typing.Any] +MllmVendor = typing.Union[ + typing.Literal["openai", "azure", "gemini", "vertexai", "xai", "qwen_omni", "openai_gpt_live"], typing.Any +] diff --git a/tests/custom/test_agentkit_session.py b/tests/custom/test_agentkit_session.py index d946f75..ea88b5d 100644 --- a/tests/custom/test_agentkit_session.py +++ b/tests/custom/test_agentkit_session.py @@ -8,6 +8,7 @@ GenericAvatar, LiveAvatarAvatar, OpenAI, + OpenAIGptLive, OpenAIRealtime, ) from agora_agent.agents.types.get_turns_agents_response import GetTurnsAgentsResponse @@ -188,6 +189,24 @@ def test_session_start_properties_applies_mllm_agent_level_defaults(): assert "max_history" not in properties["mllm"] +def test_session_start_properties_uses_greeting_message_for_openai_gpt_live(): + agent = ( + Agent(test_client()) + .with_mllm(OpenAIGptLive(api_key="gpt-live-key")) + .with_greeting("agent greeting") + ) + session = _session(agent) + + properties = session._build_start_properties( # noqa: SLF001 + {"app_id": APP_ID, "app_certificate": APP_CERTIFICATE}, + skip_vendor_validation_categories=set(), + allow_missing_vendor_categories=set(), + ) + + assert properties["mllm"]["greeting_message"] == "agent greeting" + assert "greeting" not in properties["mllm"] + + def test_session_start_properties_preserves_mllm_vendor_defaults(): agent = ( Agent(test_client()) diff --git a/tests/custom/test_asr_vendor_configs.py b/tests/custom/test_asr_vendor_configs.py new file mode 100644 index 0000000..2df37d3 --- /dev/null +++ b/tests/custom/test_asr_vendor_configs.py @@ -0,0 +1,33 @@ +from agora_agent import AresSTT, GeminiSTT +from agora_agent.agentkit.vendors.catalog import GLOBAL_VENDOR_NAMESPACE +from agora_agent.agentkit.vendors.region import GLOBAL_ASR_VENDORS + + +def test_gemini_stt_serializes_generated_params() -> None: + config = GeminiSTT( + api_key="gemini-key", + model="gemini-transcribe", + sample_rate=16000, + language="en-US", + word_timestamp=True, + ).to_config() + + assert config == { + "vendor": "gemini", + "params": { + "api_key": "gemini-key", + "model": "gemini-transcribe", + "sample_rate": 16000, + "language": "en-US", + "word_timestamp": True, + }, + } + assert "gemini" in GLOBAL_ASR_VENDORS + assert GLOBAL_VENDOR_NAMESPACE.asr["gemini"] is GeminiSTT + + +def test_ares_keywords_are_top_level() -> None: + assert AresSTT(keywords=["Agora"]).to_config() == { + "vendor": "ares", + "keywords": ["Agora"], + } diff --git a/tests/custom/test_generated_filler_words.py b/tests/custom/test_generated_filler_words.py new file mode 100644 index 0000000..b5930f4 --- /dev/null +++ b/tests/custom/test_generated_filler_words.py @@ -0,0 +1,56 @@ +from agora_agent import ( + Agent, + FillerWordsContentGeneratedConfig, + FillerWordsContentGeneratedConfigLlmProvider, + FillerWordsConfig, + FillerWordsContent, + FillerWordsContentStaticConfig, + LlmToolConfig, +) +from test_helpers import test_client + + +def test_generated_filler_types_are_public_and_constructible() -> None: + provider = FillerWordsContentGeneratedConfigLlmProvider( + base_url="https://api.openai.com/v1/chat/completions", + api_key="filler-key", + params={"model": "gpt-4o-mini"}, + ) + config = FillerWordsContentGeneratedConfig(llm_provider=provider, prompt="Generate a short filler phrase") + + assert config.llm_provider.api_key == "filler-key" + assert config.fallback_strategy == "static" + assert LlmToolConfig.__name__ == "LlmTool" + + +def test_agentkit_serializes_generated_filler_words() -> None: + generated = FillerWordsConfig( + enable=True, + content=FillerWordsContent( + mode="generated", + static_config=FillerWordsContentStaticConfig(phrases=["One moment..."]), + generated_config=FillerWordsContentGeneratedConfig( + llm_provider=FillerWordsContentGeneratedConfigLlmProvider( + base_url="https://api.openai.com/v1/chat/completions", + api_key="filler-key", + params={"model": "gpt-4o-mini"}, + ), + prompt="Generate a short filler phrase", + ), + ), + ) + properties = Agent(test_client()).with_filler_words(generated).to_properties( + channel="test-channel", + agent_uid="1", + remote_uids=[], + token="token", + skip_vendor_validation=True, + ) + + filler_words = properties.filler_words + assert filler_words is not None + assert filler_words.content is not None + assert filler_words.content.mode == "generated" + assert filler_words.content.generated_config is not None + assert filler_words.content.generated_config.prompt == "Generate a short filler phrase" + assert filler_words.content.generated_config.llm_provider.api_key == "filler-key" diff --git a/tests/custom/test_llm_tools.py b/tests/custom/test_llm_tools.py new file mode 100644 index 0000000..56e41a7 --- /dev/null +++ b/tests/custom/test_llm_tools.py @@ -0,0 +1,57 @@ +from agora_agent import AdvancedFeatures, Agent, OpenAI +from agora_agent.agentkit.vendors.cn import AliyunLLM + +from test_helpers import test_client + + +def _tool() -> dict: + return { + "type": "function", + "function": { + "name": "lookup_order", + "parameters": {"type": "object", "properties": {}}, + }, + "server": {"method": "GET", "url": "https://example.com/orders"}, + } + + +def test_global_llm_tools_use_dict_shape_and_require_explicit_enablement() -> None: + tool = _tool() + vendor = OpenAI( + api_key="openai-key", + base_url="https://api.openai.com/v1/chat/completions", + model="gpt-4o-mini", + tools=[tool], + ) + + assert vendor.to_config()["tools"] == [tool] + agent = Agent(test_client()).with_llm(vendor) + assert agent.advanced_features is None + enabled_agent = agent.with_tools() + assert enabled_agent.advanced_features is not None + assert enabled_agent.advanced_features.enable_tools is True + disabled_agent = agent.with_tools(False) + assert disabled_agent.advanced_features is not None + assert disabled_agent.advanced_features.enable_tools is False + + +def test_cn_llm_uses_the_same_tools_shape_as_global_llm() -> None: + tool = _tool() + config = AliyunLLM( + api_key="aliyun-key", + base_url="https://example.com/v1/chat/completions", + model="qwen-plus", + tools=[tool], + ).to_config() + + assert config["tools"] == [tool] + + +def test_with_tools_preserves_other_advanced_features() -> None: + agent = Agent( + test_client(), advanced_features=AdvancedFeatures(enable_sal=True) + ).with_tools() + + assert agent.advanced_features is not None + assert agent.advanced_features.enable_sal is True + assert agent.advanced_features.enable_tools is True diff --git a/tests/custom/test_openai_gpt_live.py b/tests/custom/test_openai_gpt_live.py new file mode 100644 index 0000000..66a2c62 --- /dev/null +++ b/tests/custom/test_openai_gpt_live.py @@ -0,0 +1,38 @@ +from agora_agent import Agent, OpenAIGptLive, OpenAIRealtime +from agora_agent.agentkit.vendors.catalog import GLOBAL_VENDOR_NAMESPACE +from agora_agent.agentkit.vendors.region import GLOBAL_MLLM_VENDORS + +from test_helpers import test_client + + +def test_openai_gpt_live_is_distinct_global_mllm_vendor() -> None: + config = OpenAIGptLive(api_key="openai-key").to_config() + + assert config["vendor"] == "openai_gpt_live" + assert config["url"] == "wss://api.openai.com/v1/live" + assert "openai_gpt_live" in GLOBAL_MLLM_VENDORS + assert GLOBAL_VENDOR_NAMESPACE.mllm["openai_gpt_live"] is OpenAIGptLive + assert OpenAIRealtime(api_key="openai-key").to_config()["vendor"] == "openai" + + +def test_openai_gpt_live_serializes_greeting_message() -> None: + config = OpenAIGptLive(api_key="openai-key", greeting_message="Welcome to GPT Live").to_config() + + assert config["greeting_message"] == "Welcome to GPT Live" + assert "greeting" not in config + + +def test_agent_level_greeting_uses_openai_gpt_live_greeting_message_field() -> None: + properties = Agent(test_client()).with_mllm( + OpenAIGptLive(api_key="openai-key") + ).with_greeting("Welcome to GPT Live") + + mllm = properties.to_properties( + channel="test-channel", + agent_uid="1", + remote_uids=[], + token="token", + skip_vendor_validation=True, + ).mllm + assert mllm is not None + assert mllm.greeting_message == "Welcome to GPT Live"