Skip to content

Commit 6fed34a

Browse files
theomonnomtheomonnom
committed
Merge branch 'main' into theo/v1.3
2 parents 09ac623 + bb871e3 commit 6fed34a

File tree

18 files changed

+474
-38
lines changed

18 files changed

+474
-38
lines changed

‎examples/avatar_agents/avatartalk/README.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# LiveKit AvatarTalk Avatar Agent
2+
3+
This example demonstrates how to create a animated avatar using [AvatarTalk](https://avatartalk.ai/).
4+
5+
## Usage
6+
7+
* Update the environment:
8+
9+
```bash
10+
# AvatarTalk Config
11+
export AVATARTALK_API_KEY="..."
12+
export AVATARTALK_API_URL="..."
13+
export AVATARTALK_AVATAR="..."
14+
export AVATARTALK_EMOTION="..."
15+
16+
# OpenAI config (or other models, tts, stt)
17+
export OPENAI_API_KEY="..."
18+
19+
# LiveKit config
20+
export LIVEKIT_API_KEY="..."
21+
export LIVEKIT_API_SECRET="..."
22+
export LIVEKIT_URL="..."
23+
```
24+
25+
* Start the agent worker:
26+
27+
```bash
28+
python examples/avatar_agents/avatartalk/agent_worker.py dev
29+
```

‎examples/avatar_agents/avatartalk/agent_worker.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import os
2+
3+
from dotenv import load_dotenv
4+
from openai.types.beta.realtime.session import TurnDetection
5+
6+
from livekit.agents import Agent, AgentSession, JobContext, WorkerOptions, WorkerType, cli
7+
from livekit.plugins import avatartalk, openai
8+
9+
load_dotenv()
10+
11+
12+
async def entrypoint(ctx: JobContext):
13+
await ctx.connect()
14+
15+
session = AgentSession(
16+
llm=openai.realtime.RealtimeModel(
17+
voice="ash",
18+
turn_detection=TurnDetection(
19+
type="server_vad",
20+
threshold=0.5,
21+
prefix_padding_ms=300,
22+
silence_duration_ms=500,
23+
create_response=True,
24+
interrupt_response=True,
25+
),
26+
)
27+
)
28+
29+
avatar = avatartalk.AvatarSession(
30+
api_url=os.getenv("AVATARTALK_API_URL"),
31+
avatar=os.getenv("AVATARTALK_AVATAR"),
32+
emotion=os.getenv("AVATARTALK_EMOTION"),
33+
api_secret=os.getenv("AVATARTALK_API_KEY"),
34+
)
35+
36+
await avatar.start(session, room=ctx.room)
37+
await session.start(
38+
agent=Agent(
39+
instructions=(
40+
"You are a helpful AI assistant designed to "
41+
"communicate with users in multiple languages. "
42+
"Your primary directive is to always respond in "
43+
"the same language that the user writes in."
44+
)
45+
),
46+
room=ctx.room,
47+
)
48+
49+
50+
if __name__ == "__main__":
51+
cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, worker_type=WorkerType.ROOM))

‎livekit-agents/livekit/agents/cli/cli.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1499,8 +1499,10 @@ def download_files() -> None:
14991499
# c.print(" ")
15001500

15011501
for plugin in Plugin.registered_plugins:
1502-
logger.info(f"Downloading files for {plugin}")
1502+
logger.info(f"Downloading files for {plugin.package}")
15031503
plugin.download_files()
1504+
logger.info(f"Finished downloading files for {plugin.package}")
1505+
15041506

15051507
except CLIError as e:
15061508
c.print(" ")

‎livekit-plugins/livekit-plugins-avatartalk/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# AvatarTalk virtual avatar plugin for LiveKit Agents
2+
3+
Support for the [AvatarTalk](https://avatartalk.ai/) virtual avatar.
4+
5+
See [https://docs.livekit.io/agents/integrations/avatar/avatartalk/](https://docs.livekit.io/agents/integrations/avatar/avatartalk/) for more information.
6+

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/__init__.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2025 LiveKit, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""AvatarTalk virtual avatar plugin for LiveKit Agents
16+
17+
See https://docs.livekit.io/agents/integrations/avatar/avatartalk/ for more information.
18+
"""
19+
20+
from .api import AvatarTalkException
21+
from .avatar import AvatarSession
22+
from .version import __version__
23+
24+
__all__ = [
25+
"AvatarTalkException",
26+
"AvatarSession",
27+
"__version__",
28+
]
29+
30+
from livekit.agents import Plugin
31+
32+
from .log import logger
33+
34+
35+
class AvatarTalkPlugin(Plugin):
36+
def __init__(self) -> None:
37+
super().__init__(__name__, __version__, __package__, logger)
38+
39+
40+
Plugin.register_plugin(AvatarTalkPlugin())

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/api.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import logging
2+
import os
3+
from typing import Any
4+
5+
import aiohttp
6+
7+
from livekit.agents import NOT_GIVEN, NotGivenOr
8+
9+
logger = logging.getLogger(__name__)
10+
11+
DEFAULT_API_URL = "https://api.avatartalk.ai"
12+
13+
14+
class AvatarTalkException(Exception):
15+
"""Exception for AvatarTalkAPI errors"""
16+
17+
18+
class AvatarTalkAPI:
19+
def __init__(
20+
self,
21+
api_url: NotGivenOr[str] = NOT_GIVEN,
22+
api_key: NotGivenOr[str] = NOT_GIVEN,
23+
):
24+
self._api_url = api_url or DEFAULT_API_URL
25+
avatartalk_api_key = api_key or os.getenv("AVATARTALK_API_KEY")
26+
if avatartalk_api_key is None:
27+
raise AvatarTalkException("AVATARTALK_API_KEY must be set")
28+
29+
self._api_key = avatartalk_api_key
30+
self._headers = {"Authorization": f"Bearer {self._api_key}"}
31+
32+
async def _request(self, method: str, path: str, **kwargs):
33+
async with aiohttp.ClientSession() as session:
34+
async with session.request(
35+
method, f"{self._api_url}{path}", headers=self._headers, **kwargs
36+
) as response:
37+
if response.ok:
38+
return await response.json()
39+
else:
40+
r = await response.json()
41+
raise AvatarTalkException(f"API request failed: {response.status} {r}")
42+
43+
async def start_session(
44+
self,
45+
livekit_url: str,
46+
avatar: str,
47+
emotion: str,
48+
room_name: str,
49+
livekit_listener_token: str,
50+
livekit_room_token: str,
51+
agent_identity: str,
52+
) -> dict[str, Any]:
53+
return await self._request(
54+
"POST",
55+
"/livekit/create-session",
56+
json={
57+
"livekit_url": livekit_url,
58+
"avatar": avatar,
59+
"emotion": emotion,
60+
"room_name": room_name,
61+
"room_token": livekit_room_token,
62+
"listener_token": livekit_listener_token,
63+
"agent_identity": agent_identity,
64+
},
65+
)
66+
67+
async def stop_session(self, task_id: str) -> dict[str, Any]:
68+
return await self._request("DELETE", f"/livekit/delete-session/{task_id}")

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/avatar.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import os
2+
from typing import Optional
3+
4+
from livekit import api, rtc
5+
from livekit.agents import NOT_GIVEN, AgentSession, NotGivenOr, get_job_context
6+
from livekit.agents.types import ATTRIBUTE_PUBLISH_ON_BEHALF
7+
from livekit.agents.voice.avatar import DataStreamAudioOutput
8+
9+
from .api import AvatarTalkAPI, AvatarTalkException
10+
from .log import logger
11+
12+
_AVATAR_AGENT_IDENTITY = "avatartalk-agent"
13+
_AVATAR_AGENT_NAME = "avatartalk-agent"
14+
DEFAULT_AVATAR_NAME = "japanese_man"
15+
DEFAULT_AVATAR_EMOTION = "expressive"
16+
SAMPLE_RATE = 16000
17+
18+
19+
class AvatarSession:
20+
"""AvatarTalkAPI avatar session"""
21+
22+
def __init__(
23+
self,
24+
*,
25+
api_url: NotGivenOr[str] = NOT_GIVEN,
26+
api_secret: NotGivenOr[str] = NOT_GIVEN,
27+
avatar: NotGivenOr[str | None] = NOT_GIVEN,
28+
emotion: NotGivenOr[str | None] = NOT_GIVEN,
29+
avatar_participant_identity: NotGivenOr[str | None] = NOT_GIVEN,
30+
avatar_participant_name: NotGivenOr[str | None] = NOT_GIVEN,
31+
):
32+
self._avatartalk_api = AvatarTalkAPI(api_url, api_secret)
33+
self._avatar = avatar or (os.getenv("AVATARTALK_AVATAR") or DEFAULT_AVATAR_NAME)
34+
self._emotion = emotion or (os.getenv("AVATARTALK_EMOTION") or DEFAULT_AVATAR_EMOTION)
35+
self._avatar_participant_identity = avatar_participant_identity or _AVATAR_AGENT_IDENTITY
36+
self._avatar_participant_name = avatar_participant_name or _AVATAR_AGENT_NAME
37+
self._agent_track = None
38+
39+
def __generate_lk_token(
40+
self,
41+
livekit_api_key: str,
42+
livekit_api_secret: str,
43+
room: rtc.Room,
44+
participant_identity: str,
45+
participant_name: str,
46+
as_agent: bool = True,
47+
local_participant_identity: Optional[str] = None,
48+
):
49+
token = (
50+
api.AccessToken(api_key=livekit_api_key, api_secret=livekit_api_secret)
51+
.with_identity(participant_identity)
52+
.with_name(participant_name)
53+
.with_grants(api.VideoGrants(room_join=True, room=room.name))
54+
.with_attributes({ATTRIBUTE_PUBLISH_ON_BEHALF: local_participant_identity})
55+
)
56+
if as_agent:
57+
token = token.with_kind("agent")
58+
59+
if local_participant_identity is not None:
60+
token = token.with_attributes({ATTRIBUTE_PUBLISH_ON_BEHALF: local_participant_identity})
61+
62+
return token.to_jwt()
63+
64+
async def start(
65+
self,
66+
agent_session: AgentSession,
67+
room: rtc.Room,
68+
*,
69+
livekit_url: NotGivenOr[str | None] = NOT_GIVEN,
70+
livekit_api_key: NotGivenOr[str | None] = NOT_GIVEN,
71+
livekit_api_secret: NotGivenOr[str | None] = NOT_GIVEN,
72+
):
73+
livekit_url = livekit_url or (os.getenv("LIVEKIT_URL") or NOT_GIVEN)
74+
livekit_api_key = livekit_api_key or (os.getenv("LIVEKIT_API_KEY") or NOT_GIVEN)
75+
livekit_api_secret = livekit_api_secret or (os.getenv("LIVEKIT_API_SECRET") or NOT_GIVEN)
76+
if not livekit_url or not livekit_api_key or not livekit_api_secret:
77+
raise AvatarTalkException(
78+
"livekit_url, livekit_api_key, and livekit_api_secret must be set by arguments or environment variables"
79+
)
80+
81+
session_task_mapping = {}
82+
83+
try:
84+
job_ctx = get_job_context()
85+
local_participant_identity = job_ctx.token_claims().identity
86+
87+
async def _shutdown_session():
88+
if room.name not in session_task_mapping:
89+
return
90+
await self._avatartalk_api.stop_session(session_task_mapping[room.name])
91+
92+
job_ctx.add_shutdown_callback(_shutdown_session)
93+
except (RuntimeError, KeyError):
94+
if not room.isconnected():
95+
raise AvatarTalkException(
96+
"local participant identity not found in token, and room is not connected"
97+
) from None
98+
local_participant_identity = room.local_participant.identity
99+
100+
livekit_token = self.__generate_lk_token(
101+
livekit_api_key,
102+
livekit_api_secret,
103+
room,
104+
self._avatar_participant_identity,
105+
self._avatar_participant_name,
106+
as_agent=True,
107+
local_participant_identity=local_participant_identity,
108+
)
109+
110+
livekit_listener_token = self.__generate_lk_token(
111+
livekit_api_key,
112+
livekit_api_secret,
113+
room,
114+
"listener",
115+
"listener",
116+
as_agent=False,
117+
local_participant_identity=None,
118+
)
119+
120+
logger.debug(
121+
"Starting Avatartalk agent session",
122+
extra={"avatar": self._avatar, "room_name": room.name},
123+
)
124+
try:
125+
resp = await self._avatartalk_api.start_session(
126+
livekit_url=livekit_url,
127+
avatar=self._avatar,
128+
emotion=self._emotion,
129+
room_name=room.name,
130+
livekit_listener_token=livekit_listener_token,
131+
livekit_room_token=livekit_token,
132+
agent_identity=local_participant_identity,
133+
)
134+
135+
self.conversation_id = resp["task_id"]
136+
logger.debug(
137+
"Avatartalk agent session started",
138+
extra={
139+
"avatar": self._avatar,
140+
"emotion": self._emotion,
141+
"room_name": room.name,
142+
"task_id": self.conversation_id,
143+
},
144+
)
145+
session_task_mapping[room.name] = self.conversation_id
146+
147+
agent_session.output.audio = DataStreamAudioOutput(
148+
room=room,
149+
destination_identity="listener",
150+
sample_rate=SAMPLE_RATE,
151+
# wait_remote_track=rtc.TrackKind.KIND_VIDEO,
152+
)
153+
except AvatarTalkException as e:
154+
logger.error(e)

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/log.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import logging
2+
3+
logger = logging.getLogger("livekit.plugins.avatartalk")

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/py.typed‎

Whitespace-only changes.

‎livekit-plugins/livekit-plugins-avatartalk/livekit/plugins/avatartalk/version.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2025 LiveKit, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
__version__ = "1.0.0"

0 commit comments

Comments
 (0)