Inference gateway integration (#22)
This commit is contained in:
Ben Cherry
+35
-43
@@ -2,21 +2,17 @@ import logging
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from livekit.agents import (
|
||||
NOT_GIVEN,
|
||||
Agent,
|
||||
AgentFalseInterruptionEvent,
|
||||
AgentSession,
|
||||
JobContext,
|
||||
JobProcess,
|
||||
MetricsCollectedEvent,
|
||||
RoomInputOptions,
|
||||
RunContext,
|
||||
WorkerOptions,
|
||||
cli,
|
||||
metrics,
|
||||
)
|
||||
from livekit.agents.llm import function_tool
|
||||
from livekit.plugins import cartesia, deepgram, noise_cancellation, openai, silero
|
||||
from livekit.plugins import noise_cancellation, silero
|
||||
from livekit.plugins.turn_detector.multilingual import MultilingualModel
|
||||
|
||||
logger = logging.getLogger("agent")
|
||||
@@ -27,27 +23,28 @@ load_dotenv(".env.local")
|
||||
class Assistant(Agent):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
instructions="""You are a helpful voice AI assistant.
|
||||
instructions="""You are a helpful voice AI assistant. The user is interacting with you via voice, even if you perceive the conversation as text.
|
||||
You eagerly assist users with their questions by providing information from your extensive knowledge.
|
||||
Your responses are concise, to the point, and without any complex formatting or punctuation including emojis, asterisks, or other symbols.
|
||||
You are curious, friendly, and have a sense of humor.""",
|
||||
)
|
||||
|
||||
# all functions annotated with @function_tool will be passed to the LLM when this
|
||||
# agent is active
|
||||
@function_tool
|
||||
async def lookup_weather(self, context: RunContext, location: str):
|
||||
"""Use this tool to look up current weather information in the given location.
|
||||
|
||||
If the location is not supported by the weather service, the tool will indicate this. You must tell the user the location's weather is unavailable.
|
||||
|
||||
Args:
|
||||
location: The location to look up weather information for (e.g. city name)
|
||||
"""
|
||||
|
||||
logger.info(f"Looking up weather for {location}")
|
||||
|
||||
return "sunny with a temperature of 70 degrees."
|
||||
# To add tools, use the @function_tool decorator.
|
||||
# Here's an example that adds a simple weather tool.
|
||||
# You also have to add `from livekit.agents.llm import function_tool, RunContext` to the top of this file
|
||||
# @function_tool
|
||||
# async def lookup_weather(self, context: RunContext, location: str):
|
||||
# """Use this tool to look up current weather information in the given location.
|
||||
#
|
||||
# If the location is not supported by the weather service, the tool will indicate this. You must tell the user the location's weather is unavailable.
|
||||
#
|
||||
# Args:
|
||||
# location: The location to look up weather information for (e.g. city name)
|
||||
# """
|
||||
#
|
||||
# logger.info(f"Looking up weather for {location}")
|
||||
#
|
||||
# return "sunny with a temperature of 70 degrees."
|
||||
|
||||
|
||||
def prewarm(proc: JobProcess):
|
||||
@@ -61,17 +58,17 @@ async def entrypoint(ctx: JobContext):
|
||||
"room": ctx.room.name,
|
||||
}
|
||||
|
||||
# Set up a voice AI pipeline using OpenAI, Cartesia, Deepgram, and the LiveKit turn detector
|
||||
# Set up a voice AI pipeline using OpenAI, Cartesia, AssemblyAI, and the LiveKit turn detector
|
||||
session = AgentSession(
|
||||
# A Large Language Model (LLM) is your agent's brain, processing user input and generating a response
|
||||
# See all providers at https://docs.livekit.io/agents/integrations/llm/
|
||||
llm=openai.LLM(model="gpt-4o-mini"),
|
||||
# Speech-to-text (STT) is your agent's ears, turning the user's speech into text that the LLM can understand
|
||||
# See all providers at https://docs.livekit.io/agents/integrations/stt/
|
||||
stt=deepgram.STT(model="nova-3", language="multi"),
|
||||
# See all available models at https://docs.livekit.io/agents/models/stt/
|
||||
stt="assemblyai/universal-streaming:en",
|
||||
# A Large Language Model (LLM) is your agent's brain, processing user input and generating a response
|
||||
# See all available models at https://docs.livekit.io/agents/models/llm/
|
||||
llm="openai/gpt-4.1-mini",
|
||||
# Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear
|
||||
# See all providers at https://docs.livekit.io/agents/integrations/tts/
|
||||
tts=cartesia.TTS(voice="6f84f4b8-58a2-430c-8c79-688dad597532"),
|
||||
# See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/
|
||||
tts="cartesia/sonic-2:9626c31c-bec5-4cca-baa8-f8ba9e84c8bc",
|
||||
# VAD and turn detection are used to determine when the user is speaking and when the agent should respond
|
||||
# See more at https://docs.livekit.io/agents/build/turns
|
||||
turn_detection=MultilingualModel(),
|
||||
@@ -81,19 +78,16 @@ async def entrypoint(ctx: JobContext):
|
||||
preemptive_generation=True,
|
||||
)
|
||||
|
||||
# To use a realtime model instead of a voice pipeline, use the following session setup instead:
|
||||
# To use a realtime model instead of a voice pipeline, use the following session setup instead.
|
||||
# (Note: This is for the OpenAI Realtime API. For other providers, see https://docs.livekit.io/agents/models/realtime/))
|
||||
# 1. Install livekit-agents[openai]
|
||||
# 2. Set OPENAI_API_KEY in .env.local
|
||||
# 3. Add `from livekit.plugins import openai` to the top of this file
|
||||
# 4. Use the following session setup instead of the version above
|
||||
# session = AgentSession(
|
||||
# # See all providers at https://docs.livekit.io/agents/integrations/realtime/
|
||||
# llm=openai.realtime.RealtimeModel(voice="marin")
|
||||
# )
|
||||
|
||||
# sometimes background noise could interrupt the agent session, these are considered false positive interruptions
|
||||
# when it's detected, you may resume the agent's speech
|
||||
@session.on("agent_false_interruption")
|
||||
def _on_agent_false_interruption(ev: AgentFalseInterruptionEvent):
|
||||
logger.info("false positive interruption, resuming")
|
||||
session.generate_reply(instructions=ev.extra_instructions or NOT_GIVEN)
|
||||
|
||||
# Metrics collection, to measure pipeline performance
|
||||
# For more information, see https://docs.livekit.io/agents/build/metrics/
|
||||
usage_collector = metrics.UsageCollector()
|
||||
@@ -110,9 +104,9 @@ async def entrypoint(ctx: JobContext):
|
||||
ctx.add_shutdown_callback(log_usage)
|
||||
|
||||
# # Add a virtual avatar to the session, if desired
|
||||
# # For other providers, see https://docs.livekit.io/agents/integrations/avatar/
|
||||
# # For other providers, see https://docs.livekit.io/agents/models/avatar/
|
||||
# avatar = hedra.AvatarSession(
|
||||
# avatar_id="...", # See https://docs.livekit.io/agents/integrations/avatar/hedra
|
||||
# avatar_id="...", # See https://docs.livekit.io/agents/models/avatar/plugins/hedra
|
||||
# )
|
||||
# # Start the avatar and wait for it to join
|
||||
# await avatar.start(session, room=ctx.room)
|
||||
@@ -122,9 +116,7 @@ async def entrypoint(ctx: JobContext):
|
||||
agent=Assistant(),
|
||||
room=ctx.room,
|
||||
room_input_options=RoomInputOptions(
|
||||
# LiveKit Cloud enhanced noise cancellation
|
||||
# - If self-hosting, omit this parameter
|
||||
# - For telephony applications, use `BVCTelephony` for best results
|
||||
# For telephony applications, use `BVCTelephony` for best results
|
||||
noise_cancellation=noise_cancellation.BVC(),
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user