andrei-stoica
/
ai_sandbox
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

playing back response

This commit is contained in:
Andrei Stoica 2024-02-25 13:40:08 -05:00
parent baab95660b
commit ebcfa7e19e
3 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
speech-speech/backend/api.py
View File

@ -28,7 +28,7 @@ class Conversation(BaseModel):
@app.post("/get-text") @app.post("/get-text")
def get_text(response: Response, audio: bytes = File()): def stt(audio: bytes = File()):
with open("audio", "wb") as f: with open("audio", "wb") as f:
f.write(audio) f.write(audio)
# transcript = openAI_clinet.audio.transcriptions.create( # transcript = openAI_clinet.audio.transcriptions.create(
@ -43,7 +43,7 @@ def get_text(response: Response, audio: bytes = File()):
@app.post("/conversation") @app.post("/conversation")
async def get_next_response(request: Request, response: Response): async def get_next_response(request: Request):
# role = "test" # role = "test"
# res_msg = "temp test response" # res_msg = "temp test response"
messages = await request.json() messages = await request.json()
@ -58,8 +58,8 @@ async def get_next_response(request: Request, response: Response):
return {"role": role, "content": res_msg} return {"role": role, "content": res_msg}
@app.post("/speak", response_class=FileResponse) @app.get("/speak", response_class=FileResponse)
def tts(text: str, response: Response): def tts(text: str):
res = openAI_clinet.audio.speech.create( res = openAI_clinet.audio.speech.create(
model="tts-1", model="tts-1",
voice="nova", voice="nova",

3
speech-speech/frontend/index.html
View File

@ -2,9 +2,8 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite + React + TS</title> <title>Speach to Speech AI example</title>
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

36
speech-speech/frontend/src/App.tsx
View File

@ -12,23 +12,10 @@ type ChatMsg = {
content: string; content: string;
}; };
function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
let audioBlobs = []; let audioBlobs = [];
let streamBeingCaptured: MediaStream | null = null; let streamBeingCaptured: MediaStream | null = null;
let mediaRecorder: MediaRecorder | null = null; let mediaRecorder: MediaRecorder | null = null;
let chat: Array<ChatMsg> = [{
role: "system",
content: "You are a helpful assistant.",
}];
function get_mic() { function get_mic() {
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
@ -69,6 +56,22 @@ function playRecord() {
audio.play(); audio.play();
} }
function playMsg(msg: ChatMsg) {
const audio = new Audio("http://100.82.51.22:8001/speak?" + new URLSearchParams({text: msg.content}));
console.log("loading audio and playing?")
audio.play();
}
function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) { function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
const bottomRef = useRef(null); const bottomRef = useRef(null);
@ -86,7 +89,8 @@ function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
<div className="content-center space-y-2 divide-y-4"> <div className="content-center space-y-2 divide-y-4">
{props.chat.filter((m: ChatMsg) => m.role != "system").map(( {props.chat.filter((m: ChatMsg) => m.role != "system").map((
m: ChatMsg, m: ChatMsg,
) => <Msg msg={m} />)} i: number,
) => <Msg key={i} msg={m} />)}
</div> </div>
<div ref={bottomRef} /> <div ref={bottomRef} />
</div> </div>
@ -142,6 +146,8 @@ function Controls(props: { setChatStateFn: any; chat: Array[ChatMsg] }) {
}).then((res) => res.json()) }).then((res) => res.json())
.then((res) => { .then((res) => {
props.setChatStateFn((curState) => [...curState, res]); props.setChatStateFn((curState) => [...curState, res]);
console.log("attempting to play result")
playMsg(res)
}); });
}); });
} }