andrei-stoica
/
ai_sandbox
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

playing back response

This commit is contained in:
Andrei Stoica 2024-02-25 13:40:08 -05:00
parent baab95660b
commit ebcfa7e19e
3 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
speech-speech/backend/api.py
View File

@ -28,7 +28,7 @@ class Conversation(BaseModel):
@app.post("/get-text")
def get_text(response: Response, audio: bytes = File()):
def stt(audio: bytes = File()):
with open("audio", "wb") as f:
f.write(audio)
# transcript = openAI_clinet.audio.transcriptions.create(
@ -43,7 +43,7 @@ def get_text(response: Response, audio: bytes = File()):
@app.post("/conversation")
async def get_next_response(request: Request, response: Response):
async def get_next_response(request: Request):
# role = "test"
# res_msg = "temp test response"
messages = await request.json()
@ -58,8 +58,8 @@ async def get_next_response(request: Request, response: Response):
return {"role": role, "content": res_msg}
@app.post("/speak", response_class=FileResponse)
def tts(text: str, response: Response):
@app.get("/speak", response_class=FileResponse)
def tts(text: str):
res = openAI_clinet.audio.speech.create(
model="tts-1",
voice="nova",

3
speech-speech/frontend/index.html
View File

@ -2,9 +2,8 @@
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite + React + TS</title>
<title>Speach to Speech AI example</title>
</head>
<body>
<div id="root"></div>

38
speech-speech/frontend/src/App.tsx
View File

@ -12,23 +12,10 @@ type ChatMsg = {
content: string;
};
function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
let audioBlobs = [];
let streamBeingCaptured: MediaStream | null = null;
let mediaRecorder: MediaRecorder | null = null;
let chat: Array<ChatMsg> = [{
role: "system",
content: "You are a helpful assistant.",
}];
function get_mic() {
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
@ -65,10 +52,26 @@ function stopRecord() {
function playRecord() {
const audioBlob = new Blob(audioBlobs, { type: "audio/webm" });
const audioUrl = URL.createObjectURL(audioBlob);
const audio = new Audio(audioUrl);
const audio = new Audio(audioUrl);
audio.play();
}
function playMsg(msg: ChatMsg) {
const audio = new Audio("http://100.82.51.22:8001/speak?" + new URLSearchParams({text: msg.content}));
console.log("loading audio and playing?")
audio.play();
}
function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
const bottomRef = useRef(null);
@ -86,7 +89,8 @@ function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
<div className="content-center space-y-2 divide-y-4">
{props.chat.filter((m: ChatMsg) => m.role != "system").map((
m: ChatMsg,
) => <Msg msg={m} />)}
i: number,
) => <Msg key={i} msg={m} />)}
</div>
<div ref={bottomRef} />
</div>
@ -142,6 +146,8 @@ function Controls(props: { setChatStateFn: any; chat: Array[ChatMsg] }) {
}).then((res) => res.json())
.then((res) => {
props.setChatStateFn((curState) => [...curState, res]);
console.log("attempting to play result")
playMsg(res)
});
});
}