Compare commits

..

2 Commits

Author SHA1 Message Date
Andrei Stoica 64bb9f9db3 cleaning up my sins 2024-02-26 11:31:01 -05:00
Andrei Stoica 42c605d992 cleanup 2024-02-25 17:47:25 -05:00
2 changed files with 136 additions and 112 deletions

View File

@ -1,34 +1,23 @@
import { useEffect, useRef, useState } from "react"; import { useState } from "react";
import { import { ChatMsg, Controls, Feed, Header } from "./components.tsx";
TbBrandOpenai,
TbMicrophone2,
TbPlayerPlay,
TbPlayerStop,
} from "react-icons/tb";
import "./App.css"; import "./App.css";
type ChatMsg = { let audioBlobs: Array<Blob> = [];
role: string;
content: string;
};
let audioBlobs = [];
let streamBeingCaptured: MediaStream | null = null; let streamBeingCaptured: MediaStream | null = null;
let mediaRecorder: MediaRecorder | null = null; let mediaRecorder: MediaRecorder | null = null;
function get_mic() { function get_mic() {
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
console.log("getUserMedia supported."); console.log("getUserMedia supported.");
return navigator.mediaDevices.getUserMedia({ audio: true }); return navigator.mediaDevices.getUserMedia({ audio: true });
} else {
console.log("getUserMedia not supported on your browser!");
} }
throw "getUserMedia not supported on your browser!";
} }
function startRecord() { function startRecord() {
audioBlobs = []; audioBlobs = [];
get_mic().then((stream) => { get_mic().then((stream) => {
console.log("got mic");
streamBeingCaptured = stream; streamBeingCaptured = stream;
mediaRecorder = new MediaRecorder(stream); mediaRecorder = new MediaRecorder(stream);
console.log("Starting Recording"); console.log("Starting Recording");
@ -40,6 +29,12 @@ function startRecord() {
} }
function stopRecord() { function stopRecord() {
if (!mediaRecorder) {
throw "MediaRecorder not set";
}
if (!streamBeingCaptured) {
throw "Stream not set";
}
mediaRecorder.stop(); mediaRecorder.stop();
streamBeingCaptured.getTracks() streamBeingCaptured.getTracks()
.forEach((track) => track.stop()); .forEach((track) => track.stop());
@ -57,62 +52,19 @@ function playRecord() {
} }
function playMsg(msg: ChatMsg) { function playMsg(msg: ChatMsg) {
const audio = new Audio("http://100.82.51.22:8001/speak?" + new URLSearchParams({text: msg.content})); const audio = new Audio(
console.log("loading audio and playing?") "http://100.82.51.22:8001/speak?" +
new URLSearchParams({ text: msg.content }),
);
console.log("loading audio and playing?");
audio.play(); audio.play();
} }
function App() {
function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
const bottomRef = useRef(null);
const scrollToBottom = () => {
bottomRef.current?.scrollIntoView({ behavior: "smooth" });
};
useEffect(() => {
scrollToBottom();
console.log("scroll?");
});
return (
<div className="feed grow self-center w-5/6 max-w-screen-lg px-6 py-3 overflow-scroll">
<div className="content-center space-y-2 divide-y-4">
{props.chat.filter((m: ChatMsg) => m.role != "system").map((
m: ChatMsg,
i: number,
) => <Msg key={i} msg={m} />)}
</div>
<div ref={bottomRef} />
</div>
);
}
function Msg(props: { msg: ChatMsg }) {
return (
<div className="Messege text-lg">
<span className="font-bold">
{props.msg.role.toUpperCase()}:
</span>
<br />
<span className="ml-8">
{props.msg.content}
</span>
</div>
);
}
function Controls(props: { setChatStateFn: any; chat: Array[ChatMsg] }) {
const [recordState, setRecordState] = useState(false); const [recordState, setRecordState] = useState(false);
const [chatState, setChatState] = useState([{
role: "system",
content: "You are a helpful assistant.",
}]);
function toggleRecord() { function toggleRecord() {
if (recordState == false) { if (recordState == false) {
@ -132,61 +84,27 @@ function Controls(props: { setChatStateFn: any; chat: Array[ChatMsg] }) {
"body": formData, "body": formData,
}).then((res) => res.json()) }).then((res) => res.json())
.then((res) => { .then((res) => {
console.log(res); setChatState((curState: Array<ChatMsg>) => [
props.setChatStateFn((curState) => [
...curState, ...curState,
{ "role": "user", "content": res["user-transcript"] }, { "role": "user", "content": res["user-transcript"] },
]); ]);
fetch("http://100.82.51.22:8001/conversation", { fetch("http://100.82.51.22:8001/conversation", {
"method": "POST", "method": "POST",
"body": JSON.stringify([...props.chat, { "body": JSON.stringify([...chatState, {
"role": "user", "role": "user",
"content": res["user-transcript"], "content": res["user-transcript"],
}]), }]),
}).then((res) => res.json()) }).then((res) => res.json())
.then((res) => { .then((res) => {
props.setChatStateFn((curState) => [...curState, res]); setChatState((
console.log("attempting to play result") curState: Array<ChatMsg>,
playMsg(res) ) => [...curState, res]);
console.log("attempting to play result");
playMsg(res);
}); });
}); });
} }
return (
<div className="controls self-center flex justify-evenly p-5 text-5xl border-2 border-b-0 w-1/2 max-w-screen-sm min-w-fit">
<button
onClick={() => toggleRecord()}
className={"inline-flex " + (recordState ? "text-red-500" : "")}
>
{recordState ? <TbPlayerStop /> : <TbMicrophone2 />}
{recordState ? "STOP" : "REC"}
</button>
<button
onClick={() => playRecord()}
className="inline-flex text-green-500"
>
<TbPlayerPlay /> PLAY
</button>
<button
onClick={() => {
sendAudio();
}}
className="inline-flex"
>
<TbBrandOpenai /> SEND
</button>
</div>
);
}
function App() {
const [chatState, setChatState] = useState([{
role: "system",
content: "You are a helpful assistant.",
}]);
return ( return (
<> <>
<div className="h-screen center flex flex-col"> <div className="h-screen center flex flex-col">
@ -195,7 +113,12 @@ function App() {
<hr className="mx-3 border-t-4" /> <hr className="mx-3 border-t-4" />
</div> </div>
<Feed chat={chatState} setChatStateFn={setChatState} /> <Feed chat={chatState} setChatStateFn={setChatState} />
<Controls setChatStateFn={setChatState} chat={chatState} /> <Controls
recButtonOnClick={toggleRecord}
recordState={recordState}
playButtonOnClick={playRecord}
sendButtonOnClick={sendAudio}
/>
</div> </div>
</> </>
); );

View File

@ -0,0 +1,101 @@
import { useEffect, useRef } from "react";
import {
TbBrandOpenai,
TbMicrophone2,
TbPlayerPlay,
TbPlayerStop,
} from "react-icons/tb";
export type ChatMsg = {
role: string;
content: string;
};
export function Header() {
return (
<header className="header p-3">
<div className="title text-5xl font-extrabold">
Speach to Speech AI example
</div>
</header>
);
}
export function Feed(props: { chat: Array<ChatMsg>; setChatStateFn: any }) {
const bottomRef = useRef<any>(null);
const scrollToBottom = () => {
if (bottomRef.current) {
bottomRef.current.scrollIntoView({ behavior: "smooth" });
}
};
useEffect(() => {
scrollToBottom();
console.log("scroll?");
});
return (
<div className="feed grow self-center w-5/6 max-w-screen-lg px-6 py-3 overflow-scroll">
<div className="content-center space-y-2 divide-y-4">
{props.chat.filter((m: ChatMsg) => m.role != "system").map((
m: ChatMsg,
i: number,
) => <Msg key={i} msg={m} />)}
</div>
<div ref={bottomRef} />
</div>
);
}
export function Msg(props: { msg: ChatMsg }) {
return (
<div className="Messege text-lg">
<span className="font-bold">
{props.msg.role.toUpperCase()}:
</span>
<br />
<span className="ml-8">
{props.msg.content}
</span>
</div>
);
}
export function Controls(
props: {
recButtonOnClick: Function;
recordState: Boolean;
playButtonOnClick: Function;
sendButtonOnClick: Function;
},
) {
return (
<div className="controls self-center flex justify-evenly p-5 text-5xl border-2 border-b-0 w-1/2 max-w-screen-sm min-w-fit">
<button
onClick={() => props.recButtonOnClick()}
className={"inline-flex " + (props.recordState ? "text-red-500" : "")}
>
{props.recordState ? <TbPlayerStop /> : <TbMicrophone2 />}
{props.recordState ? "STOP" : "REC"}
</button>
<button
onClick={() => props.playButtonOnClick()}
className="inline-flex text-green-500"
>
<TbPlayerPlay /> PLAY
</button>
<button
onClick={() => {
props.sendButtonOnClick();
}}
className="inline-flex"
>
<TbBrandOpenai /> SEND
</button>
</div>
);
}