speech-speech/frontend/src/App.tsx

@@ -1,23 +1,34 @@
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
-import { ChatMsg, Controls, Feed, Header } from "./components.tsx";
+import {
+  TbBrandOpenai,
+  TbMicrophone2,
+  TbPlayerPlay,
+  TbPlayerStop,
+} from "react-icons/tb";
 import "./App.css";
 
-let audioBlobs: Array<Blob> = [];
+type ChatMsg = {
+  role: string;
+  content: string;
+};
+
+let audioBlobs = [];
 let streamBeingCaptured: MediaStream | null = null;
 let mediaRecorder: MediaRecorder | null = null;
 
+
 function get_mic() {
   if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
     console.log("getUserMedia supported.");
     return navigator.mediaDevices.getUserMedia({ audio: true });
+  } else {
+    console.log("getUserMedia not supported on your browser!");
   }
-  throw "getUserMedia not supported on your browser!";
 }
 
 function startRecord() {
   audioBlobs = [];
   get_mic().then((stream) => {
-    console.log("got mic");
     streamBeingCaptured = stream;
     mediaRecorder = new MediaRecorder(stream);
     console.log("Starting Recording");
@@ -29,12 +40,6 @@ function startRecord() {
 }
 
 function stopRecord() {
-  if (!mediaRecorder) {
-    throw "MediaRecorder not set";
-  }
-  if (!streamBeingCaptured) {
-    throw "Stream not set";
-  }
   mediaRecorder.stop();
   streamBeingCaptured.getTracks()
     .forEach((track) => track.stop());
@@ -47,24 +52,67 @@ function stopRecord() {
 function playRecord() {
   const audioBlob = new Blob(audioBlobs, { type: "audio/webm" });
   const audioUrl = URL.createObjectURL(audioBlob);
-  const audio = new Audio(audioUrl);
+  const audio =  new Audio(audioUrl);
   audio.play();
 }
 
 function playMsg(msg: ChatMsg) {
-  const audio = new Audio(
+  const audio = new Audio("http://100.82.51.22:8001/speak?" + new URLSearchParams({text: msg.content}));
-    "http://100.82.51.22:8001/speak?" +
+  console.log("loading audio and playing?")
-    new URLSearchParams({ text: msg.content }),
-  );
-  console.log("loading audio and playing?");
   audio.play();
 }
-function App() {
+
+function Header() {
+  return (
+    <header className="header p-3">
+      <div className="title text-5xl font-extrabold">
+        Speach to Speech AI example
+      </div>
+    </header>
+  );
+}
+
+function Feed(props: { chat: Array[ChatMsg]; setChatStateFn: any }) {
+  const bottomRef = useRef(null);
+
+  const scrollToBottom = () => {
+    bottomRef.current?.scrollIntoView({ behavior: "smooth" });
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+    console.log("scroll?");
+  });
+
+  return (
+    <div className="feed grow self-center w-5/6 max-w-screen-lg px-6 py-3 overflow-scroll">
+      <div className="content-center  space-y-2 divide-y-4">
+        {props.chat.filter((m: ChatMsg) => m.role != "system").map((
+          m: ChatMsg,
+          i: number,
+        ) => <Msg key={i} msg={m} />)}
+      </div>
+      <div ref={bottomRef} />
+    </div>
+  );
+}
+
+function Msg(props: { msg: ChatMsg }) {
+  return (
+    <div className="Messege text-lg">
+      <span className="font-bold">
+        {props.msg.role.toUpperCase()}:
+      </span>
+      <br />
+      <span className="ml-8">
+        {props.msg.content}
+      </span>
+    </div>
+  );
+}
+
+function Controls(props: { setChatStateFn: any; chat: Array[ChatMsg] }) {
   const [recordState, setRecordState] = useState(false);
-  const [chatState, setChatState] = useState([{
-    role: "system",
-    content: "You are a helpful assistant.",
-  }]);
 
   function toggleRecord() {
     if (recordState == false) {
@@ -84,27 +132,61 @@ function App() {
       "body": formData,
     }).then((res) => res.json())
       .then((res) => {
-        setChatState((curState: Array<ChatMsg>) => [
+        console.log(res);
+        props.setChatStateFn((curState) => [
           ...curState,
           { "role": "user", "content": res["user-transcript"] },
         ]);
         fetch("http://100.82.51.22:8001/conversation", {
           "method": "POST",
-          "body": JSON.stringify([...chatState, {
+          "body": JSON.stringify([...props.chat, {
             "role": "user",
             "content": res["user-transcript"],
           }]),
         }).then((res) => res.json())
           .then((res) => {
-            setChatState((
+            props.setChatStateFn((curState) => [...curState, res]);
-              curState: Array<ChatMsg>,
+            console.log("attempting to play result")
-            ) => [...curState, res]);
+            playMsg(res)
-            console.log("attempting to play result");
-            playMsg(res);
           });
       });
   }
 
+  return (
+    <div className="controls self-center flex justify-evenly p-5 text-5xl border-2 border-b-0 w-1/2 max-w-screen-sm min-w-fit">
+      <button
+        onClick={() => toggleRecord()}
+        className={"inline-flex " + (recordState ? "text-red-500" : "")}
+      >
+        {recordState ? <TbPlayerStop /> : <TbMicrophone2 />}
+        {recordState ? "STOP" : "REC"}
+      </button>
+
+      <button
+        onClick={() => playRecord()}
+        className="inline-flex text-green-500"
+      >
+        <TbPlayerPlay /> PLAY
+      </button>
+
+      <button
+        onClick={() => {
+          sendAudio();
+        }}
+        className="inline-flex"
+      >
+        <TbBrandOpenai /> SEND
+      </button>
+    </div>
+  );
+}
+
+function App() {
+  const [chatState, setChatState] = useState([{
+    role: "system",
+    content: "You are a helpful assistant.",
+  }]);
+
   return (
     <>
       <div className="h-screen center flex flex-col">
@@ -113,12 +195,7 @@ function App() {
           <hr className="mx-3 border-t-4" />
         </div>
         <Feed chat={chatState} setChatStateFn={setChatState} />
-        <Controls
+        <Controls setChatStateFn={setChatState} chat={chatState} />
-          recButtonOnClick={toggleRecord}
-          recordState={recordState}
-          playButtonOnClick={playRecord}
-          sendButtonOnClick={sendAudio}
-        />
       </div>
     </>
   );

speech-speech/frontend/src/components.tsx

@@ -1,101 +0,0 @@
-import { useEffect, useRef } from "react";
-import {
-  TbBrandOpenai,
-  TbMicrophone2,
-  TbPlayerPlay,
-  TbPlayerStop,
-} from "react-icons/tb";
-
-
-export type ChatMsg = {
-  role: string;
-  content: string;
-};
-
-export function Header() {
-  return (
-    <header className="header p-3">
-      <div className="title text-5xl font-extrabold">
-        Speach to Speech AI example
-      </div>
-    </header>
-  );
-}
-
-export function Feed(props: { chat: Array<ChatMsg>; setChatStateFn: any }) {
-  const bottomRef = useRef<any>(null);
-
-  const scrollToBottom = () => {
-    if (bottomRef.current) {
-      bottomRef.current.scrollIntoView({ behavior: "smooth" });
-    }
-  };
-
-  useEffect(() => {
-    scrollToBottom();
-    console.log("scroll?");
-  });
-
-  return (
-    <div className="feed grow self-center w-5/6 max-w-screen-lg px-6 py-3 overflow-scroll">
-      <div className="content-center  space-y-2 divide-y-4">
-        {props.chat.filter((m: ChatMsg) => m.role != "system").map((
-          m: ChatMsg,
-          i: number,
-        ) => <Msg key={i} msg={m} />)}
-      </div>
-      <div ref={bottomRef} />
-    </div>
-  );
-}
-
-export function Msg(props: { msg: ChatMsg }) {
-  return (
-    <div className="Messege text-lg">
-      <span className="font-bold">
-        {props.msg.role.toUpperCase()}:
-      </span>
-      <br />
-      <span className="ml-8">
-        {props.msg.content}
-      </span>
-    </div>
-  );
-}
-
-export function Controls(
-  props: {
-    recButtonOnClick: Function;
-    recordState: Boolean;
-    playButtonOnClick: Function;
-    sendButtonOnClick: Function;
-  },
-) {
-  return (
-    <div className="controls self-center flex justify-evenly p-5 text-5xl border-2 border-b-0 w-1/2 max-w-screen-sm min-w-fit">
-      <button
-        onClick={() => props.recButtonOnClick()}
-        className={"inline-flex " + (props.recordState ? "text-red-500" : "")}
-      >
-        {props.recordState ? <TbPlayerStop /> : <TbMicrophone2 />}
-        {props.recordState ? "STOP" : "REC"}
-      </button>
-
-      <button
-        onClick={() => props.playButtonOnClick()}
-        className="inline-flex text-green-500"
-      >
-        <TbPlayerPlay /> PLAY
-      </button>
-
-      <button
-        onClick={() => {
-          props.sendButtonOnClick();
-        }}
-        className="inline-flex"
-      >
-        <TbBrandOpenai /> SEND
-      </button>
-    </div>
-  );
-}