From 3c0a9b150b43f3d58eeec8591a653004994dd934 Mon Sep 17 00:00:00 2001
From: Andrei Stoica <andrei.stoica.365@gmail.com>
Date: Wed, 28 Feb 2024 11:55:34 -0500
Subject: [PATCH] audio playback for both user and assistant

---
 speech-speech/frontend/src/App.tsx        | 29 +++++++++---------
 speech-speech/frontend/src/components.tsx | 37 ++++++++++++++---------
 2 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/speech-speech/frontend/src/App.tsx b/speech-speech/frontend/src/App.tsx
index 27597e7..7319a70 100644
--- a/speech-speech/frontend/src/App.tsx
+++ b/speech-speech/frontend/src/App.tsx
@@ -2,6 +2,7 @@ import { useState } from "react";
 import { ChatMsg, Controls, Feed, Header } from "./components.tsx";
 import "./App.css";
 
+let userAudio: Array<Blob> = [];
 let audioBlobs: Array<Blob> = [];
 let streamBeingCaptured: MediaStream | null = null;
 let mediaRecorder: MediaRecorder | null = null;
@@ -51,15 +52,6 @@ function playRecord() {
   audio.play();
 }
 
-//function playMsg(msg: ChatMsg) {
-//  const audio = new Audio(
-//    "/speak?" +
-//    new URLSearchParams({ text: msg.content }),
-//  );
-//  console.log("loading audio and playing?");
-//  audio.play();
-//}
-
 function App() {
   const [recordState, setRecordState] = useState(false);
   const [chatState, setChatState] = useState([{
@@ -78,8 +70,10 @@ function App() {
   }
 
   function sendAudio() {
-    var formData = new FormData();
-    formData.append("audio", new Blob(audioBlobs, { type: "audio/webm" }));
+    let formData = new FormData();
+    let audio = new Blob(audioBlobs, { type: "audio/webm" });
+    userAudio.push(audio);
+    formData.append("audio", audio);
     fetch("/get-text", {
       "method": "POST",
       "body": formData,
@@ -87,7 +81,11 @@ function App() {
       .then((res) => {
         setChatState((curState: Array<ChatMsg>) => [
           ...curState,
-          { "role": "user", "content": res["user-transcript"] },
+          {
+            "role": "user",
+            "content": res["user-transcript"],
+            "audio": URL.createObjectURL(userAudio[userAudio.length - 1]),
+          },
         ]);
         fetch("/conversation", {
           "method": "POST",
@@ -99,9 +97,10 @@ function App() {
           .then((res) => {
             setChatState((
               curState: Array<ChatMsg>,
-            ) => [...curState, res]);
-           // console.log("attempting to play result");
-            //playMsg(res);
+            ) => [...curState, {
+              ...res,
+              "audio": "/speak?" + new URLSearchParams({ text: res.content }),
+            }]);
           });
       });
   }
diff --git a/speech-speech/frontend/src/components.tsx b/speech-speech/frontend/src/components.tsx
index d1cd6e8..3ce115a 100644
--- a/speech-speech/frontend/src/components.tsx
+++ b/speech-speech/frontend/src/components.tsx
@@ -9,7 +9,7 @@ import {
 export type ChatMsg = {
   role: string;
   content: string;
-  audio: any = null;
+  audio?: string;
 };
 
 export function Header() {
@@ -50,18 +50,23 @@ export function Feed(props: { chat: Array<ChatMsg>; setChatStateFn: any }) {
 }
 
 export function Msg(props: { msg: ChatMsg }) {
-  let audio;
-  if (props.msg.role == "assistant") {
-    audio = (
-      <audio
-        controls
-        autoPlay
-        src={"/speak?" + new URLSearchParams({ text: props.msg.content })}
-      />
-    );
-  } else if (props.msg.audio){
-    <div />;
-  }
+  //let audio;
+  //if (props.msg.role == "assistant") {
+  //  audio = (
+  //    <audio
+  //      controls
+  //      autoPlay
+  //      src={"/speak?" + new URLSearchParams({ text: props.msg.content })}
+  //    />
+  //  );
+  //} else if (props.msg.audio) {
+  //  audio = (
+  //    <audio
+  //      controls
+  //      src={props.msg.audio}
+  //    />
+  //  );
+  //}
   return (
     <div className="Messege text-lg">
       <span className="font-bold">
@@ -71,7 +76,11 @@ export function Msg(props: { msg: ChatMsg }) {
       <span className="ml-8">
         {props.msg.content}
       </span>
-      {audio}
+      <audio
+        controls
+        autoPlay={props.msg.role == "assistant"}
+        src={props.msg.audio}
+      />
     </div>
   );
 }