- {[...Array(BAR_COUNT)].map((_, index) => (
-
- ))}
+
+
+ {messages.map((msg, index) => (
+
+
+ {msg.role === "user" ? "You" : "AI"}:
+
+
+ {msg.content}
+
-
+ ))}
+ {isLoading && (
+
+ )}
-
- {/* 底部状态信息 */}
-
-
支持唤醒词："你好千问"
-
-
- {/* 呼吸圆点指示器 */}
-
- {/* 扩散波纹效果 */}
- {isListening && (
-
- )}
-
-
{getStatusText()}
-
-
- {/* 音频播放 */}
-
+
+
);
};
-export default VoiceAssistant;
+export default VoiceAssistant;
\ No newline at end of file
diff --git a/src/components/VoiceAssistant1.0.tsx b/src/components/VoiceAssistant1.0.tsx
new file mode 100644
index 0000000..325afb8
--- /dev/null
+++ b/src/components/VoiceAssistant1.0.tsx
@@ -0,0 +1,420 @@
+import { useState, useRef, useCallback } from "react";
+
+interface ProcessState {
+ recording: boolean;
+ transcribing: boolean;
+ generating: boolean;
+ synthesizing: boolean;
+ error?: string;
+ thinking: boolean;
+ speaking: boolean;
+}
+
+interface VoiceAssistantProps {
+ greeting: string;
+}
+
+const ANALYSER_FFT_SIZE = 128;
+const VOLUME_SENSITIVITY = 1.5;
+const SMOOTHING_FACTOR = 0.7;
+const BAR_COUNT = 12;
+
+const VoiceAssistant = ({ greeting }: VoiceAssistantProps) => {
+ const [isListening, setIsListening] = useState(false);
+ const [processState, setProcessState] = useState
({
+ recording: false,
+ transcribing: false,
+ generating: false,
+ synthesizing: false,
+ error: undefined,
+ thinking: false,
+ speaking: false,
+ });
+ const [asrText, setAsrText] = useState("");
+ const [answerText, setAnswerText] = useState("");
+ const mediaRecorder = useRef(null);
+ const audioChunks = useRef([]);
+ const audioElement = useRef(null);
+ const barsRef = useRef(null);
+ const mediaStreamRef = useRef(null);
+ const audioContextRef = useRef(null);
+ const analyserRef = useRef(null);
+ const animationFrameRef = useRef(null);
+ const dataArrayRef = useRef(null);
+ const lastValuesRef = useRef(new Array(BAR_COUNT).fill(10));
+ const updateState = (newState: Partial) => {
+ setProcessState((prev) => ({ ...prev, ...newState }));
+ };
+
+ const cleanupAudio = useCallback(async () => {
+ mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
+ if (audioContextRef.current?.state !== "closed") {
+ await audioContextRef.current?.close();
+ }
+ if (animationFrameRef.current) {
+ cancelAnimationFrame(animationFrameRef.current);
+ animationFrameRef.current = null;
+ }
+ }, []);
+ const initializeAudioContext = useCallback(() => {
+ const AudioContextClass =
+ window.AudioContext || (window as any).webkitAudioContext;
+ audioContextRef.current = new AudioContextClass();
+ analyserRef.current = audioContextRef.current.createAnalyser();
+ analyserRef.current.fftSize = ANALYSER_FFT_SIZE;
+ analyserRef.current.smoothingTimeConstant = SMOOTHING_FACTOR;
+ dataArrayRef.current = new Uint8Array(
+ analyserRef.current.frequencyBinCount
+ );
+ }, []);
+
+ const startRecording = async () => {
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: { sampleRate: 16000, channelCount: 1, sampleSize: 16 },
+ });
+
+ mediaRecorder.current = new MediaRecorder(stream);
+ audioChunks.current = [];
+
+ mediaRecorder.current.ondataavailable = (e) => {
+ audioChunks.current.push(e.data);
+ };
+
+ mediaRecorder.current.start(500);
+ updateState({ recording: true, error: undefined });
+ } catch (err) {
+ updateState({ error: "麦克风访问失败，请检查权限设置" });
+ }
+ };
+
+ const stopRecording = async () => {
+ if (!mediaRecorder.current) return;
+ mediaRecorder.current.stop();
+ // 更新状态为未录音
+ updateState({ recording: false });
+ mediaRecorder.current.onstop = async () => {
+ try {
+ const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" });
+ await processAudio(audioBlob);
+ } finally {
+ audioChunks.current = [];
+ }
+ };
+ };
+
+ /*const processAudio = async (audioBlob: Blob) => {
+ // 处理音频的函数
+ const formData = new FormData();
+ formData.append("audio", audioBlob, "recording.wav");
+ try {
+ updateState({ transcribing: true }); // 设置转录状态为true
+ // 发送请求到后端
+ const asrResponse = await fetch("http://localhost:5000/asr", {
+ method: "POST",
+ body: formData,
+ });
+ // 如果请求失败,则抛出错误
+ if (!asrResponse.ok) throw new Error("语音识别失败");
+ // 获取后端返回的文本
+ const asrData = await asrResponse.json();
+ setAsrText(asrData.asr_text);
+ updateState({ transcribing: false, thinking: true });
+
+ // 发送请求到后端，生成回答
+ const generateResponse = await fetch("http://localhost:5000/generate", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({ asr_text: asrData.asr_text }),
+ });
+
+ if (!generateResponse.ok) throw new Error("生成回答失败");
+
+ const generateData = await generateResponse.json(); //获取生成的回答，设置为answerText
+ setAnswerText(generateData.answer_text);
+ updateState({ thinking: false, synthesizing: true });
+
+ // 播放合成的音频，增加可视化效果
+ if (audioElement.current) {
+ //设置说话状态
+ updateState({ synthesizing: false, speaking: true }); // 替代setIsSpeaking(true)
+ initializeAudioContext(); // 初始化音频上下文
+ // 播放合成的音频
+ //audioElement.current.src = `http://localhost:5000${generateData.audio_url}`;
+ const audio = new Audio(
+ `http://localhost:5000${generateData.audio_url}`
+ ); // 创建音频元素
+ const source = audioContextRef.current!.createMediaElementSource(audio); // 创建音频源
+ source.connect(analyserRef.current!); // 连接到分析器
+ analyserRef.current!.connect(audioContextRef.current!.destination); // 连接到目标
+ //播放结束设置说话状态为false
+ audio.onended = () => {
+ updateState({ speaking: false }); // 替代setIsSpeaking(false)
+ };
+ try {
+ await audio.play(); // 播放音频
+ startVisualization(); // 开始可视化效果
+ } catch (err) {
+ console.error("播放失败:", err);
+ updateState({ error: "音频播放失败" });
+ }
+ }
+ } catch (err) {
+ updateState({ error: err instanceof Error ? err.message : "未知错误" });
+ } finally {
+ updateState({
+ transcribing: false,
+ generating: false,
+ synthesizing: false,
+ });
+ }
+ };*/
+ const processAudio = async (audioBlob: Blob) => {
+ const formData = new FormData();
+ formData.append("audio", audioBlob, "recording.wav");
+
+ try {
+ updateState({ transcribing: true });
+
+ // Step 1: 语音识别
+ const asrResponse = await fetch("http://localhost:5000/asr", {
+ method: "POST",
+ body: formData,
+ });
+ if (!asrResponse.ok) throw new Error("语音识别失败");
+ const asrData = await asrResponse.json();
+ setAsrText(asrData.asr_text);
+ updateState({ transcribing: false, thinking: true });
+
+ // Step 2: 获取大模型回复（新增独立请求）
+ const generateTextResponse = await fetch("http://localhost:5000/generate_text", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ asr_text: asrData.asr_text }),
+ });
+ if (!generateTextResponse.ok) throw new Error("生成回答失败");
+ const textData = await generateTextResponse.json();
+ setAnswerText(textData.answer_text); // 立即显示回复文本
+ updateState({ thinking: false, synthesizing: true });
+
+ // Step 3: 单独请求语音合成（新增）
+ const generateAudioResponse = await fetch("http://localhost:5000/generate_audio", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ answer_text: textData.answer_text }),
+ });
+ if (!generateAudioResponse.ok) throw new Error("语音合成失败");
+ const audioData = await generateAudioResponse.json();
+
+ // 播放音频
+ if (audioElement.current) {
+ updateState({ synthesizing: false, speaking: true });
+ initializeAudioContext();
+
+ const audio = new Audio(`http://localhost:5000${audioData.audio_url}`);
+ const source = audioContextRef.current!.createMediaElementSource(audio);
+ source.connect(analyserRef.current!);
+ analyserRef.current!.connect(audioContextRef.current!.destination);
+
+ audio.onended = () => {
+ updateState({ speaking: false });
+ };
+
+ try {
+ await audio.play();
+ startVisualization();
+ } catch (err) {
+ console.error("播放失败:", err);
+ updateState({ error: "音频播放失败" });
+ }
+ }
+ } catch (err) {
+ updateState({ error: err instanceof Error ? err.message : "未知错误" });
+ } finally {
+ updateState({
+ transcribing: false,
+ generating: false,
+ synthesizing: false,
+ });
+ }
+ };
+
+ // 状态提示更新
+ const getStatusText = () => {
+ if (processState.error) return processState.error;
+ if (processState.recording) return "请说... 🎤";
+ if (processState.transcribing) return "识别音频中... 🔍";
+ if (processState.thinking) return "正在思考中... 💡";
+ if (processState.synthesizing) return "合成语音中... 🎶"; // 更新状态提示
+ if (processState.speaking) return "说话中... 🗣📢";
+ return "对话未开始🎙️";
+ };
+
+ /*const getStatusText = () => {
+ if (processState.error) return processState.error;
+ if (processState.recording) return "请说... 🎤"; //录音
+ if (processState.transcribing) return "识别音频中... 🔍"; //语音转文字
+ if (processState.thinking) return "正在思考中... 💡"; // 等待AI回复
+ if (processState.generating) return "生成回答中... 💡"; // AI以文字形式回复中//未使用
+ if (processState.synthesizing) return "整理话语中... 🎶"; //收到AI回复，正在合成语音//未使用
+ if (processState.speaking) return "说话中... 🗣📢"; // 播放合成后的语音
+ return "对话未开始🎙️";
+ };*/
+
+ const startVisualization = useCallback(() => {
+ if (!analyserRef.current || !dataArrayRef.current || !barsRef.current) {
+ console.warn("可视化组件未就绪");
+ return;
+ }
+
+ if (animationFrameRef.current) {
+ cancelAnimationFrame(animationFrameRef.current);
+ animationFrameRef.current = null;
+ }
+
+ const bufferLength = analyserRef.current.frequencyBinCount;
+ const updateBars = () => {
+ try {
+ analyserRef.current!.getByteFrequencyData(dataArrayRef.current!);
+
+ const bars = barsRef.current!.children;
+ for (let i = 0; i < bars.length; i++) {
+ const bar = bars[i] as HTMLElement;
+ const dataIndex = Math.floor((i / BAR_COUNT) * (bufferLength / 2));
+ const rawValue =
+ (dataArrayRef.current![dataIndex] / 255) * 100 * VOLUME_SENSITIVITY;
+
+ const smoothValue = Math.min(
+ 100,
+ Math.max(10, rawValue * 0.6 + lastValuesRef.current[i] * 0.4)
+ );
+ lastValuesRef.current[i] = smoothValue;
+
+ bar.style.cssText = `
+ height: ${smoothValue}%;
+ transform: scaleY(${0.8 + (smoothValue / 100) * 0.6});
+ transition: ${i === 0 ? "none" : "height 50ms linear"};
+ `;
+ }
+
+ animationFrameRef.current = requestAnimationFrame(updateBars);
+ } catch (err) {
+ console.error("可视化更新失败:", err);
+ }
+ };
+
+ animationFrameRef.current = requestAnimationFrame(updateBars);
+ }, [analyserRef, dataArrayRef, barsRef]);
+
+ // 切换监听状态
+ const toggleListening = useCallback(async () => {
+ if (isListening) {
+ // 如果正在监听
+ await cleanupAudio(); // 清理现有音频
+ } else {
+ // 否则
+ try {
+ // 尝试
+ await cleanupAudio(); // 清理现有音频
+ initializeAudioContext(); // 初始化音频上下文
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: { noiseSuppression: true, echoCancellation: true },
+ });
+ mediaStreamRef.current = stream; // 设置媒体流
+ const source = audioContextRef.current!.createMediaStreamSource(stream);
+ source.connect(analyserRef.current!); // 只连接到分析器，不连接到目标
+ //analyserRef.current!.connect(audioContextRef.current!.destination); // 连接到目标
+ startVisualization(); // 开始可视化
+ } catch (err) {
+ console.error("初始化失败:", err);
+ updateState({ error: "音频初始化失败" });
+ }
+ }
+ setIsListening((prev) => !prev);
+ }, [isListening, cleanupAudio, initializeAudioContext, startVisualization]);
+
+ return (
+
+ {/* 问候语 */}
+
{greeting}
+ {/* 较小较细的字体显示{asrText || "等待语音输入..."}*/}
+
{asrText || "等待中..."}
+ {/*较小较细的字体显示{answerText || "等待生成回答..."}*/}
+
+ {answerText || "AI助手待命中"}
+
+
+ {/* 音频波形可视化 */}
+
+
+
+
+ {/* 底部状态信息 */}
+
+
支持唤醒词："你好千问"
+
+
+ {/* 呼吸圆点指示器 */}
+
+ {/* 扩散波纹效果 */}
+ {isListening && (
+
+ )}
+
+
{getStatusText()}
+
+
+ {/* 音频播放 */}
+
+