fix(conversation): 离屏不丢回复、列表预热 WS 与非阻塞进入聊天

- 后端:文本/转写后 AI 生成改为独立任务,避免断连取消整轮;按需 TTS 等与 WS 改动
- 前端:RealtimeSession 重绑 UI 时恢复流式 buffer;列表 onPressIn/挂载预热、已有会话立即 push
- 同步会话相关类型、i18n、测试与 env/资源等累计改动

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Kevin
2026-05-08 17:28:31 +08:00
parent 5dac3efd52
commit d0c26242db
44 changed files with 1209 additions and 212 deletions

View File

@@ -174,6 +174,8 @@ interface UseRealtimeSessionOptions {
onTtsSegment?: (payload: TtsSegmentPayload) => void;
/** 用户发出下一条文本/语音成功后调用,用于恢复接受 TTS 片段(打断后丢弃迟到片段) */
onTtsPlaybackResume?: () => void;
/** 本条发送是否请求了「本轮助手朗读」,用于仅在该轮自动播放 WS TTS */
onUserSendTtsPreference?: (requestedTts: boolean) => void;
}
const MIN_RECORDING_DURATION_SEC = 1;
@@ -192,10 +194,19 @@ interface RealtimeSessionState {
/** 已发出用户消息,尚未收到助手首段流式文本(用于「正在回复」气泡) */
awaitingAssistantReply: boolean;
error: string | null;
sendText: (text: string) => void;
sendVoiceMessage: (uri: string, durationMs: number) => Promise<boolean>;
sendText: (text: string, options?: { ttsThisTurn?: boolean }) => void;
sendVoiceMessage: (
uri: string,
durationMs: number,
options?: { ttsThisTurn?: boolean },
) => Promise<boolean>;
sendEndConversation: () => void;
sendTtsCancel: () => void;
requestAssistantSegmentTts: (body: {
assistantMessageId: string;
segmentIndex: number;
segmentText?: string;
}) => boolean;
}
export function useRealtimeSession({
@@ -203,6 +214,7 @@ export function useRealtimeSession({
enabled = true,
onTtsSegment,
onTtsPlaybackResume,
onUserSendTtsPreference,
}: UseRealtimeSessionOptions): RealtimeSessionState {
const queryClient = useQueryClient();
const sessionRef = useRef<RealtimeSession | null>(null);
@@ -301,15 +313,17 @@ export function useRealtimeSession({
}, [conversationId, enabled, queryClient, foregroundResumeGeneration]);
const sendText = useCallback(
(text: string) => {
(text: string, options?: { ttsThisTurn?: boolean }) => {
if (!sessionRef.current) return;
const sent = sessionRef.current.sendText(text);
const sent = sessionRef.current.sendText(text, options);
if (!sent) {
setError('消息发送失败,连接未就绪');
return;
}
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
setAwaitingAssistantReply(true);
onTtsPlaybackResume?.();
@@ -342,11 +356,15 @@ export function useRealtimeSession({
},
);
},
[conversationId, queryClient, onTtsPlaybackResume],
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
);
const sendVoiceMessage = useCallback(
async (uri: string, durationMs: number): Promise<boolean> => {
async (
uri: string,
durationMs: number,
options?: { ttsThisTurn?: boolean },
): Promise<boolean> => {
const session = sessionRef.current;
if (!session) return false;
@@ -363,12 +381,15 @@ export function useRealtimeSession({
clientSegmentId: `${voiceSessionId}-0`,
isLast: true,
duration: durationSec,
ttsThisTurn: options?.ttsThisTurn,
});
if (!sent) {
setError('语音发送失败,连接未就绪');
return false;
}
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
setAwaitingAssistantReply(true);
const localId = `pending_voice_${Date.now()}`;
await voiceSegmentStore.recordSentSegment({
@@ -413,7 +434,7 @@ export function useRealtimeSession({
return false;
}
},
[conversationId, queryClient, onTtsPlaybackResume],
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
);
const sendEndConversation = useCallback(() => {
@@ -424,6 +445,15 @@ export function useRealtimeSession({
sessionRef.current?.sendTtsCancel();
}, []);
const requestAssistantSegmentTts = useCallback(
(body: {
assistantMessageId: string;
segmentIndex: number;
segmentText?: string;
}) => sessionRef.current?.requestAssistantSegmentTts(body) ?? false,
[],
);
return {
connectionState,
streamingMessage,
@@ -433,5 +463,6 @@ export function useRealtimeSession({
sendVoiceMessage,
sendEndConversation,
sendTtsCancel,
requestAssistantSegmentTts,
};
}