71bf62166e
- useRealtimeSession：恢复 uiRef 同步与 AppState 后台/前台重连，收窄长连 effect 依赖，修正 send API 类型 - 会话页：用派生条件控制 AssistantTypingBubble，避免缓存已有助手末条仍显示 typing - RealtimeSession：在 commit 落缓存时统一 clearAssistantPendingUi，避免漏掉流式完成帧 Co-authored-by: Cursor <cursoragent@cursor.com>
514 lines
16 KiB
TypeScript
514 lines
16 KiB
TypeScript
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||
import { File, Paths } from 'expo-file-system';
|
||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||
import { AppState, type AppStateStatus } from 'react-native';
|
||
|
||
import i18n from '@/i18n';
|
||
import type { TopicSuggestion, WsConnectionState } from '@/core/ws/types';
|
||
|
||
import { conversationApi } from './api';
|
||
import {
|
||
acquireBackgroundConversationWs,
|
||
disposeAllBackgroundConversationWs,
|
||
disposeBackgroundConversationWs,
|
||
releaseConversationWsUi,
|
||
} from './conversation-ws-background-pool';
|
||
import { conversationMessagesRepository } from './conversation-messages-repository';
|
||
import { conversationKeys } from './query-keys';
|
||
import { takePreparedRealtimeSession } from './prepared-session-registry';
|
||
import {
|
||
type ErrorCallback,
|
||
type RealtimeSessionUiOwner,
|
||
type StreamingTextCallback,
|
||
type TtsSegmentPayload,
|
||
type RealtimeSession,
|
||
} from './realtime-session';
|
||
import {
|
||
type ConversationListItem,
|
||
type MessageItem,
|
||
type StreamingAgentMessage,
|
||
} from './types';
|
||
import { voiceSegmentStore } from '@/features/voice/voice-segment-store';
|
||
|
||
/** Expo `File` 需要规范 `file://` URI；部分录音 API 会返回裸绝对路径。 */
|
||
function ensureFileUri(uri: string): string {
|
||
const u = uri.trim();
|
||
if (u.startsWith('file://')) return u;
|
||
if (u.startsWith('/')) return `file://${u}`;
|
||
return u;
|
||
}
|
||
|
||
function guessAudioExtension(uri: string): string {
|
||
const pathOnly = uri.split('?')[0] ?? uri;
|
||
const m = /\.[^/.]+$/u.exec(pathOnly);
|
||
return m ? m[0] : '.m4a';
|
||
}
|
||
|
||
/**
|
||
* 使用主包 `File`/`Paths`（见 Expo 文档：新 File 与旧 readAsStringAsync 互操作示例）。
|
||
* 先 copy 到 cache 下唯一文件名再 `base64()`，避免直接读源路径时偶发读到陈旧/错误内容。
|
||
*/
|
||
async function readRecordingPayload(uri: string): Promise<string> {
|
||
const resolved = ensureFileUri(uri);
|
||
const source = new File(resolved);
|
||
if (!source.exists) {
|
||
throw new Error('recording file missing');
|
||
}
|
||
const stagedName = `voice-upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}${guessAudioExtension(resolved)}`;
|
||
const staged = new File(Paths.cache, stagedName);
|
||
try {
|
||
source.copy(staged);
|
||
} catch {
|
||
return await source.base64();
|
||
}
|
||
try {
|
||
return await staged.base64();
|
||
} finally {
|
||
try {
|
||
staged.delete();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─── Query hooks ───
|
||
|
||
// TODO: 连接不上后端时 isLoading 可能一直为 true，需加超时或展示错误态
|
||
export function useConversations() {
|
||
return useQuery({
|
||
queryKey: conversationKeys.lists(),
|
||
queryFn: () => conversationApi.list(),
|
||
});
|
||
}
|
||
|
||
export function useConversationDetail(conversationId: string) {
|
||
return useQuery({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
queryFn: () => conversationApi.detail(conversationId),
|
||
enabled: !!conversationId,
|
||
});
|
||
}
|
||
|
||
export function useMessages(conversationId: string) {
|
||
return useQuery({
|
||
queryKey: conversationKeys.messages(conversationId),
|
||
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
|
||
enabled: !!conversationId,
|
||
});
|
||
}
|
||
|
||
// ─── Mutation hooks ───
|
||
|
||
export function useCreateConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: () => conversationApi.create(),
|
||
onSuccess: (newConversation) => {
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
const startedMs = Date.parse(newConversation.started_at);
|
||
const now = Date.now();
|
||
const item: ConversationListItem = {
|
||
id: newConversation.id,
|
||
title: i18n.t('agentName', { ns: 'conversation' }),
|
||
avatarUrl: null,
|
||
latestMessagePreview: '',
|
||
latestMessageTime: now,
|
||
startedAt: Number.isFinite(startedMs) ? startedMs : now,
|
||
unreadCount: 0,
|
||
isDefaultAssistant: true,
|
||
hasUserMessage: false,
|
||
};
|
||
return [item, ...(old ?? [])];
|
||
},
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
export function useDeleteConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: (conversationId: string) =>
|
||
conversationApi.delete(conversationId),
|
||
onSuccess: async (_, conversationId) => {
|
||
disposeBackgroundConversationWs(conversationId);
|
||
await voiceSegmentStore.clearConversation(conversationId);
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => old?.filter((item) => item.id !== conversationId),
|
||
);
|
||
queryClient.removeQueries({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
});
|
||
queryClient.removeQueries({
|
||
queryKey: conversationKeys.messages(conversationId),
|
||
});
|
||
},
|
||
});
|
||
}
|
||
|
||
export function useEndConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: (conversationId: string) => conversationApi.end(conversationId),
|
||
onSuccess: (_, conversationId) => {
|
||
queryClient.invalidateQueries({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
});
|
||
queryClient.invalidateQueries({
|
||
queryKey: conversationKeys.lists(),
|
||
});
|
||
},
|
||
});
|
||
}
|
||
|
||
// ─── Realtime session hook ───
|
||
|
||
interface UseRealtimeSessionOptions {
|
||
conversationId: string;
|
||
enabled?: boolean;
|
||
onTtsSegment?: (payload: TtsSegmentPayload) => void;
|
||
/** 用户发出下一条文本/语音成功后调用，用于恢复接受 TTS 片段（打断后丢弃迟到片段） */
|
||
onTtsPlaybackResume?: () => void;
|
||
/** 本条发送是否请求了「本轮助手朗读」，用于仅在该轮自动播放 WS TTS */
|
||
onUserSendTtsPreference?: (requestedTts: boolean) => void;
|
||
}
|
||
|
||
const MIN_RECORDING_DURATION_SEC = 1;
|
||
|
||
function generateUUID(): string {
|
||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||
const r = (Math.random() * 16) | 0;
|
||
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
||
return v.toString(16);
|
||
});
|
||
}
|
||
|
||
interface RealtimeSessionState {
|
||
connectionState: WsConnectionState;
|
||
streamingMessage: StreamingAgentMessage | null;
|
||
/** 已发出用户消息，尚未收到助手首段流式文本（用于「正在回复」气泡） */
|
||
awaitingAssistantReply: boolean;
|
||
error: string | null;
|
||
/** 服务端下发的 quick-start 话题 chips；用户首次发文本/语音后清空 */
|
||
topicSuggestions: TopicSuggestion[];
|
||
dismissTopicSuggestions: () => void;
|
||
sendText: (text: string, options?: { ttsThisTurn?: boolean }) => void;
|
||
sendVoiceMessage: (
|
||
uri: string,
|
||
durationMs: number,
|
||
options?: { ttsThisTurn?: boolean },
|
||
) => Promise<boolean>;
|
||
sendEndConversation: () => void;
|
||
sendTtsCancel: () => void;
|
||
requestAssistantSegmentTts: (body: {
|
||
assistantMessageId: string;
|
||
segmentIndex: number;
|
||
segmentText?: string;
|
||
}) => boolean;
|
||
}
|
||
|
||
export function useRealtimeSession({
|
||
conversationId,
|
||
enabled = true,
|
||
onTtsSegment,
|
||
onTtsPlaybackResume,
|
||
onUserSendTtsPreference,
|
||
}: UseRealtimeSessionOptions): RealtimeSessionState {
|
||
const queryClient = useQueryClient();
|
||
const sessionRef = useRef<RealtimeSession | null>(null);
|
||
const uiOwnerRef = useRef<RealtimeSessionUiOwner>(
|
||
Symbol('conversation-screen-ui'),
|
||
);
|
||
const uiRef = useRef({
|
||
handleStreamingText: (() => {}) as StreamingTextCallback,
|
||
handleError: (() => {}) as ErrorCallback,
|
||
onTtsSegment: undefined as
|
||
| ((payload: TtsSegmentPayload) => void)
|
||
| undefined,
|
||
});
|
||
|
||
const [connectionState, setConnectionState] =
|
||
useState<WsConnectionState>('disconnected');
|
||
const [streamingMessage, setStreamingMessage] =
|
||
useState<StreamingAgentMessage | null>(null);
|
||
const [awaitingAssistantReply, setAwaitingAssistantReply] = useState(false);
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [topicSuggestions, setTopicSuggestions] = useState<TopicSuggestion[]>(
|
||
[],
|
||
);
|
||
|
||
const [foregroundResumeGeneration, setForegroundResumeGeneration] =
|
||
useState(0);
|
||
const needsResumeAfterBackgroundRef = useRef(false);
|
||
|
||
const handleStreamingText: StreamingTextCallback = useCallback(
|
||
(text, isComplete) => {
|
||
if (text.trim().length > 0) {
|
||
setAwaitingAssistantReply(false);
|
||
}
|
||
if (isComplete) {
|
||
setStreamingMessage(null);
|
||
setAwaitingAssistantReply(false);
|
||
return;
|
||
}
|
||
/**
|
||
* 空文本 + 未完成时不能写成 `{text: '', isComplete: false}`：
|
||
* UI 会渲染一只空 `StreamingBubbles`（pulsing 气泡 + 光标），看上去与
|
||
* 「正在回复…」typing 气泡难以区分，且会一直挂在底部不消失。
|
||
*/
|
||
if (text.length === 0) {
|
||
return;
|
||
}
|
||
setStreamingMessage({ text, isComplete });
|
||
},
|
||
[],
|
||
);
|
||
|
||
const handleError: ErrorCallback = useCallback((message) => {
|
||
setAwaitingAssistantReply(false);
|
||
setError(message);
|
||
}, []);
|
||
|
||
const handleTopicSuggestions = useCallback(
|
||
(payload: { suggestions: TopicSuggestion[] }) => {
|
||
setTopicSuggestions(payload.suggestions);
|
||
},
|
||
[],
|
||
);
|
||
|
||
const dismissTopicSuggestions = useCallback(() => {
|
||
setTopicSuggestions([]);
|
||
}, []);
|
||
|
||
/** 每轮 render 写入 ref：`attachUiCallbacks` 的包装函数始终读最新实现（TTS / 流式 / 错误）。 */
|
||
uiRef.current.handleStreamingText = handleStreamingText;
|
||
uiRef.current.handleError = handleError;
|
||
uiRef.current.onTtsSegment = onTtsSegment;
|
||
|
||
useEffect(() => {
|
||
if (!enabled || !conversationId) return;
|
||
|
||
const sub = AppState.addEventListener('change', (next: AppStateStatus) => {
|
||
if (next === 'background') {
|
||
needsResumeAfterBackgroundRef.current = true;
|
||
disposeAllBackgroundConversationWs();
|
||
sessionRef.current = null;
|
||
setConnectionState('disconnected');
|
||
setStreamingMessage(null);
|
||
setAwaitingAssistantReply(false);
|
||
} else if (next === 'active' && needsResumeAfterBackgroundRef.current) {
|
||
needsResumeAfterBackgroundRef.current = false;
|
||
setForegroundResumeGeneration((g) => g + 1);
|
||
}
|
||
});
|
||
|
||
return () => sub.remove();
|
||
}, [enabled, conversationId]);
|
||
|
||
/**
|
||
* 依赖保持最小：`handleStreamingText` / `onTtsSegment` 等由上方 `uiRef` 每轮 render 同步，
|
||
* 避免因父组件 `enqueue` 等导致 identity 变化而反复 teardown WS（短时 noop 回调会丢 TTS）。
|
||
*/
|
||
useEffect(() => {
|
||
if (!enabled || !conversationId) return;
|
||
|
||
const prepared = takePreparedRealtimeSession(conversationId);
|
||
const session = acquireBackgroundConversationWs(
|
||
conversationId,
|
||
queryClient,
|
||
prepared,
|
||
);
|
||
|
||
session.attachUiCallbacks(
|
||
{
|
||
onStreamingText: (text, isComplete) => {
|
||
uiRef.current.handleStreamingText(text, isComplete);
|
||
},
|
||
onTtsSegment: (payload) => uiRef.current.onTtsSegment?.(payload),
|
||
onTopicSuggestions: (payload) =>
|
||
handleTopicSuggestions({ suggestions: payload.suggestions }),
|
||
onError: (message, code) => uiRef.current.handleError(message, code),
|
||
onStateChange: setConnectionState,
|
||
},
|
||
uiOwnerRef.current,
|
||
);
|
||
|
||
sessionRef.current = session;
|
||
setConnectionState(session.getConnectionState());
|
||
|
||
return () => {
|
||
releaseConversationWsUi(session, uiOwnerRef.current);
|
||
sessionRef.current = null;
|
||
setConnectionState('disconnected');
|
||
setStreamingMessage(null);
|
||
setAwaitingAssistantReply(false);
|
||
setTopicSuggestions([]);
|
||
};
|
||
}, [conversationId, enabled, queryClient, foregroundResumeGeneration]);
|
||
|
||
const sendText = useCallback(
|
||
(text: string, options?: { ttsThisTurn?: boolean }) => {
|
||
if (!sessionRef.current) return;
|
||
|
||
const sent = sessionRef.current.sendText(text, options);
|
||
if (!sent) {
|
||
setError('消息发送失败，连接未就绪');
|
||
return;
|
||
}
|
||
|
||
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
|
||
|
||
setAwaitingAssistantReply(true);
|
||
setTopicSuggestions([]);
|
||
onTtsPlaybackResume?.();
|
||
|
||
const localId = `pending_${Date.now()}`;
|
||
|
||
queryClient.setQueryData<MessageItem[]>(
|
||
conversationKeys.messages(conversationId),
|
||
(old) => {
|
||
const msg: MessageItem = {
|
||
id: localId,
|
||
conversationId,
|
||
content: text,
|
||
senderType: 'user',
|
||
timestamp: Date.now(),
|
||
messageType: 'text',
|
||
};
|
||
return [...(old ?? []), msg];
|
||
},
|
||
);
|
||
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
if (!old) return old;
|
||
return old.map((item) =>
|
||
item.id === conversationId
|
||
? { ...item, hasUserMessage: true }
|
||
: item,
|
||
);
|
||
},
|
||
);
|
||
},
|
||
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
|
||
);
|
||
|
||
const sendVoiceMessage = useCallback(
|
||
async (
|
||
uri: string,
|
||
durationMs: number,
|
||
options?: { ttsThisTurn?: boolean },
|
||
): Promise<boolean> => {
|
||
const session = sessionRef.current;
|
||
if (!session) return false;
|
||
|
||
const durationSec = Math.round(durationMs / 1000);
|
||
if (durationSec < MIN_RECORDING_DURATION_SEC) return false;
|
||
|
||
try {
|
||
const base64 = await readRecordingPayload(uri);
|
||
if (!base64) return false;
|
||
|
||
const voiceSessionId = generateUUID();
|
||
const sent = session.sendAudioSegment(base64, 0, {
|
||
voiceSessionId,
|
||
clientSegmentId: `${voiceSessionId}-0`,
|
||
isLast: true,
|
||
duration: durationSec,
|
||
ttsThisTurn: options?.ttsThisTurn,
|
||
});
|
||
if (!sent) {
|
||
setError('语音发送失败，连接未就绪');
|
||
return false;
|
||
}
|
||
|
||
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
|
||
|
||
setAwaitingAssistantReply(true);
|
||
setTopicSuggestions([]);
|
||
const localId = `pending_voice_${Date.now()}`;
|
||
await voiceSegmentStore.recordSentSegment({
|
||
voiceSessionId,
|
||
conversationId,
|
||
fileUri: uri,
|
||
durationMs,
|
||
});
|
||
queryClient.setQueryData<MessageItem[]>(
|
||
conversationKeys.messages(conversationId),
|
||
(old) => {
|
||
const msg: MessageItem = {
|
||
id: localId,
|
||
conversationId,
|
||
content: '',
|
||
senderType: 'user',
|
||
timestamp: Date.now(),
|
||
messageType: 'voice',
|
||
voiceSessionId,
|
||
durationSeconds: durationSec,
|
||
audioUri: uri,
|
||
};
|
||
return [...(old ?? []), msg];
|
||
},
|
||
);
|
||
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
if (!old) return old;
|
||
return old.map((item) =>
|
||
item.id === conversationId
|
||
? { ...item, hasUserMessage: true }
|
||
: item,
|
||
);
|
||
},
|
||
);
|
||
onTtsPlaybackResume?.();
|
||
return true;
|
||
} catch {
|
||
setError('语音文件读取失败');
|
||
return false;
|
||
}
|
||
},
|
||
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
|
||
);
|
||
|
||
const sendEndConversation = useCallback(() => {
|
||
sessionRef.current?.sendEndConversation();
|
||
}, []);
|
||
|
||
const sendTtsCancel = useCallback(() => {
|
||
sessionRef.current?.sendTtsCancel();
|
||
}, []);
|
||
|
||
const requestAssistantSegmentTts = useCallback(
|
||
(body: {
|
||
assistantMessageId: string;
|
||
segmentIndex: number;
|
||
segmentText?: string;
|
||
}) => sessionRef.current?.requestAssistantSegmentTts(body) ?? false,
|
||
[],
|
||
);
|
||
|
||
return {
|
||
connectionState,
|
||
streamingMessage,
|
||
awaitingAssistantReply,
|
||
error,
|
||
topicSuggestions,
|
||
dismissTopicSuggestions,
|
||
sendText,
|
||
sendVoiceMessage,
|
||
sendEndConversation,
|
||
sendTtsCancel,
|
||
requestAssistantSegmentTts,
|
||
};
|
||
}
|