Merge branch 'development' into claude/agent-proactive-chat-UYHu9
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
import { api } from '@/core/api/client';
|
||||
|
||||
import type {
|
||||
AvatarPresetItem,
|
||||
ChangePasswordRequest,
|
||||
ChangePhoneRequest,
|
||||
LoginRequest,
|
||||
RegisterRequest,
|
||||
ResetPasswordRequest,
|
||||
SetAvatarPresetRequest,
|
||||
SmsLoginRequest,
|
||||
SmsRegisterRequest,
|
||||
SmsRequest,
|
||||
@@ -90,4 +92,14 @@ export const authApi = {
|
||||
uploadAvatar(file: FormData) {
|
||||
return api.post<UserInfo>(`${AUTH}/me/avatar`, { body: file });
|
||||
},
|
||||
|
||||
fetchAvatarPresets() {
|
||||
return api.get<AvatarPresetItem[]>(`${AUTH}/avatar-presets`, {
|
||||
skipAuth: true,
|
||||
});
|
||||
},
|
||||
|
||||
setAvatarPreset(body: SetAvatarPresetRequest) {
|
||||
return api.put<UserInfo>(`${AUTH}/me/avatar/preset`, { body });
|
||||
},
|
||||
} as const;
|
||||
|
||||
77
app-expo/src/features/auth/avatar-upload-form-data.ts
Normal file
77
app-expo/src/features/auth/avatar-upload-form-data.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import type * as ImagePicker from 'expo-image-picker';
|
||||
import { Platform } from 'react-native';
|
||||
|
||||
type AvatarMime = 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
|
||||
function inferMimeFromUri(uri: string): AvatarMime {
|
||||
const u = uri.toLowerCase();
|
||||
if (u.endsWith('.png')) return 'image/png';
|
||||
if (u.endsWith('.webp')) return 'image/webp';
|
||||
return 'image/jpeg';
|
||||
}
|
||||
|
||||
function coerceMime(value: string | null | undefined, uri: string): AvatarMime {
|
||||
if (
|
||||
value === 'image/jpeg' ||
|
||||
value === 'image/png' ||
|
||||
value === 'image/webp'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
return inferMimeFromUri(uri);
|
||||
}
|
||||
|
||||
function mimeToFilename(mime: AvatarMime): string {
|
||||
switch (mime) {
|
||||
case 'image/png':
|
||||
return 'avatar.png';
|
||||
case 'image/webp':
|
||||
return 'avatar.webp';
|
||||
default:
|
||||
return 'avatar.jpg';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建与后端 `POST /api/auth/me/avatar` 约定的 multipart(字段名 `file`)。
|
||||
* Native:`{ uri, name, type }`;Web:`File`,避免 RN FormData 在 Web 上不识别 `uri`。
|
||||
*/
|
||||
export async function buildAvatarUploadFormData(
|
||||
asset: ImagePicker.ImagePickerAsset,
|
||||
): Promise<FormData> {
|
||||
const uri = asset.uri;
|
||||
const mime = coerceMime(asset.mimeType, uri);
|
||||
const filename = mimeToFilename(mime);
|
||||
const form = new FormData();
|
||||
|
||||
if (Platform.OS === 'web') {
|
||||
const webFile = asset.file;
|
||||
if (
|
||||
webFile instanceof File &&
|
||||
(webFile.type === 'image/jpeg' ||
|
||||
webFile.type === 'image/png' ||
|
||||
webFile.type === 'image/webp')
|
||||
) {
|
||||
form.append(
|
||||
'file',
|
||||
webFile,
|
||||
webFile.name || mimeToFilename(coerceMime(webFile.type, uri)),
|
||||
);
|
||||
return form;
|
||||
}
|
||||
|
||||
const res = await fetch(uri);
|
||||
const blob = await res.blob();
|
||||
const type = coerceMime(blob.type, uri);
|
||||
form.append('file', new File([blob], mimeToFilename(type), { type }));
|
||||
return form;
|
||||
}
|
||||
|
||||
form.append('file', {
|
||||
uri,
|
||||
name: filename,
|
||||
type: mime,
|
||||
} as unknown as Blob);
|
||||
|
||||
return form;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { useCallback } from 'react';
|
||||
|
||||
import { AuthError } from '@/core/api/types';
|
||||
import { tokenManager } from '@/core/auth/token-manager';
|
||||
import { disposeAllBackgroundConversationWs } from '@/features/conversation/conversation-ws-background-pool';
|
||||
|
||||
import { authApi } from './api';
|
||||
import type {
|
||||
@@ -14,6 +15,7 @@ import type {
|
||||
SmsRegisterRequest,
|
||||
SmsRequest,
|
||||
TokenResponse,
|
||||
UpdateNicknameRequest,
|
||||
UserInfo,
|
||||
} from './types';
|
||||
|
||||
@@ -24,6 +26,16 @@ export const authKeys = {
|
||||
tokenCheck: ['auth', 'token-check'] as const,
|
||||
};
|
||||
|
||||
const PROFILE_QUERY_PREFIX = ['profile'] as const;
|
||||
|
||||
function syncSessionAndProfileQueries(
|
||||
queryClient: ReturnType<typeof useQueryClient>,
|
||||
user: UserInfo,
|
||||
) {
|
||||
queryClient.setQueryData(authKeys.session, user);
|
||||
queryClient.invalidateQueries({ queryKey: PROFILE_QUERY_PREFIX });
|
||||
}
|
||||
|
||||
// ─── useSession ───
|
||||
|
||||
/**
|
||||
@@ -162,6 +174,44 @@ export function useSmsCode() {
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Avatar / nickname ───
|
||||
|
||||
export function useAvatarPresets() {
|
||||
return useQuery({
|
||||
queryKey: ['avatar-presets'],
|
||||
queryFn: () => authApi.fetchAvatarPresets(),
|
||||
staleTime: 60 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useUpdateNickname() {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation({
|
||||
mutationFn: (body: UpdateNicknameRequest) => authApi.updateNickname(body),
|
||||
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
|
||||
});
|
||||
}
|
||||
|
||||
export function useUploadAvatar() {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation({
|
||||
mutationFn: (form: FormData) => authApi.uploadAvatar(form),
|
||||
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
|
||||
});
|
||||
}
|
||||
|
||||
export function useSetAvatarPreset() {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation({
|
||||
mutationFn: (presetId: string) =>
|
||||
authApi.setAvatarPreset({ preset_id: presetId }),
|
||||
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
|
||||
});
|
||||
}
|
||||
|
||||
// ─── useLogout ───
|
||||
|
||||
/**
|
||||
@@ -185,6 +235,7 @@ export function useLogout() {
|
||||
}
|
||||
},
|
||||
onSettled: async () => {
|
||||
disposeAllBackgroundConversationWs();
|
||||
await tokenManager.clearTokens();
|
||||
queryClient.clear();
|
||||
queryClient.setQueryData(authKeys.tokenCheck, false);
|
||||
|
||||
@@ -79,6 +79,15 @@ export interface UpdateNicknameRequest {
|
||||
nickname: string;
|
||||
}
|
||||
|
||||
export interface AvatarPresetItem {
|
||||
id: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface SetAvatarPresetRequest {
|
||||
preset_id: string;
|
||||
}
|
||||
|
||||
// ─── Session state ───
|
||||
|
||||
export type SessionStatus =
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { QueryClient } from '@tanstack/react-query';
|
||||
import { AppState, type AppStateStatus } from 'react-native';
|
||||
|
||||
import { RealtimeSession } from './realtime-session';
|
||||
|
||||
type Slot = { conversationId: string; session: RealtimeSession };
|
||||
|
||||
let slot: Slot | null = null;
|
||||
|
||||
/** 与常见聊天 App 一致:仅当应用进入 background 时断开长连(避免后台挂 socket);inactive 不处理以减少控制中心等短暂打断 */
|
||||
let backgroundUnsubscribe: (() => void) | null = null;
|
||||
|
||||
function installBackgroundLifecycleOnce(): void {
|
||||
if (backgroundUnsubscribe) return;
|
||||
const sub = AppState.addEventListener('change', (next: AppStateStatus) => {
|
||||
if (next === 'background') {
|
||||
disposeAllBackgroundConversationWs();
|
||||
}
|
||||
});
|
||||
backgroundUnsubscribe = () => sub.remove();
|
||||
}
|
||||
|
||||
function disposeSlot(): void {
|
||||
if (!slot) return;
|
||||
slot.session.dispose();
|
||||
slot = null;
|
||||
}
|
||||
|
||||
const offScreenUi = {
|
||||
onStreamingText: () => {},
|
||||
onTtsSegment: () => {},
|
||||
onError: () => {},
|
||||
onStateChange: () => {},
|
||||
};
|
||||
|
||||
/** 离屏:保持 WebSocket,去掉 UI 回调,避免列表页播 TTS 或对已卸载组件 setState */
|
||||
export function releaseConversationWsUi(session: RealtimeSession): void {
|
||||
session.attachUiCallbacks({
|
||||
onStreamingText: offScreenUi.onStreamingText,
|
||||
onTtsSegment: offScreenUi.onTtsSegment,
|
||||
onError: offScreenUi.onError,
|
||||
onStateChange: offScreenUi.onStateChange,
|
||||
});
|
||||
}
|
||||
|
||||
/** 删除会话等场景:关闭对应长连 */
|
||||
export function disposeBackgroundConversationWs(conversationId: string): void {
|
||||
if (slot?.conversationId === conversationId) {
|
||||
disposeSlot();
|
||||
}
|
||||
}
|
||||
|
||||
/** 登出 / 清账号:关闭池中连接 */
|
||||
export function disposeAllBackgroundConversationWs(): void {
|
||||
disposeSlot();
|
||||
}
|
||||
|
||||
/**
|
||||
* 单槽:仅保留「最近进入」的一个会话长连。换会话会dispose旧槽;同会话返回池中实例。
|
||||
*/
|
||||
export function acquireBackgroundConversationWs(
|
||||
conversationId: string,
|
||||
queryClient: QueryClient,
|
||||
prepared: RealtimeSession | null,
|
||||
): RealtimeSession {
|
||||
installBackgroundLifecycleOnce();
|
||||
if (prepared) {
|
||||
if (
|
||||
slot &&
|
||||
(slot.conversationId !== conversationId || slot.session !== prepared)
|
||||
) {
|
||||
disposeSlot();
|
||||
}
|
||||
slot = { conversationId, session: prepared };
|
||||
return prepared;
|
||||
}
|
||||
|
||||
if (slot?.conversationId === conversationId) {
|
||||
void slot.session.connect();
|
||||
return slot.session;
|
||||
}
|
||||
|
||||
disposeSlot();
|
||||
const session = new RealtimeSession({ conversationId, queryClient });
|
||||
slot = { conversationId, session };
|
||||
void session.connect();
|
||||
return session;
|
||||
}
|
||||
147
app-expo/src/features/conversation/entry-warmup.ts
Normal file
147
app-expo/src/features/conversation/entry-warmup.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import type { QueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { acquireBackgroundConversationWs } from './conversation-ws-background-pool';
|
||||
import { conversationMessagesRepository } from './conversation-messages-repository';
|
||||
import { conversationKeys } from './query-keys';
|
||||
import { registerPreparedRealtimeSession } from './prepared-session-registry';
|
||||
import { RealtimeSession } from './realtime-session';
|
||||
import type { MessageItem } from './types';
|
||||
|
||||
const OPENING_WARMUP_TIMEOUT_MS = 50_000;
|
||||
const CACHE_POLL_MS = 120;
|
||||
|
||||
function cacheHasAssistantMessage(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
): boolean {
|
||||
const data = queryClient.getQueryData<MessageItem[]>(
|
||||
conversationKeys.messages(conversationId),
|
||||
);
|
||||
return (data ?? []).some((m) => m.senderType === 'assistant');
|
||||
}
|
||||
|
||||
function waitForAssistantInCache(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
timeoutMs: number,
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const schedule = () => {
|
||||
if (cacheHasAssistantMessage(queryClient, conversationId)) {
|
||||
resolve(true);
|
||||
return;
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
setTimeout(schedule, CACHE_POLL_MS);
|
||||
};
|
||||
schedule();
|
||||
});
|
||||
}
|
||||
|
||||
export async function prefetchConversationMessages(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
): Promise<void> {
|
||||
await queryClient.prefetchQuery({
|
||||
queryKey: conversationKeys.messages(conversationId),
|
||||
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
|
||||
});
|
||||
}
|
||||
|
||||
const offscreenUiCallbacks = {
|
||||
onStreamingText: () => {},
|
||||
onTtsSegment: () => {},
|
||||
onError: () => {},
|
||||
onStateChange: () => {},
|
||||
};
|
||||
|
||||
const inflightPrewarms = new Set<string>();
|
||||
|
||||
/**
|
||||
* 列表页/卡片按下时的预热:保持后台 WS 连接,并触发消息缓存填充。
|
||||
* 与 `warmupConversationOpening` 不同:不等待开场白、不阻塞调用方,仅适用于"已有消息"的会话。
|
||||
*/
|
||||
export function prewarmConversationSession(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
): void {
|
||||
if (!conversationId) return;
|
||||
const session = acquireBackgroundConversationWs(
|
||||
conversationId,
|
||||
queryClient,
|
||||
null,
|
||||
);
|
||||
// 预热阶段没有挂载的 UI,先用空回调占位;聊天页 mount 时会重新 attach。
|
||||
session.attachUiCallbacks(offscreenUiCallbacks);
|
||||
if (inflightPrewarms.has(conversationId)) return;
|
||||
const cached = queryClient.getQueryData<MessageItem[]>(
|
||||
conversationKeys.messages(conversationId),
|
||||
);
|
||||
// 已有缓存就交给 React Query staleTime 决定是否刷新;只对首次进入做后台预取
|
||||
if (cached && cached.length > 0) return;
|
||||
inflightPrewarms.add(conversationId);
|
||||
void prefetchConversationMessages(queryClient, conversationId).finally(() => {
|
||||
inflightPrewarms.delete(conversationId);
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshConversationMessagesForWarmup(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
): Promise<void> {
|
||||
await queryClient.fetchQuery({
|
||||
queryKey: conversationKeys.messages(conversationId),
|
||||
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
|
||||
staleTime: 0,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 在会话列表阶段连接 WS 并等待首条助手开场写入 React Query;成功后挂起会话供聊天页接棒。
|
||||
* 超时或失败则 dispose,由聊天页自行重连(服务端若已写入 history 不会重复开场)。
|
||||
*/
|
||||
export async function warmupConversationOpening(
|
||||
queryClient: QueryClient,
|
||||
conversationId: string,
|
||||
): Promise<void> {
|
||||
/**
|
||||
* 先走 REST 历史预取:若 access token 已过期,API client 会在这里刷新 token;
|
||||
* 也避免 Redis/DB 已有开场白但本地缓存仍为空时继续等 WS。
|
||||
*/
|
||||
await refreshConversationMessagesForWarmup(queryClient, conversationId);
|
||||
|
||||
if (cacheHasAssistantMessage(queryClient, conversationId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const session = new RealtimeSession({
|
||||
conversationId,
|
||||
queryClient,
|
||||
});
|
||||
|
||||
session.attachUiCallbacks({
|
||||
onStreamingText: () => {},
|
||||
onTtsSegment: () => {},
|
||||
onError: () => {},
|
||||
onStateChange: () => {},
|
||||
});
|
||||
|
||||
await session.connect();
|
||||
|
||||
const ok = await waitForAssistantInCache(
|
||||
queryClient,
|
||||
conversationId,
|
||||
OPENING_WARMUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (ok) {
|
||||
registerPreparedRealtimeSession(conversationId, session);
|
||||
await prefetchConversationMessages(queryClient, conversationId);
|
||||
} else {
|
||||
session.dispose();
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,25 @@
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { File, Paths } from 'expo-file-system';
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { AppState, type AppStateStatus } from 'react-native';
|
||||
|
||||
import type { TopicSuggestion, WsConnectionState } from '@/core/ws/types';
|
||||
|
||||
import { conversationApi } from './api';
|
||||
import {
|
||||
acquireBackgroundConversationWs,
|
||||
disposeAllBackgroundConversationWs,
|
||||
disposeBackgroundConversationWs,
|
||||
releaseConversationWsUi,
|
||||
} from './conversation-ws-background-pool';
|
||||
import { conversationMessagesRepository } from './conversation-messages-repository';
|
||||
import { conversationKeys } from './query-keys';
|
||||
import { takePreparedRealtimeSession } from './prepared-session-registry';
|
||||
import {
|
||||
RealtimeSession,
|
||||
type ErrorCallback,
|
||||
type StreamingTextCallback,
|
||||
type TtsSegmentPayload,
|
||||
type RealtimeSession,
|
||||
} from './realtime-session';
|
||||
import {
|
||||
type ConversationListItem,
|
||||
@@ -126,6 +134,7 @@ export function useDeleteConversation() {
|
||||
mutationFn: (conversationId: string) =>
|
||||
conversationApi.delete(conversationId),
|
||||
onSuccess: async (_, conversationId) => {
|
||||
disposeBackgroundConversationWs(conversationId);
|
||||
await voiceSegmentStore.clearConversation(conversationId);
|
||||
queryClient.setQueryData<ConversationListItem[]>(
|
||||
conversationKeys.lists(),
|
||||
@@ -165,6 +174,8 @@ interface UseRealtimeSessionOptions {
|
||||
onTtsSegment?: (payload: TtsSegmentPayload) => void;
|
||||
/** 用户发出下一条文本/语音成功后调用,用于恢复接受 TTS 片段(打断后丢弃迟到片段) */
|
||||
onTtsPlaybackResume?: () => void;
|
||||
/** 本条发送是否请求了「本轮助手朗读」,用于仅在该轮自动播放 WS TTS */
|
||||
onUserSendTtsPreference?: (requestedTts: boolean) => void;
|
||||
}
|
||||
|
||||
const MIN_RECORDING_DURATION_SEC = 1;
|
||||
@@ -190,6 +201,11 @@ interface RealtimeSessionState {
|
||||
sendVoiceMessage: (uri: string, durationMs: number) => Promise<boolean>;
|
||||
sendEndConversation: () => void;
|
||||
sendTtsCancel: () => void;
|
||||
requestAssistantSegmentTts: (body: {
|
||||
assistantMessageId: string;
|
||||
segmentIndex: number;
|
||||
segmentText?: string;
|
||||
}) => boolean;
|
||||
}
|
||||
|
||||
export function useRealtimeSession({
|
||||
@@ -197,9 +213,17 @@ export function useRealtimeSession({
|
||||
enabled = true,
|
||||
onTtsSegment,
|
||||
onTtsPlaybackResume,
|
||||
onUserSendTtsPreference,
|
||||
}: UseRealtimeSessionOptions): RealtimeSessionState {
|
||||
const queryClient = useQueryClient();
|
||||
const sessionRef = useRef<RealtimeSession | null>(null);
|
||||
const uiRef = useRef({
|
||||
handleStreamingText: (() => {}) as StreamingTextCallback,
|
||||
handleError: (() => {}) as ErrorCallback,
|
||||
onTtsSegment: undefined as
|
||||
| ((payload: TtsSegmentPayload) => void)
|
||||
| undefined,
|
||||
});
|
||||
|
||||
const [connectionState, setConnectionState] =
|
||||
useState<WsConnectionState>('disconnected');
|
||||
@@ -211,6 +235,10 @@ export function useRealtimeSession({
|
||||
[],
|
||||
);
|
||||
|
||||
const [foregroundResumeGeneration, setForegroundResumeGeneration] =
|
||||
useState(0);
|
||||
const needsResumeAfterBackgroundRef = useRef(false);
|
||||
|
||||
const handleStreamingText: StreamingTextCallback = useCallback(
|
||||
(text, isComplete) => {
|
||||
if (text.trim().length > 0) {
|
||||
@@ -245,7 +273,8 @@ export function useRealtimeSession({
|
||||
useEffect(() => {
|
||||
if (!enabled || !conversationId) return;
|
||||
|
||||
const session = new RealtimeSession({
|
||||
const prepared = takePreparedRealtimeSession(conversationId);
|
||||
const session = acquireBackgroundConversationWs(
|
||||
conversationId,
|
||||
queryClient,
|
||||
onStreamingText: handleStreamingText,
|
||||
@@ -256,10 +285,10 @@ export function useRealtimeSession({
|
||||
});
|
||||
|
||||
sessionRef.current = session;
|
||||
session.connect();
|
||||
setConnectionState(session.getConnectionState());
|
||||
|
||||
return () => {
|
||||
session.dispose();
|
||||
releaseConversationWsUi(session);
|
||||
sessionRef.current = null;
|
||||
setConnectionState('disconnected');
|
||||
setStreamingMessage(null);
|
||||
@@ -277,15 +306,17 @@ export function useRealtimeSession({
|
||||
]);
|
||||
|
||||
const sendText = useCallback(
|
||||
(text: string) => {
|
||||
(text: string, options?: { ttsThisTurn?: boolean }) => {
|
||||
if (!sessionRef.current) return;
|
||||
|
||||
const sent = sessionRef.current.sendText(text);
|
||||
const sent = sessionRef.current.sendText(text, options);
|
||||
if (!sent) {
|
||||
setError('消息发送失败,连接未就绪');
|
||||
return;
|
||||
}
|
||||
|
||||
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
|
||||
|
||||
setAwaitingAssistantReply(true);
|
||||
setTopicSuggestions([]);
|
||||
onTtsPlaybackResume?.();
|
||||
@@ -319,11 +350,15 @@ export function useRealtimeSession({
|
||||
},
|
||||
);
|
||||
},
|
||||
[conversationId, queryClient, onTtsPlaybackResume],
|
||||
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
|
||||
);
|
||||
|
||||
const sendVoiceMessage = useCallback(
|
||||
async (uri: string, durationMs: number): Promise<boolean> => {
|
||||
async (
|
||||
uri: string,
|
||||
durationMs: number,
|
||||
options?: { ttsThisTurn?: boolean },
|
||||
): Promise<boolean> => {
|
||||
const session = sessionRef.current;
|
||||
if (!session) return false;
|
||||
|
||||
@@ -340,12 +375,15 @@ export function useRealtimeSession({
|
||||
clientSegmentId: `${voiceSessionId}-0`,
|
||||
isLast: true,
|
||||
duration: durationSec,
|
||||
ttsThisTurn: options?.ttsThisTurn,
|
||||
});
|
||||
if (!sent) {
|
||||
setError('语音发送失败,连接未就绪');
|
||||
return false;
|
||||
}
|
||||
|
||||
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
|
||||
|
||||
setAwaitingAssistantReply(true);
|
||||
setTopicSuggestions([]);
|
||||
const localId = `pending_voice_${Date.now()}`;
|
||||
@@ -391,7 +429,7 @@ export function useRealtimeSession({
|
||||
return false;
|
||||
}
|
||||
},
|
||||
[conversationId, queryClient, onTtsPlaybackResume],
|
||||
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
|
||||
);
|
||||
|
||||
const sendEndConversation = useCallback(() => {
|
||||
@@ -402,6 +440,15 @@ export function useRealtimeSession({
|
||||
sessionRef.current?.sendTtsCancel();
|
||||
}, []);
|
||||
|
||||
const requestAssistantSegmentTts = useCallback(
|
||||
(body: {
|
||||
assistantMessageId: string;
|
||||
segmentIndex: number;
|
||||
segmentText?: string;
|
||||
}) => sessionRef.current?.requestAssistantSegmentTts(body) ?? false,
|
||||
[],
|
||||
);
|
||||
|
||||
return {
|
||||
connectionState,
|
||||
streamingMessage,
|
||||
@@ -413,5 +460,6 @@ export function useRealtimeSession({
|
||||
sendVoiceMessage,
|
||||
sendEndConversation,
|
||||
sendTtsCancel,
|
||||
requestAssistantSegmentTts,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { RealtimeSession } from './realtime-session';
|
||||
|
||||
const preparedByConversationId = new Map<string, RealtimeSession>();
|
||||
|
||||
/** 列表页预热完成后挂起会话,聊天页挂载时接棒并删除登记 */
|
||||
export function registerPreparedRealtimeSession(
|
||||
conversationId: string,
|
||||
session: RealtimeSession,
|
||||
): void {
|
||||
const old = preparedByConversationId.get(conversationId);
|
||||
if (old && old !== session) {
|
||||
old.dispose();
|
||||
}
|
||||
preparedByConversationId.set(conversationId, session);
|
||||
}
|
||||
|
||||
/** 取出即视为消费;若无则返回 null */
|
||||
export function takePreparedRealtimeSession(
|
||||
conversationId: string,
|
||||
): RealtimeSession | null {
|
||||
const session = preparedByConversationId.get(conversationId);
|
||||
if (!session) return null;
|
||||
preparedByConversationId.delete(conversationId);
|
||||
return session;
|
||||
}
|
||||
|
||||
/** 预热成功后导航失败时释放挂起连接,避免僵尸 WebSocket */
|
||||
export function abandonPreparedRealtimeSession(conversationId: string): void {
|
||||
const session = preparedByConversationId.get(conversationId);
|
||||
if (!session) return;
|
||||
preparedByConversationId.delete(conversationId);
|
||||
session.dispose();
|
||||
}
|
||||
@@ -16,6 +16,13 @@ import { assistantSegmentMessageId, lastSegmentPreview } from './message-split';
|
||||
import { conversationKeys } from './query-keys';
|
||||
import type { ConversationListItem, MessageItem } from './types';
|
||||
|
||||
/** 与落库助手消息 id、会话页 `durableAssistantIdForBubble` 的 uuid 判断一致 */
|
||||
function looksLikeUuidAssistantMessageId(id: string): boolean {
|
||||
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
export type StreamingTextCallback = (text: string, isComplete: boolean) => void;
|
||||
export type ErrorCallback = (message: string, code?: string) => void;
|
||||
export type TopicSuggestionsCallback = (payload: {
|
||||
@@ -32,6 +39,8 @@ export type TtsSegmentPayload = {
|
||||
total?: number;
|
||||
/** 服务端持久化后的助手消息 id,用于与气泡 listKey / 消息 id 对齐 */
|
||||
assistantMessageId?: string;
|
||||
/** 用户点喇叭按需下发时为 true,应加入播放队列(即使未开「本轮朗读」) */
|
||||
manual?: boolean;
|
||||
};
|
||||
|
||||
interface RealtimeSessionOptions {
|
||||
@@ -65,12 +74,25 @@ export class RealtimeSession {
|
||||
private onTtsSegment?: (payload: TtsSegmentPayload) => void;
|
||||
private onTopicSuggestions?: TopicSuggestionsCallback;
|
||||
private onError?: ErrorCallback;
|
||||
private uiStateListener?: WsStateListener;
|
||||
private unsubEvent: (() => void) | null = null;
|
||||
private unsubState: (() => void) | null = null;
|
||||
|
||||
private streamingBuffer = '';
|
||||
/** 单段回复且服务端带 `assistant_message_id` 时用于落缓存 id */
|
||||
private pendingAssistantMessageId: string | null = null;
|
||||
private destroyed = false;
|
||||
|
||||
/** 本条用户消息是否请求「先 TTS 再出字」的助手轮次 */
|
||||
private assistantTurnTtsSync = false;
|
||||
private pendingTtsByKey = new Map<string, TtsSegmentPayload>();
|
||||
|
||||
private static bufferedTtsKey(
|
||||
assistantMessageId: string | undefined,
|
||||
index: number,
|
||||
): string {
|
||||
return `${assistantMessageId ?? '_'}:${index}`;
|
||||
}
|
||||
|
||||
constructor(options: RealtimeSessionOptions) {
|
||||
this.client = new WsClient(options.conversationId);
|
||||
@@ -80,11 +102,38 @@ export class RealtimeSession {
|
||||
this.onTtsSegment = options.onTtsSegment;
|
||||
this.onTopicSuggestions = options.onTopicSuggestions;
|
||||
this.onError = options.onError;
|
||||
this.uiStateListener = options.onStateChange;
|
||||
|
||||
this.unsubEvent = this.client.onEvent(this.handleEvent);
|
||||
|
||||
if (options.onStateChange) {
|
||||
this.unsubState = this.client.onStateChange(options.onStateChange);
|
||||
this.unsubState = this.client.onStateChange((state) => {
|
||||
this.uiStateListener?.(state);
|
||||
});
|
||||
}
|
||||
|
||||
/** 列表预热接棒或刷新 UI 订阅时替换回调,不重建 WebSocket */
|
||||
attachUiCallbacks(options: {
|
||||
onStreamingText?: StreamingTextCallback;
|
||||
onTtsSegment?: (payload: TtsSegmentPayload) => void;
|
||||
onError?: ErrorCallback;
|
||||
onStateChange?: WsStateListener;
|
||||
}): void {
|
||||
if (this.destroyed) return;
|
||||
if (options.onStreamingText !== undefined) {
|
||||
this.onStreamingText = options.onStreamingText;
|
||||
}
|
||||
if (options.onTtsSegment !== undefined) {
|
||||
this.onTtsSegment = options.onTtsSegment;
|
||||
}
|
||||
if (options.onError !== undefined) {
|
||||
this.onError = options.onError;
|
||||
}
|
||||
if (options.onStateChange !== undefined) {
|
||||
this.uiStateListener = options.onStateChange;
|
||||
options.onStateChange(this.client.getState());
|
||||
}
|
||||
if (!this.assistantTurnTtsSync && this.streamingBuffer.trim().length > 0) {
|
||||
this.onStreamingText?.(this.streamingBuffer, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,15 +146,20 @@ export class RealtimeSession {
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
this.flushStreamingBufferIfPending();
|
||||
this.resetAssistantTtsSyncState();
|
||||
this.unsubEvent?.();
|
||||
this.unsubState?.();
|
||||
this.client.dispose();
|
||||
}
|
||||
|
||||
/** Returns true if the message was sent over the socket. */
|
||||
sendText(text: string): boolean {
|
||||
return this.client.sendText(text);
|
||||
sendText(text: string, options?: { ttsThisTurn?: boolean }): boolean {
|
||||
const tts = !!options?.ttsThisTurn;
|
||||
this.assistantTurnTtsSync = tts;
|
||||
return this.client.sendText(text, { ttsThisTurn: tts });
|
||||
}
|
||||
|
||||
sendAudioSegment(
|
||||
@@ -116,8 +170,11 @@ export class RealtimeSession {
|
||||
clientSegmentId?: string;
|
||||
isLast?: boolean;
|
||||
duration?: number;
|
||||
ttsThisTurn?: boolean;
|
||||
},
|
||||
): boolean {
|
||||
const tts = !!options?.ttsThisTurn;
|
||||
this.assistantTurnTtsSync = tts;
|
||||
return this.client.send({
|
||||
type: 'audio_segment',
|
||||
data: {
|
||||
@@ -127,6 +184,7 @@ export class RealtimeSession {
|
||||
client_segment_id: options?.clientSegmentId,
|
||||
is_last: options?.isLast,
|
||||
duration: options?.duration,
|
||||
...(options?.ttsThisTurn === true ? { tts_this_turn: true } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -137,6 +195,7 @@ export class RealtimeSession {
|
||||
|
||||
/** 通知服务端停止当前轮次后续 TTS 合成与下发(与客户端 stop 队列配合) */
|
||||
sendTtsCancel(): boolean {
|
||||
this.resetAssistantTtsSyncState();
|
||||
return this.client.sendTtsCancel();
|
||||
}
|
||||
|
||||
@@ -144,8 +203,40 @@ export class RealtimeSession {
|
||||
return this.client.getState();
|
||||
}
|
||||
|
||||
requestAssistantSegmentTts(body: {
|
||||
assistantMessageId: string;
|
||||
segmentIndex: number;
|
||||
segmentText?: string;
|
||||
}): boolean {
|
||||
return this.client.sendTtsRequest(body);
|
||||
}
|
||||
|
||||
// ─── Internal ───
|
||||
|
||||
private resetAssistantTtsSyncState(): void {
|
||||
this.assistantTurnTtsSync = false;
|
||||
this.pendingTtsByKey.clear();
|
||||
}
|
||||
|
||||
private flushBufferedTtsIfSync(
|
||||
assistantMessageId: string | undefined,
|
||||
index: number,
|
||||
): void {
|
||||
if (!this.assistantTurnTtsSync) return;
|
||||
const key = RealtimeSession.bufferedTtsKey(assistantMessageId, index);
|
||||
const payload = this.pendingTtsByKey.get(key);
|
||||
if (payload) {
|
||||
this.pendingTtsByKey.delete(key);
|
||||
this.onTtsSegment?.(payload);
|
||||
}
|
||||
}
|
||||
|
||||
private finishAssistantTurnIfLastSegment(index: number, total: number): void {
|
||||
if (index >= total - 1) {
|
||||
this.resetAssistantTtsSyncState();
|
||||
}
|
||||
}
|
||||
|
||||
private handleEvent: WsEventListener = (event: WsEvent) => {
|
||||
if (event.kind === 'agent_response') {
|
||||
this.handleAgentChunk(event);
|
||||
@@ -155,14 +246,26 @@ export class RealtimeSession {
|
||||
if (event.kind === 'tts_audio_received') {
|
||||
const b64 = event.audioBase64?.trim();
|
||||
const url = event.audioUrl?.trim();
|
||||
if (b64 || url) {
|
||||
this.onTtsSegment?.({
|
||||
audioBase64: b64 || undefined,
|
||||
audioUrl: url || undefined,
|
||||
index: event.index,
|
||||
total: event.total,
|
||||
assistantMessageId: event.assistantMessageId,
|
||||
});
|
||||
if (!b64 && !url) {
|
||||
return;
|
||||
}
|
||||
const payload: TtsSegmentPayload = {
|
||||
audioBase64: b64 || undefined,
|
||||
audioUrl: url || undefined,
|
||||
index: event.index,
|
||||
total: event.total,
|
||||
assistantMessageId: event.assistantMessageId,
|
||||
manual: event.manual,
|
||||
};
|
||||
if (this.assistantTurnTtsSync && !payload.manual) {
|
||||
const idx = event.index ?? 0;
|
||||
const key = RealtimeSession.bufferedTtsKey(
|
||||
event.assistantMessageId,
|
||||
idx,
|
||||
);
|
||||
this.pendingTtsByKey.set(key, payload);
|
||||
} else {
|
||||
this.onTtsSegment?.(payload);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -179,6 +282,7 @@ export class RealtimeSession {
|
||||
handleWsEvent(this.queryClient, event);
|
||||
|
||||
if (event.kind === 'session_error') {
|
||||
this.resetAssistantTtsSyncState();
|
||||
this.onError?.(event.message, event.code);
|
||||
}
|
||||
};
|
||||
@@ -202,14 +306,19 @@ export class RealtimeSession {
|
||||
|
||||
const total = event.total ?? 1;
|
||||
const index = event.index ?? 0;
|
||||
const sync = this.assistantTurnTtsSync;
|
||||
|
||||
if (total > 1) {
|
||||
const id =
|
||||
event.assistantMessageId != null
|
||||
? assistantSegmentMessageId(event.assistantMessageId, index)
|
||||
: `${this.conversationId}_agent_${Date.now()}_${index}`;
|
||||
if (sync) {
|
||||
this.flushBufferedTtsIfSync(event.assistantMessageId, index);
|
||||
}
|
||||
this.commitOneAssistantMessage(event.text, id);
|
||||
this.onStreamingText?.(event.text, true);
|
||||
this.finishAssistantTurnIfLastSegment(index, total);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -218,18 +327,30 @@ export class RealtimeSession {
|
||||
}
|
||||
|
||||
this.streamingBuffer += event.text;
|
||||
|
||||
// 与 coerced index/total 对齐:若服务端只带 text、省略 index/total,旧逻辑会 isComplete=false,永远不落库
|
||||
const isComplete = index >= total - 1;
|
||||
|
||||
this.onStreamingText?.(this.streamingBuffer, isComplete);
|
||||
if (!sync) {
|
||||
this.onStreamingText?.(this.streamingBuffer, isComplete);
|
||||
}
|
||||
|
||||
if (isComplete) {
|
||||
const assistantId =
|
||||
event.assistantMessageId ?? this.pendingAssistantMessageId;
|
||||
const id =
|
||||
this.pendingAssistantMessageId ??
|
||||
`${this.conversationId}_agent_${Date.now()}`;
|
||||
this.commitStreamingBufferWithId(id);
|
||||
if (sync) {
|
||||
this.flushBufferedTtsIfSync(assistantId ?? undefined, 0);
|
||||
this.commitStreamingBufferWithId(id);
|
||||
const visible =
|
||||
this.streamingBuffer.trim().length > 0 ? this.streamingBuffer : '…';
|
||||
this.onStreamingText?.(visible, true);
|
||||
} else {
|
||||
this.commitStreamingBufferWithId(id);
|
||||
}
|
||||
this.streamingBuffer = '';
|
||||
this.pendingAssistantMessageId = null;
|
||||
this.finishAssistantTurnIfLastSegment(0, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,6 +364,9 @@ export class RealtimeSession {
|
||||
senderType: 'assistant',
|
||||
timestamp: Date.now(),
|
||||
messageType: 'text',
|
||||
...(looksLikeUuidAssistantMessageId(id)
|
||||
? { durableMessageId: id }
|
||||
: {}),
|
||||
};
|
||||
return [...(old ?? []), message];
|
||||
});
|
||||
@@ -256,7 +380,6 @@ export class RealtimeSession {
|
||||
}
|
||||
|
||||
const fullText = this.streamingBuffer;
|
||||
this.streamingBuffer = '';
|
||||
const content = fullText.trim().length > 0 ? fullText : '…';
|
||||
|
||||
const messagesKey = conversationKeys.messages(this.conversationId);
|
||||
@@ -268,6 +391,9 @@ export class RealtimeSession {
|
||||
senderType: 'assistant',
|
||||
timestamp: Date.now(),
|
||||
messageType: 'text',
|
||||
...(looksLikeUuidAssistantMessageId(messageId)
|
||||
? { durableMessageId: messageId }
|
||||
: {}),
|
||||
};
|
||||
return [...(old ?? []), message];
|
||||
});
|
||||
@@ -299,6 +425,7 @@ export class RealtimeSession {
|
||||
this.pendingAssistantMessageId ??
|
||||
`${this.conversationId}_agent_${Date.now()}`;
|
||||
this.commitStreamingBufferWithId(id);
|
||||
this.streamingBuffer = '';
|
||||
this.pendingAssistantMessageId = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +72,8 @@ export interface MessageItem {
|
||||
audioUri?: string;
|
||||
/** 助手 TTS 已上传的 COS URL 列表(与后端 `ttsAudioUrls` 一致),用于不重合成重复朗读 */
|
||||
ttsAudioUrls?: string[];
|
||||
/** 落库后的助手消息 id(REST 历史同步),用于按需 TTS 请求 */
|
||||
durableMessageId?: string;
|
||||
}
|
||||
|
||||
export interface OrganizeResponse {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { router } from 'expo-router';
|
||||
|
||||
import { tokenManager } from '@/core/auth/token-manager';
|
||||
import { disposeAllBackgroundConversationWs } from '@/features/conversation/conversation-ws-background-pool';
|
||||
import { authKeys } from '@/features/auth/hooks';
|
||||
|
||||
import { profileApi } from './api';
|
||||
@@ -97,6 +98,7 @@ export function usePurgeUserData() {
|
||||
return useMutation({
|
||||
mutationFn: (body: PurgeUserDataRequest) => profileApi.purgeUserData(body),
|
||||
onSuccess: async () => {
|
||||
disposeAllBackgroundConversationWs();
|
||||
await tokenManager.clearTokens();
|
||||
queryClient.clear();
|
||||
queryClient.setQueryData(authKeys.tokenCheck, false);
|
||||
|
||||
@@ -15,6 +15,10 @@ interface UsePlayerResult {
|
||||
enqueue: (item: PlaybackItem) => void;
|
||||
/** Replace queue and play this item (e.g. user voice bubble vs other sources). */
|
||||
enqueueExclusive: (item: PlaybackItem) => Promise<void>;
|
||||
/** Pause native playback without draining queue(与 stop 清空队列不同)。 */
|
||||
pausePlayback: () => void;
|
||||
/** Continue after pausePlayback(需 status === 'paused') */
|
||||
resumePlayback: () => void;
|
||||
stop: () => void;
|
||||
}
|
||||
|
||||
@@ -68,9 +72,11 @@ export function usePlayer(): UsePlayerResult {
|
||||
useEffect(() => {
|
||||
if (!currentSource || !player) return;
|
||||
if (!playerStatus.isLoaded) return;
|
||||
/** 先于 isLoaded「抢暂停」时需保留暂停,避免本条自动 play 覆盖 pause */
|
||||
if (status === 'paused') return;
|
||||
player.play();
|
||||
isPlayingRef.current = true;
|
||||
}, [currentSource, player, playerStatus.isLoaded]);
|
||||
}, [currentSource, player, playerStatus.isLoaded, status]);
|
||||
|
||||
const playNext = useCallback(async () => {
|
||||
if (isPlayNextInProgressRef.current) return;
|
||||
@@ -114,6 +120,7 @@ export function usePlayer(): UsePlayerResult {
|
||||
|
||||
// Detect playback completion → advance queue(必须曾 playing,避免换源瞬间沿用上一条的 duration/currentTime)
|
||||
useEffect(() => {
|
||||
if (status === 'paused') return;
|
||||
if (!currentSource || !isPlayingRef.current) return;
|
||||
|
||||
const { playing, currentTime, duration } = playerStatus;
|
||||
@@ -128,7 +135,32 @@ export function usePlayer(): UsePlayerResult {
|
||||
isPlayingRef.current = false;
|
||||
playNext();
|
||||
}
|
||||
}, [playerStatus, currentSource, playNext]);
|
||||
}, [playerStatus, currentSource, playNext, status]);
|
||||
|
||||
const pausePlayback = useCallback(() => {
|
||||
setStatus((s) => {
|
||||
if (s !== 'playing') return s;
|
||||
if (player) {
|
||||
player.pause();
|
||||
}
|
||||
isPlayingRef.current = false;
|
||||
return 'paused';
|
||||
});
|
||||
}, [player]);
|
||||
|
||||
const resumePlayback = useCallback(async () => {
|
||||
if (status !== 'paused') return;
|
||||
const acquired = await audioFocus.acquireForPlayback();
|
||||
if (!acquired) {
|
||||
setStatus('idle');
|
||||
return;
|
||||
}
|
||||
if (!player) return;
|
||||
if (!playerStatus.isLoaded) return;
|
||||
player.play();
|
||||
setStatus('playing');
|
||||
isPlayingRef.current = true;
|
||||
}, [status, player, playerStatus.isLoaded]);
|
||||
|
||||
// Subscribe to audioFocus owner changes for recorder → idle recovery
|
||||
useEffect(() => {
|
||||
@@ -205,6 +237,8 @@ export function usePlayer(): UsePlayerResult {
|
||||
currentPlaybackItem,
|
||||
enqueue,
|
||||
enqueueExclusive,
|
||||
pausePlayback,
|
||||
resumePlayback,
|
||||
stop,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user