Merge branch 'development' into claude/agent-proactive-chat-UYHu9

This commit is contained in:
Sully
2026-05-11 13:07:18 +08:00
committed by GitHub
128 changed files with 4827 additions and 1144 deletions

View File

@@ -1,11 +1,13 @@
import { api } from '@/core/api/client';
import type {
AvatarPresetItem,
ChangePasswordRequest,
ChangePhoneRequest,
LoginRequest,
RegisterRequest,
ResetPasswordRequest,
SetAvatarPresetRequest,
SmsLoginRequest,
SmsRegisterRequest,
SmsRequest,
@@ -90,4 +92,14 @@ export const authApi = {
uploadAvatar(file: FormData) {
return api.post<UserInfo>(`${AUTH}/me/avatar`, { body: file });
},
fetchAvatarPresets() {
return api.get<AvatarPresetItem[]>(`${AUTH}/avatar-presets`, {
skipAuth: true,
});
},
setAvatarPreset(body: SetAvatarPresetRequest) {
return api.put<UserInfo>(`${AUTH}/me/avatar/preset`, { body });
},
} as const;

View File

@@ -0,0 +1,77 @@
import type * as ImagePicker from 'expo-image-picker';
import { Platform } from 'react-native';
type AvatarMime = 'image/jpeg' | 'image/png' | 'image/webp';
function inferMimeFromUri(uri: string): AvatarMime {
const u = uri.toLowerCase();
if (u.endsWith('.png')) return 'image/png';
if (u.endsWith('.webp')) return 'image/webp';
return 'image/jpeg';
}
function coerceMime(value: string | null | undefined, uri: string): AvatarMime {
if (
value === 'image/jpeg' ||
value === 'image/png' ||
value === 'image/webp'
) {
return value;
}
return inferMimeFromUri(uri);
}
function mimeToFilename(mime: AvatarMime): string {
switch (mime) {
case 'image/png':
return 'avatar.png';
case 'image/webp':
return 'avatar.webp';
default:
return 'avatar.jpg';
}
}
/**
* 构建与后端 `POST /api/auth/me/avatar` 约定的 multipart字段名 `file`)。
* Native`{ uri, name, type }`Web`File`,避免 RN FormData 在 Web 上不识别 `uri`。
*/
export async function buildAvatarUploadFormData(
asset: ImagePicker.ImagePickerAsset,
): Promise<FormData> {
const uri = asset.uri;
const mime = coerceMime(asset.mimeType, uri);
const filename = mimeToFilename(mime);
const form = new FormData();
if (Platform.OS === 'web') {
const webFile = asset.file;
if (
webFile instanceof File &&
(webFile.type === 'image/jpeg' ||
webFile.type === 'image/png' ||
webFile.type === 'image/webp')
) {
form.append(
'file',
webFile,
webFile.name || mimeToFilename(coerceMime(webFile.type, uri)),
);
return form;
}
const res = await fetch(uri);
const blob = await res.blob();
const type = coerceMime(blob.type, uri);
form.append('file', new File([blob], mimeToFilename(type), { type }));
return form;
}
form.append('file', {
uri,
name: filename,
type: mime,
} as unknown as Blob);
return form;
}

View File

@@ -4,6 +4,7 @@ import { useCallback } from 'react';
import { AuthError } from '@/core/api/types';
import { tokenManager } from '@/core/auth/token-manager';
import { disposeAllBackgroundConversationWs } from '@/features/conversation/conversation-ws-background-pool';
import { authApi } from './api';
import type {
@@ -14,6 +15,7 @@ import type {
SmsRegisterRequest,
SmsRequest,
TokenResponse,
UpdateNicknameRequest,
UserInfo,
} from './types';
@@ -24,6 +26,16 @@ export const authKeys = {
tokenCheck: ['auth', 'token-check'] as const,
};
const PROFILE_QUERY_PREFIX = ['profile'] as const;
function syncSessionAndProfileQueries(
queryClient: ReturnType<typeof useQueryClient>,
user: UserInfo,
) {
queryClient.setQueryData(authKeys.session, user);
queryClient.invalidateQueries({ queryKey: PROFILE_QUERY_PREFIX });
}
// ─── useSession ───
/**
@@ -162,6 +174,44 @@ export function useSmsCode() {
});
}
// ─── Avatar / nickname ───
export function useAvatarPresets() {
return useQuery({
queryKey: ['avatar-presets'],
queryFn: () => authApi.fetchAvatarPresets(),
staleTime: 60 * 60 * 1000,
});
}
export function useUpdateNickname() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (body: UpdateNicknameRequest) => authApi.updateNickname(body),
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
});
}
export function useUploadAvatar() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (form: FormData) => authApi.uploadAvatar(form),
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
});
}
export function useSetAvatarPreset() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (presetId: string) =>
authApi.setAvatarPreset({ preset_id: presetId }),
onSuccess: (user) => syncSessionAndProfileQueries(queryClient, user),
});
}
// ─── useLogout ───
/**
@@ -185,6 +235,7 @@ export function useLogout() {
}
},
onSettled: async () => {
disposeAllBackgroundConversationWs();
await tokenManager.clearTokens();
queryClient.clear();
queryClient.setQueryData(authKeys.tokenCheck, false);

View File

@@ -79,6 +79,15 @@ export interface UpdateNicknameRequest {
nickname: string;
}
export interface AvatarPresetItem {
id: string;
url: string;
}
export interface SetAvatarPresetRequest {
preset_id: string;
}
// ─── Session state ───
export type SessionStatus =

View File

@@ -0,0 +1,88 @@
import type { QueryClient } from '@tanstack/react-query';
import { AppState, type AppStateStatus } from 'react-native';
import { RealtimeSession } from './realtime-session';
type Slot = { conversationId: string; session: RealtimeSession };
let slot: Slot | null = null;
/** 与常见聊天 App 一致:仅当应用进入 background 时断开长连(避免后台挂 socketinactive 不处理以减少控制中心等短暂打断 */
let backgroundUnsubscribe: (() => void) | null = null;
function installBackgroundLifecycleOnce(): void {
if (backgroundUnsubscribe) return;
const sub = AppState.addEventListener('change', (next: AppStateStatus) => {
if (next === 'background') {
disposeAllBackgroundConversationWs();
}
});
backgroundUnsubscribe = () => sub.remove();
}
function disposeSlot(): void {
if (!slot) return;
slot.session.dispose();
slot = null;
}
const offScreenUi = {
onStreamingText: () => {},
onTtsSegment: () => {},
onError: () => {},
onStateChange: () => {},
};
/** 离屏:保持 WebSocket去掉 UI 回调,避免列表页播 TTS 或对已卸载组件 setState */
export function releaseConversationWsUi(session: RealtimeSession): void {
session.attachUiCallbacks({
onStreamingText: offScreenUi.onStreamingText,
onTtsSegment: offScreenUi.onTtsSegment,
onError: offScreenUi.onError,
onStateChange: offScreenUi.onStateChange,
});
}
/** 删除会话等场景:关闭对应长连 */
export function disposeBackgroundConversationWs(conversationId: string): void {
if (slot?.conversationId === conversationId) {
disposeSlot();
}
}
/** 登出 / 清账号:关闭池中连接 */
export function disposeAllBackgroundConversationWs(): void {
disposeSlot();
}
/**
* 单槽仅保留「最近进入」的一个会话长连。换会话会dispose旧槽同会话返回池中实例。
*/
export function acquireBackgroundConversationWs(
conversationId: string,
queryClient: QueryClient,
prepared: RealtimeSession | null,
): RealtimeSession {
installBackgroundLifecycleOnce();
if (prepared) {
if (
slot &&
(slot.conversationId !== conversationId || slot.session !== prepared)
) {
disposeSlot();
}
slot = { conversationId, session: prepared };
return prepared;
}
if (slot?.conversationId === conversationId) {
void slot.session.connect();
return slot.session;
}
disposeSlot();
const session = new RealtimeSession({ conversationId, queryClient });
slot = { conversationId, session };
void session.connect();
return session;
}

View File

@@ -0,0 +1,147 @@
import type { QueryClient } from '@tanstack/react-query';
import { acquireBackgroundConversationWs } from './conversation-ws-background-pool';
import { conversationMessagesRepository } from './conversation-messages-repository';
import { conversationKeys } from './query-keys';
import { registerPreparedRealtimeSession } from './prepared-session-registry';
import { RealtimeSession } from './realtime-session';
import type { MessageItem } from './types';
const OPENING_WARMUP_TIMEOUT_MS = 50_000;
const CACHE_POLL_MS = 120;
function cacheHasAssistantMessage(
queryClient: QueryClient,
conversationId: string,
): boolean {
const data = queryClient.getQueryData<MessageItem[]>(
conversationKeys.messages(conversationId),
);
return (data ?? []).some((m) => m.senderType === 'assistant');
}
function waitForAssistantInCache(
queryClient: QueryClient,
conversationId: string,
timeoutMs: number,
): Promise<boolean> {
return new Promise((resolve) => {
const deadline = Date.now() + timeoutMs;
const schedule = () => {
if (cacheHasAssistantMessage(queryClient, conversationId)) {
resolve(true);
return;
}
if (Date.now() >= deadline) {
resolve(false);
return;
}
setTimeout(schedule, CACHE_POLL_MS);
};
schedule();
});
}
export async function prefetchConversationMessages(
queryClient: QueryClient,
conversationId: string,
): Promise<void> {
await queryClient.prefetchQuery({
queryKey: conversationKeys.messages(conversationId),
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
});
}
const offscreenUiCallbacks = {
onStreamingText: () => {},
onTtsSegment: () => {},
onError: () => {},
onStateChange: () => {},
};
const inflightPrewarms = new Set<string>();
/**
* 列表页/卡片按下时的预热:保持后台 WS 连接,并触发消息缓存填充。
* 与 `warmupConversationOpening` 不同:不等待开场白、不阻塞调用方,仅适用于"已有消息"的会话。
*/
export function prewarmConversationSession(
queryClient: QueryClient,
conversationId: string,
): void {
if (!conversationId) return;
const session = acquireBackgroundConversationWs(
conversationId,
queryClient,
null,
);
// 预热阶段没有挂载的 UI先用空回调占位聊天页 mount 时会重新 attach。
session.attachUiCallbacks(offscreenUiCallbacks);
if (inflightPrewarms.has(conversationId)) return;
const cached = queryClient.getQueryData<MessageItem[]>(
conversationKeys.messages(conversationId),
);
// 已有缓存就交给 React Query staleTime 决定是否刷新;只对首次进入做后台预取
if (cached && cached.length > 0) return;
inflightPrewarms.add(conversationId);
void prefetchConversationMessages(queryClient, conversationId).finally(() => {
inflightPrewarms.delete(conversationId);
});
}
async function refreshConversationMessagesForWarmup(
queryClient: QueryClient,
conversationId: string,
): Promise<void> {
await queryClient.fetchQuery({
queryKey: conversationKeys.messages(conversationId),
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
staleTime: 0,
});
}
/**
* 在会话列表阶段连接 WS 并等待首条助手开场写入 React Query成功后挂起会话供聊天页接棒。
* 超时或失败则 dispose由聊天页自行重连服务端若已写入 history 不会重复开场)。
*/
export async function warmupConversationOpening(
queryClient: QueryClient,
conversationId: string,
): Promise<void> {
/**
* 先走 REST 历史预取:若 access token 已过期API client 会在这里刷新 token
* 也避免 Redis/DB 已有开场白但本地缓存仍为空时继续等 WS。
*/
await refreshConversationMessagesForWarmup(queryClient, conversationId);
if (cacheHasAssistantMessage(queryClient, conversationId)) {
return;
}
const session = new RealtimeSession({
conversationId,
queryClient,
});
session.attachUiCallbacks({
onStreamingText: () => {},
onTtsSegment: () => {},
onError: () => {},
onStateChange: () => {},
});
await session.connect();
const ok = await waitForAssistantInCache(
queryClient,
conversationId,
OPENING_WARMUP_TIMEOUT_MS,
);
if (ok) {
registerPreparedRealtimeSession(conversationId, session);
await prefetchConversationMessages(queryClient, conversationId);
} else {
session.dispose();
}
}

View File

@@ -1,17 +1,25 @@
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { File, Paths } from 'expo-file-system';
import { useCallback, useEffect, useRef, useState } from 'react';
import { AppState, type AppStateStatus } from 'react-native';
import type { TopicSuggestion, WsConnectionState } from '@/core/ws/types';
import { conversationApi } from './api';
import {
acquireBackgroundConversationWs,
disposeAllBackgroundConversationWs,
disposeBackgroundConversationWs,
releaseConversationWsUi,
} from './conversation-ws-background-pool';
import { conversationMessagesRepository } from './conversation-messages-repository';
import { conversationKeys } from './query-keys';
import { takePreparedRealtimeSession } from './prepared-session-registry';
import {
RealtimeSession,
type ErrorCallback,
type StreamingTextCallback,
type TtsSegmentPayload,
type RealtimeSession,
} from './realtime-session';
import {
type ConversationListItem,
@@ -126,6 +134,7 @@ export function useDeleteConversation() {
mutationFn: (conversationId: string) =>
conversationApi.delete(conversationId),
onSuccess: async (_, conversationId) => {
disposeBackgroundConversationWs(conversationId);
await voiceSegmentStore.clearConversation(conversationId);
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
@@ -165,6 +174,8 @@ interface UseRealtimeSessionOptions {
onTtsSegment?: (payload: TtsSegmentPayload) => void;
/** 用户发出下一条文本/语音成功后调用,用于恢复接受 TTS 片段(打断后丢弃迟到片段) */
onTtsPlaybackResume?: () => void;
/** 本条发送是否请求了「本轮助手朗读」,用于仅在该轮自动播放 WS TTS */
onUserSendTtsPreference?: (requestedTts: boolean) => void;
}
const MIN_RECORDING_DURATION_SEC = 1;
@@ -190,6 +201,11 @@ interface RealtimeSessionState {
sendVoiceMessage: (uri: string, durationMs: number) => Promise<boolean>;
sendEndConversation: () => void;
sendTtsCancel: () => void;
requestAssistantSegmentTts: (body: {
assistantMessageId: string;
segmentIndex: number;
segmentText?: string;
}) => boolean;
}
export function useRealtimeSession({
@@ -197,9 +213,17 @@ export function useRealtimeSession({
enabled = true,
onTtsSegment,
onTtsPlaybackResume,
onUserSendTtsPreference,
}: UseRealtimeSessionOptions): RealtimeSessionState {
const queryClient = useQueryClient();
const sessionRef = useRef<RealtimeSession | null>(null);
const uiRef = useRef({
handleStreamingText: (() => {}) as StreamingTextCallback,
handleError: (() => {}) as ErrorCallback,
onTtsSegment: undefined as
| ((payload: TtsSegmentPayload) => void)
| undefined,
});
const [connectionState, setConnectionState] =
useState<WsConnectionState>('disconnected');
@@ -211,6 +235,10 @@ export function useRealtimeSession({
[],
);
const [foregroundResumeGeneration, setForegroundResumeGeneration] =
useState(0);
const needsResumeAfterBackgroundRef = useRef(false);
const handleStreamingText: StreamingTextCallback = useCallback(
(text, isComplete) => {
if (text.trim().length > 0) {
@@ -245,7 +273,8 @@ export function useRealtimeSession({
useEffect(() => {
if (!enabled || !conversationId) return;
const session = new RealtimeSession({
const prepared = takePreparedRealtimeSession(conversationId);
const session = acquireBackgroundConversationWs(
conversationId,
queryClient,
onStreamingText: handleStreamingText,
@@ -256,10 +285,10 @@ export function useRealtimeSession({
});
sessionRef.current = session;
session.connect();
setConnectionState(session.getConnectionState());
return () => {
session.dispose();
releaseConversationWsUi(session);
sessionRef.current = null;
setConnectionState('disconnected');
setStreamingMessage(null);
@@ -277,15 +306,17 @@ export function useRealtimeSession({
]);
const sendText = useCallback(
(text: string) => {
(text: string, options?: { ttsThisTurn?: boolean }) => {
if (!sessionRef.current) return;
const sent = sessionRef.current.sendText(text);
const sent = sessionRef.current.sendText(text, options);
if (!sent) {
setError('消息发送失败,连接未就绪');
return;
}
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
setAwaitingAssistantReply(true);
setTopicSuggestions([]);
onTtsPlaybackResume?.();
@@ -319,11 +350,15 @@ export function useRealtimeSession({
},
);
},
[conversationId, queryClient, onTtsPlaybackResume],
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
);
const sendVoiceMessage = useCallback(
async (uri: string, durationMs: number): Promise<boolean> => {
async (
uri: string,
durationMs: number,
options?: { ttsThisTurn?: boolean },
): Promise<boolean> => {
const session = sessionRef.current;
if (!session) return false;
@@ -340,12 +375,15 @@ export function useRealtimeSession({
clientSegmentId: `${voiceSessionId}-0`,
isLast: true,
duration: durationSec,
ttsThisTurn: options?.ttsThisTurn,
});
if (!sent) {
setError('语音发送失败,连接未就绪');
return false;
}
onUserSendTtsPreference?.(options?.ttsThisTurn === true);
setAwaitingAssistantReply(true);
setTopicSuggestions([]);
const localId = `pending_voice_${Date.now()}`;
@@ -391,7 +429,7 @@ export function useRealtimeSession({
return false;
}
},
[conversationId, queryClient, onTtsPlaybackResume],
[conversationId, queryClient, onTtsPlaybackResume, onUserSendTtsPreference],
);
const sendEndConversation = useCallback(() => {
@@ -402,6 +440,15 @@ export function useRealtimeSession({
sessionRef.current?.sendTtsCancel();
}, []);
const requestAssistantSegmentTts = useCallback(
(body: {
assistantMessageId: string;
segmentIndex: number;
segmentText?: string;
}) => sessionRef.current?.requestAssistantSegmentTts(body) ?? false,
[],
);
return {
connectionState,
streamingMessage,
@@ -413,5 +460,6 @@ export function useRealtimeSession({
sendVoiceMessage,
sendEndConversation,
sendTtsCancel,
requestAssistantSegmentTts,
};
}

View File

@@ -0,0 +1,33 @@
import type { RealtimeSession } from './realtime-session';
const preparedByConversationId = new Map<string, RealtimeSession>();
/** 列表页预热完成后挂起会话,聊天页挂载时接棒并删除登记 */
export function registerPreparedRealtimeSession(
conversationId: string,
session: RealtimeSession,
): void {
const old = preparedByConversationId.get(conversationId);
if (old && old !== session) {
old.dispose();
}
preparedByConversationId.set(conversationId, session);
}
/** 取出即视为消费;若无则返回 null */
export function takePreparedRealtimeSession(
conversationId: string,
): RealtimeSession | null {
const session = preparedByConversationId.get(conversationId);
if (!session) return null;
preparedByConversationId.delete(conversationId);
return session;
}
/** 预热成功后导航失败时释放挂起连接,避免僵尸 WebSocket */
export function abandonPreparedRealtimeSession(conversationId: string): void {
const session = preparedByConversationId.get(conversationId);
if (!session) return;
preparedByConversationId.delete(conversationId);
session.dispose();
}

View File

@@ -16,6 +16,13 @@ import { assistantSegmentMessageId, lastSegmentPreview } from './message-split';
import { conversationKeys } from './query-keys';
import type { ConversationListItem, MessageItem } from './types';
/** 与落库助手消息 id、会话页 `durableAssistantIdForBubble` 的 uuid 判断一致 */
function looksLikeUuidAssistantMessageId(id: string): boolean {
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
id,
);
}
export type StreamingTextCallback = (text: string, isComplete: boolean) => void;
export type ErrorCallback = (message: string, code?: string) => void;
export type TopicSuggestionsCallback = (payload: {
@@ -32,6 +39,8 @@ export type TtsSegmentPayload = {
total?: number;
/** 服务端持久化后的助手消息 id用于与气泡 listKey / 消息 id 对齐 */
assistantMessageId?: string;
/** 用户点喇叭按需下发时为 true应加入播放队列即使未开「本轮朗读」 */
manual?: boolean;
};
interface RealtimeSessionOptions {
@@ -65,12 +74,25 @@ export class RealtimeSession {
private onTtsSegment?: (payload: TtsSegmentPayload) => void;
private onTopicSuggestions?: TopicSuggestionsCallback;
private onError?: ErrorCallback;
private uiStateListener?: WsStateListener;
private unsubEvent: (() => void) | null = null;
private unsubState: (() => void) | null = null;
private streamingBuffer = '';
/** 单段回复且服务端带 `assistant_message_id` 时用于落缓存 id */
private pendingAssistantMessageId: string | null = null;
private destroyed = false;
/** 本条用户消息是否请求「先 TTS 再出字」的助手轮次 */
private assistantTurnTtsSync = false;
private pendingTtsByKey = new Map<string, TtsSegmentPayload>();
private static bufferedTtsKey(
assistantMessageId: string | undefined,
index: number,
): string {
return `${assistantMessageId ?? '_'}:${index}`;
}
constructor(options: RealtimeSessionOptions) {
this.client = new WsClient(options.conversationId);
@@ -80,11 +102,38 @@ export class RealtimeSession {
this.onTtsSegment = options.onTtsSegment;
this.onTopicSuggestions = options.onTopicSuggestions;
this.onError = options.onError;
this.uiStateListener = options.onStateChange;
this.unsubEvent = this.client.onEvent(this.handleEvent);
if (options.onStateChange) {
this.unsubState = this.client.onStateChange(options.onStateChange);
this.unsubState = this.client.onStateChange((state) => {
this.uiStateListener?.(state);
});
}
/** 列表预热接棒或刷新 UI 订阅时替换回调,不重建 WebSocket */
attachUiCallbacks(options: {
onStreamingText?: StreamingTextCallback;
onTtsSegment?: (payload: TtsSegmentPayload) => void;
onError?: ErrorCallback;
onStateChange?: WsStateListener;
}): void {
if (this.destroyed) return;
if (options.onStreamingText !== undefined) {
this.onStreamingText = options.onStreamingText;
}
if (options.onTtsSegment !== undefined) {
this.onTtsSegment = options.onTtsSegment;
}
if (options.onError !== undefined) {
this.onError = options.onError;
}
if (options.onStateChange !== undefined) {
this.uiStateListener = options.onStateChange;
options.onStateChange(this.client.getState());
}
if (!this.assistantTurnTtsSync && this.streamingBuffer.trim().length > 0) {
this.onStreamingText?.(this.streamingBuffer, false);
}
}
@@ -97,15 +146,20 @@ export class RealtimeSession {
}
dispose(): void {
if (this.destroyed) return;
this.destroyed = true;
this.flushStreamingBufferIfPending();
this.resetAssistantTtsSyncState();
this.unsubEvent?.();
this.unsubState?.();
this.client.dispose();
}
/** Returns true if the message was sent over the socket. */
sendText(text: string): boolean {
return this.client.sendText(text);
sendText(text: string, options?: { ttsThisTurn?: boolean }): boolean {
const tts = !!options?.ttsThisTurn;
this.assistantTurnTtsSync = tts;
return this.client.sendText(text, { ttsThisTurn: tts });
}
sendAudioSegment(
@@ -116,8 +170,11 @@ export class RealtimeSession {
clientSegmentId?: string;
isLast?: boolean;
duration?: number;
ttsThisTurn?: boolean;
},
): boolean {
const tts = !!options?.ttsThisTurn;
this.assistantTurnTtsSync = tts;
return this.client.send({
type: 'audio_segment',
data: {
@@ -127,6 +184,7 @@ export class RealtimeSession {
client_segment_id: options?.clientSegmentId,
is_last: options?.isLast,
duration: options?.duration,
...(options?.ttsThisTurn === true ? { tts_this_turn: true } : {}),
},
});
}
@@ -137,6 +195,7 @@ export class RealtimeSession {
/** 通知服务端停止当前轮次后续 TTS 合成与下发(与客户端 stop 队列配合) */
sendTtsCancel(): boolean {
this.resetAssistantTtsSyncState();
return this.client.sendTtsCancel();
}
@@ -144,8 +203,40 @@ export class RealtimeSession {
return this.client.getState();
}
requestAssistantSegmentTts(body: {
assistantMessageId: string;
segmentIndex: number;
segmentText?: string;
}): boolean {
return this.client.sendTtsRequest(body);
}
// ─── Internal ───
private resetAssistantTtsSyncState(): void {
this.assistantTurnTtsSync = false;
this.pendingTtsByKey.clear();
}
private flushBufferedTtsIfSync(
assistantMessageId: string | undefined,
index: number,
): void {
if (!this.assistantTurnTtsSync) return;
const key = RealtimeSession.bufferedTtsKey(assistantMessageId, index);
const payload = this.pendingTtsByKey.get(key);
if (payload) {
this.pendingTtsByKey.delete(key);
this.onTtsSegment?.(payload);
}
}
private finishAssistantTurnIfLastSegment(index: number, total: number): void {
if (index >= total - 1) {
this.resetAssistantTtsSyncState();
}
}
private handleEvent: WsEventListener = (event: WsEvent) => {
if (event.kind === 'agent_response') {
this.handleAgentChunk(event);
@@ -155,14 +246,26 @@ export class RealtimeSession {
if (event.kind === 'tts_audio_received') {
const b64 = event.audioBase64?.trim();
const url = event.audioUrl?.trim();
if (b64 || url) {
this.onTtsSegment?.({
audioBase64: b64 || undefined,
audioUrl: url || undefined,
index: event.index,
total: event.total,
assistantMessageId: event.assistantMessageId,
});
if (!b64 && !url) {
return;
}
const payload: TtsSegmentPayload = {
audioBase64: b64 || undefined,
audioUrl: url || undefined,
index: event.index,
total: event.total,
assistantMessageId: event.assistantMessageId,
manual: event.manual,
};
if (this.assistantTurnTtsSync && !payload.manual) {
const idx = event.index ?? 0;
const key = RealtimeSession.bufferedTtsKey(
event.assistantMessageId,
idx,
);
this.pendingTtsByKey.set(key, payload);
} else {
this.onTtsSegment?.(payload);
}
return;
}
@@ -179,6 +282,7 @@ export class RealtimeSession {
handleWsEvent(this.queryClient, event);
if (event.kind === 'session_error') {
this.resetAssistantTtsSyncState();
this.onError?.(event.message, event.code);
}
};
@@ -202,14 +306,19 @@ export class RealtimeSession {
const total = event.total ?? 1;
const index = event.index ?? 0;
const sync = this.assistantTurnTtsSync;
if (total > 1) {
const id =
event.assistantMessageId != null
? assistantSegmentMessageId(event.assistantMessageId, index)
: `${this.conversationId}_agent_${Date.now()}_${index}`;
if (sync) {
this.flushBufferedTtsIfSync(event.assistantMessageId, index);
}
this.commitOneAssistantMessage(event.text, id);
this.onStreamingText?.(event.text, true);
this.finishAssistantTurnIfLastSegment(index, total);
return;
}
@@ -218,18 +327,30 @@ export class RealtimeSession {
}
this.streamingBuffer += event.text;
// 与 coerced index/total 对齐:若服务端只带 text、省略 index/total旧逻辑会 isComplete=false永远不落库
const isComplete = index >= total - 1;
this.onStreamingText?.(this.streamingBuffer, isComplete);
if (!sync) {
this.onStreamingText?.(this.streamingBuffer, isComplete);
}
if (isComplete) {
const assistantId =
event.assistantMessageId ?? this.pendingAssistantMessageId;
const id =
this.pendingAssistantMessageId ??
`${this.conversationId}_agent_${Date.now()}`;
this.commitStreamingBufferWithId(id);
if (sync) {
this.flushBufferedTtsIfSync(assistantId ?? undefined, 0);
this.commitStreamingBufferWithId(id);
const visible =
this.streamingBuffer.trim().length > 0 ? this.streamingBuffer : '…';
this.onStreamingText?.(visible, true);
} else {
this.commitStreamingBufferWithId(id);
}
this.streamingBuffer = '';
this.pendingAssistantMessageId = null;
this.finishAssistantTurnIfLastSegment(0, 1);
}
}
@@ -243,6 +364,9 @@ export class RealtimeSession {
senderType: 'assistant',
timestamp: Date.now(),
messageType: 'text',
...(looksLikeUuidAssistantMessageId(id)
? { durableMessageId: id }
: {}),
};
return [...(old ?? []), message];
});
@@ -256,7 +380,6 @@ export class RealtimeSession {
}
const fullText = this.streamingBuffer;
this.streamingBuffer = '';
const content = fullText.trim().length > 0 ? fullText : '…';
const messagesKey = conversationKeys.messages(this.conversationId);
@@ -268,6 +391,9 @@ export class RealtimeSession {
senderType: 'assistant',
timestamp: Date.now(),
messageType: 'text',
...(looksLikeUuidAssistantMessageId(messageId)
? { durableMessageId: messageId }
: {}),
};
return [...(old ?? []), message];
});
@@ -299,6 +425,7 @@ export class RealtimeSession {
this.pendingAssistantMessageId ??
`${this.conversationId}_agent_${Date.now()}`;
this.commitStreamingBufferWithId(id);
this.streamingBuffer = '';
this.pendingAssistantMessageId = null;
}
}

View File

@@ -72,6 +72,8 @@ export interface MessageItem {
audioUri?: string;
/** 助手 TTS 已上传的 COS URL 列表(与后端 `ttsAudioUrls` 一致),用于不重合成重复朗读 */
ttsAudioUrls?: string[];
/** 落库后的助手消息 idREST 历史同步),用于按需 TTS 请求 */
durableMessageId?: string;
}
export interface OrganizeResponse {

View File

@@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { router } from 'expo-router';
import { tokenManager } from '@/core/auth/token-manager';
import { disposeAllBackgroundConversationWs } from '@/features/conversation/conversation-ws-background-pool';
import { authKeys } from '@/features/auth/hooks';
import { profileApi } from './api';
@@ -97,6 +98,7 @@ export function usePurgeUserData() {
return useMutation({
mutationFn: (body: PurgeUserDataRequest) => profileApi.purgeUserData(body),
onSuccess: async () => {
disposeAllBackgroundConversationWs();
await tokenManager.clearTokens();
queryClient.clear();
queryClient.setQueryData(authKeys.tokenCheck, false);

View File

@@ -15,6 +15,10 @@ interface UsePlayerResult {
enqueue: (item: PlaybackItem) => void;
/** Replace queue and play this item (e.g. user voice bubble vs other sources). */
enqueueExclusive: (item: PlaybackItem) => Promise<void>;
/** Pause native playback without draining queue与 stop 清空队列不同)。 */
pausePlayback: () => void;
/** Continue after pausePlayback需 status === 'paused' */
resumePlayback: () => void;
stop: () => void;
}
@@ -68,9 +72,11 @@ export function usePlayer(): UsePlayerResult {
useEffect(() => {
if (!currentSource || !player) return;
if (!playerStatus.isLoaded) return;
/** 先于 isLoaded「抢暂停」时需保留暂停避免本条自动 play 覆盖 pause */
if (status === 'paused') return;
player.play();
isPlayingRef.current = true;
}, [currentSource, player, playerStatus.isLoaded]);
}, [currentSource, player, playerStatus.isLoaded, status]);
const playNext = useCallback(async () => {
if (isPlayNextInProgressRef.current) return;
@@ -114,6 +120,7 @@ export function usePlayer(): UsePlayerResult {
// Detect playback completion → advance queue必须曾 playing避免换源瞬间沿用上一条的 duration/currentTime
useEffect(() => {
if (status === 'paused') return;
if (!currentSource || !isPlayingRef.current) return;
const { playing, currentTime, duration } = playerStatus;
@@ -128,7 +135,32 @@ export function usePlayer(): UsePlayerResult {
isPlayingRef.current = false;
playNext();
}
}, [playerStatus, currentSource, playNext]);
}, [playerStatus, currentSource, playNext, status]);
const pausePlayback = useCallback(() => {
setStatus((s) => {
if (s !== 'playing') return s;
if (player) {
player.pause();
}
isPlayingRef.current = false;
return 'paused';
});
}, [player]);
const resumePlayback = useCallback(async () => {
if (status !== 'paused') return;
const acquired = await audioFocus.acquireForPlayback();
if (!acquired) {
setStatus('idle');
return;
}
if (!player) return;
if (!playerStatus.isLoaded) return;
player.play();
setStatus('playing');
isPlayingRef.current = true;
}, [status, player, playerStatus.isLoaded]);
// Subscribe to audioFocus owner changes for recorder → idle recovery
useEffect(() => {
@@ -205,6 +237,8 @@ export function usePlayer(): UsePlayerResult {
currentPlaybackItem,
enqueue,
enqueueExclusive,
pausePlayback,
resumePlayback,
stop,
};
}