数据库 - 新增迁移 0003:timeline_events.memory_source_id 外键 → memory_sources,便于按 ingest 源做时间线幂等 后端 - 记忆 - 新增 ingest 后 LLM 富化(摘要/事实/时间线),可配置开关与最大字符数 - 新增证据包组装:合并 chunk、摘要、事实、时间线、故事等检索结果;支持空 query 时是否仍带 rolling 等开关 - repo/retriever/service/router/schemas/summarizer/timeline/extractor 等扩展;文档 memory-retrieval.md 更新 后端 - 对话 WS - 增加 PING/PONG;分段 ASR 日志与空音频处理;转写失败与「无助手回复」错误提示更明确 - 助手多段回复持久化使用统一分隔符,与分段逻辑一致 后端 - Agent - reply_limits:按 [SPLIT] 与段落拆段,并保证非空 fallback,供 WS 与 TTS 多段下发 后端 - 回忆录任务 - transcript ingest 记录 source_id;任务成功结?
391 lines
11 KiB
TypeScript
391 lines
11 KiB
TypeScript
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||
import { File, Paths } from 'expo-file-system';
|
||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||
|
||
import type { WsConnectionState } from '@/core/ws/types';
|
||
|
||
import { conversationApi } from './api';
|
||
import { conversationMessagesRepository } from './conversation-messages-repository';
|
||
import { conversationKeys } from './query-keys';
|
||
import {
|
||
RealtimeSession,
|
||
type ErrorCallback,
|
||
type StreamingTextCallback,
|
||
type TtsSegmentPayload,
|
||
} from './realtime-session';
|
||
import {
|
||
type ConversationListItem,
|
||
type MessageItem,
|
||
type StreamingAgentMessage,
|
||
} from './types';
|
||
import { voiceSegmentStore } from '@/features/voice/voice-segment-store';
|
||
|
||
/** Expo `File` 需要规范 `file://` URI;部分录音 API 会返回裸绝对路径。 */
|
||
function ensureFileUri(uri: string): string {
|
||
const u = uri.trim();
|
||
if (u.startsWith('file://')) return u;
|
||
if (u.startsWith('/')) return `file://${u}`;
|
||
return u;
|
||
}
|
||
|
||
function guessAudioExtension(uri: string): string {
|
||
const pathOnly = uri.split('?')[0] ?? uri;
|
||
const m = /\.[^/.]+$/u.exec(pathOnly);
|
||
return m ? m[0] : '.m4a';
|
||
}
|
||
|
||
/**
|
||
* 使用主包 `File`/`Paths`(见 Expo 文档:新 File 与旧 readAsStringAsync 互操作示例)。
|
||
* 先 copy 到 cache 下唯一文件名再 `base64()`,避免直接读源路径时偶发读到陈旧/错误内容。
|
||
*/
|
||
async function readRecordingPayload(uri: string): Promise<string> {
|
||
const resolved = ensureFileUri(uri);
|
||
const source = new File(resolved);
|
||
if (!source.exists) {
|
||
throw new Error('recording file missing');
|
||
}
|
||
const stagedName = `voice-upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}${guessAudioExtension(resolved)}`;
|
||
const staged = new File(Paths.cache, stagedName);
|
||
try {
|
||
source.copy(staged);
|
||
} catch {
|
||
return await source.base64();
|
||
}
|
||
try {
|
||
return await staged.base64();
|
||
} finally {
|
||
try {
|
||
staged.delete();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─── Query hooks ───
|
||
|
||
// TODO: 连接不上后端时 isLoading 可能一直为 true,需加超时或展示错误态
|
||
export function useConversations() {
|
||
return useQuery({
|
||
queryKey: conversationKeys.lists(),
|
||
queryFn: () => conversationApi.list(),
|
||
});
|
||
}
|
||
|
||
export function useConversationDetail(conversationId: string) {
|
||
return useQuery({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
queryFn: () => conversationApi.detail(conversationId),
|
||
enabled: !!conversationId,
|
||
});
|
||
}
|
||
|
||
export function useMessages(conversationId: string) {
|
||
return useQuery({
|
||
queryKey: conversationKeys.messages(conversationId),
|
||
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
|
||
enabled: !!conversationId,
|
||
});
|
||
}
|
||
|
||
// ─── Mutation hooks ───
|
||
|
||
export function useCreateConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: () => conversationApi.create(),
|
||
onSuccess: (newConversation) => {
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
const item: ConversationListItem = {
|
||
id: newConversation.id,
|
||
title: '岁月知己',
|
||
avatarUrl: null,
|
||
latestMessagePreview: '',
|
||
latestMessageTime: Date.now(),
|
||
unreadCount: 0,
|
||
isDefaultAssistant: true,
|
||
hasUserMessage: false,
|
||
};
|
||
return [item, ...(old ?? [])];
|
||
},
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
export function useDeleteConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: (conversationId: string) =>
|
||
conversationApi.delete(conversationId),
|
||
onSuccess: async (_, conversationId) => {
|
||
await voiceSegmentStore.clearConversation(conversationId);
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => old?.filter((item) => item.id !== conversationId),
|
||
);
|
||
queryClient.removeQueries({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
});
|
||
queryClient.removeQueries({
|
||
queryKey: conversationKeys.messages(conversationId),
|
||
});
|
||
},
|
||
});
|
||
}
|
||
|
||
export function useEndConversation() {
|
||
const queryClient = useQueryClient();
|
||
|
||
return useMutation({
|
||
mutationFn: (conversationId: string) => conversationApi.end(conversationId),
|
||
onSuccess: (_, conversationId) => {
|
||
queryClient.invalidateQueries({
|
||
queryKey: conversationKeys.detail(conversationId),
|
||
});
|
||
queryClient.invalidateQueries({
|
||
queryKey: conversationKeys.lists(),
|
||
});
|
||
},
|
||
});
|
||
}
|
||
|
||
// ─── Realtime session hook ───
|
||
|
||
interface UseRealtimeSessionOptions {
|
||
conversationId: string;
|
||
enabled?: boolean;
|
||
onTtsSegment?: (payload: TtsSegmentPayload) => void;
|
||
/** 用户发出下一条文本/语音成功后调用,用于恢复接受 TTS 片段(打断后丢弃迟到片段) */
|
||
onTtsPlaybackResume?: () => void;
|
||
}
|
||
|
||
const MIN_RECORDING_DURATION_SEC = 1;
|
||
|
||
function generateUUID(): string {
|
||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||
const r = (Math.random() * 16) | 0;
|
||
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
||
return v.toString(16);
|
||
});
|
||
}
|
||
|
||
interface RealtimeSessionState {
|
||
connectionState: WsConnectionState;
|
||
streamingMessage: StreamingAgentMessage | null;
|
||
/** 已发出用户消息,尚未收到助手首段流式文本(用于「正在回复」气泡) */
|
||
awaitingAssistantReply: boolean;
|
||
error: string | null;
|
||
sendText: (text: string) => void;
|
||
sendVoiceMessage: (uri: string, durationMs: number) => Promise<boolean>;
|
||
sendEndConversation: () => void;
|
||
sendTtsCancel: () => void;
|
||
}
|
||
|
||
export function useRealtimeSession({
|
||
conversationId,
|
||
enabled = true,
|
||
onTtsSegment,
|
||
onTtsPlaybackResume,
|
||
}: UseRealtimeSessionOptions): RealtimeSessionState {
|
||
const queryClient = useQueryClient();
|
||
const sessionRef = useRef<RealtimeSession | null>(null);
|
||
|
||
const [connectionState, setConnectionState] =
|
||
useState<WsConnectionState>('disconnected');
|
||
const [streamingMessage, setStreamingMessage] =
|
||
useState<StreamingAgentMessage | null>(null);
|
||
const [awaitingAssistantReply, setAwaitingAssistantReply] = useState(false);
|
||
const [error, setError] = useState<string | null>(null);
|
||
|
||
const handleStreamingText: StreamingTextCallback = useCallback(
|
||
(text, isComplete) => {
|
||
if (text.trim().length > 0) {
|
||
setAwaitingAssistantReply(false);
|
||
}
|
||
if (isComplete) {
|
||
setStreamingMessage(null);
|
||
setAwaitingAssistantReply(false);
|
||
return;
|
||
}
|
||
setStreamingMessage({ text, isComplete });
|
||
},
|
||
[],
|
||
);
|
||
|
||
const handleError: ErrorCallback = useCallback((message) => {
|
||
setAwaitingAssistantReply(false);
|
||
setError(message);
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
if (!enabled || !conversationId) return;
|
||
|
||
const session = new RealtimeSession({
|
||
conversationId,
|
||
queryClient,
|
||
onStreamingText: handleStreamingText,
|
||
onTtsSegment,
|
||
onError: handleError,
|
||
onStateChange: setConnectionState,
|
||
});
|
||
|
||
sessionRef.current = session;
|
||
session.connect();
|
||
|
||
return () => {
|
||
session.dispose();
|
||
sessionRef.current = null;
|
||
setConnectionState('disconnected');
|
||
setStreamingMessage(null);
|
||
setAwaitingAssistantReply(false);
|
||
};
|
||
}, [
|
||
conversationId,
|
||
enabled,
|
||
queryClient,
|
||
handleStreamingText,
|
||
handleError,
|
||
onTtsSegment,
|
||
]);
|
||
|
||
const sendText = useCallback(
|
||
(text: string) => {
|
||
if (!sessionRef.current) return;
|
||
|
||
const sent = sessionRef.current.sendText(text);
|
||
if (!sent) {
|
||
setError('消息发送失败,连接未就绪');
|
||
return;
|
||
}
|
||
|
||
setAwaitingAssistantReply(true);
|
||
onTtsPlaybackResume?.();
|
||
|
||
const localId = `pending_${Date.now()}`;
|
||
|
||
queryClient.setQueryData<MessageItem[]>(
|
||
conversationKeys.messages(conversationId),
|
||
(old) => {
|
||
const msg: MessageItem = {
|
||
id: localId,
|
||
conversationId,
|
||
content: text,
|
||
senderType: 'user',
|
||
timestamp: Date.now(),
|
||
messageType: 'text',
|
||
};
|
||
return [...(old ?? []), msg];
|
||
},
|
||
);
|
||
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
if (!old) return old;
|
||
return old.map((item) =>
|
||
item.id === conversationId
|
||
? { ...item, hasUserMessage: true }
|
||
: item,
|
||
);
|
||
},
|
||
);
|
||
},
|
||
[conversationId, queryClient, onTtsPlaybackResume],
|
||
);
|
||
|
||
const sendVoiceMessage = useCallback(
|
||
async (uri: string, durationMs: number): Promise<boolean> => {
|
||
const session = sessionRef.current;
|
||
if (!session) return false;
|
||
|
||
const durationSec = Math.round(durationMs / 1000);
|
||
if (durationSec < MIN_RECORDING_DURATION_SEC) return false;
|
||
|
||
try {
|
||
const base64 = await readRecordingPayload(uri);
|
||
if (!base64) return false;
|
||
|
||
const voiceSessionId = generateUUID();
|
||
const sent = session.sendAudioSegment(base64, 0, {
|
||
voiceSessionId,
|
||
clientSegmentId: `${voiceSessionId}-0`,
|
||
isLast: true,
|
||
duration: durationSec,
|
||
});
|
||
if (!sent) {
|
||
setError('语音发送失败,连接未就绪');
|
||
return false;
|
||
}
|
||
|
||
setAwaitingAssistantReply(true);
|
||
const localId = `pending_voice_${Date.now()}`;
|
||
await voiceSegmentStore.recordSentSegment({
|
||
voiceSessionId,
|
||
conversationId,
|
||
fileUri: uri,
|
||
durationMs,
|
||
});
|
||
queryClient.setQueryData<MessageItem[]>(
|
||
conversationKeys.messages(conversationId),
|
||
(old) => {
|
||
const msg: MessageItem = {
|
||
id: localId,
|
||
conversationId,
|
||
content: '',
|
||
senderType: 'user',
|
||
timestamp: Date.now(),
|
||
messageType: 'voice',
|
||
voiceSessionId,
|
||
durationSeconds: durationSec,
|
||
audioUri: uri,
|
||
};
|
||
return [...(old ?? []), msg];
|
||
},
|
||
);
|
||
|
||
queryClient.setQueryData<ConversationListItem[]>(
|
||
conversationKeys.lists(),
|
||
(old) => {
|
||
if (!old) return old;
|
||
return old.map((item) =>
|
||
item.id === conversationId
|
||
? { ...item, hasUserMessage: true }
|
||
: item,
|
||
);
|
||
},
|
||
);
|
||
onTtsPlaybackResume?.();
|
||
return true;
|
||
} catch {
|
||
setError('语音文件读取失败');
|
||
return false;
|
||
}
|
||
},
|
||
[conversationId, queryClient, onTtsPlaybackResume],
|
||
);
|
||
|
||
const sendEndConversation = useCallback(() => {
|
||
sessionRef.current?.sendEndConversation();
|
||
}, []);
|
||
|
||
const sendTtsCancel = useCallback(() => {
|
||
sessionRef.current?.sendTtsCancel();
|
||
}, []);
|
||
|
||
return {
|
||
connectionState,
|
||
streamingMessage,
|
||
awaitingAssistantReply,
|
||
error,
|
||
sendText,
|
||
sendVoiceMessage,
|
||
sendEndConversation,
|
||
sendTtsCancel,
|
||
};
|
||
}
|