Files
life-echo/app-expo/src/features/conversation/hooks.ts
Kevin e4bf0710c7 feat(memory,conversation): 记忆富化/证据包、时间线幂等字段与对话分段全链路
数据库
- 新增迁移 0003:timeline_events.memory_source_id 外键 → memory_sources,便于按 ingest 源做时间线幂等

后端 - 记忆
- 新增 ingest 后 LLM 富化(摘要/事实/时间线),可配置开关与最大字符数
- 新增证据包组装:合并 chunk、摘要、事实、时间线、故事等检索结果;支持空 query 时是否仍带 rolling 等开关
- repo/retriever/service/router/schemas/summarizer/timeline/extractor 等扩展;文档 memory-retrieval.md 更新

后端 - 对话 WS
- 增加 PING/PONG;分段 ASR 日志与空音频处理;转写失败与「无助手回复」错误提示更明确
- 助手多段回复持久化使用统一分隔符,与分段逻辑一致

后端 - Agent
- reply_limits:按 [SPLIT] 与段落拆段,并保证非空 fallback,供 WS 与 TTS 多段下发

后端 - 回忆录任务
- transcript ingest 记录 source_id;任务成功结?
2026-03-27 16:24:43 +08:00

391 lines
11 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { File, Paths } from 'expo-file-system';
import { useCallback, useEffect, useRef, useState } from 'react';
import type { WsConnectionState } from '@/core/ws/types';
import { conversationApi } from './api';
import { conversationMessagesRepository } from './conversation-messages-repository';
import { conversationKeys } from './query-keys';
import {
RealtimeSession,
type ErrorCallback,
type StreamingTextCallback,
type TtsSegmentPayload,
} from './realtime-session';
import {
type ConversationListItem,
type MessageItem,
type StreamingAgentMessage,
} from './types';
import { voiceSegmentStore } from '@/features/voice/voice-segment-store';
/** Expo `File` 需要规范 `file://` URI部分录音 API 会返回裸绝对路径。 */
function ensureFileUri(uri: string): string {
const u = uri.trim();
if (u.startsWith('file://')) return u;
if (u.startsWith('/')) return `file://${u}`;
return u;
}
function guessAudioExtension(uri: string): string {
const pathOnly = uri.split('?')[0] ?? uri;
const m = /\.[^/.]+$/u.exec(pathOnly);
return m ? m[0] : '.m4a';
}
/**
* 使用主包 `File`/`Paths`(见 Expo 文档:新 File 与旧 readAsStringAsync 互操作示例)。
* 先 copy 到 cache 下唯一文件名再 `base64()`,避免直接读源路径时偶发读到陈旧/错误内容。
*/
async function readRecordingPayload(uri: string): Promise<string> {
const resolved = ensureFileUri(uri);
const source = new File(resolved);
if (!source.exists) {
throw new Error('recording file missing');
}
const stagedName = `voice-upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}${guessAudioExtension(resolved)}`;
const staged = new File(Paths.cache, stagedName);
try {
source.copy(staged);
} catch {
return await source.base64();
}
try {
return await staged.base64();
} finally {
try {
staged.delete();
} catch {
// ignore
}
}
}
// ─── Query hooks ───
// TODO: 连接不上后端时 isLoading 可能一直为 true需加超时或展示错误态
export function useConversations() {
return useQuery({
queryKey: conversationKeys.lists(),
queryFn: () => conversationApi.list(),
});
}
export function useConversationDetail(conversationId: string) {
return useQuery({
queryKey: conversationKeys.detail(conversationId),
queryFn: () => conversationApi.detail(conversationId),
enabled: !!conversationId,
});
}
export function useMessages(conversationId: string) {
return useQuery({
queryKey: conversationKeys.messages(conversationId),
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
enabled: !!conversationId,
});
}
// ─── Mutation hooks ───
export function useCreateConversation() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: () => conversationApi.create(),
onSuccess: (newConversation) => {
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
(old) => {
const item: ConversationListItem = {
id: newConversation.id,
title: '岁月知己',
avatarUrl: null,
latestMessagePreview: '',
latestMessageTime: Date.now(),
unreadCount: 0,
isDefaultAssistant: true,
hasUserMessage: false,
};
return [item, ...(old ?? [])];
},
);
},
});
}
export function useDeleteConversation() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (conversationId: string) =>
conversationApi.delete(conversationId),
onSuccess: async (_, conversationId) => {
await voiceSegmentStore.clearConversation(conversationId);
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
(old) => old?.filter((item) => item.id !== conversationId),
);
queryClient.removeQueries({
queryKey: conversationKeys.detail(conversationId),
});
queryClient.removeQueries({
queryKey: conversationKeys.messages(conversationId),
});
},
});
}
export function useEndConversation() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (conversationId: string) => conversationApi.end(conversationId),
onSuccess: (_, conversationId) => {
queryClient.invalidateQueries({
queryKey: conversationKeys.detail(conversationId),
});
queryClient.invalidateQueries({
queryKey: conversationKeys.lists(),
});
},
});
}
// ─── Realtime session hook ───
interface UseRealtimeSessionOptions {
conversationId: string;
enabled?: boolean;
onTtsSegment?: (payload: TtsSegmentPayload) => void;
/** 用户发出下一条文本/语音成功后调用,用于恢复接受 TTS 片段(打断后丢弃迟到片段) */
onTtsPlaybackResume?: () => void;
}
const MIN_RECORDING_DURATION_SEC = 1;
function generateUUID(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = (Math.random() * 16) | 0;
const v = c === 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
interface RealtimeSessionState {
connectionState: WsConnectionState;
streamingMessage: StreamingAgentMessage | null;
/** 已发出用户消息,尚未收到助手首段流式文本(用于「正在回复」气泡) */
awaitingAssistantReply: boolean;
error: string | null;
sendText: (text: string) => void;
sendVoiceMessage: (uri: string, durationMs: number) => Promise<boolean>;
sendEndConversation: () => void;
sendTtsCancel: () => void;
}
export function useRealtimeSession({
conversationId,
enabled = true,
onTtsSegment,
onTtsPlaybackResume,
}: UseRealtimeSessionOptions): RealtimeSessionState {
const queryClient = useQueryClient();
const sessionRef = useRef<RealtimeSession | null>(null);
const [connectionState, setConnectionState] =
useState<WsConnectionState>('disconnected');
const [streamingMessage, setStreamingMessage] =
useState<StreamingAgentMessage | null>(null);
const [awaitingAssistantReply, setAwaitingAssistantReply] = useState(false);
const [error, setError] = useState<string | null>(null);
const handleStreamingText: StreamingTextCallback = useCallback(
(text, isComplete) => {
if (text.trim().length > 0) {
setAwaitingAssistantReply(false);
}
if (isComplete) {
setStreamingMessage(null);
setAwaitingAssistantReply(false);
return;
}
setStreamingMessage({ text, isComplete });
},
[],
);
const handleError: ErrorCallback = useCallback((message) => {
setAwaitingAssistantReply(false);
setError(message);
}, []);
useEffect(() => {
if (!enabled || !conversationId) return;
const session = new RealtimeSession({
conversationId,
queryClient,
onStreamingText: handleStreamingText,
onTtsSegment,
onError: handleError,
onStateChange: setConnectionState,
});
sessionRef.current = session;
session.connect();
return () => {
session.dispose();
sessionRef.current = null;
setConnectionState('disconnected');
setStreamingMessage(null);
setAwaitingAssistantReply(false);
};
}, [
conversationId,
enabled,
queryClient,
handleStreamingText,
handleError,
onTtsSegment,
]);
const sendText = useCallback(
(text: string) => {
if (!sessionRef.current) return;
const sent = sessionRef.current.sendText(text);
if (!sent) {
setError('消息发送失败,连接未就绪');
return;
}
setAwaitingAssistantReply(true);
onTtsPlaybackResume?.();
const localId = `pending_${Date.now()}`;
queryClient.setQueryData<MessageItem[]>(
conversationKeys.messages(conversationId),
(old) => {
const msg: MessageItem = {
id: localId,
conversationId,
content: text,
senderType: 'user',
timestamp: Date.now(),
messageType: 'text',
};
return [...(old ?? []), msg];
},
);
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
(old) => {
if (!old) return old;
return old.map((item) =>
item.id === conversationId
? { ...item, hasUserMessage: true }
: item,
);
},
);
},
[conversationId, queryClient, onTtsPlaybackResume],
);
const sendVoiceMessage = useCallback(
async (uri: string, durationMs: number): Promise<boolean> => {
const session = sessionRef.current;
if (!session) return false;
const durationSec = Math.round(durationMs / 1000);
if (durationSec < MIN_RECORDING_DURATION_SEC) return false;
try {
const base64 = await readRecordingPayload(uri);
if (!base64) return false;
const voiceSessionId = generateUUID();
const sent = session.sendAudioSegment(base64, 0, {
voiceSessionId,
clientSegmentId: `${voiceSessionId}-0`,
isLast: true,
duration: durationSec,
});
if (!sent) {
setError('语音发送失败,连接未就绪');
return false;
}
setAwaitingAssistantReply(true);
const localId = `pending_voice_${Date.now()}`;
await voiceSegmentStore.recordSentSegment({
voiceSessionId,
conversationId,
fileUri: uri,
durationMs,
});
queryClient.setQueryData<MessageItem[]>(
conversationKeys.messages(conversationId),
(old) => {
const msg: MessageItem = {
id: localId,
conversationId,
content: '',
senderType: 'user',
timestamp: Date.now(),
messageType: 'voice',
voiceSessionId,
durationSeconds: durationSec,
audioUri: uri,
};
return [...(old ?? []), msg];
},
);
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
(old) => {
if (!old) return old;
return old.map((item) =>
item.id === conversationId
? { ...item, hasUserMessage: true }
: item,
);
},
);
onTtsPlaybackResume?.();
return true;
} catch {
setError('语音文件读取失败');
return false;
}
},
[conversationId, queryClient, onTtsPlaybackResume],
);
const sendEndConversation = useCallback(() => {
sessionRef.current?.sendEndConversation();
}, []);
const sendTtsCancel = useCallback(() => {
sessionRef.current?.sendTtsCancel();
}, []);
return {
connectionState,
streamingMessage,
awaitingAssistantReply,
error,
sendText,
sendVoiceMessage,
sendEndConversation,
sendTtsCancel,
};
}