kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复：CI 部署环境与 ref 错配、迁移碎片化、图片意图 source_span、章节物化脏版式、会话历史与本地语音不一致

Browse Source 
新增：TTS 上传 COS 与分片、章节 reading_segments 物化与快照、markdown 清洗、会话消息 repository、语音 store 重构与相关测试
This commit is contained in:
Kevin
2026-03-20 16:36:42 +08:00
parent 7317bf10cd
commit 8af37e5e8e
65 changed files with 1704 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
app-expo/src/app/(main)/chapter/[id].tsx
View File

@@ -1,3 +1,5 @@
import { Image } from 'expo-image';
import { LinearGradient } from 'expo-linear-gradient';
import { router, useLocalSearchParams } from 'expo-router';
import { Settings, Trash2, X } from 'lucide-react-native';
import React, { useState } from 'react';
@@ -18,7 +20,11 @@ import { Icon } from '@/components/ui/icon';
import { Text } from '@/components/ui/text';
import { ScreenHeader } from '@/components/screen-header';
import { ScreenGutter } from '@/constants/layout';
import { MarkdownRenderer } from '@/features/memoir/markdown-renderer';
import {
MarkdownRenderer,
ReadingMarkdownHorizontalRuleInColumn,
} from '@/features/memoir/markdown-renderer';
import type { ChapterReadingSegment } from '@/features/memoir/types';
import { cn } from '@/lib/utils';
import { useChapterDetail, useDeleteChapter } from '@/features/memoir/hooks';
@@ -34,6 +40,98 @@ const READING_COLORS = {
outlineVariant: 'rgba(121, 117, 127, 0.3)',
};
/** 章节封面 hero：全宽无水平留白，与 MarkdownRenderer 内 hero 同比例与渐变 */
function ChapterCoverHero({
coverImageUrl,
backgroundColor,
}: {
coverImageUrl: string;
backgroundColor: string;
}) {
const [loadFailed, setLoadFailed] = useState(false);
if (loadFailed) {
return null;
}
return (
<View
style={{
width: '100%',
aspectRatio: 4 / 5,
overflow: 'hidden',
}}
>
<Image
source={{ uri: coverImageUrl }}
accessibilityLabel="Chapter cover"
onError={() => setLoadFailed(true)}
style={{
width: '100%',
height: '100%',
objectFit: 'cover',
}}
/>
<LinearGradient
colors={['transparent', backgroundColor]}
locations={[0.3, 1]}
style={{
position: 'absolute',
left: 0,
right: 0,
top: 0,
bottom: 0,
}}
/>
</View>
);
}
/** 故事段末配图：与 Markdown 内嵌图同比例圆角；无独立主图时不渲染（避免与正文 asset 重复已由后端省略） */
function StorySegmentCover({
asset,
contentWidth,
}: {
asset: NonNullable<ChapterReadingSegment['cover_image']>;
contentWidth: number;
}) {
const url = asset?.url;
const wrap = {
maxWidth: contentWidth,
alignSelf: 'center' as const,
width: '100%' as const,
paddingHorizontal: 20,
marginTop: 20,
marginBottom: 0,
};
if (url) {
return (
<View style={wrap}>
<Image
source={{ uri: url }}
style={{
width: '100%',
aspectRatio: 16 / 9,
borderRadius: 12,
}}
accessibilityLabel={asset?.description ?? 'Story illustration'}
/>
</View>
);
}
return (
<View style={wrap}>
<View
style={{
width: '100%',
aspectRatio: 16 / 9,
borderRadius: 12,
backgroundColor: 'rgba(121, 117, 127, 0.12)',
}}
accessibilityLabel="Story illustration placeholder"
/>
</View>
);
}
type FontSize = 'small' | 'default' | 'large';
type FontFamily = 'serif' | 'sans';
type BackgroundTheme = 'white' | 'sepia';
@@ -306,6 +404,9 @@ export default function ChapterScreen() {
const coverImageUrl = chapter.cover_image?.url ?? null;
const canonicalMarkdown = (chapter.canonical_markdown ?? '').trim();
const renderedAssets = chapter.rendered_assets ?? chapter.images ?? [];
const readingSegments = chapter.reading_segments;
const useReadingSegments =
Array.isArray(readingSegments) && readingSegments.length > 0;
/** 与 ScreenHeader（reading、useSafeArea）可视高度对齐，避免返回栏与首屏内容之间出现空隙 */
const headerOccupiedHeight = Math.max(insets.top, 12) + 56;
@@ -335,7 +436,7 @@ export default function ChapterScreen() {
variant="reading"
absolute
backgroundColor={bgColor}
title={chapter.title}
title={useReadingSegments ? '' : chapter.title}
backAccessibilityLabel={t('chapterReading.back')}
right={
<View style={{ flexDirection: 'row', alignItems: 'center', gap: 4 }}>
@@ -382,15 +483,76 @@ export default function ChapterScreen() {
showsVerticalScrollIndicator={false}
style={{ backgroundColor: bgColor }}
>
<MarkdownRenderer
markdown={canonicalMarkdown}
renderedAssets={renderedAssets}
coverImageUrl={coverImageUrl}
fontSize={fontSize}
fontFamily={fontFamily}
backgroundColor={bgColor}
contentWidth={contentWidth}
/>
{useReadingSegments ? (
<View style={{ flex: 1 }}>
{coverImageUrl ? (
<ChapterCoverHero
coverImageUrl={coverImageUrl}
backgroundColor={bgColor}
/>
) : null}
<View
style={{
maxWidth: contentWidth,
alignSelf: 'center',
width: '100%',
paddingHorizontal: 20,
marginTop: coverImageUrl ? -28 : 0,
paddingTop: coverImageUrl ? 40 : 8,
paddingBottom: 20,
}}
>
<Text
selectable
style={{
fontSize: 26,
fontWeight: '700',
color: READING_COLORS.primary,
lineHeight: 34,
letterSpacing: -0.4,
}}
>
{chapter.title}
</Text>
</View>
{readingSegments!.map((seg, i) => (
<View key={seg.story_id}>
<MarkdownRenderer
markdown={(seg.body_markdown ?? '').trim()}
renderedAssets={renderedAssets}
coverImageUrl={null}
fontSize={fontSize}
fontFamily={fontFamily}
backgroundColor={bgColor}
contentWidth={contentWidth}
enableDropCap={i === 0}
showBottomDivider={false}
/>
{seg.cover_image ? (
<StorySegmentCover
asset={seg.cover_image}
contentWidth={contentWidth}
/>
) : null}
{i < readingSegments!.length - 1 ? (
<ReadingMarkdownHorizontalRuleInColumn
contentWidth={contentWidth}
/>
) : null}
</View>
))}
</View>
) : (
<MarkdownRenderer
markdown={canonicalMarkdown}
renderedAssets={renderedAssets}
coverImageUrl={coverImageUrl}
fontSize={fontSize}
fontFamily={fontFamily}
backgroundColor={bgColor}
contentWidth={contentWidth}
/>
)}
</ScrollView>
<ReadingSettingsModal

189
app-expo/src/app/(main)/conversation/[id].tsx
View File

@@ -1,14 +1,6 @@
import { Image } from 'expo-image';
import { useLocalSearchParams } from 'expo-router';
import {
Mic,
Pause,
Play,
PlusCircle,
Type,
Volume2,
X,
} from 'lucide-react-native';
import { Mic, Pause, Play, PlusCircle, Type, X } from 'lucide-react-native';
import React, { useCallback, useEffect, useRef, useState } from 'react';
import type {
NativeSyntheticEvent,
@@ -37,10 +29,9 @@ import { ScreenHeader } from '@/components/screen-header';
import { useThemeColors } from '@/hooks/use-theme-colors';
import { useMessages, useRealtimeSession } from '@/features/conversation/hooks';
import type { MessageItem } from '@/features/conversation/types';
import { audioFocus } from '@/core/audio/audio-focus';
import { isVoiceMessage } from '@/features/conversation/types';
import { usePlayer } from '@/features/voice/hooks/use-player';
import { useRecorder } from '@/features/voice/hooks/use-recorder';
import { useAudioPlayer, useAudioPlayerStatus } from 'expo-audio';
// Life-Echo chat colors (from HTML reference)
const CHAT_COLORS = {
@@ -109,13 +100,21 @@ function MessageBubble({
item,
agentName,
meLabel,
currentPlaybackUri,
playbackIsPlaying,
onPlayVoiceExclusive,
onPausePlayback,
}: {
item: MessageItem;
agentName: string;
meLabel: string;
currentPlaybackUri: string | null;
playbackIsPlaying: boolean;
onPlayVoiceExclusive: (uri: string) => void;
onPausePlayback: () => void;
}) {
const isUser = item.senderType === 'user';
const isVoice = item.messageType === 'voice';
const isVoice = isVoiceMessage(item);
return (
<View style={[styles.messageRow, isUser && styles.messageRowReverse]}>
@@ -140,19 +139,39 @@ function MessageBubble({
>
{isUser ? meLabel : agentName}
</Text>
<View
style={[
styles.bubble,
isUser ? styles.bubbleUser : styles.bubbleAgent,
]}
>
{isVoice ? (
{isVoice ? (
<View
style={[
styles.bubble,
isUser ? styles.bubbleUser : styles.bubbleAgent,
]}
>
<VoiceMessageBubble
durationSeconds={item.durationSeconds ?? 0}
audioUri={item.audioUri}
isUser={isUser}
isPlaying={
!!item.audioUri &&
playbackIsPlaying &&
currentPlaybackUri === item.audioUri
}
onPlayPress={() => {
if (!item.audioUri) return;
if (playbackIsPlaying && currentPlaybackUri === item.audioUri) {
onPausePlayback();
} else {
onPlayVoiceExclusive(item.audioUri);
}
}}
/>
) : (
</View>
) : (
<View
style={[
styles.bubble,
isUser ? styles.bubbleUser : styles.bubbleAgent,
]}
>
<Text
selectable
style={[
@@ -162,8 +181,8 @@ function MessageBubble({
>
{item.content}
</Text>
)}
</View>
</View>
)}
</View>
</View>
);
@@ -250,33 +269,18 @@ function VoiceMessageBubble({
durationSeconds,
audioUri,
isUser,
isPlaying,
onPlayPress,
}: {
durationSeconds: number;
audioUri?: string;
isUser: boolean;
isPlaying: boolean;
onPlayPress: () => void;
}) {
const player = useAudioPlayer(null);
const status = useAudioPlayerStatus(player);
useEffect(() => {
const { playing, currentTime, duration } = status;
const finished = !playing && duration > 0 && currentTime >= duration - 0.05;
if (finished) {
void audioFocus.release();
}
}, [status]);
const handlePlayPause = useCallback(async () => {
if (!audioUri) return;
if (status.playing) {
player.pause();
} else {
const acquired = await audioFocus.acquireForPlayback();
if (!acquired) return;
player.replace(audioUri);
player.play();
}
}, [audioUri, player, status.playing]);
const handlePlayPause = useCallback(() => {
onPlayPress();
}, [onPlayPress]);
return (
<View
@@ -292,11 +296,11 @@ function VoiceMessageBubble({
pressed && { opacity: 0.7 },
]}
disabled={!audioUri}
accessibilityLabel={status.playing ? 'Pause' : 'Play'}
accessibilityLabel={isPlaying ? 'Pause' : 'Play'}
accessibilityRole="button"
>
<Icon
as={status.playing ? Pause : Play}
as={isPlaying ? Pause : Play}
size={24}
color={isUser ? CHAT_COLORS.onSurface : CHAT_COLORS.onPrimary}
/>
@@ -609,12 +613,44 @@ export default function ConversationScreen() {
const { t } = useTranslation('conversation');
const { t: tApp } = useTranslation('app');
const { data: messages } = useMessages(id);
const { enqueueTtsAudio, status: playerStatus } = usePlayer();
const {
enqueue,
enqueueExclusive,
stop,
status: playerStatus,
currentSource,
} = usePlayer();
const handleTtsSegment = useCallback(
(p: { audioBase64?: string; audioUrl?: string }) => {
if (p.audioBase64) {
void enqueue({
uri: `data:audio/mp3;base64,${p.audioBase64}`,
label: 'TTS',
});
} else if (p.audioUrl) {
void enqueue({ uri: p.audioUrl, label: 'TTS' });
}
},
[enqueue],
);
const handlePlayVoiceExclusive = useCallback(
(uri: string) => {
void enqueueExclusive({ uri, label: 'voice' });
},
[enqueueExclusive],
);
const handlePausePlayback = useCallback(() => {
void stop();
}, [stop]);
const { connectionState, streamingMessage, sendText, sendVoiceMessage } =
useRealtimeSession({
conversationId: id ?? '',
enabled: !!id,
onTtsAudio: enqueueTtsAudio,
onTtsSegment: handleTtsSegment,
});
const handleRecordingComplete = useCallback(
@@ -643,30 +679,6 @@ export default function ConversationScreen() {
const onShow = (e: { endCoordinates: { height: number } }) => {
setIsKeyboardVisible(true);
setKeyboardHeight(e.endCoordinates.height);
// #region agent log
void fetch(
'http://127.0.0.1:7446/ingest/e6437b8c-57a6-4b5a-9fdd-4a69aa1b3a6c',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Debug-Session-Id': 'a82a4c',
},
body: JSON.stringify({
sessionId: 'a82a4c',
hypothesisId: 'H3',
location: 'ConversationScreen:keyboardShow',
message: 'keyboard open; composer uses insetBottom 0 when text+kb',
data: {
kbH: e.endCoordinates.height,
insetBottom: insets.bottom,
os: Platform.OS,
},
timestamp: Date.now(),
}),
},
).catch(() => {});
// #endregion
InteractionManager.runAfterInteractions(() => {
listRef.current?.scrollToEnd({ animated: true });
});
@@ -706,26 +718,6 @@ export default function ConversationScreen() {
sendText(text);
setInput('');
setInputResetKey((k) => k + 1);
// #region agent log
void fetch(
'http://127.0.0.1:7446/ingest/e6437b8c-57a6-4b5a-9fdd-4a69aa1b3a6c',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Debug-Session-Id': 'a82a4c',
},
body: JSON.stringify({
sessionId: 'a82a4c',
hypothesisId: 'H4',
location: 'ConversationScreen:handleSend',
message: 'cleared input + bumped textInputKey',
data: { sentLen: text.length },
timestamp: Date.now(),
}),
},
).catch(() => {});
// #endregion
};
const connectionLabel =
@@ -752,14 +744,6 @@ export default function ConversationScreen() {
title={
<View style={styles.headerTitleBlock}>
<Text style={styles.headerTitle}>{tApp('name')}</Text>
{playerStatus === 'playing' && (
<Icon
as={Volume2}
size={18}
color={CHAT_COLORS.primary}
style={{ marginRight: 6 }}
/>
)}
<View
style={[
styles.statusBadge,
@@ -795,6 +779,10 @@ export default function ConversationScreen() {
item={item}
agentName={t('agentName')}
meLabel={t('me')}
currentPlaybackUri={currentSource}
playbackIsPlaying={playerStatus === 'playing'}
onPlayVoiceExclusive={handlePlayVoiceExclusive}
onPausePlayback={handlePausePlayback}
/>
)}
onContentSizeChange={() =>
@@ -930,6 +918,7 @@ const styles = StyleSheet.create({
alignItems: 'flex-start',
gap: 10,
marginBottom: 16,
overflow: 'visible',
},
messageRowReverse: {
flexDirection: 'row-reverse',

3
app-expo/src/core/ws/client.ts
View File

@@ -46,6 +46,9 @@ function mapServerMessage(raw: RawServerMessage): WsEvent | null {
kind: 'tts_audio_received',
conversationId: cid,
audioBase64: d.audio_base64 as string,
audioUrl: d.audio_url as string | undefined,
index: d.index as number | undefined,
total: d.total as number | undefined,
};
case 'end_conversation':

3
app-expo/src/core/ws/types.ts
View File

@@ -60,6 +60,9 @@ export interface TtsAudioReceivedEvent {
kind: 'tts_audio_received';
conversationId: string;
audioBase64: string;
audioUrl?: string;
index?: number;
total?: number;
}
export interface ConversationEndedEvent {

44
app-expo/src/features/conversation/conversation-messages-repository.ts Normal file
View File

@@ -0,0 +1,44 @@
import { conversationApi } from './api';
import { isVoiceMessage, type MessageItem } from './types';
import { voiceSegmentStore } from '@/features/voice/voice-segment-store';
/**
* 会话消息读端口：REST 历史 + 本机语音回放路径装配。
* 单一入口，供 React Query 与测试注入替代实现。
*/
export interface ConversationMessagesPort {
loadMessages(conversationId: string): Promise<MessageItem[]>;
}
async function attachLocalVoicePlayback(
conversationId: string,
server: MessageItem[],
): Promise<MessageItem[]> {
const playback =
await voiceSegmentStore.listPlaybackForConversation(conversationId);
if (playback.length === 0) return server;
const byVoice = new Map(playback.map((p) => [p.voiceSessionId, p]));
return server.map((m) => {
if (!isVoiceMessage(m)) return m;
const sid = m.voiceSessionId;
if (!sid) return m;
const row = byVoice.get(sid);
if (!row) return m;
const sec = Math.max(1, Math.round(row.durationMs / 1000));
return {
...m,
audioUri: row.fileUri,
durationSeconds:
m.durationSeconds != null && m.durationSeconds > 0
? m.durationSeconds
: sec,
};
});
}
export const conversationMessagesRepository: ConversationMessagesPort = {
async loadMessages(conversationId: string): Promise<MessageItem[]> {
const server = await conversationApi.messages(conversationId);
return attachLocalVoicePlayback(conversationId, server);
},
};

33
app-expo/src/features/conversation/event-handlers.ts
View File

@@ -4,7 +4,11 @@ import i18n from '@/i18n';
import type { WsEvent } from '@/core/ws/types';
import { conversationKeys } from './query-keys';
import type { ConversationListItem, MessageItem } from './types';
import {
type ConversationListItem,
type MessageItem,
isVoiceMessage,
} from './types';
function nowMs(): number {
return Date.now();
@@ -43,14 +47,27 @@ function handleTranscript(
if (!old?.length) return old ?? [];
const lastUser = [...old].reverse().find((m) => m.senderType === 'user');
const isPendingVoice =
lastUser?.id?.startsWith('pending_voice_') &&
lastUser?.messageType === 'voice';
lastUser?.id?.startsWith('pending_voice_') && isVoiceMessage(lastUser);
if (isPendingVoice && lastUser) {
return old.map((m) =>
m.id === lastUser.id && event.audioDuration != null
? { ...m, durationSeconds: Math.round(event.audioDuration) }
: m,
);
return old.map((m) => {
if (m.id !== lastUser.id) return m;
const serverSec =
event.audioDuration != null && event.audioDuration > 0
? Math.round(event.audioDuration)
: null;
const localSec =
m.durationSeconds != null && m.durationSeconds > 0
? m.durationSeconds
: undefined;
const nextDuration = serverSec ?? localSec;
return {
...m,
...(nextDuration != null && nextDuration > 0
? { durationSeconds: nextDuration }
: {}),
...(event.text?.trim() ? { content: event.text.trim() } : {}),
};
});
}
return old;
});

31
app-expo/src/features/conversation/hooks.ts
View File

@@ -5,17 +5,20 @@ import { useCallback, useEffect, useRef, useState } from 'react';
import type { WsConnectionState } from '@/core/ws/types';
import { conversationApi } from './api';
import { conversationMessagesRepository } from './conversation-messages-repository';
import { conversationKeys } from './query-keys';
import {
RealtimeSession,
type ErrorCallback,
type StreamingTextCallback,
type TtsSegmentPayload,
} from './realtime-session';
import type {
ConversationListItem,
MessageItem,
StreamingAgentMessage,
import {
type ConversationListItem,
type MessageItem,
type StreamingAgentMessage,
} from './types';
import { voiceSegmentStore } from '@/features/voice/voice-segment-store';
// ─── Query hooks ───
@@ -38,7 +41,7 @@ export function useConversationDetail(conversationId: string) {
export function useMessages(conversationId: string) {
return useQuery({
queryKey: conversationKeys.messages(conversationId),
queryFn: () => conversationApi.messages(conversationId),
queryFn: () => conversationMessagesRepository.loadMessages(conversationId),
enabled: !!conversationId,
});
}
@@ -76,7 +79,8 @@ export function useDeleteConversation() {
return useMutation({
mutationFn: (conversationId: string) =>
conversationApi.delete(conversationId),
onSuccess: (_, conversationId) => {
onSuccess: async (_, conversationId) => {
await voiceSegmentStore.clearConversation(conversationId);
queryClient.setQueryData<ConversationListItem[]>(
conversationKeys.lists(),
(old) => old?.filter((item) => item.id !== conversationId),
@@ -112,7 +116,7 @@ export function useEndConversation() {
interface UseRealtimeSessionOptions {
conversationId: string;
enabled?: boolean;
onTtsAudio?: (audioBase64: string) => void;
onTtsSegment?: (payload: TtsSegmentPayload) => void;
}
const MIN_RECORDING_DURATION_SEC = 1;
@@ -137,7 +141,7 @@ interface RealtimeSessionState {
export function useRealtimeSession({
conversationId,
enabled = true,
onTtsAudio,
onTtsSegment,
}: UseRealtimeSessionOptions): RealtimeSessionState {
const queryClient = useQueryClient();
const sessionRef = useRef<RealtimeSession | null>(null);
@@ -170,7 +174,7 @@ export function useRealtimeSession({
conversationId,
queryClient,
onStreamingText: handleStreamingText,
onTtsAudio,
onTtsSegment,
onError: handleError,
onStateChange: setConnectionState,
});
@@ -190,7 +194,7 @@ export function useRealtimeSession({
queryClient,
handleStreamingText,
handleError,
onTtsAudio,
onTtsSegment,
]);
const sendText = useCallback(
@@ -249,6 +253,12 @@ export function useRealtimeSession({
}
const localId = `pending_voice_${Date.now()}`;
await voiceSegmentStore.recordSentSegment({
voiceSessionId,
conversationId,
fileUri: uri,
durationMs,
});
queryClient.setQueryData<MessageItem[]>(
conversationKeys.messages(conversationId),
(old) => {
@@ -259,6 +269,7 @@ export function useRealtimeSession({
senderType: 'user',
timestamp: Date.now(),
messageType: 'voice',
voiceSessionId,
durationSeconds: durationSec,
audioUri: uri,
};

22
app-expo/src/features/conversation/realtime-session.ts
View File

@@ -14,11 +14,18 @@ import type { ConversationListItem, MessageItem } from './types';
export type StreamingTextCallback = (text: string, isComplete: boolean) => void;
export type ErrorCallback = (message: string, code?: string) => void;
/** WebSocket `tts_audio`：服务端可能只带 base64、只带 COS URL，或两者都有 */
export type TtsSegmentPayload = {
audioBase64?: string;
audioUrl?: string;
};
interface RealtimeSessionOptions {
conversationId: string;
queryClient: QueryClient;
onStreamingText?: StreamingTextCallback;
onTtsAudio?: (audioBase64: string) => void;
/** 收到 TTS 片段时入队播放（与「气泡上的手动朗读按钮」无关） */
onTtsSegment?: (payload: TtsSegmentPayload) => void;
onError?: ErrorCallback;
onStateChange?: WsStateListener;
}
@@ -39,7 +46,7 @@ export class RealtimeSession {
private conversationId: string;
private queryClient: QueryClient;
private onStreamingText?: StreamingTextCallback;
private onTtsAudio?: (audioBase64: string) => void;
private onTtsSegment?: (payload: TtsSegmentPayload) => void;
private onError?: ErrorCallback;
private unsubEvent: (() => void) | null = null;
private unsubState: (() => void) | null = null;
@@ -51,7 +58,7 @@ export class RealtimeSession {
this.conversationId = options.conversationId;
this.queryClient = options.queryClient;
this.onStreamingText = options.onStreamingText;
this.onTtsAudio = options.onTtsAudio;
this.onTtsSegment = options.onTtsSegment;
this.onError = options.onError;
this.unsubEvent = this.client.onEvent(this.handleEvent);
@@ -121,7 +128,14 @@ export class RealtimeSession {
}
if (event.kind === 'tts_audio_received') {
this.onTtsAudio?.(event.audioBase64);
const b64 = event.audioBase64?.trim();
const url = event.audioUrl?.trim();
if (b64 || url) {
this.onTtsSegment?.({
audioBase64: b64 || undefined,
audioUrl: url || undefined,
});
}
return;
}

13
app-expo/src/features/conversation/types.ts
View File

@@ -48,16 +48,23 @@ export interface EndConversationResponse {
duration_seconds: number;
}
/** 后端历史里语音为 `audio`，与本地乐观更新的 `voice` 同义 */
export function isVoiceMessage(m: Pick<MessageItem, 'messageType'>): boolean {
return m.messageType === 'voice' || m.messageType === 'audio';
}
export interface MessageItem {
id: string;
conversationId: string;
content: string;
senderType: 'user' | 'assistant';
timestamp: number;
messageType: 'text' | 'voice';
/** 语音消息时长（秒），仅 messageType='voice' 时有值 */
messageType: 'text' | 'voice' | 'audio';
/** 与 WS audio_segment 的 voice_session_id 一致，用于关联本地录音文件 */
voiceSessionId?: string;
/** 语音消息时长（秒），语音消息（voice/audio）时有值 */
durationSeconds?: number;
/** 语音文件本地 URI，用于回放，仅 messageType='voice' 时有值 */
/** 语音文件本地 URI，用于回放，仅本地乐观语音条有值 */
audioUri?: string;
}

105
app-expo/src/features/memoir/markdown-renderer.tsx
View File

@@ -98,6 +98,85 @@ const READING_COLORS = {
horizontalRule: 'rgba(121, 117, 127, 0.42)',
};
const HR_HEIGHT = Platform.OS === 'android' ? 2 : 1;
/** 与 Markdown `---` / hr 规则完全一致，供分段阅读插在故事之间 */
export function ReadingMarkdownHorizontalRule() {
return (
<View
style={{
width: '100%',
alignSelf: 'stretch',
height: HR_HEIGHT,
backgroundColor: READING_COLORS.horizontalRule,
marginVertical: 28,
}}
/>
);
}
/** 章末细装饰线（与 MarkdownRenderer showBottomDivider 一致） */
export function ReadingMarkdownEndDivider() {
return (
<View
style={{
paddingTop: 24,
paddingBottom: 24,
alignItems: 'center',
}}
>
<View
style={{
width: 96,
height: 1,
backgroundColor: READING_COLORS.divider,
}}
/>
</View>
);
}
/** 与正文列宽对齐的 `---`（避免单独一行时宽度为 0） */
export function ReadingMarkdownHorizontalRuleInColumn({
contentWidth,
}: {
contentWidth: number;
}) {
return (
<View style={{ width: '100%', paddingHorizontal: 20 }}>
<View
style={{
maxWidth: contentWidth,
alignSelf: 'center',
width: '100%',
}}
>
<ReadingMarkdownHorizontalRule />
</View>
</View>
);
}
export function ReadingMarkdownEndDividerInColumn({
contentWidth,
}: {
contentWidth: number;
}) {
return (
<View style={{ width: '100%', paddingHorizontal: 20 }}>
<View
style={{
maxWidth: contentWidth,
alignSelf: 'center',
width: '100%',
}}
>
<ReadingMarkdownEndDivider />
</View>
</View>
);
}
const FONT_FAMILIES = {
serif:
Platform.select({ ios: 'Georgia', android: 'serif', default: 'serif' }) ??
@@ -121,6 +200,10 @@ export interface MarkdownRendererProps {
fontFamily: 'serif' | 'sans';
backgroundColor: string;
contentWidth: number;
/** 多故事分段时仅首段下沉首字 */
enableDropCap?: boolean;
/** 文末装饰分隔线（分段中间可关） */
showBottomDivider?: boolean;
}
export function MarkdownRenderer({
@@ -131,6 +214,8 @@ export function MarkdownRenderer({
fontFamily,
backgroundColor,
contentWidth,
enableDropCap = true,
showBottomDivider = true,
}: MarkdownRendererProps) {
const processedMarkdown = React.useMemo(
() => replaceImagePlaceholders(markdown, renderedAssets),
@@ -213,7 +298,7 @@ export function MarkdownRenderer({
width: '100%',
alignSelf: 'stretch',
backgroundColor: READING_COLORS.horizontalRule,
height: Platform.OS === 'android' ? 2 : StyleSheet.hairlineWidth,
height: HR_HEIGHT,
marginVertical: 28,
},
image: {
@@ -249,16 +334,9 @@ export function MarkdownRenderer({
return {
hr: (node: { key: string }) => (
<View
key={node.key}
style={{
width: '100%',
alignSelf: 'stretch',
height: Platform.OS === 'android' ? 2 : StyleSheet.hairlineWidth,
backgroundColor: READING_COLORS.horizontalRule,
marginVertical: 28,
}}
/>
<View key={node.key}>
<ReadingMarkdownHorizontalRule />
</View>
),
image: (
node: { key: string; attributes: Record<string, string | undefined> },
@@ -330,6 +408,7 @@ export function MarkdownRenderer({
const afterWs = content.slice(leadingWs.length);
const docFirstParagraph = isFirstParagraphUnderBody(paragraph, body);
const wantDropCap =
enableDropCap &&
docFirstParagraph &&
!dropCapConsumedRef.current &&
afterWs.length > 0;
@@ -377,7 +456,7 @@ export function MarkdownRenderer({
);
},
};
}, [bodySize, fontFam, lineHeight, markdownStyles.image]);
}, [bodySize, enableDropCap, fontFam, lineHeight, markdownStyles.image]);
return (
<>
@@ -435,7 +514,7 @@ export function MarkdownRenderer({
</Markdown>
) : null}
{processedMarkdown.trim().length > 0 && (
{showBottomDivider && processedMarkdown.trim().length > 0 && (
<View
style={{
paddingTop: 24,

9
app-expo/src/features/memoir/types.ts
View File

@@ -49,6 +49,13 @@ export interface ChapterSection {
image: ImageAsset | null;
}
/** 章节详情：与 chapter_story_links 顺序一致，每段故事正文 + 主配图 */
export interface ChapterReadingSegment {
story_id: string;
body_markdown: string;
cover_image: ImageAsset | null;
}
export interface Chapter {
id: string;
title: string;
@@ -68,6 +75,8 @@ export interface Chapter {
canonical_markdown?: string | null;
/** 图片等资源映射，与 canonical_markdown 配合使用 */
rendered_assets?: ImageAsset[];
/** 有 story 编排时的分段阅读（正文不含故事标题，配图按故事） */
reading_segments?: ChapterReadingSegment[];
updated_at: string | null;
is_new: boolean;
source_segments: unknown[];

65
app-expo/src/features/voice/hooks/use-player.ts
View File

@@ -8,8 +8,11 @@ import type { PlaybackItem, PlayerStatus } from '../types';
interface UsePlayerResult {
status: PlayerStatus;
queueLength: number;
enqueueTtsAudio: (audioBase64: string) => void;
/** Current playback source URI (file, https, or data URL). */
currentSource: string | null;
enqueue: (item: PlaybackItem) => void;
/** Replace queue and play this item (e.g. user voice bubble vs other sources). */
enqueueExclusive: (item: PlaybackItem) => Promise<void>;
stop: () => void;
}
@@ -29,8 +32,12 @@ export function usePlayer(): UsePlayerResult {
const isPlayingRef = useRef(false);
const wasBlockedByRecorderRef = useRef(false);
const isPlayNextInProgressRef = useRef(false);
/** 同步反映「当前是否正在播放某条 URI」；enqueue 不能依赖 state，否则 await stop() 后仍为陈旧闭包。 */
const playbackActiveUriRef = useRef<string | null>(null);
/** 当前 source 是否已进入过 playing=true，避免换源瞬间 playerStatus 仍带上一首的 duration 而误判「已播完」。 */
const trackHasPlayedRef = useRef(false);
const player = useAudioPlayer(currentSource);
const player = useAudioPlayer(currentSource, { downloadFirst: false });
const playerStatus = useAudioPlayerStatus(player);
// Start playback when a new source is set
@@ -46,6 +53,7 @@ export function usePlayer(): UsePlayerResult {
isPlayNextInProgressRef.current = true;
try {
if (queueRef.current.length === 0) {
playbackActiveUriRef.current = null;
setCurrentSource(null);
setStatus('idle');
setQueueLength(0);
@@ -64,20 +72,33 @@ export function usePlayer(): UsePlayerResult {
const next = queueRef.current.shift()!;
setQueueLength(queueRef.current.length);
setStatus('playing');
trackHasPlayedRef.current = false;
playbackActiveUriRef.current = next.uri;
setCurrentSource(next.uri);
} finally {
isPlayNextInProgressRef.current = false;
}
}, []);
// Detect playback completion → advance queue
useEffect(() => {
if (playerStatus.playing) {
trackHasPlayedRef.current = true;
}
}, [playerStatus.playing]);
// Detect playback completion → advance queue（必须曾 playing，避免换源瞬间沿用上一条的 duration/currentTime）
useEffect(() => {
if (!currentSource || !isPlayingRef.current) return;
const { playing, currentTime, duration } = playerStatus;
const finished = !playing && duration > 0 && currentTime >= duration - 0.05;
const finished =
trackHasPlayedRef.current &&
!playing &&
duration > 0 &&
currentTime >= duration - 0.05;
if (finished) {
trackHasPlayedRef.current = false;
isPlayingRef.current = false;
playNext();
}
@@ -107,19 +128,31 @@ export function usePlayer(): UsePlayerResult {
queueRef.current.push(item);
setQueueLength(queueRef.current.length);
if (status === 'idle' && !currentSource) {
const shouldKick =
queueRef.current.length === 1 && playbackActiveUriRef.current === null;
if (shouldKick) {
await playNext();
}
},
[status, currentSource, playNext],
[playNext],
);
const enqueueTtsAudio = useCallback(
(audioBase64: string) => {
const uri = `data:audio/mp3;base64,${audioBase64}`;
enqueue({ uri, label: 'TTS' });
const enqueueExclusive = useCallback(
async (item: PlaybackItem) => {
queueRef.current = [item];
setQueueLength(1);
isPlayingRef.current = false;
if (player) {
player.pause();
}
playbackActiveUriRef.current = null;
setCurrentSource(null);
setStatus('idle');
await audioFocus.release();
await playNext();
},
[enqueue],
[player, playNext],
);
const stop = useCallback(async () => {
@@ -131,10 +164,18 @@ export function usePlayer(): UsePlayerResult {
player.pause();
}
playbackActiveUriRef.current = null;
setCurrentSource(null);
setStatus('idle');
await audioFocus.release();
}, [player]);
return { status, queueLength, enqueueTtsAudio, enqueue, stop };
return {
status,
queueLength,
currentSource,
enqueue,
enqueueExclusive,
stop,
};
}

2
app-expo/src/features/voice/types.ts
View File

@@ -11,7 +11,7 @@ export interface SegmenterConfig {
fixedDurationMs: number;
}
// ─── Segment outbox ───
// ─── 本地语音分段（outbox + 已发送可回放元数据，见 voice-segment-store）───
export type SegmentOutboxStatus = 'pending' | 'sending' | 'sent' | 'failed';

98
app-expo/src/features/voice/segment-outbox.ts → app-expo/src/features/voice/voice-segment-store.ts
View File

@@ -18,9 +18,29 @@ const CREATE_TABLE_SQL = `
let initialized = false;
async function migrateLegacyVoiceMessageLocal(): Promise<void> {
const rows = await querySql<{ name: string }>(
`SELECT name FROM sqlite_master WHERE type='table' AND name='voice_message_local'`,
);
if (rows.length === 0) return;
const now = Date.now();
await executeSql(
`INSERT OR IGNORE INTO segment_outbox (conversation_id, voice_session_id, segment_index, file_uri, duration_ms, status, retry_count, created_at)
SELECT conversation_id, voice_session_id, 0, file_uri, duration_ms, 'sent', 0, ?
FROM voice_message_local`,
[now],
);
await executeSql(`DROP TABLE IF EXISTS voice_message_local`);
}
async function ensureTable(): Promise<void> {
if (initialized) return;
await executeSql(CREATE_TABLE_SQL);
await executeSql(
`CREATE UNIQUE INDEX IF NOT EXISTS uq_segment_outbox_voice_session_segment
ON segment_outbox(voice_session_id, segment_index)`,
);
await migrateLegacyVoiceMessageLocal();
initialized = true;
}
@@ -38,14 +58,17 @@ function mapRow(row: Record<string, unknown>): SegmentOutboxEntry {
};
}
export interface VoicePlaybackRow {
voiceSessionId: string;
fileUri: string;
durationMs: number;
}
/**
* SQLite-backed outbox for voice segments.
* Stores metadata + queue state only — audio files live on the local filesystem.
*
* State machine: pending → sending → sent | failed
* Single writer pattern enforced by serializing all writes through this module.
* 本地语音分段：outbox（pending→sent）与可回放元数据共用同一张表。
* 音频文件在文件系统；`status=sent` 行保留用于按 voice_session_id 关联 REST 历史。
*/
export const segmentOutbox = {
export const voiceSegmentStore = {
async enqueue(
entry: Omit<
SegmentOutboxEntry,
@@ -68,6 +91,53 @@ export const segmentOutbox = {
return result.lastInsertRowId;
},
/** 发送成功后写入（或覆盖）同一条 voice+segment，用于回放与 outbox 终态统一 */
async recordSentSegment(entry: {
conversationId: string;
voiceSessionId: string;
segmentIndex?: number;
fileUri: string;
durationMs: number;
}): Promise<void> {
await ensureTable();
const segmentIndex = entry.segmentIndex ?? 0;
const now = Date.now();
await executeSql(
`INSERT INTO segment_outbox (conversation_id, voice_session_id, segment_index, file_uri, duration_ms, status, retry_count, created_at)
VALUES (?, ?, ?, ?, ?, 'sent', 0, ?)
ON CONFLICT(voice_session_id, segment_index) DO UPDATE SET
conversation_id = excluded.conversation_id,
file_uri = excluded.file_uri,
duration_ms = excluded.duration_ms,
status = 'sent',
retry_count = 0`,
[
entry.conversationId,
entry.voiceSessionId,
segmentIndex,
entry.fileUri,
entry.durationMs,
now,
],
);
},
async listPlaybackForConversation(
conversationId: string,
): Promise<VoicePlaybackRow[]> {
await ensureTable();
const rows = await querySql<Record<string, unknown>>(
`SELECT voice_session_id AS voiceSessionId, file_uri AS fileUri, duration_ms AS durationMs
FROM segment_outbox WHERE conversation_id = ? AND status = 'sent'`,
[conversationId],
);
return rows.map((r) => ({
voiceSessionId: r.voiceSessionId as string,
fileUri: r.fileUri as string,
durationMs: r.durationMs as number,
}));
},
async getPending(conversationId?: string): Promise<SegmentOutboxEntry[]> {
await ensureTable();
const sql = conversationId
@@ -110,24 +180,18 @@ export const segmentOutbox = {
await executeSql(sql, params);
},
async clearSent(conversationId?: string): Promise<void> {
await ensureTable();
const sql = conversationId
? `DELETE FROM segment_outbox WHERE status = 'sent' AND conversation_id = ?`
: `DELETE FROM segment_outbox WHERE status = 'sent'`;
const params = conversationId ? [conversationId] : [];
await executeSql(sql, params);
},
async clearAll(conversationId: string): Promise<void> {
async clearConversation(conversationId: string): Promise<void> {
await ensureTable();
await executeSql(`DELETE FROM segment_outbox WHERE conversation_id = ?`, [
conversationId,
]);
},
/** For testing — reset the initialization flag. */
/** @internal 测试用 */
_resetForTest(): void {
initialized = false;
},
} as const;
/** @deprecated 使用 voiceSegmentStore */
export const segmentOutbox = voiceSegmentStore;