Skip to content

Commit 2df5044

Browse files
committed
feat(classroom,feynman): local ASR IPC groundwork, AI error retry, nginx health timeout
- feat(classroom): whitelist local_asr_* IPC channels in preload; add AsrModelPrompt download guide card; fix audioBase64 race condition (capture before async slot) - fix(feynman): add retry/resubmit buttons on AI evaluation & question error states; mode-aware AI tooltip; neutral message while gateway status detecting - fix(nginx): raise /health proxy_read_timeout 5s->12s to avoid false fallback when parallel provider pings exceed 5s
1 parent 506db0a commit 2df5044

12 files changed

Lines changed: 234 additions & 21 deletions

File tree

‎client/electron/preload.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,14 @@ const ALLOWED_CHANNELS = [
9898
'performance:set-mode',
9999
// P3-18 性能诊断 IPC channel
100100
'perf:get-metrics',
101+
// 本地 ASR(sherpa-onnx)IPC channel
102+
'local_asr_transcribe',
103+
'local_asr_get_config',
104+
'local_asr_update_config',
105+
'local_asr_check_available',
106+
'local_asr_get_models',
107+
'local_asr_download_model',
108+
'local_asr_delete_model',
101109
] as const;
102110

103111
/** 允许渲染进程监听的事件 channel 白名单(主进程 → 渲染进程推送) */
@@ -122,6 +130,8 @@ const ALLOWED_EVENT_CHANNELS = [
122130
'ai:stream:chunk',
123131
'ai:stream:end',
124132
'ai:stream:error',
133+
// 本地 ASR 模型下载进度推送
134+
'local_asr_download_progress',
125135
] as const;
126136

127137
/** 允许渲染进程单向发送的 channel 白名单(渲染进程 → 主进程,fire-and-forget) */
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/**
2+
* 本地 ASR 模型下载引导卡片
3+
*
4+
* 首次进入课堂助手时检测本地 ASR 模型是否已下载,
5+
* 未下载则显示引导卡片供用户选择下载(可选增强,不阻断使用)。
6+
* 用户关闭后记录 localStorage 不再重复弹出。
7+
*
8+
* @ai-context: 可选增强组件:检测 local_asr_check_available IPC,
9+
* 未下载时展示模型选择与下载进度;关闭后持久化到 localStorage。
10+
* 下载进度经 local_asr_download_progress 事件推送。
11+
*/
12+
import { useState, useEffect, useCallback } from 'react';
13+
import { Download, X, HardDrive, CheckCircle2, Loader2 } from 'lucide-react';
14+
import { cn } from '@/lib/utils';
15+
16+
const DISMISS_KEY = 'kb_asr_model_prompt_dismissed';
17+
18+
interface AsrModelInfo {
19+
engine: string;
20+
id: string;
21+
label: string;
22+
description: string;
23+
size: string;
24+
ready: boolean;
25+
}
26+
27+
interface DownloadProgress {
28+
engine: string;
29+
progress: number;
30+
}
31+
32+
export function AsrModelPrompt() {
33+
const [visible, setVisible] = useState(false);
34+
const [models, setModels] = useState<AsrModelInfo[]>([]);
35+
const [downloading, setDownloading] = useState<string | null>(null);
36+
const [progress, setProgress] = useState(0);
37+
38+
// 初始检测:是否已关闭过 / 模型是否已下载
39+
useEffect(() => {
40+
if (localStorage.getItem(DISMISS_KEY)) return;
41+
if (!window.electronAPI) return;
42+
43+
window.electronAPI.invoke('local_asr_get_models').then((result: unknown) => {
44+
const data = result as { models: AsrModelInfo[] };
45+
if (data?.models) {
46+
setModels(data.models);
47+
// 至少有一个模型已下载则不再提示
48+
const anyDownloaded = data.models.some((m) => m.ready);
49+
if (!anyDownloaded) setVisible(true);
50+
}
51+
}).catch(() => { /* sherpa-onnx 未安装时静默跳过 */ });
52+
}, []);
53+
54+
// 监听下载进度事件
55+
useEffect(() => {
56+
if (!window.electronAPI) return;
57+
const off = window.electronAPI.on('local_asr_download_progress', (...args: unknown[]) => {
58+
const data = args[0] as DownloadProgress;
59+
setProgress(data.progress);
60+
if (data.progress >= 100) {
61+
setDownloading(null);
62+
setProgress(0);
63+
// 刷新模型状态
64+
window.electronAPI.invoke('local_asr_get_models').then((result: unknown) => {
65+
const d = result as { models: AsrModelInfo[] };
66+
if (d?.models) setModels(d.models);
67+
}).catch(() => {});
68+
}
69+
});
70+
return off;
71+
}, []);
72+
73+
const handleDownload = useCallback(async (engine: string) => {
74+
if (!window.electronAPI || downloading) return;
75+
setDownloading(engine);
76+
setProgress(0);
77+
try {
78+
await window.electronAPI.invoke('local_asr_download_model', { engine, useMirror: true });
79+
} catch (err) {
80+
console.warn('[AsrModelPrompt] 下载失败:', err);
81+
setDownloading(null);
82+
}
83+
}, [downloading]);
84+
85+
const handleDismiss = useCallback(() => {
86+
setVisible(false);
87+
localStorage.setItem(DISMISS_KEY, '1');
88+
}, []);
89+
90+
if (!visible) return null;
91+
92+
return (
93+
<div className="mx-4 mt-3 p-4 rounded-kb-lg bg-bg-elevated border border-border/40 shadow-kb-sm">
94+
<div className="flex items-start justify-between">
95+
<div className="flex items-center gap-2">
96+
<HardDrive className="w-4 h-4 text-brand-500" strokeWidth={1.5} />
97+
<span className="text-b2 font-medium text-text-primary">下载本地语音识别模型</span>
98+
</div>
99+
<button
100+
onClick={handleDismiss}
101+
className="p-1 rounded-kb-sm text-text-tertiary hover:text-text-secondary hover:bg-bg-secondary transition-colors"
102+
aria-label="关闭提示"
103+
>
104+
<X className="w-3.5 h-3.5" strokeWidth={1.5} />
105+
</button>
106+
</div>
107+
108+
<p className="mt-1.5 text-c1 text-text-tertiary">
109+
下载后可完全离线转写语音,无需联网、零 API 费用。不下载也可正常使用(云端转写)。
110+
</p>
111+
112+
<div className="mt-3 space-y-2">
113+
{models.map((model) => (
114+
<div key={model.id} className="flex items-center gap-3 p-2.5 rounded-kb-md bg-bg-secondary/50">
115+
<div className="flex-1 min-w-0">
116+
<p className="text-b3 font-medium text-text-primary truncate">{model.label}</p>
117+
<p className="text-c1 text-text-tertiary mt-0.5">{model.description} · {model.size}</p>
118+
</div>
119+
{model.ready ? (
120+
<span className="flex items-center gap-1 text-c1 text-emerald-600 flex-shrink-0">
121+
<CheckCircle2 className="w-3.5 h-3.5" strokeWidth={1.5} />
122+
已下载
123+
</span>
124+
) : downloading === model.engine ? (
125+
<span className="flex items-center gap-1.5 text-c1 text-brand-600 flex-shrink-0">
126+
<Loader2 className="w-3.5 h-3.5 animate-spin" strokeWidth={1.5} />
127+
{progress}%
128+
</span>
129+
) : (
130+
<button
131+
onClick={() => handleDownload(model.engine)}
132+
disabled={!!downloading}
133+
className={cn(
134+
'flex items-center gap-1 px-2.5 py-1.5 rounded-kb-sm text-c1 font-medium transition-colors flex-shrink-0',
135+
downloading
136+
? 'bg-bg-secondary text-text-tertiary cursor-not-allowed'
137+
: 'bg-brand-600 text-white hover:bg-brand-700',
138+
)}
139+
>
140+
<Download className="w-3 h-3" strokeWidth={1.5} />
141+
下载
142+
</button>
143+
)}
144+
</div>
145+
))}
146+
</div>
147+
148+
{/* 下载进度条 */}
149+
{downloading && (
150+
<div className="mt-3 h-1.5 rounded-full bg-bg-secondary overflow-hidden">
151+
<div
152+
className="h-full rounded-full bg-brand-500 transition-all duration-300"
153+
style={{ width: `${progress}%` }}
154+
/>
155+
</div>
156+
)}
157+
</div>
158+
);
159+
}

‎client/src/features/classroom/hooks/useClassroomEvents.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,12 +199,15 @@ export function useClassroomEvents({
199199
}));
200200

201201
// 后台流式 ASR 转写(不阻塞采集,受并发控制)
202-
if (!seg.audioBase64) return;
202+
// ⚠️ 必须同步捕获 audioBase64:vadMarker 在 emit 后立即清空原对象字段释放内存,
203+
// 而 slot.then() 是微任务(异步),此时 seg.audioBase64 已为 ''。
204+
const audioData = seg.audioBase64;
205+
if (!audioData) return;
203206
const slot = asr.acquire();
204207
if (!slot) return; // 队列已满且丢弃了本段
205208
slot.then(() => {
206209
transcribeWithRetry({
207-
audio_base64: seg.audioBase64,
210+
audio_base64: audioData,
208211
language: toAsrLanguage(language),
209212
sample_rate: 16000,
210213
channels: 1,

‎client/src/features/classroom/hooks/useSessionControl.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { useCallback } from 'react';
1212
import { requireGatewayUrl } from '@/lib/ai/config';
1313
import { soundPlayer } from '@/lib/audio/SoundPlayer';
1414
import { analyzePartial } from '@/lib/ai/sessionAnalyzer';
15+
import { refreshLocalAsrStatus } from '../utils/asrTranscriber';
1516
import { getAudioSourcePreference } from '@/lib/capture/audioSourcePreference';
1617
import type { AudioSourceKind } from '@/lib/capture/audioSourceStrategy';
1718
import type {
@@ -96,6 +97,9 @@ export function useSessionControl({
9697

9798
await probeGateway();
9899

100+
// 刷新本地 ASR 可用性缓存(会话开始时检测一次,避免每段都 IPC 查询)
101+
refreshLocalAsrStatus().catch(() => { /* 静默失败,降级走云端 */ });
102+
99103
// Path C 全程录制:走独立 IPC 通道,不启动截图/音频流水线
100104
if (capturePath === 'full_record') {
101105
await captureManager.startSession({

‎client/src/features/classroom/pages/ClassroomPage.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import { CourseInfoCard } from '../components/CourseInfoCard';
2727
import { SmartCapturePanel } from '@/features/notes/components/SmartCapturePanel';
2828
import { AnalysisPreview } from '@/features/notes/components/AnalysisPreview';
2929
import { VideoRecordPanel } from '@/features/notes/components/VideoRecordPanel';
30+
import { AsrModelPrompt } from '../components/AsrModelPrompt';
3031

3132
export default function ClassroomPage() {
3233
const capture = useClassroomCapture();
@@ -158,6 +159,8 @@ export default function ClassroomPage() {
158159
{showIdleGuide ? (
159160
/* 空态:当前配置说明书 + 课程信息卡 */
160161
<div className="flex-1 overflow-y-auto">
162+
{/* 本地 ASR 模型下载引导(首次进入时显示,可选增强) */}
163+
<AsrModelPrompt />
161164
<IdleGuidePanel
162165
capturePath={capture.capturePath}
163166
mode={capture.mode}

‎client/src/features/classroom/utils/asrTranscriber.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ async function transcribeCloud(payload: TranscribePayload): Promise<string | nul
102102
* ASR 转写(本地优先,云端降级)
103103
*
104104
* 策略:
105-
* 1. 本地 ASR 可用 → IPC 调用 whisper.cpp(零成本、离线可用)
105+
* 1. 本地 ASR 可用 → IPC 调用 sherpa-onnx(零成本、离线可用)
106106
* 2. 本地失败 + fallbackToCloud → 降级到云端网关(15s 超时,重试 1 次)
107107
* 3. 本地不可用 → 直接走云端
108108
*/

‎client/src/features/feynman/components/AIEvaluationResult.tsx‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* 评估的综合评分/维度进度条/优势/待改进/建议。数据来自 useAIEvaluate,
66
* 评分阈值文案(≥80 出色 / ≥60 较好)为产品定义,勿随意调整。
77
*/
8-
import { X, Sparkles, CheckCircle2, Circle, RotateCcw } from 'lucide-react';
8+
import { X, Sparkles, CheckCircle2, Circle, RotateCcw, RefreshCw } from 'lucide-react';
99
import { AIThinkingIndicator } from '@/components/ui/AIThinkingIndicator';
1010
import { cn } from '@/lib/utils';
1111
import type { EvaluateResult } from '@/lib/ai/types';
@@ -19,10 +19,12 @@ interface AIEvaluationResultProps {
1919
onGoSettings: () => void;
2020
/** v0.30: 重置 AI 反馈(用户建议) */
2121
onReset?: () => void;
22+
/** v0.31: 评估失败后重新评估 */
23+
onRetry?: () => void;
2224
}
2325

2426
export function AIEvaluationResult({
25-
loading, error, needsConfig, data, onClose, onGoSettings, onReset,
27+
loading, error, needsConfig, data, onClose, onGoSettings, onReset, onRetry,
2628
}: AIEvaluationResultProps) {
2729
return (
2830
<div className={cn(
@@ -64,14 +66,25 @@ export function AIEvaluationResult({
6466
{error && !loading && (
6567
<div className="p-3 rounded-kb-md bg-semantic-error/10 border border-semantic-error/20 text-b2 text-semantic-error">
6668
{error}
67-
{needsConfig && (
68-
<button
69-
onClick={onGoSettings}
70-
className="mt-2 block text-b3 underline hover:no-underline"
71-
>
72-
前往设置页配置 API Key
73-
</button>
74-
)}
69+
<div className="flex items-center gap-3 mt-2">
70+
{onRetry && (
71+
<button
72+
onClick={onRetry}
73+
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-kb-md text-b3 font-medium bg-bg-secondary text-text-secondary border border-border/50 hover:bg-bg-tertiary hover:text-text-primary active:scale-95 transition-all duration-kb-fast"
74+
>
75+
<RefreshCw className="w-3.5 h-3.5" strokeWidth={1.5} />
76+
重新评估
77+
</button>
78+
)}
79+
{needsConfig && (
80+
<button
81+
onClick={onGoSettings}
82+
className="text-b3 underline hover:no-underline"
83+
>
84+
前往设置页配置 API Key
85+
</button>
86+
)}
87+
</div>
7588
</div>
7689
)}
7790

‎client/src/features/feynman/components/AIQuestionPanel.tsx‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
* onSubmit 回调注入。理解度评分阈值文案(≥8 举一反三 / ≥6 较好)为
77
* 产品定义。数据来自 useAIFeynmanQuestion / useAIFeynmanEvaluateAnswers。
88
*/
9-
import { X, Check, Sparkles, MessageCircle, CheckCircle2, Circle } from 'lucide-react';
9+
import { X, Check, Sparkles, MessageCircle, CheckCircle2, Circle, RefreshCw } from 'lucide-react';
1010
import { AIThinkingIndicator } from '@/components/ui/AIThinkingIndicator';
1111
import { Button } from '@/components/ui';
1212
import { cn } from '@/lib/utils';
@@ -172,14 +172,23 @@ export function AIQuestionPanel({
172172
{answerEvalError && !answerEvalLoading && (
173173
<div className="p-3 rounded-kb-md bg-semantic-error/10 border border-semantic-error/20 text-b2 text-semantic-error">
174174
{answerEvalError}
175-
{answerEvalNeedsConfig && (
175+
<div className="flex items-center gap-3 mt-2">
176176
<button
177-
onClick={onGoSettings}
178-
className="mt-2 block text-b3 underline hover:no-underline"
177+
onClick={onSubmit}
178+
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-kb-md text-b3 font-medium bg-bg-secondary text-text-secondary border border-border/50 hover:bg-bg-tertiary hover:text-text-primary active:scale-95 transition-all duration-kb-fast"
179179
>
180-
前往设置页配置 API Key
180+
<RefreshCw className="w-3.5 h-3.5" strokeWidth={1.5} />
181+
重新提交
181182
</button>
182-
)}
183+
{answerEvalNeedsConfig && (
184+
<button
185+
onClick={onGoSettings}
186+
className="text-b3 underline hover:no-underline"
187+
>
188+
前往设置页配置 API Key
189+
</button>
190+
)}
191+
</div>
183192
</div>
184193
)}
185194

‎client/src/features/feynman/components/FeynmanChrome.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { motion, useReducedMotion } from 'framer-motion';
99
import { ArrowLeft, ArrowRight, Check, HelpCircle, CheckCircle2 } from 'lucide-react';
1010
import { Button, Skeleton, EmptyState } from '@/components/ui';
1111
import { AIButton } from '@/components/ui/AIButton';
12+
import { useModeState } from '@/hooks/useMode';
1213
import { cn } from '@/lib/utils';
1314

1415
// ── 环境光 ──
@@ -52,6 +53,7 @@ interface FeynmanTopBarProps {
5253
export function FeynmanTopBar({
5354
concept, hasExplanation, isCompleted, aiEvalLoading, onBack, onRescue, onAIEval,
5455
}: FeynmanTopBarProps) {
56+
const { config } = useModeState();
5557
return (
5658
<motion.div
5759
className="flex items-center gap-kb-sm px-kb-md py-3 border-b border-border/50 flex-shrink-0 relative z-10"
@@ -89,7 +91,7 @@ export function FeynmanTopBar({
8991
size="sm"
9092
loading={aiEvalLoading}
9193
disabled={aiEvalLoading}
92-
tooltip="请先完成讲解内容"
94+
tooltip={config.aiEnabled ? '请先完成讲解内容' : '当前模式未启用 AI 功能,请切换到混合模式'}
9395
onClick={onAIEval}
9496
title={aiEvalLoading ? 'AI 评估中…' : 'AI 评估讲解质量'}
9597
>

‎client/src/features/feynman/pages/FeynmanSessionPage.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ export default function FeynmanSessionPage() {
200200
onClose={() => setShowAIEval(false)}
201201
onGoSettings={() => navigate('/settings')}
202202
onReset={resetAIResults}
203+
onRetry={handleAIEval}
203204
/>
204205
)}
205206

0 commit comments

Comments
 (0)