diff --git a/apps/course/catalog.json b/apps/course/catalog.json index 3a62dcc..887f267 100644 --- a/apps/course/catalog.json +++ b/apps/course/catalog.json @@ -4,8 +4,8 @@ "order": 1, "title": "文字怎样变成 Token？", "outcome": "观察同一句话如何分成片段并映射为编号；区分字符、词和 Token。", - "status": "planned", - "route": null, + "status": "reviewed", + "route": "/learn/tokenization/", "published": false }, { diff --git a/apps/site/learn/tokenization/LESSON.md b/apps/site/learn/tokenization/LESSON.md new file mode 100644 index 0000000..c8a2343 --- /dev/null +++ b/apps/site/learn/tokenization/LESSON.md @@ -0,0 +1,25 @@ +# Token 分词教学场景 + +核心问题：一个词如何按已有规则变成编号序列？46秒从固定英文词开始，展开基础字符，按三条规则合并相邻片段，检查Token边界，查表得到ID，最后解码还原。 + +表现：上方刻度保留输入字符，下方片段宽度准确覆盖字符跨度；颜色取自片段起点，合并后边界改变而输入不变。宽度不编码概率。编号沿原顺序出现，指引跟随当前片段。 + +规则来自 [Hugging Face BPE 教学章节](https://huggingface.co/learn/llm-course/en/chapter6/5) 的已训练示例：u+g、u+n、h+ug。本实现为独立教学代码。词表编号从0起设定，不是生产模型ID。切换0–3条规则代表不同教学版本，绝不是对每个输入重新训练。本节不实现字节级编码、空格处理、中文或特殊Token，不运行语言模型。 + +反事实实验：固定 hugs，规则0条得到4个Token；规则1条得到3个；规则3条得到hug|s两个Token。字符数一直为4，解码一直为hugs。无匹配的u+n规则在hugs上是无操作。 + +迁移检验：输入unhug，先预测有几个Token，再解释u|n|h|ug → un|h|ug → un|hug，最后在图中验证[8,9]。错误答案提供中间步骤。 + +测试命令：node apps/site/learn/tokenization/model.test.mjs。独立黄金值覆盖全部五个词；检查所有规则版本与每0.1秒状态下的字符跨度守恒、顺序、解码、反向跳转、无匹配规则和输入范围。 + +教学评审：角色、规则、输出均可见；分词和理解/生成明确区分；提供受控实验与新词预测。这是开发者教学适配评审，没有真实学习者效果研究。浏览器验收在发布前补充。 + +复用请求：用单舞台、浅色细线的可操作图解，解释一个词如何按固定BPE合并规则变成编号。每次只解释当前对象，能暂停、倒退、换词和切换规则版本；输出来源、简化边界和独立验证。 + +## 2026-10-27 验收记录 + +独立模型测试和全站npm run check通过，既有721条行星约束没有回归。桌面完整播放到46/46恢复配置；正反向跳转能清除/重建片段和编号；390×844手机截图核对正文、片段、ID与解码均可读。手机上的错误与正确预测都能给出推导，验证按钮返回unhug、规则3条和[8,9]。已修复SVG留白导致的箭头偏差、片段表面的多余对角线，并让无匹配规则明确显示跳过。 + +静态阶段按阶段/规则/输入缓存SVG，时间轴继续由同一绝对时钟推进；没有每帧重建相同图形。CSS支持减少动态效果，隐藏标签页有暂停处理；本轮云浏览器没有独立实测系统减少动态效果开关和标签页可见性切换，不宣称它们实测通过。没有实体手机、屏幕阅读器或真实学习者研究。 + +规则滑块实测发现暂停重绘会覆盖事件中的新值，已先捕获输入再暂停；最终浏览器复测通过：0条输出[2,6,1,5]共4片段，3条恢复[9,5]共2片段。 diff --git a/apps/site/learn/tokenization/SCENE_SPEC.json b/apps/site/learn/tokenization/SCENE_SPEC.json new file mode 100644 index 0000000..6e94afc --- /dev/null +++ b/apps/site/learn/tokenization/SCENE_SPEC.json @@ -0,0 +1,67 @@ +{ + "question": "同一个词怎样按固定合并规则变成 Token 编号？", + "durationSeconds": 46, + "objects": [ + { + "id": "input", + "meaning": "一个预设英文词" + }, + { + "id": "span", + "meaning": "覆盖输入连续字母的片段；宽度表示字母跨度，不表示概率" + }, + { + "id": "id", + "meaning": "该教学词表的固定索引" + } + ], + "relationships": [ + "每轮合并只覆盖相邻匹配片段", + "所有片段拼接始终等于输入", + "同词表解码编号等于输入", + "规则版本改变时Token数量可能改变" + ], + "inputs": { + "word": [ + "hugs", + "hug", + "pug", + "unhug", + "bun" + ], + "limit": [ + 0, + 1, + 2, + 3 + ] + }, + "outputs": [ + "pieces", + "ids", + "decoded", + "trace" + ], + "simplifications": [ + "已确定规则，非本轮训练", + "预设ASCII英文词；不处理空格、中文、字节级BPE与特殊Token", + "编号与分词结果均非生产模型词表" + ], + "sources": [ + "https://huggingface.co/learn/llm-course/en/chapter6/5" + ], + "acceptance": [ + "独立黄金分词与ID", + "字符跨度守恒", + "解码可逆", + "无匹配规则不改变状态", + "反向seek确定性", + "真实桌面/手机验收" + ], + "untested": [ + "真实学习者效果", + "实体手机", + "独立浏览器引擎", + "屏幕阅读器实机" + ] +} diff --git a/apps/site/learn/tokenization/app.mjs b/apps/site/learn/tokenization/app.mjs new file mode 100644 index 0000000..cc6585b --- /dev/null +++ b/apps/site/learn/tokenization/app.mjs @@ -0,0 +1,21 @@ +import {WORDS,RULES,VOCAB,STARTS,DURATION,sample,narration} from './model.mjs'; +import {createStage} from './stage.mjs'; +const $=s=>document.querySelector(s),$$=s=>[...document.querySelectorAll(s)],stage=createStage($('#artwork')),motion=matchMedia('(prefers-reduced-motion: reduce)'); +let word='hugs',limit=3,time=0,playing=false,raf=0,last=0,lastCopy='',dead=false; +const names=['固定输入','基础符号','按规则合并','Token 边界','查词表编号','解码还原']; +function focus(on){$('#app').classList.toggle('is-playing',on);$$('.secondary').forEach(x=>{x.inert=on;x.setAttribute('aria-hidden',String(on));});} +function anchor(){const box=$('.exhibit').getBoundingClientRect(),bubble=$('#callout').getBoundingClientRect(),art=$('#artwork');art.style.top=Math.max(185,bubble.bottom-box.top+13)+'px';const a=art.getBoundingClientRect(),point=stage.projectAnchor();if(!point)return;const x=a.left-box.left+point.x*a.width,y=a.top-box.top+point.y*a.height,cx=Math.max(bubble.width/2+17,Math.min(box.width-bubble.width/2-17,x));$('#callout').style.left=cx+'px';$('#leader').setAttribute('viewBox',`0 0 ${box.width} ${box.height}`);const sy=bubble.bottom-box.top;$('#leader-line').setAttribute('d',`M${cx} ${sy}C${cx} ${sy+20},${x} ${y-25},${x} ${y-8}`);$('#leader-dot').setAttribute('cx',x);$('#leader-dot').setAttribute('cy',y);} +function render(){if(dead)return;const f=sample(time,{word,limit}),c=narration(f),key=JSON.stringify(c);stage.render(f);if(key!==lastCopy){lastCopy=key;$('#copy-eyebrow').textContent=c.eyebrow;$('#copy-title').textContent=c.title;$('#copy-body').textContent=c.body;}$('#phase-label').textContent=`0${f.phase+1} / 06 · ${names[f.phase]}`;$('#counts').textContent=`${word.length} 字母 · ${f.phase<3?"当前 "+f.current.length+" 片段":f.pieces.length+" Token"}`;$('#rules').value=limit;$('#rule-value').textContent=limit;$$('[data-word]').forEach(b=>b.setAttribute('aria-pressed',String(b.dataset.word===word)));$('#timeline').value=time;$('#timeline').style.setProperty('--fill',`${time/DURATION*100}%`);$('#time').textContent=`00:${String(Math.floor(time)).padStart(2,'0')} / 00:46`;$('#play-label').textContent=playing?'暂停':time>=DURATION?'再看一遍':time?'继续讲解':'播放讲解';$('#play-icon').textContent=playing?'Ⅱ':'▶';$('#play').setAttribute('aria-pressed',String(playing));$('#back').disabled=time===0;$('#next').disabled=time===DURATION;$('#play-note').textContent=playing?'配置已收起 · 空格暂停':'六个步骤 · 片段与编号';$('#challenge').hidden=timeseek(+e.target.value);$('#back').onclick=()=>seek(STARTS.filter(t=>tseek(STARTS.find(t=>t>time+.1)??DURATION);$('#reset').onclick=()=>{stop();time=0;word='hugs';limit=3;render();}; +$$('[data-word]').forEach(b=>b.onclick=()=>{stop();word=b.dataset.word;render();});$('#rules').oninput=e=>{const value=+e.target.value;stop();limit=value;render();};$('#experiment').onclick=()=>{seek(30);$('#rules').focus();}; +function info(){stop();$('#info').showModal();}$('#about').onclick=info;$('#show-rules').onclick=info;$('#close-info').onclick=()=>$('#info').close(); +$('#challenge').onclick=()=>{stop();$('#feedback').textContent='';$('#quiz-try').hidden=true;$$('[data-answer]').forEach(b=>b.removeAttribute('aria-pressed'));$('#quiz').showModal();}; +$$('[data-answer]').forEach(b=>b.onclick=()=>{$$('[data-answer]').forEach(x=>x.setAttribute('aria-pressed',String(x===b)));$('#feedback').textContent=(+b.dataset.answer===2?'预测正确。':'会得到 2 个 Token。')+'先得到 u | n | h | ug，再合并成 un | h | ug，最后得到 un | hug，编号是 [8, 9]。字母数仍然是 5。';$('#quiz-try').hidden=false;}); +$('#quiz-try').onclick=()=>{$('#quiz').close();word='unhug';limit=3;time=46;render();};$('#close-quiz').onclick=()=>$('#quiz').close(); +document.addEventListener('visibilitychange',()=>{if(document.hidden&&playing)stop();});document.addEventListener('keydown',e=>{if($('#info').open||$('#quiz').open)return;if(e.key==='Escape')stop();if(e.code==='Space'&&!['INPUT','BUTTON','A'].includes(e.target.tagName)){e.preventDefault();play();}}); +const observer=new ResizeObserver(()=>{stage.resize();anchor();});observer.observe($('#artwork'));$('.exhibit').addEventListener('transitionend',anchor);window.addEventListener('pagehide',e=>{stop();if(!e.persisted){dead=true;observer.disconnect();stage.dispose();}});render(); diff --git a/apps/site/learn/tokenization/index.html b/apps/site/learn/tokenization/index.html new file mode 100644 index 0000000..f3924c1 --- /dev/null +++ b/apps/site/learn/tokenization/index.html @@ -0,0 +1,2 @@ +文字怎样变成 Token？ · Explain3D
e· Explain3DAI 原理课堂 / 01

FROM TEXT TO TOKEN IDS

文字，怎样变成 Token？

看见片段的边界，再查到对应编号。

教学 BPE · 预设英文词 · 未运行语言模型
六个步骤 · 一条因果关系00:00 / 00:46
+
Explain3D · AI 原理课堂规则来自教学示例 · ID 仅属于本词表

A SMALL BPE TOKENIZER

先有规则，再处理输入。

本页实现一个字符级 BPE 教学分词器。三条已确定的合并规则按优先顺序应用：u + g → ug；u + n → un；h + ug → hug。没有匹配的相邻片段时，该规则不产生变化。

规则顺序参考 Hugging Face 的小型 BPE 教学示例。原示例通过语料频率学习规则，本页仅演示已有规则的应用，不演示训练。滑块切换的是教学分词器的规则版本，不表示真实模型每次输入都重新训练。

ID 0…9：b, g, h, n, p, s, u, ug, un, hug

本页只接受五个预设英文词，不处理空格、中文、特殊 Token、Unicode 归一化或未知字符。真实 tokenizer 可能使用字节级 BPE、WordPiece、Unigram 等方法。这里的分词结果与 ID 不能当作 ChatGPT、Claude 或其他模型的真实结果。

算法与示例来源 ↗

分词的输出是编号序列；它不等于理解词义，也不等于生成下一个 Token。

PREDICT · TRY · EXPLAIN

换个词，能自己推一次吗？

启用全部三条规则，输入 unhug。按照 u+g、u+n、h+ug 的顺序，它会得到几个 Token？

 \ No newline at end of file diff --git a/apps/site/learn/tokenization/model.mjs b/apps/site/learn/tokenization/model.mjs new file mode 100644 index 0000000..395b957 --- /dev/null +++ b/apps/site/learn/tokenization/model.mjs @@ -0,0 +1,25 @@ +export const WORDS=['hugs','hug','pug','unhug','bun']; +export const VOCAB=['b','g','h','n','p','s','u','ug','un','hug']; +export const RULES=[['u','g'],['u','n'],['h','ug']]; +export const STARTS=[0,6,13,25,34,41],DURATION=46; +export function encode(word,limit=3){ + if(!WORDS.includes(word)||!Number.isInteger(limit)||limit<0||limit>3)throw new RangeError('Use a supported teaching word and 0–3 rules'); + let pieces=[...word].map((text,start)=>({text,start,end:start+1}));const trace=[pieces]; + for(const [a,b] of RULES.slice(0,limit)){ + const next=[];for(let i=0;iVOCAB.indexOf(p.text)),decoded:pieces.map(p=>p.text).join('')}; +} +export function sample(time,{word='hugs',limit=3}={}){ + if(!Number.isFinite(time))throw new RangeError('Finite time required');const t=Math.max(0,Math.min(DURATION,time)),phase=STARTS.findLastIndex(s=>s<=t),progress=(t-STARTS[phase])/((STARTS[phase+1]??DURATION)-STARTS[phase]); + const final=encode(word,limit),step=phase<2?0:phase===2?Math.min(limit,Math.floor(progress*(limit+1))):limit; + return {...final,time:t,phase,progress,step,current:phase===0?[{text:word,start:0,end:word.length}]:final.trace[step],rule:phase===2&&stepp.text===f.rule[0]&&f.current[i+1]?.text===f.rule[1])?'合并这两个相邻片段。':'此输入没有这个相邻组合，本条跳过。'}规则在输入前已经确定。`:'本版本的规则已用完。没有匹配的规则会保留片段，不能为了凑成整个词而随意拼接。'], + ['Token 边界','一个片段，就是本版本的一个 Token。',`${f.word} 现在分成 ${f.pieces.length} 个 Token。片段可以小于一个词，也可能刚好等于一个词。`], + ['词表查找','把片段查成编号，顺序保持不变。','每个片段在这张词表中有固定 ID。编号是索引，不是含义、分数或概率；换词表后编号也可能不同。'], + ['还原检查','编号可以查回片段，再拼回输入。',`这次得到 [${f.ids.join(', ')}]。按同一张词表解码，仍然是 ${f.decoded}。分词没有生成新内容，也没有解释词义。`] + ][f.phase];return {eyebrow:'看这里 · '+texts[0],title:texts[1],body:texts[2]};} diff --git a/apps/site/learn/tokenization/model.test.mjs b/apps/site/learn/tokenization/model.test.mjs new file mode 100644 index 0000000..9d6aa50 --- /dev/null +++ b/apps/site/learn/tokenization/model.test.mjs @@ -0,0 +1,8 @@ +import assert from 'node:assert/strict'; +import {WORDS,VOCAB,encode,sample,DURATION} from './model.mjs'; +const golden={hugs:['hug','s'],hug:['hug'],pug:['p','ug'],unhug:['un','hug'],bun:['b','un']}; +for(const word of WORDS){assert.deepEqual(encode(word).pieces.map(p=>p.text),golden[word]);for(let k=0;k<=3;k++){const f=encode(word,k);assert.equal(f.decoded,word);assert.equal(f.ids.map(i=>VOCAB[i]).join(''),word);for(const row of f.trace){assert.equal(row.map(x=>x.text).join(''),word);let pos=0;for(const p of row){assert.equal(p.start,pos);assert.equal(p.text,word.slice(p.start,p.end));pos=p.end;}assert.equal(pos,word.length);}}} +assert.deepEqual(encode('hugs').ids,[9,5]);assert.deepEqual(encode('unhug').ids,[8,9]);assert.equal(encode('hugs',0).pieces.length,4);assert.equal(encode('hugs',1).pieces.length,3);assert.deepEqual(encode('hugs',1).pieces,encode('hugs',2).pieces); +for(const word of WORDS)for(let limit=0;limit<=3;limit++)for(let t=0;t<=DURATION;t+=.1){const a=sample(t,{word,limit});assert(a.current.every(p=>p.text===word.slice(p.start,p.end)));assert.equal(a.current.map(p=>p.text).join(''),word);sample(46,{word,limit});assert.deepEqual(sample(t,{word,limit}),a);} +assert.throws(()=>encode('未知'));assert.throws(()=>encode('hugs',4));assert.equal(sample(-1).time,0);assert.equal(sample(99).time,DURATION); +console.log('Tokenization: independent golden segmentations/IDs, span conservation, reversible decode, rule no-op, limits and backward seeking pass.'); diff --git a/apps/site/learn/tokenization/stage.mjs b/apps/site/learn/tokenization/stage.mjs new file mode 100644 index 0000000..177d6e6 --- /dev/null +++ b/apps/site/learn/tokenization/stage.mjs @@ -0,0 +1,28 @@ +import {VOCAB,RULES} from './model.mjs'; +const NS='http://www.w3.org/2000/svg',colors=['#6b8fc6','#9b84ca','#73a995','#c58b73','#87a7b4',' #a3b68c']; +const node=(tag,attrs,parent)=>{const n=document.createElementNS(NS,tag);for(const[k,v]of Object.entries(attrs))n.setAttribute(k,v);parent.append(n);return n;}; +export function createStage(host){ + const svg=node('svg',{viewBox:'0 0 900 360',role:'img','aria-label':'文字片段按规则合并，再查表变成编号'},host);svg.style.cssText='width:100%;height:100%;overflow:visible';let anchors={},last,lastKey=''; + function render(f){last=f;const mobile=host.clientWidth<600,W=mobile?500:900,key=[f.phase,f.step,f.word,f.limit,W].join('|');if(key===lastKey)return;lastKey=key;svg.setAttribute('viewBox',`0 0 ${W} 360`);svg.replaceChildren();const text=(x,y,s,size=18,fill='#6d687b',anchor='middle')=>{const t=node('text',{x,y,'font-size':size,fill,'text-anchor':anchor,'font-family':'ui-monospace,monospace'},svg);t.textContent=s;return t;}; + const xs=(start,end)=>40+(W-80)*(start+end)/2/f.word.length,barX=40,barW=W-80; + text(W/2,25,f.phase===0?'固定输入 / 一个英文词':f.phase===4?'同一张教学词表 / 固定索引':f.phase===5?'解码 / 保持原来的顺序':'同一个输入 / 观察边界怎样变化',mobile?13:15,'#9c8aab'); + // Engraved input rail stays spatially aligned with the token spans. + node('path',{d:`M38 72H${W-35}l8 -6H46Z`,fill:'#fbfcff',stroke:'#d3d9e2','stroke-width':.7},svg); + [...f.word].forEach((char,i)=>{text(xs(i,i+1),61,char,20,'#9a94a3');node('line',{x1:xs(i,i+1),x2:xs(i,i+1),y1:77,y2:96,stroke:'#ded7e5','stroke-dasharray':'2 4'},svg);}); + const rowY=115,rowH=70; + f.current.forEach((p,i)=>{const x=barX+barW*p.start/f.word.length+3,w=barW*(p.end-p.start)/f.word.length-6,c=colors[p.start%colors.length],hot=f.rule&&f.current[i+1]&&p.text===f.rule[0]&&f.current[i+1].text===f.rule[1]; + node('path',{d:`M${x} ${rowY}l7 -6h${w}l-7 6Z`,fill:'#eef1f6',stroke:'#c4cedb','stroke-width':.6},svg); + node('rect',{x,y:rowY,width:w,height:rowH,rx:4,fill:c,'fill-opacity':.25,stroke:c,'stroke-width':hot?1.8:.7},svg);node('path',{d:`M${x+3} ${rowY+3}h${w-6}`,stroke:'#fff','stroke-width':1.3},svg); + text(x+w/2,rowY+43,p.text,mobile?25:30,'#536078'); + if(f.phase>=4){const id=VOCAB.indexOf(p.text);node('path',{d:`M${x+w/2} 192V220`,stroke:c,'stroke-width':1},svg);node('rect',{x:x+w/2-28,y:223,width:56,height:37,rx:5,fill:'#fbf8fc',stroke:c,'stroke-width':.7},svg);text(x+w/2,248,String(id),21,'#80669b');} + }); + if(f.phase===2){const index=Math.min(f.step,f.limit-1),rule=RULES[index];text(W/2,232,f.rule?`${f.rule[0]} + ${f.rule[1]} → ${f.rule.join('')}`:'规则应用完毕',mobile?20:24,'#8a70a3');text(W/2,265,f.limit?`按优先顺序应用规则 · 已完成 ${f.step} / ${f.limit}`:'字符版本 · 不使用合并规则',mobile?12:14,'#9b8ba7');} + if(f.phase===3)text(W/2,238,`${f.word.length} 个字母 → ${f.pieces.length} 个 Token`,mobile?19:23,'#8b72a5'); + if(f.phase===4)text(W/2,307,f.pieces.map(p=>`${p.text} ↦ ${VOCAB.indexOf(p.text)}`).join(' '),mobile?15:18,'#8b72a5'); + if(f.phase===5){text(W/2,305,`[${f.ids.join(', ')}] → ${f.decoded}`,mobile?22:27,'#80689c');text(W/2,338,'同一词表解码 · 与输入逐字相同',mobile?12:14,'#9a8ba7');} + if(f.phase<2)text(W/2,259,f.phase===0?'不是按空格数计算 Token':'本例只处理预设英文词；空格与字节级编码另有规则',mobile?13:16,'#9989a6'); + const match=f.rule?f.current.findIndex((p,i)=>p.text===f.rule[0]&&f.current[i+1]?.text===f.rule[1]):-1;const a=f.current[Math.max(0,match)];anchors={focus:match>=0?[xs(a.start,f.current[match+1].end),115]:f.phase===2?[W/2,232]:[xs(a.start,a.end)+(f.phase>=4?28:0),f.phase>=4?238:115]}; + svg.setAttribute('aria-label',`${f.word}，阶段${f.phase+1}，当前片段 ${f.current.map(p=>p.text).join('、')}${f.phase>=4?'；编号 '+f.ids.join('、'):''}。`); + } + return {render,projectAnchor:()=>{const a=anchors.focus,m=svg.getScreenCTM(),r=host.getBoundingClientRect();if(!a||!m)return null;const p=svg.createSVGPoint();p.x=a[0];p.y=a[1];const q=p.matrixTransform(m);return{x:(q.x-r.left)/r.width,y:(q.y-r.top)/r.height,visible:true};},resize(){if(last)render(last);},dispose(){svg.remove();}}; +} diff --git a/apps/site/learn/tokenization/style.css b/apps/site/learn/tokenization/style.css new file mode 100644 index 0000000..18ff12f --- /dev/null +++ b/apps/site/learn/tokenization/style.css @@ -0,0 +1,15 @@ +:root{--paper:#f7f5ef;--ink:#343743;--muted:#817b87;--accent:#8066a6;--line:#e8e3da;font-family:Inter,'PingFang SC','Microsoft YaHei',system-ui,sans-serif;color:var(--ink);background:var(--paper);font-synthesis:none}*{box-sizing:border-box}body{margin:0}button,input,a{-webkit-tap-highlight-color:transparent}button{font:inherit;cursor:pointer;border:0}button:focus-visible,input:focus-visible,a:focus-visible{outline:2px solid #8e74b5;outline-offset:4px}button:disabled{opacity:.3;cursor:default}a{color:inherit;text-decoration:none}.app{height:100svh;min-height:790px;max-width:1600px;margin:auto;padding:0 40px;display:flex;flex-direction:column}header{height:58px;display:flex;gap:24px;align-items:center;flex-shrink:0}.brand{font-weight:700;font-size:19px;letter-spacing:-.6px}.brand i{font:italic bold 28px Georgia,serif;color:var(--accent);margin-right:4px}header>span{font-size:10px;letter-spacing:1px;color:#928a98}header>.quiet{margin-left:auto}.quiet{background:transparent;color:#82768b;border-radius:7px;font-size:11px;padding:8px 10px;white-space:nowrap}.quiet:hover{color:#554362;background:#eee8ee}.intro{display:flex;align-items:flex-end;justify-content:space-between;max-height:112px;padding:10px 0 23px;overflow:hidden;flex-shrink:0}.eyebrow{font-size:9px;letter-spacing:2px;font-weight:600;color:#8e7c9b;margin:0 0 10px}h1{margin:0;font-size:clamp(27px,2.6vw,38px);font-weight:650;letter-spacing:-1.2px;line-height:1.35}h1 span{color:#7c6c8e}.intro-copy{font-size:12px;color:#817788;margin:0 0 4px}.controls{height:60px;max-height:60px;display:flex;align-items:center;gap:17px;border:1px solid #e6e0d7;background:#efede6;padding:0 18px;border-radius:13px;margin-bottom:17px;flex-shrink:0;overflow:hidden}.temperature{display:flex;align-items:center;gap:20px}.temperature label{font-size:11px;white-space:nowrap;color:#756c7c}.temperature output{font-family:ui-monospace,monospace;color:var(--accent);margin-left:12px;font-size:12px}input[type=range]{appearance:none;background:transparent;height:24px;margin:0;cursor:pointer}input[type=range]::-webkit-slider-runnable-track{height:3px;border-radius:4px;background:linear-gradient(to right,#9b85b9 var(--fill,0%),#d9d1df var(--fill,0%))}input[type=range]::-webkit-slider-thumb{appearance:none;width:12px;height:12px;border-radius:50%;background:#8b73ac;margin-top:-4.5px;box-shadow:0 0 0 3px #f7f5ef}input[type=range]::-moz-range-track{height:3px;background:#d9d1df}input[type=range]::-moz-range-progress{height:3px;background:#9b85b9}input[type=range]::-moz-range-thumb{height:12px;width:12px;border-radius:50%;border:0;background:#8b73ac}#temperature{width:155px}.presets{display:flex;gap:3px}.presets button{font-size:10px;padding:8px 11px;background:none;color:#7f7187;border-radius:6px;white-space:nowrap}.presets button[aria-pressed=true],.presets button:hover{background:#fffcf8;color:#6f5387}.divider{height:22px;width:1px;background:#ded7df;margin-left:auto}#new-draw span{margin-right:4px;font-size:15px}.experiment{font-size:11px;padding:10px 14px;background:#8b749e;color:white;border-radius:7px;white-space:nowrap}.experiment:hover{background:#78618e}.exhibit{position:relative;flex:1;min-height:420px;background:#fffdfa;border:1px solid #e9e4db;border-radius:18px;overflow:hidden;isolation:isolate}.exhibit-meta{position:absolute;top:21px;left:25px;right:25px;display:flex;justify-content:space-between;color:#84798c;font-size:10px;letter-spacing:.6px;z-index:2}#live-temperature{font-family:ui-monospace,monospace;color:#8a73a2}.context{position:absolute;top:65px;left:27px;z-index:2;max-height:80px;overflow:hidden}.context>span{font-size:9px;color:#94899c}.context p{font:14px Georgia,serif;color:#7c7186;margin:9px 0}.context b{color:#a28dbb}.callout{position:absolute;top:56px;width:380px;left:50%;transform:translateX(-50%);padding:15px 19px 11px;border:1px solid #e7dfee;background:#fffffff7;border-radius:13px;box-shadow:0 8px 26px #88779905;z-index:4}.callout>span{font-size:9px;letter-spacing:.8px;color:#9a80b4}.callout h2{font-size:18px;font-weight:650;line-height:1.5;letter-spacing:-.25px;margin:7px 0}.callout p{font-size:12px;line-height:1.8;color:#766a7e;margin:0}.calculation{font-size:10px;color:#9579ac;background:transparent;padding:8px 0 0}.calculation:hover{color:#674280}.leader{position:absolute;inset:0;width:100%;height:100%;pointer-events:none;z-index:3;color:#af97c5}.leader path{fill:none;stroke:currentColor;stroke-width:1.1}.leader circle{fill:#fffdfa;stroke:currentColor;stroke-width:1.1}.artwork{position:absolute;left:50%;bottom:39px;transform:translateX(-50%);width:min(96%,1160px);height:calc(100% - 208px);display:flex;align-items:center;justify-content:center;transition:transform .4s}.exhibit.is-detailed .artwork{transform:translateX(-50%) scale(1.3);transform-origin:center 70%}.stage-bottom{position:absolute;bottom:13px;left:25px;right:17px;display:flex;align-items:center;justify-content:space-between;z-index:5;color:#8a7f92;font-size:9px}.stage-bottom .quiet{font-size:10px}.transport{height:70px;display:flex;gap:18px;align-items:center;flex-shrink:0}.play{height:42px;width:134px;display:flex;align-items:center;justify-content:center;gap:11px;border-radius:9px;background:#3c3543;color:#fff;font-size:11px;flex-shrink:0}.play:hover{background:#62516f}#play-icon{font-size:10px}.track{flex:1;min-width:0}.track input{display:block;width:100%}.track>div{display:flex;justify-content:space-between;color:#8c7d98;font-size:9px;font-variant-numeric:tabular-nums}footer{height:30px;max-height:30px;display:flex;align-items:center;justify-content:space-between;flex-shrink:0;overflow:hidden;font-size:9px;color:#a497ac}.secondary{transition:opacity .35s,max-height .6s cubic-bezier(.22,1,.36,1),padding .6s,margin .6s}.is-playing .secondary{max-height:0;opacity:0;padding-top:0;padding-bottom:0;margin-top:0;margin-bottom:0;pointer-events:none}.is-playing .controls{border-width:0}.is-playing .exhibit{border-color:transparent}.is-playing .calculation,.is-playing #detail{display:none}.is-playing .artwork{height:calc(100% - 185px);bottom:35px}.is-playing .stage-bottom{bottom:19px}.is-playing .callout{padding-bottom:15px} +dialog{border:1px solid #e5dce9;background:#fffdf8;border-radius:18px;max-width:720px;width:94%;max-height:88svh;padding:34px;color:#4d4058;box-shadow:0 25px 100px #3e2c4829}dialog::backdrop{background:#34263e55;backdrop-filter:blur(4px)}dialog h2{font-size:25px;line-height:1.4;margin:0 0 16px;letter-spacing:-.6px}dialog p{font-size:13px;line-height:1.9;color:#81718d}dialog a{color:#8b6aa4;text-decoration:underline;text-underline-offset:3px}.close{position:absolute;right:16px;top:10px;background:none;color:#9883a6;font-size:26px}.fine{font-size:11px!important;color:#9786a2!important}.formula{font:14px/1.8 ui-monospace,monospace;background:#f0ebf3;border:1px solid #e4dce9;border-radius:8px;padding:12px 16px;margin:16px 0;color:#7b5e98;overflow-x:auto}.table-scroll{overflow-x:auto}table{width:100%;border-collapse:collapse;text-align:right;font-variant-numeric:tabular-nums;font-size:12px}th{font-size:10px;color:#95819f;font-weight:500;white-space:nowrap;padding:9px 10px;border-bottom:1px solid #e7dfeb}td{padding:14px 10px;border-bottom:1px solid #eee8f1;color:#6b567a}td:first-child,th:first-child{text-align:left}td:first-child i{width:7px;height:7px;display:inline-block;border-radius:50%;margin-right:7px}.calculation-total{font-size:12px;color:#79628a;text-align:right} +@media(max-width:1100px){.app{padding:0 24px}.intro-copy{display:none}.controls{gap:10px}.temperature{gap:12px}#temperature{width:120px}.presets button{padding:8px}.context{display:none}.divider{margin-left:auto}} +@media(max-width:700px){.app{padding:0 12px;min-height:790px}header{height:48px;gap:14px}.brand{font-size:17px}.brand i{font-size:25px}header>span{font-size:8px;letter-spacing:0}header>.quiet{font-size:9px;padding-right:0}.intro{padding:9px 0 18px;max-height:105px}h1{font-size:24px;line-height:1.45;letter-spacing:-.7px}.eyebrow{font-size:8px;margin-bottom:7px}.controls{height:98px;max-height:98px;display:grid;grid-template-columns:1fr auto;gap:8px 4px;padding:10px 12px;margin-bottom:11px}.temperature{grid-column:1/-1;grid-row:1;gap:17px}.temperature label{font-size:11px}#temperature{flex:1;width:auto}.presets{grid-row:2;gap:0}.presets button{font-size:9px;padding:7px 9px}.divider{display:none}#new-draw{display:none}.experiment{font-size:9px;padding:9px 10px}.exhibit{min-height:408px;border-radius:13px}.exhibit-meta{top:17px;left:17px;right:17px;font-size:9px}.callout{top:47px;width:calc(100% - 34px);max-width:380px;padding:13px 16px 10px}.callout h2{font-size:16px;line-height:1.5}.callout p{font-size:11px;line-height:1.7}.callout>span{font-size:8px}.calculation{font-size:9px}.artwork{width:100%;height:calc(100% - 192px);bottom:37px}.stage-bottom{left:17px;right:7px;bottom:10px;font-size:8px}.stage-bottom .quiet{font-size:9px}.transport{height:62px;gap:6px}.play{width:110px;height:39px;font-size:10px}.transport>.quiet{padding:6px}.track>div span:first-child{display:none}.track>div{justify-content:flex-end}.track input{height:23px}footer{height:27px;max-height:27px;font-size:8px}footer>span:last-child{max-width:150px;text-align:right}.is-playing{min-height:600px}.is-playing .artwork{height:calc(100% - 190px);bottom:40px}.is-playing .stage-bottom{bottom:16px}dialog{padding:30px 19px}dialog h2{font-size:22px}td,th{padding:10px 6px}td{font-size:11px}.formula{font-size:12px}} +@media(prefers-reduced-motion:reduce){*{animation:none!important;transition:none!important}}[hidden]{display:none!important} + +/* Reserve the callout before fitting the vector diagram. */ +.artwork,.is-playing .artwork{top:220px;bottom:35px;height:auto}.stage-actions{display:flex;align-items:center;gap:5px} +@media(max-width:700px){.artwork,.is-playing .artwork{top:205px;bottom:36px;height:auto}} + +.challenge-answers{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:20px 0}.challenge-answers button{background:#f1ebf5;color:#725389;border:1px solid #dfd3e8;border-radius:8px;padding:13px 8px;font:20px Georgia,serif}.challenge-answers button[aria-pressed=true]{background:#8b749e;color:white;border-color:#8b749e}#challenge-feedback:empty{display:none}#challenge-feedback{padding:15px;background:#f2f1ed;border-radius:8px;color:#665873;line-height:1.9}.stage-actions #challenge{color:#80609e} + +.is-playing #challenge{display:none} + +#rules{width:110px}.controls label{font-size:11px;white-space:nowrap;color:#83708e}.controls output{color:#8066a6;margin-left:8px}.is-playing #show-rules{display:none}.challenge-answers{grid-template-columns:repeat(3,1fr)}#feedback:empty{display:none}#feedback{background:#f0ebf3;border-radius:8px;padding:15px}.artwork svg{display:block} @media(max-width:700px){.controls{grid-template-columns:auto 1fr auto;height:102px;max-height:102px}.presets{grid-column:1/-1;grid-row:1}.controls label{grid-row:2;font-size:10px}#rules{grid-row:2;width:100%}#experiment{grid-row:2;font-size:9px}.presets button{padding:7px 12px}.stage-bottom{font-size:8px}} diff --git a/docs/ai-learning/STATUS.md b/docs/ai-learning/STATUS.md index d252aae..a4b2631 100644 --- a/docs/ai-learning/STATUS.md +++ b/docs/ai-learning/STATUS.md @@ -1,6 +1,6 @@ # AI principles course status -Updated: 2026-10-26 (Asia/Shanghai). +Updated: 2026-10-27 (Asia/Shanghai). The user authorized ten scenes, an iteration every twelve hours, independent learner-oriented review and publication after acceptance. A recurring task has been created. Each scheduled run works on one scene; failure keeps that scene active rather than counting it as complete. @@ -18,10 +18,14 @@ The user authorized ten scenes, an iteration every twelve hours, independent lea - Release source: fc03f80a1953a82d7ad104fb90307bf5cb2d131a; PR #1 merged after successful branch CI and preview. - Production deployment dpl_8KskYdCLpm2uudUGezEzk3RiFJyD reached READY for that source SHA. Main CI run 40281781432 succeeded. - Production browser: both public routes loaded; learning check answered correctly and returned to T=2, u=.6219, floor27.6%, matching the independently tested model. No sign-in required on the custom domain. -- Next iteration: tokenization. Continue the remaining nine scenes in roadmap dependency order; do not redo the released scene without a new issue. +- Next iteration after the current candidate is released: embedding. Continue the remaining nine scenes in roadmap dependency order; do not redo the released scene without a new issue. ## Remaining work Nine scenes are planned, not implemented or advertised as playable. See `roadmap.json` for topics, learning outcomes and current source status. The existing experimental autoregression and roulette pages are not silently promoted to accepted lessons. Real learner sessions, physical phones, screen-reader devices and independent browser-engine coverage have not been completed. These limits do not prevent releasing a clearly labeled teaching sample, but they must not be represented as tested. + +## Reviewed release candidate: tokenization + +Route /learn/tokenization/. A 46-second fixed-rule character-level BPE teaching model shows input, base symbols, ranked adjacent merges, boundaries, vocabulary IDs and reversible decoding. Five supported English words and four rule versions are explicit; not a production tokenizer, no Chinese/space/byte-level support. Golden splits and IDs, span conservation, rule no-op, all version/time samples and decode round trips pass. Desktop and390×844 visual/quiz checks completed. The new inputunhug predicts two tokens and verifies[8,9]. This is developer teaching review, not measured learning gains. Pending production deployment verification; do not count this candidate as published until READY and a production-route check. diff --git a/docs/ai-learning/roadmap.json b/docs/ai-learning/roadmap.json index 435c39b..699b553 100644 --- a/docs/ai-learning/roadmap.json +++ b/docs/ai-learning/roadmap.json @@ -1,6 +1,6 @@ { "version": 1, - "updated": "2026-10-26", + "updated": "2026-10-27", "target": 10, "cadenceHours": 12, "releasePolicy": "One reviewed scene per iteration; continue a failed scene before advancing.", @@ -22,8 +22,8 @@ "order": 1, "title": "文字怎样变成 Token？", "outcome": "观察同一句话如何分成片段并映射为编号；区分字符、词和 Token。", - "status": "planned", - "route": null, + "status": "reviewed", + "route": "/learn/tokenization/", "published": false }, { diff --git a/package.json b/package.json index 4df3382..7d48663 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "build": "node scripts/build.mjs", - "test": "node evals/verify.mjs && node apps/site/learn/token-choice/model.test.mjs && node evals/verify-token-choice.mjs && node evals/verify-course.mjs", + "test": "node evals/verify.mjs && node apps/site/learn/token-choice/model.test.mjs && node evals/verify-token-choice.mjs && node evals/verify-course.mjs && node apps/site/learn/tokenization/model.test.mjs", "check": "npm test && npm run build" }, "engines": { diff --git a/scripts/build.mjs b/scripts/build.mjs index d4fc7f3..55ce3cd 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -22,7 +22,7 @@ await write('start',page('让 Agent，多一个本领。','安装 Explain3D Skil await write('docs',page('怎么做，也怎么验证。','Explain3D 的结构、工作流、模板边界、自动发布与验证记录。',`

怎么做，也怎么验证。

漂亮的结果值得展示，可重复的方法更值得留下。

能力如何组成

Skill 负责把一个问题拆成对象、关系、变量、约束与叙事。运行时负责绝对时间采样、播放状态、视角、参数和配置退场。场景适配器负责各自的几何与计算。

产品采用可静态部署的网站，核心演示使用 Three.js；无 WebGL 时使用同一模型的 CPU 投影。官网不存储 API Key，也不直接运行用户提示词。

创建一个新场景

  1. 限定一个闭环问题与适合的受众。
  2. 写场景规格，注明来源、约束与简化。
  3. 选择合适的模板，重建新对象与计算关系。
  4. 共用时间、播放和交互状态，制作十秒讲解。
  5. 分别验证数学、交互、画面、键盘、手机与性能。
  6. 交付源码、提示词、运行方法与验证记录。

当前验证到哪里

已执行模型约束、时间线与静态构建检查，并在当前浏览器检查交互和响应式。当前云端验收浏览器禁用 WebGL，画面使用 CPU 兼容渲染。真实手机 GPU、方向传感器与持续帧率尚未实机验收。

标准 Skill 包已提供。Claude Code、WorkBuddy 和 DeepSeek 宿主的独立复现尚未完成；网站不会把文档适配标为验证通过。

自动发布

GitHub 保存网站、Skill、模板与提示词。Vercel Git 集成负责分支预览和正式分支发布；生产构建先执行 npm run check，模型检查失败时构建中止。每个版本可按 Git 提交追溯，正式站点绑定 explain.funagent.app。

Skill 的本地检查完成后再同步到发布目录。依赖、模型假设和第三方许可证保留在源码中。社交文案在素材页准备，发布到社交账号单独执行。

深度效果的边界

透视、遮挡、光照和运动视差能在普通屏幕中形成明显深度感；它们不让普通屏幕自动变成双眼立体硬件。倾斜控制依赖方向传感器，完整行星齿轮版可使用；其他模板支持拖动观察。

参与

提交新模板时，附带可运行场景、参数、公式或事件模型、简化说明和测试。请通过 GitHub Issues 提出问题或 Pull Request 分享作品。

仓库与贡献指南 ↗

`,'/docs/')); await write('press',page('把可以操作的理解，分享出去。','Explain3D 的真实演示、发布文案、品牌说明和素材。',`

把可以操作的理解，分享出去。

一个可体验的入口，一份能带走的方法。文案和镜头方案已经放在开源仓库中。

Explain3D 实际交互作品预览

产品一句话

让 AI 把一个原理，做成可以亲手探索的 3D 作品。

可使用的宣传材料

小红书、短视频、公众号、X、社区与 GitHub 发布文案 ↗

下载分享封面 / 下载行星齿轮场景图

演示素材的准确表述

这是可操作的实时三维原理讲解。可以说“普通屏幕里的深度感”“可拖动、可拆解”“AI 辅助生成源码”。不能据此宣称普通手机拥有真正的双眼裸眼立体显示，也不能把未验证平台说成一键必成。

`,'/press/')); await buildCourse(root,out); -const urls=['/','/learn/','/learn/token-choice/','/templates/','/create/','/start/','/docs/','/press/',...templates.map(s=>`/templates/${s.id}/`)]; +const urls=['/','/learn/','/learn/token-choice/','/learn/tokenization/','/templates/','/create/','/start/','/docs/','/press/',...templates.map(s=>`/templates/${s.id}/`)]; await fs.writeFile(path.join(out,'sitemap.xml'),`${urls.map(u=>`https://explain.funagent.app${u}`).join('')}`); await fs.writeFile(path.join(out,'robots.txt'),'User-agent: *\nAllow: /\nSitemap: https://explain.funagent.app/sitemap.xml\n'); await fs.writeFile(path.join(out,'favicon.svg'),'e');