Skip to content

Say the masked future D is excluded during the value-mixture step #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hash-panda wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Say the masked future D is excluded during the value-mixture step #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/site/learn/attention/app.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function teachingCopy(f){
else if(f.phase===3){
c.title=p.collect<1?'先求一个共同分母':`${TOKENS[p.active]} 的份额，除以同一分母`;
c.body=p.collect<1?'把各行的指数权重相加。未来 D 的指数为 0，不进入分母。':'短条按 0→100% 的共同刻度描出份额。这是混合信息的权重，不是抽取下一个 Token。';
}else if(f.phase===4){c.title=`读取 ${TOKENS[p.active]} 的 V，再汇入结果`;c.body='虚线是 V，实色是读出的份额，同一刻度。非零贡献沿连线汇入底部；零值与未来位置不发出贡献。';}
}else if(f.phase===4){c.title=p.active===3?'未来 D 被排除，贡献为 0':`读取 ${TOKENS[p.active]} 的 V，再汇入结果`;c.body='虚线是 V，实色是读出的份额，同一刻度。非零贡献沿连线汇入底部；零值与未来位置不发出贡献。';}
if(f.phase===5&&!h?.inTransit)c.body='保持 K/V 不变，试着切换查询方向。再只改未来 D 的 V：输出为何不变？查询归零后又是多少？';
return c;
}
Expand Down
2 changes: 1 addition & 1 deletion apps/site/learn/attention/stage.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export function createStage(host,{reducedMotion=()=>false}={}){
const active=pose.active,c=COLORS[active],collecting=phase===3&&!transfer?.inTransit&&pose.collect<1;
const arrived=(i,t=.9)=>i<active||i===active&&pose.travel>=t;
plate(12,4,W-24,108);
const titles=['查询 C · 一条 Q，四组 K/V',`逐项匹配 · ${TOKENS[active]}`, '先遮住未来 D，再做 Softmax',transfer?.inTransit?'分数 → 指数 → 共同分母':collecting?'共同分母 · 汇集全部指数权重':`同一个分母 · Σ exp(s) ≈ ${fmt(pose.denominator)}`,`逐项读取 · ${TOKENS[active]}`,'所有贡献相加，不是挑最大的一项'];
const titles=['查询 C · 一条 Q，四组 K/V',`逐项匹配 · ${TOKENS[active]}`, '先遮住未来 D，再做 Softmax',transfer?.inTransit?'分数 → 指数 → 共同分母':collecting?'共同分母 · 汇集全部指数权重':`同一个分母 · Σ exp(s) ≈ ${fmt(pose.denominator)}`,active===3?'未来 D · 权重为 0':`逐项读取 · ${TOKENS[active]}`,'所有贡献相加，不是挑最大的一项'];
text(W/2,25,titles[phase],mobile?17:19,phase===2?COLORS[3]:c);
let formula=`q ≈ [${fmt(f.q[0])}, 0]`;
if(phase===1)formula=mobile?`(${fmt(f.q[0])}×${f.keys[active][0]} + 0) / √2 ≈ ${fmt(f.scores[active])}`:`(${fmt(f.q[0])} × ${f.keys[active][0]} + 0 × ${f.keys[active][1]}) / √2 ≈ ${fmt(f.scores[active])}`;
Expand Down
Loading