From 0ed1504611a9e218c190a83ff2af05221000d8b6 Mon Sep 17 00:00:00 2001 From: Filipe Guerreiro Date: Tue, 28 Jul 2026 15:42:45 +0900 Subject: [PATCH 1/5] Read pages from the write-ahead log A database in WAL mode keeps recent pages in a separate "-wal" file until a checkpoint folds them in, so reading the main file alone returns the database as of the last checkpoint. For a file belonging to a running application that is arbitrarily stale: a row written seconds ago can be missing, and so can a newer page 1, which leaves the header (user_version among it) out of date too. Open, and OpenFrom when the reader exposes Name(), now index the log beside the database and serve those pages from it. Frames are accepted while their salts match the header and their checksums verify, and only frames up to the last commit frame are applied, so a torn tail or an in-flight transaction is dropped rather than read. A log SQLite would ignore (bad header checksum, mismatched page size, no commit frame) is ignored here too, leaving the main file readable on its own. testdata/wal.sqlite was checkpointed after its first row: one row and user_version 7 on its own, three rows and user_version 16 with its log applied. --- AUTHORS | 1 + CONTRIBUTORS | 1 + file.go | 45 +++++ pager.go | 57 ++++-- testdata/wal-shrink.sqlite | Bin 0 -> 34816 bytes testdata/wal-shrink.sqlite-wal | Bin 0 -> 4224 bytes testdata/wal.sqlite | Bin 0 -> 2048 bytes testdata/wal.sqlite-wal | Bin 0 -> 3176 bytes wal.go | 147 +++++++++++++++ wal_test.go | 334 +++++++++++++++++++++++++++++++++ 10 files changed, 568 insertions(+), 17 deletions(-) create mode 100644 testdata/wal-shrink.sqlite create mode 100644 testdata/wal-shrink.sqlite-wal create mode 100644 testdata/wal.sqlite create mode 100644 testdata/wal.sqlite-wal create mode 100644 wal.go create mode 100644 wal_test.go diff --git a/AUTHORS b/AUTHORS index c5c2879..e0dc4b2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -10,5 +10,6 @@ Adam Shannon Andre Renaud +Filipe Guerreiro Sebastien Binet Zellyn Hunter diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 15ac00f..d04788d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -17,5 +17,6 @@ Adam Shannon Andre Renaud +Filipe Guerreiro Sebastien Binet Zellyn Hunter diff --git a/file.go b/file.go index 510a40d..8d3e9c2 100644 --- a/file.go +++ b/file.go @@ -5,6 +5,7 @@ package sqlite3 import ( + "bytes" "fmt" "io" "os" @@ -65,6 +66,12 @@ type dbHeader struct { SqliteVersion int32 // SQLITE_VERSION_NUMBER } +// OpenFrom reads the database in f. +// +// When f is backed by a named file (an *os.File, or any reader exposing +// Name() string) the committed pages of a write-ahead log beside it are +// overlaid on the database image, so a database belonging to a running +// application reads current rather than as of its last checkpoint. func OpenFrom(f io.ReadSeeker) (*DbFile, error) { var db DbFile @@ -111,14 +118,52 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) { db.pager = newPager(f, db.PageSize(), db.NumPage()) + if named, ok := f.(interface{ Name() string }); ok { + if err := db.attachWAL(named.Name()); err != nil { + return nil, err + } + } + err = db.init() if err != nil { + db.pager.Delete() return nil, err } return &db, err } +// attachWAL overlays the write-ahead log beside the database at dbPath, if one +// holds a committed snapshot. Both the page count and the header itself can be +// superseded by the log, so they are re-read from it. +func (db *DbFile) attachWAL(dbPath string) error { + walFile, index, err := openWAL(dbPath, db.PageSize()) + if err != nil || index == nil { + return err + } + db.pager.wal = index + db.pager.walFile = walFile + db.pager.npages = index.dbSize + + if _, ok := index.offsets[1]; ok { + page, err := db.pager.Page(1) + if err != nil { + db.pager.Delete() + return err + } + dec := binary.NewDecoder(bytes.NewReader(page.buf)) + dec.Order = binary.BigEndian + if err := dec.Decode(&db.header); err != nil { + db.pager.Delete() + return err + } + } + // Last, because the commit frame is what states the page count and the + // decode above may have just put the main file's stale one back. + db.header.DbSize = int32(index.dbSize) + return nil +} + func Open(fname string) (*DbFile, error) { f, err := os.Open(fname) if err != nil { diff --git a/pager.go b/pager.go index a47ca7f..203afea 100644 --- a/pager.go +++ b/pager.go @@ -7,14 +7,17 @@ package sqlite3 import ( "fmt" "io" + "os" ) type pager struct { - f io.ReadSeeker - size int // page size in bytes - npages int // total number of pages in db - pages map[int]page // cache of pages - lru []int // list of last used pages + f io.ReadSeeker + size int // page size in bytes + npages int // total number of pages in db + pages map[int]page // cache of pages + lru []int // list of last used pages + wal *walIndex // committed pages living in the write-ahead log, if any + walFile *os.File // the -wal file backing wal } func newPager(f io.ReadSeeker, size, npages int) pager { @@ -40,21 +43,10 @@ func (p *pager) Page(i int) (page, error) { return page, fmt.Errorf("sqlite3: out of range (%d > %d)", i, p.npages) } - pos, _ := p.f.Seek(0, io.SeekCurrent) - defer p.f.Seek(pos, io.SeekStart) - buf := make([]byte, p.size) - if _, err := p.f.Seek(int64((i-1)*p.size), io.SeekStart); err != nil { + if err := p.read(i, buf); err != nil { return page, err } - n, err := p.f.Read(buf) - if err != nil { - return page, err - } - - if n != len(buf) { - return page, fmt.Errorf("sqlite3: read too few bytes") - } page.id = i page.buf = buf @@ -64,9 +56,40 @@ func (p *pager) Page(i int) (page, error) { return page, err } +// read fills buf with page i, preferring the write-ahead log's image of it +// over the one in the main database file. +func (p *pager) read(i int, buf []byte) error { + if p.wal != nil { + if off, ok := p.wal.offsets[i]; ok { + _, err := p.walFile.ReadAt(buf, off) + return err + } + } + + pos, _ := p.f.Seek(0, io.SeekCurrent) + defer p.f.Seek(pos, io.SeekStart) + + if _, err := p.f.Seek(int64((i-1)*p.size), io.SeekStart); err != nil { + return err + } + n, err := p.f.Read(buf) + if err != nil { + return err + } + if n != len(buf) { + return fmt.Errorf("sqlite3: read too few bytes") + } + return nil +} + func (p *pager) Delete() error { var err error p.pages = nil p.lru = nil + if p.walFile != nil { + err = p.walFile.Close() + p.walFile = nil + p.wal = nil + } return err } diff --git a/testdata/wal-shrink.sqlite b/testdata/wal-shrink.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..4b1337ca365b12c9992578c0a93f4d43552ae0d9 GIT binary patch literal 34816 zcmeFZQ;=-U+wI%7ZQHi(?$yR>+qP}4wr$())wXTh?%w_X_SyS<`{rDoi*w$1Giv@S zE30Nj)icJ3oS9KkO21^xos0=gY#priod}o#KmdS%U<3#V0002Meh#9a<9{`vpAN|X z_257EfdA`>|A%`p0J`WG*q;{w0lNYKSO4JO_HPCLtpFRwn7~Ef!O%?Kfr62Nl7_&^)t11~THngb+{TIWznA~NLi}F<{TT-N{~Z4NXZ^ST zZw3CX!2i1y_|M!72)qyQbNjdbTY-No@XxCNJRmSBq@kIyp{1Se&-8C>^xwbYec(fnDq5J?A=?5?fKY)h& z0Tj#+AfbK$0r3Oy|E%W0{| z;XZE=T~O|oc{0ivYP!Y*&#ZJVf!sGRFd1wj&nMk{u}T~}Y-;#joFwX1tv(5F< zl`ejcE+Up$z&U=U8P!0teAlU;Ry#~@EJ>7!7IYBFgXg?g{4mAbJY|HGa8X}=q_yjP z5gtH0!%L7NXPlR<rIV_um7IFV!ym~mjL zij_p4Q8F3L(w8cTd2x6?Che^d#RVbEpkjkP7+_FM+lVuH|- zLch(r>GWe@zL(dwr;ng@K!Kr!2Qj5^lb zQq2GkN})|I!?f*%hxXP_(;d_dbwnp*3R3W4Z;V)+#kB$wrJ7n_X+>o|qs*2mJE$x$Vu~DuFon2_A|c%9{foQLN$=K&yHyxdhMX z?E?B9Ki-qwhNBU~h$!)lI2ml7nCSW)&_GUvSoi7Dx=wX*w7Qm57y6duBtMAR=?()r zodf8LN}nh=D@v{ffB2$x z-LjD7L__n{Dfj&rdWKYkd1N4hlk=z=dVJPRz%7DmKd;3muKla|?V2s|6;Ne$9a(|O z(zqqBIkkXc8k#7HckteiuVcPjL`{00y-Vv=ODF=eT~YK+9}9-$`Rjw;g(ndYAV0o(7Hl(5eW=ma`Qbrbe+exd@~F&^8Dpl_@GV022qJ0{@#tP#x~sTEOSE zn6DpzSxp%y_N_`l@-UM|2f!mxqs211lC^BOfJ1$=$%XDk@f5f@qmHDG!O)QKyrvj9 zQ4gi^ZLHm~bJqx}W!KTC`DsICgsJ`)73kj=QDdaeXmuP#kWzW|=w!8NIDl|J@#6N9 z-y>vZ_OdEqEA<841ZAGVCXkAE4a59FA=E22Jvt!MRNyJuxYZN768nNL{)vliW-R=T zs|oHosZdmlq!W8dy5(X+N%mUzCL*v1dp4(VaQ&R(IH3$xnxgVz*+>_c^=#>*K4bnA z9qlE<%#V^;Csx}imx0K@KdKa`>Cqcn>XG*aYeO|+LLe2hu##QhP3#mgzBR$SKqq0T zw-o=Gu-()|l`Ak=hw2zJF)=4db{%dPF$?Z&CoC4AQOqa$RnVMw@FYf&X(DP`_{2a5 z+3%2sH*)>Kbi`I>L9%Q3Dhfb^{iF@&bUUdEH$nl`5H{VOoYHn^@VnK=1Yq~QxDgv6 z-favuYC?34Hd~XTmfZV~hKpj%86`K8GI~c?-*W_;8i!WGIXShNXV7UbO6rbAP_tPKCm1!%$+rEl}PpWp-u9y2qz9aISa%6 z@sU;Ron4Ac$Q)(k+9^zz8DASeto=;_) z5Gptz)TI((>UAadv-EPt!%BbF#BzH%YeLH{Pe2#? zzGMH7{l9-s{=t5n<|@(Z*LD%v9-7l!Vi@3f&vsH#_csLDg(5?ym5P@O1jx%?Uksvo zA~t4R7-KD$Y_{ztlHAn|8Jn(6Sd*RZ^0c8@G#x7-{cl!J%hTIMmuVewE&<6s=+W6J z|57uS|1CSs5WS8n=3~a*C$lJQiRwsXWqy$L8TeHzqm!RUh_2C&v?G4!nI{JKJVy=W zk-!2KO0Vn)@wg1>%j!fUcZb0QF5I93D#W%ioeMCBoFh{^O}X1%mI&=W`rKduA*9ck zNaOL66PDU4X5ol{qIQW$e%uJG76&v4^ysX61$`V+d;b7JWr*TEE~%(?%Z+gdJcvPd z&l6ti_{CxZ+a!QSGS^XexkFIrel6y;}_j@{oQds!1JSm*KN)O02Zybz^9b$;yr!)P^2ujt@IYkH zA61TU)=8}32xVE4$12Va=hb2JO=nG%daFb?@k*&Poh^Xg7cbQXmLxgX@Qgud6FnO= zw0q8*KRX;C>&B2A)OHVp6fPGYW;$#ipRDVLP_JSf>Tpq*incjgL19Ai(Cv(HdHFgS ze!a-fdnj;r1R;bCs*QT5x(=60D1zjk%~OUP7=2_J5)biB)pc~`SFg)Y5;f3m!@)S+ zvO=x1NZ(kgL838kGfXf9qhzf=+`kR9-bzMmLE!;p*US*c2mb*u0zEAon45shfAlIc z7qjJiYbq6M(9MLD48cBouo1`cq|n{rrsv_Io4CM8D1=yDK@iFlDw+QiH@+h2b0rkz0KQ_hqTtOSqG?0$$7ULEveRbm*shIN<| zxyLwdQsOv5Lk6QK-e8$rH z)Xk>EEP6=eC(oc|$yoF=X$&B9h9 zhZ<)_fMThU(u$hb{w#+T$W~1ONh0o7Z4kElBJz5X<-%@>o?!83(B!+x1&RV<=F|$A z$1!1{sPK^f%1TD{o5b4&A3vd>VXp^6U7*U00Ep-Dic3ZcIhB5I?5RriWr4_|Wngrt=+B{~lWY8=uN@t?gph4P3V?lb3 zvA6`SoS?OZ`A}Kbuct!Qin;9!Pa6-9^FfxA!jCl@N|5i%viLwDbC$lv&O}&G18>v8 zv&JFZnk;SLVgn?UgpFoLQ1C{fS4hJ zqk`3za-nj^H}fb7VXQLMH!u)fA;f&=n6Wk~XMJw!*tE_L!jd*~K&TNYz#E=n=t*}C zo<8xxMaJFkn)|CEcUlR-E{KUwLU6&S2=9XUB1a%H_>U^~%i*xR%xm}6zA#Ql-{U#l z0k>7yKHKUbKwF#ppbHSDyCb-#c{1nO+?O8sB^M(+kxR6-%hj~w_}tSBtHg#g19<@VC966W*j(c(Kr8Jw9lCrk z!7^bd0yhD}7Q`#f>&405a*30OV)i7Z5KxJOG?U}!%ZNx8g>iOlO#OP;n3jqZy|Un9 z{E46~AQLLdRoC$%cz5R-smVyn=I%NDk*YPtW;>pUG2u1+d{wkv7N+8+8>;-nL{x19 zTCSD3GJfc@c>L+g_Pot)Hczkr%=|n{f>^f_}G-+lJgo>tz2;y?Yntf(1f7H0IjOBU_!UtB^#suZ)*16rO0g6US8r) zD%?*=Dgy+CRVnzOKMafwttP9(@NS~58iY%ko+p0M#`zolh{G+q+P2*;tD*98#Ys}= zDrQ!=qmgPp1o2KjdwBRWmo@`bQ!#*<#J_H=+N*3fU)l9_2w4|_c zXHfHupK|W{INS{aT6xIqWlS7J5uZgSR8!PT&KnxhCj^XmR=0fd;}f5bG^)n0|8^qYLzPptlm}G z=z(-9DUvd*;v5!;4EdwVExRs6PMTpdUrahR63>!YZ!pFKrxr#MXx8RDB;S{L#7S7e$)~#oc>24dM|EX60E!$S1Se1|bXulJ7?=fNw8ucK4$6Li; znlD&OuBH8^7z**)uHkXuW9a*_yoj&A}<>=KUNmf&Hu7SJ%1 zrb*HWy=?l-*0ZzLI~B)&C>BNkift-WsR06y*z!kC_Z;8@q0$h1Hh?m9uqV;fPS=_n z<6%OIdK<|A5O!Tt7%pq`z1Esgpx;Q8RCsqXO2k#8&tkD5U3{nMxvJtbi!nr47)pm@ zoSVyoJ2~>ETwSJ&g^E$bQlPPBPeF9^Qp}ax-sh2Z=x;mry{rXa@^?bX$>Wuk;o`*q zX9Mwnw|~y{Kg^HQ`gb`A!Exyj@$86ft1%D{&3?x-`U}l_tVW?G5LGrSo72wUsbHvLqn?lbC9^RaF8)}!*xW1I&36PaL1JmKxwBC;ri6*ePdPkIRkUwl zZE7}uHl#DR3i8c^c|n!F$ex0LAWuw!eak-C)|-(9>VFk=zK=WDI0;Tz1LY1976&3j z|ESUc>0ZysBM2&%`pPSNB@!fx0{JX8k(ro#hD~jv{&+~b`#5uE5Mj97htM+yRh$c6 za}FQ0IBqG+QHAHq-vM@_>Dh4Ylhs}Y4#17%zbzhqMh)NCitN7s8hRG-$wl0iXR}8= zHzen7tt{fABIQyUDZbe!PfrJ8d7}lS0osD;MXX{;A4mg1=GmIKJ3w98d)JKIXC|@k zx;rLk3IS(V0=~wxT8|nb7Z8Jk#cAyQOY8)Rc{4V3+dQ6npT3aV$ud%FRXJ%1gxERL zeHe=FG#9H@Su>9FKo3wiFBFPajo^VODsf~Tdtkh9c%H{GFT7ouQy&k^U;zQbpSmiy zT^k{9Lx7^An9kv$pzn_%kgXKF3^J$}vFDDRaV7?Dy;Z-HTn_LfK4>6aY?HEeDqbwk ztW^1DrSiN=s$5V+CsvH>(cr8_n^+6ayr-YmQ&7y~MINYK(tDnzk=sLSSl_#vCnX1Z z8}MV3@AyR0onU1cx(NtMry|!w)4?BiMP)IXw#+91dfSas-lEC>9sZxrKPUfiKTa!* zgV$AF{KLm2A<{y>I=4HI6F4^i%z8d2Qb$^N9=-3Y3k7}ZWWy*|o@2@%rFDR2Ml2~N z(903t3kBgQDj}1R?diP?El_T&2;i)i8=W+-Ae*HZTSI%0Y)M9(9Qh_#$p}Rc z{>|=*BW_Zt`iW?9t+;^OlY>`L&_r-UF@S&19;)(jJ<=C}Y#1&MUa)pDC<=@@xroYR z-hYRg$h)R(UJwN=hWq5Ml8w-VG4%^Ss;62lJH`BQ-oum!?@cgf*A+v%vVxDisJ}1? z?{6i>2%Ytbn2(H1J>!l>hbu#^;LVCvQf|(DX}Q8$ z1WfK{D__vqdy0p{r9V5>G2}?p_F{rxPC3A7hkLBQ!^K!@fY0x6ck@<3L^@UvukGG`HstDq)t)A>oer_d-4N>Moh<)G00D zLb7pXR?7GKXvgwPcKADqI$a(qN9hTBm^3xxI{z4Fn2shiqZyr=C_d zJYjvclUC8N32kcNl08lAu`H!tLx6cQ@OVO>9px!p37UYWEoJAbFXWA?M|oq!_*VVN ze_9D6TdK<`H{0ItOOE_o)s-4KSoV-E=bxE>gde9l`P&zB1KuWP+G$Ut$31@TzXP?H zr(IUjrTkL**pWN?0*9(ef82FoAnmt@MFt~iNfxfNMW}}4zp{LHv)DJe^Y-sAK}TZt z^*0#UsngMEh4y0d?SKA+v$abk?n|3Pw+xp}DuUQxpOZERaR;L~9>SKMP)p{{4)aEb z+E$}AZC}-@W^R4*B;KM>IBV~jRJtdP5J@W!-8j4?(HrJQtJnr83ZXkqb^OSHC@pW8 z$xv?h2Tuq6ZaT3bf#Z)g170^=i(?g`dqhmg5aG_v< zuO6KvlLQ#CFoW%Uvg0UIU&Crlo2qw&Dam>JzFy@&2-TgvPd(+ttl#bSvogn(r1}fq z!5$;tfHth55S1%|zDjT-mm)P==)Bn!n}}Ra)|Rd=wyZAiAd{IPKM|EXEia>bWv1y> zF0A^c#iYz{Xsq(b*BXH&1I0oI1UqP#a19b__v91|fHZ)zRbfQ02&0lJljah-d9%gZdqMky8Rdjl zG6z!(e z^fByLJb_=g7W7a3|jChkFfOL}&Z93Z_7V zg>+wCne5}Lhq$9na0=^OVUB#Q!Gw+e6?4^$Q;MX?hw6Gda&7t-{#Qgh6w=JiTZB#1 zGuvX-;tCg@T{H7w+2sU;Y}xW>P7F8r9-HTRxG*={Mx) z?mPV}>MUSjeH|g&2{&!$6ADId+@>$Wj4+YqrQAKPzP=8zz%AJ4BI7OAMv1k0mM9fi z!5tJs)&I==BmFo{lPsgZN*bh0m!QrX%Z#rR|3KL&Y`4su+R%&UIiyMS+d2tNpO#n3YQPZ??YUzkiBpqBN=8DG>#cvSp$@_BX z-r9(DK5tAnRq?tg3Szpfx5+Wf8{vzyEq%A*ju4pg+ zwd#4~@5WNl%`*Bn?RE3CN=*8ONp|U0v|P@R$4d>fWOe3SJJ4Y^YhQs=h3kLP1tKH< zsDh5SRH$wccWEKtIpo;}q%KI&1o>K@0K|(os%e@4r#fM>88RcF)@+0R2r8tPwNDL> zxNH54djv1gRE=c{5>&!->O0?M+KRz0>Fyez>>%t|s zN;BvmD%eSsU{%y+Dtl$rnz*yfy+%C+0jZrR z98B5#@*;Wm9Kr*rP%vLL>1+}N+|~8`fD$lYdJQw%|Blk$k=UMNvV7Tk)qvk9W5cZE z93sm>XQw`mH;gunToGN(#byB( z8@^q;B>5zS*(L$gfQU%>^$7agSW-M8&4~XuL#V+t)=)zv&f9w6)Q~SoFEc)TQm3yv zkE8V~WZvO15}XuAyE&I_3l1;&$R=Ji9GmFO&I(Kl<9FM7um3+Y|0q9B>w2~1;>!y7 z^^WDp@0t1c!Ew9nPgHMi!Pmv^7e$}O7k?g7y?(@qN3R#o4I{KFJ`+nqg0Md(@;0Z9 zrsD%$e2s-@fn0d|!#JQ21;qu6(_|`uwfBOG$((K!#~ubNe8NUnx7pvWvrgINyJxC^ zJ9dz_yMSF63L~A>KDPJI%nQ3;E6`DBUd}e&6tE+2H$Yq@qbe}qA*JkDV>U4s&BP17 z=nGxVX$vGVY51t4S^28rza>51Co7WtgKQ+26bUSk|2}JI=$plz3UtC(PTlIPvsv<( zI;;iN?@@`%;I5>G9np_;Iq5R`*f^41aBl#qs&=}t$8>bc(^9u9Z?2M6=#cSU)V!@K zdxzc4Ep{AjW7Vq==Xhx@b0`mL7xT7P($$@3l69@>M_ak8e)AYuf(fImG?-bj&_qwB zPP@`eL%MsF$I)s>J)WyChn&Naiy)0e`hR>){TbcZJ(~fB3Uh2a&q%`F6A`AvRa1?p`~f(1 zRgVH;vo;9rQB&x`hHbWv+zK3Z>ueYM7j4_{X6Mp5*G15AL(Lja_KvhCIMB!>1hqtG zw3N2>P@9q8m4v2FiH(g8tTTLWr9>>mE?iD!Ie2Mc7)9%J&*m~pxB|vc8&2*q?I378 z4FvUOc~Z8)sMi&u=l6!Y{mAzIWiL^!5Qq&PN~nJdtbEiRGH^a*_4ft5SMB8apF8@t{K975yK;8OVg0iP5HoLEi9M&&e-pj7tBnTg zz3uJo$}On+z#12P1}&5C2REtxGxLx3K!nc=4BL)Tbc-K$_4 z)r?Tt&b)7+75~48gZxplLHQ~SyEM{O65vq>(8{+yDK1%*ykiMN_2QA$5g~){wt(#V z_CE9}nezK^#Ll#lyno(~h5dqd!WE@?_CU133&Ru1V$Q_OR9ctzfp+&2<2vu`G+IC|A;6N0uQq;Sy)sWpFqC!zdZ2iy9R$m>U9S_7{K~WO8k8koEe3vEZqZ#_U>zg@C4mRvgOCxBt^>fR6 z6t5=x$#sz4{uGR*~;-whGIW-l&2n2IY8`4-2O13mCn zR6Y@wqe!x$bslYuc3Ux9E_y%pg4lkhbVh-b`?!OdVj{UGX z07pCKI)@Rc*a0f|WTMTwm+aAX%AcAn8k~cAc}jQ+_!VGqhongEYij^Kt>P}q5laV4 zgp$+ml;=w0W>`J7IK^yP_MnmTxCK5N>F}!q9Xp7s0faS!CePSxMFwm(WueBhJf49w zaRp`@63MsESmJjBa?J|?xTC+EW4b;eG`{n;(GQN&ay0RlEuNP>{6S{5gYv4elA4zS zrXNmgWR+yH0*1a}_5Y}l6V)21|HE!b+A<$oiK}|YWUjUG?e?>852qvDyE;8 z`YSb;Y~n!51~y|iXwvQS!)vR`b$*@bTwUPvrX&gX!E&XC{IS%I2@r;yOmb1Q#^1C+f9I{0b8|>rSqQJBf1&r&m%>l0u4SddI3N;+$E%SlPP+BdyeV= z?GwQ~Y}2&H++hqcFp6QZ_mELZk+0O&V(ma@!qMELfd$2jaF2`FP@njQOs6F@tkE$!1 zVaWwPxnI5xww7qqb~hemqp??kH{MREq$O9%6S^Ht#b{``3nSG zY_9l=vm^uZV}b!djtH%&Pim&qTxPqp7Tnf@Q8m(93O&W=LF>yUNocI9_#HAFP!R() z8PeEfPVMDeFH6r{gf^aaK;Csuo-l>=wy-R&QJhY2UdLN((nuQM*~A>`pwjH}BGI>l zkKGi^$TN^J{n>_VK@+Y*ROJX)ER_}04qStNuJQEm?8{pM7po8Fj+kTv#}p@I{@3n06T>!rm+#Gd0BnV?azflrEkhs47M@yMYWd0HQ|iux!i z3PGjUMmsz3hr+vAgu{Gq2OV2xBAwX^AT-Iu63N)m=>GD3H3UA?1Av&=!#b-)==1Hz zhjEp%MqI#v{5!UzuY}Qw+K8>wFAmyljNl!Ke6YxxKruc=gVZf2Q5}}Bb}BpGk-zb6 zaS9FiF1inL)=f8(HAngwFL3oLCuM|S(D&rmI6K;23jfUfWBoX7I_=Ls$Usb=63W}B z(Z~0%PQfL*LX8aZO8L0)&HPPI>NB;7tWB}_s8#0#r2ZkI{P;DtT5Lq~-z5(*UcNdM zLstHIO5h_3*3sw~JmQweRs$&%?5OjBB34%)q`!^rGdujJ534@{PobuvW6-wZY+I_! z*jX6KIyb%@8YzyZpd=z}^TEPPL;kJ{FrJq8O@gc+4tH}@U`~evsd}>{DDe56#7mAn zpdygWme3z`*_WHM^798y@j>8dU;A>>YwHQ7@9BT6ykuK9; zQfL^qzA4q2>U9n<$n{`(#G~Y7dXwcB^KSre4A(KTbP)AFl@n|2R=sM5l;1&ua^|;w zVL$JcjBtSzbI#J3FDnRo+5m`4Re!Z)R9PJ1 zrA&u#32MkXNB!=%wRO^u)^R{lYXrj7P_3n?&QW46loqXxOfLByc{T_OZ}e0y^lk7x zn~-WuZ!}@}sJr&2V#QU!$Z06vj-SOx6o`!Zqe`+VZx~>Q-vP3dSx45Y=_tiQq7REE zQb~@6O4UN{Bx;(8keg%m!==xYC*@f@I`it@J4BOm=e7Sj9iP6FfzYRSoM$*3XR~=& zbWnC7_Yz0urm4i7aT6VeU(%!jZ2ejtLLyc(q7`bM6B)eroE&4U3!w+ljMIyqz3*=s z5j(VdG82ISR&@hRZuWMeAi}c!g?cnwPy?<9wiWwrAEzhO)%D`Hd1x_X5;5+T-YG3? zwQjI@`v|2C$#p+^DGECo-nXJJr#obbThvmF)m;5iY@Br`$76C)nlK}Gy0>&Yshe-N zI1id0BoxpUFo}LFfL0)`J~^v-avDn*>M_KU;H(o{ z{mKDs4Tgxl=7?LNwcEjH5&ZVBTq=Aa-$nA|{7jA;;gT-O%9~&*lzdZ|VCG0Cxv34A z3PvKB;}xjZuAy)s){NwyMjfTVp#`zEf>Fg>YFk~`3{uzS~!)_bI}ri!)>8f+Idmu%`xjD=i(E-&~`?;efjGJA%@hpR*(**#Tg!p|X}rh>Ol}dj3;=vzO-V;<6HL z-COfs)+g1m1+>dl*RrJ*;X~RO*~-*u^W{{WM6s#9(nD&4V!kcVc2)+kNv+#hpkXDh z1w#!-MzCCgo{|392IDtzsslys;00aB8wiA#5Sl?dURSi$8^$#&Eg0d3(2=XLsSJI zn|A72Pl3qTKdRV1JQ&JiTjxv=5~qm5i}bylK6$@XRv^e)%QalTXzZK)Y?4y|>08f# zGzUFC4VCRLF`-|(cY^#egCS>IH646mFJ_*#iRLt=zG>)`<2z^%7AHX* z*K-8|xIKY&#X!Ew=(n?nd~wi&TP3HJj?CPMM}V**+bu2uAI=Xy1f7XM91vOP?Hs>H zYy=)S`fZFsuCA#Aixo_VO$yBY+tw-%K`KnIo0DD~QkC$|AYX1JAcb4;_&%t~`|#H0 zc&~1-SoZ@>*-YVAnlv-SNQRyWHIKW4jYPMM_2A%9%p>1&hZT-@X8xsb5Mv$4^et-P%%}L4@ zdeh(hscX@Fq!vD8q>SRjr4aLU!$0Mx=B7%W_g;a@Hv}~Tcj6KYNl?PXqhj3u%>3j1 zIE`wM?I>n&{hXQ`OYN$+qOEIh8iY>aOg65K?>yB>ET)J$hOcydA!~udx;;LB>>+h9 zoTIr%*RypClQo8=BZVS0Ur>RxSr#vjAVrZ@W))WRz+fr$-KDtY5$b#c?o|FJY7L9+ zqXzJbyb3LIUeX)!{+;QLbK8|U^H4o`sm~DZ`gQuOsbxCo@FB8jmE4nD2ZF z$ScM9@uLMYX(D}N z(#Gsc{92iKdhU?TG4=S>a=$Q|X5`duprC=sxIe1sbn-QQnV`3J zOM{Y%-O7~^I*8N-QnrAmzFYSOwGuAN+KN2qTxW1&clf!X2rG*wYFT^0bzoI4p;e^^ zYZACwiZvhaK`C8p>9Xic)g)97MtyTscX9_jyC#p)WE+1uD=SSGGZphw*lo_OaFcf* zY@_8Isl4uLzDzX0>2NI!20f_*gP#CV4MjKT%F0PoP26T6>iqWWgZ#7bf&^V__EFRC zz`B5AzBFJt6~^b!^!sB>kaET44YTOIKPoqpiB0tiEz+J9Hg$z|R=kz~5lUIz3CL#V{Uvj@+t}QhyKnRHG6k`K<@N*$nd!fPam%mW^z7ian^E@ZB%9`X30%b%{o>N9IT0zY2CKb;@pPmcL@i3 zi%@1cbfj5=x%<7Auy70sLj#iIpT4RJH~3vA1+QZ7Lw>z5;*Aoz31hp=Rx}q5wqvLD zu(9ocjr`zI3UheE^L7G^Z4F>==z`F$7#-8=IZikgD=Jok=pSmpY6zB zhDJ=+Ht;XkVmb*V9|?=V$|K4tf_JffAD&7(eE$6@p{~h!q!cCbPR$)4X zbv~T}Ug}xWnnoYxA7F(>x~&xy`jAWab>P9--tU*n-r_>kO3NqZykvbMPpRDv31V%} z+)t=I3{d!8nw~YXAtjilZfc%}{>njoN;qzo9EGCcV+0cC95(^=^LR_))WJQP6sn4%i0C8Wur64N`L9Qu9#~ zqKG2KOt4$ zeVFB^5yX4jld>p7B%5*4j3T?U4E^`22J}P|Bl&6jYS;~#suTS2`#~L%2U;H**F>kF zPAqR+Cw=!BW?u`Q3za9nz-rib6DUn?GqHShl(tDsWuVWO%3eXMXnTvU=cBA&N>d1U z0v%||v{7naG^?VFGjplc&}~W}GX9S$?Yg)wBkqVmwQ(Q6WDct8^t9;wNGDN0x*FBG8HTiRDy^8_v%EuByj}ZLLc@d2k`QyfTk*TxBwFdfN>9C1 zOg$+BGG-Fb{2H*H_e2?@t>vl;QtiVkUARyc*t|2#? zotBbx8Oy>T=~X$yR2mmn#s)G#prAL~qES)WnBxSWwEFVz?nI7N6;<_~;UZQQg{=Bi zxz-!L{+b|0?*jh0@Pd7_QYVg+!WkO1WIgmCPNbxSdj1vN6;@F;GFE2&U1-qn2bKAk zC8e|wWlqj&5!?q)&%LJtorwZI>{GMi%1v7a6$v;wS_cFjNM)Z-BTB<5VvOW|{p767 z@(gX9=sOldDo?3L$n&fA?{On#@7|ub;svYdh|yc=9=3D+$~b1RqU2R)}mlJTtc9 zi85oEiwI>`)sSherHH?qk; zrJzyes&I8VcH#k>uNX9Ko@2r70fBX+1@(>|D(pJB{83Qr`Vqd&$i=mi;v#}9_wFDM;+J(4Zm$A@nP`){e^p-Um1 z%+T@mc~9Cy&)hvtIGPa$?9#%h>~9)TSJo+%yDL7(S*wkTMgDuB0wdcbPcnA~#bg#Y z4pl7dfrMrcDE7#m4B~O>u{ex~{ZN(H_B5lr;s{)pT3)0)-bY?MSk$H1)hUF@T`3p- z*4ite4Xl3Nr2Bq4m|0dhcZdkyyOce6i^vro7;;G5$S`IfmETkx&-mW<>GBhifLDOa zA}JaLvdyEh{cSO$M+EjV$70pye*jv05N(wxzs=6d;R=iy-)3Yb%vOJc2MYjIw%8|- z<`}B6#*bTpwww+_2%j#A?K#{s8IyF0O00(P*pjN}0oWwN)3PgCBMd2MU6ac$ub>U= z%VxRbdT|TJMR2X3Ob}o(?ng{6%&)qG36IM| zBwzl5gD0!t!3(hYx;KCA1OhW!;ZXjGti6!fvnw;n_ZE7%$dG)*K65es2IPP4BlLog zp#BVaY1^+&S`XwrJwcn~p!m8}>+_h!`d79dGr7fvt)pd;Ghy0It;v5pNlK#V6X5O` zVRxT4{-MC3Cj3KoGAYNAYYY>`hbw4ctGT|8`A-3*f@6mCMI6BpYB=ZNPWR}7UjIO- zx2E{cXj8lo0aR7xa`E4~L`!})_j)}~&57Kc`B4-qkCRf+X-Q9y^wrYeyu;4{-Kl|m zjt9l{P4%H54&aJicK*s{XOZHEzw+ST4{oef{=k(mX7|}pg*3!Loh}L`3$|1|hlSBU zVCp`S^BhchK7T36dr#AqJg~Jbo#&3oVz3gmNHt7x(d%bsij7{vNjV;CcL?=l1?%Vv z8t`Jcx17wokR6vB2UBwXwA5AnO*eb$>LvOf_x*R9Q-q{|4DwF zmfbC%eA5Pgr~46Cn`J&IRXyeM|*frg<7cGs92|}n}H5k7R zK=fR=T8S)KqGts@EOo^1wD_w5vc>G$bw$aubXcxNEXB@ zxV|9x@h^u<%im1@?S?R4oMZ73e+fY-8)o~J+~^wp&ocU(T-l-qze?=R&R?IVqoib>ZR~(`Ce=3g@~VO#fcCBN-bgNM>5in+bK})9v?PmL zEBGi?`HC~K1nuWx>2!n4d}za8&u5!b9<{d)U?(QDL>c2jPF^Yh%tjEbxmE*2Um)c#(@q;s$iEkG0HFt0m+m(&3( zb8AYM)}z}i4BeJhX5axr7iaqZE@mdOq6_EKs>+NI!Yc5Xx>OaN`%`CIKK)eU0Y{cwemhq-Bv_O)`aOnc!)12j; zcc4h9A$yEJYmPxoBZ!Jhq?nAEk0l6RXkQ<9hcRbQl8Jmp)DosEZ6 z1BMo;w1g}2$a8Q`?uU^Ffyypt-l_DOhxs`0_%fO7d@9cDY_@iNrD#$wNQ!@BFXs`c zFuK?Z+S>ma#=))%YRBMJmN@wnC15_r!B0alS81CEK-0^A7kJ1X|0&7ZA_nvO;MFX*a@cY znb{gxZ0}{!Z${QYz`y>+Txx`*ln@PBdk`_1b@gC$e`$LDz(`Zlna+@=k%adV$A$*8Dy;b zT(E&?eNcpMnvMI+mP+{!4~`w6EE!|O#I?zCmkD)V(e|`|G#^Q|@IAoAJt^s9vqo{z za=q8Ib)Eflp|@4!%!SE%B-J?{rUEIDe4Ux>I&KrTJ5Q-0dy?1hEmgGk#NV%7h|7Jt zd`*Dj%ZkB7ny*`V{@$5!zH(cBJc$Ya6_a{&(^i4my8yp2BR)WiU9Yp?Idm`NrmjaJ zzS)FhwMwdBm1rPcBN{Hn3{Mq^O!}kB!=sicxy?|dw#<8)bT}C`@4)GnQ#vN{8y9R1 zD7uA*y$@K(_+FpsJ%5%BGuM&FUi&RJ!$W@)Ex3coLZPKGsE(kzo5}F8cSk)-(g>|1`6t}%4}|g z9pAb#JSVQA;rtfCRj~MMVpRk=)|=6jqcF-7q7NZpdCK@n(|T=9ELG+{&q9x^x~NAw z&N?>dLo*&1scx4|C?n}|-$|0O9ZGbh?n`bg$m1Lp?u zh{*l8Tj)ZNJ5P{I zbdNtj`;DttIfWguxUtOSgS%!FpwtMOQIjjCeWr0jg>ALO%gdTgZB68$ScPr7{at!# z55Wyp2NHixUyM;pqsKo_h_J1mJH0J4qN8MJYU1T0AKtjegpMH~_W|72fobX(U8qzF zttgvQY0_i0^z^nbv7lf0!kkg~ggX56J}+Z?)wxfBDcF#c^5xxO5YvZXjJw#ZM4(S4*KWL6GW!ydjSjhUFb?AYwrQ)mRLRUO73VR<9$kY}RbYJ3+(MA4G{g5C zNw3|sy4e}sJ$O>W%*&WHoK)q)izG2^{l!WIl8vPKW=8 zVWvBdHr?Hh?(S}e>5j?iZl=4BZjLrJrgJ){9frfy)Yp&Szv1&Y-1l?6uJ`r6o`*7> zniD4O%Qk<@9F4HP+UOdxLOou2G+)}^ZIsf1w+seoTIX%rXpr5)kUc=iEw+h$Dlfyq zU%lNSgJuWDx58)JrRN*PXTsII#yiPF*E!Y08)7LIOXJX*SBiF~gq)!)*0BHR{s;5B zX)?;rF>d^2wb zdE77*dERSgzc;C--<0b!J6^bwyn4U);p&!wRh^_LRZF*a8}YG$Fbs-P25tNhNB)Zl zqpq`S4^uapEVd*_b%0`U%(FQcn-~vf_r&mxZbs?Eo$=4F@p2QyCwbupMtWx;pYN zi}10XdI*eF#IJIKSxzkMj@hNl#F7*p$hF9s0D~>gF`-iu&rM=NtIZ=m#VllI8X+lH z{lxwcjv)0l0v|*Ypc-XJBBqD>TTT14sm~B3QeMumtZD32?j<`!A9sOfv;aREp~|6c z2r;l+L)eeonkflm=qji+q&7^F>!eMKZ+KPw1)auK9r zr4d42??RhPvZy!cdX#29WzdC&_m^iPOS$U$YhKNtj~^`zoBmnrnq&qlOD&6i>r<-Z z3d5I%@8LmPnEmm3HxVWmZy2dLPOI840j(eN-ep&u|qwt-#x9flj3o2E=hi?%Zb{hye(4)faohEdw`r893f??na&(W=B+86^WVJxtna49R-cuRnpHO^mdta6 zD*$_h=73Ve$u36F%k>>$JKwEv8{wK%os<qkg#g)+*G7)hNNmd7tpwAg>CXh`$v*Ih1-~?=HGGD&6AIz39L3a(t0tQ z_rp*wsv72h6HI45RrGY?p)X|&1a+MN6BGqTKDc^L5__Xvgu6wKC^MD0{GlFkJ-J{z z{k6dj-NK!mp1iZ4&s=UITmk9OsfA{G-f35OB}KI1+3Y2edfO&$wAT51#+CWLfzddg zNFi%1NhjWDl>%`YW=4`f*CVyx`e zCG?l(jT-r*xJS0KqkA2Qt9uv5!#9+(b; z-5)8^aG*98Z_a6N3af>&l%Y>~{&ela??(MaS#i_csd1*9OpiY3JBma<^wLd5SdU9R z&Ozq!trzVU?T7FFIkuPe?U+0J%qkJqm62x3E3-IiyQ_aS%{2U4V9ZO4?r z$Q`)c^`7@BF(%B4&n7{j3^{oGstrC`M2h-5({#2m(^o!yer=X)5(&ERP5agf zr!P*~Cltgqq2)Tja=Fv)Mpe3PH~L-SlidbA_qHvzY$v3SSu5%lOmb<^OuXT)Fz)nNgpNuy3TVawp z3Jb4PE@~IZ;krNY=x#uN09TZH(&v*6NSUGj^C$akfBHr}em*f3Zqb3h36GUNL z3}Fb45&*L7(=7$$0~#rO>zN@(<<@+fP}=7nl3(YB^<-sc$ZwEu{3BL|5*vFkv$`ZTc&JwOBu2RSmH3#K}p6VB)m%JYGsA zFW%1LEXJowle}*3HS=IB))g^G)}#bHmLD+tA6`z44sDO|h36ylMFUVC zL9Ko8@FXZ0lz?w4cz(ZKeoF>*-I!_n+I{Vt6IKbt{g#*~$ zsl0A)*sO?+mle%mj)2r}65Ev4N{tpF&!bDEdx<|Yw%ZH!G|Vi%W0C|GqK+SroOkB6 zdz8)>SasqV8}W_3sOSVF7@&Td6v}9)>+k6Mwvd)EhIB-^;FDm9v&u3Z@M6)>`i8xz z=~^_-tumK2O^VyZNSvD0^R&i9|ETv_s)MxoOZ1d|-6{X~z1qL?Sa`YZH0_a3T?v|X zKhgTbAJ>qAqLqpnizCc*FXm|64SqFLfDn@qPT{n$yV(@m*JHAOq~oZCK)9=Wh*fs& z!hh4N@_)>Zh#AJQPpt|W>kPRn2=y<9P=p=}du1@x(%tIpw+++jUEMQsIqwx2?(I6U zh0tdsb>;M7;pY23CjCtG`)v|xp2~&Un}(Lc)f)otWY-(NTEVz^!lC{0JDw-nA7kFC z|5_%fX1cU-x-PT~6=jfSDr;?pou$g+;O>Wpe9na~`F8@?43Nf^TISioc8v_A?QQm7 z#}VSKew=cGmgrBts@UeWI!k>;LSqR#)_MSH@c9*9h4bqP+{c^$X8t+fO=AOh5`^U+ zH2!q>Iz3cr5EC4;5OWx5z082_G?)QL(?}>_CDPO*qBj&Cs_6HqZ|U7i&}Lr4h%G1C z9+JI7kI`qrP7-WXd7BQ$`YDGmyo{dBr6tlx31)+IQ4U66;iOXWr>>)8GTneEqd0DB zUS3r|=sYwGbq)nu>b~ZfaDw~=ek8PuB+|c3g2+yc%kR&5AU3FG^l;cxR!Lq_Eq~MIvW{-ZR3!Y z<{|_2(V>Hmphkm(Ted@(bp2nkGHL$N?Twx~IdiUh<7TWWzRVdNj{WkYOvz*co`3`h z2jyy)qw;0(MEMAa7vw;yI$Zxm>`{n^JW^f@7Ps`qgualljfwamnpZ3X#q&u*FM%uH zk4sXn^B*!x#Iltr&u4Kr(RQ9u4vAndW(W6F6sX}WZ7BpypO;Usl+dvhgxRR|2tzaa$!Szt)`PnAv$j^MP#tNZ zrs0_9L}%tvVtH121mzdp^(RF4CY*?0J6TetC*MSxaX~fW38N<8d5VB+zd*fO7*a`8INu|dKVk}? z&&oEmTL>5E3D^FztIZNQBj}%zST!;dt33${H1|)vC@T=I5mPu6=mym(s+5@~LQJjT zM1z^qZMMxXYcG(|)1`$$V%_a;&$nwAkCy#Dy~jMiJsddX(yGW43>Z1K5rSp!)Vl$d zPf*crMUJ%nlDDZNPt-iDj_Chp{<+^xOR0RMOwvg;38Tnx4G_6^*QFu2`jn<{hSjX+ zskIpluE6=egTh*Ytv`xsAY2C^5!z;^bwpv~kcVZ*K zRoQJBc$>m}@wq+b8nzK(Vs`EGD`}uJx!5#jdB?3{JF55WHe8lvuXz)7NgF%Q)iMFx zH7jnP?Q{}h0wWKR?nl9Oj-JUoe&RmTocW4ij5JSAKGR2d8(gK|AY`jUhQ+szA)znj z5us6Z0yt-7yFGGtX&El=gJG*kYF;Y8jU~l17ka%$*pZ;>w=ST` zANgtlf^57ceRmmE&|XHXP~-hyhDn_hvNB$#-5>}02iw_mvXRw(1sZ>Aj~&f|@K@*} zy-kfbZ!=5mF2ey_?^K3Svnsr`NJ^j0T=k0R`1$@p{+>^uWI2OhRPL6RK*cp5ivvYW zGtIH?_q1B7DfC5;)$|j^Q8UG&nK30upiTU)YlKpjB$wsPGZ9XyhJ#B5_u!h6K9LzX zkslFHvNr)|A^n5hSepk;X*#PDSwN;6ufS}QOG{0^ zJr}j%d=c>s0c-Y9xvC(`Fihe^T?P9WLCshrHg}QbOiF0$sm-N4x zf8KY~NYY!3=jsW9LMPk1=o`Mse{FdP1#RVTQqFcz1oP;PMjT!SLY1Fc%kPey2Bd(l zjL#Q$!P3~Av!9c7zS1Vmfq_vMlNE3Yk6|UCfO~VTa-P}^7Xwd=fnOeEG(vBKv@SNe z3vmT90qFe9v1t&TwkJ#M2XEQDxJ8s+!m?y83Z{ZKxz*aeUqU)tvl<4vf4#> zk7hvP0MHj)QOx4~b;PXCtmo95a)@(_)y8a~p4y3m{=Rrb{|2XC2fi$}(_KP<51t8y zKxtS2VS|2!RJhOwGFR*m0r!*9#|s@MGUt?wT-wZ`xpLNeHn-m4!qcxg5nEo^oYj1m z8jBUFTX)^>T{FrScyj1s$i~zae0Ljz*+snXSCAz+j_&yNn*r z&IGyJWbA8Ob-|`4zK1$WZxsx1r5)nso9m)ngZO6P>$jT!WBh&pi|3sRcp>!rM8agj z#hxoBoKD{ADVnO*CqPfiIQcX zQG)Nf4#h2u!Qs*uhDt_fM~6RUL@|^j$4m1Nie}#_O}}gj{PwzS#vs{g^mW`ykK*l} zp{ViJ1#9gLcq9V9g*@W5v=b_;yl-upRZFw_?W>+T2mqEJMOjCPW_x%sBSbl&5O=3{$s@EqXoXI{y#ts9k4ZP zmNGdvLw6#S^TX#d^1U6EitPTc$NvJ#z|9U!LDV5jCc15z;Gl@(kSHHfSSd+_KLT9F zJE$OhA&7EWpJ>Plh%IZ`l25NR|u_h^t#@%k_mm7L*tTCqXzmT(= z@QVC~qZQy@dlr`O#WS*I!w(`+%8^U z*=xASoO?q?R3otM`IuFcb-hyFFo6lhKpvB|PpiEa^B3E^ld|Mls8-*NvXBl{Q+M%Q z$~GH~t$?W5wNQ1ebl0k~$P%x@MEb{TcdKu$6=qbDI&%1&MvaJVQlcHk>53SG} z;qx%o39Tcru+n@pOs=xwDACVw0N*6xdpR2#l2=kjTdlo!0pKRJ^TwVnV_{rj`PKU1#~Q3q*K$_wEzlY8Una&J2vnEq4d|s_Tk+;LI({yJjny$hTg8xUc zM-lDtLNAE}eD&YVzred`c^$slWb;_D9bfY^0H8nHLdU;qZgzDeG8WR<$T_y!#!h@& z4rk9}b_{KN)+{B<`gmzA#4LGShkT}(l4-tH0$QMtZh^ihQIQEmYEdZApxRspk>||7 zsFfu+&KBN>oRK)ujme89QfxXkPkBZ_uJ4AE7|;zT%Q1B4k9U6ZYY+N3nII{6ytFY; zugzgnsVGV!teej>0%1s_f#nT_Tcp=_KhEHrmYLrT)Aj)$8Ba!lp-GgVuA$cZ#`&jQd=n*UoCt+Jm%03?79}HvrmBJXY)i3ea`J*%QU2$IUh8`JxU#+gSZ(65T; z4N%`iPb{^!wzGt-5= zQ-7I)^l}=R>kMj`U?NRaW0?pQ?mgsnvUp!Xm&WA_zcD_^Sgfq@t1}Qb1R|Hh0_{@5 z0l;@EPlK<(z-Pcgv;z};O@6H_J6EO~Ds(GI$|$-h{9@8hA5%>Cmqrw>E+7*gE2z*P z67!SM46`*REf5B@Q;W_vAhW?37U|qDxt;R_yM0n|C$PXOS-35s_A{i~?ZRp)wrxMj zo0ih9Hn`>EciNJPz7!;l0q+;i*vEi@we-5iCOwo3m z-7j}|2gH_oTRt#2XbMs*ew0}EA5Q=XnGJCic>aC`^eR!M@ft(p)gzbeJ1XK6yaB%q zDM8C|N%^;#;C*t`_oz zb1c0C<9s-#+?n&Z>a&^*!Xw%(SSxgbZ(#E@m22zm7kQ)O&?lGS356_b|ES`!`&euP z2*M88DyI^V?YeU`({V~ess$ok8%nAb-+HKcN2iLoJM}lYR)ik8C>fZpM%1%vt%oj* z_53BZN#p1D8xk5WCFJL}{64KbkoQ?>PF_SxvOuD-Q8yYz+g|H`wErXYZkqB;k~j6~ z{7({ZOhKYl7)<{V2g&Ozkgjl_!|zx6mj~DFfG+Ez6-^)co*mAYOP;Oc+8w?axn;7> zu^C9JkEfhao!OI=)5n2PI_I*da)h4%NhZwBWSo{bh{~NY&LF~62IVOym15w(@_!cN;yW}31&Q+G#*$*>!erlKL47E%*XT3EwN1Ep_I>LKaMO`|A znKU}|a;OL&_q6yeQO%)f&ZVo2dKdp;L6#4RpnYp|vLahOOlk-V+4guJ7-5osflIkz zsmnN`g02!J%$-`>mO}>F_>LD<%(R28`(x%-CrTf^4`uV{=ukz^jKDe%irkv1zT{Qx?&jd{r^w!or+P!tMCnSO1n#9ZD3T{ALTzQ zVwR56c@;lOQc?T(JR|u`846jeIxwnu?pg{wTFEE-76RA6>g;8s&v~-=(m1z2WTl*t z{3p)81&XNX8qGj&lA2TrSVwmLPa&irru;S_ajgxNNuX+OTN$v~hz4 zJpZv7Dt(BZX(A!U4`2>17WjtK0KAUkxyW&!Yep>>WqPwAS{}s_wl9{H8t*RsCu92| zwMx_VYGURCfXkBz%9$?x=TX%I<#SusIYT3N$kfE?*J5vlP$`W_ZwDhoD({i6jN@8j zWAWjq_(!Fg756a~j5ZxNBORPDO#2|ERUR7Q>=`%5B;6(ycW>JQKO3HrDo~&`J;Ng2 zE7Do&RHVOpcY%S~@ters3->>t*cB>Q(N7~TDmk>t+geo2*r(?Cq?qDSEK zrQrlsF-*;Nu1kkY0I_S`y}$eqw4QV@p9CITdb8c>Hl)2GnPT_K^yT(L3uxN(6A{%& zp;Kb*(ZCYPZFVyw&)@pT?#N6M|EF|ddZUaQCHjeL7c`R>tU0IP3Wn-}q2Gkq%EAWTvG`&fC zw86G+hU3rhM5G%@PWBby(|eWCaKgiG7Gg?W>su};Px5ZX?n#bl&#DXl{Vd83?WMr5+{b$2TqApB0H%U|1aH25?_PinN0emI-1_m>MPo*MEI*K(9T!Sdak(9(B5e9~%W+eiOIzMbiXMev|J6 zSXt5s0tc94&fs#hQ-0sU^>j?-0LDG^7&KQ1z;(cW+@D`re9A4DqlFNqV(}7;az0HX z1Ez=3aNJMKk5Em47jn{IsvdF;UVXp^VAI;}m;2U7k|1m|<`9Fqgv)=oiz2>YhimuK1tT8UH)_b~K+ z!0$6w`BwfO_6b1@0B8#uPynKNi*P+=8*ctSCfA9hoFfo@i zFs}m=qhK@yMnk}qiA`KmlF_{+F)1fCEi)%4wFpA8I0v~phPWz(I6C>bDnP_E3KCNk zN>VFIG+h|k#6?9JbHUn5l5z||m>ES2NJJw)FIAx|u_!qsu}H(vKvPGdqtI5#mT zCo``^6D{~b`5IncbliNtV}I2ZGjQex6JY5fhrcu*ILtr!nhDr!9104!47h)frJ2+#Zur;KOf0cpxG4s`-;*n8z tj)uT!2;d0;9wug4#>_n3^28k7L?(VlW>wDQjMU`pf_z{%A~gkBCjj_@a6kY6 literal 0 HcmV?d00001 diff --git a/testdata/wal.sqlite b/testdata/wal.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..d186e3597faebfafef1bb0f6e1364eec2118bb6d GIT binary patch literal 2048 zcmWFz^vNtqRY=P(%1ta$FlJz3U}EBNP*7lC05TaEn1C1t7=a9s7z}{;>@W@vnuS5H z_XjUfl972G0}zdZ(GVC70kT8Dg^^8MRFpBdBrz!`wInIW5QLeWgIpa$TopnboqSvs zKq4CXd8rCziABj7iA5TQ2AVnwCFS`F#kq+&IhlDSn#k5e^5$a(=EumQqwLWT7!848 Z5(4~;%&MHp8L7$H1^K}2pPGW~F9576B{%>8 literal 0 HcmV?d00001 diff --git a/testdata/wal.sqlite-wal b/testdata/wal.sqlite-wal new file mode 100644 index 0000000000000000000000000000000000000000..c241242e188e1925b8506bd46409ba40ddf7b37a GIT binary patch literal 3176 zcmXr7XKP~6eI&uaz`_6mU6O?ll-an0w>rqoJ)rh06)406!Z7i=*r^6JZ3WT1AVKD< z49t&##3&dIfzc2c`XRu>#4O90nWtNxn4_D>#LvjA%9)&znw(vbpP5&Znu6v*lybqq zmNzm%q9qknE-*762bT*#Y7~ryz-R~zy%6AGMwA3e%%qhIj6mKNjx5aQ_M(a6tB iRVYg=O3p|u(l9j8)KMrY&sQkUP0YywHW)OKtp@<+tx*^N literal 0 HcmV?d00001 diff --git a/wal.go b/wal.go new file mode 100644 index 0000000..04484b8 --- /dev/null +++ b/wal.go @@ -0,0 +1,147 @@ +// Copyright 2017 The go-sqlite Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "encoding/binary" + "io" + "os" +) + +// Write-ahead log reader. +// +// A database in WAL mode keeps recent pages in a separate "-wal" file and only +// folds them back into the main file at a checkpoint. Reading the main file +// alone therefore returns whatever the last checkpoint left behind, which for a +// database belonging to a running application can be arbitrarily stale. +// +// Format: https://sqlite.org/fileformat2.html#walformat +const ( + walHeaderSize = 32 + walFrameHeaderSize = 24 + + // Low bit set means the checksums are big-endian, per wal.c's + // SQLITE_BIGENDIAN test. The format page's prose reads the other way round. + walMagicLittleEndian = 0x377f0682 + walMagicBigEndian = 0x377f0683 +) + +// walIndex locates the most recent committed image of each page in a WAL. +type walIndex struct { + // offsets maps a page number to the offset of its page data in the WAL. + offsets map[int]int64 + // dbSize is the size of the database in pages after the last commit frame, + // which supersedes the size recorded in the main file's header. + dbSize int +} + +// readWALIndex scans the WAL in f and indexes every page belonging to a +// committed transaction, newest image winning. It returns a nil index for a +// log SQLite would itself ignore, such as the stale bytes a checkpoint leaves +// behind; only I/O failures come back as errors. +func readWALIndex(f io.ReaderAt, pageSize int) (*walIndex, error) { + header := make([]byte, walHeaderSize) + if _, err := f.ReadAt(header, 0); err != nil { + if err == io.EOF { + return nil, nil + } + return nil, err + } + + var bigEndian bool + switch binary.BigEndian.Uint32(header[0:4]) { + case walMagicLittleEndian: + bigEndian = false + case walMagicBigEndian: + bigEndian = true + default: + return nil, nil + } + + // A torn or reset header means there is no snapshot to read. + s0, s1 := walChecksum(bigEndian, 0, 0, header[0:24]) + if s0 != binary.BigEndian.Uint32(header[24:28]) || s1 != binary.BigEndian.Uint32(header[28:32]) { + return nil, nil + } + if int(binary.BigEndian.Uint32(header[8:12])) != pageSize { + return nil, nil + } + salt := header[16:24] + + index := &walIndex{offsets: make(map[int]int64)} + // Frames after the last commit frame belong to a transaction that was never + // committed, so they are staged here and only merged when a commit is seen. + pending := make(map[int]int64) + + frame := make([]byte, walFrameHeaderSize+pageSize) + for offset := int64(walHeaderSize); ; offset += int64(len(frame)) { + if _, err := f.ReadAt(frame, offset); err != nil { + if err == io.EOF { + break + } + return nil, err + } + // A salt mismatch marks where a later checkpoint restarted the log and + // left older frames behind. + if string(frame[8:16]) != string(salt) { + break + } + c0, c1 := walChecksum(bigEndian, s0, s1, frame[0:8]) + c0, c1 = walChecksum(bigEndian, c0, c1, frame[walFrameHeaderSize:]) + if c0 != binary.BigEndian.Uint32(frame[16:20]) || c1 != binary.BigEndian.Uint32(frame[20:24]) { + break + } + s0, s1 = c0, c1 + + pending[int(binary.BigEndian.Uint32(frame[0:4]))] = offset + walFrameHeaderSize + if dbSize := binary.BigEndian.Uint32(frame[4:8]); dbSize != 0 { + for page, at := range pending { + index.offsets[page] = at + } + pending = make(map[int]int64) + index.dbSize = int(dbSize) + } + } + + if len(index.offsets) == 0 { + return nil, nil + } + return index, nil +} + +// walChecksum continues SQLite's running WAL checksum over b, which must be a +// whole number of 8-byte blocks. +func walChecksum(bigEndian bool, s0, s1 uint32, b []byte) (uint32, uint32) { + order := binary.ByteOrder(binary.LittleEndian) + if bigEndian { + order = binary.BigEndian + } + for i := 0; i+8 <= len(b); i += 8 { + s0 += order.Uint32(b[i:i+4]) + s1 + s1 += order.Uint32(b[i+4:i+8]) + s0 + } + return s0, s1 +} + +// openWAL indexes the write-ahead log next to the database at dbPath, +// returning a nil file and index when there is no snapshot to read. +// +// A log that cannot be opened at all is treated as absent, since a permission +// denial or a sharing violation on it says nothing about the main file, which +// stays perfectly readable. An I/O fault on a log already open says the +// opposite, so those errors propagate rather than silently serving stale pages +// in place of a snapshot that is really there. +func openWAL(dbPath string, pageSize int) (*os.File, *walIndex, error) { + f, err := os.Open(dbPath + "-wal") + if err != nil { + return nil, nil, nil + } + index, err := readWALIndex(f, pageSize) + if err != nil || index == nil { + f.Close() + return nil, nil, err + } + return f, index, nil +} diff --git a/wal_test.go b/wal_test.go new file mode 100644 index 0000000..e018a53 --- /dev/null +++ b/wal_test.go @@ -0,0 +1,334 @@ +// Copyright 2017 The go-sqlite Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "encoding/binary" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +// testdata/wal.sqlite is a database left in WAL mode by a still-open +// connection. Its main file was checkpointed after the first row only, so +// reading it alone yields one row and user_version 7; the log carries two more +// rows and user_version 16. +// +// Do not open the fixture with the sqlite3 CLI or any writing client: that +// checkpoints the log into the main file and empties it, which is precisely +// the state the fixture exists to not be in. TestOpenWithoutWAL is what +// catches it, by way of reporting three rows where it wanted one. +const ( + walFixture = "testdata/wal.sqlite" + walStaleUserVersion = 7 + walUserVersion = 16 +) + +var ( + // walStaleRows is what the main file holds on its own, walRows what the + // log adds to it. + walStaleRows = []string{"checkpointed"} + walRows = []string{"checkpointed", "in-wal-a", "in-wal-b"} +) + +func rowsInTbl1(t *testing.T, db *DbFile) []string { + t.Helper() + var got []string + err := db.VisitTableRecords("tbl1", func(_ *int64, rec Record) error { + got = append(got, rec.Values[0].(string)) + return nil + }) + if err != nil { + t.Fatalf("visiting tbl1: %v", err) + } + return got +} + +// copyDB copies the fixture database into dir, taking the log with it only when +// withWAL is set, and returns the path of the copy. +func copyDB(t *testing.T, dir string, withWAL bool) string { + t.Helper() + dst := filepath.Join(dir, "wal.sqlite") + copyFile(t, walFixture, dst) + if withWAL { + copyFile(t, walFixture+"-wal", dst+"-wal") + } + return dst +} + +func copyFile(t *testing.T, src, dst string) { + t.Helper() + b, err := ioutil.ReadFile(src) + if err != nil { + t.Fatalf("reading %s: %v", src, err) + } + if err := ioutil.WriteFile(dst, b, 0644); err != nil { + t.Fatalf("writing %s: %v", dst, err) + } +} + +func TestOpenWithWAL(t *testing.T) { + db, err := Open(walFixture) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walRows) { + t.Errorf("rows = %q, want %q; the write-ahead log was not applied", got, walRows) + } + // The log also carries a newer page 1, so the header must come from it. + if got := db.UserVersion(); got != walUserVersion { + t.Errorf("UserVersion() = %d, want %d", got, walUserVersion) + } +} + +// Without the log beside it the same main file is a valid, older database. +func TestOpenWithoutWAL(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + db, err := Open(copyDB(t, dir, false)) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { + t.Errorf("rows = %q, want %q", got, walStaleRows) + } + if got := db.UserVersion(); got != walStaleUserVersion { + t.Errorf("UserVersion() = %d, want %d", got, walStaleUserVersion) + } +} + +// A reader without a path to work from cannot find the log, and must still read +// the main file rather than fail. +func TestOpenFromUnnamedIgnoresWAL(t *testing.T) { + f, err := os.Open(walFixture) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + db, err := OpenFrom(struct{ readSeeker }{f}) + if err != nil { + t.Fatalf("OpenFrom: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { + t.Errorf("rows = %q, want %q", got, walStaleRows) + } +} + +// readSeeker hides *os.File's Name method so OpenFrom sees an anonymous reader. +type readSeeker interface { + Read([]byte) (int, error) + Seek(int64, int) (int64, error) +} + +// A torn frame can appear at the end of any log a writer is still appending +// to, and must be dropped without taking the committed snapshot with it. +func TestWALStopsAtTornTail(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := copyDB(t, dir, true) + header, err := ioutil.ReadFile(path + "-wal") + if err != nil { + t.Fatal(err) + } + log, err := os.OpenFile(path+"-wal", os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + t.Fatal(err) + } + // Right size, right salt, so only the checksum can reject it. A frame + // carrying the wrong salt would be turned away one check earlier. + torn := make([]byte, walFrameHeaderSize+1024) + copy(torn[8:16], header[16:24]) + if _, err := log.Write(torn); err != nil { + t.Fatal(err) + } + log.Close() + + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walRows) { + t.Errorf("rows = %q, want %q", got, walRows) + } +} + +// A log whose header does not checksum is one SQLite would ignore — typically a +// log reset by a checkpoint — and it must not make the database unreadable. +func TestWALWithBadHeaderIgnored(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := copyDB(t, dir, true) + log, err := os.OpenFile(path+"-wal", os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + // Corrupt the salt, which the header checksum covers. + if _, err := log.WriteAt([]byte{0xff, 0xff, 0xff, 0xff}, 16); err != nil { + t.Fatal(err) + } + log.Close() + + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { + t.Errorf("rows = %q, want %q", got, walStaleRows) + } +} + +// openWAL treats a log it cannot read as absent, so the main file still reads. +func TestWALUnreadableIgnored(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root reads the log regardless of its mode") + } + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := copyDB(t, dir, true) + if err := os.Chmod(path+"-wal", 0); err != nil { + t.Fatal(err) + } + + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { + t.Errorf("rows = %q, want %q", got, walStaleRows) + } +} + +// SQLite writes a log's checksums in the byte order of the machine that +// created it, so a log from a big-endian host uses the other magic number and +// the other word order. There is no fixture from such a host to hand, so this +// rewrites the little-endian one into that form, computing the checksums here +// rather than through walChecksum so that transposing the two magic constants +// fails the test instead of cancelling out. +func TestWALBigEndianChecksums(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := copyDB(t, dir, true) + log, err := ioutil.ReadFile(path + "-wal") + if err != nil { + t.Fatal(err) + } + + sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { + for i := 0; i+8 <= len(b); i += 8 { + s0 += binary.BigEndian.Uint32(b[i:i+4]) + s1 + s1 += binary.BigEndian.Uint32(b[i+4:i+8]) + s0 + } + return s0, s1 + } + + binary.BigEndian.PutUint32(log[0:4], walMagicBigEndian) + s0, s1 := sum(0, 0, log[0:24]) + binary.BigEndian.PutUint32(log[24:28], s0) + binary.BigEndian.PutUint32(log[28:32], s1) + + frame := walFrameHeaderSize + 1024 + for off := walHeaderSize; off+frame <= len(log); off += frame { + f := log[off : off+frame] + s0, s1 = sum(s0, s1, f[0:8]) + s0, s1 = sum(s0, s1, f[walFrameHeaderSize:]) + binary.BigEndian.PutUint32(f[16:20], s0) + binary.BigEndian.PutUint32(f[20:24], s1) + } + if err := ioutil.WriteFile(path+"-wal", log, 0644); err != nil { + t.Fatal(err) + } + + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walRows) { + t.Errorf("rows = %q, want %q", got, walRows) + } +} + +// A log can shrink the database as well as grow it, so the page count has to +// come from the commit frame rather than the main file's header. In +// testdata/wal-shrink.sqlite an auto-vacuuming database dropped 60 rows in the +// logged transaction, taking it from 34 pages to 4. +func TestWALShrinksDatabase(t *testing.T) { + db, err := Open("testdata/wal-shrink.sqlite") + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got, want := db.NumPage(), 4; got != want { + t.Errorf("NumPage() = %d, want %d", got, want) + } + if _, err := db.pager.Page(5); err == nil { + t.Error("Page(5) succeeded past the end of the shrunk database") + } + want := []string{"checkpointed", "in-wal-a"} + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, want) { + t.Errorf("rows = %q, want %q", got, want) + } +} + +// An empty -wal file is what a freshly checkpointed database leaves behind. +func TestWALEmptyIgnored(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := copyDB(t, dir, false) + if err := ioutil.WriteFile(path+"-wal", nil, 0644); err != nil { + t.Fatal(err) + } + + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { + t.Errorf("rows = %q, want %q", got, walStaleRows) + } +} From b0b84f8067b6da9b28f63f2b29cfe7e9d5251917 Mon Sep 17 00:00:00 2001 From: Filipe Guerreiro Date: Wed, 29 Jul 2026 07:46:15 +0900 Subject: [PATCH 2/5] Fold the WAL file handle into walIndex The pager held the index and the file it points into as two fields set together, cleared together, and with one guarded on the other to be dereferenced. Nothing enforced that pairing. The file now lives on walIndex, which drops a return value from openWAL and a branch from pager.Delete. Cleanup on a failed attachWAL moves to the call site, matching the init path directly below it, and DbFile.Close now returns the error from closing the log instead of discarding it. In the tests, copyDB owns the temporary directory it copies into and hands back a cleanup func, and assertRows replaces the open-and-compare block that five of them repeated verbatim. --- file.go | 17 +++---- pager.go | 26 +++++------ wal.go | 15 ++++-- wal_test.go | 130 ++++++++++++++++------------------------------------ 4 files changed, 70 insertions(+), 118 deletions(-) diff --git a/file.go b/file.go index 8d3e9c2..5c0e4bb 100644 --- a/file.go +++ b/file.go @@ -120,6 +120,7 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) { if named, ok := f.(interface{ Name() string }); ok { if err := db.attachWAL(named.Name()); err != nil { + db.pager.Delete() return nil, err } } @@ -135,26 +136,24 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) { // attachWAL overlays the write-ahead log beside the database at dbPath, if one // holds a committed snapshot. Both the page count and the header itself can be -// superseded by the log, so they are re-read from it. +// superseded by the log, so they are re-read from it. On error the log is left +// attached for the caller to close through the pager. func (db *DbFile) attachWAL(dbPath string) error { - walFile, index, err := openWAL(dbPath, db.PageSize()) + index, err := openWAL(dbPath, db.PageSize()) if err != nil || index == nil { return err } db.pager.wal = index - db.pager.walFile = walFile db.pager.npages = index.dbSize if _, ok := index.offsets[1]; ok { page, err := db.pager.Page(1) if err != nil { - db.pager.Delete() return err } dec := binary.NewDecoder(bytes.NewReader(page.buf)) dec.Order = binary.BigEndian if err := dec.Decode(&db.header); err != nil { - db.pager.Delete() return err } } @@ -180,11 +179,13 @@ func Open(fname string) (*DbFile, error) { } func (db *DbFile) Close() error { - db.pager.Delete() + err := db.pager.Delete() if db.close != nil { - return db.close() + if errDB := db.close(); errDB != nil { + return errDB + } } - return nil + return err } // PageSize returns the database page size in bytes diff --git a/pager.go b/pager.go index 203afea..6543cdf 100644 --- a/pager.go +++ b/pager.go @@ -7,17 +7,15 @@ package sqlite3 import ( "fmt" "io" - "os" ) type pager struct { - f io.ReadSeeker - size int // page size in bytes - npages int // total number of pages in db - pages map[int]page // cache of pages - lru []int // list of last used pages - wal *walIndex // committed pages living in the write-ahead log, if any - walFile *os.File // the -wal file backing wal + f io.ReadSeeker + size int // page size in bytes + npages int // total number of pages in db + pages map[int]page // cache of pages + lru []int // list of last used pages + wal *walIndex // committed pages living in the write-ahead log, if any } func newPager(f io.ReadSeeker, size, npages int) pager { @@ -33,10 +31,9 @@ func newPager(f io.ReadSeeker, size, npages int) pager { } func (p *pager) Page(i int) (page, error) { - var err error page, ok := p.pages[i] if ok { - return page, err + return page, nil } if i > p.npages { @@ -53,7 +50,7 @@ func (p *pager) Page(i int) (page, error) { p.pages[i] = page p.lru = append(p.lru, i) - return page, err + return page, nil } // read fills buf with page i, preferring the write-ahead log's image of it @@ -61,7 +58,7 @@ func (p *pager) Page(i int) (page, error) { func (p *pager) read(i int, buf []byte) error { if p.wal != nil { if off, ok := p.wal.offsets[i]; ok { - _, err := p.walFile.ReadAt(buf, off) + _, err := p.wal.f.ReadAt(buf, off) return err } } @@ -86,9 +83,8 @@ func (p *pager) Delete() error { var err error p.pages = nil p.lru = nil - if p.walFile != nil { - err = p.walFile.Close() - p.walFile = nil + if p.wal != nil { + err = p.wal.f.Close() p.wal = nil } return err diff --git a/wal.go b/wal.go index 04484b8..ad0fc12 100644 --- a/wal.go +++ b/wal.go @@ -30,6 +30,9 @@ const ( // walIndex locates the most recent committed image of each page in a WAL. type walIndex struct { + // f is the log the offsets point into. readWALIndex leaves it nil, since + // it indexes any reader; openWAL sets it to the file it opened. + f *os.File // offsets maps a page number to the offset of its page data in the WAL. offsets map[int]int64 // dbSize is the size of the database in pages after the last commit frame, @@ -126,22 +129,24 @@ func walChecksum(bigEndian bool, s0, s1 uint32, b []byte) (uint32, uint32) { } // openWAL indexes the write-ahead log next to the database at dbPath, -// returning a nil file and index when there is no snapshot to read. +// returning a nil index when there is no snapshot to read. The caller owns the +// returned index's file and must close it. // // A log that cannot be opened at all is treated as absent, since a permission // denial or a sharing violation on it says nothing about the main file, which // stays perfectly readable. An I/O fault on a log already open says the // opposite, so those errors propagate rather than silently serving stale pages // in place of a snapshot that is really there. -func openWAL(dbPath string, pageSize int) (*os.File, *walIndex, error) { +func openWAL(dbPath string, pageSize int) (*walIndex, error) { f, err := os.Open(dbPath + "-wal") if err != nil { - return nil, nil, nil + return nil, nil } index, err := readWALIndex(f, pageSize) if err != nil || index == nil { f.Close() - return nil, nil, err + return nil, err } - return f, index, nil + index.f = f + return index, nil } diff --git a/wal_test.go b/wal_test.go index e018a53..05b76fb 100644 --- a/wal_test.go +++ b/wal_test.go @@ -48,16 +48,34 @@ func rowsInTbl1(t *testing.T, db *DbFile) []string { return got } -// copyDB copies the fixture database into dir, taking the log with it only when -// withWAL is set, and returns the path of the copy. -func copyDB(t *testing.T, dir string, withWAL bool) string { +// copyDB copies the fixture database into a temporary directory, taking the +// log with it only when withWAL is set. Tests mutate the copy, never the +// fixture. The returned func removes the directory. +func copyDB(t *testing.T, withWAL bool) (string, func()) { t.Helper() + dir, err := ioutil.TempDir("", "sqlite3-wal") + if err != nil { + t.Fatal(err) + } dst := filepath.Join(dir, "wal.sqlite") copyFile(t, walFixture, dst) if withWAL { copyFile(t, walFixture+"-wal", dst+"-wal") } - return dst + return dst, func() { os.RemoveAll(dir) } +} + +// assertRows opens path and checks tbl1 holds exactly want. +func assertRows(t *testing.T, path string, want []string) { + t.Helper() + db, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, want) { + t.Errorf("rows = %q, want %q", got, want) + } } func copyFile(t *testing.T, src, dst string) { @@ -89,13 +107,10 @@ func TestOpenWithWAL(t *testing.T) { // Without the log beside it the same main file is a valid, older database. func TestOpenWithoutWAL(t *testing.T) { - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) + path, cleanup := copyDB(t, false) + defer cleanup() - db, err := Open(copyDB(t, dir, false)) + db, err := Open(path) if err != nil { t.Fatalf("Open: %v", err) } @@ -138,13 +153,8 @@ type readSeeker interface { // A torn frame can appear at the end of any log a writer is still appending // to, and must be dropped without taking the committed snapshot with it. func TestWALStopsAtTornTail(t *testing.T) { - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - path := copyDB(t, dir, true) + path, cleanup := copyDB(t, true) + defer cleanup() header, err := ioutil.ReadFile(path + "-wal") if err != nil { t.Fatal(err) @@ -162,27 +172,14 @@ func TestWALStopsAtTornTail(t *testing.T) { } log.Close() - db, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer db.Close() - - if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walRows) { - t.Errorf("rows = %q, want %q", got, walRows) - } + assertRows(t, path, walRows) } // A log whose header does not checksum is one SQLite would ignore — typically a // log reset by a checkpoint — and it must not make the database unreadable. func TestWALWithBadHeaderIgnored(t *testing.T) { - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - path := copyDB(t, dir, true) + path, cleanup := copyDB(t, true) + defer cleanup() log, err := os.OpenFile(path+"-wal", os.O_WRONLY, 0644) if err != nil { t.Fatal(err) @@ -193,15 +190,7 @@ func TestWALWithBadHeaderIgnored(t *testing.T) { } log.Close() - db, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer db.Close() - - if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { - t.Errorf("rows = %q, want %q", got, walStaleRows) - } + assertRows(t, path, walStaleRows) } // openWAL treats a log it cannot read as absent, so the main file still reads. @@ -209,26 +198,13 @@ func TestWALUnreadableIgnored(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads the log regardless of its mode") } - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - path := copyDB(t, dir, true) + path, cleanup := copyDB(t, true) + defer cleanup() if err := os.Chmod(path+"-wal", 0); err != nil { t.Fatal(err) } - db, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer db.Close() - - if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { - t.Errorf("rows = %q, want %q", got, walStaleRows) - } + assertRows(t, path, walStaleRows) } // SQLite writes a log's checksums in the byte order of the machine that @@ -238,13 +214,8 @@ func TestWALUnreadableIgnored(t *testing.T) { // rather than through walChecksum so that transposing the two magic constants // fails the test instead of cancelling out. func TestWALBigEndianChecksums(t *testing.T) { - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - path := copyDB(t, dir, true) + path, cleanup := copyDB(t, true) + defer cleanup() log, err := ioutil.ReadFile(path + "-wal") if err != nil { t.Fatal(err) @@ -275,15 +246,7 @@ func TestWALBigEndianChecksums(t *testing.T) { t.Fatal(err) } - db, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer db.Close() - - if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walRows) { - t.Errorf("rows = %q, want %q", got, walRows) - } + assertRows(t, path, walRows) } // A log can shrink the database as well as grow it, so the page count has to @@ -311,24 +274,11 @@ func TestWALShrinksDatabase(t *testing.T) { // An empty -wal file is what a freshly checkpointed database leaves behind. func TestWALEmptyIgnored(t *testing.T) { - dir, err := ioutil.TempDir("", "sqlite3-wal") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - path := copyDB(t, dir, false) + path, cleanup := copyDB(t, false) + defer cleanup() if err := ioutil.WriteFile(path+"-wal", nil, 0644); err != nil { t.Fatal(err) } - db, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer db.Close() - - if got := rowsInTbl1(t, db); !reflect.DeepEqual(got, walStaleRows) { - t.Errorf("rows = %q, want %q", got, walStaleRows) - } + assertRows(t, path, walStaleRows) } From 6a59ca1414f0218ecbaeab2b9c931f0d39990141 Mon Sep 17 00:00:00 2001 From: Filipe Guerreiro Date: Wed, 29 Jul 2026 08:54:14 +0900 Subject: [PATCH 3/5] Validate the log before trusting its pages Open panics on a log whose page 1 is zeroed. The magic check in OpenFrom runs against the main file's page 1, never against the log's replacement for it, so a page size of zero reaches the pager and page.Kind() indexes an empty buffer. Re-check the magic and the page size after the decode. Three frame fields went unvalidated: - a page number of zero, which wal.c rejects outright - a format version other than 3007000, which would parse a future format with today's frame rules - a commit page count above MaxInt32, which became a negative NumPage A checkpoint can also restart the log between the scan and a later page read, leaving an indexed offset pointing into another generation's frame. Re-read the frame header on each page and check that the page number and salt still match, so that errors instead of returning a wrong page. Structural, no behaviour change: build the index before the pager so the page count derives once, put the offsets and the file handle behind walIndex methods, and pass the checksum word order as a binary.ByteOrder. Each rejection has a test through readWALIndex. The new wal-grow fixture covers a log that grows the database past the end of the main file, which is the ordinary state of a database with a live writer. --- file.go | 57 ++++++--- pager.go | 8 +- testdata/wal-grow.sqlite | Bin 0 -> 2048 bytes testdata/wal-grow.sqlite-wal | Bin 0 -> 6320 bytes wal.go | 96 ++++++++++---- wal_test.go | 241 +++++++++++++++++++++++++++++++++-- 6 files changed, 340 insertions(+), 62 deletions(-) create mode 100644 testdata/wal-grow.sqlite create mode 100644 testdata/wal-grow.sqlite-wal diff --git a/file.go b/file.go index 5c0e4bb..b20f890 100644 --- a/file.go +++ b/file.go @@ -116,14 +116,23 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) { ) } - db.pager = newPager(f, db.PageSize(), db.NumPage()) - + var wal *walIndex if named, ok := f.(interface{ Name() string }); ok { - if err := db.attachWAL(named.Name()); err != nil { - db.pager.Delete() + wal, err = openWAL(named.Name(), db.PageSize()) + if err != nil { return nil, err } } + if wal != nil { + // Before the pager, so that the page count it is built with is the + // log's rather than the main file's stale one. + if err := db.applyWALHeader(wal); err != nil { + wal.Close() + return nil, err + } + } + + db.pager = newPager(f, db.PageSize(), db.NumPage(), wal) err = db.init() if err != nil { @@ -134,32 +143,38 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) { return &db, err } -// attachWAL overlays the write-ahead log beside the database at dbPath, if one -// holds a committed snapshot. Both the page count and the header itself can be -// superseded by the log, so they are re-read from it. On error the log is left -// attached for the caller to close through the pager. -func (db *DbFile) attachWAL(dbPath string) error { - index, err := openWAL(dbPath, db.PageSize()) - if err != nil || index == nil { +// applyWALHeader supersedes the header with the log's newer image of page 1, +// when the log carries one, and takes the page count from its last commit +// frame either way. +func (db *DbFile) applyWALHeader(wal *walIndex) error { + pageSize := db.PageSize() + buf := make([]byte, pageSize) + ok, err := wal.page(1, buf) + if err != nil { return err } - db.pager.wal = index - db.pager.npages = index.dbSize - - if _, ok := index.offsets[1]; ok { - page, err := db.pager.Page(1) - if err != nil { - return err - } - dec := binary.NewDecoder(bytes.NewReader(page.buf)) + if ok { + dec := binary.NewDecoder(bytes.NewReader(buf)) dec.Order = binary.BigEndian if err := dec.Decode(&db.header); err != nil { return err } + // The magic check above ran against the main file's page 1, so the + // log's replacement for it has to clear the same bar. A page size of + // zero would otherwise reach the pager and read empty pages forever. + if string(db.header.Magic[:]) != sqlite3Magic { + return fmt.Errorf("sqlite3: invalid file header in write-ahead log") + } + if db.PageSize() != pageSize { + return fmt.Errorf( + "sqlite3: write-ahead log page 1 changes the page size (%d -> %d)", + pageSize, db.PageSize(), + ) + } } // Last, because the commit frame is what states the page count and the // decode above may have just put the main file's stale one back. - db.header.DbSize = int32(index.dbSize) + db.header.DbSize = int32(wal.dbSize) return nil } diff --git a/pager.go b/pager.go index 6543cdf..a0e0c53 100644 --- a/pager.go +++ b/pager.go @@ -18,13 +18,14 @@ type pager struct { wal *walIndex // committed pages living in the write-ahead log, if any } -func newPager(f io.ReadSeeker, size, npages int) pager { +func newPager(f io.ReadSeeker, size, npages int, wal *walIndex) pager { pager := pager{ f: f, size: size, npages: npages, pages: make(map[int]page, npages), lru: make([]int, 0, 2), + wal: wal, } return pager @@ -57,8 +58,7 @@ func (p *pager) Page(i int) (page, error) { // over the one in the main database file. func (p *pager) read(i int, buf []byte) error { if p.wal != nil { - if off, ok := p.wal.offsets[i]; ok { - _, err := p.wal.f.ReadAt(buf, off) + if ok, err := p.wal.page(i, buf); ok || err != nil { return err } } @@ -84,7 +84,7 @@ func (p *pager) Delete() error { p.pages = nil p.lru = nil if p.wal != nil { - err = p.wal.f.Close() + err = p.wal.Close() p.wal = nil } return err diff --git a/testdata/wal-grow.sqlite b/testdata/wal-grow.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..b3f8f16d2617a6c73ceaf81dde4596bf85d00912 GIT binary patch literal 2048 zcmWFz^vNtqRY=P(%1ta$FlJz3U}EBNP*7lCU|@n`AO!}DK#~Q@22mhBI}#rolZipE z_XjUXG4oLdAQ}auAut*O)D8h%MmBL#QAYcc#H5_mlB66%5N2`?a&-)GRS0o(@^MuF ziD(ongt$h8Xd+t=$(v6Yn4chvj3$g6oB77w+nMFEo<4Ap$N1<;q421i^HP009646WGRBQP_cj@OU9saP27nl z8a0}@fQmar6O9m%C{cq(qb4r6Z$V>%QKM(TcMdf1m;N&I(n-JXy-w%r&D_)WMO(|R zvg$tVWmzQ_9k9JqJZRYK>;5=a-u2NZ=03NM&Z)wM^!&F6TXv0EyD~d_=8P3>P1$9u z*R(8Z%X;)|fR2;0S^83-M=3qhqwxIW-~8_r1usu8{=;XmM%{F*f-Wk(&!UgwQCxxI z3jEhAP^ZdzR#mBqZHpE+H?=Kpb_x%eJS$hy;DD+x9b~qUa!?F^n5*2=X9d$^dMcOGw=r-h0o!A zcmrO6XWmdrlY0&?YHLKUJs*4<)%53A5l*0sb z%rTzM9N}c<5F3~StY`Ld60?UBnO&T~?BIB28^@&_Bsi8i#xcwhCd?tm%mGHsK8DO5 z2Fxz{%no|YHo7VM2|CO%+RPD-W)5)_bAWZsK8|Gea0Ii9!D%rOpS zj&KNbh^H|JIGEYTYGw}yF}s*$c5ooGjRR746YS3%V?X8y`!a{vhdIFB%s%#F_OK_j zi#?bftYWsYd&*9N-I!zS${b-A<`6qG2iS?($4X`oE0|sE$n2nDwlfHQPrT2QOeV!R z(;>wuQ=Vd&DN8ZPl&0usN>cPPIz=}FDLR!3hAdg$v}9%L>J_WnnwGYk3&;NlNasSs zms6g3a@%8#g}E@OW+uM z2Oq;>H~=rglkgyHhwEW8w89cNpQge%7!B1>1s&9H>Ie0SI;{4qJv0;UQ@5#W)CSe8 z7OGh)ry`mNeN~0^r*+i&jOM|9Yq#|nWt@7Lyn%ry7R~WulSi@0m&CK1nEDdL)qB97T0 zVw)>ioW!gbiOo8Z$gC9!O`AwyE*J658WGQ2CgPe_5yz|+vCS$LJ25RHv1t~G%u12a zTq+Wn6(YV_F5;O>L|n5>#4$}GHZD!KQIg;i<`@?0%i~AGrM>Z zvx65h+c+=fIKjEhG0tI*@B-!#&u0$sJZ2vonLV7%?BXnD2WK+dcy7v3g6A;Dcs6r{ zXEBF3gE_!6nSDHi*~96~?x~yX_`QPCbS|{!YB!E-`Jj7YE=;g)w)7YJJ$+EWs2``P zaI4;?*XiYY9?gUa+Sb*&yDp=N@C|$jhu~#+isr$ca4l?vW>^5{LIe0P6nX)w-)S0r ztln08)pP1$b+@`vZBeaiv1(LPRIG-peyWmgOZ;RVvEH=~SbMA;*1h!NCmkj~J$*;G zHf_ymsW}O4&8cFvHK&Ntn8=CIm^fXG#>8YX8WRm-G$!iBXiQ8JqcJg2j9MnJ#nduh zj9SKtQOj5{Y8fL&Er}Sl#A4JEiBU@^MlFFDwfJliwRmFG;)+p=BStN@7`2QRqn1%( z)KVu#EhELKWrP^D3}*|erB;kuYQ(5zm>9JT6{D6RV$^b)7_|%*qn2thY8fO(Em^jJ zS_X2|^<|QX>w*nUK6Y^ZR_=d5!Ji6eUcmL&m$Ik6uqrDdI TJGNhuq8C@-zg~fVao_PTk1m1B literal 0 HcmV?d00001 diff --git a/wal.go b/wal.go index ad0fc12..90be478 100644 --- a/wal.go +++ b/wal.go @@ -6,7 +6,9 @@ package sqlite3 import ( "encoding/binary" + "fmt" "io" + "math" "os" ) @@ -26,20 +28,51 @@ const ( // SQLITE_BIGENDIAN test. The format page's prose reads the other way round. walMagicLittleEndian = 0x377f0682 walMagicBigEndian = 0x377f0683 + + // The only format version wal.c writes or accepts. + walFormatVersion = 3007000 ) // walIndex locates the most recent committed image of each page in a WAL. type walIndex struct { - // f is the log the offsets point into. readWALIndex leaves it nil, since - // it indexes any reader; openWAL sets it to the file it opened. + // f is the log the offsets point into; nil until openWAL sets it + // (readWALIndex indexes any reader), so every index reaching a pager has one. f *os.File - // offsets maps a page number to the offset of its page data in the WAL. + // offsets maps a page number to the offset of its frame in the WAL. offsets map[int]int64 + // salt is the header's, repeated in every frame belonging to this + // generation of the log. + salt [8]byte // dbSize is the size of the database in pages after the last commit frame, // which supersedes the size recorded in the main file's header. dbSize int } +// page reads the log's image of page i into buf, reporting whether the log +// carries one. A page it does not carry is left to the main database file. +func (w *walIndex) page(i int, buf []byte) (bool, error) { + off, ok := w.offsets[i] + if !ok { + return false, nil + } + // A checkpoint can restart the log between the scan and this read, leaving + // the offset pointing into a later generation's frame; re-reading the frame + // header turns that into an error instead of a wrong page. + var header [walFrameHeaderSize]byte + if _, err := w.f.ReadAt(header[:], off); err != nil { + return true, err + } + if binary.BigEndian.Uint32(header[0:4]) != uint32(i) || string(header[8:16]) != string(w.salt[:]) { + return true, fmt.Errorf("sqlite3: write-ahead log changed while being read") + } + _, err := w.f.ReadAt(buf, off+walFrameHeaderSize) + return true, err +} + +func (w *walIndex) Close() error { + return w.f.Close() +} + // readWALIndex scans the WAL in f and indexes every page belonging to a // committed transaction, newest image winning. It returns a nil index for a // log SQLite would itself ignore, such as the stale bytes a checkpoint leaves @@ -53,27 +86,32 @@ func readWALIndex(f io.ReaderAt, pageSize int) (*walIndex, error) { return nil, err } - var bigEndian bool + var order binary.ByteOrder = binary.LittleEndian switch binary.BigEndian.Uint32(header[0:4]) { case walMagicLittleEndian: - bigEndian = false case walMagicBigEndian: - bigEndian = true + order = binary.BigEndian default: return nil, nil } + // A later format may keep the magic and reuse these fields differently. + if binary.BigEndian.Uint32(header[4:8]) != walFormatVersion { + return nil, nil + } // A torn or reset header means there is no snapshot to read. - s0, s1 := walChecksum(bigEndian, 0, 0, header[0:24]) + s0, s1 := walChecksum(order, 0, 0, header[0:24]) if s0 != binary.BigEndian.Uint32(header[24:28]) || s1 != binary.BigEndian.Uint32(header[28:32]) { return nil, nil } + // The pager reads fixed-size pages, so a log written at another page size + // is no more usable here than an absent one. if int(binary.BigEndian.Uint32(header[8:12])) != pageSize { return nil, nil } - salt := header[16:24] index := &walIndex{offsets: make(map[int]int64)} + copy(index.salt[:], header[16:24]) // Frames after the last commit frame belong to a transaction that was never // committed, so they are staged here and only merged when a commit is seen. pending := make(map[int]int64) @@ -88,18 +126,28 @@ func readWALIndex(f io.ReaderAt, pageSize int) (*walIndex, error) { } // A salt mismatch marks where a later checkpoint restarted the log and // left older frames behind. - if string(frame[8:16]) != string(salt) { + if string(frame[8:16]) != string(index.salt[:]) { break } - c0, c1 := walChecksum(bigEndian, s0, s1, frame[0:8]) - c0, c1 = walChecksum(bigEndian, c0, c1, frame[walFrameHeaderSize:]) + c0, c1 := walChecksum(order, s0, s1, frame[0:8]) + c0, c1 = walChecksum(order, c0, c1, frame[walFrameHeaderSize:]) if c0 != binary.BigEndian.Uint32(frame[16:20]) || c1 != binary.BigEndian.Uint32(frame[20:24]) { break } s0, s1 = c0, c1 - pending[int(binary.BigEndian.Uint32(frame[0:4]))] = offset + walFrameHeaderSize - if dbSize := binary.BigEndian.Uint32(frame[4:8]); dbSize != 0 { + // Page 0 does not exist; wal.c rejects such a frame outright. + pgno := binary.BigEndian.Uint32(frame[0:4]) + if pgno == 0 { + break + } + dbSize := binary.BigEndian.Uint32(frame[4:8]) + if dbSize > math.MaxInt32 { + break + } + + pending[int(pgno)] = offset + if dbSize != 0 { for page, at := range pending { index.offsets[page] = at } @@ -115,12 +163,8 @@ func readWALIndex(f io.ReaderAt, pageSize int) (*walIndex, error) { } // walChecksum continues SQLite's running WAL checksum over b, which must be a -// whole number of 8-byte blocks. -func walChecksum(bigEndian bool, s0, s1 uint32, b []byte) (uint32, uint32) { - order := binary.ByteOrder(binary.LittleEndian) - if bigEndian { - order = binary.BigEndian - } +// whole number of 8-byte blocks. The log records its own word order. +func walChecksum(order binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32) { for i := 0; i+8 <= len(b); i += 8 { s0 += order.Uint32(b[i:i+4]) + s1 s1 += order.Uint32(b[i+4:i+8]) + s0 @@ -128,15 +172,13 @@ func walChecksum(bigEndian bool, s0, s1 uint32, b []byte) (uint32, uint32) { return s0, s1 } -// openWAL indexes the write-ahead log next to the database at dbPath, -// returning a nil index when there is no snapshot to read. The caller owns the -// returned index's file and must close it. +// openWAL indexes the write-ahead log next to the database at dbPath, returning +// a nil index when there is no snapshot to read; the caller owns the returned +// index's file and must close it. // -// A log that cannot be opened at all is treated as absent, since a permission -// denial or a sharing violation on it says nothing about the main file, which -// stays perfectly readable. An I/O fault on a log already open says the -// opposite, so those errors propagate rather than silently serving stale pages -// in place of a snapshot that is really there. +// A log that cannot be opened is treated as absent (a permission or sharing +// error there says nothing about the main file), but an I/O fault on a log +// already open propagates rather than silently serving stale pages. func openWAL(dbPath string, pageSize int) (*walIndex, error) { f, err := os.Open(dbPath + "-wal") if err != nil { diff --git a/wal_test.go b/wal_test.go index 05b76fb..48a7e1c 100644 --- a/wal_test.go +++ b/wal_test.go @@ -5,6 +5,7 @@ package sqlite3 import ( + "bytes" "encoding/binary" "io/ioutil" "os" @@ -207,12 +208,10 @@ func TestWALUnreadableIgnored(t *testing.T) { assertRows(t, path, walStaleRows) } -// SQLite writes a log's checksums in the byte order of the machine that -// created it, so a log from a big-endian host uses the other magic number and -// the other word order. There is no fixture from such a host to hand, so this -// rewrites the little-endian one into that form, computing the checksums here -// rather than through walChecksum so that transposing the two magic constants -// fails the test instead of cancelling out. +// SQLite writes a log's checksums in the byte order of the machine that created +// it; there is no big-endian fixture, so this flips a little-endian one and +// recomputes the checksums by hand rather than via walChecksum, so that +// transposing the two magic constants fails instead of cancelling out. func TestWALBigEndianChecksums(t *testing.T) { path, cleanup := copyDB(t, true) defer cleanup() @@ -249,10 +248,8 @@ func TestWALBigEndianChecksums(t *testing.T) { assertRows(t, path, walRows) } -// A log can shrink the database as well as grow it, so the page count has to -// come from the commit frame rather than the main file's header. In -// testdata/wal-shrink.sqlite an auto-vacuuming database dropped 60 rows in the -// logged transaction, taking it from 34 pages to 4. +// testdata/wal-shrink.sqlite: an auto-vacuuming transaction drops 60 rows and +// takes the database from 34 pages to 4. The shrink case for walIndex.dbSize. func TestWALShrinksDatabase(t *testing.T) { db, err := Open("testdata/wal-shrink.sqlite") if err != nil { @@ -272,6 +269,230 @@ func TestWALShrinksDatabase(t *testing.T) { } } +// testdata/wal-grow.sqlite is 2 pages on disk; the logged transaction inserts +// 200 rows and commits at 5. The grow case for walIndex.dbSize, and the +// ordinary state of a database belonging to a running application. +func TestWALGrowsDatabase(t *testing.T) { + db, err := Open("testdata/wal-grow.sqlite") + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got, want := db.NumPage(), 5; got != want { + t.Errorf("NumPage() = %d, want %d", got, want) + } + // Page 5 lives only in the log; the main file stops after page 2. + if _, err := db.pager.Page(5); err != nil { + t.Errorf("Page(5): %v", err) + } + if got, want := len(rowsInTbl1(t, db)), 201; got != want { + t.Errorf("len(rows) = %d, want %d", got, want) + } +} + +// A log restarted under a reader must fail the read, rather than serve a frame +// from the new generation as the page it used to be. +func TestWALDetectsRestartUnderReader(t *testing.T) { + path, cleanup := copyDB(t, true) + defer cleanup() + + index, err := openWAL(path, 1024) + if err != nil { + t.Fatalf("openWAL: %v", err) + } + if index == nil { + t.Fatal("openWAL returned no index for the fixture log") + } + defer index.Close() + + buf := make([]byte, 1024) + if ok, err := index.page(1, buf); !ok || err != nil { + t.Fatalf("page(1) = %v, %v; want true, nil", ok, err) + } + + // Rewrite the salt of the frame the index points at, as a restarted log + // would. The index still holds the old salt. + log, err := os.OpenFile(path+"-wal", os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + if _, err := log.WriteAt([]byte{0, 0, 0, 0, 0, 0, 0, 0}, index.offsets[1]+8); err != nil { + t.Fatal(err) + } + log.Close() + + if ok, err := index.page(1, buf); !ok || err == nil { + t.Errorf("page(1) = %v, %v; want true and an error", ok, err) + } +} + +// A zeroed page 1 in the log must fail Open, rather than reach the pager as a +// page size of zero. +func TestWALRejectsBadHeaderPage(t *testing.T) { + path, cleanup := copyDB(t, true) + defer cleanup() + log, err := ioutil.ReadFile(path + "-wal") + if err != nil { + t.Fatal(err) + } + + // The fixture's last frame carries page 1. Blanking its payload keeps the + // frame well-formed once the checksums are recomputed over it. + last := walHeaderSize + for last+2*(walFrameHeaderSize+1024) <= len(log) { + last += walFrameHeaderSize + 1024 + } + if got := binary.BigEndian.Uint32(log[last : last+4]); got != 1 { + t.Fatalf("last frame carries page %d, want page 1", got) + } + payload := log[last+walFrameHeaderSize:] + for i := range payload { + payload[i] = 0 + } + resealWAL(t, log, 1024) + if err := ioutil.WriteFile(path+"-wal", log, 0644); err != nil { + t.Fatal(err) + } + + db, err := Open(path) + if err == nil { + db.Close() + t.Fatal("Open accepted a log whose page 1 is not a database header") + } +} + +// resealWAL recomputes the little-endian checksum chain over log in place, so +// a test can edit frame payloads and still hand back a log that verifies. +func resealWAL(t *testing.T, log []byte, pageSize int) { + t.Helper() + sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { + for i := 0; i+8 <= len(b); i += 8 { + s0 += binary.LittleEndian.Uint32(b[i:i+4]) + s1 + s1 += binary.LittleEndian.Uint32(b[i+4:i+8]) + s0 + } + return s0, s1 + } + s0, s1 := sum(0, 0, log[0:24]) + binary.BigEndian.PutUint32(log[24:28], s0) + binary.BigEndian.PutUint32(log[28:32], s1) + frame := walFrameHeaderSize + pageSize + for off := walHeaderSize; off+frame <= len(log); off += frame { + f := log[off : off+frame] + s0, s1 = sum(s0, s1, f[0:8]) + s0, s1 = sum(s0, s1, f[walFrameHeaderSize:]) + binary.BigEndian.PutUint32(f[16:20], s0) + binary.BigEndian.PutUint32(f[20:24], s1) + } +} + +// walFrame is one frame to assemble into a synthetic log. A dbSize of zero +// makes it a non-commit frame. +type walFrame struct { + pgno uint32 + dbSize uint32 +} + +// buildWAL assembles a little-endian write-ahead log. The checksums are +// computed here rather than through walChecksum, so that a bug in the latter +// cannot cancel itself out. magic and version are parameters because rejecting +// the wrong ones is most of what these tests check. +func buildWAL(magic, version uint32, pageSize int, frames []walFrame) []byte { + sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { + for i := 0; i+8 <= len(b); i += 8 { + s0 += binary.LittleEndian.Uint32(b[i:i+4]) + s1 + s1 += binary.LittleEndian.Uint32(b[i+4:i+8]) + s0 + } + return s0, s1 + } + + log := make([]byte, walHeaderSize) + binary.BigEndian.PutUint32(log[0:4], magic) + binary.BigEndian.PutUint32(log[4:8], version) + binary.BigEndian.PutUint32(log[8:12], uint32(pageSize)) + copy(log[16:24], []byte("saltsalt")) + s0, s1 := sum(0, 0, log[0:24]) + binary.BigEndian.PutUint32(log[24:28], s0) + binary.BigEndian.PutUint32(log[28:32], s1) + + for _, f := range frames { + frame := make([]byte, walFrameHeaderSize+pageSize) + binary.BigEndian.PutUint32(frame[0:4], f.pgno) + binary.BigEndian.PutUint32(frame[4:8], f.dbSize) + copy(frame[8:16], log[16:24]) + s0, s1 = sum(s0, s1, frame[0:8]) + s0, s1 = sum(s0, s1, frame[walFrameHeaderSize:]) + binary.BigEndian.PutUint32(frame[16:20], s0) + binary.BigEndian.PutUint32(frame[20:24], s1) + log = append(log, frame...) + } + return log +} + +// Logs SQLite would refuse, or this package cannot apply. Each must read as +// "no snapshot here" rather than as an error, so the main file still opens. +func TestReadWALIndexRejects(t *testing.T) { + const pageSize = 1024 + commit := []walFrame{{pgno: 1, dbSize: 1}} + + // Frames laid out at the size readWALIndex is called with, but a header + // declaring another. Everything else about the log verifies, so only the + // page-size check can turn it away. + mislabelled := buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, commit) + binary.BigEndian.PutUint32(mislabelled[8:12], 512) + resealWAL(t, mislabelled, pageSize) + + tests := []struct { + name string + log []byte + }{ + {"wrong magic", buildWAL(0xdeadbeef, walFormatVersion, pageSize, commit)}, + {"future format version", buildWAL(walMagicLittleEndian, walFormatVersion+1, pageSize, commit)}, + {"other page size", mislabelled}, + {"page zero", buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, []walFrame{{pgno: 0, dbSize: 1}})}, + {"page count overflows int32", buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, []walFrame{{pgno: 1, dbSize: 1 << 31}})}, + {"no commit frame", buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, []walFrame{{pgno: 1}})}, + {"header only", buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, nil)}, + } + for _, tt := range tests { + index, err := readWALIndex(bytes.NewReader(tt.log), pageSize) + if err != nil { + t.Errorf("%s: readWALIndex: %v", tt.name, err) + continue + } + if index != nil { + t.Errorf("%s: indexed %d pages, want no index", tt.name, len(index.offsets)) + } + } +} + +// The positive control for the table above, and for the rule that only frames +// up to the last commit frame count. +func TestReadWALIndexAppliesCommittedFrames(t *testing.T) { + const pageSize = 1024 + log := buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, []walFrame{ + {pgno: 1}, + {pgno: 2, dbSize: 2}, + {pgno: 3}, // after the commit, so uncommitted + }) + + index, err := readWALIndex(bytes.NewReader(log), pageSize) + if err != nil { + t.Fatalf("readWALIndex: %v", err) + } + if index == nil { + t.Fatal("readWALIndex returned no index for a committed log") + } + frame := int64(walFrameHeaderSize + pageSize) + want := map[int]int64{1: walHeaderSize, 2: walHeaderSize + frame} + if !reflect.DeepEqual(index.offsets, want) { + t.Errorf("offsets = %v, want %v", index.offsets, want) + } + if index.dbSize != 2 { + t.Errorf("dbSize = %d, want 2", index.dbSize) + } +} + // An empty -wal file is what a freshly checkpointed database leaves behind. func TestWALEmptyIgnored(t *testing.T) { path, cleanup := copyDB(t, false) From f6e8204a67ffecc604576497dbebc6a7769d2b8e Mon Sep 17 00:00:00 2001 From: Filipe Guerreiro Date: Wed, 29 Jul 2026 09:07:41 +0900 Subject: [PATCH 4/5] Stop a hostile log from sizing the page cache Reading the page count from the log before building the pager, as the previous commit does, also made it attacker-controlled at the point of allocation: newPager hints its cache map with npages, so a 1KB crafted -wal committing at MaxInt32 pages reaches make(map[int]page, 2147483647). Open thrashes for 19s and is killed. Drop the hint. The cache only ever holds the pages actually read, and the main file's header could already reach the same call. Tests: page 1 with the wrong magic, and page 1 changing the page size, were each masked by whichever check fired first, and the log's fall-through to the main file was never exercised at all, since every fixture happens to log every page it commits. The three copies of the checksum chain collapse to one helper taking the word order, which keeps the property that a bug in walChecksum cannot cancel itself out. --- file.go | 4 +- pager.go | 8 ++- wal.go | 5 +- wal_test.go | 195 ++++++++++++++++++++++++++++++++++------------------ 4 files changed, 138 insertions(+), 74 deletions(-) diff --git a/file.go b/file.go index b20f890..264c99b 100644 --- a/file.go +++ b/file.go @@ -172,8 +172,8 @@ func (db *DbFile) applyWALHeader(wal *walIndex) error { ) } } - // Last, because the commit frame is what states the page count and the - // decode above may have just put the main file's stale one back. + // Last: the decode above may have just put the main file's stale page + // count back. db.header.DbSize = int32(wal.dbSize) return nil } diff --git a/pager.go b/pager.go index a0e0c53..c23d201 100644 --- a/pager.go +++ b/pager.go @@ -23,9 +23,11 @@ func newPager(f io.ReadSeeker, size, npages int, wal *walIndex) pager { f: f, size: size, npages: npages, - pages: make(map[int]page, npages), - lru: make([]int, 0, 2), - wal: wal, + // No size hint: npages comes from a file header, and a hostile one can + // ask for MaxInt32 pages. The cache only ever holds pages read. + pages: make(map[int]page), + lru: make([]int, 0, 2), + wal: wal, } return pager diff --git a/wal.go b/wal.go index 90be478..d1748a8 100644 --- a/wal.go +++ b/wal.go @@ -56,8 +56,9 @@ func (w *walIndex) page(i int, buf []byte) (bool, error) { return false, nil } // A checkpoint can restart the log between the scan and this read, leaving - // the offset pointing into a later generation's frame; re-reading the frame - // header turns that into an error instead of a wrong page. + // the offset pointing into a later generation's frame. Re-reading the frame + // header catches that in every case but a restart that draws the same salt, + // which is as much as can be had without rewalking the checksum chain. var header [walFrameHeaderSize]byte if _, err := w.f.ReadAt(header[:], off); err != nil { return true, err diff --git a/wal_test.go b/wal_test.go index 48a7e1c..1f60807 100644 --- a/wal_test.go +++ b/wal_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" ) @@ -209,9 +210,9 @@ func TestWALUnreadableIgnored(t *testing.T) { } // SQLite writes a log's checksums in the byte order of the machine that created -// it; there is no big-endian fixture, so this flips a little-endian one and -// recomputes the checksums by hand rather than via walChecksum, so that -// transposing the two magic constants fails instead of cancelling out. +// it, so a log from a big-endian host carries the other magic number and the +// other word order. There is no fixture from such a host to hand, so this makes +// one out of the little-endian fixture. func TestWALBigEndianChecksums(t *testing.T) { path, cleanup := copyDB(t, true) defer cleanup() @@ -220,27 +221,8 @@ func TestWALBigEndianChecksums(t *testing.T) { t.Fatal(err) } - sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { - for i := 0; i+8 <= len(b); i += 8 { - s0 += binary.BigEndian.Uint32(b[i:i+4]) + s1 - s1 += binary.BigEndian.Uint32(b[i+4:i+8]) + s0 - } - return s0, s1 - } - binary.BigEndian.PutUint32(log[0:4], walMagicBigEndian) - s0, s1 := sum(0, 0, log[0:24]) - binary.BigEndian.PutUint32(log[24:28], s0) - binary.BigEndian.PutUint32(log[28:32], s1) - - frame := walFrameHeaderSize + 1024 - for off := walHeaderSize; off+frame <= len(log); off += frame { - f := log[off : off+frame] - s0, s1 = sum(s0, s1, f[0:8]) - s0, s1 = sum(s0, s1, f[walFrameHeaderSize:]) - binary.BigEndian.PutUint32(f[16:20], s0) - binary.BigEndian.PutUint32(f[20:24], s1) - } + sealWAL(binary.BigEndian, log, 1024) if err := ioutil.WriteFile(path+"-wal", log, 0644); err != nil { t.Fatal(err) } @@ -327,49 +309,79 @@ func TestWALDetectsRestartUnderReader(t *testing.T) { } } -// A zeroed page 1 in the log must fail Open, rather than reach the pager as a -// page size of zero. +// The log's page 1 replaces the database header, so it has to clear the same +// bar the main file's page 1 did. A zeroed one used to reach the pager as a +// page size of zero and panic there; one with the wrong magic is not a +// database header at all. func TestWALRejectsBadHeaderPage(t *testing.T) { - path, cleanup := copyDB(t, true) - defer cleanup() - log, err := ioutil.ReadFile(path + "-wal") + real1, err := ioutil.ReadFile(walFixture) if err != nil { t.Fatal(err) } + wrongMagic := append([]byte(nil), real1[:1024]...) + wrongMagic[0] = 'X' - // The fixture's last frame carries page 1. Blanking its payload keeps the - // frame well-formed once the checksums are recomputed over it. - last := walHeaderSize - for last+2*(walFrameHeaderSize+1024) <= len(log) { - last += walFrameHeaderSize + 1024 - } - if got := binary.BigEndian.Uint32(log[last : last+4]); got != 1 { - t.Fatalf("last frame carries page %d, want page 1", got) + tests := []struct { + name string + page1 []byte + }{ + // Zeroed is the panic that actually happened. Wrong magic keeps a + // valid page size, so only the magic check can turn it away. + {"zeroed", nil}, + {"wrong magic", wrongMagic}, } - payload := log[last+walFrameHeaderSize:] - for i := range payload { - payload[i] = 0 + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, cleanup := copyDB(t, false) + defer cleanup() + writeWAL(t, path, buildWAL(walMagicLittleEndian, walFormatVersion, 1024, + []walFrame{{pgno: 1, dbSize: 2, page: tt.page1}})) + + db, err := Open(path) + if err == nil { + db.Close() + t.Error("Open accepted a log whose page 1 is not a database header") + } + }) } - resealWAL(t, log, 1024) - if err := ioutil.WriteFile(path+"-wal", log, 0644); err != nil { +} + +// A page 1 that is a valid header but disagrees with the main file about the +// page size cannot be applied to a pager already sized for the main file. +func TestWALRejectsHeaderPageResize(t *testing.T) { + path, cleanup := copyDB(t, false) + defer cleanup() + main, err := ioutil.ReadFile(path) + if err != nil { t.Fatal(err) } + // The real page 1, so the magic still passes, with only the page-size + // field changed. + page1 := append([]byte(nil), main[:1024]...) + binary.BigEndian.PutUint16(page1[16:18], 512) + writeWAL(t, path, buildWAL(walMagicLittleEndian, walFormatVersion, 1024, + []walFrame{{pgno: 1, dbSize: 2, page: page1}})) + db, err := Open(path) if err == nil { db.Close() - t.Fatal("Open accepted a log whose page 1 is not a database header") + t.Fatal("Open accepted a log whose page 1 changes the page size") + } + if !strings.Contains(err.Error(), "changes the page size") { + t.Errorf("Open: %v, want a page-size error", err) } } -// resealWAL recomputes the little-endian checksum chain over log in place, so -// a test can edit frame payloads and still hand back a log that verifies. -func resealWAL(t *testing.T, log []byte, pageSize int) { - t.Helper() +// sealWAL recomputes the checksum chain over log in place, so a test can edit a +// header field or a frame payload and still hand back a log that verifies. The +// chain is spelled out here rather than called through walChecksum, so that a +// bug in the latter cannot cancel itself out. +func sealWAL(order binary.ByteOrder, log []byte, pageSize int) { sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { for i := 0; i+8 <= len(b); i += 8 { - s0 += binary.LittleEndian.Uint32(b[i:i+4]) + s1 - s1 += binary.LittleEndian.Uint32(b[i+4:i+8]) + s0 + s0 += order.Uint32(b[i:i+4]) + s1 + s1 += order.Uint32(b[i+4:i+8]) + s0 } return s0, s1 } @@ -387,23 +399,21 @@ func resealWAL(t *testing.T, log []byte, pageSize int) { } // walFrame is one frame to assemble into a synthetic log. A dbSize of zero -// makes it a non-commit frame. +// makes it a non-commit frame; a nil page leaves the payload zeroed. type walFrame struct { pgno uint32 dbSize uint32 + page []byte } -// buildWAL assembles a little-endian write-ahead log. The checksums are -// computed here rather than through walChecksum, so that a bug in the latter -// cannot cancel itself out. magic and version are parameters because rejecting -// the wrong ones is most of what these tests check. +// buildWAL assembles a write-ahead log and seals it, so the result verifies as +// it stands. magic and version are parameters because rejecting the wrong ones +// is most of what these tests check; the word order follows the magic, as it +// does in a real log. func buildWAL(magic, version uint32, pageSize int, frames []walFrame) []byte { - sum := func(s0, s1 uint32, b []byte) (uint32, uint32) { - for i := 0; i+8 <= len(b); i += 8 { - s0 += binary.LittleEndian.Uint32(b[i:i+4]) + s1 - s1 += binary.LittleEndian.Uint32(b[i+4:i+8]) + s0 - } - return s0, s1 + order := binary.ByteOrder(binary.LittleEndian) + if magic == walMagicBigEndian { + order = binary.BigEndian } log := make([]byte, walHeaderSize) @@ -411,24 +421,75 @@ func buildWAL(magic, version uint32, pageSize int, frames []walFrame) []byte { binary.BigEndian.PutUint32(log[4:8], version) binary.BigEndian.PutUint32(log[8:12], uint32(pageSize)) copy(log[16:24], []byte("saltsalt")) - s0, s1 := sum(0, 0, log[0:24]) - binary.BigEndian.PutUint32(log[24:28], s0) - binary.BigEndian.PutUint32(log[28:32], s1) for _, f := range frames { frame := make([]byte, walFrameHeaderSize+pageSize) binary.BigEndian.PutUint32(frame[0:4], f.pgno) binary.BigEndian.PutUint32(frame[4:8], f.dbSize) copy(frame[8:16], log[16:24]) - s0, s1 = sum(s0, s1, frame[0:8]) - s0, s1 = sum(s0, s1, frame[walFrameHeaderSize:]) - binary.BigEndian.PutUint32(frame[16:20], s0) - binary.BigEndian.PutUint32(frame[20:24], s1) + copy(frame[walFrameHeaderSize:], f.page) log = append(log, frame...) } + sealWAL(order, log, pageSize) return log } +// writeWAL lays log beside the database at path. +func writeWAL(t *testing.T, path string, log []byte) { + t.Helper() + if err := ioutil.WriteFile(path+"-wal", log, 0644); err != nil { + t.Fatal(err) + } +} + +// The log overlays the main file rather than replacing it, so a page it does +// not carry still has to come from the main file. Every other fixture happens +// to log every page it commits, which never exercises the fall-through. +func TestWALFallsThroughToMainFile(t *testing.T) { + path, cleanup := copyDB(t, false) + defer cleanup() + main, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + // A log carrying page 1 unchanged and committing at the main file's size. + // The rows live on page 2, which only the main file has. + writeWAL(t, path, buildWAL(walMagicLittleEndian, walFormatVersion, 1024, + []walFrame{{pgno: 1, dbSize: 2, page: main[:1024]}})) + + assertRows(t, path, walStaleRows) +} + +// A checkpoint restarts the log with a fresh salt and writes over the old +// frames in place, so the scan has to stop at the first frame carrying a salt +// other than the header's rather than read into the previous generation. +func TestReadWALIndexStopsAtSaltChange(t *testing.T) { + const pageSize = 1024 + log := buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, []walFrame{ + {pgno: 1, dbSize: 1}, + {pgno: 2, dbSize: 2}, + }) + second := walHeaderSize + walFrameHeaderSize + pageSize + copy(log[second+8:second+16], []byte("OTHERSLT")) + sealWAL(binary.LittleEndian, log, pageSize) + + index, err := readWALIndex(bytes.NewReader(log), pageSize) + if err != nil { + t.Fatalf("readWALIndex: %v", err) + } + if index == nil { + t.Fatal("readWALIndex dropped the frames before the salt change") + } + want := map[int]int64{1: walHeaderSize} + if !reflect.DeepEqual(index.offsets, want) { + t.Errorf("offsets = %v, want %v", index.offsets, want) + } + if index.dbSize != 1 { + t.Errorf("dbSize = %d, want 1", index.dbSize) + } +} + // Logs SQLite would refuse, or this package cannot apply. Each must read as // "no snapshot here" rather than as an error, so the main file still opens. func TestReadWALIndexRejects(t *testing.T) { @@ -440,7 +501,7 @@ func TestReadWALIndexRejects(t *testing.T) { // page-size check can turn it away. mislabelled := buildWAL(walMagicLittleEndian, walFormatVersion, pageSize, commit) binary.BigEndian.PutUint32(mislabelled[8:12], 512) - resealWAL(t, mislabelled, pageSize) + sealWAL(binary.LittleEndian, mislabelled, pageSize) tests := []struct { name string From c3f1db8d9b98d817adf5f1a8a3862359a2bf7f62 Mon Sep 17 00:00:00 2001 From: Filipe Guerreiro Date: Wed, 29 Jul 2026 09:18:11 +0900 Subject: [PATCH 5/5] Report a truncated log as a restart, not an I/O fault A checkpoint restarts the log either by overwriting it under a new salt or, for wal_checkpoint(TRUNCATE) and journal_size_limit, by emptying it. Only the first was recognised. The second leaves every indexed offset past the end of the file, so the read came back as a bare io.EOF and read like a disk problem rather than the restart it is. A short read at an offset the scan already reached can only mean the log shrank, so map it onto the same error as a salt mismatch. Also stop the pager cache comment implying a bound it does not have: lru is appended to and never read, so nothing evicts. --- pager.go | 3 ++- wal.go | 34 ++++++++++++++++++++++++++-------- wal_test.go | 30 ++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/pager.go b/pager.go index c23d201..594b1bd 100644 --- a/pager.go +++ b/pager.go @@ -24,7 +24,8 @@ func newPager(f io.ReadSeeker, size, npages int, wal *walIndex) pager { size: size, npages: npages, // No size hint: npages comes from a file header, and a hostile one can - // ask for MaxInt32 pages. The cache only ever holds pages read. + // ask for MaxInt32 pages. The cache grows with the pages actually + // read, not with the count the header claims. pages: make(map[int]page), lru: make([]int, 0, 2), wal: wal, diff --git a/wal.go b/wal.go index d1748a8..2d41e93 100644 --- a/wal.go +++ b/wal.go @@ -6,12 +6,17 @@ package sqlite3 import ( "encoding/binary" - "fmt" + "errors" "io" "math" "os" ) +// errWALChanged reports that the log was checkpointed and restarted after it +// was indexed, so the offsets no longer describe it. Reopening the database +// picks up the new generation. +var errWALChanged = errors.New("sqlite3: write-ahead log changed while being read") + // Write-ahead log reader. // // A database in WAL mode keeps recent pages in a separate "-wal" file and only @@ -56,18 +61,31 @@ func (w *walIndex) page(i int, buf []byte) (bool, error) { return false, nil } // A checkpoint can restart the log between the scan and this read, leaving - // the offset pointing into a later generation's frame. Re-reading the frame - // header catches that in every case but a restart that draws the same salt, - // which is as much as can be had without rewalking the checksum chain. + // the offset pointing into a later generation's frame, or past the end of a + // log that was truncated. Both are caught here, all but a restart that + // draws the same salt, which would need the checksum chain rewalked per + // read to see. var header [walFrameHeaderSize]byte if _, err := w.f.ReadAt(header[:], off); err != nil { - return true, err + return true, walReadError(err) } if binary.BigEndian.Uint32(header[0:4]) != uint32(i) || string(header[8:16]) != string(w.salt[:]) { - return true, fmt.Errorf("sqlite3: write-ahead log changed while being read") + return true, errWALChanged + } + if _, err := w.f.ReadAt(buf, off+walFrameHeaderSize); err != nil { + return true, walReadError(err) + } + return true, nil +} + +// walReadError reports a short read at an offset the scan already reached as a +// restart rather than as an I/O fault, since only a truncation can shorten the +// log and that is how a checkpoint restarts it. +func walReadError(err error) error { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return errWALChanged } - _, err := w.f.ReadAt(buf, off+walFrameHeaderSize) - return true, err + return err } func (w *walIndex) Close() error { diff --git a/wal_test.go b/wal_test.go index 1f60807..5d82395 100644 --- a/wal_test.go +++ b/wal_test.go @@ -304,8 +304,34 @@ func TestWALDetectsRestartUnderReader(t *testing.T) { } log.Close() - if ok, err := index.page(1, buf); !ok || err == nil { - t.Errorf("page(1) = %v, %v; want true and an error", ok, err) + if ok, err := index.page(1, buf); !ok || err != errWALChanged { + t.Errorf("page(1) = %v, %v; want true, %v", ok, err, errWALChanged) + } +} + +// PRAGMA wal_checkpoint(TRUNCATE) restarts the log by emptying it, which leaves +// every indexed offset past the end rather than pointing at a wrong frame. A +// bare io.EOF here would read as an I/O fault instead of a restart. +func TestWALDetectsTruncationUnderReader(t *testing.T) { + path, cleanup := copyDB(t, true) + defer cleanup() + + index, err := openWAL(path, 1024) + if err != nil { + t.Fatalf("openWAL: %v", err) + } + if index == nil { + t.Fatal("openWAL returned no index for the fixture log") + } + defer index.Close() + + if err := os.Truncate(path+"-wal", 0); err != nil { + t.Fatal(err) + } + + buf := make([]byte, 1024) + if ok, err := index.page(1, buf); !ok || err != errWALChanged { + t.Errorf("page(1) = %v, %v; want true, %v", ok, err, errWALChanged) } }