Read the write-ahead log

AUTHORS

@@ -10,5 +10,6 @@
 
 Adam Shannon <adamkshannon@gmail.com>
 Andre Renaud <andre@ignavus.net>
+Filipe Guerreiro <filipe.m.guerreiro@gmail.com>
 Sebastien Binet <binet@cern.ch> <seb.binet@gmail.com>
 Zellyn Hunter <zellyn@gmail.com>

CONTRIBUTORS

@@ -17,5 +17,6 @@
 
 Adam Shannon <adamkshannon@gmail.com>
 Andre Renaud <andre@ignavus.net>
+Filipe Guerreiro <filipe.m.guerreiro@gmail.com>
 Sebastien Binet <binet@cern.ch> <seb.binet@gmail.com>
 Zellyn Hunter <zellyn@gmail.com>

file.go

@@ -5,6 +5,7 @@
 package sqlite3
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -65,6 +66,12 @@ type dbHeader struct {
 	SqliteVersion int32    // SQLITE_VERSION_NUMBER
 }
 
+// OpenFrom reads the database in f.
+//
+// When f is backed by a named file (an *os.File, or any reader exposing
+// Name() string) the committed pages of a write-ahead log beside it are
+// overlaid on the database image, so a database belonging to a running
+// application reads current rather than as of its last checkpoint.
 func OpenFrom(f io.ReadSeeker) (*DbFile, error) {
 	var db DbFile
 
@@ -109,16 +116,68 @@ func OpenFrom(f io.ReadSeeker) (*DbFile, error) {
 		)
 	}
 
-	db.pager = newPager(f, db.PageSize(), db.NumPage())
+	var wal *walIndex
+	if named, ok := f.(interface{ Name() string }); ok {
+		wal, err = openWAL(named.Name(), db.PageSize())
+		if err != nil {
+			return nil, err
+		}
+	}
+	if wal != nil {
+		// Before the pager, so that the page count it is built with is the
+		// log's rather than the main file's stale one.
+		if err := db.applyWALHeader(wal); err != nil {
+			wal.Close()
+			return nil, err
+		}
+	}
+
+	db.pager = newPager(f, db.PageSize(), db.NumPage(), wal)
 
 	err = db.init()
 	if err != nil {
+		db.pager.Delete()
 		return nil, err
 	}
 
 	return &db, err
 }
 
+// applyWALHeader supersedes the header with the log's newer image of page 1,
+// when the log carries one, and takes the page count from its last commit
+// frame either way.
+func (db *DbFile) applyWALHeader(wal *walIndex) error {
+	pageSize := db.PageSize()
+	buf := make([]byte, pageSize)
+	ok, err := wal.page(1, buf)
+	if err != nil {
+		return err
+	}
+	if ok {
+		dec := binary.NewDecoder(bytes.NewReader(buf))
+		dec.Order = binary.BigEndian
+		if err := dec.Decode(&db.header); err != nil {
+			return err
+		}
+		// The magic check above ran against the main file's page 1, so the
+		// log's replacement for it has to clear the same bar. A page size of
+		// zero would otherwise reach the pager and read empty pages forever.
+		if string(db.header.Magic[:]) != sqlite3Magic {
+			return fmt.Errorf("sqlite3: invalid file header in write-ahead log")
+		}
+		if db.PageSize() != pageSize {
+			return fmt.Errorf(
+				"sqlite3: write-ahead log page 1 changes the page size (%d -> %d)",
+				pageSize, db.PageSize(),
+			)
+		}
+	}
+	// Last: the decode above may have just put the main file's stale page
+	// count back.
+	db.header.DbSize = int32(wal.dbSize)
+	return nil
+}
+
 func Open(fname string) (*DbFile, error) {
 	f, err := os.Open(fname)
 	if err != nil {
@@ -135,11 +194,13 @@ func Open(fname string) (*DbFile, error) {
 }
 
 func (db *DbFile) Close() error {
-	db.pager.Delete()
+	err := db.pager.Delete()
 	if db.close != nil {
-		return db.close()
+		if errDB := db.close(); errDB != nil {
+			return errDB
+		}
 	}
-	return nil
+	return err
 }
 
 // PageSize returns the database page size in bytes

pager.go

@@ -15,58 +15,80 @@ type pager struct {
 	npages int          // total number of pages in db
 	pages  map[int]page // cache of pages
 	lru    []int        // list of last used pages
+	wal    *walIndex    // committed pages living in the write-ahead log, if any
 }
 
-func newPager(f io.ReadSeeker, size, npages int) pager {
+func newPager(f io.ReadSeeker, size, npages int, wal *walIndex) pager {
 	pager := pager{
 		f:      f,
 		size:   size,
 		npages: npages,
-		pages:  make(map[int]page, npages),
-		lru:    make([]int, 0, 2),
+		// No size hint: npages comes from a file header, and a hostile one can
+		// ask for MaxInt32 pages. The cache grows with the pages actually
+		// read, not with the count the header claims.
+		pages: make(map[int]page),
+		lru:   make([]int, 0, 2),
+		wal:   wal,
 	}
 
 	return pager
 }
 
 func (p *pager) Page(i int) (page, error) {
-	var err error
 	page, ok := p.pages[i]
 	if ok {
-		return page, err
+		return page, nil
 	}
 
 	if i > p.npages {
 		return page, fmt.Errorf("sqlite3: out of range (%d > %d)", i, p.npages)
 	}
 
+	buf := make([]byte, p.size)
+	if err := p.read(i, buf); err != nil {
+		return page, err
+	}
+
+	page.id = i
+	page.buf = buf
+
+	p.pages[i] = page
+	p.lru = append(p.lru, i)
+	return page, nil
+}
+
+// read fills buf with page i, preferring the write-ahead log's image of it
+// over the one in the main database file.
+func (p *pager) read(i int, buf []byte) error {
+	if p.wal != nil {
+		if ok, err := p.wal.page(i, buf); ok || err != nil {
+			return err
+		}
+	}
+
 	pos, _ := p.f.Seek(0, io.SeekCurrent)
 	defer p.f.Seek(pos, io.SeekStart)
 
-	buf := make([]byte, p.size)
 	if _, err := p.f.Seek(int64((i-1)*p.size), io.SeekStart); err != nil {
-		return page, err
+		return err
 	}
 	n, err := p.f.Read(buf)
 	if err != nil {
-		return page, err
+		return err
 	}
-
 	if n != len(buf) {
-		return page, fmt.Errorf("sqlite3: read too few bytes")
+		return fmt.Errorf("sqlite3: read too few bytes")
 	}
-
-	page.id = i
-	page.buf = buf
-
-	p.pages[i] = page
-	p.lru = append(p.lru, i)
-	return page, err
+	return nil
 }
 
 func (p *pager) Delete() error {
 	var err error
 	p.pages = nil
 	p.lru = nil
+	if p.wal != nil {
+		err = p.wal.Close()
+		p.wal = nil
+	}
 	return err
 }