diff --git a/go.mod b/go.mod index 7ff5309..b117a0c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 - github.com/btcsuite/btcwallet/walletdb v1.0.0 + github.com/btcsuite/btcwallet/walletdb v1.2.0 github.com/btcsuite/btcwallet/wtxmgr v1.0.0 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 09627e9..3f267ec 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= diff --git a/goclean.sh b/goclean.sh index 0a123f9..a01f733 100755 --- a/goclean.sh +++ b/goclean.sh @@ -7,7 +7,7 @@ set -ex -test_targets=$(go list ./...) +test_targets=$(go list -deps ./... | grep 'btcwallet') # Automatic checks test -z "$(go fmt $test_targets | tee /dev/stderr)" diff --git a/walletdb/bdb/db.go b/walletdb/bdb/db.go index 046c6ac..cf3fc2e 100644 --- a/walletdb/bdb/db.go +++ b/walletdb/bdb/db.go @@ -68,7 +68,7 @@ func (tx *transaction) ReadWriteBucket(key []byte) walletdb.ReadWriteBucket { } func (tx *transaction) CreateTopLevelBucket(key []byte) (walletdb.ReadWriteBucket, error) { - boltBucket, err := tx.boltTx.CreateBucket(key) + boltBucket, err := tx.boltTx.CreateBucketIfNotExists(key) if err != nil { return nil, convertErr(err) } @@ -231,6 +231,21 @@ func (b *bucket) Tx() walletdb.ReadWriteTx { } } +// NextSequence returns an autoincrementing integer for the bucket. +func (b *bucket) NextSequence() (uint64, error) { + return (*bbolt.Bucket)(b).NextSequence() +} + +// SetSequence updates the sequence number for the bucket. +func (b *bucket) SetSequence(v uint64) error { + return (*bbolt.Bucket)(b).SetSequence(v) +} + +// Sequence returns the current integer for the bucket without incrementing it. +func (b *bucket) Sequence() uint64 { + return (*bbolt.Bucket)(b).Sequence() +} + // cursor represents a cursor over key/value pairs and nested buckets of a // bucket. // @@ -327,6 +342,19 @@ func (db *db) Close() error { return convertErr((*bbolt.DB)(db).Close()) } +// Batch is similar to the package-level Update method, but it will attempt to +// optismitcally combine the invocation of several transaction functions into a +// single db write transaction. +// +// This function is part of the walletdb.Db interface implementation. +func (db *db) Batch(f func(tx walletdb.ReadWriteTx) error) error { + return (*bbolt.DB)(db).Batch(func(btx *bbolt.Tx) error { + interfaceTx := transaction{btx} + + return f(&interfaceTx) + }) +} + // filesExists reports whether the named file or directory exists. func fileExists(name string) bool { if _, err := os.Stat(name); err != nil { diff --git a/walletdb/bdb/driver_test.go b/walletdb/bdb/driver_test.go index c233abe..8fdcab1 100644 --- a/walletdb/bdb/driver_test.go +++ b/walletdb/bdb/driver_test.go @@ -23,7 +23,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure that attempting to open a database that doesn't exist returns // the expected error. wantErr := walletdb.ErrDbDoesNotExist - if _, err := walletdb.Open(dbType, "noexist.db"); err != wantErr { + if _, err := walletdb.Open(dbType, "noexist.db", true); err != wantErr { t.Errorf("Open: did not receive expected error - got %v, "+ "want %v", err, wantErr) return @@ -32,7 +32,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure that attempting to open a database with the wrong number of // parameters returns the expected error. wantErr = fmt.Errorf("invalid arguments to %s.Open -- expected "+ - "database path", dbType) + "database path and no-freelist-sync option", dbType) if _, err := walletdb.Open(dbType, 1, 2, 3); err.Error() != wantErr.Error() { t.Errorf("Open: did not receive expected error - got %v, "+ "want %v", err, wantErr) @@ -43,7 +43,7 @@ func TestCreateOpenFail(t *testing.T) { // the first parameter returns the expected error. wantErr = fmt.Errorf("first argument to %s.Open is invalid -- "+ "expected database path string", dbType) - if _, err := walletdb.Open(dbType, 1); err.Error() != wantErr.Error() { + if _, err := walletdb.Open(dbType, 1, true); err.Error() != wantErr.Error() { t.Errorf("Open: did not receive expected error - got %v, "+ "want %v", err, wantErr) return @@ -52,7 +52,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure that attempting to create a database with the wrong number of // parameters returns the expected error. wantErr = fmt.Errorf("invalid arguments to %s.Create -- expected "+ - "database path", dbType) + "database path and no-freelist-sync option", dbType) if _, err := walletdb.Create(dbType, 1, 2, 3); err.Error() != wantErr.Error() { t.Errorf("Create: did not receive expected error - got %v, "+ "want %v", err, wantErr) @@ -63,7 +63,7 @@ func TestCreateOpenFail(t *testing.T) { // the first parameter returns the expected error. wantErr = fmt.Errorf("first argument to %s.Create is invalid -- "+ "expected database path string", dbType) - if _, err := walletdb.Create(dbType, 1); err.Error() != wantErr.Error() { + if _, err := walletdb.Create(dbType, 1, true); err.Error() != wantErr.Error() { t.Errorf("Create: did not receive expected error - got %v, "+ "want %v", err, wantErr) return @@ -72,7 +72,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure operations against a closed database return the expected // error. dbPath := "createfail.db" - db, err := walletdb.Create(dbType, dbPath) + db, err := walletdb.Create(dbType, dbPath, true) if err != nil { t.Errorf("Create: unexpected error: %v", err) return @@ -93,7 +93,7 @@ func TestCreateOpenFail(t *testing.T) { func TestPersistence(t *testing.T) { // Create a new database to run tests against. dbPath := "persistencetest.db" - db, err := walletdb.Create(dbType, dbPath) + db, err := walletdb.Create(dbType, dbPath, true) if err != nil { t.Errorf("Failed to create test database (%s) %v", dbType, err) return diff --git a/walletdb/db_test.go b/walletdb/db_test.go index 95ce2de..48b60c3 100644 --- a/walletdb/db_test.go +++ b/walletdb/db_test.go @@ -55,7 +55,7 @@ func TestAddDuplicateDriver(t *testing.T) { } dbPath := "dupdrivertest.db" - db, err := walletdb.Create(dbType, dbPath) + db, err := walletdb.Create(dbType, dbPath, true) if err != nil { t.Errorf("failed to create database: %v", err) return diff --git a/walletdb/example_test.go b/walletdb/example_test.go index 73dda14..4cb5134 100644 --- a/walletdb/example_test.go +++ b/walletdb/example_test.go @@ -28,7 +28,7 @@ func ExampleCreate() { // this, but it's done here in the example to ensure the example cleans // up after itself. dbPath := filepath.Join(os.TempDir(), "examplecreate.db") - db, err := walletdb.Create("bdb", dbPath) + db, err := walletdb.Create("bdb", dbPath, true) if err != nil { fmt.Println(err) return @@ -47,7 +47,7 @@ var exampleNum = 0 func exampleLoadDB() (walletdb.DB, func(), error) { dbName := fmt.Sprintf("exampleload%d.db", exampleNum) dbPath := filepath.Join(os.TempDir(), dbName) - db, err := walletdb.Create("bdb", dbPath) + db, err := walletdb.Create("bdb", dbPath, true) if err != nil { return nil, nil, err } @@ -111,7 +111,7 @@ func Example_basicUsage() { // this, but it's done here in the example to ensure the example cleans // up after itself. dbPath := filepath.Join(os.TempDir(), "exampleusage.db") - db, err := walletdb.Create("bdb", dbPath) + db, err := walletdb.Create("bdb", dbPath, true) if err != nil { fmt.Println(err) return diff --git a/walletdb/go.mod b/walletdb/go.mod index 5daf526..59f6db3 100644 --- a/walletdb/go.mod +++ b/walletdb/go.mod @@ -7,5 +7,6 @@ require ( github.com/coreos/bbolt v1.3.3 github.com/davecgh/go-spew v1.1.1 go.etcd.io/bbolt v1.3.3 // indirect + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect ) diff --git a/walletdb/go.sum b/walletdb/go.sum index e68003e..ca356ac 100644 --- a/walletdb/go.sum +++ b/walletdb/go.sum @@ -6,5 +6,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/walletdb/interface.go b/walletdb/interface.go index f89fa91..40fb19e 100644 --- a/walletdb/interface.go +++ b/walletdb/interface.go @@ -7,7 +7,10 @@ package walletdb -import "io" +import ( + "fmt" + "io" +) // ReadTx represents a database transaction that can only be used for reads. If // a database update must occur, use a ReadWriteTx. @@ -126,6 +129,16 @@ type ReadWriteBucket interface { // Tx returns the bucket's transaction. Tx() ReadWriteTx + + // NextSequence returns an autoincrementing integer for the bucket. + NextSequence() (uint64, error) + + // SetSequence updates the sequence number for the bucket. + SetSequence(v uint64) error + + // Sequence returns the current integer for the bucket without + // incrementing it. + Sequence() uint64 } // ReadCursor represents a bucket cursor that can be positioned at the start or @@ -191,6 +204,18 @@ type DB interface { Close() error } +// BatchDB is a special version of the main DB interface that allos the caller +// to specify write transactions that should be combine dtoegether if multiple +// goroutines are calling the Batch method. +type BatchDB interface { + DB + + // Batch is similar to the package-level Update method, but it will + // attempt to optismitcally combine the invocation of several + // transaction functions into a single db write transaction. + Batch(func(tx ReadWriteTx) error) error +} + // View opens a database read transaction and executes the function f with the // transaction passed as a parameter. After f exits, the transaction is rolled // back. If f errors, its error is returned, not a rollback error (if any @@ -200,11 +225,20 @@ func View(db DB, f func(tx ReadTx) error) error { if err != nil { return err } + + // Make sure the transaction rolls back in the event of a panic. + defer func() { + if tx != nil { + tx.Rollback() + } + }() + err = f(tx) rollbackErr := tx.Rollback() if err != nil { return err } + if rollbackErr != nil { return rollbackErr } @@ -222,6 +256,14 @@ func Update(db DB, f func(tx ReadWriteTx) error) error { if err != nil { return err } + + // Make sure the transaction rolls back in the event of a panic. + defer func() { + if tx != nil { + tx.Rollback() + } + }() + err = f(tx) if err != nil { // Want to return the original error, not a rollback error if @@ -229,9 +271,27 @@ func Update(db DB, f func(tx ReadWriteTx) error) error { _ = tx.Rollback() return err } + return tx.Commit() } +// Batch opens a database read/write transaction and executes the function f +// with the transaction passed as a parameter. After f exits, if f did not +// error, the transaction is committed. Otherwise, if f did error, the +// transaction is rolled back. If the rollback fails, the original error +// returned by f is still returned. If the commit fails, the commit error is +// returned. +// +// Batch is only useful when there are multiple goroutines calling it. +func Batch(db DB, f func(tx ReadWriteTx) error) error { + batchDB, ok := db.(BatchDB) + if !ok { + return fmt.Errorf("need batch") + } + + return batchDB.Batch(f) +} + // Driver defines a structure for backend drivers to use when they registered // themselves as a backend which implements the Db interface. type Driver struct { diff --git a/walletdb/walletdbtest/interface.go b/walletdb/walletdbtest/interface.go index 8a2beb1..fe338a7 100644 --- a/walletdb/walletdbtest/interface.go +++ b/walletdb/walletdbtest/interface.go @@ -5,9 +5,11 @@ package walletdbtest import ( + "bytes" "fmt" "os" "reflect" + "sync" "github.com/btcsuite/btcwallet/walletdb" ) @@ -104,6 +106,51 @@ func testNestedReadWriteBucket(tc *testContext, testBucket walletdb.ReadWriteBuc return true } +// testSequence tests that the sequence related methods work as expected. +func testSequence(tc *testContext, testBucket walletdb.ReadWriteBucket) bool { + // Obtaining the current sequence twice should give us the same value. + seqNo1 := testBucket.Sequence() + seqNo2 := testBucket.Sequence() + if seqNo1 != seqNo2 { + tc.t.Errorf("Sequence: seq has incremented") + return false + } + + // Incrementing to the next sequence should give us a value one larger + // than the prior number. + seqNo3, err := testBucket.NextSequence() + if err != nil { + tc.t.Errorf("Sequence: unexpected error: %v", err) + return false + } + if seqNo3 != seqNo2+1 { + tc.t.Errorf("Sequence: expected seq no of %v, instead got %v", + seqNo2+1, seqNo3) + return false + } + + // We should be able to modify the sequence base number. + newBase := uint64(100) + if err := testBucket.SetSequence(newBase); err != nil { + tc.t.Errorf("Sequence: unexpected error: %v", err) + return false + } + + // Any offset from this new sequence should now be properly reflected. + seqNo4, err := testBucket.NextSequence() + if err != nil { + tc.t.Errorf("Sequence: unexpected error: %v", err) + return false + } + if seqNo4 != newBase+1 { + tc.t.Errorf("Sequence: expected seq no of %v, instead got %v", + newBase+1, seqNo4) + return false + } + + return true +} + // testReadWriteBucketInterface ensures the bucket interface is working properly by // exercising all of its functions. func testReadWriteBucketInterface(tc *testContext, bucket walletdb.ReadWriteBucket) bool { @@ -164,6 +211,11 @@ func testReadWriteBucketInterface(tc *testContext, bucket walletdb.ReadWriteBuck return false } + // Test that the sequence methods work as expected. + if !testSequence(tc, bucket) { + return false + } + // Ensure creating a new bucket works as expected. testBucketName := []byte("testbucket") testBucket, err := bucket.CreateBucket(testBucketName) @@ -676,9 +728,73 @@ func testAdditionalErrors(tc *testContext) bool { return true } +// testBatchInterface tests that if the target database implements the batch +// method, then the method functions as expected. +func testBatchInterface(tc *testContext) bool { + // If the database doesn't support the batch super-set of the + // interface, then we're done here. + batchDB, ok := tc.db.(walletdb.BatchDB) + if !ok { + return true + } + + const numGoroutines = 5 + errChan := make(chan error, numGoroutines) + + var wg sync.WaitGroup + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + err := walletdb.Batch(batchDB, func(tx walletdb.ReadWriteTx) error { + b, err := tx.CreateTopLevelBucket([]byte("test")) + if err != nil { + return err + } + + byteI := []byte{byte(i)} + return b.Put(byteI, byteI) + }) + errChan <- err + }(i) + } + + wg.Wait() + close(errChan) + + for err := range errChan { + if err != nil { + tc.t.Errorf("Batch: unexpected error: %v", err) + return false + } + } + + err := walletdb.View(batchDB, func(tx walletdb.ReadTx) error { + b := tx.ReadBucket([]byte("test")) + + for i := 0; i < numGoroutines; i++ { + byteI := []byte{byte(i)} + if v := b.Get(byteI); v == nil { + return fmt.Errorf("key %v not present", byteI) + } else if !bytes.Equal(v, byteI) { + return fmt.Errorf("key %v not equal to value: "+ + "%v", byteI, v) + } + } + + return nil + }) + if err != nil { + tc.t.Errorf("Batch: unexpected error: %v", err) + return false + } + + return true +} + // TestInterface performs all interfaces tests for this database driver. func TestInterface(t Tester, dbType, dbPath string) { - db, err := walletdb.Create(dbType, dbPath) + db, err := walletdb.Create(dbType, dbPath, true) if err != nil { t.Errorf("Failed to create test database (%s) %v", dbType, err) return @@ -704,4 +820,9 @@ func TestInterface(t Tester, dbType, dbPath string) { if !testAdditionalErrors(&context) { return } + + // If applicable, also test the behavior of the Batch call. + if !testBatchInterface(&context) { + return + } } diff --git a/wtxmgr/go.mod b/wtxmgr/go.mod index 9f754a0..9d04e24 100644 --- a/wtxmgr/go.mod +++ b/wtxmgr/go.mod @@ -6,5 +6,5 @@ require ( github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d - github.com/btcsuite/btcwallet/walletdb v1.0.0 + github.com/btcsuite/btcwallet/walletdb v1.2.0 ) diff --git a/wtxmgr/go.sum b/wtxmgr/go.sum index 8400709..363cc23 100644 --- a/wtxmgr/go.sum +++ b/wtxmgr/go.sum @@ -5,8 +5,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw= -github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= +github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y= +github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= diff --git a/wtxmgr/tx_test.go b/wtxmgr/tx_test.go index ce1f32c..84a6023 100644 --- a/wtxmgr/tx_test.go +++ b/wtxmgr/tx_test.go @@ -51,7 +51,7 @@ func testDB() (walletdb.DB, func(), error) { if err != nil { return nil, func() {}, err } - db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db")) + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true) return db, func() { os.RemoveAll(tmpDir) }, err } @@ -63,7 +63,7 @@ func testStore() (*Store, walletdb.DB, func(), error) { return nil, nil, func() {}, err } - db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db")) + db, err := walletdb.Create("bdb", filepath.Join(tmpDir, "db"), true) if err != nil { os.RemoveAll(tmpDir) return nil, nil, nil, err