From 454d290b68a160cc858ce6df8e3a3d41b290c4ac Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 2 Nov 2014 15:06:11 -0600 Subject: [PATCH] Convert waddrmgr to new walletdb package. This commit converts the waddrmgr package to use the new walletdb package semantics. Since waddrmgr no longer controls the database, it is unable to make a copy of the database and return it as the old ExportWatchingOnly function required. As a result, it has been renamed to ConvertToWatchingOnly and it now modifies the namespace provided to it. The idea is that the caller which does control the database can now make a copy of the database, get the waddrmgr namespace in the database copy and invoke the new function to modify it. This also works well with other packages that might also need to make modifications for watching-only mode. In addition, the following changes are made: - All places that worked with database paths now work with the walletdb.Namespace interface - The managerTx code is replaced to use the walletdb.Tx interface - The code which checks if the manager already exists is updated to work with the walletdb.Namespace interface - The LatestDbVersion constant is now LatestMgrVersion since it no longer controls the database --- waddrmgr/common_test.go | 83 ++++++++- waddrmgr/db.go | 381 +++++++++++++++------------------------ waddrmgr/error.go | 4 +- waddrmgr/manager.go | 296 ++++++++++++++++-------------- waddrmgr/manager_test.go | 120 ++++++------ waddrmgr/sync.go | 7 +- 6 files changed, 462 insertions(+), 429 deletions(-) diff --git a/waddrmgr/common_test.go b/waddrmgr/common_test.go index e52acca..59ef2e8 100644 --- a/waddrmgr/common_test.go +++ b/waddrmgr/common_test.go @@ -18,9 +18,15 @@ package waddrmgr_test import ( "encoding/hex" + "io/ioutil" + "os" + "path/filepath" "testing" + "github.com/conformal/btcnet" "github.com/conformal/btcwallet/waddrmgr" + "github.com/conformal/btcwallet/walletdb" + _ "github.com/conformal/btcwallet/walletdb/bdb" ) var ( @@ -36,6 +42,14 @@ var ( privPassphrase = []byte("81lUHXnOMZ@?XXd7O9xyDIWIbXX-lj") pubPassphrase2 = []byte("-0NV4P~VSJBWbunw}% @@ -1035,9 +1053,9 @@ func (mtx *managerTx) FetchSyncedTo() (*BlockStamp, error) { return &bs, nil } -// PutSyncedTo stores the provided synced to blockstamp to the database. -func (mtx *managerTx) PutSyncedTo(bs *BlockStamp) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putSyncedTo stores the provided synced to blockstamp to the database. +func putSyncedTo(tx walletdb.Tx, bs *BlockStamp) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized synced to format is: // @@ -1055,10 +1073,10 @@ func (mtx *managerTx) PutSyncedTo(bs *BlockStamp) error { return nil } -// FetchStartBlock loads the start block stamp for the manager from the +// fetchStartBlock loads the start block stamp for the manager from the // database. -func (mtx *managerTx) FetchStartBlock() (*BlockStamp, error) { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +func fetchStartBlock(tx walletdb.Tx) (*BlockStamp, error) { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized start block format is: // @@ -1076,9 +1094,9 @@ func (mtx *managerTx) FetchStartBlock() (*BlockStamp, error) { return &bs, nil } -// PutStartBlock stores the provided start block stamp to the database. -func (mtx *managerTx) PutStartBlock(bs *BlockStamp) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putStartBlock stores the provided start block stamp to the database. +func putStartBlock(tx walletdb.Tx, bs *BlockStamp) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized start block format is: // @@ -1096,10 +1114,10 @@ func (mtx *managerTx) PutStartBlock(bs *BlockStamp) error { return nil } -// FetchRecentBlocks returns the height of the most recent block height and +// fetchRecentBlocks returns the height of the most recent block height and // hashes of the most recent blocks. -func (mtx *managerTx) FetchRecentBlocks() (int32, []btcwire.ShaHash, error) { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +func fetchRecentBlocks(tx walletdb.Tx) (int32, []btcwire.ShaHash, error) { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized recent blocks format is: // @@ -1127,9 +1145,9 @@ func (mtx *managerTx) FetchRecentBlocks() (int32, []btcwire.ShaHash, error) { return recentHeight, recentHashes, nil } -// PutStartBlock stores the provided start block stamp to the database. -func (mtx *managerTx) PutRecentBlocks(recentHeight int32, recentHashes []btcwire.ShaHash) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putRecentBlocks stores the provided start block stamp to the database. +func putRecentBlocks(tx walletdb.Tx, recentHeight int32, recentHashes []btcwire.ShaHash) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized recent blocks format is: // @@ -1154,166 +1172,71 @@ func (mtx *managerTx) PutRecentBlocks(recentHeight int32, recentHashes []btcwire return nil } -// managerDB provides transactional facilities to read and write the address -// manager data to a bolt database. -type managerDB struct { - db *bolt.DB - version uint32 - created time.Time -} - -// Close releases all database resources. All transactions must be closed -// before closing the database. -func (db *managerDB) Close() error { - if err := db.db.Close(); err != nil { - str := "failed to close database" - return managerError(ErrDatabase, str, err) - } - - return nil -} - -// View executes the passed function within the context of a managed read-only -// transaction. Any error that is returned from the passed function is returned -// from this function. -func (db *managerDB) View(fn func(tx *managerTx) error) error { - err := db.db.View(func(tx *bolt.Tx) error { - return fn((*managerTx)(tx)) - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database read transaction" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// Update executes the passed function within the context of a read-write -// managed transaction. The transaction is committed if no error is returned -// from the function. On the other hand, the entire transaction is rolled back -// if an error is returned. Any error that is returned from the passed function -// or returned from the commit is returned from this function. -func (db *managerDB) Update(fn func(tx *managerTx) error) error { - err := db.db.Update(func(tx *bolt.Tx) error { - return fn((*managerTx)(tx)) - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database write transaction" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// CopyDB copies the entire database to the provided new database path. A -// reader transaction is maintained during the copy so it is safe to continue -// using the database while a copy is in progress. -func (db *managerDB) CopyDB(newDbPath string) error { - err := db.db.View(func(tx *bolt.Tx) error { - if err := tx.CopyFile(newDbPath, 0600); err != nil { - str := "failed to copy database" - return managerError(ErrDatabase, str, err) - } - +// managerExists returns whether or not the manager has already been created +// in the given database namespace. +func managerExists(namespace walletdb.Namespace) (bool, error) { + var exists bool + err := namespace.View(func(tx walletdb.Tx) error { + mainBucket := tx.RootBucket().Bucket(mainBucketName) + exists = mainBucket != nil return nil }) if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database copy" - return managerError(ErrDatabase, str, err) - } - return err + str := fmt.Sprintf("failed to obtain database view: %v", err) + return false, managerError(ErrDatabase, str, err) } - - return nil + return exists, nil } -// WriteTo writes the entire database to the provided writer. A reader -// transaction is maintained during the copy so it is safe to continue using the -// database while a copy is in progress. -func (db *managerDB) WriteTo(w io.Writer) error { - err := db.db.View(func(tx *bolt.Tx) error { - if err := tx.Copy(w); err != nil { - str := "failed to copy database" - return managerError(ErrDatabase, str, err) - } - - return nil - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database copy" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// openOrCreateDB opens the database at the provided path or creates and +// upgradeManager opens the manager using the specified namespace or creates and // initializes it if it does not already exist. It also provides facilities to -// upgrade the database to newer versions. -func openOrCreateDB(dbPath string) (*managerDB, error) { - db, err := bolt.Open(dbPath, 0600, nil) - if err != nil { - str := "failed to open database" - return nil, managerError(ErrDatabase, str, err) - } - +// upgrade the data in the namespace to newer versions. +func upgradeManager(namespace walletdb.Namespace) error { // Initialize the buckets and main db fields as needed. var version uint32 var createDate uint64 - err = db.Update(func(tx *bolt.Tx) error { - mainBucket, err := tx.CreateBucketIfNotExists(mainBucketName) + err := namespace.Update(func(tx walletdb.Tx) error { + rootBucket := tx.RootBucket() + mainBucket, err := rootBucket.CreateBucketIfNotExists( + mainBucketName) if err != nil { str := "failed to create main bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(addrBucketName) + _, err = rootBucket.CreateBucketIfNotExists(addrBucketName) if err != nil { str := "failed to create address bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(acctBucketName) + _, err = rootBucket.CreateBucketIfNotExists(acctBucketName) if err != nil { str := "failed to create account bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(addrAcctIdxBucketName) + _, err = rootBucket.CreateBucketIfNotExists(addrAcctIdxBucketName) if err != nil { str := "failed to create address index bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(syncBucketName) + _, err = rootBucket.CreateBucketIfNotExists(syncBucketName) if err != nil { str := "failed to create sync bucket" return managerError(ErrDatabase, str, err) } - // Save the most recent database version if it isn't already + // Save the most recent manager version if it isn't already // there, otherwise keep track of it for potential upgrades. - verBytes := mainBucket.Get(dbVersionName) + verBytes := mainBucket.Get(mgrVersionName) if verBytes == nil { - version = LatestDbVersion + version = LatestMgrVersion var buf [4]byte binary.LittleEndian.PutUint32(buf[:], version) - err := mainBucket.Put(dbVersionName, buf[:]) + err := mainBucket.Put(mgrVersionName, buf[:]) if err != nil { str := "failed to store latest database version" return managerError(ErrDatabase, str, err) @@ -1322,12 +1245,12 @@ func openOrCreateDB(dbPath string) (*managerDB, error) { version = binary.LittleEndian.Uint32(verBytes) } - createBytes := mainBucket.Get(dbCreateDateName) + createBytes := mainBucket.Get(mgrCreateDateName) if createBytes == nil { createDate = uint64(time.Now().Unix()) var buf [8]byte binary.LittleEndian.PutUint64(buf[:], createDate) - err := mainBucket.Put(dbCreateDateName, buf[:]) + err := mainBucket.Put(mgrCreateDateName, buf[:]) if err != nil { str := "failed to store database creation time" return managerError(ErrDatabase, str, err) @@ -1340,17 +1263,13 @@ func openOrCreateDB(dbPath string) (*managerDB, error) { }) if err != nil { str := "failed to update database" - return nil, managerError(ErrDatabase, str, err) + return managerError(ErrDatabase, str, err) } - // Upgrade the database as needed. - if version < LatestDbVersion { + // Upgrade the manager as needed. + if version < LatestMgrVersion { // No upgrades yet. } - return &managerDB{ - db: db, - version: version, - created: time.Unix(int64(createDate), 0), - }, nil + return nil } diff --git a/waddrmgr/error.go b/waddrmgr/error.go index c2fbba4..3f6efa9 100644 --- a/waddrmgr/error.go +++ b/waddrmgr/error.go @@ -74,10 +74,10 @@ const ( // key type has been selected. ErrInvalidKeyType - // ErrNoExist indicates the specified database does not exist. + // ErrNoExist indicates the manager does not exist. ErrNoExist - // ErrAlreadyExists indicates the specified database already exists. + // ErrAlreadyExists indicates the specified manager already exists. ErrAlreadyExists // ErrCoinTypeTooHigh indicates the coin type specified in the provided diff --git a/waddrmgr/manager.go b/waddrmgr/manager.go index fccb68c..71f8214 100644 --- a/waddrmgr/manager.go +++ b/waddrmgr/manager.go @@ -18,8 +18,6 @@ package waddrmgr import ( "fmt" - "io" - "os" "sync" "github.com/conformal/btcec" @@ -27,6 +25,7 @@ import ( "github.com/conformal/btcutil" "github.com/conformal/btcutil/hdkeychain" "github.com/conformal/btcwallet/snacl" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -204,7 +203,7 @@ var newCryptoKey = defaultNewCryptoKey type Manager struct { mtx sync.RWMutex - db *managerDB + namespace walletdb.Namespace net *btcnet.Params addrs map[addrKey]ManagedAddress syncState syncState @@ -309,9 +308,9 @@ func (m *Manager) zeroSensitivePublicData() { m.masterKeyPub.Zero() } -// Close cleanly shuts down the underlying database and syncs all data. It also -// makes a best try effort to remove and zero all private key and sensitive -// public key material associated with the address manager. +// Close cleanly shuts down the manager. It makes a best try effort to remove +// and zero all private key and sensitive public key material associated with +// the address manager from memory. func (m *Manager) Close() error { m.mtx.Lock() defer m.mtx.Unlock() @@ -324,10 +323,6 @@ func (m *Manager) Close() error { // Attempt to clear sensitive public key material from memory too. m.zeroSensitivePublicData() - if err := m.db.Close(); err != nil { - return err - } - m.closed = true return nil } @@ -408,13 +403,13 @@ func (m *Manager) loadAccountInfo(account uint32) (*accountInfo, error) { // The account is either invalid or just wasn't cached, so attempt to // load the information from the database. var rowInterface interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterface, err = tx.FetchAccountInfo(account) + rowInterface, err = fetchAccountInfo(tx, account) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Ensure the account type is a BIP0044 account. @@ -596,13 +591,13 @@ func (m *Manager) rowInterfaceToManaged(rowInterface interface{}) (ManagedAddres func (m *Manager) loadAndCacheAddress(address btcutil.Address) (ManagedAddress, error) { // Attempt to load the raw address information from the database. var rowInterface interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterface, err = tx.FetchAddress(address.ScriptAddress()) + rowInterface, err = fetchAddress(tx, address.ScriptAddress()) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Create a new managed address for the specific type of address based @@ -732,16 +727,16 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private // Save the new keys and params to the the db in a single // transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutCryptoKeys(nil, encPriv, encScript) + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putCryptoKeys(tx, nil, encPriv, encScript) if err != nil { return err } - return tx.PutMasterKeyParams(nil, newKeyParams) + return putMasterKeyParams(tx, nil, newKeyParams) }) if err != nil { - return err + return maybeConvertDbError(err) } // Now that the db has been successfully updated, clear the old @@ -761,16 +756,16 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private // Save the new keys and params to the the db in a single // transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutCryptoKeys(encryptedPub, nil, nil) + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putCryptoKeys(tx, encryptedPub, nil, nil) if err != nil { return err } - return tx.PutMasterKeyParams(newKeyParams, nil) + return putMasterKeyParams(tx, newKeyParams, nil) }) if err != nil { - return err + return maybeConvertDbError(err) } // Now that the db has been successfully updated, clear the old @@ -782,53 +777,85 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private return nil } -// ExportWatchingOnly creates a new watching-only address manager backed by a -// database at the provided path. A watching-only address manager has all -// private keys removed which means it is not possible to create transactions -// which spend funds. -func (m *Manager) ExportWatchingOnly(newDbPath string, pubPassphrase []byte) (*Manager, error) { - m.mtx.RLock() - defer m.mtx.RUnlock() +// ConvertToWatchingOnly converts the current address manager to a locked +// watching-only address manager. +// +// WARNING: This function removes private keys from the existing address manager +// which means they will no longer be available. Typically the caller will make +// a copy of the existing wallet database and modify the copy since otherwise it +// would mean permanent loss of any imported private keys and scripts. +// +// Executing this function on a manager that is already watching-only will have +// no effect. +func (m *Manager) ConvertToWatchingOnly(pubPassphrase []byte) error { + m.mtx.Lock() + defer m.mtx.Unlock() - // Return an error if the specified database already exists. - if fileExists(newDbPath) { - return nil, managerError(ErrAlreadyExists, errAlreadyExists, nil) - } - - // Copy the existing manager database to the provided path. - if err := m.db.CopyDB(newDbPath); err != nil { - return nil, err - } - - // Open the copied database. - watchingDb, err := openOrCreateDB(newDbPath) - if err != nil { - return nil, err + // Exit now if the manager is already watching-only. + if m.watchingOnly { + return nil } // Remove all private key material and mark the new database as watching // only. - err = watchingDb.Update(func(tx *managerTx) error { - if err := tx.DeletePrivateKeys(); err != nil { + err := m.namespace.Update(func(tx walletdb.Tx) error { + if err := deletePrivateKeys(tx); err != nil { return err } - return tx.PutWatchingOnly(true) + return putWatchingOnly(tx, true) }) if err != nil { - return nil, err + return maybeConvertDbError(err) } - return loadManager(watchingDb, pubPassphrase, m.net, m.config) -} + // Lock the manager to remove all clear text private key material from + // memory if needed. + if !m.locked { + m.lock() + } -// Export writes the manager database to the provided writer. -func (m *Manager) Export(w io.Writer) error { - m.mtx.RLock() - defer m.mtx.RUnlock() + // This section clears and removes the encrypted private key material + // that is ordinarily used to unlock the manager. Since the the manager + // is being converted to watching-only, the encrypted private key + // material is no longer needed. + + // Clear and remove all of the encrypted acount private keys. + for _, acctInfo := range m.acctInfo { + zero(acctInfo.acctKeyEncrypted) + acctInfo.acctKeyEncrypted = nil + } + + // Clear and remove encrypted private keys and encrypted scripts from + // all address entries. + for _, ma := range m.addrs { + switch addr := ma.(type) { + case *managedAddress: + zero(addr.privKeyEncrypted) + addr.privKeyEncrypted = nil + case *scriptAddress: + zero(addr.scriptEncrypted) + addr.scriptEncrypted = nil + } + } + + // Clear and remove encrypted private and script crypto keys. + zero(m.cryptoKeyScriptEncrypted) + m.cryptoKeyScriptEncrypted = nil + m.cryptoKeyScript = nil + zero(m.cryptoKeyPrivEncrypted) + m.cryptoKeyPrivEncrypted = nil + m.cryptoKeyPriv = nil + + // The master private key is derived from a passphrase when the manager + // is unlocked, so there is no encrypted version to zero. However, + // it is no longer needed, so nil it. + m.masterKeyPriv = nil + + // Mark the manager watching-only. + m.watchingOnly = true + return nil - // Copy the existing manager database to the provided path. - return m.db.WriteTo(w) } // existsAddress returns whether or not the passed address is known to the @@ -843,12 +870,12 @@ func (m *Manager) existsAddress(addressID []byte) (bool, error) { // Check the database if not already found above. var exists bool - err := m.db.View(func(tx *managerTx) error { - exists = tx.ExistsAddress(addressID) + err := m.namespace.View(func(tx walletdb.Tx) error { + exists = existsAddress(tx, addressID) return nil }) if err != nil { - return false, err + return false, maybeConvertDbError(err) } return exists, nil @@ -928,15 +955,15 @@ func (m *Manager) ImportPrivateKey(wif *btcutil.WIF, bs *BlockStamp) (ManagedPub // Save the new imported address to the db and update start block (if // needed) in a single transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutImportedAddress(pubKeyHash, ImportedAddrAccount, + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putImportedAddress(tx, pubKeyHash, ImportedAddrAccount, ssNone, encryptedPubKey, encryptedPrivKey) if err != nil { return err } if updateStartBlock { - return tx.PutStartBlock(bs) + return putStartBlock(tx, bs) } return nil @@ -1035,21 +1062,21 @@ func (m *Manager) ImportScript(script []byte, bs *BlockStamp) (ManagedScriptAddr // Save the new imported address to the db and update start block (if // needed) in a single transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutScriptAddress(scriptHash, ImportedAddrAccount, + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putScriptAddress(tx, scriptHash, ImportedAddrAccount, ssNone, encryptedHash, encryptedScript) if err != nil { return err } if updateStartBlock { - return tx.PutStartBlock(bs) + return putStartBlock(tx, bs) } return nil }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Now that the database has been updated, update the start block in @@ -1309,12 +1336,12 @@ func (m *Manager) nextAddresses(account uint32, numAddresses uint32, internal bo // Now that all addresses have been successfully generated, update the // database in a single transaction. - err = m.db.Update(func(tx *managerTx) error { + err = m.namespace.Update(func(tx walletdb.Tx) error { for _, info := range addressInfo { ma := info.managedAddr addressID := ma.Address().ScriptAddress() - err := tx.PutChainedAddress(addressID, account, - ssFull, info.branch, info.index) + err := putChainedAddress(tx, addressID, account, ssFull, + info.branch, info.index) if err != nil { return err } @@ -1323,7 +1350,7 @@ func (m *Manager) nextAddresses(account uint32, numAddresses uint32, internal bo return nil }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Finally update the next address tracking and add the addresses to the @@ -1450,13 +1477,13 @@ func (m *Manager) AllActiveAddresses() ([]btcutil.Address, error) { // Load the raw address information from the database. var rowInterfaces []interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterfaces, err = tx.FetchAllAddresses() + rowInterfaces, err = fetchAllAddresses(tx) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } addrs := make([]btcutil.Address, 0, len(rowInterfaces)) @@ -1481,7 +1508,7 @@ func (m *Manager) AllActiveAddresses() ([]btcutil.Address, error) { // This function MUST be called with the manager lock held for reads. func (m *Manager) selectCryptoKey(keyType CryptoKeyType) (EncryptorDecryptor, error) { if keyType == CKTPrivate || keyType == CKTScript { - // The manager must be unlocked to encrypt with the private keys. + // The manager must be unlocked to work with the private keys. if m.locked || m.watchingOnly { return nil, managerError(ErrLocked, errLocked, nil) } @@ -1542,13 +1569,14 @@ func (m *Manager) Decrypt(keyType CryptoKeyType, in []byte) ([]byte, error) { } // newManager returns a new locked address manager with the given parameters. -func newManager(db *managerDB, net *btcnet.Params, masterKeyPub *snacl.SecretKey, - masterKeyPriv *snacl.SecretKey, cryptoKeyPub EncryptorDecryptor, - cryptoKeyPrivEncrypted, cryptoKeyScriptEncrypted []byte, - syncInfo *syncState, config *Options) *Manager { +func newManager(namespace walletdb.Namespace, net *btcnet.Params, + masterKeyPub *snacl.SecretKey, masterKeyPriv *snacl.SecretKey, + cryptoKeyPub EncryptorDecryptor, cryptoKeyPrivEncrypted, + cryptoKeyScriptEncrypted []byte, syncInfo *syncState, + config *Options) *Manager { return &Manager{ - db: db, + namespace: namespace, net: net, addrs: make(map[addrKey]ManagedAddress), syncState: *syncInfo, @@ -1565,16 +1593,6 @@ func newManager(db *managerDB, net *btcnet.Params, masterKeyPub *snacl.SecretKey } } -// filesExists reports whether the named file or directory exists. -func fileExists(name string) bool { - if _, err := os.Stat(name); err != nil { - if os.IsNotExist(err) { - return false - } - } - return true -} - // deriveAccountKey derives the extended key for an account according to the // hierarchy described by BIP0044 given the master node. // @@ -1642,7 +1660,7 @@ func checkBranchKeys(acctKey *hdkeychain.ExtendedKey) error { // loadManager returns a new address manager that results from loading it from // the passed opened database. The public passphrase is required to decrypt the // public keys. -func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { +func loadManager(namespace walletdb.Namespace, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { // Perform all database lookups in a read-only view. var watchingOnly bool var masterKeyPubParams, masterKeyPrivParams []byte @@ -1650,43 +1668,43 @@ func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config var syncedTo, startBlock *BlockStamp var recentHeight int32 var recentHashes []btcwire.ShaHash - err := db.View(func(tx *managerTx) error { + err := namespace.View(func(tx walletdb.Tx) error { // Load whether or not the manager is watching-only from the db. var err error - watchingOnly, err = tx.FetchWatchingOnly() + watchingOnly, err = fetchWatchingOnly(tx) if err != nil { return err } // Load the master key params from the db. masterKeyPubParams, masterKeyPrivParams, err = - tx.FetchMasterKeyParams() + fetchMasterKeyParams(tx) if err != nil { return err } // Load the crypto keys from the db. cryptoKeyPubEnc, cryptoKeyPrivEnc, cryptoKeyScriptEnc, err = - tx.FetchCryptoKeys() + fetchCryptoKeys(tx) if err != nil { return err } // Load the sync state from the db. - syncedTo, err = tx.FetchSyncedTo() + syncedTo, err = fetchSyncedTo(tx) if err != nil { return err } - startBlock, err = tx.FetchStartBlock() + startBlock, err = fetchStartBlock(tx) if err != nil { return err } - recentHeight, recentHashes, err = tx.FetchRecentBlocks() + recentHeight, recentHashes, err = fetchRecentBlocks(tx) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // When not a watching-only manager, set the master private key params, @@ -1728,31 +1746,38 @@ func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config // Create new address manager with the given parameters. Also, override // the defaults for the additional fields which are not specified in the // call to new with the values loaded from the database. - mgr := newManager(db, net, &masterKeyPub, &masterKeyPriv, cryptoKeyPub, - cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, config) + mgr := newManager(namespace, net, &masterKeyPub, &masterKeyPriv, + cryptoKeyPub, cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, + config) mgr.watchingOnly = watchingOnly return mgr, nil } -// Open loads an existing address manager from the given database path. The -// public passphrase is required to decrypt the public keys used to protect the -// public information such as addresses. This is important since access to -// BIP0032 extended keys means it is possible to generate all future addresses. +// Open loads an existing address manager from the given namespace. The public +// passphrase is required to decrypt the public keys used to protect the public +// information such as addresses. This is important since access to BIP0032 +// extended keys means it is possible to generate all future addresses. // // If a config structure is passed to the function, that configuration // will override the defaults. // // A ManagerError with an error code of ErrNoExist will be returned if the -// passed database does not exist. -func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { - // Return an error if the specified database does not exist. - if !fileExists(dbPath) { +// passed manager does not exist in the specified namespace. +func Open(namespace walletdb.Namespace, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { + // Return an error if the manager has NOT already been created in the + // given database namespace. + exists, err := managerExists(namespace) + if err != nil { + return nil, err + } + if !exists { str := "the specified address manager does not exist" return nil, managerError(ErrNoExist, str, nil) } - db, err := openOrCreateDB(dbPath) - if err != nil { + // Upgrade the manager to the latest version as needed. This includes + // the initial creation. + if err := upgradeManager(namespace); err != nil { return nil, err } @@ -1760,10 +1785,10 @@ func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Optio config = defaultConfig } - return loadManager(db, pubPassphrase, net, config) + return loadManager(namespace, pubPassphrase, net, config) } -// Create returns a new locked address manager at the given database path. The +// Create returns a new locked address manager in the given namespace. The // seed must conform to the standards described in hdkeychain.NewMaster and will // be used to create the master root node from which all hierarchical // deterministic addresses are derived. This allows all chained addresses in @@ -1778,16 +1803,22 @@ func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Optio // If a config structure is passed to the function, that configuration // will override the defaults. // -// A ManagerError with an error code of ErrAlreadyExists will be returned if the -// passed database already exists. -func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { - // Return an error if the specified database already exists. - if fileExists(dbPath) { +// A ManagerError with an error code of ErrAlreadyExists will be returned the +// address manager already exists in the specified namespace. +func Create(namespace walletdb.Namespace, seed, pubPassphrase, privPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { + // Return an error if the manager has already been created in the given + // database namespace. + exists, err := managerExists(namespace) + if err != nil { + return nil, err + } + if exists { return nil, managerError(ErrAlreadyExists, errAlreadyExists, nil) } - db, err := openOrCreateDB(dbPath) - if err != nil { + // Upgrade the manager to the latest version as needed. This includes + // the initial creation. + if err := upgradeManager(namespace); err != nil { return nil, err } @@ -1911,17 +1942,17 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn syncInfo := newSyncState(createdAt, createdAt, recentHeight, recentHashes) // Perform all database updates in a single transaction. - err = db.Update(func(tx *managerTx) error { + err = namespace.Update(func(tx walletdb.Tx) error { // Save the master key params to the database. pubParams := masterKeyPub.Marshal() privParams := masterKeyPriv.Marshal() - err = tx.PutMasterKeyParams(pubParams, privParams) + err = putMasterKeyParams(tx, pubParams, privParams) if err != nil { return err } // Save the encrypted crypto keys to the database. - err = tx.PutCryptoKeys(cryptoKeyPubEnc, cryptoKeyPrivEnc, + err = putCryptoKeys(tx, cryptoKeyPubEnc, cryptoKeyPrivEnc, cryptoKeyScriptEnc) if err != nil { return err @@ -1929,38 +1960,38 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn // Save the fact this is not a watching-only address manager to // the database. - err = tx.PutWatchingOnly(false) + err = putWatchingOnly(tx, false) if err != nil { return err } // Save the initial synced to state. - err = tx.PutSyncedTo(&syncInfo.syncedTo) + err = putSyncedTo(tx, &syncInfo.syncedTo) if err != nil { return err } - err = tx.PutStartBlock(&syncInfo.startBlock) + err = putStartBlock(tx, &syncInfo.startBlock) if err != nil { return err } // Save the initial recent blocks state. - err = tx.PutRecentBlocks(recentHeight, recentHashes) + err = putRecentBlocks(tx, recentHeight, recentHashes) if err != nil { return err } // Save the information for the default account to the database. - err = tx.PutAccountInfo(defaultAccountNum, acctPubEnc, + err = putAccountInfo(tx, defaultAccountNum, acctPubEnc, acctPrivEnc, 0, 0, "") if err != nil { return err } - return tx.PutNumAccounts(1) + return putNumAccounts(tx, 1) }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // The new address manager is locked by default, so clear the master, @@ -1968,6 +1999,7 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn masterKeyPriv.Zero() cryptoKeyPriv.Zero() cryptoKeyScript.Zero() - return newManager(db, net, masterKeyPub, masterKeyPriv, cryptoKeyPub, - cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, config), nil + return newManager(namespace, net, masterKeyPub, masterKeyPriv, + cryptoKeyPub, cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, + config), nil } diff --git a/waddrmgr/manager_test.go b/waddrmgr/manager_test.go index c8b5f28..0fa9ca5 100644 --- a/waddrmgr/manager_test.go +++ b/waddrmgr/manager_test.go @@ -19,7 +19,6 @@ package waddrmgr_test import ( "encoding/hex" "fmt" - "io/ioutil" "os" "reflect" "testing" @@ -27,6 +26,7 @@ import ( "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwallet/waddrmgr" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -52,6 +52,7 @@ func newShaHash(hexStr string) *btcwire.ShaHash { // spent. type testContext struct { t *testing.T + db walletdb.DB manager *waddrmgr.Manager account uint32 create bool @@ -84,12 +85,6 @@ func testNamePrefix(tc *testContext) string { return prefix + fmt.Sprintf("account #%d", tc.account) } -var fastScrypt = &waddrmgr.Options{ - ScryptN: 16, - ScryptR: 8, - ScryptP: 1, -} - // testManagedPubKeyAddress ensures the data returned by all exported functions // provided by the passed managed p ublic key address matches the corresponding // fields in the provided expected address. @@ -1134,32 +1129,52 @@ func testManagerAPI(tc *testContext) { testChangePassphrase(tc) } -// testExportWatchingOnly tests various facets of a watching-only address -// manager such as running the full set of API tests against a newly exported -// copy as well as when it is opened. -func testExportWatchingOnly(tc *testContext) bool { - // Export the manager as watching-only. +// testWatchingOnly tests various facets of a watching-only address +// manager such as running the full set of API tests against a newly converted +// copy as well as when it is opened from an existing namespace. +func testWatchingOnly(tc *testContext) bool { + // Make a copy of the current database so the copy can be converted to + // watching only. woMgrName := "mgrtestwo.bin" _ = os.Remove(woMgrName) - mgr, err := tc.manager.ExportWatchingOnly(woMgrName, pubPassphrase) + fi, err := os.OpenFile(woMgrName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { - tc.t.Errorf("ExportWatchingOnly: unexpected error: %v", err) + tc.t.Errorf("%v", err) return false } + if err := tc.db.Copy(fi); err != nil { + fi.Close() + tc.t.Errorf("%v", err) + return false + } + fi.Close() defer os.Remove(woMgrName) - // NOTE: Not using deferred close here since part of the tests is - // explicitly closing the manager and then opening the existing one. - // Exporting to an existing manager should fail. - _, err = tc.manager.ExportWatchingOnly(woMgrName, pubPassphrase) - if !checkManagerError(tc.t, "Export watching-only", err, waddrmgr.ErrAlreadyExists) { - mgr.Close() + // Open the new database copy and get the address manager namespace. + db, namespace, err := openDbNamespace(woMgrName) + if err != nil { + tc.t.Errorf("openDbNamespace: unexpected error: %v", err) + return false + } + defer db.Close() + + // Open the manager using the namespace and convert it to watching-only. + mgr, err := waddrmgr.Open(namespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) + if err != nil { + tc.t.Errorf("%v", err) + return false + } + if err := mgr.ConvertToWatchingOnly(pubPassphrase); err != nil { + tc.t.Errorf("%v", err) return false } - // Run all of the manager API tests and close the manager. + // Run all of the manager API tests against the converted manager and + // close it. testManagerAPI(&testContext{ t: tc.t, + db: db, manager: mgr, account: 0, create: false, @@ -1168,7 +1183,8 @@ func testExportWatchingOnly(tc *testContext) bool { mgr.Close() // Open the watching-only manager and run all the tests again. - mgr, err = waddrmgr.Open(woMgrName, pubPassphrase, &btcnet.MainNetParams, fastScrypt) + mgr, err = waddrmgr.Open(namespace, pubPassphrase, &btcnet.MainNetParams, + fastScrypt) if err != nil { tc.t.Errorf("Open Watching-Only: unexpected error: %v", err) return false @@ -1177,6 +1193,7 @@ func testExportWatchingOnly(tc *testContext) bool { testManagerAPI(&testContext{ t: tc.t, + db: db, manager: mgr, account: 0, create: false, @@ -1436,31 +1453,39 @@ func testSync(tc *testContext) bool { // It makes use of a test context because the address manager is persistent and // much of the testing involves having specific state. func TestManager(t *testing.T) { + dbName := "mgrtest.bin" + _ = os.Remove(dbName) + db, mgrNamespace, err := createDbNamespace(dbName) + if err != nil { + t.Errorf("createDbNamespace: unexpected error: %v", err) + return + } + defer os.Remove(dbName) + defer db.Close() + // Open manager that does not exist to ensure the expected error is // returned. - mgrName := "mgrtest.bin" - _ = os.Remove(mgrName) - _, err := waddrmgr.Open(mgrName, pubPassphrase, &btcnet.MainNetParams, - fastScrypt) + _, err = waddrmgr.Open(mgrNamespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) if !checkManagerError(t, "Open non-existant", err, waddrmgr.ErrNoExist) { return } // Create a new manager. - mgr, err := waddrmgr.Create(mgrName, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) + mgr, err := waddrmgr.Create(mgrNamespace, seed, pubPassphrase, + privPassphrase, &btcnet.MainNetParams, fastScrypt) if err != nil { t.Errorf("Create: unexpected error: %v", err) return } - defer os.Remove(mgrName) + // NOTE: Not using deferred close here since part of the tests is // explicitly closing the manager and then opening the existing one. // Attempt to create the manager again to ensure the expected error is // returned. - _, err = waddrmgr.Create(mgrName, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) + _, err = waddrmgr.Create(mgrNamespace, seed, pubPassphrase, + privPassphrase, &btcnet.MainNetParams, fastScrypt) if !checkManagerError(t, "Create existing", err, waddrmgr.ErrAlreadyExists) { mgr.Close() return @@ -1470,6 +1495,7 @@ func TestManager(t *testing.T) { // manager after they've completed testManagerAPI(&testContext{ t: t, + db: db, manager: mgr, account: 0, create: true, @@ -1479,8 +1505,8 @@ func TestManager(t *testing.T) { // Open the manager and run all the tests again in open mode which // avoids reinserting new addresses like the create mode tests do. - mgr, err = waddrmgr.Open(mgrName, pubPassphrase, &btcnet.MainNetParams, - fastScrypt) + mgr, err = waddrmgr.Open(mgrNamespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) if err != nil { t.Errorf("Open: unexpected error: %v", err) return @@ -1489,6 +1515,7 @@ func TestManager(t *testing.T) { tc := &testContext{ t: t, + db: db, manager: mgr, account: 0, create: false, @@ -1498,7 +1525,7 @@ func TestManager(t *testing.T) { // Now that the address manager has been tested in both the newly // created and opened modes, test a watching-only version. - testExportWatchingOnly(tc) + testWatchingOnly(tc) // Ensure that the manager sync state functionality works as expected. testSync(tc) @@ -1510,31 +1537,6 @@ func TestManager(t *testing.T) { } } -// setupManager creates a new address manager and returns a teardown function -// that should be invoked to ensure it is closed and removed upon completion. -func setupManager(t *testing.T) (tearDownFunc func(), mgr *waddrmgr.Manager) { - t.Parallel() - - // Create a new manager. - // We create the file and immediately delete it, as the waddrmgr - // needs to be doing the creating. - file, err := ioutil.TempDir("", "mgrtest") - if err != nil { - t.Fatalf("Failed to create db file: %v", err) - } - _ = os.Remove(file) - mgr, err = waddrmgr.Create(file, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) - if err != nil { - t.Fatalf("Failed to create Manager: %v", err) - } - tearDownFunc = func() { - mgr.Close() - os.Remove(file) - } - return tearDownFunc, mgr -} - // TestEncryptDecryptErrors ensures that errors which occur while encrypting and // decrypting data return the expected errors. func TestEncryptDecryptErrors(t *testing.T) { diff --git a/waddrmgr/sync.go b/waddrmgr/sync.go index 0f1234b..0e56318 100644 --- a/waddrmgr/sync.go +++ b/waddrmgr/sync.go @@ -19,6 +19,7 @@ package waddrmgr import ( "sync" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -210,13 +211,13 @@ func (m *Manager) SetSyncedTo(bs *BlockStamp) error { } // Update the database. - err := m.db.Update(func(tx *managerTx) error { - err := tx.PutSyncedTo(bs) + err := m.namespace.Update(func(tx walletdb.Tx) error { + err := putSyncedTo(tx, bs) if err != nil { return err } - return tx.PutRecentBlocks(recentHeight, recentHashes) + return putRecentBlocks(tx, recentHeight, recentHashes) }) if err != nil { return err