diff --git a/waddrmgr/common_test.go b/waddrmgr/common_test.go index e52acca..59ef2e8 100644 --- a/waddrmgr/common_test.go +++ b/waddrmgr/common_test.go @@ -18,9 +18,15 @@ package waddrmgr_test import ( "encoding/hex" + "io/ioutil" + "os" + "path/filepath" "testing" + "github.com/conformal/btcnet" "github.com/conformal/btcwallet/waddrmgr" + "github.com/conformal/btcwallet/walletdb" + _ "github.com/conformal/btcwallet/walletdb/bdb" ) var ( @@ -36,6 +42,14 @@ var ( privPassphrase = []byte("81lUHXnOMZ@?XXd7O9xyDIWIbXX-lj") pubPassphrase2 = []byte("-0NV4P~VSJBWbunw}% @@ -1035,9 +1053,9 @@ func (mtx *managerTx) FetchSyncedTo() (*BlockStamp, error) { return &bs, nil } -// PutSyncedTo stores the provided synced to blockstamp to the database. -func (mtx *managerTx) PutSyncedTo(bs *BlockStamp) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putSyncedTo stores the provided synced to blockstamp to the database. +func putSyncedTo(tx walletdb.Tx, bs *BlockStamp) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized synced to format is: // @@ -1055,10 +1073,10 @@ func (mtx *managerTx) PutSyncedTo(bs *BlockStamp) error { return nil } -// FetchStartBlock loads the start block stamp for the manager from the +// fetchStartBlock loads the start block stamp for the manager from the // database. -func (mtx *managerTx) FetchStartBlock() (*BlockStamp, error) { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +func fetchStartBlock(tx walletdb.Tx) (*BlockStamp, error) { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized start block format is: // @@ -1076,9 +1094,9 @@ func (mtx *managerTx) FetchStartBlock() (*BlockStamp, error) { return &bs, nil } -// PutStartBlock stores the provided start block stamp to the database. -func (mtx *managerTx) PutStartBlock(bs *BlockStamp) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putStartBlock stores the provided start block stamp to the database. +func putStartBlock(tx walletdb.Tx, bs *BlockStamp) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized start block format is: // @@ -1096,10 +1114,10 @@ func (mtx *managerTx) PutStartBlock(bs *BlockStamp) error { return nil } -// FetchRecentBlocks returns the height of the most recent block height and +// fetchRecentBlocks returns the height of the most recent block height and // hashes of the most recent blocks. -func (mtx *managerTx) FetchRecentBlocks() (int32, []btcwire.ShaHash, error) { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +func fetchRecentBlocks(tx walletdb.Tx) (int32, []btcwire.ShaHash, error) { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized recent blocks format is: // @@ -1127,9 +1145,9 @@ func (mtx *managerTx) FetchRecentBlocks() (int32, []btcwire.ShaHash, error) { return recentHeight, recentHashes, nil } -// PutStartBlock stores the provided start block stamp to the database. -func (mtx *managerTx) PutRecentBlocks(recentHeight int32, recentHashes []btcwire.ShaHash) error { - bucket := (*bolt.Tx)(mtx).Bucket(syncBucketName) +// putRecentBlocks stores the provided start block stamp to the database. +func putRecentBlocks(tx walletdb.Tx, recentHeight int32, recentHashes []btcwire.ShaHash) error { + bucket := tx.RootBucket().Bucket(syncBucketName) // The serialized recent blocks format is: // @@ -1154,166 +1172,71 @@ func (mtx *managerTx) PutRecentBlocks(recentHeight int32, recentHashes []btcwire return nil } -// managerDB provides transactional facilities to read and write the address -// manager data to a bolt database. -type managerDB struct { - db *bolt.DB - version uint32 - created time.Time -} - -// Close releases all database resources. All transactions must be closed -// before closing the database. -func (db *managerDB) Close() error { - if err := db.db.Close(); err != nil { - str := "failed to close database" - return managerError(ErrDatabase, str, err) - } - - return nil -} - -// View executes the passed function within the context of a managed read-only -// transaction. Any error that is returned from the passed function is returned -// from this function. -func (db *managerDB) View(fn func(tx *managerTx) error) error { - err := db.db.View(func(tx *bolt.Tx) error { - return fn((*managerTx)(tx)) - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database read transaction" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// Update executes the passed function within the context of a read-write -// managed transaction. The transaction is committed if no error is returned -// from the function. On the other hand, the entire transaction is rolled back -// if an error is returned. Any error that is returned from the passed function -// or returned from the commit is returned from this function. -func (db *managerDB) Update(fn func(tx *managerTx) error) error { - err := db.db.Update(func(tx *bolt.Tx) error { - return fn((*managerTx)(tx)) - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database write transaction" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// CopyDB copies the entire database to the provided new database path. A -// reader transaction is maintained during the copy so it is safe to continue -// using the database while a copy is in progress. -func (db *managerDB) CopyDB(newDbPath string) error { - err := db.db.View(func(tx *bolt.Tx) error { - if err := tx.CopyFile(newDbPath, 0600); err != nil { - str := "failed to copy database" - return managerError(ErrDatabase, str, err) - } - +// managerExists returns whether or not the manager has already been created +// in the given database namespace. +func managerExists(namespace walletdb.Namespace) (bool, error) { + var exists bool + err := namespace.View(func(tx walletdb.Tx) error { + mainBucket := tx.RootBucket().Bucket(mainBucketName) + exists = mainBucket != nil return nil }) if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database copy" - return managerError(ErrDatabase, str, err) - } - return err + str := fmt.Sprintf("failed to obtain database view: %v", err) + return false, managerError(ErrDatabase, str, err) } - - return nil + return exists, nil } -// WriteTo writes the entire database to the provided writer. A reader -// transaction is maintained during the copy so it is safe to continue using the -// database while a copy is in progress. -func (db *managerDB) WriteTo(w io.Writer) error { - err := db.db.View(func(tx *bolt.Tx) error { - if err := tx.Copy(w); err != nil { - str := "failed to copy database" - return managerError(ErrDatabase, str, err) - } - - return nil - }) - if err != nil { - // Ensure the returned error is a ManagerError. - if _, ok := err.(ManagerError); !ok { - str := "failed during database copy" - return managerError(ErrDatabase, str, err) - } - return err - } - - return nil -} - -// openOrCreateDB opens the database at the provided path or creates and +// upgradeManager opens the manager using the specified namespace or creates and // initializes it if it does not already exist. It also provides facilities to -// upgrade the database to newer versions. -func openOrCreateDB(dbPath string) (*managerDB, error) { - db, err := bolt.Open(dbPath, 0600, nil) - if err != nil { - str := "failed to open database" - return nil, managerError(ErrDatabase, str, err) - } - +// upgrade the data in the namespace to newer versions. +func upgradeManager(namespace walletdb.Namespace) error { // Initialize the buckets and main db fields as needed. var version uint32 var createDate uint64 - err = db.Update(func(tx *bolt.Tx) error { - mainBucket, err := tx.CreateBucketIfNotExists(mainBucketName) + err := namespace.Update(func(tx walletdb.Tx) error { + rootBucket := tx.RootBucket() + mainBucket, err := rootBucket.CreateBucketIfNotExists( + mainBucketName) if err != nil { str := "failed to create main bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(addrBucketName) + _, err = rootBucket.CreateBucketIfNotExists(addrBucketName) if err != nil { str := "failed to create address bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(acctBucketName) + _, err = rootBucket.CreateBucketIfNotExists(acctBucketName) if err != nil { str := "failed to create account bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(addrAcctIdxBucketName) + _, err = rootBucket.CreateBucketIfNotExists(addrAcctIdxBucketName) if err != nil { str := "failed to create address index bucket" return managerError(ErrDatabase, str, err) } - _, err = tx.CreateBucketIfNotExists(syncBucketName) + _, err = rootBucket.CreateBucketIfNotExists(syncBucketName) if err != nil { str := "failed to create sync bucket" return managerError(ErrDatabase, str, err) } - // Save the most recent database version if it isn't already + // Save the most recent manager version if it isn't already // there, otherwise keep track of it for potential upgrades. - verBytes := mainBucket.Get(dbVersionName) + verBytes := mainBucket.Get(mgrVersionName) if verBytes == nil { - version = LatestDbVersion + version = LatestMgrVersion var buf [4]byte binary.LittleEndian.PutUint32(buf[:], version) - err := mainBucket.Put(dbVersionName, buf[:]) + err := mainBucket.Put(mgrVersionName, buf[:]) if err != nil { str := "failed to store latest database version" return managerError(ErrDatabase, str, err) @@ -1322,12 +1245,12 @@ func openOrCreateDB(dbPath string) (*managerDB, error) { version = binary.LittleEndian.Uint32(verBytes) } - createBytes := mainBucket.Get(dbCreateDateName) + createBytes := mainBucket.Get(mgrCreateDateName) if createBytes == nil { createDate = uint64(time.Now().Unix()) var buf [8]byte binary.LittleEndian.PutUint64(buf[:], createDate) - err := mainBucket.Put(dbCreateDateName, buf[:]) + err := mainBucket.Put(mgrCreateDateName, buf[:]) if err != nil { str := "failed to store database creation time" return managerError(ErrDatabase, str, err) @@ -1340,17 +1263,13 @@ func openOrCreateDB(dbPath string) (*managerDB, error) { }) if err != nil { str := "failed to update database" - return nil, managerError(ErrDatabase, str, err) + return managerError(ErrDatabase, str, err) } - // Upgrade the database as needed. - if version < LatestDbVersion { + // Upgrade the manager as needed. + if version < LatestMgrVersion { // No upgrades yet. } - return &managerDB{ - db: db, - version: version, - created: time.Unix(int64(createDate), 0), - }, nil + return nil } diff --git a/waddrmgr/error.go b/waddrmgr/error.go index c2fbba4..3f6efa9 100644 --- a/waddrmgr/error.go +++ b/waddrmgr/error.go @@ -74,10 +74,10 @@ const ( // key type has been selected. ErrInvalidKeyType - // ErrNoExist indicates the specified database does not exist. + // ErrNoExist indicates the manager does not exist. ErrNoExist - // ErrAlreadyExists indicates the specified database already exists. + // ErrAlreadyExists indicates the specified manager already exists. ErrAlreadyExists // ErrCoinTypeTooHigh indicates the coin type specified in the provided diff --git a/waddrmgr/manager.go b/waddrmgr/manager.go index fccb68c..71f8214 100644 --- a/waddrmgr/manager.go +++ b/waddrmgr/manager.go @@ -18,8 +18,6 @@ package waddrmgr import ( "fmt" - "io" - "os" "sync" "github.com/conformal/btcec" @@ -27,6 +25,7 @@ import ( "github.com/conformal/btcutil" "github.com/conformal/btcutil/hdkeychain" "github.com/conformal/btcwallet/snacl" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -204,7 +203,7 @@ var newCryptoKey = defaultNewCryptoKey type Manager struct { mtx sync.RWMutex - db *managerDB + namespace walletdb.Namespace net *btcnet.Params addrs map[addrKey]ManagedAddress syncState syncState @@ -309,9 +308,9 @@ func (m *Manager) zeroSensitivePublicData() { m.masterKeyPub.Zero() } -// Close cleanly shuts down the underlying database and syncs all data. It also -// makes a best try effort to remove and zero all private key and sensitive -// public key material associated with the address manager. +// Close cleanly shuts down the manager. It makes a best try effort to remove +// and zero all private key and sensitive public key material associated with +// the address manager from memory. func (m *Manager) Close() error { m.mtx.Lock() defer m.mtx.Unlock() @@ -324,10 +323,6 @@ func (m *Manager) Close() error { // Attempt to clear sensitive public key material from memory too. m.zeroSensitivePublicData() - if err := m.db.Close(); err != nil { - return err - } - m.closed = true return nil } @@ -408,13 +403,13 @@ func (m *Manager) loadAccountInfo(account uint32) (*accountInfo, error) { // The account is either invalid or just wasn't cached, so attempt to // load the information from the database. var rowInterface interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterface, err = tx.FetchAccountInfo(account) + rowInterface, err = fetchAccountInfo(tx, account) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Ensure the account type is a BIP0044 account. @@ -596,13 +591,13 @@ func (m *Manager) rowInterfaceToManaged(rowInterface interface{}) (ManagedAddres func (m *Manager) loadAndCacheAddress(address btcutil.Address) (ManagedAddress, error) { // Attempt to load the raw address information from the database. var rowInterface interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterface, err = tx.FetchAddress(address.ScriptAddress()) + rowInterface, err = fetchAddress(tx, address.ScriptAddress()) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Create a new managed address for the specific type of address based @@ -732,16 +727,16 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private // Save the new keys and params to the the db in a single // transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutCryptoKeys(nil, encPriv, encScript) + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putCryptoKeys(tx, nil, encPriv, encScript) if err != nil { return err } - return tx.PutMasterKeyParams(nil, newKeyParams) + return putMasterKeyParams(tx, nil, newKeyParams) }) if err != nil { - return err + return maybeConvertDbError(err) } // Now that the db has been successfully updated, clear the old @@ -761,16 +756,16 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private // Save the new keys and params to the the db in a single // transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutCryptoKeys(encryptedPub, nil, nil) + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putCryptoKeys(tx, encryptedPub, nil, nil) if err != nil { return err } - return tx.PutMasterKeyParams(newKeyParams, nil) + return putMasterKeyParams(tx, newKeyParams, nil) }) if err != nil { - return err + return maybeConvertDbError(err) } // Now that the db has been successfully updated, clear the old @@ -782,53 +777,85 @@ func (m *Manager) ChangePassphrase(oldPassphrase, newPassphrase []byte, private return nil } -// ExportWatchingOnly creates a new watching-only address manager backed by a -// database at the provided path. A watching-only address manager has all -// private keys removed which means it is not possible to create transactions -// which spend funds. -func (m *Manager) ExportWatchingOnly(newDbPath string, pubPassphrase []byte) (*Manager, error) { - m.mtx.RLock() - defer m.mtx.RUnlock() +// ConvertToWatchingOnly converts the current address manager to a locked +// watching-only address manager. +// +// WARNING: This function removes private keys from the existing address manager +// which means they will no longer be available. Typically the caller will make +// a copy of the existing wallet database and modify the copy since otherwise it +// would mean permanent loss of any imported private keys and scripts. +// +// Executing this function on a manager that is already watching-only will have +// no effect. +func (m *Manager) ConvertToWatchingOnly(pubPassphrase []byte) error { + m.mtx.Lock() + defer m.mtx.Unlock() - // Return an error if the specified database already exists. - if fileExists(newDbPath) { - return nil, managerError(ErrAlreadyExists, errAlreadyExists, nil) - } - - // Copy the existing manager database to the provided path. - if err := m.db.CopyDB(newDbPath); err != nil { - return nil, err - } - - // Open the copied database. - watchingDb, err := openOrCreateDB(newDbPath) - if err != nil { - return nil, err + // Exit now if the manager is already watching-only. + if m.watchingOnly { + return nil } // Remove all private key material and mark the new database as watching // only. - err = watchingDb.Update(func(tx *managerTx) error { - if err := tx.DeletePrivateKeys(); err != nil { + err := m.namespace.Update(func(tx walletdb.Tx) error { + if err := deletePrivateKeys(tx); err != nil { return err } - return tx.PutWatchingOnly(true) + return putWatchingOnly(tx, true) }) if err != nil { - return nil, err + return maybeConvertDbError(err) } - return loadManager(watchingDb, pubPassphrase, m.net, m.config) -} + // Lock the manager to remove all clear text private key material from + // memory if needed. + if !m.locked { + m.lock() + } -// Export writes the manager database to the provided writer. -func (m *Manager) Export(w io.Writer) error { - m.mtx.RLock() - defer m.mtx.RUnlock() + // This section clears and removes the encrypted private key material + // that is ordinarily used to unlock the manager. Since the the manager + // is being converted to watching-only, the encrypted private key + // material is no longer needed. + + // Clear and remove all of the encrypted acount private keys. + for _, acctInfo := range m.acctInfo { + zero(acctInfo.acctKeyEncrypted) + acctInfo.acctKeyEncrypted = nil + } + + // Clear and remove encrypted private keys and encrypted scripts from + // all address entries. + for _, ma := range m.addrs { + switch addr := ma.(type) { + case *managedAddress: + zero(addr.privKeyEncrypted) + addr.privKeyEncrypted = nil + case *scriptAddress: + zero(addr.scriptEncrypted) + addr.scriptEncrypted = nil + } + } + + // Clear and remove encrypted private and script crypto keys. + zero(m.cryptoKeyScriptEncrypted) + m.cryptoKeyScriptEncrypted = nil + m.cryptoKeyScript = nil + zero(m.cryptoKeyPrivEncrypted) + m.cryptoKeyPrivEncrypted = nil + m.cryptoKeyPriv = nil + + // The master private key is derived from a passphrase when the manager + // is unlocked, so there is no encrypted version to zero. However, + // it is no longer needed, so nil it. + m.masterKeyPriv = nil + + // Mark the manager watching-only. + m.watchingOnly = true + return nil - // Copy the existing manager database to the provided path. - return m.db.WriteTo(w) } // existsAddress returns whether or not the passed address is known to the @@ -843,12 +870,12 @@ func (m *Manager) existsAddress(addressID []byte) (bool, error) { // Check the database if not already found above. var exists bool - err := m.db.View(func(tx *managerTx) error { - exists = tx.ExistsAddress(addressID) + err := m.namespace.View(func(tx walletdb.Tx) error { + exists = existsAddress(tx, addressID) return nil }) if err != nil { - return false, err + return false, maybeConvertDbError(err) } return exists, nil @@ -928,15 +955,15 @@ func (m *Manager) ImportPrivateKey(wif *btcutil.WIF, bs *BlockStamp) (ManagedPub // Save the new imported address to the db and update start block (if // needed) in a single transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutImportedAddress(pubKeyHash, ImportedAddrAccount, + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putImportedAddress(tx, pubKeyHash, ImportedAddrAccount, ssNone, encryptedPubKey, encryptedPrivKey) if err != nil { return err } if updateStartBlock { - return tx.PutStartBlock(bs) + return putStartBlock(tx, bs) } return nil @@ -1035,21 +1062,21 @@ func (m *Manager) ImportScript(script []byte, bs *BlockStamp) (ManagedScriptAddr // Save the new imported address to the db and update start block (if // needed) in a single transaction. - err = m.db.Update(func(tx *managerTx) error { - err := tx.PutScriptAddress(scriptHash, ImportedAddrAccount, + err = m.namespace.Update(func(tx walletdb.Tx) error { + err := putScriptAddress(tx, scriptHash, ImportedAddrAccount, ssNone, encryptedHash, encryptedScript) if err != nil { return err } if updateStartBlock { - return tx.PutStartBlock(bs) + return putStartBlock(tx, bs) } return nil }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Now that the database has been updated, update the start block in @@ -1309,12 +1336,12 @@ func (m *Manager) nextAddresses(account uint32, numAddresses uint32, internal bo // Now that all addresses have been successfully generated, update the // database in a single transaction. - err = m.db.Update(func(tx *managerTx) error { + err = m.namespace.Update(func(tx walletdb.Tx) error { for _, info := range addressInfo { ma := info.managedAddr addressID := ma.Address().ScriptAddress() - err := tx.PutChainedAddress(addressID, account, - ssFull, info.branch, info.index) + err := putChainedAddress(tx, addressID, account, ssFull, + info.branch, info.index) if err != nil { return err } @@ -1323,7 +1350,7 @@ func (m *Manager) nextAddresses(account uint32, numAddresses uint32, internal bo return nil }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // Finally update the next address tracking and add the addresses to the @@ -1450,13 +1477,13 @@ func (m *Manager) AllActiveAddresses() ([]btcutil.Address, error) { // Load the raw address information from the database. var rowInterfaces []interface{} - err := m.db.View(func(tx *managerTx) error { + err := m.namespace.View(func(tx walletdb.Tx) error { var err error - rowInterfaces, err = tx.FetchAllAddresses() + rowInterfaces, err = fetchAllAddresses(tx) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } addrs := make([]btcutil.Address, 0, len(rowInterfaces)) @@ -1481,7 +1508,7 @@ func (m *Manager) AllActiveAddresses() ([]btcutil.Address, error) { // This function MUST be called with the manager lock held for reads. func (m *Manager) selectCryptoKey(keyType CryptoKeyType) (EncryptorDecryptor, error) { if keyType == CKTPrivate || keyType == CKTScript { - // The manager must be unlocked to encrypt with the private keys. + // The manager must be unlocked to work with the private keys. if m.locked || m.watchingOnly { return nil, managerError(ErrLocked, errLocked, nil) } @@ -1542,13 +1569,14 @@ func (m *Manager) Decrypt(keyType CryptoKeyType, in []byte) ([]byte, error) { } // newManager returns a new locked address manager with the given parameters. -func newManager(db *managerDB, net *btcnet.Params, masterKeyPub *snacl.SecretKey, - masterKeyPriv *snacl.SecretKey, cryptoKeyPub EncryptorDecryptor, - cryptoKeyPrivEncrypted, cryptoKeyScriptEncrypted []byte, - syncInfo *syncState, config *Options) *Manager { +func newManager(namespace walletdb.Namespace, net *btcnet.Params, + masterKeyPub *snacl.SecretKey, masterKeyPriv *snacl.SecretKey, + cryptoKeyPub EncryptorDecryptor, cryptoKeyPrivEncrypted, + cryptoKeyScriptEncrypted []byte, syncInfo *syncState, + config *Options) *Manager { return &Manager{ - db: db, + namespace: namespace, net: net, addrs: make(map[addrKey]ManagedAddress), syncState: *syncInfo, @@ -1565,16 +1593,6 @@ func newManager(db *managerDB, net *btcnet.Params, masterKeyPub *snacl.SecretKey } } -// filesExists reports whether the named file or directory exists. -func fileExists(name string) bool { - if _, err := os.Stat(name); err != nil { - if os.IsNotExist(err) { - return false - } - } - return true -} - // deriveAccountKey derives the extended key for an account according to the // hierarchy described by BIP0044 given the master node. // @@ -1642,7 +1660,7 @@ func checkBranchKeys(acctKey *hdkeychain.ExtendedKey) error { // loadManager returns a new address manager that results from loading it from // the passed opened database. The public passphrase is required to decrypt the // public keys. -func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { +func loadManager(namespace walletdb.Namespace, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { // Perform all database lookups in a read-only view. var watchingOnly bool var masterKeyPubParams, masterKeyPrivParams []byte @@ -1650,43 +1668,43 @@ func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config var syncedTo, startBlock *BlockStamp var recentHeight int32 var recentHashes []btcwire.ShaHash - err := db.View(func(tx *managerTx) error { + err := namespace.View(func(tx walletdb.Tx) error { // Load whether or not the manager is watching-only from the db. var err error - watchingOnly, err = tx.FetchWatchingOnly() + watchingOnly, err = fetchWatchingOnly(tx) if err != nil { return err } // Load the master key params from the db. masterKeyPubParams, masterKeyPrivParams, err = - tx.FetchMasterKeyParams() + fetchMasterKeyParams(tx) if err != nil { return err } // Load the crypto keys from the db. cryptoKeyPubEnc, cryptoKeyPrivEnc, cryptoKeyScriptEnc, err = - tx.FetchCryptoKeys() + fetchCryptoKeys(tx) if err != nil { return err } // Load the sync state from the db. - syncedTo, err = tx.FetchSyncedTo() + syncedTo, err = fetchSyncedTo(tx) if err != nil { return err } - startBlock, err = tx.FetchStartBlock() + startBlock, err = fetchStartBlock(tx) if err != nil { return err } - recentHeight, recentHashes, err = tx.FetchRecentBlocks() + recentHeight, recentHashes, err = fetchRecentBlocks(tx) return err }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // When not a watching-only manager, set the master private key params, @@ -1728,31 +1746,38 @@ func loadManager(db *managerDB, pubPassphrase []byte, net *btcnet.Params, config // Create new address manager with the given parameters. Also, override // the defaults for the additional fields which are not specified in the // call to new with the values loaded from the database. - mgr := newManager(db, net, &masterKeyPub, &masterKeyPriv, cryptoKeyPub, - cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, config) + mgr := newManager(namespace, net, &masterKeyPub, &masterKeyPriv, + cryptoKeyPub, cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, + config) mgr.watchingOnly = watchingOnly return mgr, nil } -// Open loads an existing address manager from the given database path. The -// public passphrase is required to decrypt the public keys used to protect the -// public information such as addresses. This is important since access to -// BIP0032 extended keys means it is possible to generate all future addresses. +// Open loads an existing address manager from the given namespace. The public +// passphrase is required to decrypt the public keys used to protect the public +// information such as addresses. This is important since access to BIP0032 +// extended keys means it is possible to generate all future addresses. // // If a config structure is passed to the function, that configuration // will override the defaults. // // A ManagerError with an error code of ErrNoExist will be returned if the -// passed database does not exist. -func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { - // Return an error if the specified database does not exist. - if !fileExists(dbPath) { +// passed manager does not exist in the specified namespace. +func Open(namespace walletdb.Namespace, pubPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { + // Return an error if the manager has NOT already been created in the + // given database namespace. + exists, err := managerExists(namespace) + if err != nil { + return nil, err + } + if !exists { str := "the specified address manager does not exist" return nil, managerError(ErrNoExist, str, nil) } - db, err := openOrCreateDB(dbPath) - if err != nil { + // Upgrade the manager to the latest version as needed. This includes + // the initial creation. + if err := upgradeManager(namespace); err != nil { return nil, err } @@ -1760,10 +1785,10 @@ func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Optio config = defaultConfig } - return loadManager(db, pubPassphrase, net, config) + return loadManager(namespace, pubPassphrase, net, config) } -// Create returns a new locked address manager at the given database path. The +// Create returns a new locked address manager in the given namespace. The // seed must conform to the standards described in hdkeychain.NewMaster and will // be used to create the master root node from which all hierarchical // deterministic addresses are derived. This allows all chained addresses in @@ -1778,16 +1803,22 @@ func Open(dbPath string, pubPassphrase []byte, net *btcnet.Params, config *Optio // If a config structure is passed to the function, that configuration // will override the defaults. // -// A ManagerError with an error code of ErrAlreadyExists will be returned if the -// passed database already exists. -func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { - // Return an error if the specified database already exists. - if fileExists(dbPath) { +// A ManagerError with an error code of ErrAlreadyExists will be returned the +// address manager already exists in the specified namespace. +func Create(namespace walletdb.Namespace, seed, pubPassphrase, privPassphrase []byte, net *btcnet.Params, config *Options) (*Manager, error) { + // Return an error if the manager has already been created in the given + // database namespace. + exists, err := managerExists(namespace) + if err != nil { + return nil, err + } + if exists { return nil, managerError(ErrAlreadyExists, errAlreadyExists, nil) } - db, err := openOrCreateDB(dbPath) - if err != nil { + // Upgrade the manager to the latest version as needed. This includes + // the initial creation. + if err := upgradeManager(namespace); err != nil { return nil, err } @@ -1911,17 +1942,17 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn syncInfo := newSyncState(createdAt, createdAt, recentHeight, recentHashes) // Perform all database updates in a single transaction. - err = db.Update(func(tx *managerTx) error { + err = namespace.Update(func(tx walletdb.Tx) error { // Save the master key params to the database. pubParams := masterKeyPub.Marshal() privParams := masterKeyPriv.Marshal() - err = tx.PutMasterKeyParams(pubParams, privParams) + err = putMasterKeyParams(tx, pubParams, privParams) if err != nil { return err } // Save the encrypted crypto keys to the database. - err = tx.PutCryptoKeys(cryptoKeyPubEnc, cryptoKeyPrivEnc, + err = putCryptoKeys(tx, cryptoKeyPubEnc, cryptoKeyPrivEnc, cryptoKeyScriptEnc) if err != nil { return err @@ -1929,38 +1960,38 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn // Save the fact this is not a watching-only address manager to // the database. - err = tx.PutWatchingOnly(false) + err = putWatchingOnly(tx, false) if err != nil { return err } // Save the initial synced to state. - err = tx.PutSyncedTo(&syncInfo.syncedTo) + err = putSyncedTo(tx, &syncInfo.syncedTo) if err != nil { return err } - err = tx.PutStartBlock(&syncInfo.startBlock) + err = putStartBlock(tx, &syncInfo.startBlock) if err != nil { return err } // Save the initial recent blocks state. - err = tx.PutRecentBlocks(recentHeight, recentHashes) + err = putRecentBlocks(tx, recentHeight, recentHashes) if err != nil { return err } // Save the information for the default account to the database. - err = tx.PutAccountInfo(defaultAccountNum, acctPubEnc, + err = putAccountInfo(tx, defaultAccountNum, acctPubEnc, acctPrivEnc, 0, 0, "") if err != nil { return err } - return tx.PutNumAccounts(1) + return putNumAccounts(tx, 1) }) if err != nil { - return nil, err + return nil, maybeConvertDbError(err) } // The new address manager is locked by default, so clear the master, @@ -1968,6 +1999,7 @@ func Create(dbPath string, seed, pubPassphrase, privPassphrase []byte, net *btcn masterKeyPriv.Zero() cryptoKeyPriv.Zero() cryptoKeyScript.Zero() - return newManager(db, net, masterKeyPub, masterKeyPriv, cryptoKeyPub, - cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, config), nil + return newManager(namespace, net, masterKeyPub, masterKeyPriv, + cryptoKeyPub, cryptoKeyPrivEnc, cryptoKeyScriptEnc, syncInfo, + config), nil } diff --git a/waddrmgr/manager_test.go b/waddrmgr/manager_test.go index c8b5f28..0fa9ca5 100644 --- a/waddrmgr/manager_test.go +++ b/waddrmgr/manager_test.go @@ -19,7 +19,6 @@ package waddrmgr_test import ( "encoding/hex" "fmt" - "io/ioutil" "os" "reflect" "testing" @@ -27,6 +26,7 @@ import ( "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwallet/waddrmgr" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -52,6 +52,7 @@ func newShaHash(hexStr string) *btcwire.ShaHash { // spent. type testContext struct { t *testing.T + db walletdb.DB manager *waddrmgr.Manager account uint32 create bool @@ -84,12 +85,6 @@ func testNamePrefix(tc *testContext) string { return prefix + fmt.Sprintf("account #%d", tc.account) } -var fastScrypt = &waddrmgr.Options{ - ScryptN: 16, - ScryptR: 8, - ScryptP: 1, -} - // testManagedPubKeyAddress ensures the data returned by all exported functions // provided by the passed managed p ublic key address matches the corresponding // fields in the provided expected address. @@ -1134,32 +1129,52 @@ func testManagerAPI(tc *testContext) { testChangePassphrase(tc) } -// testExportWatchingOnly tests various facets of a watching-only address -// manager such as running the full set of API tests against a newly exported -// copy as well as when it is opened. -func testExportWatchingOnly(tc *testContext) bool { - // Export the manager as watching-only. +// testWatchingOnly tests various facets of a watching-only address +// manager such as running the full set of API tests against a newly converted +// copy as well as when it is opened from an existing namespace. +func testWatchingOnly(tc *testContext) bool { + // Make a copy of the current database so the copy can be converted to + // watching only. woMgrName := "mgrtestwo.bin" _ = os.Remove(woMgrName) - mgr, err := tc.manager.ExportWatchingOnly(woMgrName, pubPassphrase) + fi, err := os.OpenFile(woMgrName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { - tc.t.Errorf("ExportWatchingOnly: unexpected error: %v", err) + tc.t.Errorf("%v", err) return false } + if err := tc.db.Copy(fi); err != nil { + fi.Close() + tc.t.Errorf("%v", err) + return false + } + fi.Close() defer os.Remove(woMgrName) - // NOTE: Not using deferred close here since part of the tests is - // explicitly closing the manager and then opening the existing one. - // Exporting to an existing manager should fail. - _, err = tc.manager.ExportWatchingOnly(woMgrName, pubPassphrase) - if !checkManagerError(tc.t, "Export watching-only", err, waddrmgr.ErrAlreadyExists) { - mgr.Close() + // Open the new database copy and get the address manager namespace. + db, namespace, err := openDbNamespace(woMgrName) + if err != nil { + tc.t.Errorf("openDbNamespace: unexpected error: %v", err) + return false + } + defer db.Close() + + // Open the manager using the namespace and convert it to watching-only. + mgr, err := waddrmgr.Open(namespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) + if err != nil { + tc.t.Errorf("%v", err) + return false + } + if err := mgr.ConvertToWatchingOnly(pubPassphrase); err != nil { + tc.t.Errorf("%v", err) return false } - // Run all of the manager API tests and close the manager. + // Run all of the manager API tests against the converted manager and + // close it. testManagerAPI(&testContext{ t: tc.t, + db: db, manager: mgr, account: 0, create: false, @@ -1168,7 +1183,8 @@ func testExportWatchingOnly(tc *testContext) bool { mgr.Close() // Open the watching-only manager and run all the tests again. - mgr, err = waddrmgr.Open(woMgrName, pubPassphrase, &btcnet.MainNetParams, fastScrypt) + mgr, err = waddrmgr.Open(namespace, pubPassphrase, &btcnet.MainNetParams, + fastScrypt) if err != nil { tc.t.Errorf("Open Watching-Only: unexpected error: %v", err) return false @@ -1177,6 +1193,7 @@ func testExportWatchingOnly(tc *testContext) bool { testManagerAPI(&testContext{ t: tc.t, + db: db, manager: mgr, account: 0, create: false, @@ -1436,31 +1453,39 @@ func testSync(tc *testContext) bool { // It makes use of a test context because the address manager is persistent and // much of the testing involves having specific state. func TestManager(t *testing.T) { + dbName := "mgrtest.bin" + _ = os.Remove(dbName) + db, mgrNamespace, err := createDbNamespace(dbName) + if err != nil { + t.Errorf("createDbNamespace: unexpected error: %v", err) + return + } + defer os.Remove(dbName) + defer db.Close() + // Open manager that does not exist to ensure the expected error is // returned. - mgrName := "mgrtest.bin" - _ = os.Remove(mgrName) - _, err := waddrmgr.Open(mgrName, pubPassphrase, &btcnet.MainNetParams, - fastScrypt) + _, err = waddrmgr.Open(mgrNamespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) if !checkManagerError(t, "Open non-existant", err, waddrmgr.ErrNoExist) { return } // Create a new manager. - mgr, err := waddrmgr.Create(mgrName, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) + mgr, err := waddrmgr.Create(mgrNamespace, seed, pubPassphrase, + privPassphrase, &btcnet.MainNetParams, fastScrypt) if err != nil { t.Errorf("Create: unexpected error: %v", err) return } - defer os.Remove(mgrName) + // NOTE: Not using deferred close here since part of the tests is // explicitly closing the manager and then opening the existing one. // Attempt to create the manager again to ensure the expected error is // returned. - _, err = waddrmgr.Create(mgrName, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) + _, err = waddrmgr.Create(mgrNamespace, seed, pubPassphrase, + privPassphrase, &btcnet.MainNetParams, fastScrypt) if !checkManagerError(t, "Create existing", err, waddrmgr.ErrAlreadyExists) { mgr.Close() return @@ -1470,6 +1495,7 @@ func TestManager(t *testing.T) { // manager after they've completed testManagerAPI(&testContext{ t: t, + db: db, manager: mgr, account: 0, create: true, @@ -1479,8 +1505,8 @@ func TestManager(t *testing.T) { // Open the manager and run all the tests again in open mode which // avoids reinserting new addresses like the create mode tests do. - mgr, err = waddrmgr.Open(mgrName, pubPassphrase, &btcnet.MainNetParams, - fastScrypt) + mgr, err = waddrmgr.Open(mgrNamespace, pubPassphrase, + &btcnet.MainNetParams, fastScrypt) if err != nil { t.Errorf("Open: unexpected error: %v", err) return @@ -1489,6 +1515,7 @@ func TestManager(t *testing.T) { tc := &testContext{ t: t, + db: db, manager: mgr, account: 0, create: false, @@ -1498,7 +1525,7 @@ func TestManager(t *testing.T) { // Now that the address manager has been tested in both the newly // created and opened modes, test a watching-only version. - testExportWatchingOnly(tc) + testWatchingOnly(tc) // Ensure that the manager sync state functionality works as expected. testSync(tc) @@ -1510,31 +1537,6 @@ func TestManager(t *testing.T) { } } -// setupManager creates a new address manager and returns a teardown function -// that should be invoked to ensure it is closed and removed upon completion. -func setupManager(t *testing.T) (tearDownFunc func(), mgr *waddrmgr.Manager) { - t.Parallel() - - // Create a new manager. - // We create the file and immediately delete it, as the waddrmgr - // needs to be doing the creating. - file, err := ioutil.TempDir("", "mgrtest") - if err != nil { - t.Fatalf("Failed to create db file: %v", err) - } - _ = os.Remove(file) - mgr, err = waddrmgr.Create(file, seed, pubPassphrase, privPassphrase, - &btcnet.MainNetParams, fastScrypt) - if err != nil { - t.Fatalf("Failed to create Manager: %v", err) - } - tearDownFunc = func() { - mgr.Close() - os.Remove(file) - } - return tearDownFunc, mgr -} - // TestEncryptDecryptErrors ensures that errors which occur while encrypting and // decrypting data return the expected errors. func TestEncryptDecryptErrors(t *testing.T) { diff --git a/waddrmgr/sync.go b/waddrmgr/sync.go index 0f1234b..0e56318 100644 --- a/waddrmgr/sync.go +++ b/waddrmgr/sync.go @@ -19,6 +19,7 @@ package waddrmgr import ( "sync" + "github.com/conformal/btcwallet/walletdb" "github.com/conformal/btcwire" ) @@ -210,13 +211,13 @@ func (m *Manager) SetSyncedTo(bs *BlockStamp) error { } // Update the database. - err := m.db.Update(func(tx *managerTx) error { - err := tx.PutSyncedTo(bs) + err := m.namespace.Update(func(tx walletdb.Tx) error { + err := putSyncedTo(tx, bs) if err != nil { return err } - return tx.PutRecentBlocks(recentHeight, recentHashes) + return putRecentBlocks(tx, recentHeight, recentHashes) }) if err != nil { return err