[lbry] Rework claimtrie CLIs

1. Ditch in-house block repo, and use btcd blocks database.
2. Commands switch from args to flags.
3. Revive chain recording and replaying, which will be part of CI pipeline.
4. Refactor cleanup the CLI skeletons.
5. Support DataDir, and testnet/regtes (not tested yet).

TODOs:

  1. Remove hardcoded test/development params, and pass them from flags.
  2. Make output more sensible.
  3. Add debug level flag.
  4. Add MerkleTrie implementation switch.
  5. Refactor periodic progess/status reporting for long run-time tasks.
  ...

claimtrie/claimtrie.go

@@ -247,10 +247,12 @@ func (ct *ClaimTrie) AppendBlock() error {
 		updateNames = append(updateNames, newName) // TODO: make sure using the temporalRepo batch is actually faster
 		updateHeights = append(updateHeights, nextUpdate)
 	}
+	if len(updateNames) != 0 {
 		err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights)
 		if err != nil {
 			return errors.Wrap(err, "temporal repo set")
 		}
+	}
 
 	hitFork := ct.updateTrieForHashForkIfNecessary()
 

claimtrie/cmd/cmd/block.go

@@ -2,157 +2,97 @@ package cmd
 
 import (
 	"fmt"
-	"log"
-	"path/filepath"
-	"strconv"
-
-	"github.com/btcsuite/btcd/claimtrie/block/blockrepo"
-	"github.com/btcsuite/btcd/claimtrie/merkletrie"
-	"github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo"
-	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
 
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(blockCmd)
+	rootCmd.AddCommand(NewBlocCommands())
-
-	blockCmd.AddCommand(blockLastCmd)
-	blockCmd.AddCommand(blockListCmd)
-	blockCmd.AddCommand(blockNameCmd)
 }
 
-var blockCmd = &cobra.Command{
+func NewBlocCommands() *cobra.Command {
+
+	cmd := &cobra.Command{
 		Use:   "block",
 		Short: "Block related commands",
 	}
 
-var blockLastCmd = &cobra.Command{
+	cmd.AddCommand(NewBlockBestCommand())
-	Use:   "last",
+	cmd.AddCommand(NewBlockListCommand())
-	Short: "Show the Merkle Hash of the last block",
+
+	return cmd
+}
+
+func NewBlockBestCommand() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "best",
+		Short: "Show the height and hash of the best block",
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
+			db, err := loadBlocksDB()
 			if err != nil {
-			log.Fatalf("can't open reported block repo: %s", err)
+				return errors.Wrapf(err, "load blocks database")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
 			}
 
-		last, err := repo.Load()
+			state := chain.BestSnapshot()
-		if err != nil {
+			fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String())
-			return fmt.Errorf("load previous height")
-		}
-
-		hash, err := repo.Get(last)
-		if err != nil {
-			return fmt.Errorf("load changes from repo: %w", err)
-		}
-
-		fmt.Printf("blk %-7d: %s\n", last, hash.String())
 
 			return nil
 		},
 	}
 
-var blockListCmd = &cobra.Command{
+	return cmd
-	Use:   "list <from_height> [<to_height>]",
+}
-	Short: "List the Merkle Hash of block in a range of heights",
+
-	Args:  cobra.RangeArgs(1, 2),
+func NewBlockListCommand() *cobra.Command {
+
+	var fromHeight int32
+	var toHeight int32
+
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: "List merkle hash of blocks between <from_height> <to_height>",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
+			db, err := loadBlocksDB()
 			if err != nil {
-			log.Fatalf("can't open reported block repo: %s", err)
+				return errors.Wrapf(err, "load blocks database")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
 			}
 
-		fromHeight, err := strconv.Atoi(args[0])
+			if toHeight > chain.BestSnapshot().Height {
-		if err != nil {
+				toHeight = chain.BestSnapshot().Height
-			return fmt.Errorf("invalid args")
 			}
 
-		toHeight := fromHeight + 1
+			for ht := fromHeight; ht <= toHeight; ht++ {
-		if len(args) == 2 {
+				hash, err := chain.BlockHashByHeight(ht)
-			toHeight, err = strconv.Atoi(args[1])
 				if err != nil {
-				return fmt.Errorf("invalid args")
+					return errors.Wrapf(err, "load hash for %d", ht)
 				}
-		}
+				fmt.Printf("Block %7d: %s\n", ht, hash.String())
-
-		last, err := repo.Load()
-		if err != nil {
-			return fmt.Errorf("load previous height")
-		}
-
-		if toHeight >= int(last) {
-			toHeight = int(last)
-		}
-
-		for i := fromHeight; i < toHeight; i++ {
-			hash, err := repo.Get(int32(i))
-			if err != nil {
-				return fmt.Errorf("load changes from repo: %w", err)
-			}
-			fmt.Printf("blk %-7d: %s\n", i, hash.String())
 			}
 
 			return nil
 		},
 	}
 
-var blockNameCmd = &cobra.Command{
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
-	Use:   "vertex <height> <name>",
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
-	Short: "List the claim and child hashes at vertex name of block at height",
+	cmd.Flags().SortFlags = false
-	Args:  cobra.RangeArgs(2, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path))
+	return cmd
-		if err != nil {
-			return fmt.Errorf("can't open reported block repo: %w", err)
-		}
-		defer repo.Close()
-
-		height, err := strconv.Atoi(args[0])
-		if err != nil {
-			return fmt.Errorf("invalid args")
-		}
-
-		last, err := repo.Load()
-		if err != nil {
-			return fmt.Errorf("load previous height: %w", err)
-		}
-
-		if last < int32(height) {
-			return fmt.Errorf("requested height is unavailable")
-		}
-
-		hash, err := repo.Get(int32(height))
-		if err != nil {
-			return fmt.Errorf("load previous height: %w", err)
-		}
-
-		trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("can't open merkle trie repo: %w", err)
-		}
-
-		trie := merkletrie.NewPersistentTrie(nil, trieRepo)
-		defer trie.Close()
-		trie.SetRoot(hash, nil)
-		if len(args) > 1 {
-			trie.Dump(args[1])
-		} else {
-			tmpRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path))
-			if err != nil {
-				return fmt.Errorf("can't open temporal repo: %w", err)
-			}
-			nodes, err := tmpRepo.NodesAt(int32(height))
-			if err != nil {
-				return fmt.Errorf("can't read temporal repo at %d: %w", height, err)
-			}
-			for _, name := range nodes {
-				fmt.Printf("Name: %s, ", string(name))
-				trie.Dump(string(name))
-			}
-		}
-		return nil
-	},
 }

claimtrie/cmd/cmd/chain.go

@@ -2,70 +2,72 @@ package cmd
 
 import (
 	"fmt"
-	"math"
 	"os"
 	"path/filepath"
-	"strconv"
+	"sync"
+	"time"
 
+	"github.com/btcsuite/btcd/blockchain"
 	"github.com/btcsuite/btcd/claimtrie"
-	"github.com/btcsuite/btcd/claimtrie/block"
-	"github.com/btcsuite/btcd/claimtrie/block/blockrepo"
 	"github.com/btcsuite/btcd/claimtrie/chain/chainrepo"
 	"github.com/btcsuite/btcd/claimtrie/change"
 	"github.com/btcsuite/btcd/claimtrie/config"
+	"github.com/btcsuite/btcd/database"
+	_ "github.com/btcsuite/btcd/database/ffldb"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(chainCmd)
+	rootCmd.AddCommand(NewChainCommands())
-
-	chainCmd.AddCommand(chainDumpCmd)
-	chainCmd.AddCommand(chainReplayCmd)
 }
 
-var chainCmd = &cobra.Command{
+func NewChainCommands() *cobra.Command {
+
+	cmd := &cobra.Command{
 		Use:   "chain",
 		Short: "chain related command",
 	}
 
-var chainDumpCmd = &cobra.Command{
+	cmd.AddCommand(NewChainDumpCommand())
-	Use:   "dump  <fromHeight> [<toHeight>]",
+	cmd.AddCommand(NewChainReplayCommand())
-	Short: "dump changes from <fromHeight> to [<toHeight>]",
+	cmd.AddCommand(NewChainConvertCommand())
-	Args:  cobra.RangeArgs(1, 2),
+
+	return cmd
+}
+
+func NewChainDumpCommand() *cobra.Command {
+
+	var fromHeight int32
+	var toHeight int32
+
+	cmd := &cobra.Command{
+		Use:   "dump",
+		Short: "Dump the chain changes between <fromHeight> and <toHeight>",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		fromHeight, err := strconv.Atoi(args[0])
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+			log.Debugf("Open chain repo: %q", dbPath)
+			chainRepo, err := chainrepo.NewPebble(dbPath)
 			if err != nil {
-			return fmt.Errorf("invalid args")
+				return errors.Wrapf(err, "open chain repo")
 			}
 
-		toHeight := fromHeight + 1
+			for height := fromHeight; height <= toHeight; height++ {
-		if len(args) == 2 {
+				changes, err := chainRepo.Load(height)
-			toHeight, err = strconv.Atoi(args[1])
+				if errors.Is(err, pebble.ErrNotFound) {
-			if err != nil {
-				return fmt.Errorf("invalid args")
-			}
-		}
-
-		chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
-		}
-
-		for height := fromHeight; height < toHeight; height++ {
-			changes, err := chainRepo.Load(int32(height))
-			if err == pebble.ErrNotFound {
 					continue
 				}
 				if err != nil {
-				return fmt.Errorf("load commands: %w", err)
+					return errors.Wrapf(err, "load charnges for height: %d")
 				}
-
 				for _, chg := range changes {
-				if int(chg.Height) > height {
-					break
-				}
 					showChange(chg)
 				}
 			}
@@ -74,61 +76,72 @@ var chainDumpCmd = &cobra.Command{
 		},
 	}
 
-var chainReplayCmd = &cobra.Command{
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
-	Use:   "replay <height>",
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
-	Short: "Replay the chain up to <height>",
+	cmd.Flags().SortFlags = false
-	Args:  cobra.RangeArgs(0, 1),
+
+	return cmd
+}
+
+func NewChainReplayCommand() *cobra.Command {
+
+	var fromHeight int32
+	var toHeight int32
+
+	cmd := &cobra.Command{
+		Use:   "replay",
+		Short: "Replay the chain changes between <fromHeight> and <toHeight>",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		fmt.Printf("not working until we pass record flag to claimtrie\n")
+			for _, dbName := range []string{
-
+				cfg.BlockRepoPebble.Path,
-		fromHeight := 2
+				cfg.NodeRepoPebble.Path,
-		toHeight := int(math.MaxInt32)
+				cfg.MerkleTrieRepoPebble.Path,
-
+				cfg.TemporalRepoPebble.Path,
-		var err error
+			} {
-		if len(args) == 1 {
+				dbPath := filepath.Join(dataDir, netName, "claim_dbs", dbName)
-			toHeight, err = strconv.Atoi(args[0])
+				log.Debugf("Delete repo: %q", dbPath)
+				err := os.RemoveAll(dbPath)
 				if err != nil {
-				return fmt.Errorf("invalid args")
+					return errors.Wrapf(err, "delete repo: %q", dbPath)
 				}
 			}
 
-		err = os.RemoveAll(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+			log.Debugf("Open chain repo: %q", dbPath)
+			chainRepo, err := chainrepo.NewPebble(dbPath)
 			if err != nil {
-			return fmt.Errorf("delete node repo: %w", err)
+				return errors.Wrapf(err, "open chain repo")
-		}
-
-		fmt.Printf("Deleted node repo\n")
-
-		chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open change repo: %w", err)
-		}
-
-		reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open block repo: %w", err)
 			}
 
 			cfg := config.DefaultConfig
+			cfg.RamTrie = true
+			cfg.DataDir = filepath.Join(dataDir, netName)
+
 			ct, err := claimtrie.New(cfg)
 			if err != nil {
-			return fmt.Errorf("create claimtrie: %w", err)
+				return errors.Wrapf(err, "create claimtrie")
 			}
 			defer ct.Close()
 
-		err = ct.ResetHeight(int32(fromHeight - 1))
+			db, err := loadBlocksDB()
 			if err != nil {
-			return fmt.Errorf("reset claimtrie height: %w", err)
+				return errors.Wrapf(err, "load blocks database")
 			}
 
-		for height := int32(fromHeight); height < int32(toHeight); height++ {
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
+			}
 
-			changes, err := chainRepo.Load(height)
+			for ht := fromHeight; ht < toHeight; ht++ {
-			if err == pebble.ErrNotFound {
+
+				changes, err := chainRepo.Load(ht + 1)
+				if errors.Is(err, pebble.ErrNotFound) {
 					// do nothing.
 				} else if err != nil {
-				return fmt.Errorf("load from change repo: %w", err)
+					return errors.Wrapf(err, "load changes for block %d", ht)
 				}
 
 				for _, chg := range changes {
@@ -136,31 +149,27 @@ var chainReplayCmd = &cobra.Command{
 					switch chg.Type {
 					case change.AddClaim:
 						err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount)
-
 					case change.UpdateClaim:
 						err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
-
 					case change.SpendClaim:
 						err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID)
-
 					case change.AddSupport:
 						err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
-
 					case change.SpendSupport:
 						err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID)
-
 					default:
-					err = fmt.Errorf("invalid change: %v", chg)
+						err = errors.Errorf("invalid change type: %v", chg)
 					}
 
 					if err != nil {
-					return fmt.Errorf("execute change %v: %w", chg, err)
+						return errors.Wrapf(err, "execute change %v", chg)
 					}
 				}
-			err = appendBlock(ct, reportedBlockRepo)
+				err = appendBlock(ct, chain)
 				if err != nil {
-				return err
+					return errors.Wrapf(err, "appendBlock")
 				}
+
 				if ct.Height()%1000 == 0 {
 					fmt.Printf("block: %d\n", ct.Height())
 				}
@@ -170,23 +179,246 @@ var chainReplayCmd = &cobra.Command{
 		},
 	}
 
-func appendBlock(ct *claimtrie.ClaimTrie, blockRepo block.Repo) error {
+	// FIXME
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height")
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height")
+	cmd.Flags().SortFlags = false
+
+	return cmd
+}
+
+func appendBlock(ct *claimtrie.ClaimTrie, chain *blockchain.BlockChain) error {
 
 	err := ct.AppendBlock()
 	if err != nil {
-		return fmt.Errorf("append block: %w", err)
+		return errors.Wrapf(err, "append block: %w")
 	}
 
-	height := ct.Height()
+	block, err := chain.BlockByHeight(ct.Height())
-
-	hash, err := blockRepo.Get(height)
 	if err != nil {
-		return fmt.Errorf("load from block repo: %w", err)
+		return errors.Wrapf(err, "load from block repo: %w")
 	}
+	hash := block.MsgBlock().Header.ClaimTrie
 
-	if *ct.MerkleHash() != *hash {
+	if *ct.MerkleHash() != hash {
-		return fmt.Errorf("hash mismatched at height %5d: exp: %s, got: %s", height, hash, ct.MerkleHash())
+		return errors.Errorf("hash mismatched at height %5d: exp: %s, got: %s", ct.Height(), hash, ct.MerkleHash())
 	}
 
 	return nil
 }
+
+func NewChainConvertCommand() *cobra.Command {
+
+	var height int32
+
+	cmd := &cobra.Command{
+		Use:   "convert",
+		Short: "convert changes from to <height>",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			db, err := loadBlocksDB()
+			if err != nil {
+				return errors.Wrapf(err, "load block db")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load block db")
+			}
+
+			converter := chainConverter{
+				db:          db,
+				chain:       chain,
+				blockChan:   make(chan *btcutil.Block, 1000),
+				changesChan: make(chan []change.Change, 1000),
+				wg:          &sync.WaitGroup{},
+			}
+
+			startTime := time.Now()
+			err = converter.start()
+			if err != nil {
+				return errors.Wrapf(err, "start Converter")
+			}
+
+			converter.wait()
+			log.Infof("Convert chain: took %s", time.Since(startTime))
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&height, "height", 0, "Height")
+
+	return cmd
+}
+
+type chainConverter struct {
+	db    database.DB
+	chain *blockchain.BlockChain
+
+	blockChan   chan *btcutil.Block
+	changesChan chan []change.Change
+
+	wg *sync.WaitGroup
+
+	statBlocksFetched   int
+	statBlocksProcessed int
+	statChangesSaved    int
+}
+
+func (cc *chainConverter) wait() {
+	cc.wg.Wait()
+}
+
+func (cb *chainConverter) start() error {
+
+	go cb.reportStats()
+
+	cb.wg.Add(3)
+	go cb.getBlock()
+	go cb.processBlock()
+	go cb.saveChanges()
+
+	return nil
+}
+
+func (cb *chainConverter) getBlock() {
+	defer cb.wg.Done()
+	defer close(cb.blockChan)
+
+	toHeight := int32(200000)
+	fmt.Printf("blocks: %d\n", cb.chain.BestSnapshot().Height)
+
+	if toHeight > cb.chain.BestSnapshot().Height {
+		toHeight = cb.chain.BestSnapshot().Height
+
+	}
+
+	for ht := int32(0); ht < toHeight; ht++ {
+		block, err := cb.chain.BlockByHeight(ht)
+		if err != nil {
+			log.Errorf("load changes from repo: %w", err)
+			return
+		}
+		cb.statBlocksFetched++
+		cb.blockChan <- block
+	}
+}
+
+func (cb *chainConverter) processBlock() {
+	defer cb.wg.Done()
+	defer close(cb.changesChan)
+
+	view := blockchain.NewUtxoViewpoint()
+	for block := range cb.blockChan {
+		var changes []change.Change
+		for _, tx := range block.Transactions() {
+			view.AddTxOuts(tx, block.Height())
+
+			if blockchain.IsCoinBase(tx) {
+				continue
+			}
+
+			for _, txIn := range tx.MsgTx().TxIn {
+				op := txIn.PreviousOutPoint
+				e := view.LookupEntry(op)
+				if e == nil {
+					log.Criticalf("Missing input in view for %s", op.String())
+				}
+				cs, err := txscript.DecodeClaimScript(e.PkScript())
+				if err == txscript.ErrNotClaimScript {
+					continue
+				}
+				if err != nil {
+					log.Criticalf("Can't parse claim script: %s", err)
+				}
+
+				chg := change.Change{
+					Height:   block.Height(),
+					Name:     cs.Name(),
+					OutPoint: txIn.PreviousOutPoint,
+				}
+
+				switch cs.Opcode() {
+				case txscript.OP_CLAIMNAME:
+					chg.Type = change.SpendClaim
+					chg.ClaimID = change.NewClaimID(chg.OutPoint)
+				case txscript.OP_UPDATECLAIM:
+					chg.Type = change.SpendClaim
+					copy(chg.ClaimID[:], cs.ClaimID())
+				case txscript.OP_SUPPORTCLAIM:
+					chg.Type = change.SpendSupport
+					copy(chg.ClaimID[:], cs.ClaimID())
+				}
+
+				changes = append(changes, chg)
+			}
+
+			op := *wire.NewOutPoint(tx.Hash(), 0)
+			for i, txOut := range tx.MsgTx().TxOut {
+				cs, err := txscript.DecodeClaimScript(txOut.PkScript)
+				if err == txscript.ErrNotClaimScript {
+					continue
+				}
+
+				op.Index = uint32(i)
+				chg := change.Change{
+					Height:   block.Height(),
+					Name:     cs.Name(),
+					OutPoint: op,
+					Amount:   txOut.Value,
+				}
+
+				switch cs.Opcode() {
+				case txscript.OP_CLAIMNAME:
+					chg.Type = change.AddClaim
+					chg.ClaimID = change.NewClaimID(op)
+				case txscript.OP_SUPPORTCLAIM:
+					chg.Type = change.AddSupport
+					copy(chg.ClaimID[:], cs.ClaimID())
+				case txscript.OP_UPDATECLAIM:
+					chg.Type = change.UpdateClaim
+					copy(chg.ClaimID[:], cs.ClaimID())
+				}
+				changes = append(changes, chg)
+			}
+		}
+		cb.statBlocksProcessed++
+
+		if len(changes) != 0 {
+			cb.changesChan <- changes
+		}
+	}
+}
+
+func (cb *chainConverter) saveChanges() {
+	defer cb.wg.Done()
+
+	dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+	chainRepo, err := chainrepo.NewPebble(dbPath)
+	if err != nil {
+		log.Errorf("open chain repo: %s", err)
+		return
+	}
+	defer chainRepo.Close()
+
+	for changes := range cb.changesChan {
+		err = chainRepo.Save(changes[0].Height, changes)
+		if err != nil {
+			log.Errorf("save to chain repo: %s", err)
+			return
+		}
+		cb.statChangesSaved++
+	}
+}
+
+func (cb *chainConverter) reportStats() {
+	tick := time.NewTicker(5 * time.Second)
+	for range tick.C {
+		log.Infof("block : %7d / %7d,  changes saved: %d",
+			cb.statBlocksFetched, cb.statBlocksProcessed, cb.statChangesSaved)
+
+	}
+}

claimtrie/cmd/cmd/helper.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/btcsuite/btcd/blockchain"
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/txscript"
+
+	"github.com/cockroachdb/errors"
+)
+
+func loadBlocksDB() (database.DB, error) {
+
+	dbPath := filepath.Join(dataDir, netName, "blocks_ffldb")
+	log.Infof("Loading blocks database: %s", dbPath)
+	db, err := database.Open("ffldb", dbPath, chainPramas().Net)
+	if err != nil {
+		return nil, errors.Wrapf(err, "open blocks database")
+	}
+
+	return db, nil
+}
+
+func loadChain(db database.DB) (*blockchain.BlockChain, error) {
+	paramsCopy := chaincfg.MainNetParams
+
+	log.Infof("Loading chain from database")
+
+	startTime := time.Now()
+	chain, err := blockchain.New(&blockchain.Config{
+		DB:          db,
+		ChainParams: &paramsCopy,
+		TimeSource:  blockchain.NewMedianTime(),
+		SigCache:    txscript.NewSigCache(1000),
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "create blockchain")
+	}
+
+	log.Infof("Loaded chain from database (%s)", time.Since(startTime))
+
+	return chain, err
+
+}
+
+func chainPramas() chaincfg.Params {
+
+	// Make a copy so the user won't modify the global instance.
+	params := chaincfg.MainNetParams
+	switch netName {
+	case "mainnet":
+		params = chaincfg.MainNetParams
+	case "testnet":
+		params = chaincfg.TestNet3Params
+	case "regtest":
+		params = chaincfg.RegressionNetParams
+	}
+	return params
+}

claimtrie/cmd/cmd/merkletrie.go

@@ -0,0 +1,105 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/btcsuite/btcd/claimtrie/merkletrie"
+	"github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo"
+	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
+
+	"github.com/cockroachdb/errors"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(NewTrieCommands())
+}
+
+func NewTrieCommands() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "trie",
+		Short: "MerkleTrie related commands",
+	}
+
+	cmd.AddCommand(NewTrieNameCommand())
+
+	return cmd
+}
+
+func NewTrieNameCommand() *cobra.Command {
+
+	var height int32
+	var name string
+
+	cmd := &cobra.Command{
+		Use:   "name",
+		Short: "List the claim and child hashes at vertex name of block at height",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			db, err := loadBlocksDB()
+			if err != nil {
+				return errors.Wrapf(err, "load blocks database")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
+			}
+
+			state := chain.BestSnapshot()
+			fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String())
+
+			if height > state.Height {
+				return errors.New("requested height is unavailable")
+			}
+
+			hash := state.Hash
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.MerkleTrieRepoPebble.Path)
+			log.Debugf("Open merkletrie repo: %q", dbPath)
+			trieRepo, err := merkletrierepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open merkle trie repo")
+			}
+
+			trie := merkletrie.NewPersistentTrie(nil, trieRepo)
+			defer trie.Close()
+
+			trie.SetRoot(&hash, nil)
+
+			if len(name) > 1 {
+				trie.Dump(name)
+				return nil
+			}
+
+			dbPath = filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path)
+			log.Debugf("Open temporal repo: %q", dbPath)
+			tmpRepo, err := temporalrepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open temporal repo")
+			}
+
+			nodes, err := tmpRepo.NodesAt(height)
+			if err != nil {
+				return errors.Wrapf(err, "read temporal repo at %d", height)
+			}
+
+			for _, name := range nodes {
+				fmt.Printf("Name: %s, ", string(name))
+				trie.Dump(string(name))
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&height, "height", 0, "Height")
+	cmd.Flags().StringVar(&name, "name", "", "Name")
+	cmd.Flags().SortFlags = false
+
+	return cmd
+}

claimtrie/cmd/cmd/node.go

@@ -4,56 +4,58 @@ import (
 	"fmt"
 	"math"
 	"path/filepath"
-	"strconv"
 
 	"github.com/btcsuite/btcd/claimtrie/change"
 	"github.com/btcsuite/btcd/claimtrie/node"
 	"github.com/btcsuite/btcd/claimtrie/node/noderepo"
+	"github.com/cockroachdb/errors"
 
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(nodeCmd)
+	rootCmd.AddCommand(NewNodeCommands())
-
-	nodeCmd.AddCommand(nodeDumpCmd)
-	nodeCmd.AddCommand(nodeReplayCmd)
-	nodeCmd.AddCommand(nodeChildrenCmd)
 }
 
-var nodeCmd = &cobra.Command{
+func NewNodeCommands() *cobra.Command {
+
+	cmd := &cobra.Command{
 		Use:   "node",
 		Short: "Replay the application of changes on a node up to certain height",
 	}
 
-var nodeDumpCmd = &cobra.Command{
+	cmd.AddCommand(NewNodeDumpCommand())
-	Use:   "dump <node_name> [<height>]",
+	cmd.AddCommand(NewNodeReplayCommand())
+	cmd.AddCommand(NewNodeChildrenCommand())
+
+	return cmd
+}
+
+func NewNodeDumpCommand() *cobra.Command {
+
+	var name string
+	var height int32
+
+	cmd := &cobra.Command{
+		Use:   "dump",
 		Short: "Replay the application of changes on a node up to certain height",
-	Args:  cobra.RangeArgs(1, 2),
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
 			if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
+				return errors.Wrapf(err, "open node repo")
-		}
-
-		name := args[0]
-		height := math.MaxInt32
-
-		if len(args) == 2 {
-			height, err = strconv.Atoi(args[1])
-			if err != nil {
-				return fmt.Errorf("invalid args")
-			}
 			}
 
 			changes, err := repo.LoadChanges([]byte(name))
 			if err != nil {
-			return fmt.Errorf("load commands: %w", err)
+				return errors.Wrapf(err, "load commands")
 			}
 
 			for _, chg := range changes {
-			if int(chg.Height) > height {
+				if chg.Height > height {
 					break
 				}
 				showChange(chg)
@@ -63,36 +65,41 @@ var nodeDumpCmd = &cobra.Command{
 		},
 	}
 
-var nodeReplayCmd = &cobra.Command{
+	cmd.Flags().StringVar(&name, "name", "", "Name")
-	Use:   "replay <node_name> [<height>]",
+	cmd.MarkFlagRequired("name")
-	Short: "Replay the application of changes on a node up to certain height",
+	cmd.Flags().Int32Var(&height, "height", math.MaxInt32, "Height")
-	Args:  cobra.RangeArgs(1, 2),
+
+	return cmd
+}
+
+func NewNodeReplayCommand() *cobra.Command {
+
+	var name string
+	var height int32
+
+	cmd := &cobra.Command{
+		Use:   "replay",
+		Short: "Replay the changes of <name> up to <height>",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
 			if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
+				return errors.Wrapf(err, "open node repo")
-		}
-
-		name := []byte(args[0])
-		height := math.MaxInt32
-
-		if len(args) == 2 {
-			height, err = strconv.Atoi(args[1])
-			if err != nil {
-				return fmt.Errorf("invalid args")
-			}
 			}
 
 			bm, err := node.NewBaseManager(repo)
 			if err != nil {
-			return fmt.Errorf("create node manager: %w", err)
+				return errors.Wrapf(err, "create node manager")
 			}
+
 			nm := node.NewNormalizingManager(bm)
 
-		n, err := nm.NodeAt(int32(height), name)
+			n, err := nm.NodeAt(height, []byte(name))
 			if err != nil || n == nil {
-			return fmt.Errorf("get node: %w", err)
+				return errors.Wrapf(err, "get node: %s", name)
 			}
 
 			showNode(n)
@@ -100,24 +107,48 @@ var nodeReplayCmd = &cobra.Command{
 		},
 	}
 
-var nodeChildrenCmd = &cobra.Command{
+	cmd.Flags().StringVar(&name, "name", "", "Name")
-	Use:   "children <node_name>",
+	cmd.MarkFlagRequired("name")
-	Short: "Show all the children names of a given node name",
+	cmd.Flags().Int32Var(&height, "height", 0, "Height (inclusive)")
-	Args:  cobra.RangeArgs(1, 1),
+	cmd.Flags().SortFlags = false
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+	return cmd
-		if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
 }
 
-		repo.IterateChildren([]byte(args[0]), func(changes []change.Change) bool {
+func NewNodeChildrenCommand() *cobra.Command {
-			// TODO: dump all the changes?
+
+	var name string
+
+	cmd := &cobra.Command{
+		Use:   "children",
+		Short: "Show all the children names of a given node name",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open node repo")
+			}
+
+			fn := func(changes []change.Change) bool {
 				fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height,
 					changes[len(changes)-1].Height)
 				return true
-		})
+			}
+
+			err = repo.IterateChildren([]byte(name), fn)
+			if err != nil {
+				return errors.Wrapf(err, "iterate children: %s", name)
+			}
 
 			return nil
 		},
 	}
+
+	cmd.Flags().StringVar(&name, "name", "", "Name")
+	cmd.MarkFlagRequired("name")
+
+	return cmd
+}

claimtrie/cmd/cmd/root.go

@@ -1,25 +1,55 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/btcsuite/btcd/claimtrie/config"
 	"github.com/btcsuite/btcd/claimtrie/param"
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btclog"
 
 	"github.com/spf13/cobra"
 )
 
-var cfg = config.DefaultConfig
+var (
+	log     btclog.Logger
+	cfg     = config.DefaultConfig
+	netName string
+	dataDir string
+)
 
-func init() {
+var rootCmd = NewRootCommand()
-	param.SetNetwork(wire.MainNet)
-}
 
-var rootCmd = &cobra.Command{
+func NewRootCommand() *cobra.Command {
+
+	cmd := &cobra.Command{
 		Use:          "claimtrie",
 		Short:        "ClaimTrie Command Line Interface",
 		SilenceUsage: true,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			switch netName {
+			case "mainnet":
+				param.SetNetwork(wire.MainNet)
+			case "testnet":
+				param.SetNetwork(wire.TestNet3)
+			case "regtest":
+				param.SetNetwork(wire.TestNet)
+			}
+		},
+	}
+
+	cmd.PersistentFlags().StringVar(&netName, "netname", "mainnet", "Net name")
+	cmd.PersistentFlags().StringVar(&dataDir, "datadir", cfg.DataDir, "Data dir")
+
+	return cmd
 }
 
 func Execute() {
+
+	backendLogger := btclog.NewBackend(os.Stdout)
+	defer os.Stdout.Sync()
+	log = backendLogger.Logger("CMDL")
+	log.SetLevel(btclog.LevelDebug)
+
 	rootCmd.Execute() // nolint : errchk
 }

claimtrie/cmd/cmd/temporal.go

@@ -1,62 +1,56 @@
 package cmd
 
 import (
-	"fmt"
+	"path/filepath"
-	"log"
-	"strconv"
 
 	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
 
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(temporalCmd)
+	rootCmd.AddCommand(NewTemporalCommand())
 }
 
-var temporalCmd = &cobra.Command{
+func NewTemporalCommand() *cobra.Command {
-	Use:   "temporal <from_height> [<to_height>]]",
+
+	var fromHeight int32
+	var toHeight int32
+
+	cmd := &cobra.Command{
+		Use:   "temporal",
 		Short: "List which nodes are update in a range of heights",
-	Args:  cobra.RangeArgs(1, 2),
+		Args:  cobra.NoArgs,
-	RunE:  runListNodes,
+		RunE: func(cmd *cobra.Command, args []string) error {
-}
 
-func runListNodes(cmd *cobra.Command, args []string) error {
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path)
-
+			log.Debugf("Open temporal repo: %s", dbPath)
-	repo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path)
+			repo, err := temporalrepo.NewPebble(dbPath)
 			if err != nil {
-		log.Fatalf("can't open reported block repo: %s", err)
+				return errors.Wrapf(err, "open temporal repo")
 			}
 
-	fromHeight, err := strconv.Atoi(args[0])
+			for ht := fromHeight; ht < toHeight; ht++ {
+				names, err := repo.NodesAt(ht)
 				if err != nil {
-		return fmt.Errorf("invalid args")
+					return errors.Wrapf(err, "get node names from temporal")
-	}
-
-	toHeight := fromHeight + 1
-	if len(args) == 2 {
-		toHeight, err = strconv.Atoi(args[1])
-		if err != nil {
-			return fmt.Errorf("invalid args")
-		}
-	}
-
-	for height := fromHeight; height < toHeight; height++ {
-		names, err := repo.NodesAt(int32(height))
-		if err != nil {
-			return fmt.Errorf("get node names from temporal")
 				}
 
 				if len(names) == 0 {
 					continue
 				}
 
-		fmt.Printf("%7d: %q", height, names[0])
+				showTemporalNames(ht, names)
-		for _, name := range names[1:] {
-			fmt.Printf(", %q ", name)
-		}
-		fmt.Printf("\n")
 			}
 
 			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
+	cmd.Flags().SortFlags = false
+
+	return cmd
 }

claimtrie/cmd/cmd/ui.go

@@ -14,8 +14,8 @@ var status = map[node.Status]string{
 	node.Deactivated: "Deactivated",
 }
 
-func changeName(c change.ChangeType) string {
+func changeType(c change.ChangeType) string {
-	switch c { // can't this be done via reflection?
+	switch c {
 	case change.AddClaim:
 		return "AddClaim"
 	case change.SpendClaim:
@@ -31,8 +31,8 @@ func changeName(c change.ChangeType) string {
 }
 
 func showChange(chg change.Change) {
-	fmt.Printf(">>> Height: %6d: %s for %04s, %d, %s\n",
+	fmt.Printf(">>> Height: %6d: %s for %04s, %15d, %s - %s\n",
-		chg.Height, changeName(chg.Type), chg.ClaimID.String(), chg.Amount, chg.OutPoint.String())
+		chg.Height, changeType(chg.Type), chg.ClaimID, chg.Amount, chg.OutPoint, chg.Name)
 }
 
 func showClaim(c *node.Claim, n *node.Node) {
@@ -42,12 +42,12 @@ func showClaim(c *node.Claim, n *node.Node) {
 	}
 
 	fmt.Printf("%s  C  ID: %s, TXO: %s\n   %5d/%-5d, Status: %9s, Amount: %15d, Support Amount: %15d\n",
-		mark, c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()])
+		mark, c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()])
 }
 
 func showSupport(c *node.Claim) {
 	fmt.Printf("    S id: %s, op: %s, %5d/%-5d, %9s, amt: %15d\n",
-		c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount)
+		c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount)
 }
 
 func showNode(n *node.Node) {
@@ -66,3 +66,11 @@ func showNode(n *node.Node) {
 	}
 	fmt.Printf("\n\n")
 }
+
+func showTemporalNames(height int32, names [][]byte) {
+	fmt.Printf("%7d: %q", height, names[0])
+	for _, name := range names[1:] {
+		fmt.Printf(", %q ", name)
+	}
+	fmt.Printf("\n")
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/btcsuite/goleveldb v1.0.0
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792
 	github.com/btcsuite/winsvc v1.0.0
+	github.com/cockroachdb/errors v1.8.1
 	github.com/cockroachdb/pebble v0.0.0-20210525181856-e45797baeb78
 	github.com/davecgh/go-spew v1.1.1
 	github.com/decred/dcrd/lru v1.0.0