diff --git a/claimtrie/claimtrie.go b/claimtrie/claimtrie.go index 28b6f010..4ba666d2 100644 --- a/claimtrie/claimtrie.go +++ b/claimtrie/claimtrie.go @@ -247,9 +247,11 @@ func (ct *ClaimTrie) AppendBlock() error { updateNames = append(updateNames, newName) // TODO: make sure using the temporalRepo batch is actually faster updateHeights = append(updateHeights, nextUpdate) } - err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights) - if err != nil { - return errors.Wrap(err, "temporal repo set") + if len(updateNames) != 0 { + err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights) + if err != nil { + return errors.Wrap(err, "temporal repo set") + } } hitFork := ct.updateTrieForHashForkIfNecessary() diff --git a/claimtrie/cmd/cmd/block.go b/claimtrie/cmd/cmd/block.go index 5a2db0f4..d0ee7719 100644 --- a/claimtrie/cmd/cmd/block.go +++ b/claimtrie/cmd/cmd/block.go @@ -2,157 +2,97 @@ package cmd import ( "fmt" - "log" - "path/filepath" - "strconv" - - "github.com/btcsuite/btcd/claimtrie/block/blockrepo" - "github.com/btcsuite/btcd/claimtrie/merkletrie" - "github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo" - "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(blockCmd) - - blockCmd.AddCommand(blockLastCmd) - blockCmd.AddCommand(blockListCmd) - blockCmd.AddCommand(blockNameCmd) + rootCmd.AddCommand(NewBlocCommands()) } -var blockCmd = &cobra.Command{ - Use: "block", - Short: "Block related commands", +func NewBlocCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "block", + Short: "Block related commands", + } + + cmd.AddCommand(NewBlockBestCommand()) + cmd.AddCommand(NewBlockListCommand()) + + return cmd } -var blockLastCmd = &cobra.Command{ - Use: "last", - Short: "Show the Merkle Hash of the last block", - RunE: func(cmd *cobra.Command, args []string) error { +func NewBlockBestCommand() *cobra.Command { - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } + cmd := &cobra.Command{ + Use: "best", + Short: "Show the height and hash of the best block", + RunE: func(cmd *cobra.Command, args []string) error { - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height") - } - - hash, err := repo.Get(last) - if err != nil { - return fmt.Errorf("load changes from repo: %w", err) - } - - fmt.Printf("blk %-7d: %s\n", last, hash.String()) - - return nil - }, -} - -var blockListCmd = &cobra.Command{ - Use: "list []", - Short: "List the Merkle Hash of block in a range of heights", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } - - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) + db, err := loadBlocksDB() if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "load blocks database") } - } + defer db.Close() - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height") - } - - if toHeight >= int(last) { - toHeight = int(last) - } - - for i := fromHeight; i < toHeight; i++ { - hash, err := repo.Get(int32(i)) + chain, err := loadChain(db) if err != nil { - return fmt.Errorf("load changes from repo: %w", err) + return errors.Wrapf(err, "load chain") } - fmt.Printf("blk %-7d: %s\n", i, hash.String()) - } - return nil - }, + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + return nil + }, + } + + return cmd } -var blockNameCmd = &cobra.Command{ - Use: "vertex ", - Short: "List the claim and child hashes at vertex name of block at height", - Args: cobra.RangeArgs(2, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewBlockListCommand() *cobra.Command { - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path)) - if err != nil { - return fmt.Errorf("can't open reported block repo: %w", err) - } - defer repo.Close() + var fromHeight int32 + var toHeight int32 - height, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } + cmd := &cobra.Command{ + Use: "list", + Short: "List merkle hash of blocks between ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height: %w", err) - } - - if last < int32(height) { - return fmt.Errorf("requested height is unavailable") - } - - hash, err := repo.Get(int32(height)) - if err != nil { - return fmt.Errorf("load previous height: %w", err) - } - - trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path)) - if err != nil { - return fmt.Errorf("can't open merkle trie repo: %w", err) - } - - trie := merkletrie.NewPersistentTrie(nil, trieRepo) - defer trie.Close() - trie.SetRoot(hash, nil) - if len(args) > 1 { - trie.Dump(args[1]) - } else { - tmpRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path)) + db, err := loadBlocksDB() if err != nil { - return fmt.Errorf("can't open temporal repo: %w", err) + return errors.Wrapf(err, "load blocks database") } - nodes, err := tmpRepo.NodesAt(int32(height)) + defer db.Close() + + chain, err := loadChain(db) if err != nil { - return fmt.Errorf("can't read temporal repo at %d: %w", height, err) + return errors.Wrapf(err, "load chain") } - for _, name := range nodes { - fmt.Printf("Name: %s, ", string(name)) - trie.Dump(string(name)) + + if toHeight > chain.BestSnapshot().Height { + toHeight = chain.BestSnapshot().Height } - } - return nil - }, + + for ht := fromHeight; ht <= toHeight; ht++ { + hash, err := chain.BlockHashByHeight(ht) + if err != nil { + return errors.Wrapf(err, "load hash for %d", ht) + } + fmt.Printf("Block %7d: %s\n", ht, hash.String()) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } diff --git a/claimtrie/cmd/cmd/chain.go b/claimtrie/cmd/cmd/chain.go index 36e75a93..626ce852 100644 --- a/claimtrie/cmd/cmd/chain.go +++ b/claimtrie/cmd/cmd/chain.go @@ -2,191 +2,423 @@ package cmd import ( "fmt" - "math" "os" "path/filepath" - "strconv" + "sync" + "time" + "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/claimtrie" - "github.com/btcsuite/btcd/claimtrie/block" - "github.com/btcsuite/btcd/claimtrie/block/blockrepo" "github.com/btcsuite/btcd/claimtrie/chain/chainrepo" "github.com/btcsuite/btcd/claimtrie/change" "github.com/btcsuite/btcd/claimtrie/config" + "github.com/btcsuite/btcd/database" + _ "github.com/btcsuite/btcd/database/ffldb" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(chainCmd) - - chainCmd.AddCommand(chainDumpCmd) - chainCmd.AddCommand(chainReplayCmd) + rootCmd.AddCommand(NewChainCommands()) } -var chainCmd = &cobra.Command{ - Use: "chain", - Short: "chain related command", +func NewChainCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "chain", + Short: "chain related command", + } + + cmd.AddCommand(NewChainDumpCommand()) + cmd.AddCommand(NewChainReplayCommand()) + cmd.AddCommand(NewChainConvertCommand()) + + return cmd } -var chainDumpCmd = &cobra.Command{ - Use: "dump []", - Short: "dump changes from to []", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewChainDumpCommand() *cobra.Command { - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } + var fromHeight int32 + var toHeight int32 - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) + cmd := &cobra.Command{ + Use: "dump", + Short: "Dump the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + log.Debugf("Open chain repo: %q", dbPath) + chainRepo, err := chainrepo.NewPebble(dbPath) if err != nil { - return fmt.Errorf("invalid args") - } - } - - chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } - - for height := fromHeight; height < toHeight; height++ { - changes, err := chainRepo.Load(int32(height)) - if err == pebble.ErrNotFound { - continue - } - if err != nil { - return fmt.Errorf("load commands: %w", err) + return errors.Wrapf(err, "open chain repo") } - for _, chg := range changes { - if int(chg.Height) > height { - break + for height := fromHeight; height <= toHeight; height++ { + changes, err := chainRepo.Load(height) + if errors.Is(err, pebble.ErrNotFound) { + continue } - showChange(chg) - } - } - - return nil - }, -} - -var chainReplayCmd = &cobra.Command{ - Use: "replay ", - Short: "Replay the chain up to ", - Args: cobra.RangeArgs(0, 1), - RunE: func(cmd *cobra.Command, args []string) error { - - fmt.Printf("not working until we pass record flag to claimtrie\n") - - fromHeight := 2 - toHeight := int(math.MaxInt32) - - var err error - if len(args) == 1 { - toHeight, err = strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - } - - err = os.RemoveAll(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("delete node repo: %w", err) - } - - fmt.Printf("Deleted node repo\n") - - chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open change repo: %w", err) - } - - reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open block repo: %w", err) - } - - cfg := config.DefaultConfig - ct, err := claimtrie.New(cfg) - if err != nil { - return fmt.Errorf("create claimtrie: %w", err) - } - defer ct.Close() - - err = ct.ResetHeight(int32(fromHeight - 1)) - if err != nil { - return fmt.Errorf("reset claimtrie height: %w", err) - } - - for height := int32(fromHeight); height < int32(toHeight); height++ { - - changes, err := chainRepo.Load(height) - if err == pebble.ErrNotFound { - // do nothing. - } else if err != nil { - return fmt.Errorf("load from change repo: %w", err) - } - - for _, chg := range changes { - - switch chg.Type { - case change.AddClaim: - err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount) - - case change.UpdateClaim: - err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) - - case change.SpendClaim: - err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID) - - case change.AddSupport: - err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) - - case change.SpendSupport: - err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID) - - default: - err = fmt.Errorf("invalid change: %v", chg) - } - if err != nil { - return fmt.Errorf("execute change %v: %w", chg, err) + return errors.Wrapf(err, "load charnges for height: %d") + } + for _, chg := range changes { + showChange(chg) } } - err = appendBlock(ct, reportedBlockRepo) - if err != nil { - return err - } - if ct.Height()%1000 == 0 { - fmt.Printf("block: %d\n", ct.Height()) - } - } - return nil - }, + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } -func appendBlock(ct *claimtrie.ClaimTrie, blockRepo block.Repo) error { +func NewChainReplayCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + for _, dbName := range []string{ + cfg.BlockRepoPebble.Path, + cfg.NodeRepoPebble.Path, + cfg.MerkleTrieRepoPebble.Path, + cfg.TemporalRepoPebble.Path, + } { + dbPath := filepath.Join(dataDir, netName, "claim_dbs", dbName) + log.Debugf("Delete repo: %q", dbPath) + err := os.RemoveAll(dbPath) + if err != nil { + return errors.Wrapf(err, "delete repo: %q", dbPath) + } + } + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + log.Debugf("Open chain repo: %q", dbPath) + chainRepo, err := chainrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open chain repo") + } + + cfg := config.DefaultConfig + cfg.RamTrie = true + cfg.DataDir = filepath.Join(dataDir, netName) + + ct, err := claimtrie.New(cfg) + if err != nil { + return errors.Wrapf(err, "create claimtrie") + } + defer ct.Close() + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + for ht := fromHeight; ht < toHeight; ht++ { + + changes, err := chainRepo.Load(ht + 1) + if errors.Is(err, pebble.ErrNotFound) { + // do nothing. + } else if err != nil { + return errors.Wrapf(err, "load changes for block %d", ht) + } + + for _, chg := range changes { + + switch chg.Type { + case change.AddClaim: + err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount) + case change.UpdateClaim: + err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendClaim: + err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID) + case change.AddSupport: + err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendSupport: + err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID) + default: + err = errors.Errorf("invalid change type: %v", chg) + } + + if err != nil { + return errors.Wrapf(err, "execute change %v", chg) + } + } + err = appendBlock(ct, chain) + if err != nil { + return errors.Wrapf(err, "appendBlock") + } + + if ct.Height()%1000 == 0 { + fmt.Printf("block: %d\n", ct.Height()) + } + } + + return nil + }, + } + + // FIXME + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height") + cmd.Flags().SortFlags = false + + return cmd +} + +func appendBlock(ct *claimtrie.ClaimTrie, chain *blockchain.BlockChain) error { err := ct.AppendBlock() if err != nil { - return fmt.Errorf("append block: %w", err) + return errors.Wrapf(err, "append block: %w") } - height := ct.Height() - - hash, err := blockRepo.Get(height) + block, err := chain.BlockByHeight(ct.Height()) if err != nil { - return fmt.Errorf("load from block repo: %w", err) + return errors.Wrapf(err, "load from block repo: %w") } + hash := block.MsgBlock().Header.ClaimTrie - if *ct.MerkleHash() != *hash { - return fmt.Errorf("hash mismatched at height %5d: exp: %s, got: %s", height, hash, ct.MerkleHash()) + if *ct.MerkleHash() != hash { + return errors.Errorf("hash mismatched at height %5d: exp: %s, got: %s", ct.Height(), hash, ct.MerkleHash()) } return nil } + +func NewChainConvertCommand() *cobra.Command { + + var height int32 + + cmd := &cobra.Command{ + Use: "convert", + Short: "convert changes from to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load block db") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load block db") + } + + converter := chainConverter{ + db: db, + chain: chain, + blockChan: make(chan *btcutil.Block, 1000), + changesChan: make(chan []change.Change, 1000), + wg: &sync.WaitGroup{}, + } + + startTime := time.Now() + err = converter.start() + if err != nil { + return errors.Wrapf(err, "start Converter") + } + + converter.wait() + log.Infof("Convert chain: took %s", time.Since(startTime)) + + return nil + }, + } + + cmd.Flags().Int32Var(&height, "height", 0, "Height") + + return cmd +} + +type chainConverter struct { + db database.DB + chain *blockchain.BlockChain + + blockChan chan *btcutil.Block + changesChan chan []change.Change + + wg *sync.WaitGroup + + statBlocksFetched int + statBlocksProcessed int + statChangesSaved int +} + +func (cc *chainConverter) wait() { + cc.wg.Wait() +} + +func (cb *chainConverter) start() error { + + go cb.reportStats() + + cb.wg.Add(3) + go cb.getBlock() + go cb.processBlock() + go cb.saveChanges() + + return nil +} + +func (cb *chainConverter) getBlock() { + defer cb.wg.Done() + defer close(cb.blockChan) + + toHeight := int32(200000) + fmt.Printf("blocks: %d\n", cb.chain.BestSnapshot().Height) + + if toHeight > cb.chain.BestSnapshot().Height { + toHeight = cb.chain.BestSnapshot().Height + + } + + for ht := int32(0); ht < toHeight; ht++ { + block, err := cb.chain.BlockByHeight(ht) + if err != nil { + log.Errorf("load changes from repo: %w", err) + return + } + cb.statBlocksFetched++ + cb.blockChan <- block + } +} + +func (cb *chainConverter) processBlock() { + defer cb.wg.Done() + defer close(cb.changesChan) + + view := blockchain.NewUtxoViewpoint() + for block := range cb.blockChan { + var changes []change.Change + for _, tx := range block.Transactions() { + view.AddTxOuts(tx, block.Height()) + + if blockchain.IsCoinBase(tx) { + continue + } + + for _, txIn := range tx.MsgTx().TxIn { + op := txIn.PreviousOutPoint + e := view.LookupEntry(op) + if e == nil { + log.Criticalf("Missing input in view for %s", op.String()) + } + cs, err := txscript.DecodeClaimScript(e.PkScript()) + if err == txscript.ErrNotClaimScript { + continue + } + if err != nil { + log.Criticalf("Can't parse claim script: %s", err) + } + + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: txIn.PreviousOutPoint, + } + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.SpendClaim + chg.ClaimID = change.NewClaimID(chg.OutPoint) + case txscript.OP_UPDATECLAIM: + chg.Type = change.SpendClaim + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.SpendSupport + copy(chg.ClaimID[:], cs.ClaimID()) + } + + changes = append(changes, chg) + } + + op := *wire.NewOutPoint(tx.Hash(), 0) + for i, txOut := range tx.MsgTx().TxOut { + cs, err := txscript.DecodeClaimScript(txOut.PkScript) + if err == txscript.ErrNotClaimScript { + continue + } + + op.Index = uint32(i) + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: op, + Amount: txOut.Value, + } + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.AddClaim + chg.ClaimID = change.NewClaimID(op) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.AddSupport + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_UPDATECLAIM: + chg.Type = change.UpdateClaim + copy(chg.ClaimID[:], cs.ClaimID()) + } + changes = append(changes, chg) + } + } + cb.statBlocksProcessed++ + + if len(changes) != 0 { + cb.changesChan <- changes + } + } +} + +func (cb *chainConverter) saveChanges() { + defer cb.wg.Done() + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + chainRepo, err := chainrepo.NewPebble(dbPath) + if err != nil { + log.Errorf("open chain repo: %s", err) + return + } + defer chainRepo.Close() + + for changes := range cb.changesChan { + err = chainRepo.Save(changes[0].Height, changes) + if err != nil { + log.Errorf("save to chain repo: %s", err) + return + } + cb.statChangesSaved++ + } +} + +func (cb *chainConverter) reportStats() { + tick := time.NewTicker(5 * time.Second) + for range tick.C { + log.Infof("block : %7d / %7d, changes saved: %d", + cb.statBlocksFetched, cb.statBlocksProcessed, cb.statChangesSaved) + + } +} diff --git a/claimtrie/cmd/cmd/helper.go b/claimtrie/cmd/cmd/helper.go new file mode 100644 index 00000000..e71b4bbb --- /dev/null +++ b/claimtrie/cmd/cmd/helper.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "path/filepath" + "time" + + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/database" + "github.com/btcsuite/btcd/txscript" + + "github.com/cockroachdb/errors" +) + +func loadBlocksDB() (database.DB, error) { + + dbPath := filepath.Join(dataDir, netName, "blocks_ffldb") + log.Infof("Loading blocks database: %s", dbPath) + db, err := database.Open("ffldb", dbPath, chainPramas().Net) + if err != nil { + return nil, errors.Wrapf(err, "open blocks database") + } + + return db, nil +} + +func loadChain(db database.DB) (*blockchain.BlockChain, error) { + paramsCopy := chaincfg.MainNetParams + + log.Infof("Loading chain from database") + + startTime := time.Now() + chain, err := blockchain.New(&blockchain.Config{ + DB: db, + ChainParams: ¶msCopy, + TimeSource: blockchain.NewMedianTime(), + SigCache: txscript.NewSigCache(1000), + }) + if err != nil { + return nil, errors.Wrapf(err, "create blockchain") + } + + log.Infof("Loaded chain from database (%s)", time.Since(startTime)) + + return chain, err + +} + +func chainPramas() chaincfg.Params { + + // Make a copy so the user won't modify the global instance. + params := chaincfg.MainNetParams + switch netName { + case "mainnet": + params = chaincfg.MainNetParams + case "testnet": + params = chaincfg.TestNet3Params + case "regtest": + params = chaincfg.RegressionNetParams + } + return params +} diff --git a/claimtrie/cmd/cmd/merkletrie.go b/claimtrie/cmd/cmd/merkletrie.go new file mode 100644 index 00000000..4d6b4079 --- /dev/null +++ b/claimtrie/cmd/cmd/merkletrie.go @@ -0,0 +1,105 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/btcsuite/btcd/claimtrie/merkletrie" + "github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo" + "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewTrieCommands()) +} + +func NewTrieCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "trie", + Short: "MerkleTrie related commands", + } + + cmd.AddCommand(NewTrieNameCommand()) + + return cmd +} + +func NewTrieNameCommand() *cobra.Command { + + var height int32 + var name string + + cmd := &cobra.Command{ + Use: "name", + Short: "List the claim and child hashes at vertex name of block at height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + if height > state.Height { + return errors.New("requested height is unavailable") + } + + hash := state.Hash + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.MerkleTrieRepoPebble.Path) + log.Debugf("Open merkletrie repo: %q", dbPath) + trieRepo, err := merkletrierepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open merkle trie repo") + } + + trie := merkletrie.NewPersistentTrie(nil, trieRepo) + defer trie.Close() + + trie.SetRoot(&hash, nil) + + if len(name) > 1 { + trie.Dump(name) + return nil + } + + dbPath = filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %q", dbPath) + tmpRepo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + nodes, err := tmpRepo.NodesAt(height) + if err != nil { + return errors.Wrapf(err, "read temporal repo at %d", height) + } + + for _, name := range nodes { + fmt.Printf("Name: %s, ", string(name)) + trie.Dump(string(name)) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&height, "height", 0, "Height") + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.Flags().SortFlags = false + + return cmd +} diff --git a/claimtrie/cmd/cmd/node.go b/claimtrie/cmd/cmd/node.go index bcd1183b..49049916 100644 --- a/claimtrie/cmd/cmd/node.go +++ b/claimtrie/cmd/cmd/node.go @@ -4,120 +4,151 @@ import ( "fmt" "math" "path/filepath" - "strconv" "github.com/btcsuite/btcd/claimtrie/change" "github.com/btcsuite/btcd/claimtrie/node" "github.com/btcsuite/btcd/claimtrie/node/noderepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(nodeCmd) - - nodeCmd.AddCommand(nodeDumpCmd) - nodeCmd.AddCommand(nodeReplayCmd) - nodeCmd.AddCommand(nodeChildrenCmd) + rootCmd.AddCommand(NewNodeCommands()) } -var nodeCmd = &cobra.Command{ - Use: "node", - Short: "Replay the application of changes on a node up to certain height", +func NewNodeCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "node", + Short: "Replay the application of changes on a node up to certain height", + } + + cmd.AddCommand(NewNodeDumpCommand()) + cmd.AddCommand(NewNodeReplayCommand()) + cmd.AddCommand(NewNodeChildrenCommand()) + + return cmd } -var nodeDumpCmd = &cobra.Command{ - Use: "dump []", - Short: "Replay the application of changes on a node up to certain height", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewNodeDumpCommand() *cobra.Command { - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } + var name string + var height int32 - name := args[0] - height := math.MaxInt32 + cmd := &cobra.Command{ + Use: "dump", + Short: "Replay the application of changes on a node up to certain height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 2 { - height, err = strconv.Atoi(args[1]) + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "open node repo") } - } - changes, err := repo.LoadChanges([]byte(name)) - if err != nil { - return fmt.Errorf("load commands: %w", err) - } - - for _, chg := range changes { - if int(chg.Height) > height { - break - } - showChange(chg) - } - - return nil - }, -} - -var nodeReplayCmd = &cobra.Command{ - Use: "replay []", - Short: "Replay the application of changes on a node up to certain height", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } - - name := []byte(args[0]) - height := math.MaxInt32 - - if len(args) == 2 { - height, err = strconv.Atoi(args[1]) + changes, err := repo.LoadChanges([]byte(name)) if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "load commands") } - } - bm, err := node.NewBaseManager(repo) - if err != nil { - return fmt.Errorf("create node manager: %w", err) - } - nm := node.NewNormalizingManager(bm) + for _, chg := range changes { + if chg.Height > height { + break + } + showChange(chg) + } - n, err := nm.NodeAt(int32(height), name) - if err != nil || n == nil { - return fmt.Errorf("get node: %w", err) - } + return nil + }, + } - showNode(n) - return nil - }, + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", math.MaxInt32, "Height") + + return cmd } -var nodeChildrenCmd = &cobra.Command{ - Use: "children ", - Short: "Show all the children names of a given node name", - Args: cobra.RangeArgs(1, 1), - RunE: func(cmd *cobra.Command, args []string) error { +func NewNodeReplayCommand() *cobra.Command { - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } + var name string + var height int32 - repo.IterateChildren([]byte(args[0]), func(changes []change.Change) bool { - // TODO: dump all the changes? - fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, - changes[len(changes)-1].Height) - return true - }) + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the changes of up to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - return nil - }, + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + + bm, err := node.NewBaseManager(repo) + if err != nil { + return errors.Wrapf(err, "create node manager") + } + + nm := node.NewNormalizingManager(bm) + + n, err := nm.NodeAt(height, []byte(name)) + if err != nil || n == nil { + return errors.Wrapf(err, "get node: %s", name) + } + + showNode(n) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", 0, "Height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} + +func NewNodeChildrenCommand() *cobra.Command { + + var name string + + cmd := &cobra.Command{ + Use: "children", + Short: "Show all the children names of a given node name", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + + fn := func(changes []change.Change) bool { + fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, + changes[len(changes)-1].Height) + return true + } + + err = repo.IterateChildren([]byte(name), fn) + if err != nil { + return errors.Wrapf(err, "iterate children: %s", name) + } + + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + + return cmd } diff --git a/claimtrie/cmd/cmd/root.go b/claimtrie/cmd/cmd/root.go index be84f5d6..698d0526 100644 --- a/claimtrie/cmd/cmd/root.go +++ b/claimtrie/cmd/cmd/root.go @@ -1,25 +1,55 @@ package cmd import ( + "os" + "github.com/btcsuite/btcd/claimtrie/config" "github.com/btcsuite/btcd/claimtrie/param" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btclog" "github.com/spf13/cobra" ) -var cfg = config.DefaultConfig +var ( + log btclog.Logger + cfg = config.DefaultConfig + netName string + dataDir string +) -func init() { - param.SetNetwork(wire.MainNet) -} +var rootCmd = NewRootCommand() -var rootCmd = &cobra.Command{ - Use: "claimtrie", - Short: "ClaimTrie Command Line Interface", - SilenceUsage: true, +func NewRootCommand() *cobra.Command { + + cmd := &cobra.Command{ + Use: "claimtrie", + Short: "ClaimTrie Command Line Interface", + SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + switch netName { + case "mainnet": + param.SetNetwork(wire.MainNet) + case "testnet": + param.SetNetwork(wire.TestNet3) + case "regtest": + param.SetNetwork(wire.TestNet) + } + }, + } + + cmd.PersistentFlags().StringVar(&netName, "netname", "mainnet", "Net name") + cmd.PersistentFlags().StringVar(&dataDir, "datadir", cfg.DataDir, "Data dir") + + return cmd } func Execute() { + + backendLogger := btclog.NewBackend(os.Stdout) + defer os.Stdout.Sync() + log = backendLogger.Logger("CMDL") + log.SetLevel(btclog.LevelDebug) + rootCmd.Execute() // nolint : errchk } diff --git a/claimtrie/cmd/cmd/temporal.go b/claimtrie/cmd/cmd/temporal.go index 85f94850..0024deb0 100644 --- a/claimtrie/cmd/cmd/temporal.go +++ b/claimtrie/cmd/cmd/temporal.go @@ -1,62 +1,56 @@ package cmd import ( - "fmt" - "log" - "strconv" + "path/filepath" "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(temporalCmd) + rootCmd.AddCommand(NewTemporalCommand()) } -var temporalCmd = &cobra.Command{ - Use: "temporal []]", - Short: "List which nodes are update in a range of heights", - Args: cobra.RangeArgs(1, 2), - RunE: runListNodes, -} - -func runListNodes(cmd *cobra.Command, args []string) error { - - repo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } - - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("invalid args") - } - } - - for height := fromHeight; height < toHeight; height++ { - names, err := repo.NodesAt(int32(height)) - if err != nil { - return fmt.Errorf("get node names from temporal") - } - - if len(names) == 0 { - continue - } - - fmt.Printf("%7d: %q", height, names[0]) - for _, name := range names[1:] { - fmt.Printf(", %q ", name) - } - fmt.Printf("\n") - } - - return nil +func NewTemporalCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "temporal", + Short: "List which nodes are update in a range of heights", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %s", dbPath) + repo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + for ht := fromHeight; ht < toHeight; ht++ { + names, err := repo.NodesAt(ht) + if err != nil { + return errors.Wrapf(err, "get node names from temporal") + } + + if len(names) == 0 { + continue + } + + showTemporalNames(ht, names) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } diff --git a/claimtrie/cmd/cmd/ui.go b/claimtrie/cmd/cmd/ui.go index 28eb6fe8..b7a530a1 100644 --- a/claimtrie/cmd/cmd/ui.go +++ b/claimtrie/cmd/cmd/ui.go @@ -14,8 +14,8 @@ var status = map[node.Status]string{ node.Deactivated: "Deactivated", } -func changeName(c change.ChangeType) string { - switch c { // can't this be done via reflection? +func changeType(c change.ChangeType) string { + switch c { case change.AddClaim: return "AddClaim" case change.SpendClaim: @@ -31,8 +31,8 @@ func changeName(c change.ChangeType) string { } func showChange(chg change.Change) { - fmt.Printf(">>> Height: %6d: %s for %04s, %d, %s\n", - chg.Height, changeName(chg.Type), chg.ClaimID.String(), chg.Amount, chg.OutPoint.String()) + fmt.Printf(">>> Height: %6d: %s for %04s, %15d, %s - %s\n", + chg.Height, changeType(chg.Type), chg.ClaimID, chg.Amount, chg.OutPoint, chg.Name) } func showClaim(c *node.Claim, n *node.Node) { @@ -42,12 +42,12 @@ func showClaim(c *node.Claim, n *node.Node) { } fmt.Printf("%s C ID: %s, TXO: %s\n %5d/%-5d, Status: %9s, Amount: %15d, Support Amount: %15d\n", - mark, c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()]) + mark, c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()]) } func showSupport(c *node.Claim) { fmt.Printf(" S id: %s, op: %s, %5d/%-5d, %9s, amt: %15d\n", - c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount) + c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount) } func showNode(n *node.Node) { @@ -66,3 +66,11 @@ func showNode(n *node.Node) { } fmt.Printf("\n\n") } + +func showTemporalNames(height int32, names [][]byte) { + fmt.Printf("%7d: %q", height, names[0]) + for _, name := range names[1:] { + fmt.Printf(", %q ", name) + } + fmt.Printf("\n") +} diff --git a/go.mod b/go.mod index af85c0db..b91fe5d1 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/btcsuite/goleveldb v1.0.0 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 github.com/btcsuite/winsvc v1.0.0 + github.com/cockroachdb/errors v1.8.1 github.com/cockroachdb/pebble v0.0.0-20210525181856-e45797baeb78 github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/lru v1.0.0