From 6b55968ccd639a423090ebfaf9f291a30b0cb5b4 Mon Sep 17 00:00:00 2001 From: Roy Lee Date: Mon, 12 Jul 2021 16:08:47 -0700 Subject: [PATCH] [lbry] rework claimtrie config and param --- btcd.go | 2 +- claimtrie/claimtrie.go | 18 +++++------ claimtrie/claimtrie_test.go | 15 +++++---- claimtrie/cmd/cmd/block.go | 11 ++++--- claimtrie/cmd/cmd/chain.go | 13 +++++--- claimtrie/cmd/cmd/node.go | 5 +-- claimtrie/cmd/cmd/root.go | 5 ++- claimtrie/cmd/cmd/temporal.go | 2 +- claimtrie/config/config.go | 56 ++++++++++++++++++++-------------- claimtrie/node/manager_test.go | 2 +- claimtrie/param/general.go | 8 +---- 11 files changed, 74 insertions(+), 63 deletions(-) diff --git a/btcd.go b/btcd.go index 2e6915a8..4c54a1ff 100644 --- a/btcd.go +++ b/btcd.go @@ -148,7 +148,7 @@ func btcdMain(serverChan chan<- *server) error { return nil } - param.SetNetwork(activeNetParams.Params.Net, netName(activeNetParams)) // prep the claimtrie params + param.SetNetwork(activeNetParams.Params.Net) // prep the claimtrie params // Create server and start it. server, err := newServer(cfg.Listeners, cfg.AgentBlacklist, diff --git a/claimtrie/claimtrie.go b/claimtrie/claimtrie.go index 300298be..abce243a 100644 --- a/claimtrie/claimtrie.go +++ b/claimtrie/claimtrie.go @@ -3,6 +3,7 @@ package claimtrie import ( "bytes" "fmt" + "path/filepath" "runtime" "sort" @@ -58,18 +59,17 @@ type ClaimTrie struct { cleanups []func() error } -func New(record bool) (*ClaimTrie, error) { +func New(cfg config.Config) (*ClaimTrie, error) { - cfg := config.GenerateConfig(param.ClaimtrieDataFolder) var cleanups []func() error - blockRepo, err := blockrepo.NewPebble(cfg.BlockRepoPebble.Path) + blockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new block repo: %w", err) } cleanups = append(cleanups, blockRepo.Close) - temporalRepo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path) + temporalRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new temporal repo: %w", err) } @@ -77,7 +77,7 @@ func New(record bool) (*ClaimTrie, error) { // Initialize repository for changes to nodes. // The cleanup is delegated to the Node Manager. - nodeRepo, err := noderepo.NewPebble(cfg.NodeRepoPebble.Path) + nodeRepo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new node repo: %w", err) } @@ -91,7 +91,7 @@ func New(record bool) (*ClaimTrie, error) { // Initialize repository for MerkleTrie. // The cleanup is delegated to MerkleTrie. - trieRepo, err := merkletrierepo.NewPebble(cfg.MerkleTrieRepoPebble.Path) + trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new trie repo: %w", err) } @@ -128,15 +128,15 @@ func New(record bool) (*ClaimTrie, error) { height: previousHeight, } - if record { - chainRepo, err := chainrepo.NewPebble(cfg.ChainRepoPebble.Path) + if cfg.Record { + chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new change change repo: %w", err) } cleanups = append(cleanups, chainRepo.Close) ct.chainRepo = chainRepo - reportedBlockRepo, err := blockrepo.NewPebble(cfg.ReportedBlockRepoPebble.Path) + reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) if err != nil { return nil, fmt.Errorf("new reported block repo: %w", err) } diff --git a/claimtrie/claimtrie_test.go b/claimtrie/claimtrie_test.go index decbea0a..5945a181 100644 --- a/claimtrie/claimtrie_test.go +++ b/claimtrie/claimtrie_test.go @@ -3,6 +3,7 @@ package claimtrie import ( "testing" + "github.com/btcsuite/btcd/claimtrie/config" "github.com/btcsuite/btcd/claimtrie/merkletrie" "github.com/btcsuite/btcd/claimtrie/node" "github.com/btcsuite/btcd/claimtrie/param" @@ -13,9 +14,11 @@ import ( "github.com/stretchr/testify/require" ) +var cfg = config.DefaultConfig + func setup(t *testing.T) { - param.SetNetwork(wire.TestNet, "") - param.ClaimtrieDataFolder = t.TempDir() + param.SetNetwork(wire.TestNet) + cfg.DataDir = t.TempDir() } func b(s string) []byte { @@ -35,7 +38,7 @@ func TestFixedHashes(t *testing.T) { r := require.New(t) setup(t) - ct, err := New(false) + ct, err := New(cfg) r.NoError(err) defer func() { err = ct.Close() @@ -75,7 +78,7 @@ func TestNormalizationFork(t *testing.T) { setup(t) param.NormalizedNameForkHeight = 2 - ct, err := New(false) + ct, err := New(cfg) r.NoError(err) r.NotNil(ct) defer func() { @@ -139,7 +142,7 @@ func TestActivationsOnNormalizationFork(t *testing.T) { setup(t) param.NormalizedNameForkHeight = 4 - ct, err := New(false) + ct, err := New(cfg) r.NoError(err) r.NotNil(ct) defer func() { @@ -185,7 +188,7 @@ func TestNormalizationSortOrder(t *testing.T) { // alas, it's now part of our history; we hereby test it to keep it that way setup(t) param.NormalizedNameForkHeight = 2 - ct, err := New(false) + ct, err := New(cfg) r.NoError(err) r.NotNil(ct) defer func() { diff --git a/claimtrie/cmd/cmd/block.go b/claimtrie/cmd/cmd/block.go index 75e81f1d..4f4a8825 100644 --- a/claimtrie/cmd/cmd/block.go +++ b/claimtrie/cmd/cmd/block.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "log" + "path/filepath" "strconv" "github.com/btcsuite/btcd/claimtrie/block/blockrepo" @@ -32,7 +33,7 @@ var blockLastCmd = &cobra.Command{ Short: "Show the Merkle Hash of the last block", RunE: func(cmd *cobra.Command, args []string) error { - repo, err := blockrepo.NewPebble(localConfig.ReportedBlockRepoPebble.Path) + repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) if err != nil { log.Fatalf("can't open reported block repo: %s", err) } @@ -59,7 +60,7 @@ var blockListCmd = &cobra.Command{ Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { - repo, err := blockrepo.NewPebble(localConfig.ReportedBlockRepoPebble.Path) + repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) if err != nil { log.Fatalf("can't open reported block repo: %s", err) } @@ -104,7 +105,7 @@ var blockNameCmd = &cobra.Command{ Args: cobra.RangeArgs(2, 2), RunE: func(cmd *cobra.Command, args []string) error { - repo, err := blockrepo.NewPebble(localConfig.BlockRepoPebble.Path) + repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path)) if err != nil { return fmt.Errorf("can't open reported block repo: %w", err) } @@ -129,7 +130,7 @@ var blockNameCmd = &cobra.Command{ return fmt.Errorf("load previous height: %w", err) } - trieRepo, err := merkletrierepo.NewPebble(localConfig.MerkleTrieRepoPebble.Path) + trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path)) if err != nil { return fmt.Errorf("can't open merkle trie repo: %w", err) } @@ -140,7 +141,7 @@ var blockNameCmd = &cobra.Command{ if len(args) > 1 { trie.Dump(args[1], param.AllClaimsInMerkleForkHeight >= int32(height)) } else { - tmpRepo, err := temporalrepo.NewPebble(localConfig.TemporalRepoPebble.Path) + tmpRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path)) if err != nil { return fmt.Errorf("can't open temporal repo: %w", err) } diff --git a/claimtrie/cmd/cmd/chain.go b/claimtrie/cmd/cmd/chain.go index ae98fdba..7609e422 100644 --- a/claimtrie/cmd/cmd/chain.go +++ b/claimtrie/cmd/cmd/chain.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "os" + "path/filepath" "strconv" "github.com/btcsuite/btcd/claimtrie" @@ -11,6 +12,7 @@ import ( "github.com/btcsuite/btcd/claimtrie/block/blockrepo" "github.com/btcsuite/btcd/claimtrie/chain/chainrepo" "github.com/btcsuite/btcd/claimtrie/change" + "github.com/btcsuite/btcd/claimtrie/config" "github.com/btcsuite/btcd/claimtrie/node" "github.com/cockroachdb/pebble" @@ -48,7 +50,7 @@ var chainDumpCmd = &cobra.Command{ } } - chainRepo, err := chainrepo.NewPebble(localConfig.ChainRepoPebble.Path) + chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) if err != nil { return fmt.Errorf("open node repo: %w", err) } @@ -93,24 +95,25 @@ var chainReplayCmd = &cobra.Command{ } } - err = os.RemoveAll(localConfig.NodeRepoPebble.Path) + err = os.RemoveAll(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) if err != nil { return fmt.Errorf("delete node repo: %w", err) } fmt.Printf("Deleted node repo\n") - chainRepo, err := chainrepo.NewPebble(localConfig.ChainRepoPebble.Path) + chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) if err != nil { return fmt.Errorf("open change repo: %w", err) } - reportedBlockRepo, err := blockrepo.NewPebble(localConfig.ReportedBlockRepoPebble.Path) + reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) if err != nil { return fmt.Errorf("open block repo: %w", err) } - ct, err := claimtrie.New(false) + cfg := config.DefaultConfig + ct, err := claimtrie.New(cfg) if err != nil { return fmt.Errorf("create claimtrie: %w", err) } diff --git a/claimtrie/cmd/cmd/node.go b/claimtrie/cmd/cmd/node.go index 9af78742..a943ee41 100644 --- a/claimtrie/cmd/cmd/node.go +++ b/claimtrie/cmd/cmd/node.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "math" + "path/filepath" "strconv" "github.com/btcsuite/btcd/claimtrie/node" @@ -29,7 +30,7 @@ var nodeDumpCmd = &cobra.Command{ Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { - repo, err := noderepo.NewPebble(localConfig.NodeRepoPebble.Path) + repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) if err != nil { return fmt.Errorf("open node repo: %w", err) } @@ -66,7 +67,7 @@ var nodeReplayCmd = &cobra.Command{ Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { - repo, err := noderepo.NewPebble(localConfig.NodeRepoPebble.Path) + repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) if err != nil { return fmt.Errorf("open node repo: %w", err) } diff --git a/claimtrie/cmd/cmd/root.go b/claimtrie/cmd/cmd/root.go index 4a05534c..be84f5d6 100644 --- a/claimtrie/cmd/cmd/root.go +++ b/claimtrie/cmd/cmd/root.go @@ -8,11 +8,10 @@ import ( "github.com/spf13/cobra" ) -var localConfig *config.DBConfig +var cfg = config.DefaultConfig func init() { - param.SetNetwork(wire.MainNet, "mainnet") - localConfig = config.GenerateConfig(param.ClaimtrieDataFolder) + param.SetNetwork(wire.MainNet) } var rootCmd = &cobra.Command{ diff --git a/claimtrie/cmd/cmd/temporal.go b/claimtrie/cmd/cmd/temporal.go index 3254d4c2..85f94850 100644 --- a/claimtrie/cmd/cmd/temporal.go +++ b/claimtrie/cmd/cmd/temporal.go @@ -23,7 +23,7 @@ var temporalCmd = &cobra.Command{ func runListNodes(cmd *cobra.Command, args []string) error { - repo, err := temporalrepo.NewPebble(localConfig.TemporalRepoPebble.Path) + repo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path) if err != nil { log.Fatalf("can't open reported block repo: %s", err) } diff --git a/claimtrie/config/config.go b/claimtrie/config/config.go index 504cf074..99c2b118 100644 --- a/claimtrie/config/config.go +++ b/claimtrie/config/config.go @@ -2,33 +2,43 @@ package config import ( "path/filepath" + + "github.com/btcsuite/btcutil" ) -func GenerateConfig(folder string) *DBConfig { - return &DBConfig{ - BlockRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "blocks_pebble_db"), - }, - NodeRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "node_change_pebble_db"), - }, - TemporalRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "temporal_pebble_db"), - }, - MerkleTrieRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "merkletrie_pebble_db"), - }, - ChainRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "chain_pebble_db"), - }, - ReportedBlockRepoPebble: pebbleConfig{ - Path: filepath.Join(folder, "reported_blocks_pebble_db"), - }, - } +var DefaultConfig = Config{ + Record: false, + RamTrie: false, + + DataDir: filepath.Join(btcutil.AppDataDir("chain", false), "data", "mainnet", "claim_dbs"), + + BlockRepoPebble: pebbleConfig{ + Path: "blocks_pebble_db", + }, + NodeRepoPebble: pebbleConfig{ + Path: "node_change_pebble_db", + }, + TemporalRepoPebble: pebbleConfig{ + Path: "temporal_pebble_db", + }, + MerkleTrieRepoPebble: pebbleConfig{ + Path: "merkletrie_pebble_db", + }, + ChainRepoPebble: pebbleConfig{ + Path: "chain_pebble_db", + }, + ReportedBlockRepoPebble: pebbleConfig{ + Path: "reported_blocks_pebble_db", + }, } -// DBConfig is the container of all configurations. -type DBConfig struct { +// Config is the container of all configurations. +type Config struct { + Record bool + RamTrie bool + + DataDir string + BlockRepoPebble pebbleConfig NodeRepoPebble pebbleConfig TemporalRepoPebble pebbleConfig diff --git a/claimtrie/node/manager_test.go b/claimtrie/node/manager_test.go index b6ea51b9..0a1b557c 100644 --- a/claimtrie/node/manager_test.go +++ b/claimtrie/node/manager_test.go @@ -45,7 +45,7 @@ func TestSimpleAddClaim(t *testing.T) { r := require.New(t) - param.SetNetwork(wire.TestNet, "") + param.SetNetwork(wire.TestNet) repo, err := noderepo.NewPebble(t.TempDir()) r.NoError(err) diff --git a/claimtrie/param/general.go b/claimtrie/param/general.go index 1eff2517..0ed110a0 100644 --- a/claimtrie/param/general.go +++ b/claimtrie/param/general.go @@ -2,8 +2,6 @@ package param import ( "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" - "path/filepath" ) var ( @@ -20,16 +18,12 @@ var ( NormalizedNameForkHeight int32 AllClaimsInMerkleForkHeight int32 - - ClaimtrieDataFolder string ) -func SetNetwork(net wire.BitcoinNet, subfolder string) { +func SetNetwork(net wire.BitcoinNet) { MaxActiveDelay = 4032 ActiveDelayFactor = 32 MaxNodeManagerCacheSize = 16000 - appDir := btcutil.AppDataDir("chain", false) - ClaimtrieDataFolder = filepath.Join(appDir, "data", subfolder, "claim_dbs") switch net { case wire.MainNet: