From 88dbf2267c33e1ebe489c16b73d54cdec6f19067 Mon Sep 17 00:00:00 2001 From: Roy Lee Date: Tue, 3 Aug 2021 12:01:01 -0700 Subject: [PATCH] [lbry] cli: switch from utxoview to native map. The utxo is too big to hold in the memory. (~58 GB at 860K blocks) Since we're extracting claim scripts from an already validated database, a dumb native map saves us memory as well as overhead maintaining a uxto. This takes 8.5 minutes to extract claim scripts in 1M blocks, but still takes ~56 GB of memory. --- claimtrie/cmd/cmd/chain.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/claimtrie/cmd/cmd/chain.go b/claimtrie/cmd/cmd/chain.go index 52a92d02..f792e5f9 100644 --- a/claimtrie/cmd/cmd/chain.go +++ b/claimtrie/cmd/cmd/chain.go @@ -331,23 +331,19 @@ func (cb *chainConverter) processBlock() { defer cb.wg.Done() defer close(cb.changesChan) - view := blockchain.NewUtxoViewpoint() + utxoPubScripts := map[wire.OutPoint][]byte{} for block := range cb.blockChan { var changes []change.Change for _, tx := range block.Transactions() { - view.AddTxOuts(tx, block.Height()) if blockchain.IsCoinBase(tx) { continue } for _, txIn := range tx.MsgTx().TxIn { - op := txIn.PreviousOutPoint - e := view.LookupEntry(op) - if e == nil { - log.Criticalf("Missing input in view for %s", op.String()) - } - cs, err := txscript.DecodeClaimScript(e.PkScript()) + prevOutpoint := txIn.PreviousOutPoint + pkScript := utxoPubScripts[prevOutpoint] + cs, err := txscript.DecodeClaimScript(pkScript) if err == txscript.ErrNotClaimScript { continue } @@ -360,6 +356,7 @@ func (cb *chainConverter) processBlock() { Name: cs.Name(), OutPoint: txIn.PreviousOutPoint, } + delete(utxoPubScripts, prevOutpoint) switch cs.Opcode() { case txscript.OP_CLAIMNAME: @@ -390,6 +387,7 @@ func (cb *chainConverter) processBlock() { OutPoint: op, Amount: txOut.Value, } + utxoPubScripts[op] = txOut.PkScript switch cs.Opcode() { case txscript.OP_CLAIMNAME: