added channel/signature annotation to all TXOs, support for pagination

lbrynet/daemon/Daemon.py

@@ -1166,7 +1166,7 @@ class Daemon(AuthJSONRPCServer):
 
     @requires("wallet")
     @defer.inlineCallbacks
-    def jsonrpc_account_balance(self, account_id=None, address=None, include_unconfirmed=False):
+    def jsonrpc_account_balance(self, account_id=None, confirmations=0):
         """
         Return the balance of an account
 
@@ -1174,21 +1174,16 @@ class Daemon(AuthJSONRPCServer):
             account_balance [<account_id>] [<address> | --address=<address>] [--include_unconfirmed]
 
         Options:
-            --account_id=<account_id> : (str) If provided only the balance for this
+            --account_id=<account_id>       : (str) If provided only the balance for this
-                                        account will be given
+                                              account will be given. Otherwise default account.
-            --address=<address>       : (str) If provided only the balance for this
+            --confirmations=<confirmations> : (int) Only include transactions with this many
-                                        address will be given
+                                              confirmed blocks.
-            --include_unconfirmed     : (bool) Include unconfirmed
 
         Returns:
             (decimal) amount of lbry credits in wallet
         """
-        if address is not None:
-            raise NotImplementedError("Limiting by address needs to be re-implemented in new wallet.")
         account = self.get_account_or_default(account_id)
-        dewies = yield account.get_balance(
+        dewies = yield account.get_balance(confirmations=confirmations)
-            0 if include_unconfirmed else 6
-        )
         return dewies_to_lbc(dewies)
 
     @requires("wallet")
@@ -1513,7 +1508,7 @@ class Daemon(AuthJSONRPCServer):
         )
 
     @requires(WALLET_COMPONENT)
-    def jsonrpc_address_list(self, account_id=None):
+    def jsonrpc_address_list(self, account_id=None, offset=None, limit=None):
         """
         List account addresses
 
@@ -1522,11 +1517,26 @@ class Daemon(AuthJSONRPCServer):
 
         Options:
             --account_id=<account_id>  : (str) id of the account to use
+            --offset=<offset>          : (int) slice address list starting at offset
+            --limit=<limit>            : (int) limit number of addresses returned
 
         Returns:
             List of wallet addresses
         """
-        return self.get_account_or_default(account_id).get_addresses()
+        account = self.get_account_or_default(account_id)
+        if None not in (offset, limit):
+            constraints = {
+                'account': account,
+                'offset': offset,
+                'limit': limit
+            }
+            return {
+                "list": self.ledger.db.get_addresses(**constraints),
+                "size": self.ledger.db.get_addresses_count(**constraints),
+                "offset": offset,
+                "limit": limit
+            }
+        return account.get_addresses()
 
     @requires(WALLET_COMPONENT)
     def jsonrpc_address_unused(self, account_id=None):
@@ -2033,16 +2043,16 @@ class Daemon(AuthJSONRPCServer):
         nout = 0
         txo = tx.outputs[nout]
         log.info("Claimed a new channel! lbry://%s txid: %s nout: %d", channel_name, tx.id, nout)
-        defer.returnValue({
+        return {
             "success": True,
             "tx": tx,
             "claim_id": txo.claim_id,
-            "claim_address": self.ledger.hash160_to_address(txo.script.values['pubkey_hash']),
+            "claim_address": txo.get_address(self.ledger),
             "output": txo
-        })
+        }
 
     @requires(WALLET_COMPONENT)
-    def jsonrpc_channel_list(self):
+    def jsonrpc_channel_list(self, account_id=None, offset=None, limit=None):
         """
         Get certificate claim infos for channels that can be published to
 
@@ -2050,13 +2060,28 @@ class Daemon(AuthJSONRPCServer):
             channel_list
 
         Options:
-            None
+            --account_id=<account_id>  : (str) id of the account to use
+            --offset=<offset>          : (int) slice channel list starting at offset
+            --limit=<limit>            : (int) limit number of channels returned
 
         Returns:
             (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim
             is in the wallet.
         """
-        return self.wallet_manager.channel_list()
+        account = self.get_account_or_default(account_id)
+        if None not in (offset, limit):
+            constraints = {
+                'account': account,
+                'offset': offset,
+                'limit': limit
+            }
+            return {
+                "list": self.ledger.db.get_channels(**constraints),
+                "size": self.ledger.db.get_channels_count(**constraints),
+                "offset": offset,
+                "limit": limit
+            }
+        return account.get_channels()
 
     @requires(WALLET_COMPONENT)
     @defer.inlineCallbacks
@@ -2450,9 +2475,8 @@ class Daemon(AuthJSONRPCServer):
             claim_id, address, self.get_dewies_or_error("amount", amount) if amount else None
         )
 
-    # TODO: claim_list_mine should be merged into claim_list, but idk how to authenticate it -Grin
     @requires(WALLET_COMPONENT)
-    def jsonrpc_claim_list_mine(self, account_id=None):
+    def jsonrpc_claim_list_mine(self, account_id=None, offset=None, limit=None):
         """
         List my name claims
 
@@ -2461,6 +2485,8 @@ class Daemon(AuthJSONRPCServer):
 
         Options:
             --account_id=<account_id> : (str) id of the account to query
+            --offset=<offset>         : (int) slice claim list starting at offset
+            --limit=<limit>           : (int) limit number of claims returned
 
         Returns:
             (list) List of name claims owned by user
@@ -2484,7 +2510,20 @@ class Daemon(AuthJSONRPCServer):
                 },
            ]
         """
-        return self.get_account_or_default(account_id).get_claims()
+        account = self.get_account_or_default(account_id)
+        if None not in (offset, limit):
+            constraints = {
+                'account': account,
+                'offset': offset,
+                'limit': limit
+            }
+            return {
+                "list": self.ledger.db.get_claims(**constraints),
+                "size": self.ledger.db.get_claims_count(**constraints),
+                "offset": offset,
+                "limit": limit
+            }
+        return account.get_claims()
 
     @requires(WALLET_COMPONENT)
     @defer.inlineCallbacks
@@ -2616,7 +2655,8 @@ class Daemon(AuthJSONRPCServer):
         return response
 
     @requires(WALLET_COMPONENT)
-    def jsonrpc_transaction_list(self, account_id=None):
+    @defer.inlineCallbacks
+    def jsonrpc_transaction_list(self, account_id=None, offset=None, limit=None):
         """
         List transactions belonging to wallet
 
@@ -2625,6 +2665,8 @@ class Daemon(AuthJSONRPCServer):
 
         Options:
             --account_id=<account_id> : (str) id of the account to query
+            --offset=<offset>         : (int) slice transaction list starting at offset
+            --limit=<limit>           : (int) limit number of transactions returned
 
         Returns:
             (list) List of transactions
@@ -2672,7 +2714,21 @@ class Daemon(AuthJSONRPCServer):
             }
 
         """
-        return self.wallet_manager.get_history(self.get_account_or_default(account_id))
+        account = self.get_account_or_default(account_id)
+        if None not in (offset, limit):
+            constraints = {
+                'offset': offset,
+                'limit': limit
+            }
+            return {
+                "list": self.wallet_manager.get_history(
+                    account=account, **constraints),
+                "size": self.ledger.db.get_transactions_count(
+                    account=account, **constraints),
+                "offset": offset,
+                "limit": limit
+            }
+        return self.wallet_manager.get_history(account)
 
     @requires(WALLET_COMPONENT)
     def jsonrpc_transaction_show(self, txid):
@@ -2691,7 +2747,7 @@ class Daemon(AuthJSONRPCServer):
         return self.wallet_manager.get_transaction(txid)
 
     @requires(WALLET_COMPONENT)
-    def jsonrpc_utxo_list(self, account_id=None):
+    def jsonrpc_utxo_list(self, account_id=None, offset=None, limit=None):
         """
         List unspent transaction outputs
 
@@ -2700,6 +2756,8 @@ class Daemon(AuthJSONRPCServer):
 
         Options:
             --account_id=<account_id> : (str) id of the account to query
+            --offset=<offset>         : (int) slice utxo list starting at offset
+            --limit=<limit>           : (int) limit number of utxo returned
 
         Returns:
             (list) List of unspent transaction outputs (UTXOs)
@@ -2718,7 +2776,20 @@ class Daemon(AuthJSONRPCServer):
                 ...
             ]
         """
-        return self.get_account_or_default(account_id).get_unspent_outputs()
+        account = self.get_account_or_default(account_id)
+        if None not in (offset, limit):
+            constraints = {
+                'account': account,
+                'offset': offset,
+                'limit': limit
+            }
+            return {
+                "list": self.ledger.db.get_utxos(**constraints),
+                "size": self.ledger.db.get_utxo_count(**constraints),
+                "offset": offset,
+                "limit": limit
+            }
+        return account.get_utxos()
 
     @requires(WALLET_COMPONENT)
     def jsonrpc_block_show(self, blockhash=None, height=None):

lbrynet/daemon/json_response_encoder.py

@@ -48,6 +48,7 @@ class JSONResponseEncoder(JSONEncoder):
             output['is_change'] = txo.is_change
         if txo.is_my_account is not None:
             output['is_mine'] = txo.is_my_account
+
         if txo.script.is_claim_involved:
             output.update({
                 'name': txo.claim_name,
@@ -57,18 +58,19 @@ class JSONResponseEncoder(JSONEncoder):
                 'is_support': txo.script.is_support_claim,
                 'is_update': txo.script.is_update_claim
             })
+
+            if txo.script.is_claim_name or txo.script.is_update_claim:
+                output['value'] = txo.claim.claim_dict
+                if txo.claim_name.startswith('@'):
+                    output['has_signature'] = txo.has_signature
+
             if txo.script.is_claim_name:
-                output.update({
+                output['category'] = 'claim'
-                    'category': 'claim',
-                    'value': txo.claim.claim_dict
-                })
             elif txo.script.is_update_claim:
-                output.update({
+                output['category'] = 'update'
-                    'category': 'update',
-                    'value': txo.claim.claim_dict
-                })
             elif txo.script.is_support_claim:
                 output['category'] = 'support'
+
         return output
 
     def encode_input(self, txi):

lbrynet/wallet/account.py

@@ -120,13 +120,13 @@ class Account(BaseAccount):
             constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
         return super().get_balance(confirmations, **constraints)
 
-    def get_unspent_outputs(self, include_claims=False, **constraints):
+    def get_utxos(self, include_claims=False, **constraints):
         if not include_claims:
             constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
-        return super().get_unspent_outputs(**constraints)
+        return super().get_utxos(**constraints)
 
     def get_channels(self):
-        return super().get_unspent_outputs(
+        return super().get_utxos(
             claim_type__any={'is_claim': 1, 'is_update': 1},
             claim_name__like='@%'
         )

lbrynet/wallet/certificate.py

@@ -1,5 +0,0 @@
-from collections import namedtuple
-
-
-class Certificate(namedtuple('Certificate', ('channel', 'private_key'))):
-    pass

lbrynet/wallet/database.py

@@ -1,6 +1,5 @@
 from twisted.internet import defer
 from torba.basedatabase import BaseDatabase
-from .certificate import Certificate
 
 
 class WalletDatabase(BaseDatabase):
@@ -48,21 +47,24 @@ class WalletDatabase(BaseDatabase):
 
     @defer.inlineCallbacks
     def get_txos(self, **constraints):
-        txos = yield super().get_txos(**constraints)
+        my_account = constraints.get('my_account', constraints.get('account', None))
-        my_account = constraints.get('my_account', constraints.get('account'))
 
-        claim_ids = set()
+        txos = yield super().get_txos(**constraints)
+
+        channel_ids = set()
         for txo in txos:
             if txo.script.is_claim_name or txo.script.is_update_claim:
                 if 'publisherSignature' in txo.claim_dict:
-                    claim_ids.add(txo.claim_dict['publisherSignature']['certificateId'])
+                    channel_ids.add(txo.claim_dict['publisherSignature']['certificateId'])
+                if txo.claim_name.startswith('@') and my_account is not None:
+                    txo.signature = my_account.get_certificate_private_key(txo.ref)
 
-        if claim_ids:
+        if channel_ids:
             channels = {
                 txo.claim_id: txo for txo in
                 (yield super().get_utxos(
                     my_account=my_account,
-                    claim_id__in=claim_ids
+                    claim_id__in=channel_ids
                 ))
             }
             for txo in txos:
@@ -72,27 +74,45 @@ class WalletDatabase(BaseDatabase):
 
         return txos
 
-    def get_claims(self, **constraints):
+    @staticmethod
+    def constrain_claims(constraints):
         constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1}
+
+    def get_claims(self, **constraints):
+        self.constrain_claims(constraints)
         return self.get_utxos(**constraints)
 
-    def get_channels(self, **constraints):
+    def get_claims_count(self, **constraints):
+        self.constrain_claims(constraints)
+        return self.get_utxo_count(**constraints)
+
+    @staticmethod
+    def constrain_channels(constraints):
         if 'claim_name' not in constraints or 'claim_id' not in constraints:
             constraints['claim_name__like'] = '@%'
+
+    def get_channels(self, **constraints):
+        self.constrain_channels(constraints)
         return self.get_claims(**constraints)
 
+    def get_channels_count(self, **constraints):
+        self.constrain_channels(constraints)
+        return self.get_claims_count(**constraints)
+
     @defer.inlineCallbacks
     def get_certificates(self, private_key_accounts, exclude_without_key=False, **constraints):
         channels = yield self.get_channels(**constraints)
         certificates = []
         if private_key_accounts is not None:
             for channel in channels:
-                private_key = None
+                if not channel.has_signature:
-                for account in private_key_accounts:
+                    private_key = None
-                    private_key = account.get_certificate_private_key(channel.ref)
+                    for account in private_key_accounts:
-                    if private_key is not None:
+                        private_key = account.get_certificate_private_key(channel.ref)
-                        break
+                        if private_key is not None:
-                if private_key is None and exclude_without_key:
+                            break
-                    continue
+                    if private_key is None and exclude_without_key:
-                certificates.append(Certificate(channel, private_key))
+                        continue
+                    channel.signature = private_key
+                certificates.append(channel)
         return certificates

lbrynet/wallet/manager.py

@@ -240,9 +240,9 @@ class LbryWalletManager(BaseWalletManager):
 
     @defer.inlineCallbacks
     def address_is_mine(self, unknown_address, account):
-        for my_address in (yield account.get_addresses()):
+        match = yield self.ledger.db.get_address(address=unknown_address, account=account)
-            if unknown_address == my_address:
+        if match is not None:
-                return True
+            return True
         return False
 
     def get_transaction(self, txid: str):
@@ -250,9 +250,9 @@ class LbryWalletManager(BaseWalletManager):
 
     @staticmethod
     @defer.inlineCallbacks
-    def get_history(account: BaseAccount):
+    def get_history(account: BaseAccount, **constraints):
         headers = account.ledger.headers
-        txs: List[Transaction] = (yield account.get_transactions())
+        txs = (yield account.get_transactions(account=account, **constraints))
         history = []
         for tx in txs:
             ts = headers[tx.height]['timestamp']
@@ -301,7 +301,7 @@ class LbryWalletManager(BaseWalletManager):
 
     @staticmethod
     def get_utxos(account: BaseAccount):
-        return account.get_unspent_outputs()
+        return account.get_utxos()
 
     @defer.inlineCallbacks
     def claim_name(self, name, amount, claim_dict, certificate=None, claim_address=None):
@@ -311,9 +311,9 @@ class LbryWalletManager(BaseWalletManager):
             claim_address = yield account.receiving.get_or_create_usable_address()
         if certificate:
             claim = claim.sign(
-                certificate.private_key, claim_address, certificate.channel.claim_id
+                certificate.signature, claim_address, certificate.claim_id
             )
-        existing_claims = yield account.get_unspent_outputs(include_claims=True, claim_name=name)
+        existing_claims = yield account.get_utxos(include_claims=True, claim_name=name)
         if len(existing_claims) == 0:
             tx = yield Transaction.claim(
                 name, claim, amount, claim_address, [account], account
@@ -382,9 +382,6 @@ class LbryWalletManager(BaseWalletManager):
         # TODO: release reserved tx outputs in case anything fails by this point
         defer.returnValue(tx)
 
-    def channel_list(self):
-        return self.default_account.get_channels()
-
     def get_certificates(self, private_key_accounts, exclude_without_key=True, **constraints):
         return self.db.get_certificates(
             private_key_accounts=private_key_accounts,

lbrynet/wallet/transaction.py

@@ -19,16 +19,19 @@ class Output(BaseOutput):
     script: OutputScript
     script_class = OutputScript
 
-    __slots__ = '_claim_dict', 'channel'
+    __slots__ = '_claim_dict', 'channel', 'signature'
 
-    def __init__(self, *args, channel: Optional['Output'] = None, **kwargs) -> None:
+    def __init__(self, *args, channel: Optional['Output'] = None,
+                 signature: Optional[str] = None, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self._claim_dict = None
         self.channel = channel
+        self.signature = signature
 
     def update_annotations(self, annotated):
         super().update_annotations(annotated)
         self.channel = annotated.channel if annotated else None
+        self.signature = annotated.signature if annotated else None
 
     def get_fee(self, ledger):
         name_fee = 0
@@ -76,6 +79,10 @@ class Output(BaseOutput):
             return "{}#{}".format(self.claim_name, self.claim_id)
         raise ValueError('No claim associated.')
 
+    @property
+    def has_signature(self):
+        return self.signature is not None
+
     @classmethod
     def pay_claim_name_pubkey_hash(
             cls, amount: int, claim_name: str, claim: bytes, pubkey_hash: bytes) -> 'Output':

tests/integration/wallet/test_commands.py

@@ -113,7 +113,7 @@ class CommandTestCase(IntegrationTestCase):
         lbry_conf.settings.node_id = None
 
         await d2f(self.account.ensure_address_gap())
-        address = (await d2f(self.account.receiving.get_addresses(1, only_usable=True)))[0]
+        address = (await d2f(self.account.receiving.get_addresses(limit=1, only_usable=True)))[0]
         sendtxid = await self.blockchain.send_to_address(address, 10)
         await self.confirm_tx(sendtxid)
         await self.generate(5)
@@ -197,7 +197,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
         channels = yield self.out(self.daemon.jsonrpc_channel_list())
         self.assertEqual(len(channels), 1)
         self.assertEqual(channels[0]['name'], '@spam')
-        self.assertTrue(channels[0]['have_certificate'])
+        self.assertTrue(channels[0]['has_signature'])
 
         # As the new channel claim travels through the intertubes and makes its
         # way into the mempool and then a block and then into the claimtrie,

tests/integration/wallet/test_transactions.py

@@ -46,7 +46,7 @@ class BasicTransactionTest(IntegrationTestCase):
 
         await d2f(self.account.ensure_address_gap())
 
-        address1, address2 = await d2f(self.account.receiving.get_addresses(2, only_usable=True))
+        address1, address2 = await d2f(self.account.receiving.get_addresses(limit=2, only_usable=True))
         sendtxid1 = await self.blockchain.send_to_address(address1, 5)
         sendtxid2 = await self.blockchain.send_to_address(address2, 5)
         await self.blockchain.generate(1)

tests/unit/wallet/test_ledger.py

@@ -63,7 +63,7 @@ class BasicAccountingTests(LedgerTestCase):
             'insert', tx, address, hash160, '{}:{}:'.format(tx.id, 1)
         )
 
-        utxos = yield self.account.get_unspent_outputs()
+        utxos = yield self.account.get_utxos()
         self.assertEqual(len(utxos), 1)
 
         tx = Transaction(is_verified=True)\
@@ -74,6 +74,6 @@ class BasicAccountingTests(LedgerTestCase):
         balance = yield self.account.get_balance(0, include_claims=True)
         self.assertEqual(balance, 0)
 
-        utxos = yield self.account.get_unspent_outputs()
+        utxos = yield self.account.get_utxos()
         self.assertEqual(len(utxos), 0)
 

tests/unit/wallet/test_transaction.py

@@ -244,7 +244,7 @@ class TestTransactionSigning(unittest.TestCase):
         )
 
         yield account.ensure_address_gap()
-        address1, address2 = yield account.receiving.get_addresses(2)
+        address1, address2 = yield account.receiving.get_addresses(limit=2)
         pubkey_hash1 = self.ledger.address_to_hash160(address1)
         pubkey_hash2 = self.ledger.address_to_hash160(address2)