sync_apply progress

lbry/db/queries/base.py

@@ -41,7 +41,7 @@ def get_best_block_height():
 
 
 def insert_block(block):
-    context().get_bulk_loader().add_block(block).flush()
+    context().get_bulk_loader().add_block(block).flush(return_row_count_for_table=None)
 
 
 def get_block_headers(first, last=None):

lbry/service/api.py

@@ -14,7 +14,7 @@ from aiohttp import ClientSession
 from lbry.conf import Setting, NOT_SET
 from lbry.db import TXO_TYPES, CLAIM_TYPE_NAMES
 from lbry.db.utils import constrain_single_or_list
-from lbry.wallet import Wallet, Account, SingleKey, HierarchicalDeterministic
+from lbry.wallet import Wallet, Account, SingleKey, HierarchicalDeterministic, ENCRYPT_ON_DISK
 from lbry.blockchain import Transaction, Output, dewies_to_lbc, dict_values_to_lbc
 from lbry.stream.managed_stream import ManagedStream
 from lbry.event import EventController, EventStream
@@ -675,7 +675,7 @@ class API:
         if value and isinstance(value, str) and value[0] in ('[', '{'):
             value = json.loads(value)
         wallet.preferences[key] = value
-        wallet.save()
+        await wallet.notify_change('preference.set')
         return {key: value}
 
     WALLET_DOC = """
@@ -1151,23 +1151,23 @@ class API:
         wallet_changed = False
         if data is not None:
             added_accounts = await wallet.merge(password, data)
-            if added_accounts and self.ledger.sync.network.is_connected:
+            # if added_accounts and self.ledger.sync.network.is_connected:
-                if blocking:
+            #     if blocking:
-                    await asyncio.wait([
+            #         await asyncio.wait([
-                        a.ledger.subscribe_account(a) for a in added_accounts
+            #             a.ledger.subscribe_account(a) for a in added_accounts
-                    ])
+            #         ])
-                else:
+            #     else:
-                    for new_account in added_accounts:
+            #         for new_account in added_accounts:
-                        asyncio.create_task(self.ledger.subscribe_account(new_account))
+            #             asyncio.create_task(self.ledger.subscribe_account(new_account))
             wallet_changed = True
         if wallet.preferences.get(ENCRYPT_ON_DISK, False) and password != wallet.encryption_password:
             wallet.encryption_password = password
             wallet_changed = True
         if wallet_changed:
-            wallet.save()
+            await wallet.notify_change('sync')
         encrypted = wallet.pack(password)
         return {
-            'hash': self.sync_hash(wallet_id),
+            'hash': await self.sync_hash(wallet_id),
             'data': encrypted.decode()
         }
 
@@ -2545,9 +2545,15 @@ class API:
         Usage:
             block list <start_height> [<end_height>]
         """
-        return await self.service.sync.get_block_headers(
+        headers = await self.service.sync.get_block_headers(
             start_height=start_height, end_height=end_height
         )
+        for header in headers:
+            header.update({
+                'block_hash': hexlify(header['block_hash']),
+                'previous_hash': hexlify(header['previous_hash'])
+            })
+        return headers
 
     async def block_tip(self) -> int:  # block number at the tip of the blockchain
         """
@@ -3457,7 +3463,12 @@ class Client(API):
             else:
                 raise ValueError(f'Unknown message received: {d}')
 
-    async def send(self, method, **kwargs) -> EventStream:
+    async def send(self, method, *args, **kwargs) -> EventStream:
+        if args:
+            raise TypeError(
+                f"API client requires all named parameter arguments, "
+                f"received positional arguments instead: {args}"
+            )
         self.message_id += 1
         self.requests[self.message_id] = ec = EventController()
         await self.ws.send_json({'id': self.message_id, 'method': method, 'params': kwargs})

lbry/service/light_client.py

@@ -111,14 +111,13 @@ class FilterManager:
         self.cache = {}
 
     async def download(self):
-        filters_response = await self.client.get_address_filters(0, 500)
+        filters = await self.client.first.address_filter(start_height=0, end_height=500, granularity=0)
-        filters = await filters_response.first
+        #address = None
-        address = None
+        #address_array = [bytearray(self.db.ledger.address_to_hash160(address))]
-        address_array = [bytearray(self.client.ledger.address_to_hash160(address))]
+        #for address_filter in filters:
-        for address_filter in filters:
+        #    print(address_filter)
-            print(address_filter)
+        #    address_filter = get_address_filter(unhexlify(address_filter['filter']))
-            address_filter = get_address_filter(unhexlify(address_filter['filter']))
+        #    print(address_filter.MatchAny(address_array))
-            print(address_filter.MatchAny(address_array))
 
 
 #        address_array = [
@@ -159,19 +158,19 @@ class BlockHeaderManager:
 
     async def download(self):
         our_height = await self.db.get_best_block_height()
-        best_height = await self.client.block_tip()
+        best_height = await self.client.first.block_tip()
-        for block in await self.client.block_list(our_height+1, best_height):
+        for block in await self.client.first.block_list(start_height=our_height+1, end_height=best_height):
             await self.db.insert_block(Block(
                 height=block["height"],
                 version=0,
                 file_number=0,
-                block_hash=block["block_hash"],
+                block_hash=unhexlify(block["block_hash"]),
-                prev_block_hash=block["previous_hash"],
+                prev_block_hash=unhexlify(block["previous_hash"]),
-                merkle_root=block["merkle_root"],
+                merkle_root=b'',  # block["merkle_root"],
-                claim_trie_root=block["claim_trie_root"],
+                claim_trie_root=b'',  # block["claim_trie_root"],
                 timestamp=block["timestamp"],
-                bits=block["bits"],
+                bits=0,  # block["bits"],
-                nonce=block["nonce"],
+                nonce=0,  # block["nonce"],
                 txs=[]
             ))
 

lbry/testcase.py

@@ -27,7 +27,7 @@ from lbry.blockchain.bcd_data_stream import BCDataStream
 from lbry.blockchain.lbrycrd import Lbrycrd
 from lbry.blockchain.dewies import lbc_to_dewies
 from lbry.constants import COIN, CENT, NULL_HASH32
-from lbry.service import Daemon, FullNode, FullEndpoint, LightClient, jsonrpc_dumps_pretty
+from lbry.service import API, Daemon, Service, FullNode, FullEndpoint, LightClient, jsonrpc_dumps_pretty
 from lbry.conf import Config
 from lbry.console import Console
 from lbry.wallet import Wallet, Account
@@ -404,15 +404,8 @@ class IntegrationTestCase(AsyncioTestCase):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.ledger: Optional[RegTestLedger] = None
-        self.chain: Optional[Lbrycrd] = None
-        self.block_expected = 0
-        self._pg_db_counter = 0
         self._api_port = 5252
-        self.service = None
+        self.chain: Lbrycrd = None
-        self.api = None
-        self.wallet: Optional[Wallet] = None
-        self.account: Optional[Account] = None
 
     async def asyncSetUp(self):
         await super().asyncSetUp()
@@ -421,29 +414,37 @@ class IntegrationTestCase(AsyncioTestCase):
         self.addCleanup(self.chain.stop)
         await self.chain.start(*self.LBRYCRD_ARGS)
 
-    async def provision_db_from_environment(self, conf):
+    async def provision_db_from_environment(self, conf, uid):
-        self.db_driver = os.environ.get('TEST_DB', 'sqlite')
+        test_db = os.environ.get('TEST_DB', 'sqlite')
-        if self.db_driver == 'sqlite':
+        if test_db == 'sqlite':
             pass
-        elif self.db_driver.startswith('postgres') or self.db_driver.startswith('psycopg'):
+        elif test_db.startswith('postgres') or test_db.startswith('psycopg'):
-            self._pg_db_counter += 1
+            db_name = f'lbry_test_db_{uid}'
-            self.db_driver = 'postgresql'
-            db_name = f'lbry_test_db_{self._pg_db_counter}'
             db_connection = 'postgres:postgres@localhost:5432'
             meta_db = Database.from_url(f'postgresql://{db_connection}/postgres')
             await meta_db.drop(db_name)
             await meta_db.create(db_name)
             conf.db_url = f'postgresql://{db_connection}/{db_name}'
         else:
-            raise RuntimeError(f"Unsupported database driver: {self.db_driver}")
+            raise RuntimeError(
+                f"Unsupported TEST_DB: '{test_db}', "
+                f"valid options: sqlite, postgres."
+            )
 
     async def make_daemons_from_environment(self, **kwargs) -> Tuple[Daemon, Daemon]:
-        full_node_daemon = client_daemon = await self.make_full_node_daemon(**kwargs)
+        full_node_daemon = await self.make_full_node_daemon(**kwargs)
-        self.test_mode = os.environ.get('TEST_MODE', 'node')
+        test_mode = os.environ.get('TEST_MODE', 'node')
-        if self.test_mode == 'client':
+        if test_mode == 'node':
+            client_daemon = full_node_daemon
+        elif test_mode == 'client':
             client_daemon = await self.make_light_client_daemon(full_node_daemon, **kwargs)
-        elif self.test_mode == 'endpoint':
+        elif test_mode == 'endpoint':
             client_daemon = await self.make_full_endpoint_daemon(full_node_daemon, **kwargs)
+        else:
+            raise RuntimeError(
+                f"Unsupported TEST_MODE: '{test_mode}', "
+                f"valid options: node, client, endpoint."
+            )
         return full_node_daemon, client_daemon
 
     async def make_full_node_daemon(self, start=True, **conf_kwargs):
@@ -459,7 +460,7 @@ class IntegrationTestCase(AsyncioTestCase):
             lbrycrd_zmq=self.chain.ledger.conf.lbrycrd_zmq,
             **conf_kwargs
         )
-        await self.provision_db_from_environment(conf)
+        await self.provision_db_from_environment(conf, self._api_port)
         ledger = RegTestLedger(conf)
         service = FullNode(ledger)
         console = Console(service)
@@ -479,7 +480,7 @@ class IntegrationTestCase(AsyncioTestCase):
             full_nodes=[(full_node.conf.api_host, full_node.conf.api_port)],
             **conf_kwargs
         )
-        await self.provision_db_from_environment(conf)
+        await self.provision_db_from_environment(conf, self._api_port)
         ledger = RegTestLedger(conf)
         service = FullEndpoint(ledger)
         console = Console(service)
@@ -489,14 +490,17 @@ class IntegrationTestCase(AsyncioTestCase):
             await daemon.start()
         return daemon
 
-    async def make_light_client_daemon(self, full_node, start=True):
+    async def make_light_client_daemon(self, full_node, start=True, **conf_kwargs):
+        self._api_port += 1
         path = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, path, True)
-        self._node_port += 1
+        conf = Config.with_same_dir(path).set(
-        ledger = RegTestLedger(Config.with_same_dir(path).set(
+            blockchain="regtest",
-            api=f'localhost:{self._node_port}',
+            api=f"localhost:{self._api_port}",
-            full_nodes=[(full_node.conf.api_host, full_node.conf.api_port)]
+            full_nodes=[(full_node.conf.api_host, full_node.conf.api_port)],
-        ))
+            **conf_kwargs
+        )
+        ledger = RegTestLedger(conf)
         service = LightClient(ledger)
         console = Console(service)
         daemon = Daemon(service, console)
@@ -521,6 +525,62 @@ class IntegrationTestCase(AsyncioTestCase):
         balance = await account.get_balance()
         self.assertEqual(dewies_to_lbc(balance['available']), expected_balance)
 
+
+class FakeExchangeRateManager(ExchangeRateManager):
+
+    def __init__(self, market_feeds, rates):  # pylint: disable=super-init-not-called
+        self.market_feeds = market_feeds
+        for feed in self.market_feeds:
+            feed.last_check = time.time()
+            feed.rate = ExchangeRate(feed.market, rates[feed.market], time.time())
+
+    def start(self):
+        pass
+
+    def stop(self):
+        pass
+
+
+def get_fake_exchange_rate_manager(rates=None):
+    return FakeExchangeRateManager(
+        [LBRYFeed(), LBRYBTCFeed()],
+        rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
+    )
+
+
+class CommandTestCase(IntegrationTestCase):
+
+    VERBOSITY = logging.WARN
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.block_expected = 0
+        self.full_node_daemon: Daemon = None
+        self.daemon: Daemon = None
+        self.service: Service = None
+        self.conf: Config = None
+        self.api: API = None
+
+    async def asyncSetUp(self):
+        await super().asyncSetUp()
+        await self.generate(200, wait=False)
+
+        self.full_node_daemon, self.daemon = await self.make_daemons_from_environment()
+
+        self.service = self.daemon.service
+        self.conf = self.daemon.conf
+        self.api = self.daemon.api
+
+        self.wallet = self.service.wallets.default
+        self.account = self.wallet.accounts.default
+        address = await self.account.receiving.get_or_create_usable_address()
+
+        self.conf.upload_dir = os.path.join(self.conf.data_dir, 'uploads')
+        os.mkdir(self.conf.upload_dir)
+
+        await self.chain.send_to_address(address, '10.0')
+        await self.generate(5)
+
     def broadcast(self, tx):
         return self.ledger.broadcast(tx)
 
@@ -554,75 +614,6 @@ class IntegrationTestCase(AsyncioTestCase):
             lambda e: e.tx.id == tx.id and e.address == address
         )
 
-
-class FakeExchangeRateManager(ExchangeRateManager):
-
-    def __init__(self, market_feeds, rates):  # pylint: disable=super-init-not-called
-        self.market_feeds = market_feeds
-        for feed in self.market_feeds:
-            feed.last_check = time.time()
-            feed.rate = ExchangeRate(feed.market, rates[feed.market], time.time())
-
-    def start(self):
-        pass
-
-    def stop(self):
-        pass
-
-
-def get_fake_exchange_rate_manager(rates=None):
-    return FakeExchangeRateManager(
-        [LBRYFeed(), LBRYBTCFeed()],
-        rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
-    )
-
-
-class CommandTestCase(IntegrationTestCase):
-
-    VERBOSITY = logging.WARN
-    blob_lru_cache_size = 0
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.daemon_port = 5252
-        self.daemon = None
-        self.daemons = []
-        self.server_config = None
-        self.server_storage = None
-        self.extra_wallet_nodes = []
-        self.extra_wallet_node_port = 5281
-        self.server_blob_manager = None
-        self.server = None
-        self.reflector = None
-
-    async def asyncSetUp(self):
-        await super().asyncSetUp()
-        await self.generate(200, wait=False)
-
-        self.full_node_daemon, self.daemon = await self.make_daemons_from_environment()
-
-        self.service = self.daemon.service
-        self.ledger = self.service.ledger
-        self.api = self.daemon.api
-
-        self.wallet = self.service.wallets.default
-        self.account = self.wallet.accounts.default
-        address = await self.account.receiving.get_or_create_usable_address()
-
-        self.ledger.conf.upload_dir = os.path.join(self.ledger.conf.data_dir, 'uploads')
-        os.mkdir(self.ledger.conf.upload_dir)
-
-        await self.chain.send_to_address(address, '10.0')
-        await self.generate(5)
-
-    async def asyncTearDown(self):
-        await super().asyncTearDown()
-        for wallet_node in self.extra_wallet_nodes:
-            await wallet_node.stop(cleanup=True)
-        for daemon in self.daemons:
-            daemon.component_manager.get_component('wallet')._running = False
-            await daemon.stop()
-
     async def confirm_tx(self, txid, ledger=None):
         """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
         await self.on_transaction_id(txid, ledger)
@@ -702,7 +693,7 @@ class CommandTestCase(IntegrationTestCase):
 
     def create_upload_file(self, data, prefix=None, suffix=None):
         file_path = tempfile.mktemp(
-            prefix=prefix or "tmp", suffix=suffix or "", dir=self.ledger.conf.upload_dir
+            prefix=prefix or "tmp", suffix=suffix or "", dir=self.conf.upload_dir
         )
         with open(file_path, 'w+b') as file:
             file.write(data)

lbry/wallet/__init__.py

@@ -1,3 +1,3 @@
-from .wallet import Wallet
+from .wallet import Wallet, ENCRYPT_ON_DISK
 from .manager import WalletManager
 from .account import Account, SingleKey, HierarchicalDeterministic

lbry/wallet/wallet.py

@@ -61,6 +61,7 @@ class Wallet:
         self.supports = SupportListManager(self)
 
     async def generate_addresses(self):
+        if self.accounts:
             await asyncio.wait([
                 account.ensure_address_gap()
                 for account in self.accounts

tests/integration/commands/test_wallet.py

@@ -152,31 +152,30 @@ class WalletCommands(CommandTestCase):
 
 class WalletEncryptionAndSynchronization(IntegrationTestCase):
 
-    SEED = (
-        "carbon smart garage balance margin twelve chest "
-        "sword toast envelope bottom stomach absent"
-    )
-
     async def asyncSetUp(self):
         await super().asyncSetUp()
+        await self.chain.generate(200)
         self.full_node_daemon, self.daemon1 = await self.make_daemons_from_environment(
-            create_default_wallet=False
+            create_default_account=False
         )
         self.daemon2 = await self.make_light_client_daemon(
-            self.full_node_daemon, create_default_wallet=False
+            self.full_node_daemon,
+            create_default_account=False
         )
-        self.other_wallet = self.other_node.service.wallets.default
+        self.account1 = await self.daemon1.api.account_add('account1', seed=(
-        self.other_account = self.other_wallet.accounts.default
+            "carbon smart garage balance margin twelve chest "
-        address = await self.other_account.receiving.get_or_create_usable_address()
+            "sword toast envelope bottom stomach absent"
-        await self.chain.send_to_address(address, '10.0')
+        ))
-        await self.generate(5)
+        self.account2 = await self.daemon2.api.account_add('account2', seed=(
-        self.daemon2 = await self.add_full_node(
+            "chest sword toast envelope bottom stomach absent "
-            seed="chest sword toast envelope bottom stomach absent "
             "carbon smart garage balance margin twelve"
-        )
+        ))
-        address = (await self.daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0]
+        address = await self.account1.receiving.get_or_create_usable_address()
-        sendtxid = await self.blockchain.send_to_address(address, 1)
+        await self.chain.send_to_address(address, '10.0')
-        await self.confirm_tx(sendtxid, self.daemon2.ledger)
+        await self.chain.generate(1)
+        address = await self.account2.receiving.get_or_create_usable_address()
+        await self.chain.send_to_address(address, '1.0')
+        await self.chain.generate(1)
 
     def assertWalletEncrypted(self, wallet_path, encrypted):
         with open(wallet_path) as opened:
@@ -184,58 +183,62 @@ class WalletEncryptionAndSynchronization(IntegrationTestCase):
             self.assertEqual(wallet['accounts'][0]['private_key'][1:4] != 'prv', encrypted)
 
     async def test_sync(self):
-        daemon, daemon2 = self.daemon, self.daemon2
+        api1, api2 = self.daemon1.api, self.daemon2.api
 
         # Preferences
-        self.assertFalse(daemon.jsonrpc_preference_get())
+        self.assertFalse(await api1.preference_get())
-        self.assertFalse(daemon2.jsonrpc_preference_get())
+        self.assertFalse(await api2.preference_get())
 
-        daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]')
+        await api1.preference_set("fruit", '["peach", "apricot"]')
-        daemon.jsonrpc_preference_set("one", "1")
+        await api1.preference_set("one", "1")
-        daemon.jsonrpc_preference_set("conflict", "1")
+        await api1.preference_set("conflict", "1")
-        daemon2.jsonrpc_preference_set("another", "A")
+        await api2.preference_set("another", "A")
         await asyncio.sleep(1)
         # these preferences will win after merge since they are "newer"
-        daemon2.jsonrpc_preference_set("two", "2")
+        await api2.preference_set("two", "2")
-        daemon2.jsonrpc_preference_set("conflict", "2")
+        await api2.preference_set("conflict", "2")
-        daemon.jsonrpc_preference_set("another", "B")
+        await api1.preference_set("another", "B")
 
-        self.assertDictEqual(daemon.jsonrpc_preference_get(), {
+        self.assertDictEqual(await api1.preference_get(), {
             "one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"]
         })
-        self.assertDictEqual(daemon2.jsonrpc_preference_get(), {
+        self.assertDictEqual(await api2.preference_get(), {
             "two": "2", "conflict": "2", "another": "A"
         })
 
-        self.assertItemCount(await daemon.jsonrpc_account_list(), 1)
+        self.assertEqual(len(await api1.account_list()), 1)
 
-        data = await daemon2.jsonrpc_sync_apply('password')
+        data = await api2.sync_apply('password')
-        await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
+        await api1.sync_apply('password', data=data['data'], blocking=True)
 
-        self.assertItemCount(await daemon.jsonrpc_account_list(), 2)
+        self.assertEqual(len(await api1.account_list()), 2)
         self.assertDictEqual(
             # "two" key added and "conflict" value changed to "2"
-            daemon.jsonrpc_preference_get(),
+            await api1.preference_get(),
             {"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]}
         )
 
         # Channel Certificate
-        channel = await daemon2.jsonrpc_channel_create('@foo', '0.1')
+        tx = await api2.channel_create('@foo', '0.1')
-        await self.confirm_tx(channel.id, self.daemon2.ledger)
+        await self.chain.generate(1)
+        await asyncio.wait([
+            api1.service.wait(tx),
+            api2.service.wait(tx),
+        ])
 
         # both daemons will have the channel but only one has the cert so far
-        self.assertItemCount(await daemon.jsonrpc_channel_list(), 1)
+        self.assertEqual(len(await api1.channel_list()), 1)
-        self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
+        self.assertEqual(len(self.account1.channel_keys), 0)
-        self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1)
+        self.assertEqual(len(await api2.channel_list()), 1)
-        self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
+        self.assertEqual(len(self.account2.channel_keys), 0)
 
-        data = await daemon2.jsonrpc_sync_apply('password')
+        data = await api2.sync_apply('password')
-        await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
+        await api1.sync_apply('password', data=data['data'], blocking=True)
 
         # both daemons have the cert after sync'ing
         self.assertEqual(
-            daemon2.wallet_manager.default_account.channel_keys,
+            self.account1.channel_keys,
-            daemon.wallet_manager.default_wallet.accounts[1].channel_keys
+            self.account2.channel_keys
         )
 
     async def test_encryption_and_locking(self):