From d615f6761ae3a870422c7e2e79636e9417384719 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Sun, 5 Apr 2020 19:27:13 -0400 Subject: [PATCH] automatically batch large resolve requests --- lbry/wallet/ledger.py | 6 +++++- tests/integration/blockchain/test_claim_commands.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index acf842a6b..79e7ef6b2 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -752,7 +752,11 @@ class Ledger(metaclass=LedgerRegistry): async def resolve(self, accounts, urls, **kwargs): resolve = partial(self.network.retriable_call, self.network.resolve) - txos = (await self._inflate_outputs(resolve(urls), accounts, **kwargs))[0] + urls_copy = list(urls) + txos = [] + while urls_copy: + batch, urls_copy = urls_copy[:500], urls_copy[500:] + txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0]) assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received." result = {} for url, txo in zip(urls, txos): diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index b1a3fd2ee..561b7a76c 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -1,6 +1,7 @@ import os.path import tempfile import logging +import asyncio from binascii import unhexlify from urllib.request import urlopen @@ -79,6 +80,12 @@ class ClaimSearchCommand(ClaimTestCase): ] * 23828 self.assertListEqual([], await self.claim_search(claim_ids=claim_ids)) + # this should do nothing... if the resolve (which is retried) results in the server disconnecting, + # it kerplodes + await asyncio.wait_for(self.daemon.jsonrpc_resolve([ + f'0000000000000000000000000000000000000000{i}' for i in range(30000) + ]), 30) + # 23829 claim ids makes the request just large enough claim_ids = [ '0000000000000000000000000000000000000000',