wallet: when sweeping, do network reqs in parallel, and don't block GUI

electrum/gui/qt/main_window.py

@@ -2841,15 +2841,19 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         except InternalAddressCorruption as e:
             self.show_error(str(e))
             raise
-        try:
+        privkeys = get_pk()
-            coins, keypairs = sweep_preparations(get_pk(), self.network)
+
-        except Exception as e:  # FIXME too broad...
+        def on_success(result):
-            self.show_message(repr(e))
+            coins, keypairs = result
-            return
+            outputs = [PartialTxOutput.from_address_and_value(addr, value='!')]
-        scriptpubkey = bfh(bitcoin.address_to_script(addr))
+            self.warn_if_watching_only()
-        outputs = [PartialTxOutput(scriptpubkey=scriptpubkey, value='!')]
+            self.pay_onchain_dialog(coins, outputs, external_keypairs=keypairs)
-        self.warn_if_watching_only()
+        def on_failure(exc_info):
-        self.pay_onchain_dialog(coins, outputs, external_keypairs=keypairs)
+            self.on_error(exc_info)
+        msg = _('Preparing sweep transaction...')
+        task = lambda: self.network.run_from_another_thread(
+            sweep_preparations(privkeys, self.network))
+        WaitingDialog(self, msg, task, on_success, on_failure)
 
     def _do_import(self, title, header_layout, func):
         text = text_dialog(self, title, header_layout, _('Import'), allow_multi=True)

electrum/tests/test_wallet_vertical.py

@@ -1404,6 +1404,11 @@ class TestWalletSending(TestCaseForTestnet):
                     return [{'tx_hash': 'ac24de8b58e826f60bd7b9ba31670bdfc3e8aedb2f28d0e91599d741569e3429', 'tx_pos': 1, 'height': 1325785, 'value': 1000000}]
                 else:
                     return []
+            async def get_transaction(self, txid):
+                if txid == "ac24de8b58e826f60bd7b9ba31670bdfc3e8aedb2f28d0e91599d741569e3429":
+                    return "010000000001021b41471d6af3aa80ebe536dbf4f505a6d46af456131a8e12e1950171959b690e0f00000000fdffffff2ef29833a69863b31e884fc5e6f7b99a23b5601e14f0eb65905faa42fec0776d0000000000fdffffff02f96a070000000000160014e61b989a740056254b5f8061281ac96ca15d35e140420f00000000004341049afa8fb50f52104b381a673c6e4fb7fb54987271d0e948dd9a568bb2af6f9310a7a809ce06e09d1510e5836f20414596232e2c0be63715459fa3cf8e7092af05ac0247304402201fe20012c1c732a6a8f942c4e0feed5ed0bddfb94db736ec3d0c0d38f0f7f46a022021d690e6d2688b90b76002f4c3134981502d666211e85e8a6ca91e78405dfa3801210346fb31136ab48e6c648865264d32004b43643d01f0ba485cffac4bb0b3f739470247304402204a2473ab4b3bfc8e6b1a6b8675dc2c3d115d8c04f5df37f29779dca6d300d9db02205e72ebbccd018c67b86ae4da6b0e6222902a8de85915ed6115330b9328764b370121027a93ffc9444a12d99307318e2e538949072cb35b2aca344b8163795a022414c7d73a1400"
+                else:
+                    raise Exception("unexpected txid")
 
         privkeys = ['93NQ7CFbwTPyKDJLXe97jczw33fiLijam2SCZL3Uinz1NSbHrTu', ]
         network = NetworkMock()

electrum/wallet.py

@@ -43,6 +43,8 @@ from typing import TYPE_CHECKING, List, Optional, Tuple, Union, NamedTuple, Sequ
 from abc import ABC, abstractmethod
 import itertools
 
+from aiorpcx import TaskGroup
+
 from .i18n import _
 from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath, convert_bip32_path_to_list_of_uint32
 from .crypto import sha256
@@ -92,64 +94,79 @@ TX_STATUS = [
 ]
 
 
-def _append_utxos_to_inputs(inputs: List[PartialTxInput], network: 'Network', pubkey, txin_type, imax):
+async def _append_utxos_to_inputs(*, inputs: List[PartialTxInput], network: 'Network',
+                                  pubkey: str, txin_type: str, imax: int) -> None:
     if txin_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'):
         address = bitcoin.pubkey_to_address(txin_type, pubkey)
         scripthash = bitcoin.address_to_scripthash(address)
     elif txin_type == 'p2pk':
         script = bitcoin.public_key_to_p2pk_script(pubkey)
         scripthash = bitcoin.script_to_scripthash(script)
-        address = None
     else:
         raise Exception(f'unexpected txin_type to sweep: {txin_type}')
 
-    u = network.run_from_another_thread(network.listunspent_for_scripthash(scripthash))
+    async def append_single_utxo(item):
-    for item in u:
+        prev_tx_raw = await network.get_transaction(item['tx_hash'])
-        if len(inputs) >= imax:
+        prev_tx = Transaction(prev_tx_raw)
-            break
+        prev_txout = prev_tx.outputs()[item['tx_pos']]
+        if scripthash != bitcoin.script_to_scripthash(prev_txout.scriptpubkey.hex()):
+            raise Exception('scripthash mismatch when sweeping')
         prevout_str = item['tx_hash'] + ':%d' % item['tx_pos']
         prevout = TxOutpoint.from_str(prevout_str)
-        utxo = PartialTxInput(prevout=prevout)
+        txin = PartialTxInput(prevout=prevout)
-        utxo._trusted_value_sats = int(item['value'])
+        txin.utxo = prev_tx
-        utxo._trusted_address = address
+        txin.block_height = int(item['height'])
-        utxo.block_height = int(item['height'])
+        txin.script_type = txin_type
-        utxo.script_type = txin_type
+        txin.pubkeys = [bfh(pubkey)]
-        utxo.pubkeys = [bfh(pubkey)]
+        txin.num_sig = 1
-        utxo.num_sig = 1
         if txin_type == 'p2wpkh-p2sh':
-            utxo.redeem_script = bfh(bitcoin.p2wpkh_nested_script(pubkey))
+            txin.redeem_script = bfh(bitcoin.p2wpkh_nested_script(pubkey))
-        inputs.append(utxo)
+        inputs.append(txin)
 
-def sweep_preparations(privkeys, network: 'Network', imax=100):
+    u = await network.listunspent_for_scripthash(scripthash)
+    async with TaskGroup() as group:
+        for item in u:
+            if len(inputs) >= imax:
+                break
+            await group.spawn(append_single_utxo(item))
 
-    def find_utxos_for_privkey(txin_type, privkey, compressed):
+
+async def sweep_preparations(privkeys, network: 'Network', imax=100):
+
+    async def find_utxos_for_privkey(txin_type, privkey, compressed):
         pubkey = ecc.ECPrivkey(privkey).get_public_key_hex(compressed=compressed)
-        _append_utxos_to_inputs(inputs, network, pubkey, txin_type, imax)
+        await _append_utxos_to_inputs(
+            inputs=inputs,
+            network=network,
+            pubkey=pubkey,
+            txin_type=txin_type,
+            imax=imax)
         keypairs[pubkey] = privkey, compressed
+
     inputs = []  # type: List[PartialTxInput]
     keypairs = {}
-    for sec in privkeys:
+    async with TaskGroup() as group:
-        txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec)
+        for sec in privkeys:
-        find_utxos_for_privkey(txin_type, privkey, compressed)
+            txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec)
-        # do other lookups to increase support coverage
+            await group.spawn(find_utxos_for_privkey(txin_type, privkey, compressed))
-        if is_minikey(sec):
+            # do other lookups to increase support coverage
-            # minikeys don't have a compressed byte
+            if is_minikey(sec):
-            # we lookup both compressed and uncompressed pubkeys
+                # minikeys don't have a compressed byte
-            find_utxos_for_privkey(txin_type, privkey, not compressed)
+                # we lookup both compressed and uncompressed pubkeys
-        elif txin_type == 'p2pkh':
+                await group.spawn(find_utxos_for_privkey(txin_type, privkey, not compressed))
-            # WIF serialization does not distinguish p2pkh and p2pk
+            elif txin_type == 'p2pkh':
-            # we also search for pay-to-pubkey outputs
+                # WIF serialization does not distinguish p2pkh and p2pk
-            find_utxos_for_privkey('p2pk', privkey, compressed)
+                # we also search for pay-to-pubkey outputs
+                await group.spawn(find_utxos_for_privkey('p2pk', privkey, compressed))
     if not inputs:
-        raise Exception(_('No inputs found. (Note that inputs need to be confirmed)'))
+        raise Exception(_('No inputs found.'))
-        # FIXME actually inputs need not be confirmed now, see https://github.com/kyuupichan/electrumx/issues/365
     return inputs, keypairs
 
 
 def sweep(privkeys, *, network: 'Network', config: 'SimpleConfig',
           to_address: str, fee: int = None, imax=100,
           locktime=None, tx_version=None) -> PartialTransaction:
-    inputs, keypairs = sweep_preparations(privkeys, network, imax)
+    inputs, keypairs = network.run_from_another_thread(sweep_preparations(privkeys, network, imax))
     total = sum(txin.value_sats() for txin in inputs)
     if fee is None:
         outputs = [PartialTxOutput(scriptpubkey=bfh(bitcoin.address_to_script(to_address)),