if payment fails with UPDATE onion error, also utilise channel_update for private channels

electrum/lnbase.py

@@ -11,6 +11,8 @@ import os
 import time
 from functools import partial
 from typing import List, Tuple
+import traceback
+import sys
 
 import aiorpcx
 
@@ -848,7 +850,12 @@ class Peer(PrintError):
             # attempted_route is not persisted, so we will get here then
             self.print_error("UPDATE_FAIL_HTLC. cannot decode! attempted route is MISSING. {}".format(key))
         else:
-            await self._handle_error_code_from_failed_htlc(payload["reason"], route, channel_id, htlc_id)
+            try:
+                await self._handle_error_code_from_failed_htlc(payload["reason"], route, channel_id, htlc_id)
+            except Exception:
+                # exceptions are suppressed as failing to handle an error code
+                # should not block us from removing the htlc
+                traceback.print_exc(file=sys.stderr)
         # process update_fail_htlc on channel
         chan = self.channels[channel_id]
         chan.receive_fail_htlc(htlc_id)
@@ -858,7 +865,7 @@ class Peer(PrintError):
         await self.receive_revoke(chan)
         self.lnworker.save_channel(chan)
 
-    async def _handle_error_code_from_failed_htlc(self, error_reason, route, channel_id, htlc_id):
+    async def _handle_error_code_from_failed_htlc(self, error_reason, route: List[RouteEdge], channel_id, htlc_id):
         chan = self.channels[channel_id]
         failure_msg, sender_idx = decode_onion_error(error_reason,
                                                      [x.node_id for x in route],
@@ -879,7 +886,12 @@ class Peer(PrintError):
         if offset:
             channel_update = (258).to_bytes(length=2, byteorder="big") + data[offset:]
             message_type, payload = decode_msg(channel_update)
-            self.on_channel_update(payload)
+            try:
+                self.channel_db.on_channel_update(payload)
+            except NotFoundChanAnnouncementForUpdate:
+                # maybe it is a private channel (and data in invoice was outdated)
+                start_node_id = route[sender_idx].node_id
+                self.channel_db.add_channel_update_for_private_channel(payload, start_node_id)
         else:
             # blacklist channel after reporter node
             # TODO this should depend on the error (even more granularity)

electrum/lnrouter.py

@@ -129,7 +129,7 @@ class ChannelInfo(PrintError):
         else:
             self.policy_node2 = new_policy
 
-    def get_policy_for_node(self, node_id: bytes) -> 'ChannelInfoDirectedPolicy':
+    def get_policy_for_node(self, node_id: bytes) -> Optional['ChannelInfoDirectedPolicy']:
         if node_id == self.node_id_1:
             return self.policy_node1
         elif node_id == self.node_id_2:
@@ -285,6 +285,9 @@ class ChannelDB(JsonDB):
         self._recent_peers = []
         self._last_good_address = {}  # node_id -> LNPeerAddr
 
+        # (intentionally not persisted)
+        self._channel_updates_for_private_channels = {}  # type: Dict[Tuple[bytes, bytes], ChannelInfoDirectedPolicy]
+
         self.ca_verifier = LNChannelVerifier(network, self)
 
         self.load_data()
@@ -425,6 +428,21 @@ class ChannelDB(JsonDB):
             return  # ignore
         self.nodes[pubkey] = new_node_info
 
+    def get_routing_policy_for_channel(self, start_node_id: bytes,
+                                       short_channel_id: bytes) -> Optional[ChannelInfoDirectedPolicy]:
+        if not start_node_id or not short_channel_id: return None
+        channel_info = self.get_channel_info(short_channel_id)
+        if channel_info is not None:
+            return channel_info.get_policy_for_node(start_node_id)
+        return self._channel_updates_for_private_channels.get((start_node_id, short_channel_id))
+
+    def add_channel_update_for_private_channel(self, msg_payload: dict, start_node_id: bytes):
+        if not verify_sig_for_channel_update(msg_payload, start_node_id):
+            return  # ignore
+        short_channel_id = msg_payload['short_channel_id']
+        policy = ChannelInfoDirectedPolicy(msg_payload)
+        self._channel_updates_for_private_channels[(start_node_id, short_channel_id)] = policy
+
     def remove_channel(self, short_channel_id):
         try:
             channel_info = self._id_to_channel_info[short_channel_id]
@@ -488,7 +506,7 @@ class RouteEdge(NamedTuple("RouteEdge", [('node_id', bytes),
 
 class LNPathFinder(PrintError):
 
-    def __init__(self, channel_db):
+    def __init__(self, channel_db: ChannelDB):
         self.channel_db = channel_db
         self.blacklist = set()
 
@@ -590,12 +608,9 @@ class LNPathFinder(PrintError):
         route = []
         prev_node_id = from_node_id
         for node_id, short_channel_id in path:
-            channel_info = self.channel_db.get_channel_info(short_channel_id)
+            channel_policy = self.channel_db.get_routing_policy_for_channel(prev_node_id, short_channel_id)
-            if channel_info is None:
-                raise Exception('cannot find channel info for short_channel_id: {}'.format(bh2u(short_channel_id)))
-            channel_policy = channel_info.get_policy_for_node(prev_node_id)
             if channel_policy is None:
-                raise Exception('cannot find channel policy for short_channel_id: {}'.format(bh2u(short_channel_id)))
+                raise Exception(f'cannot find channel policy for short_channel_id: {bh2u(short_channel_id)}')
             route.append(RouteEdge(node_id,
                                    short_channel_id,
                                    channel_policy.fee_base_msat,

electrum/lnworker.py

@@ -294,10 +294,19 @@ class LNWorker(PrintError):
             # we need to shift the node pubkey by one towards the destination:
             private_route_nodes = [edge[0] for edge in private_route][1:] + [invoice_pubkey]
             private_route_rest = [edge[1:] for edge in private_route]
+            prev_node_id = border_node_pubkey
             for node_pubkey, edge_rest in zip(private_route_nodes, private_route_rest):
                 short_channel_id, fee_base_msat, fee_proportional_millionths, cltv_expiry_delta = edge_rest
+                # if we have a routing policy for this edge in the db, that takes precedence,
+                # as it is likely from a previous failure
+                channel_policy = self.channel_db.get_routing_policy_for_channel(prev_node_id, short_channel_id)
+                if channel_policy:
+                    fee_base_msat = channel_policy.fee_base_msat
+                    fee_proportional_millionths = channel_policy.fee_proportional_millionths
+                    cltv_expiry_delta = channel_policy.cltv_expiry_delta
                 route.append(RouteEdge(node_pubkey, short_channel_id, fee_base_msat, fee_proportional_millionths,
                                        cltv_expiry_delta))
+                prev_node_id = node_pubkey
             break
         # if could not find route using any hint; try without hint now
         if route is None:

electrum/tests/test_lnrouter.py

@@ -79,18 +79,18 @@ class Test_LNRouter(TestCaseForTestnet):
                                      'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'),
                                      'len': b'\x00\x00', 'features': b''}, trusted=True)
         o = lambda i: i.to_bytes(8, "big")
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
-        cdb.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
+        cdb.on_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': bfh('43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000'), 'timestamp': b'\x00\x00\x00\x00'}, trusted=True)
         self.assertNotEqual(None, path_finder.find_path_for_payment(b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 100000))