From 4c10a830f3243d154b440f0250ec7399dd4cb733 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 01:44:42 +0100 Subject: [PATCH 01/17] lnmsg: rewrite LN msg encoding/decoding --- electrum/channel_db.py | 4 +- electrum/lightning.json | 903 --------------------------------- electrum/lnchannel.py | 5 +- electrum/lnmsg.py | 319 +++++++----- electrum/lnpeer.py | 14 +- electrum/lnwire/README.md | 5 + electrum/lnwire/onion_wire.csv | 53 ++ electrum/lnwire/peer_wire.csv | 210 ++++++++ 8 files changed, 477 insertions(+), 1036 deletions(-) delete mode 100644 electrum/lightning.json create mode 100644 electrum/lnwire/README.md create mode 100644 electrum/lnwire/onion_wire.csv create mode 100644 electrum/lnwire/peer_wire.csv diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 66dec951e..54fc45460 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -321,11 +321,12 @@ class ChannelDB(SqlDB): return ret # note: currently channel announcements are trusted by default (trusted=True); - # they are not verified. Verifying them would make the gossip sync + # they are not SPV-verified. Verifying them would make the gossip sync # even slower; especially as servers will start throttling us. # It would probably put significant strain on servers if all clients # verified the complete gossip. def add_channel_announcement(self, msg_payloads, *, trusted=True): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] added = 0 @@ -499,6 +500,7 @@ class ChannelDB(SqlDB): raise Exception(f'failed verifying channel update for {short_channel_id}') def add_node_announcement(self, msg_payloads): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] new_nodes = {} diff --git a/electrum/lightning.json b/electrum/lightning.json deleted file mode 100644 index d6794e67c..000000000 --- a/electrum/lightning.json +++ /dev/null @@ -1,903 +0,0 @@ -{ - "init": { - "type": "16", - "payload": { - "gflen": { - "position": "0", - "length": "2" - }, - "globalfeatures": { - "position": "2", - "length": "gflen" - }, - "lflen": { - "position": "2+gflen", - "length": "2" - }, - "localfeatures": { - "position": "4+gflen", - "length": "lflen" - } - } - }, - "error": { - "type": "17", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "data": { - "position": "34", - "length": "len" - } - } - }, - "ping": { - "type": "18", - "payload": { - "num_pong_bytes": { - "position": "0", - "length": "2" - }, - "byteslen": { - "position": "2", - "length": "2" - }, - "ignored": { - "position": "4", - "length": "byteslen" - } - } - }, - "pong": { - "type": "19", - "payload": { - "byteslen": { - "position": "0", - "length": "2" - }, - "ignored": { - "position": "2", - "length": "byteslen" - } - } - }, - "open_channel": { - "type": "32", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "temporary_channel_id": { - "position": "32", - "length": "32" - }, - "funding_satoshis": { - "position": "64", - "length": "8" - }, - "push_msat": { - "position": "72", - "length": "8" - }, - "dust_limit_satoshis": { - "position": "80", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "88", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "96", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "104", - "length": "8" - }, - "feerate_per_kw": { - "position": "112", - "length": "4" - }, - "to_self_delay": { - "position": "116", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "118", - "length": "2" - }, - "funding_pubkey": { - "position": "120", - "length": "33" - }, - "revocation_basepoint": { - "position": "153", - "length": "33" - }, - "payment_basepoint": { - "position": "186", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "219", - "length": "33" - }, - "htlc_basepoint": { - "position": "252", - "length": "33" - }, - "first_per_commitment_point": { - "position": "285", - "length": "33" - }, - "channel_flags": { - "position": "318", - "length": "1" - }, - "shutdown_len": { - "position": "319", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "321", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "accept_channel": { - "type": "33", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "dust_limit_satoshis": { - "position": "32", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "40", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "48", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "56", - "length": "8" - }, - "minimum_depth": { - "position": "64", - "length": "4" - }, - "to_self_delay": { - "position": "68", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "70", - "length": "2" - }, - "funding_pubkey": { - "position": "72", - "length": "33" - }, - "revocation_basepoint": { - "position": "105", - "length": "33" - }, - "payment_basepoint": { - "position": "138", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "171", - "length": "33" - }, - "htlc_basepoint": { - "position": "204", - "length": "33" - }, - "first_per_commitment_point": { - "position": "237", - "length": "33" - }, - "shutdown_len": { - "position": "270", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "272", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "funding_created": { - "type": "34", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "funding_txid": { - "position": "32", - "length": "32" - }, - "funding_output_index": { - "position": "64", - "length": "2" - }, - "signature": { - "position": "66", - "length": "64" - } - } - }, - "funding_signed": { - "type": "35", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - } - } - }, - "funding_locked": { - "type": "36", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_per_commitment_point": { - "position": "32", - "length": "33" - } - } - }, - "shutdown": { - "type": "38", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "scriptpubkey": { - "position": "34", - "length": "len" - } - } - }, - "closing_signed": { - "type": "39", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "fee_satoshis": { - "position": "32", - "length": "8" - }, - "signature": { - "position": "40", - "length": "64" - } - } - }, - "update_add_htlc": { - "type": "128", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "amount_msat": { - "position": "40", - "length": "8" - }, - "payment_hash": { - "position": "48", - "length": "32" - }, - "cltv_expiry": { - "position": "80", - "length": "4" - }, - "onion_routing_packet": { - "position": "84", - "length": "1366" - } - } - }, - "update_fulfill_htlc": { - "type": "130", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "payment_preimage": { - "position": "40", - "length": "32" - } - } - }, - "update_fail_htlc": { - "type": "131", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "len": { - "position": "40", - "length": "2" - }, - "reason": { - "position": "42", - "length": "len" - } - } - }, - "update_fail_malformed_htlc": { - "type": "135", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "sha256_of_onion": { - "position": "40", - "length": "32" - }, - "failure_code": { - "position": "72", - "length": "2" - } - } - }, - "commitment_signed": { - "type": "132", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - }, - "num_htlcs": { - "position": "96", - "length": "2" - }, - "htlc_signature": { - "position": "98", - "length": "num_htlcs*64" - } - } - }, - "revoke_and_ack": { - "type": "133", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "per_commitment_secret": { - "position": "32", - "length": "32" - }, - "next_per_commitment_point": { - "position": "64", - "length": "33" - } - } - }, - "update_fee": { - "type": "134", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "feerate_per_kw": { - "position": "32", - "length": "4" - } - } - }, - "channel_reestablish": { - "type": "136", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_local_commitment_number": { - "position": "32", - "length": "8" - }, - "next_remote_revocation_number": { - "position": "40", - "length": "8" - }, - "your_last_per_commitment_secret": { - "position": "48", - "length": "32", - "feature": "option_data_loss_protect" - }, - "my_current_per_commitment_point": { - "position": "80", - "length": "33", - "feature": "option_data_loss_protect" - } - } - }, - "invalid_realm": { - "type": "PERM|1", - "payload": {} - }, - "temporary_node_failure": { - "type": "NODE|2", - "payload": {} - }, - "permanent_node_failure": { - "type": "PERM|NODE|2", - "payload": {} - }, - "required_node_feature_missing": { - "type": "PERM|NODE|3", - "payload": {} - }, - "invalid_onion_version": { - "type": "BADONION|PERM|4", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_hmac": { - "type": "BADONION|PERM|5", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_key": { - "type": "BADONION|PERM|6", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "temporary_channel_failure": { - "type": "UPDATE|7", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "permanent_channel_failure": { - "type": "PERM|8", - "payload": {} - }, - "required_channel_feature_missing": { - "type": "PERM|9", - "payload": {} - }, - "unknown_next_peer": { - "type": "PERM|10", - "payload": {} - }, - "amount_below_minimum": { - "type": "UPDATE|11", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "fee_insufficient": { - "type": "UPDATE|12", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "incorrect_cltv_expiry": { - "type": "UPDATE|13", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - }, - "len": { - "position": "4", - "length": "2" - }, - "channel_update": { - "position": "6", - "length": "len" - } - } - }, - "expiry_too_soon": { - "type": "UPDATE|14", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "unknown_payment_hash": { - "type": "PERM|15", - "payload": {} - }, - "incorrect_payment_amount": { - "type": "PERM|16", - "payload": {} - }, - "final_expiry_too_soon": { - "type": "17", - "payload": {} - }, - "final_incorrect_cltv_expiry": { - "type": "18", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - } - } - }, - "final_incorrect_htlc_amount": { - "type": "19", - "payload": { - "incoming_htlc_amt": { - "position": "0", - "length": "8" - } - } - }, - "channel_disabled": { - "type": "UPDATE|20", - "payload": {} - }, - "expiry_too_far": { - "type": "21", - "payload": {} - }, - "announcement_signatures": { - "type": "259", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "short_channel_id": { - "position": "32", - "length": "8" - }, - "node_signature": { - "position": "40", - "length": "64" - }, - "bitcoin_signature": { - "position": "104", - "length": "64" - } - } - }, - "channel_announcement": { - "type": "256", - "payload": { - "node_signature_1": { - "position": "0", - "length": "64" - }, - "node_signature_2": { - "position": "64", - "length": "64" - }, - "bitcoin_signature_1": { - "position": "128", - "length": "64" - }, - "bitcoin_signature_2": { - "position": "192", - "length": "64" - }, - "len": { - "position": "256", - "length": "2" - }, - "features": { - "position": "258", - "length": "len" - }, - "chain_hash": { - "position": "258+len", - "length": "32" - }, - "short_channel_id": { - "position": "290+len", - "length": "8" - }, - "node_id_1": { - "position": "298+len", - "length": "33" - }, - "node_id_2": { - "position": "331+len", - "length": "33" - }, - "bitcoin_key_1": { - "position": "364+len", - "length": "33" - }, - "bitcoin_key_2": { - "position": "397+len", - "length": "33" - } - } - }, - "node_announcement": { - "type": "257", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "flen": { - "position": "64", - "length": "2" - }, - "features": { - "position": "66", - "length": "flen" - }, - "timestamp": { - "position": "66+flen", - "length": "4" - }, - "node_id": { - "position": "70+flen", - "length": "33" - }, - "rgb_color": { - "position": "103+flen", - "length": "3" - }, - "alias": { - "position": "106+flen", - "length": "32" - }, - "addrlen": { - "position": "138+flen", - "length": "2" - }, - "addresses": { - "position": "140+flen", - "length": "addrlen" - } - } - }, - "channel_update": { - "type": "258", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "chain_hash": { - "position": "64", - "length": "32" - }, - "short_channel_id": { - "position": "96", - "length": "8" - }, - "timestamp": { - "position": "104", - "length": "4" - }, - "message_flags": { - "position": "108", - "length": "1" - }, - "channel_flags": { - "position": "109", - "length": "1" - }, - "cltv_expiry_delta": { - "position": "110", - "length": "2" - }, - "htlc_minimum_msat": { - "position": "112", - "length": "8" - }, - "fee_base_msat": { - "position": "120", - "length": "4" - }, - "fee_proportional_millionths": { - "position": "124", - "length": "4" - }, - "htlc_maximum_msat": { - "position": "128", - "length": "8", - "feature": "option_channel_htlc_max" - } - } - }, - "query_short_channel_ids": { - "type": "261", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "encoded_short_ids": { - "position": "34", - "length": "len" - } - } - }, - "reply_short_channel_ids_end": { - "type": "262", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "complete": { - "position": "32", - "length": "1" - } - } - }, - "query_channel_range": { - "type": "263", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - } - } - }, - "reply_channel_range": { - "type": "264", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - }, - "complete": { - "position": "40", - "length": "1" - }, - "len": { - "position": "41", - "length": "2" - }, - "encoded_short_ids": { - "position": "43", - "length": "len" - } - } - }, - "gossip_timestamp_filter": { - "type": "265", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_timestamp": { - "position": "32", - "length": "4" - }, - "timestamp_range": { - "position": "36", - "length": "4" - } - } - } -} diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index d38bd4824..bd52175b7 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -249,7 +249,8 @@ class Channel(Logger): node_ids = sorted_node_ids bitcoin_keys.reverse() - chan_ann = encode_msg("channel_announcement", + chan_ann = encode_msg( + "channel_announcement", len=0, features=b'', chain_hash=constants.net.rev_genesis_bytes(), @@ -257,7 +258,7 @@ class Channel(Logger): node_id_1=node_ids[0], node_id_2=node_ids[1], bitcoin_key_1=bitcoin_keys[0], - bitcoin_key_2=bitcoin_keys[1] + bitcoin_key_2=bitcoin_keys[1], ) self._chan_ann_without_sigs = chan_ann diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index a755756b4..2ef48ae97 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,152 +1,225 @@ -import json import os -from typing import Callable, Tuple -from collections import OrderedDict +import csv +import io +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union -def _eval_length_term(x, ma: dict) -> int: - """ - Evaluate a term of the simple language used - to specify lightning message field lengths. - If `x` is an integer, it is returned as is, - otherwise it is treated as a variable and - looked up in `ma`. +class MalformedMsg(Exception): + pass - If the value in `ma` was no integer, it is - assumed big-endian bytes and decoded. - Returns evaluated result as int - """ - try: - x = int(x) - except ValueError: - x = ma[x] - try: - x = int(x) - except ValueError: - x = int.from_bytes(x, byteorder='big') - return x +class UnknownMsgFieldType(MalformedMsg): + pass -def _eval_exp_with_ctx(exp, ctx: dict) -> int: - """ - Evaluate simple mathematical expression given - in `exp` with context (variables assigned) - from the dict `ctx`. - Returns evaluated result as int - """ - exp = str(exp) - if "*" in exp: - assert "+" not in exp - result = 1 - for term in exp.split("*"): - result *= _eval_length_term(term, ctx) - return result - return sum(_eval_length_term(x, ctx) for x in exp.split("+")) +class UnexpectedEndOfStream(MalformedMsg): + pass -def _make_handler(msg_name: str, v: dict) -> Callable[[bytes], Tuple[str, dict]]: - """ - Generate a message handler function (taking bytes) - for message type `msg_name` with specification `v` - Check lib/lightning.json, `msg_name` could be 'init', - and `v` could be +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + cur_pos = fd.tell() + end_pos = fd.seek(0, io.SEEK_END) + fd.seek(cur_pos) + if end_pos - cur_pos < n: + raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") - { type: 16, payload: { 'gflen': ..., ... }, ... } - Returns function taking bytes - """ - def handler(data: bytes) -> Tuple[str, dict]: - ma = {} # map of field name -> field data; after parsing msg - pos = 0 - for fieldname in v["payload"]: - poslenMap = v["payload"][fieldname] - if "feature" in poslenMap and pos == len(data): - continue - #assert pos == _eval_exp_with_ctx(poslenMap["position"], ma) # this assert is expensive... - length = poslenMap["length"] - length = _eval_exp_with_ctx(length, ma) - ma[fieldname] = data[pos:pos+length] - pos += length - # BOLT-01: "MUST ignore any additional data within a message beyond the length that it expects for that type." - assert pos <= len(data), (msg_name, pos, len(data)) - return msg_name, ma - return handler +# TODO return int when it makes sense +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return b"" + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected field type: {field_type!r}") + total_len = count * type_len + _assert_can_read_at_least_n_bytes(fd, total_len) + return fd.read(total_len) + + +def _write_field(*, fd: io.BytesIO, field_type: str, count: int, + value: Union[bytes, int]) -> None: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected fundamental type: {field_type!r}") + total_len = count * type_len + if isinstance(value, int) and (count == 1 or field_type == 'byte'): + value = int.to_bytes(value, length=total_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + if total_len != len(value): + raise Exception(f"unexpected field size. expected: {total_len}, got {len(value)}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + class LNSerializer: def __init__(self): - message_types = {} - path = os.path.join(os.path.dirname(__file__), 'lightning.json') - with open(path) as f: - structured = json.loads(f.read(), object_pairs_hook=OrderedDict) + self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] + self.msg_type_from_name = {} # type: Dict[str, bytes] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") + with open(path, newline='') as f: + csvreader = csv.reader(f) + for row in csvreader: + #print(f">>> {row!r}") + if row[0] == "msgtype": + msg_type_name = row[1] + msg_type_int = int(row[2]) + msg_type_bytes = msg_type_int.to_bytes(2, 'big') + assert msg_type_bytes not in self.msg_scheme_from_type, f"type collision? for {msg_type_name}" + assert msg_type_name not in self.msg_type_from_name, f"type collision? for {msg_type_name}" + row[2] = msg_type_int + self.msg_scheme_from_type[msg_type_bytes] = [tuple(row)] + self.msg_type_from_name[msg_type_name] = msg_type_bytes + elif row[0] == "msgdata": + assert msg_type_name == row[1] + self.msg_scheme_from_type[msg_type_bytes].append(tuple(row)) + else: + pass # TODO - for msg_name in structured: - v = structured[msg_name] - # these message types are skipped since their types collide - # (for example with pong, which also uses type=19) - # we don't need them yet - if msg_name in ["final_incorrect_cltv_expiry", "final_incorrect_htlc_amount"]: - continue - if len(v["payload"]) == 0: - continue - try: - num = int(v["type"]) - except ValueError: - #print("skipping", k) - continue - byts = num.to_bytes(2, 'big') - assert byts not in message_types, (byts, message_types[byts].__name__, msg_name) - names = [x.__name__ for x in message_types.values()] - assert msg_name + "_handler" not in names, (msg_name, names) - message_types[byts] = _make_handler(msg_name, v) - message_types[byts].__name__ = msg_name + "_handler" - - assert message_types[b"\x00\x10"].__name__ == "init_handler" - self.structured = structured - self.message_types = message_types - - def encode_msg(self, msg_type : str, **kwargs) -> bytes: + def encode_msg(self, msg_type: str, **kwargs) -> bytes: """ Encode kwargs into a Lightning message (bytes) of the type given in the msg_type string """ - typ = self.structured[msg_type] - data = int(typ["type"]).to_bytes(2, 'big') - lengths = {} - for k in typ["payload"]: - poslenMap = typ["payload"][k] - if k not in kwargs and "feature" in poslenMap: - continue - param = kwargs.get(k, 0) - leng = _eval_exp_with_ctx(poslenMap["length"], lengths) - try: - clone = dict(lengths) - clone.update(kwargs) - leng = _eval_exp_with_ctx(poslenMap["length"], clone) - except KeyError: - pass - try: - if not isinstance(param, bytes): - assert isinstance(param, int), "field {} is neither bytes or int".format(k) - param = param.to_bytes(leng, 'big') - except ValueError: - raise Exception("{} does not fit in {} bytes".format(k, leng)) - lengths[k] = len(param) - if lengths[k] != leng: - raise Exception("field {} is {} bytes long, should be {} bytes long".format(k, lengths[k], leng)) - data += param - return data + #print(f">>> encode_msg. msg_type={msg_type}, payload={kwargs!r}") + msg_type_bytes = self.msg_type_from_name[msg_type] + scheme = self.msg_scheme_from_type[msg_type_bytes] + with io.BytesIO() as fd: + fd.write(msg_type_bytes) + for row in scheme: + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + #print(f">>> encode_msg. msgdata. field_name={field_name!r}. field_type={field_type!r}. field_count_str={field_count_str!r}") + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = kwargs[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + try: + field_value = kwargs[field_name] + except KeyError: + if len(row) > 5: + break # optional feature field not present + else: + field_value = 0 # default mandatory fields to zero + #print(f">>> encode_msg. writing field: {field_name}. value={field_value!r}. field_type={field_type!r}. count={field_count!r}") + try: + _write_field(fd=fd, + field_type=field_type, + count=field_count, + value=field_value) + #print(f">>> encode_msg. so far: {fd.getvalue().hex()}") + except UnknownMsgFieldType as e: + pass # TODO + else: + pass # TODO + return fd.getvalue() - def decode_msg(self, data : bytes) -> Tuple[str, dict]: + def decode_msg(self, data: bytes) -> Tuple[str, dict]: """ Decode Lightning message by reading the first two bytes to determine message type. Returns message type string and parsed message contents dict """ - typ = data[:2] - k, parsed = self.message_types[typ](data[2:]) - return k, parsed + #print(f"decode_msg >>> {data.hex()}") + assert len(data) >= 2 + msg_type_bytes = data[:2] + msg_type_int = int.from_bytes(msg_type_bytes, byteorder="big", signed=False) + scheme = self.msg_scheme_from_type[msg_type_bytes] + assert scheme[0][2] == msg_type_int + msg_type_name = scheme[0][1] + parsed = {} + with io.BytesIO(data[2:]) as fd: + for row in scheme: + #print(f"row: {row!r}") + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + #print(f">> count={field_count}. parsed={parsed}") + try: + parsed[field_name] = _read_field(fd=fd, + field_type=field_type, + count=field_count) + except UnknownMsgFieldType as e: + pass # TODO + except UnexpectedEndOfStream as e: + if len(row) > 5: + break # optional feature field not present + else: + raise + else: + pass # TODO + return msg_type_name, parsed + _inst = LNSerializer() encode_msg = _inst.encode_msg diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 91853884d..2b0aa87c1 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -131,7 +131,7 @@ class Peer(Logger): async def initialize(self): if isinstance(self.transport, LNTransport): await self.transport.handshake() - self.send_message("init", gflen=0, lflen=2, localfeatures=self.localfeatures) + self.send_message("init", gflen=0, flen=2, features=self.localfeatures) self._sent_init = True self.maybe_set_initialized() @@ -201,7 +201,7 @@ class Peer(Logger): return # if they required some even flag we don't have, they will close themselves # but if we require an even flag they don't have, we close - their_localfeatures = int.from_bytes(payload['localfeatures'], byteorder="big") + their_localfeatures = int.from_bytes(payload['features'], byteorder="big") # TODO feature bit unification try: self.localfeatures = ln_compare_features(self.localfeatures, their_localfeatures) except IncompatibleLightningFeatures as e: @@ -760,16 +760,16 @@ class Peer(Logger): self.send_message( "channel_reestablish", channel_id=chan_id, - next_local_commitment_number=next_local_ctn, - next_remote_revocation_number=oldest_unrevoked_remote_ctn, + next_commitment_number=next_local_ctn, + next_revocation_number=oldest_unrevoked_remote_ctn, your_last_per_commitment_secret=last_rev_secret, my_current_per_commitment_point=latest_point) self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): sent channel_reestablish with ' f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_local_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_remote_revocation_number"], 'big') + their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') + their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -818,7 +818,7 @@ class Peer(Logger): if oldest_unrevoked_local_ctn != their_oldest_unrevoked_remote_ctn: if oldest_unrevoked_local_ctn - 1 == their_oldest_unrevoked_remote_ctn: # A node: - # if next_remote_revocation_number is equal to the commitment number of the last revoke_and_ack + # if next_revocation_number is equal to the commitment number of the last revoke_and_ack # the receiving node sent, AND the receiving node hasn't already received a closing_signed: # MUST re-send the revoke_and_ack. last_secret, last_point = chan.get_secret_and_point(LOCAL, oldest_unrevoked_local_ctn - 1) diff --git a/electrum/lnwire/README.md b/electrum/lnwire/README.md new file mode 100644 index 000000000..72fd48f37 --- /dev/null +++ b/electrum/lnwire/README.md @@ -0,0 +1,5 @@ +These files are generated from the BOLT repository: +``` +$ python3 tools/extract-formats.py 01-*.md 02-*.md 07-*.md > peer_wire.csv +$ python3 tools/extract-formats.py 04-*.md > onion_wire.csv +``` diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv new file mode 100644 index 000000000..7d0258239 --- /dev/null +++ b/electrum/lnwire/onion_wire.csv @@ -0,0 +1,53 @@ +tlvtype,tlv_payload,amt_to_forward,2 +tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64, +tlvtype,tlv_payload,outgoing_cltv_value,4 +tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32, +tlvtype,tlv_payload,short_channel_id,6 +tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id, +tlvtype,tlv_payload,payment_data,8 +tlvdata,tlv_payload,payment_data,payment_secret,byte,32 +tlvdata,tlv_payload,payment_data,total_msat,tu64, +msgtype,invalid_realm,PERM|1 +msgtype,temporary_node_failure,NODE|2 +msgtype,permanent_node_failure,PERM|NODE|2 +msgtype,required_node_feature_missing,PERM|NODE|3 +msgtype,invalid_onion_version,BADONION|PERM|4 +msgdata,invalid_onion_version,sha256_of_onion,sha256, +msgtype,invalid_onion_hmac,BADONION|PERM|5 +msgdata,invalid_onion_hmac,sha256_of_onion,sha256, +msgtype,invalid_onion_key,BADONION|PERM|6 +msgdata,invalid_onion_key,sha256_of_onion,sha256, +msgtype,temporary_channel_failure,UPDATE|7 +msgdata,temporary_channel_failure,len,u16, +msgdata,temporary_channel_failure,channel_update,byte,len +msgtype,permanent_channel_failure,PERM|8 +msgtype,required_channel_feature_missing,PERM|9 +msgtype,unknown_next_peer,PERM|10 +msgtype,amount_below_minimum,UPDATE|11 +msgdata,amount_below_minimum,htlc_msat,u64, +msgdata,amount_below_minimum,len,u16, +msgdata,amount_below_minimum,channel_update,byte,len +msgtype,fee_insufficient,UPDATE|12 +msgdata,fee_insufficient,htlc_msat,u64, +msgdata,fee_insufficient,len,u16, +msgdata,fee_insufficient,channel_update,byte,len +msgtype,incorrect_cltv_expiry,UPDATE|13 +msgdata,incorrect_cltv_expiry,cltv_expiry,u32, +msgdata,incorrect_cltv_expiry,len,u16, +msgdata,incorrect_cltv_expiry,channel_update,byte,len +msgtype,expiry_too_soon,UPDATE|14 +msgdata,expiry_too_soon,len,u16, +msgdata,expiry_too_soon,channel_update,byte,len +msgtype,incorrect_or_unknown_payment_details,PERM|15 +msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64, +msgdata,incorrect_or_unknown_payment_details,height,u32, +msgtype,final_incorrect_cltv_expiry,18 +msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32, +msgtype,final_incorrect_htlc_amount,19 +msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64, +msgtype,channel_disabled,UPDATE|20 +msgtype,expiry_too_far,21 +msgtype,invalid_onion_payload,PERM|22 +msgdata,invalid_onion_payload,type,varint, +msgdata,invalid_onion_payload,offset,u16, +msgtype,mpp_timeout,23 diff --git a/electrum/lnwire/peer_wire.csv b/electrum/lnwire/peer_wire.csv new file mode 100644 index 000000000..a128e9c71 --- /dev/null +++ b/electrum/lnwire/peer_wire.csv @@ -0,0 +1,210 @@ +msgtype,init,16 +msgdata,init,gflen,u16, +msgdata,init,globalfeatures,byte,gflen +msgdata,init,flen,u16, +msgdata,init,features,byte,flen +msgdata,init,tlvs,init_tlvs, +tlvtype,init_tlvs,networks,1 +tlvdata,init_tlvs,networks,chains,chain_hash,... +msgtype,error,17 +msgdata,error,channel_id,channel_id, +msgdata,error,len,u16, +msgdata,error,data,byte,len +msgtype,ping,18 +msgdata,ping,num_pong_bytes,u16, +msgdata,ping,byteslen,u16, +msgdata,ping,ignored,byte,byteslen +msgtype,pong,19 +msgdata,pong,byteslen,u16, +msgdata,pong,ignored,byte,byteslen +tlvtype,n1,tlv1,1 +tlvdata,n1,tlv1,amount_msat,tu64, +tlvtype,n1,tlv2,2 +tlvdata,n1,tlv2,scid,short_channel_id, +tlvtype,n1,tlv3,3 +tlvdata,n1,tlv3,node_id,point, +tlvdata,n1,tlv3,amount_msat_1,u64, +tlvdata,n1,tlv3,amount_msat_2,u64, +tlvtype,n1,tlv4,254 +tlvdata,n1,tlv4,cltv_delta,u16, +tlvtype,n2,tlv1,0 +tlvdata,n2,tlv1,amount_msat,tu64, +tlvtype,n2,tlv2,11 +tlvdata,n2,tlv2,cltv_expiry,tu32, +msgtype,open_channel,32 +msgdata,open_channel,chain_hash,chain_hash, +msgdata,open_channel,temporary_channel_id,byte,32 +msgdata,open_channel,funding_satoshis,u64, +msgdata,open_channel,push_msat,u64, +msgdata,open_channel,dust_limit_satoshis,u64, +msgdata,open_channel,max_htlc_value_in_flight_msat,u64, +msgdata,open_channel,channel_reserve_satoshis,u64, +msgdata,open_channel,htlc_minimum_msat,u64, +msgdata,open_channel,feerate_per_kw,u32, +msgdata,open_channel,to_self_delay,u16, +msgdata,open_channel,max_accepted_htlcs,u16, +msgdata,open_channel,funding_pubkey,point, +msgdata,open_channel,revocation_basepoint,point, +msgdata,open_channel,payment_basepoint,point, +msgdata,open_channel,delayed_payment_basepoint,point, +msgdata,open_channel,htlc_basepoint,point, +msgdata,open_channel,first_per_commitment_point,point, +msgdata,open_channel,channel_flags,byte, +msgdata,open_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,open_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,accept_channel,33 +msgdata,accept_channel,temporary_channel_id,byte,32 +msgdata,accept_channel,dust_limit_satoshis,u64, +msgdata,accept_channel,max_htlc_value_in_flight_msat,u64, +msgdata,accept_channel,channel_reserve_satoshis,u64, +msgdata,accept_channel,htlc_minimum_msat,u64, +msgdata,accept_channel,minimum_depth,u32, +msgdata,accept_channel,to_self_delay,u16, +msgdata,accept_channel,max_accepted_htlcs,u16, +msgdata,accept_channel,funding_pubkey,point, +msgdata,accept_channel,revocation_basepoint,point, +msgdata,accept_channel,payment_basepoint,point, +msgdata,accept_channel,delayed_payment_basepoint,point, +msgdata,accept_channel,htlc_basepoint,point, +msgdata,accept_channel,first_per_commitment_point,point, +msgdata,accept_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,accept_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,funding_created,34 +msgdata,funding_created,temporary_channel_id,byte,32 +msgdata,funding_created,funding_txid,sha256, +msgdata,funding_created,funding_output_index,u16, +msgdata,funding_created,signature,signature, +msgtype,funding_signed,35 +msgdata,funding_signed,channel_id,channel_id, +msgdata,funding_signed,signature,signature, +msgtype,funding_locked,36 +msgdata,funding_locked,channel_id,channel_id, +msgdata,funding_locked,next_per_commitment_point,point, +msgtype,shutdown,38 +msgdata,shutdown,channel_id,channel_id, +msgdata,shutdown,len,u16, +msgdata,shutdown,scriptpubkey,byte,len +msgtype,closing_signed,39 +msgdata,closing_signed,channel_id,channel_id, +msgdata,closing_signed,fee_satoshis,u64, +msgdata,closing_signed,signature,signature, +msgtype,update_add_htlc,128 +msgdata,update_add_htlc,channel_id,channel_id, +msgdata,update_add_htlc,id,u64, +msgdata,update_add_htlc,amount_msat,u64, +msgdata,update_add_htlc,payment_hash,sha256, +msgdata,update_add_htlc,cltv_expiry,u32, +msgdata,update_add_htlc,onion_routing_packet,byte,1366 +msgtype,update_fulfill_htlc,130 +msgdata,update_fulfill_htlc,channel_id,channel_id, +msgdata,update_fulfill_htlc,id,u64, +msgdata,update_fulfill_htlc,payment_preimage,byte,32 +msgtype,update_fail_htlc,131 +msgdata,update_fail_htlc,channel_id,channel_id, +msgdata,update_fail_htlc,id,u64, +msgdata,update_fail_htlc,len,u16, +msgdata,update_fail_htlc,reason,byte,len +msgtype,update_fail_malformed_htlc,135 +msgdata,update_fail_malformed_htlc,channel_id,channel_id, +msgdata,update_fail_malformed_htlc,id,u64, +msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256, +msgdata,update_fail_malformed_htlc,failure_code,u16, +msgtype,commitment_signed,132 +msgdata,commitment_signed,channel_id,channel_id, +msgdata,commitment_signed,signature,signature, +msgdata,commitment_signed,num_htlcs,u16, +msgdata,commitment_signed,htlc_signature,signature,num_htlcs +msgtype,revoke_and_ack,133 +msgdata,revoke_and_ack,channel_id,channel_id, +msgdata,revoke_and_ack,per_commitment_secret,byte,32 +msgdata,revoke_and_ack,next_per_commitment_point,point, +msgtype,update_fee,134 +msgdata,update_fee,channel_id,channel_id, +msgdata,update_fee,feerate_per_kw,u32, +msgtype,channel_reestablish,136 +msgdata,channel_reestablish,channel_id,channel_id, +msgdata,channel_reestablish,next_commitment_number,u64, +msgdata,channel_reestablish,next_revocation_number,u64, +msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32,option_data_loss_protect,option_static_remotekey +msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect,option_static_remotekey +msgtype,announcement_signatures,259 +msgdata,announcement_signatures,channel_id,channel_id, +msgdata,announcement_signatures,short_channel_id,short_channel_id, +msgdata,announcement_signatures,node_signature,signature, +msgdata,announcement_signatures,bitcoin_signature,signature, +msgtype,channel_announcement,256 +msgdata,channel_announcement,node_signature_1,signature, +msgdata,channel_announcement,node_signature_2,signature, +msgdata,channel_announcement,bitcoin_signature_1,signature, +msgdata,channel_announcement,bitcoin_signature_2,signature, +msgdata,channel_announcement,len,u16, +msgdata,channel_announcement,features,byte,len +msgdata,channel_announcement,chain_hash,chain_hash, +msgdata,channel_announcement,short_channel_id,short_channel_id, +msgdata,channel_announcement,node_id_1,point, +msgdata,channel_announcement,node_id_2,point, +msgdata,channel_announcement,bitcoin_key_1,point, +msgdata,channel_announcement,bitcoin_key_2,point, +msgtype,node_announcement,257 +msgdata,node_announcement,signature,signature, +msgdata,node_announcement,flen,u16, +msgdata,node_announcement,features,byte,flen +msgdata,node_announcement,timestamp,u32, +msgdata,node_announcement,node_id,point, +msgdata,node_announcement,rgb_color,byte,3 +msgdata,node_announcement,alias,byte,32 +msgdata,node_announcement,addrlen,u16, +msgdata,node_announcement,addresses,byte,addrlen +msgtype,channel_update,258 +msgdata,channel_update,signature,signature, +msgdata,channel_update,chain_hash,chain_hash, +msgdata,channel_update,short_channel_id,short_channel_id, +msgdata,channel_update,timestamp,u32, +msgdata,channel_update,message_flags,byte, +msgdata,channel_update,channel_flags,byte, +msgdata,channel_update,cltv_expiry_delta,u16, +msgdata,channel_update,htlc_minimum_msat,u64, +msgdata,channel_update,fee_base_msat,u32, +msgdata,channel_update,fee_proportional_millionths,u32, +msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max +msgtype,query_short_channel_ids,261,gossip_queries +msgdata,query_short_channel_ids,chain_hash,chain_hash, +msgdata,query_short_channel_ids,len,u16, +msgdata,query_short_channel_ids,encoded_short_ids,byte,len +msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs, +tlvtype,query_short_channel_ids_tlvs,query_flags,1 +tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8, +tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,... +msgtype,reply_short_channel_ids_end,262,gossip_queries +msgdata,reply_short_channel_ids_end,chain_hash,chain_hash, +msgdata,reply_short_channel_ids_end,complete,byte, +msgtype,query_channel_range,263,gossip_queries +msgdata,query_channel_range,chain_hash,chain_hash, +msgdata,query_channel_range,first_blocknum,u32, +msgdata,query_channel_range,number_of_blocks,u32, +msgdata,query_channel_range,tlvs,query_channel_range_tlvs, +tlvtype,query_channel_range_tlvs,query_option,1 +tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint, +msgtype,reply_channel_range,264,gossip_queries +msgdata,reply_channel_range,chain_hash,chain_hash, +msgdata,reply_channel_range,first_blocknum,u32, +msgdata,reply_channel_range,number_of_blocks,u32, +msgdata,reply_channel_range,complete,byte, +msgdata,reply_channel_range,len,u16, +msgdata,reply_channel_range,encoded_short_ids,byte,len +msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs, +tlvtype,reply_channel_range_tlvs,timestamps_tlv,1 +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8, +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,... +tlvtype,reply_channel_range_tlvs,checksums_tlv,3 +tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,... +subtype,channel_update_timestamps +subtypedata,channel_update_timestamps,timestamp_node_id_1,u32, +subtypedata,channel_update_timestamps,timestamp_node_id_2,u32, +subtype,channel_update_checksums +subtypedata,channel_update_checksums,checksum_node_id_1,u32, +subtypedata,channel_update_checksums,checksum_node_id_2,u32, +msgtype,gossip_timestamp_filter,265,gossip_queries +msgdata,gossip_timestamp_filter,chain_hash,chain_hash, +msgdata,gossip_timestamp_filter,first_timestamp,u32, +msgdata,gossip_timestamp_filter,timestamp_range,u32, From 3a73f6ee5cc3cca995ab1164c8083843a0bf0046 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 04:08:13 +0100 Subject: [PATCH 02/17] lnmsg.decode_msg: dict values for numbers are int, instead of BE bytes Will be useful for TLVs where it makes sense to do the conversion in lnmsg, as it might be more complicated than just int.from_bytes(). --- electrum/channel_db.py | 20 ++++++------ electrum/lnchannel.py | 12 +++---- electrum/lnmsg.py | 17 +++++++--- electrum/lnpeer.py | 57 ++++++++++++++++----------------- electrum/tests/test_lnrouter.py | 37 +++++++++++---------- 5 files changed, 75 insertions(+), 68 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 54fc45460..aef25effe 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -102,14 +102,14 @@ class Policy(NamedTuple): def from_msg(payload: dict) -> 'Policy': return Policy( key = payload['short_channel_id'] + payload['start_node'], - cltv_expiry_delta = int.from_bytes(payload['cltv_expiry_delta'], "big"), - htlc_minimum_msat = int.from_bytes(payload['htlc_minimum_msat'], "big"), - htlc_maximum_msat = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None, - fee_base_msat = int.from_bytes(payload['fee_base_msat'], "big"), - fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"), + cltv_expiry_delta = payload['cltv_expiry_delta'], + htlc_minimum_msat = payload['htlc_minimum_msat'], + htlc_maximum_msat = payload.get('htlc_maximum_msat', None), + fee_base_msat = payload['fee_base_msat'], + fee_proportional_millionths = payload['fee_proportional_millionths'], message_flags = int.from_bytes(payload['message_flags'], "big"), channel_flags = int.from_bytes(payload['channel_flags'], "big"), - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'], ) @staticmethod @@ -154,7 +154,7 @@ class NodeInfo(NamedTuple): alias = alias.decode('utf8') except: alias = '' - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias) return node_info, peer_addrs @@ -393,7 +393,7 @@ class ChannelDB(SqlDB): now = int(time.time()) for payload in payloads: short_channel_id = ShortChannelID(payload['short_channel_id']) - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] if max_age and now - timestamp > max_age: expired.append(payload) continue @@ -408,7 +408,7 @@ class ChannelDB(SqlDB): known.append(payload) # compare updates to existing database entries for payload in known: - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] start_node = payload['start_node'] short_channel_id = ShortChannelID(payload['short_channel_id']) key = (start_node, short_channel_id) @@ -673,7 +673,7 @@ class ChannelDB(SqlDB): return now = int(time.time()) remote_update_decoded = decode_msg(remote_update_raw)[1] - remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big") + remote_update_decoded['timestamp'] = now remote_update_decoded['start_node'] = node_id return Policy.from_msg(remote_update_decoded) elif node_id == chan.get_local_pubkey(): # outgoing direction (from us) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index bd52175b7..4fecf6a03 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -218,13 +218,13 @@ class Channel(Logger): short_channel_id=self.short_channel_id, channel_flags=channel_flags, message_flags=b'\x01', - cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA.to_bytes(2, byteorder="big"), - htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat.to_bytes(8, byteorder="big"), - htlc_maximum_msat=htlc_maximum_msat.to_bytes(8, byteorder="big"), - fee_base_msat=lnutil.OUR_FEE_BASE_MSAT.to_bytes(4, byteorder="big"), - fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS.to_bytes(4, byteorder="big"), + cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA, + htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat, + htlc_maximum_msat=htlc_maximum_msat, + fee_base_msat=lnutil.OUR_FEE_BASE_MSAT, + fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS, chain_hash=constants.net.rev_genesis_bytes(), - timestamp=now.to_bytes(4, byteorder="big"), + timestamp=now, ) sighash = sha256d(chan_upd[2 + 64:]) sig = ecc.ECPrivkey(self.lnworker.node_keypair.privkey).sign(sighash, ecc.sig_string_from_r_and_s) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 2ef48ae97..4e362efae 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,7 +1,7 @@ import os import csv import io -from typing import Callable, Tuple, Any, Dict, List, Sequence, Union +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional class MalformedMsg(Exception): @@ -24,8 +24,7 @@ def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") -# TODO return int when it makes sense -def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: if not fd: raise Exception() assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" if count == 0: @@ -35,10 +34,19 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: type_len = 1 elif field_type == 'u16': type_len = 2 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u32': type_len = 4 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u64': type_len = 8 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) # TODO tu16/tu32/tu64 elif field_type == 'chain_hash': type_len = 32 @@ -203,7 +211,8 @@ class LNSerializer: try: field_count = int(field_count_str) except ValueError: - field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + field_count = parsed[field_count_str] + assert isinstance(field_count, int) #print(f">> count={field_count}. parsed={parsed}") try: parsed[field_name] = _read_field(fd=fd, diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 2b0aa87c1..684cbe859 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -180,7 +180,7 @@ class Peer(Logger): self.ordered_message_queues[chan_id].put_nowait((None, {'error':payload['data']})) def on_ping(self, payload): - l = int.from_bytes(payload['num_pong_bytes'], 'big') + l = payload['num_pong_bytes'] self.send_message('pong', byteslen=l) def on_pong(self, payload): @@ -417,8 +417,8 @@ class Peer(Logger): return ids def on_reply_channel_range(self, payload): - first = int.from_bytes(payload['first_blocknum'], 'big') - num = int.from_bytes(payload['number_of_blocks'], 'big') + first = payload['first_blocknum'] + num = payload['number_of_blocks'] complete = bool(int.from_bytes(payload['complete'], 'big')) encoded = payload['encoded_short_ids'] ids = self.decode_short_ids(encoded) @@ -541,27 +541,27 @@ class Peer(Logger): ) payload = await self.wait_for_message('accept_channel', temp_channel_id) remote_per_commitment_point = payload['first_per_commitment_point'] - funding_txn_minimum_depth = int.from_bytes(payload['minimum_depth'], 'big') + funding_txn_minimum_depth = payload['minimum_depth'] if funding_txn_minimum_depth <= 0: raise Exception(f"minimum depth too low, {funding_txn_minimum_depth}") if funding_txn_minimum_depth > 30: raise Exception(f"minimum depth too high, {funding_txn_minimum_depth}") - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') + remote_dust_limit_sat = payload['dust_limit_satoshis'] remote_reserve_sat = self.validate_remote_reserve(payload["channel_reserve_satoshis"], remote_dust_limit_sat, funding_sat) if remote_dust_limit_sat > remote_reserve_sat: raise Exception(f"Remote Lightning peer reports dust_limit_sat > reserve_sat which is a BOLT-02 protocol violation.") - htlc_min = int.from_bytes(payload['htlc_minimum_msat'], 'big') + htlc_min = payload['htlc_minimum_msat'] if htlc_min > MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED: raise Exception(f"Remote Lightning peer reports htlc_minimum_msat={htlc_min} mSAT," + f" which is above Electrums required maximum limit of that parameter ({MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED} mSAT).") - remote_max = int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big') + remote_max = payload['max_htlc_value_in_flight_msat'] if remote_max < MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED: raise Exception(f"Remote Lightning peer reports max_htlc_value_in_flight_msat at only {remote_max} mSAT" + f" which is below Electrums required minimum ({MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED} mSAT).") - max_accepted_htlcs = int.from_bytes(payload["max_accepted_htlcs"], 'big') + max_accepted_htlcs = payload["max_accepted_htlcs"] if max_accepted_htlcs > 483: raise Exception("Remote Lightning peer reports max_accepted_htlcs > 483, which is a BOLT-02 protocol violation.") - remote_to_self_delay = int.from_bytes(payload['to_self_delay'], byteorder='big') + remote_to_self_delay = payload['to_self_delay'] if remote_to_self_delay > MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED: raise Exception(f"Remote Lightning peer reports to_self_delay={remote_to_self_delay}," + f" which is above Electrums required maximum ({MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED})") @@ -647,9 +647,9 @@ class Peer(Logger): # payload['channel_flags'] if payload['chain_hash'] != constants.net.rev_genesis_bytes(): raise Exception('wrong chain_hash') - funding_sat = int.from_bytes(payload['funding_satoshis'], 'big') - push_msat = int.from_bytes(payload['push_msat'], 'big') - feerate = int.from_bytes(payload['feerate_per_kw'], 'big') + funding_sat = payload['funding_satoshis'] + push_msat = payload['push_msat'] + feerate = payload['feerate_per_kw'] temp_chan_id = payload['temporary_channel_id'] local_config = self.make_local_config(funding_sat, push_msat, REMOTE) # for the first commitment transaction @@ -674,11 +674,11 @@ class Peer(Logger): first_per_commitment_point=per_commitment_point_first, ) funding_created = await self.wait_for_message('funding_created', temp_chan_id) - funding_idx = int.from_bytes(funding_created['funding_output_index'], 'big') + funding_idx = funding_created['funding_output_index'] funding_txid = bh2u(funding_created['funding_txid'][::-1]) channel_id, funding_txid_bytes = channel_id_from_funding_tx(funding_txid, funding_idx) remote_balance_sat = funding_sat * 1000 - push_msat - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') # TODO validate + remote_dust_limit_sat = payload['dust_limit_satoshis'] # TODO validate remote_reserve_sat = self.validate_remote_reserve(payload['channel_reserve_satoshis'], remote_dust_limit_sat, funding_sat) remote_config = RemoteConfig( payment_basepoint=OnlyPubkeyKeypair(payload['payment_basepoint']), @@ -686,13 +686,13 @@ class Peer(Logger): htlc_basepoint=OnlyPubkeyKeypair(payload['htlc_basepoint']), delayed_basepoint=OnlyPubkeyKeypair(payload['delayed_payment_basepoint']), revocation_basepoint=OnlyPubkeyKeypair(payload['revocation_basepoint']), - to_self_delay=int.from_bytes(payload['to_self_delay'], 'big'), + to_self_delay=payload['to_self_delay'], dust_limit_sat=remote_dust_limit_sat, - max_htlc_value_in_flight_msat=int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big'), # TODO validate - max_accepted_htlcs=int.from_bytes(payload['max_accepted_htlcs'], 'big'), # TODO validate + max_htlc_value_in_flight_msat=payload['max_htlc_value_in_flight_msat'], # TODO validate + max_accepted_htlcs=payload['max_accepted_htlcs'], # TODO validate initial_msat=remote_balance_sat, reserve_sat = remote_reserve_sat, - htlc_minimum_msat=int.from_bytes(payload['htlc_minimum_msat'], 'big'), # TODO validate + htlc_minimum_msat=payload['htlc_minimum_msat'], # TODO validate next_per_commitment_point=payload['first_per_commitment_point'], current_per_commitment_point=None, ) @@ -718,8 +718,7 @@ class Peer(Logger): chan.set_state(channel_states.OPENING) self.lnworker.add_new_channel(chan) - def validate_remote_reserve(self, payload_field: bytes, dust_limit: int, funding_sat: int) -> int: - remote_reserve_sat = int.from_bytes(payload_field, 'big') + def validate_remote_reserve(self, remote_reserve_sat: int, dust_limit: int, funding_sat: int) -> int: if remote_reserve_sat < dust_limit: raise Exception('protocol violation: reserve < dust_limit') if remote_reserve_sat > funding_sat/100: @@ -768,8 +767,8 @@ class Peer(Logger): f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') + their_next_local_ctn = msg["next_commitment_number"] + their_oldest_unrevoked_remote_ctn = msg["next_revocation_number"] their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -1005,7 +1004,7 @@ class Peer(Logger): return msg_hash, node_signature, bitcoin_signature def on_update_fail_htlc(self, chan: Channel, payload): - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] reason = payload["reason"] self.logger.info(f"on_update_fail_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_fail_htlc(htlc_id, error_bytes=reason) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) @@ -1083,7 +1082,7 @@ class Peer(Logger): def on_update_fulfill_htlc(self, chan: Channel, payload): preimage = payload["payment_preimage"] payment_hash = sha256(preimage) - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] self.logger.info(f"on_update_fulfill_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_htlc_settle(preimage, htlc_id) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) self.lnworker.save_preimage(payment_hash, preimage) @@ -1103,10 +1102,10 @@ class Peer(Logger): def on_update_add_htlc(self, chan: Channel, payload): payment_hash = payload["payment_hash"] - htlc_id = int.from_bytes(payload["id"], 'big') + htlc_id = payload["id"] self.logger.info(f"on_update_add_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") - cltv_expiry = int.from_bytes(payload["cltv_expiry"], 'big') - amount_msat_htlc = int.from_bytes(payload["amount_msat"], 'big') + cltv_expiry = payload["cltv_expiry"] + amount_msat_htlc = payload["amount_msat"] onion_packet = payload["onion_routing_packet"] if chan.get_state() != channel_states.OPEN: raise RemoteMisbehaving(f"received update_add_htlc while chan.get_state() != OPEN. state was {chan.get_state()}") @@ -1258,7 +1257,7 @@ class Peer(Logger): self.maybe_send_commitment(chan) def on_update_fee(self, chan: Channel, payload): - feerate = int.from_bytes(payload["feerate_per_kw"], "big") + feerate = payload["feerate_per_kw"] chan.update_fee(feerate, False) async def maybe_update_fee(self, chan: Channel): @@ -1378,7 +1377,7 @@ class Peer(Logger): while True: # FIXME: the remote SHOULD send closing_signed, but some don't. cs_payload = await self.wait_for_message('closing_signed', chan.channel_id) - their_fee = int.from_bytes(cs_payload['fee_satoshis'], 'big') + their_fee = cs_payload['fee_satoshis'] if their_fee > max_fee: raise Exception(f'the proposed fee exceeds the base fee of the latest commitment transaction {is_local, their_fee, max_fee}') their_sig = cs_payload['signature'] diff --git a/electrum/tests/test_lnrouter.py b/electrum/tests/test_lnrouter.py index db93b925a..59b6996fc 100644 --- a/electrum/tests/test_lnrouter.py +++ b/electrum/tests/test_lnrouter.py @@ -57,46 +57,45 @@ class Test_LNRouter(TestCaseForTestnet): 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02cccccccccccccccccccccccccccccccc', 'short_channel_id': bfh('0000000000000001'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) self.assertEqual(cdb.num_channels, 1) cdb.add_channel_announcement({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000002'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'short_channel_id': bfh('0000000000000003'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02cccccccccccccccccccccccccccccccc', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02cccccccccccccccccccccccccccccccc', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000004'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02dddddddddddddddddddddddddddddddd', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000005'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000006'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) - o = lambda i: i.to_bytes(8, "big") - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) + 'len': 0, 'features': b''}, trusted=True) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 99999999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) path = path_finder.find_path_for_payment(b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 100000) self.assertEqual([(b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', b'\x00\x00\x00\x00\x00\x00\x00\x03'), (b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', b'\x00\x00\x00\x00\x00\x00\x00\x02'), From 69497522630b95a632875cf87bbeec7d1d7b3242 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 13 Mar 2020 21:20:31 +0100 Subject: [PATCH 03/17] lnmsg: initial TLV implementation --- electrum/lnmsg.py | 272 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 239 insertions(+), 33 deletions(-) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 4e362efae..49afb7319 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -2,6 +2,7 @@ import os import csv import io from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional +from collections import OrderedDict class MalformedMsg(Exception): @@ -16,12 +17,56 @@ class UnexpectedEndOfStream(MalformedMsg): pass -def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: +class FieldEncodingNotMinimal(MalformedMsg): + pass + + +class UnknownMandatoryTLVRecordType(MalformedMsg): + pass + + +def _num_remaining_bytes_to_read(fd: io.BytesIO) -> int: cur_pos = fd.tell() end_pos = fd.seek(0, io.SEEK_END) fd.seek(cur_pos) - if end_pos - cur_pos < n: - raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") + return end_pos - cur_pos + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + nremaining = _num_remaining_bytes_to_read(fd) + if nremaining < n: + raise UnexpectedEndOfStream(f"wants to read {n} bytes but only {nremaining} bytes left") + + +def bigsize_from_int(i: int) -> bytes: + assert i >= 0, i + if i < 0xfd: + return int.to_bytes(i, length=1, byteorder="big", signed=False) + elif i < 0x1_0000: + return b"\xfd" + int.to_bytes(i, length=2, byteorder="big", signed=False) + elif i < 0x1_0000_0000: + return b"\xfe" + int.to_bytes(i, length=4, byteorder="big", signed=False) + else: + return b"\xff" + int.to_bytes(i, length=8, byteorder="big", signed=False) + + +def read_int_from_bigsize(fd: io.BytesIO) -> Optional[int]: + try: + first = fd.read(1)[0] + except IndexError: + return None # end of file + if first < 0xfd: + return first + elif first == 0xfd: + _assert_can_read_at_least_n_bytes(fd, 2) + return int.from_bytes(fd.read(2), byteorder="big", signed=False) + elif first == 0xfe: + _assert_can_read_at_least_n_bytes(fd, 4) + return int.from_bytes(fd.read(4), byteorder="big", signed=False) + elif first == 0xff: + _assert_can_read_at_least_n_bytes(fd, 8) + return int.from_bytes(fd.read(8), byteorder="big", signed=False) + raise Exception() def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: @@ -32,22 +77,36 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, type_len = None if field_type == 'byte': type_len = 1 - elif field_type == 'u16': - type_len = 2 + elif field_type in ('u16', 'u32', 'u64'): + if field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + else: + assert field_type == 'u64' + type_len = 8 assert count == 1, count _assert_can_read_at_least_n_bytes(fd, type_len) return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u32': - type_len = 4 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u64': - type_len = 8 + raw = fd.read(type_len) + if len(raw) > 0 and raw[0] == 0x00: + raise FieldEncodingNotMinimal() + return int.from_bytes(raw, byteorder="big", signed=False) + elif field_type == 'varint': assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - # TODO tu16/tu32/tu64 + val = read_int_from_bigsize(fd) + if val is None: + raise UnexpectedEndOfStream() + return val elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -82,7 +141,35 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, type_len = 4 elif field_type == 'u64': type_len = 8 - # TODO tu16/tu32/tu64 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 + assert count == 1, count + if isinstance(value, int): + value = int.to_bytes(value, length=type_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + while len(value) > 0 and value[0] == 0x00: + value = value[1:] + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return + elif field_type == 'varint': + assert count == 1, count + if isinstance(value, int): + value = bigsize_from_int(value) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -109,16 +196,55 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") +def _read_tlv_record(*, fd: io.BytesIO) -> Tuple[int, bytes]: + if not fd: raise Exception() + tlv_type = _read_field(fd=fd, field_type="varint", count=1) + tlv_len = _read_field(fd=fd, field_type="varint", count=1) + tlv_val = _read_field(fd=fd, field_type="byte", count=tlv_len) + return tlv_type, tlv_val + + +def _write_tlv_record(*, fd: io.BytesIO, tlv_type: int, tlv_val: bytes) -> None: + if not fd: raise Exception() + tlv_len = len(tlv_val) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_type) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_len) + _write_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) + + +def _resolve_field_count(field_count_str: str, *, vars_dict: dict) -> int: + if field_count_str == "": + field_count = 1 + elif field_count_str == "...": + raise NotImplementedError() # TODO... + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = vars_dict[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + return field_count + + class LNSerializer: def __init__(self): + # TODO msg_type could be 'int' everywhere... self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] self.msg_type_from_name = {} # type: Dict[str, bytes] + + self.in_tlv_stream_get_tlv_record_scheme_from_type = {} # type: Dict[str, Dict[int, List[Sequence[str]]]] + self.in_tlv_stream_get_record_type_from_name = {} # type: Dict[str, Dict[str, int]] + self.in_tlv_stream_get_record_name_from_type = {} # type: Dict[str, Dict[int, str]] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") with open(path, newline='') as f: csvreader = csv.reader(f) for row in csvreader: #print(f">>> {row!r}") if row[0] == "msgtype": + # msgtype,,[,