From 4c10a830f3243d154b440f0250ec7399dd4cb733 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 01:44:42 +0100 Subject: [PATCH] lnmsg: rewrite LN msg encoding/decoding --- electrum/channel_db.py | 4 +- electrum/lightning.json | 903 --------------------------------- electrum/lnchannel.py | 5 +- electrum/lnmsg.py | 319 +++++++----- electrum/lnpeer.py | 14 +- electrum/lnwire/README.md | 5 + electrum/lnwire/onion_wire.csv | 53 ++ electrum/lnwire/peer_wire.csv | 210 ++++++++ 8 files changed, 477 insertions(+), 1036 deletions(-) delete mode 100644 electrum/lightning.json create mode 100644 electrum/lnwire/README.md create mode 100644 electrum/lnwire/onion_wire.csv create mode 100644 electrum/lnwire/peer_wire.csv diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 66dec951e..54fc45460 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -321,11 +321,12 @@ class ChannelDB(SqlDB): return ret # note: currently channel announcements are trusted by default (trusted=True); - # they are not verified. Verifying them would make the gossip sync + # they are not SPV-verified. Verifying them would make the gossip sync # even slower; especially as servers will start throttling us. # It would probably put significant strain on servers if all clients # verified the complete gossip. def add_channel_announcement(self, msg_payloads, *, trusted=True): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] added = 0 @@ -499,6 +500,7 @@ class ChannelDB(SqlDB): raise Exception(f'failed verifying channel update for {short_channel_id}') def add_node_announcement(self, msg_payloads): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] new_nodes = {} diff --git a/electrum/lightning.json b/electrum/lightning.json deleted file mode 100644 index d6794e67c..000000000 --- a/electrum/lightning.json +++ /dev/null @@ -1,903 +0,0 @@ -{ - "init": { - "type": "16", - "payload": { - "gflen": { - "position": "0", - "length": "2" - }, - "globalfeatures": { - "position": "2", - "length": "gflen" - }, - "lflen": { - "position": "2+gflen", - "length": "2" - }, - "localfeatures": { - "position": "4+gflen", - "length": "lflen" - } - } - }, - "error": { - "type": "17", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "data": { - "position": "34", - "length": "len" - } - } - }, - "ping": { - "type": "18", - "payload": { - "num_pong_bytes": { - "position": "0", - "length": "2" - }, - "byteslen": { - "position": "2", - "length": "2" - }, - "ignored": { - "position": "4", - "length": "byteslen" - } - } - }, - "pong": { - "type": "19", - "payload": { - "byteslen": { - "position": "0", - "length": "2" - }, - "ignored": { - "position": "2", - "length": "byteslen" - } - } - }, - "open_channel": { - "type": "32", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "temporary_channel_id": { - "position": "32", - "length": "32" - }, - "funding_satoshis": { - "position": "64", - "length": "8" - }, - "push_msat": { - "position": "72", - "length": "8" - }, - "dust_limit_satoshis": { - "position": "80", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "88", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "96", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "104", - "length": "8" - }, - "feerate_per_kw": { - "position": "112", - "length": "4" - }, - "to_self_delay": { - "position": "116", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "118", - "length": "2" - }, - "funding_pubkey": { - "position": "120", - "length": "33" - }, - "revocation_basepoint": { - "position": "153", - "length": "33" - }, - "payment_basepoint": { - "position": "186", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "219", - "length": "33" - }, - "htlc_basepoint": { - "position": "252", - "length": "33" - }, - "first_per_commitment_point": { - "position": "285", - "length": "33" - }, - "channel_flags": { - "position": "318", - "length": "1" - }, - "shutdown_len": { - "position": "319", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "321", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "accept_channel": { - "type": "33", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "dust_limit_satoshis": { - "position": "32", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "40", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "48", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "56", - "length": "8" - }, - "minimum_depth": { - "position": "64", - "length": "4" - }, - "to_self_delay": { - "position": "68", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "70", - "length": "2" - }, - "funding_pubkey": { - "position": "72", - "length": "33" - }, - "revocation_basepoint": { - "position": "105", - "length": "33" - }, - "payment_basepoint": { - "position": "138", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "171", - "length": "33" - }, - "htlc_basepoint": { - "position": "204", - "length": "33" - }, - "first_per_commitment_point": { - "position": "237", - "length": "33" - }, - "shutdown_len": { - "position": "270", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "272", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "funding_created": { - "type": "34", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "funding_txid": { - "position": "32", - "length": "32" - }, - "funding_output_index": { - "position": "64", - "length": "2" - }, - "signature": { - "position": "66", - "length": "64" - } - } - }, - "funding_signed": { - "type": "35", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - } - } - }, - "funding_locked": { - "type": "36", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_per_commitment_point": { - "position": "32", - "length": "33" - } - } - }, - "shutdown": { - "type": "38", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "scriptpubkey": { - "position": "34", - "length": "len" - } - } - }, - "closing_signed": { - "type": "39", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "fee_satoshis": { - "position": "32", - "length": "8" - }, - "signature": { - "position": "40", - "length": "64" - } - } - }, - "update_add_htlc": { - "type": "128", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "amount_msat": { - "position": "40", - "length": "8" - }, - "payment_hash": { - "position": "48", - "length": "32" - }, - "cltv_expiry": { - "position": "80", - "length": "4" - }, - "onion_routing_packet": { - "position": "84", - "length": "1366" - } - } - }, - "update_fulfill_htlc": { - "type": "130", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "payment_preimage": { - "position": "40", - "length": "32" - } - } - }, - "update_fail_htlc": { - "type": "131", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "len": { - "position": "40", - "length": "2" - }, - "reason": { - "position": "42", - "length": "len" - } - } - }, - "update_fail_malformed_htlc": { - "type": "135", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "sha256_of_onion": { - "position": "40", - "length": "32" - }, - "failure_code": { - "position": "72", - "length": "2" - } - } - }, - "commitment_signed": { - "type": "132", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - }, - "num_htlcs": { - "position": "96", - "length": "2" - }, - "htlc_signature": { - "position": "98", - "length": "num_htlcs*64" - } - } - }, - "revoke_and_ack": { - "type": "133", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "per_commitment_secret": { - "position": "32", - "length": "32" - }, - "next_per_commitment_point": { - "position": "64", - "length": "33" - } - } - }, - "update_fee": { - "type": "134", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "feerate_per_kw": { - "position": "32", - "length": "4" - } - } - }, - "channel_reestablish": { - "type": "136", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_local_commitment_number": { - "position": "32", - "length": "8" - }, - "next_remote_revocation_number": { - "position": "40", - "length": "8" - }, - "your_last_per_commitment_secret": { - "position": "48", - "length": "32", - "feature": "option_data_loss_protect" - }, - "my_current_per_commitment_point": { - "position": "80", - "length": "33", - "feature": "option_data_loss_protect" - } - } - }, - "invalid_realm": { - "type": "PERM|1", - "payload": {} - }, - "temporary_node_failure": { - "type": "NODE|2", - "payload": {} - }, - "permanent_node_failure": { - "type": "PERM|NODE|2", - "payload": {} - }, - "required_node_feature_missing": { - "type": "PERM|NODE|3", - "payload": {} - }, - "invalid_onion_version": { - "type": "BADONION|PERM|4", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_hmac": { - "type": "BADONION|PERM|5", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_key": { - "type": "BADONION|PERM|6", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "temporary_channel_failure": { - "type": "UPDATE|7", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "permanent_channel_failure": { - "type": "PERM|8", - "payload": {} - }, - "required_channel_feature_missing": { - "type": "PERM|9", - "payload": {} - }, - "unknown_next_peer": { - "type": "PERM|10", - "payload": {} - }, - "amount_below_minimum": { - "type": "UPDATE|11", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "fee_insufficient": { - "type": "UPDATE|12", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "incorrect_cltv_expiry": { - "type": "UPDATE|13", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - }, - "len": { - "position": "4", - "length": "2" - }, - "channel_update": { - "position": "6", - "length": "len" - } - } - }, - "expiry_too_soon": { - "type": "UPDATE|14", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "unknown_payment_hash": { - "type": "PERM|15", - "payload": {} - }, - "incorrect_payment_amount": { - "type": "PERM|16", - "payload": {} - }, - "final_expiry_too_soon": { - "type": "17", - "payload": {} - }, - "final_incorrect_cltv_expiry": { - "type": "18", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - } - } - }, - "final_incorrect_htlc_amount": { - "type": "19", - "payload": { - "incoming_htlc_amt": { - "position": "0", - "length": "8" - } - } - }, - "channel_disabled": { - "type": "UPDATE|20", - "payload": {} - }, - "expiry_too_far": { - "type": "21", - "payload": {} - }, - "announcement_signatures": { - "type": "259", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "short_channel_id": { - "position": "32", - "length": "8" - }, - "node_signature": { - "position": "40", - "length": "64" - }, - "bitcoin_signature": { - "position": "104", - "length": "64" - } - } - }, - "channel_announcement": { - "type": "256", - "payload": { - "node_signature_1": { - "position": "0", - "length": "64" - }, - "node_signature_2": { - "position": "64", - "length": "64" - }, - "bitcoin_signature_1": { - "position": "128", - "length": "64" - }, - "bitcoin_signature_2": { - "position": "192", - "length": "64" - }, - "len": { - "position": "256", - "length": "2" - }, - "features": { - "position": "258", - "length": "len" - }, - "chain_hash": { - "position": "258+len", - "length": "32" - }, - "short_channel_id": { - "position": "290+len", - "length": "8" - }, - "node_id_1": { - "position": "298+len", - "length": "33" - }, - "node_id_2": { - "position": "331+len", - "length": "33" - }, - "bitcoin_key_1": { - "position": "364+len", - "length": "33" - }, - "bitcoin_key_2": { - "position": "397+len", - "length": "33" - } - } - }, - "node_announcement": { - "type": "257", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "flen": { - "position": "64", - "length": "2" - }, - "features": { - "position": "66", - "length": "flen" - }, - "timestamp": { - "position": "66+flen", - "length": "4" - }, - "node_id": { - "position": "70+flen", - "length": "33" - }, - "rgb_color": { - "position": "103+flen", - "length": "3" - }, - "alias": { - "position": "106+flen", - "length": "32" - }, - "addrlen": { - "position": "138+flen", - "length": "2" - }, - "addresses": { - "position": "140+flen", - "length": "addrlen" - } - } - }, - "channel_update": { - "type": "258", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "chain_hash": { - "position": "64", - "length": "32" - }, - "short_channel_id": { - "position": "96", - "length": "8" - }, - "timestamp": { - "position": "104", - "length": "4" - }, - "message_flags": { - "position": "108", - "length": "1" - }, - "channel_flags": { - "position": "109", - "length": "1" - }, - "cltv_expiry_delta": { - "position": "110", - "length": "2" - }, - "htlc_minimum_msat": { - "position": "112", - "length": "8" - }, - "fee_base_msat": { - "position": "120", - "length": "4" - }, - "fee_proportional_millionths": { - "position": "124", - "length": "4" - }, - "htlc_maximum_msat": { - "position": "128", - "length": "8", - "feature": "option_channel_htlc_max" - } - } - }, - "query_short_channel_ids": { - "type": "261", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "encoded_short_ids": { - "position": "34", - "length": "len" - } - } - }, - "reply_short_channel_ids_end": { - "type": "262", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "complete": { - "position": "32", - "length": "1" - } - } - }, - "query_channel_range": { - "type": "263", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - } - } - }, - "reply_channel_range": { - "type": "264", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - }, - "complete": { - "position": "40", - "length": "1" - }, - "len": { - "position": "41", - "length": "2" - }, - "encoded_short_ids": { - "position": "43", - "length": "len" - } - } - }, - "gossip_timestamp_filter": { - "type": "265", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_timestamp": { - "position": "32", - "length": "4" - }, - "timestamp_range": { - "position": "36", - "length": "4" - } - } - } -} diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index d38bd4824..bd52175b7 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -249,7 +249,8 @@ class Channel(Logger): node_ids = sorted_node_ids bitcoin_keys.reverse() - chan_ann = encode_msg("channel_announcement", + chan_ann = encode_msg( + "channel_announcement", len=0, features=b'', chain_hash=constants.net.rev_genesis_bytes(), @@ -257,7 +258,7 @@ class Channel(Logger): node_id_1=node_ids[0], node_id_2=node_ids[1], bitcoin_key_1=bitcoin_keys[0], - bitcoin_key_2=bitcoin_keys[1] + bitcoin_key_2=bitcoin_keys[1], ) self._chan_ann_without_sigs = chan_ann diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index a755756b4..2ef48ae97 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,152 +1,225 @@ -import json import os -from typing import Callable, Tuple -from collections import OrderedDict +import csv +import io +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union -def _eval_length_term(x, ma: dict) -> int: - """ - Evaluate a term of the simple language used - to specify lightning message field lengths. - If `x` is an integer, it is returned as is, - otherwise it is treated as a variable and - looked up in `ma`. +class MalformedMsg(Exception): + pass - If the value in `ma` was no integer, it is - assumed big-endian bytes and decoded. - Returns evaluated result as int - """ - try: - x = int(x) - except ValueError: - x = ma[x] - try: - x = int(x) - except ValueError: - x = int.from_bytes(x, byteorder='big') - return x +class UnknownMsgFieldType(MalformedMsg): + pass -def _eval_exp_with_ctx(exp, ctx: dict) -> int: - """ - Evaluate simple mathematical expression given - in `exp` with context (variables assigned) - from the dict `ctx`. - Returns evaluated result as int - """ - exp = str(exp) - if "*" in exp: - assert "+" not in exp - result = 1 - for term in exp.split("*"): - result *= _eval_length_term(term, ctx) - return result - return sum(_eval_length_term(x, ctx) for x in exp.split("+")) +class UnexpectedEndOfStream(MalformedMsg): + pass -def _make_handler(msg_name: str, v: dict) -> Callable[[bytes], Tuple[str, dict]]: - """ - Generate a message handler function (taking bytes) - for message type `msg_name` with specification `v` - Check lib/lightning.json, `msg_name` could be 'init', - and `v` could be +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + cur_pos = fd.tell() + end_pos = fd.seek(0, io.SEEK_END) + fd.seek(cur_pos) + if end_pos - cur_pos < n: + raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") - { type: 16, payload: { 'gflen': ..., ... }, ... } - Returns function taking bytes - """ - def handler(data: bytes) -> Tuple[str, dict]: - ma = {} # map of field name -> field data; after parsing msg - pos = 0 - for fieldname in v["payload"]: - poslenMap = v["payload"][fieldname] - if "feature" in poslenMap and pos == len(data): - continue - #assert pos == _eval_exp_with_ctx(poslenMap["position"], ma) # this assert is expensive... - length = poslenMap["length"] - length = _eval_exp_with_ctx(length, ma) - ma[fieldname] = data[pos:pos+length] - pos += length - # BOLT-01: "MUST ignore any additional data within a message beyond the length that it expects for that type." - assert pos <= len(data), (msg_name, pos, len(data)) - return msg_name, ma - return handler +# TODO return int when it makes sense +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return b"" + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected field type: {field_type!r}") + total_len = count * type_len + _assert_can_read_at_least_n_bytes(fd, total_len) + return fd.read(total_len) + + +def _write_field(*, fd: io.BytesIO, field_type: str, count: int, + value: Union[bytes, int]) -> None: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected fundamental type: {field_type!r}") + total_len = count * type_len + if isinstance(value, int) and (count == 1 or field_type == 'byte'): + value = int.to_bytes(value, length=total_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + if total_len != len(value): + raise Exception(f"unexpected field size. expected: {total_len}, got {len(value)}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + class LNSerializer: def __init__(self): - message_types = {} - path = os.path.join(os.path.dirname(__file__), 'lightning.json') - with open(path) as f: - structured = json.loads(f.read(), object_pairs_hook=OrderedDict) + self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] + self.msg_type_from_name = {} # type: Dict[str, bytes] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") + with open(path, newline='') as f: + csvreader = csv.reader(f) + for row in csvreader: + #print(f">>> {row!r}") + if row[0] == "msgtype": + msg_type_name = row[1] + msg_type_int = int(row[2]) + msg_type_bytes = msg_type_int.to_bytes(2, 'big') + assert msg_type_bytes not in self.msg_scheme_from_type, f"type collision? for {msg_type_name}" + assert msg_type_name not in self.msg_type_from_name, f"type collision? for {msg_type_name}" + row[2] = msg_type_int + self.msg_scheme_from_type[msg_type_bytes] = [tuple(row)] + self.msg_type_from_name[msg_type_name] = msg_type_bytes + elif row[0] == "msgdata": + assert msg_type_name == row[1] + self.msg_scheme_from_type[msg_type_bytes].append(tuple(row)) + else: + pass # TODO - for msg_name in structured: - v = structured[msg_name] - # these message types are skipped since their types collide - # (for example with pong, which also uses type=19) - # we don't need them yet - if msg_name in ["final_incorrect_cltv_expiry", "final_incorrect_htlc_amount"]: - continue - if len(v["payload"]) == 0: - continue - try: - num = int(v["type"]) - except ValueError: - #print("skipping", k) - continue - byts = num.to_bytes(2, 'big') - assert byts not in message_types, (byts, message_types[byts].__name__, msg_name) - names = [x.__name__ for x in message_types.values()] - assert msg_name + "_handler" not in names, (msg_name, names) - message_types[byts] = _make_handler(msg_name, v) - message_types[byts].__name__ = msg_name + "_handler" - - assert message_types[b"\x00\x10"].__name__ == "init_handler" - self.structured = structured - self.message_types = message_types - - def encode_msg(self, msg_type : str, **kwargs) -> bytes: + def encode_msg(self, msg_type: str, **kwargs) -> bytes: """ Encode kwargs into a Lightning message (bytes) of the type given in the msg_type string """ - typ = self.structured[msg_type] - data = int(typ["type"]).to_bytes(2, 'big') - lengths = {} - for k in typ["payload"]: - poslenMap = typ["payload"][k] - if k not in kwargs and "feature" in poslenMap: - continue - param = kwargs.get(k, 0) - leng = _eval_exp_with_ctx(poslenMap["length"], lengths) - try: - clone = dict(lengths) - clone.update(kwargs) - leng = _eval_exp_with_ctx(poslenMap["length"], clone) - except KeyError: - pass - try: - if not isinstance(param, bytes): - assert isinstance(param, int), "field {} is neither bytes or int".format(k) - param = param.to_bytes(leng, 'big') - except ValueError: - raise Exception("{} does not fit in {} bytes".format(k, leng)) - lengths[k] = len(param) - if lengths[k] != leng: - raise Exception("field {} is {} bytes long, should be {} bytes long".format(k, lengths[k], leng)) - data += param - return data + #print(f">>> encode_msg. msg_type={msg_type}, payload={kwargs!r}") + msg_type_bytes = self.msg_type_from_name[msg_type] + scheme = self.msg_scheme_from_type[msg_type_bytes] + with io.BytesIO() as fd: + fd.write(msg_type_bytes) + for row in scheme: + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + #print(f">>> encode_msg. msgdata. field_name={field_name!r}. field_type={field_type!r}. field_count_str={field_count_str!r}") + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = kwargs[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + try: + field_value = kwargs[field_name] + except KeyError: + if len(row) > 5: + break # optional feature field not present + else: + field_value = 0 # default mandatory fields to zero + #print(f">>> encode_msg. writing field: {field_name}. value={field_value!r}. field_type={field_type!r}. count={field_count!r}") + try: + _write_field(fd=fd, + field_type=field_type, + count=field_count, + value=field_value) + #print(f">>> encode_msg. so far: {fd.getvalue().hex()}") + except UnknownMsgFieldType as e: + pass # TODO + else: + pass # TODO + return fd.getvalue() - def decode_msg(self, data : bytes) -> Tuple[str, dict]: + def decode_msg(self, data: bytes) -> Tuple[str, dict]: """ Decode Lightning message by reading the first two bytes to determine message type. Returns message type string and parsed message contents dict """ - typ = data[:2] - k, parsed = self.message_types[typ](data[2:]) - return k, parsed + #print(f"decode_msg >>> {data.hex()}") + assert len(data) >= 2 + msg_type_bytes = data[:2] + msg_type_int = int.from_bytes(msg_type_bytes, byteorder="big", signed=False) + scheme = self.msg_scheme_from_type[msg_type_bytes] + assert scheme[0][2] == msg_type_int + msg_type_name = scheme[0][1] + parsed = {} + with io.BytesIO(data[2:]) as fd: + for row in scheme: + #print(f"row: {row!r}") + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + #print(f">> count={field_count}. parsed={parsed}") + try: + parsed[field_name] = _read_field(fd=fd, + field_type=field_type, + count=field_count) + except UnknownMsgFieldType as e: + pass # TODO + except UnexpectedEndOfStream as e: + if len(row) > 5: + break # optional feature field not present + else: + raise + else: + pass # TODO + return msg_type_name, parsed + _inst = LNSerializer() encode_msg = _inst.encode_msg diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 91853884d..2b0aa87c1 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -131,7 +131,7 @@ class Peer(Logger): async def initialize(self): if isinstance(self.transport, LNTransport): await self.transport.handshake() - self.send_message("init", gflen=0, lflen=2, localfeatures=self.localfeatures) + self.send_message("init", gflen=0, flen=2, features=self.localfeatures) self._sent_init = True self.maybe_set_initialized() @@ -201,7 +201,7 @@ class Peer(Logger): return # if they required some even flag we don't have, they will close themselves # but if we require an even flag they don't have, we close - their_localfeatures = int.from_bytes(payload['localfeatures'], byteorder="big") + their_localfeatures = int.from_bytes(payload['features'], byteorder="big") # TODO feature bit unification try: self.localfeatures = ln_compare_features(self.localfeatures, their_localfeatures) except IncompatibleLightningFeatures as e: @@ -760,16 +760,16 @@ class Peer(Logger): self.send_message( "channel_reestablish", channel_id=chan_id, - next_local_commitment_number=next_local_ctn, - next_remote_revocation_number=oldest_unrevoked_remote_ctn, + next_commitment_number=next_local_ctn, + next_revocation_number=oldest_unrevoked_remote_ctn, your_last_per_commitment_secret=last_rev_secret, my_current_per_commitment_point=latest_point) self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): sent channel_reestablish with ' f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_local_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_remote_revocation_number"], 'big') + their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') + their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -818,7 +818,7 @@ class Peer(Logger): if oldest_unrevoked_local_ctn != their_oldest_unrevoked_remote_ctn: if oldest_unrevoked_local_ctn - 1 == their_oldest_unrevoked_remote_ctn: # A node: - # if next_remote_revocation_number is equal to the commitment number of the last revoke_and_ack + # if next_revocation_number is equal to the commitment number of the last revoke_and_ack # the receiving node sent, AND the receiving node hasn't already received a closing_signed: # MUST re-send the revoke_and_ack. last_secret, last_point = chan.get_secret_and_point(LOCAL, oldest_unrevoked_local_ctn - 1) diff --git a/electrum/lnwire/README.md b/electrum/lnwire/README.md new file mode 100644 index 000000000..72fd48f37 --- /dev/null +++ b/electrum/lnwire/README.md @@ -0,0 +1,5 @@ +These files are generated from the BOLT repository: +``` +$ python3 tools/extract-formats.py 01-*.md 02-*.md 07-*.md > peer_wire.csv +$ python3 tools/extract-formats.py 04-*.md > onion_wire.csv +``` diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv new file mode 100644 index 000000000..7d0258239 --- /dev/null +++ b/electrum/lnwire/onion_wire.csv @@ -0,0 +1,53 @@ +tlvtype,tlv_payload,amt_to_forward,2 +tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64, +tlvtype,tlv_payload,outgoing_cltv_value,4 +tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32, +tlvtype,tlv_payload,short_channel_id,6 +tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id, +tlvtype,tlv_payload,payment_data,8 +tlvdata,tlv_payload,payment_data,payment_secret,byte,32 +tlvdata,tlv_payload,payment_data,total_msat,tu64, +msgtype,invalid_realm,PERM|1 +msgtype,temporary_node_failure,NODE|2 +msgtype,permanent_node_failure,PERM|NODE|2 +msgtype,required_node_feature_missing,PERM|NODE|3 +msgtype,invalid_onion_version,BADONION|PERM|4 +msgdata,invalid_onion_version,sha256_of_onion,sha256, +msgtype,invalid_onion_hmac,BADONION|PERM|5 +msgdata,invalid_onion_hmac,sha256_of_onion,sha256, +msgtype,invalid_onion_key,BADONION|PERM|6 +msgdata,invalid_onion_key,sha256_of_onion,sha256, +msgtype,temporary_channel_failure,UPDATE|7 +msgdata,temporary_channel_failure,len,u16, +msgdata,temporary_channel_failure,channel_update,byte,len +msgtype,permanent_channel_failure,PERM|8 +msgtype,required_channel_feature_missing,PERM|9 +msgtype,unknown_next_peer,PERM|10 +msgtype,amount_below_minimum,UPDATE|11 +msgdata,amount_below_minimum,htlc_msat,u64, +msgdata,amount_below_minimum,len,u16, +msgdata,amount_below_minimum,channel_update,byte,len +msgtype,fee_insufficient,UPDATE|12 +msgdata,fee_insufficient,htlc_msat,u64, +msgdata,fee_insufficient,len,u16, +msgdata,fee_insufficient,channel_update,byte,len +msgtype,incorrect_cltv_expiry,UPDATE|13 +msgdata,incorrect_cltv_expiry,cltv_expiry,u32, +msgdata,incorrect_cltv_expiry,len,u16, +msgdata,incorrect_cltv_expiry,channel_update,byte,len +msgtype,expiry_too_soon,UPDATE|14 +msgdata,expiry_too_soon,len,u16, +msgdata,expiry_too_soon,channel_update,byte,len +msgtype,incorrect_or_unknown_payment_details,PERM|15 +msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64, +msgdata,incorrect_or_unknown_payment_details,height,u32, +msgtype,final_incorrect_cltv_expiry,18 +msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32, +msgtype,final_incorrect_htlc_amount,19 +msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64, +msgtype,channel_disabled,UPDATE|20 +msgtype,expiry_too_far,21 +msgtype,invalid_onion_payload,PERM|22 +msgdata,invalid_onion_payload,type,varint, +msgdata,invalid_onion_payload,offset,u16, +msgtype,mpp_timeout,23 diff --git a/electrum/lnwire/peer_wire.csv b/electrum/lnwire/peer_wire.csv new file mode 100644 index 000000000..a128e9c71 --- /dev/null +++ b/electrum/lnwire/peer_wire.csv @@ -0,0 +1,210 @@ +msgtype,init,16 +msgdata,init,gflen,u16, +msgdata,init,globalfeatures,byte,gflen +msgdata,init,flen,u16, +msgdata,init,features,byte,flen +msgdata,init,tlvs,init_tlvs, +tlvtype,init_tlvs,networks,1 +tlvdata,init_tlvs,networks,chains,chain_hash,... +msgtype,error,17 +msgdata,error,channel_id,channel_id, +msgdata,error,len,u16, +msgdata,error,data,byte,len +msgtype,ping,18 +msgdata,ping,num_pong_bytes,u16, +msgdata,ping,byteslen,u16, +msgdata,ping,ignored,byte,byteslen +msgtype,pong,19 +msgdata,pong,byteslen,u16, +msgdata,pong,ignored,byte,byteslen +tlvtype,n1,tlv1,1 +tlvdata,n1,tlv1,amount_msat,tu64, +tlvtype,n1,tlv2,2 +tlvdata,n1,tlv2,scid,short_channel_id, +tlvtype,n1,tlv3,3 +tlvdata,n1,tlv3,node_id,point, +tlvdata,n1,tlv3,amount_msat_1,u64, +tlvdata,n1,tlv3,amount_msat_2,u64, +tlvtype,n1,tlv4,254 +tlvdata,n1,tlv4,cltv_delta,u16, +tlvtype,n2,tlv1,0 +tlvdata,n2,tlv1,amount_msat,tu64, +tlvtype,n2,tlv2,11 +tlvdata,n2,tlv2,cltv_expiry,tu32, +msgtype,open_channel,32 +msgdata,open_channel,chain_hash,chain_hash, +msgdata,open_channel,temporary_channel_id,byte,32 +msgdata,open_channel,funding_satoshis,u64, +msgdata,open_channel,push_msat,u64, +msgdata,open_channel,dust_limit_satoshis,u64, +msgdata,open_channel,max_htlc_value_in_flight_msat,u64, +msgdata,open_channel,channel_reserve_satoshis,u64, +msgdata,open_channel,htlc_minimum_msat,u64, +msgdata,open_channel,feerate_per_kw,u32, +msgdata,open_channel,to_self_delay,u16, +msgdata,open_channel,max_accepted_htlcs,u16, +msgdata,open_channel,funding_pubkey,point, +msgdata,open_channel,revocation_basepoint,point, +msgdata,open_channel,payment_basepoint,point, +msgdata,open_channel,delayed_payment_basepoint,point, +msgdata,open_channel,htlc_basepoint,point, +msgdata,open_channel,first_per_commitment_point,point, +msgdata,open_channel,channel_flags,byte, +msgdata,open_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,open_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,accept_channel,33 +msgdata,accept_channel,temporary_channel_id,byte,32 +msgdata,accept_channel,dust_limit_satoshis,u64, +msgdata,accept_channel,max_htlc_value_in_flight_msat,u64, +msgdata,accept_channel,channel_reserve_satoshis,u64, +msgdata,accept_channel,htlc_minimum_msat,u64, +msgdata,accept_channel,minimum_depth,u32, +msgdata,accept_channel,to_self_delay,u16, +msgdata,accept_channel,max_accepted_htlcs,u16, +msgdata,accept_channel,funding_pubkey,point, +msgdata,accept_channel,revocation_basepoint,point, +msgdata,accept_channel,payment_basepoint,point, +msgdata,accept_channel,delayed_payment_basepoint,point, +msgdata,accept_channel,htlc_basepoint,point, +msgdata,accept_channel,first_per_commitment_point,point, +msgdata,accept_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,accept_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,funding_created,34 +msgdata,funding_created,temporary_channel_id,byte,32 +msgdata,funding_created,funding_txid,sha256, +msgdata,funding_created,funding_output_index,u16, +msgdata,funding_created,signature,signature, +msgtype,funding_signed,35 +msgdata,funding_signed,channel_id,channel_id, +msgdata,funding_signed,signature,signature, +msgtype,funding_locked,36 +msgdata,funding_locked,channel_id,channel_id, +msgdata,funding_locked,next_per_commitment_point,point, +msgtype,shutdown,38 +msgdata,shutdown,channel_id,channel_id, +msgdata,shutdown,len,u16, +msgdata,shutdown,scriptpubkey,byte,len +msgtype,closing_signed,39 +msgdata,closing_signed,channel_id,channel_id, +msgdata,closing_signed,fee_satoshis,u64, +msgdata,closing_signed,signature,signature, +msgtype,update_add_htlc,128 +msgdata,update_add_htlc,channel_id,channel_id, +msgdata,update_add_htlc,id,u64, +msgdata,update_add_htlc,amount_msat,u64, +msgdata,update_add_htlc,payment_hash,sha256, +msgdata,update_add_htlc,cltv_expiry,u32, +msgdata,update_add_htlc,onion_routing_packet,byte,1366 +msgtype,update_fulfill_htlc,130 +msgdata,update_fulfill_htlc,channel_id,channel_id, +msgdata,update_fulfill_htlc,id,u64, +msgdata,update_fulfill_htlc,payment_preimage,byte,32 +msgtype,update_fail_htlc,131 +msgdata,update_fail_htlc,channel_id,channel_id, +msgdata,update_fail_htlc,id,u64, +msgdata,update_fail_htlc,len,u16, +msgdata,update_fail_htlc,reason,byte,len +msgtype,update_fail_malformed_htlc,135 +msgdata,update_fail_malformed_htlc,channel_id,channel_id, +msgdata,update_fail_malformed_htlc,id,u64, +msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256, +msgdata,update_fail_malformed_htlc,failure_code,u16, +msgtype,commitment_signed,132 +msgdata,commitment_signed,channel_id,channel_id, +msgdata,commitment_signed,signature,signature, +msgdata,commitment_signed,num_htlcs,u16, +msgdata,commitment_signed,htlc_signature,signature,num_htlcs +msgtype,revoke_and_ack,133 +msgdata,revoke_and_ack,channel_id,channel_id, +msgdata,revoke_and_ack,per_commitment_secret,byte,32 +msgdata,revoke_and_ack,next_per_commitment_point,point, +msgtype,update_fee,134 +msgdata,update_fee,channel_id,channel_id, +msgdata,update_fee,feerate_per_kw,u32, +msgtype,channel_reestablish,136 +msgdata,channel_reestablish,channel_id,channel_id, +msgdata,channel_reestablish,next_commitment_number,u64, +msgdata,channel_reestablish,next_revocation_number,u64, +msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32,option_data_loss_protect,option_static_remotekey +msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect,option_static_remotekey +msgtype,announcement_signatures,259 +msgdata,announcement_signatures,channel_id,channel_id, +msgdata,announcement_signatures,short_channel_id,short_channel_id, +msgdata,announcement_signatures,node_signature,signature, +msgdata,announcement_signatures,bitcoin_signature,signature, +msgtype,channel_announcement,256 +msgdata,channel_announcement,node_signature_1,signature, +msgdata,channel_announcement,node_signature_2,signature, +msgdata,channel_announcement,bitcoin_signature_1,signature, +msgdata,channel_announcement,bitcoin_signature_2,signature, +msgdata,channel_announcement,len,u16, +msgdata,channel_announcement,features,byte,len +msgdata,channel_announcement,chain_hash,chain_hash, +msgdata,channel_announcement,short_channel_id,short_channel_id, +msgdata,channel_announcement,node_id_1,point, +msgdata,channel_announcement,node_id_2,point, +msgdata,channel_announcement,bitcoin_key_1,point, +msgdata,channel_announcement,bitcoin_key_2,point, +msgtype,node_announcement,257 +msgdata,node_announcement,signature,signature, +msgdata,node_announcement,flen,u16, +msgdata,node_announcement,features,byte,flen +msgdata,node_announcement,timestamp,u32, +msgdata,node_announcement,node_id,point, +msgdata,node_announcement,rgb_color,byte,3 +msgdata,node_announcement,alias,byte,32 +msgdata,node_announcement,addrlen,u16, +msgdata,node_announcement,addresses,byte,addrlen +msgtype,channel_update,258 +msgdata,channel_update,signature,signature, +msgdata,channel_update,chain_hash,chain_hash, +msgdata,channel_update,short_channel_id,short_channel_id, +msgdata,channel_update,timestamp,u32, +msgdata,channel_update,message_flags,byte, +msgdata,channel_update,channel_flags,byte, +msgdata,channel_update,cltv_expiry_delta,u16, +msgdata,channel_update,htlc_minimum_msat,u64, +msgdata,channel_update,fee_base_msat,u32, +msgdata,channel_update,fee_proportional_millionths,u32, +msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max +msgtype,query_short_channel_ids,261,gossip_queries +msgdata,query_short_channel_ids,chain_hash,chain_hash, +msgdata,query_short_channel_ids,len,u16, +msgdata,query_short_channel_ids,encoded_short_ids,byte,len +msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs, +tlvtype,query_short_channel_ids_tlvs,query_flags,1 +tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8, +tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,... +msgtype,reply_short_channel_ids_end,262,gossip_queries +msgdata,reply_short_channel_ids_end,chain_hash,chain_hash, +msgdata,reply_short_channel_ids_end,complete,byte, +msgtype,query_channel_range,263,gossip_queries +msgdata,query_channel_range,chain_hash,chain_hash, +msgdata,query_channel_range,first_blocknum,u32, +msgdata,query_channel_range,number_of_blocks,u32, +msgdata,query_channel_range,tlvs,query_channel_range_tlvs, +tlvtype,query_channel_range_tlvs,query_option,1 +tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint, +msgtype,reply_channel_range,264,gossip_queries +msgdata,reply_channel_range,chain_hash,chain_hash, +msgdata,reply_channel_range,first_blocknum,u32, +msgdata,reply_channel_range,number_of_blocks,u32, +msgdata,reply_channel_range,complete,byte, +msgdata,reply_channel_range,len,u16, +msgdata,reply_channel_range,encoded_short_ids,byte,len +msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs, +tlvtype,reply_channel_range_tlvs,timestamps_tlv,1 +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8, +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,... +tlvtype,reply_channel_range_tlvs,checksums_tlv,3 +tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,... +subtype,channel_update_timestamps +subtypedata,channel_update_timestamps,timestamp_node_id_1,u32, +subtypedata,channel_update_timestamps,timestamp_node_id_2,u32, +subtype,channel_update_checksums +subtypedata,channel_update_checksums,checksum_node_id_1,u32, +subtypedata,channel_update_checksums,checksum_node_id_2,u32, +msgtype,gossip_timestamp_filter,265,gossip_queries +msgdata,gossip_timestamp_filter,chain_hash,chain_hash, +msgdata,gossip_timestamp_filter,first_timestamp,u32, +msgdata,gossip_timestamp_filter,timestamp_range,u32,