diff --git a/lbry/service/json_encoder.py b/lbry/service/json_encoder.py new file mode 100644 index 000000000..032aab2e9 --- /dev/null +++ b/lbry/service/json_encoder.py @@ -0,0 +1,332 @@ +import logging +from decimal import Decimal +from binascii import hexlify, unhexlify +from datetime import datetime, date +from json import JSONEncoder + +from google.protobuf.message import DecodeError + +from lbry.schema.claim import Claim +from lbry.wallet.wallet import Wallet, Account +from lbry.blockchain.transaction import Transaction, Output +from lbry.crypto.bip32 import PubKey +from lbry.blockchain.dewies import dewies_to_lbc +from lbry.stream.managed_stream import ManagedStream + + +log = logging.getLogger(__name__) + + +output_doc = { + 'txid': "hash of transaction in hex", + 'nout': "position in the transaction", + 'height': "block where transaction was recorded", + 'amount': "value of the txo as a decimal", + 'address': "address of who can spend the txo", + 'confirmations': "number of confirmed blocks", + 'is_change': "payment to change address, only available when it can be determined", + 'is_received': "true if txo was sent from external account to this account", + 'is_spent': "true if txo is spent", + 'is_mine': "payment to one of your accounts, only available when it can be determined", + 'type': "one of 'claim', 'support' or 'purchase'", + 'name': "when type is 'claim' or 'support', this is the claim name", + 'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id", + 'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'", + 'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload", + 'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc", + 'protobuf': "hex encoded raw protobuf version of 'value' field", + 'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL", + 'claim': "for purchase outputs only, metadata of purchased claim", + 'reposted_claim': "for repost claims only, metadata of claim being reposted", + 'signing_channel': "for signed claims only, metadata of signing channel", + 'is_channel_signature_valid': "for signed claims only, whether signature is valid", + 'purchase_receipt': "metadata for the purchase transaction associated with this claim" +} + + +transaction_doc = { + 'txid': "hash of transaction in hex", + 'height': "block where transaction was recorded", + 'inputs': [output_doc], + 'outputs': [output_doc], + 'total_input': "sum of inputs as a decimal", + 'total_output': "sum of outputs, sans fee, as a decimal", + 'total_fee': "fee amount", + 'hex': "entire transaction encoded in hex", +} + + +account_doc = { + 'id': 'account_id', + 'is_default': 'this account is used by default', + 'ledger': 'name of crypto currency and network', + 'name': 'optional account name', + 'seed': 'human friendly words from which account can be recreated', + 'encrypted': 'if account is encrypted', + 'private_key': 'extended private key', + 'public_key': 'extended public key', + 'address_generator': 'settings for generating addresses', + 'modified_on': 'date of last modification to account settings' +} + + +wallet_doc = { + 'id': 'wallet_id', + 'name': 'optional wallet name', +} + + +managedstream_doc = { + 'streaming_url': '(str) url to stream the file using range requests', + 'completed': '(bool) true if download is completed', + 'file_name': '(str) name of file', + 'download_directory': '(str) download directory', + 'points_paid': '(float) credit paid to download file', + 'stopped': '(bool) true if download is stopped', + 'stream_hash': '(str) stream hash of file', + 'stream_name': '(str) stream name', + 'suggested_file_name': '(str) suggested file name', + 'sd_hash': '(str) sd hash of file', + 'download_path': '(str) download path of file', + 'mime_type': '(str) mime type of file', + 'key': '(str) key attached to file', + 'total_bytes_lower_bound': '(int) lower bound file size in bytes', + 'total_bytes': '(int) file upper bound size in bytes', + 'written_bytes': '(int) written size in bytes', + 'blobs_completed': '(int) number of fully downloaded blobs', + 'blobs_in_stream': '(int) total blobs on stream', + 'blobs_remaining': '(int) total blobs remaining to download', + 'status': '(str) downloader status', + 'claim_id': '(str) None if claim is not found else the claim id', + 'txid': '(str) None if claim is not found else the transaction id', + 'nout': '(int) None if claim is not found else the transaction output index', + 'outpoint': '(str) None if claim is not found else the tx and output', + 'metadata': '(dict) None if claim is not found else the claim metadata', + 'channel_claim_id': '(str) None if claim is not found or not signed', + 'channel_name': '(str) None if claim is not found or not signed', + 'claim_name': '(str) None if claim is not found else the claim name' +} + + +address_doc = { + +} + + +def encode_pagination_doc(items): + return { + "page": "Page number of the current items.", + "page_size": "Number of items to show on a page.", + "total_pages": "Total number of pages.", + "total_items": "Total number of items.", + "items": [items], + } + + +class JSONResponseEncoder(JSONEncoder): + + def __init__(self, *args, service, include_protobuf=False, **kwargs): + super().__init__(*args, **kwargs) + self.service = service + self.include_protobuf = include_protobuf + + def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements + if isinstance(obj, Account): + return self.encode_account(obj) + if isinstance(obj, Wallet): + return self.encode_wallet(obj) + if isinstance(obj, ManagedStream): + return self.encode_file(obj) + if isinstance(obj, Transaction): + return self.encode_transaction(obj) + if isinstance(obj, Output): + return self.encode_output(obj) + if isinstance(obj, Claim): + return self.encode_claim(obj) + if isinstance(obj, PubKey): + return obj.extended_key_string() + if isinstance(obj, date): + return obj.isoformat() + if isinstance(obj, datetime): + return obj.strftime("%Y%m%dT%H:%M:%S") + if isinstance(obj, Decimal): + return float(obj) + if isinstance(obj, bytes): + return obj.decode() + return super().default(obj) + + def encode_transaction(self, tx): + return { + 'txid': tx.id, + 'height': tx.height, + 'inputs': [self.encode_input(txo) for txo in tx.inputs], + 'outputs': [self.encode_output(txo) for txo in tx.outputs], + 'total_input': dewies_to_lbc(tx.input_sum), + 'total_output': dewies_to_lbc(tx.input_sum - tx.fee), + 'total_fee': dewies_to_lbc(tx.fee), + 'hex': hexlify(tx.raw).decode(), + } + + def encode_output(self, txo, check_signature=True): + if not txo: + return + tx_height = txo.tx_ref.height + best_height = 0#self.ledger.headers.height + output = { + 'txid': txo.tx_ref.id, + 'nout': txo.position, + 'height': tx_height, + 'amount': dewies_to_lbc(txo.amount), + 'address': txo.get_address(self.service.ledger) if txo.has_address else None, + 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, + 'timestamp': 0 #self.ledger.headers.estimated_timestamp(tx_height) + } + if txo.is_spent is not None: + output['is_spent'] = txo.is_spent + if txo.is_my_output is not None: + output['is_my_output'] = txo.is_my_output + if txo.is_my_input is not None: + output['is_my_input'] = txo.is_my_input + if txo.sent_supports is not None: + output['sent_supports'] = dewies_to_lbc(txo.sent_supports) + if txo.sent_tips is not None: + output['sent_tips'] = dewies_to_lbc(txo.sent_tips) + if txo.received_tips is not None: + output['received_tips'] = dewies_to_lbc(txo.received_tips) + if txo.is_internal_transfer is not None: + output['is_internal_transfer'] = txo.is_internal_transfer + + if txo.script.is_claim_name: + output['type'] = 'claim' + output['claim_op'] = 'create' + elif txo.script.is_update_claim: + output['type'] = 'claim' + output['claim_op'] = 'update' + elif txo.script.is_support_claim: + output['type'] = 'support' + elif txo.script.is_return_data: + output['type'] = 'data' + elif txo.purchase is not None: + output['type'] = 'purchase' + output['claim_id'] = txo.purchased_claim_id + if txo.purchased_claim is not None: + output['claim'] = self.encode_output(txo.purchased_claim) + else: + output['type'] = 'payment' + + if txo.script.is_claim_involved: + output.update({ + 'name': txo.claim_name, + 'normalized_name': txo.normalized_name, + 'claim_id': txo.claim_id, + 'permanent_url': txo.permanent_url, + 'meta': self.encode_claim_meta(txo.meta.copy()) + }) + if 'short_url' in output['meta']: + output['short_url'] = output['meta'].pop('short_url') + if 'canonical_url' in output['meta']: + output['canonical_url'] = output['meta'].pop('canonical_url') + if txo.claims is not None: + output['claims'] = [self.encode_output(o) for o in txo.claims] + if txo.reposted_claim is not None: + output['reposted_claim'] = self.encode_output(txo.reposted_claim) + if txo.script.is_claim_name or txo.script.is_update_claim: + try: + output['value'] = txo.claim + output['value_type'] = txo.claim.claim_type + if self.include_protobuf: + output['protobuf'] = hexlify(txo.claim.to_bytes()) + if txo.purchase_receipt is not None: + output['purchase_receipt'] = self.encode_output(txo.purchase_receipt) + if txo.claim.is_channel: + output['has_signing_key'] = txo.has_private_key + if check_signature and txo.claim.is_signed: + if txo.channel is not None: + output['signing_channel'] = self.encode_output(txo.channel) + output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.service.ledger) + else: + output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id} + output['is_channel_signature_valid'] = False + except DecodeError: + pass + return output + + def encode_claim_meta(self, meta): + for key, value in meta.items(): + if key.endswith('_amount'): + if isinstance(value, int): + meta[key] = dewies_to_lbc(value) + if 0 < meta.get('creation_height', 0) <= 0: #self.ledger.headers.height: + meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height']) + return meta + + def encode_input(self, txi): + return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else { + 'txid': txi.txo_ref.tx_ref.id, + 'nout': txi.txo_ref.position + } + + def encode_account(self, account): + result = account.to_dict() + result['id'] = account.id + result.pop('certificates', None) + #result['is_default'] = self.ledger.accounts[0] == account + return result + + @staticmethod + def encode_wallet(wallet): + return { + 'id': wallet.id, + 'name': wallet.name + } + + def encode_file(self, managed_stream): + output_exists = managed_stream.output_file_exists + tx_height = managed_stream.stream_claim_info.height + best_height = 0 #self.ledger.headers.height + return { + 'streaming_url': managed_stream.stream_url, + 'completed': managed_stream.completed, + 'file_name': managed_stream.file_name if output_exists else None, + 'download_directory': managed_stream.download_directory if output_exists else None, + 'download_path': managed_stream.full_path if output_exists else None, + 'points_paid': 0.0, + 'stopped': not managed_stream.running, + 'stream_hash': managed_stream.stream_hash, + 'stream_name': managed_stream.descriptor.stream_name, + 'suggested_file_name': managed_stream.descriptor.suggested_file_name, + 'sd_hash': managed_stream.descriptor.sd_hash, + 'mime_type': managed_stream.mime_type, + 'key': managed_stream.descriptor.key, + 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(), + 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(), + 'written_bytes': managed_stream.written_bytes, + 'blobs_completed': managed_stream.blobs_completed, + 'blobs_in_stream': managed_stream.blobs_in_stream, + 'blobs_remaining': managed_stream.blobs_remaining, + 'status': managed_stream.status, + 'claim_id': managed_stream.claim_id, + 'txid': managed_stream.txid, + 'nout': managed_stream.nout, + 'outpoint': managed_stream.outpoint, + 'metadata': managed_stream.metadata, + 'protobuf': managed_stream.metadata_protobuf, + 'channel_claim_id': managed_stream.channel_claim_id, + 'channel_name': managed_stream.channel_name, + 'claim_name': managed_stream.claim_name, + 'content_fee': managed_stream.content_fee, + 'purchase_receipt': self.encode_output(managed_stream.purchase_receipt), + 'added_on': managed_stream.added_on, + 'height': tx_height, + 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, + 'timestamp': 0, #self.ledger.headers.estimated_timestamp(tx_height), + 'is_fully_reflected': managed_stream.is_fully_reflected + } + + def encode_claim(self, claim): + encoded = getattr(claim, claim.claim_type).to_dict() + if 'public_key' in encoded: + encoded['public_key_id'] = self.service.ledger.public_key_to_address( + unhexlify(encoded['public_key']) + ) + return encoded diff --git a/lbry/service/parser.py b/lbry/service/parser.py new file mode 100644 index 000000000..d0cbb0d26 --- /dev/null +++ b/lbry/service/parser.py @@ -0,0 +1,246 @@ +import json +import inspect +import textwrap +import tokenize +import token +from io import BytesIO +from typing import Tuple, List + +from lbry.service import api +from lbry.service import json_encoder + + +def parse_description(desc) -> dict: + lines = iter(desc.splitlines()) + parts = {'text': []} + current = parts['text'] + for line in lines: + if line.strip() in ('Usage:', 'Options:', 'Returns:'): + current = parts.setdefault(line.strip().lower()[:-1], []) + else: + if line.strip(): + current.append(line) + return parts + + +def parse_type(tokens: List) -> Tuple[str, str]: + type_ = [tokens.pop(0).string] + if tokens and tokens[0].string == '[': + while tokens[0].string != ']': + type_.append(tokens.pop(0).string) + type_.append(tokens.pop(0).string) + json_ = None + if type_ == ['StrOrList']: + type_ = ['str, list'] + elif type_[0] == 'Paginated': + json_ = json_encoder.encode_pagination_doc( + getattr(json_encoder, f'{type_[2].lower()}_doc') + ) + return ''.join(type_), json_ + # obj_type = result[1:-1] + # if '[' in obj_type: + # sub_type = obj_type[obj_type.index('[') + 1:-1] + # obj_type = obj_type[:obj_type.index('[')] + # if obj_type == 'Paginated': + # obj_def = encode_pagination_doc(RETURN_DOCS[sub_type]) + # elif obj_type == 'List': + # obj_def = [RETURN_DOCS[sub_type]] + # else: + # raise NameError(f'Unknown return type: {obj_type}') + # else: + # obj_def = RETURN_DOCS[obj_type] + # return indent(json.dumps(obj_def, indent=4), ' ' * 12) + + +def parse_argument(tokens, method_name='') -> dict: + arg = { + 'name': tokens.pop(0).string, + 'desc': [] + } + if arg['name'] == 'self': + return {} + if tokens[0].string == ':': + tokens.pop(0) + type_tokens = [] + while tokens[0].string not in ('=', ',', ')') and tokens[0].type != token.COMMENT: + type_tokens.append(tokens.pop(0)) + arg['type'] = parse_type(type_tokens)[0] + if tokens[0].string == '=': + tokens.pop(0) + default = tokens.pop(0) + if default.type == token.NAME: + default_value = eval(default.string) + if default_value is not None: + arg['default'] = default_value + elif default.type == token.NUMBER: + arg['default'] = int(default.string) + else: + arg['default'] = default.string + if tokens[0].string == ',': + tokens.pop(0) + if 'type' not in arg: + if 'default' in arg: + arg['type'] = type(arg['default']).__name__ + else: + raise Exception(f"No type and no default value for {arg['name']} in {method_name}") + for part in tokens: + if part.type == token.COMMENT: + arg['desc'].append(part.string[1:].strip()) + return arg + + +def produce_argument_tokens(src: str): + in_args = False + in_comment = False + parsed = [] + for t in tokenize.tokenize(BytesIO(src.encode()).readline): + if t.string == '(': + in_args = True + elif in_args: + if not in_comment and t.string == ',': + in_comment = True + elif in_comment and (t.type == token.NAME or t.string == '**'): + yield parsed + in_comment = False + parsed = [] + if t.type in (token.NAME, token.OP, token.COMMENT, token.STRING, token.NUMBER): + parsed.append(t) + if t.string == ')': + yield parsed + break + + +def parse_return(tokens) -> dict: + d = {'desc': []} + if tokens[0].string == '->': + tokens.pop(0) + type_tokens = [] + while tokens[0].string != ':': + type_tokens.append(tokens.pop(0)) + d['type'], _ = parse_type(type_tokens) + if _: + d['json'] = _ + assert tokens.pop(0).string == ':' + for part in tokens: + if part.type == token.COMMENT: + d['desc'].append(part.string[1:].strip()) + return d + + +def produce_return_tokens(src: str): + in_return = False + parsed = [] + for t in tokenize.tokenize(BytesIO(src.encode()).readline): + if t.string == ')': + in_return = True + elif in_return: + if t.type == token.INDENT: + break + else: + parsed.append(t) + return parsed + + +def parse_method(method, expanders: dict) -> dict: + d = { + 'name': method.__name__, + 'desc': parse_description(textwrap.dedent(method.__doc__)) if method.__doc__ else '', + 'method': method, + 'arguments': [], + 'returns': None + } + src = inspect.getsource(method) + for tokens in produce_argument_tokens(src): + if tokens[0].string == '**': + tokens.pop(0) + expander_name = tokens.pop(0).string[:-7] + if expander_name not in expanders: + raise Exception(f"Expander '{expander_name}' not found, used by {d['name']}.") + expander = expanders[expander_name] + d['arguments'].extend(expander) + else: + arg = parse_argument(tokens, d['name']) + if arg: + d['arguments'].append(arg) + d['returns'] = parse_return(produce_return_tokens(src)) + return d + + +def get_expanders(): + expanders = {} + for e in api.kwarg_expanders: + expanders[e.__name__] = parse_method(e, expanders)['arguments'] + return expanders + + +def get_groups(cls): + return { + group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip() + for group_name in dir(cls) if group_name.endswith('_DOC') + } + + +def get_methods(cls): + expanders = get_expanders() + return { + method: parse_method(getattr(cls, method), expanders) + for method in dir(cls) if not method.endswith('_DOC') and not method.startswith('_') + } + + +def generate_options(method, indent): + flags = [] + for arg in method['arguments']: + if arg['type'] == 'bool': + flags.append(f"--{arg['name']}") + else: + flags.append(f"--{arg['name']}=<{arg['name']}>") + max_len = max(len(f) for f in flags) + 1 + flags = [f.ljust(max_len) for f in flags] + options = [] + for flag, arg in zip(flags, method['arguments']): + line = [f"{indent}{flag}: ({arg['type']}) {' '.join(arg['desc'])}"] + if 'default' in arg: + line.append(f" [default: {arg['default']}]") + options.append(''.join(line)) + return options + + +def augment_description(command): + pass + + + +def get_api_definitions(cls): + groups = get_groups(cls) + commands = get_methods(cls) + for name, command in commands.items(): + parts = name.split('_') + if parts[0] in groups: + command['name'] = '_'.join(parts[1:]) + command['group'] = parts[0] + #command['desc'] = + return {'groups': groups, 'commands': commands} + + +def write(fp): + fp.write('# DO NOT EDIT: GENERATED FILE\n') + fp.write(f'interface = ') + defs = get_api_definitions(api.API) + for c in defs['commands'].values(): + del c['method'] + j = json.dumps(defs, indent=4) + j = j.replace(': false', ': False') + j = j.replace(': true', ': True') + j = j.replace(': null', ': None') + fp.write(j) + fp.write('\n') + + +def main(): + with open('metadata.py', 'w') as fp: + write(fp) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/service/test_parser.py b/tests/unit/service/test_parser.py new file mode 100644 index 000000000..f202a67e4 --- /dev/null +++ b/tests/unit/service/test_parser.py @@ -0,0 +1,134 @@ +from unittest import TestCase +from lbry.service.api import Paginated, Wallet +from lbry.service.parser import ( + parse_method, get_expanders, get_api_definitions, + generate_options +) + + +class FakeAPI: + + THING_DOC = "thing doc" + + def thing_create( + self, + name: str, # the name + value1='hi', # the first value + value2=9 # the second value + ) -> str: # thing name + """create command doc""" + + def thing_list( + self, + value1: str = None, # the first value + value2: int = None, # the second value + value3=False, # a bool + # multi-line + **pagination_kwargs + ) -> Paginated[Wallet]: # list of wallets + """list command doc""" + + def not_grouped(self) -> str: # some string + """ + group command doc + + Usage: + not_grouped [--foo] + + Options: + --foo : (bool) blah + + Returns: + (str) blah + """ + + +class TestParser(TestCase): + maxDiff = None + + def test_parse_method(self): + expanders = get_expanders() + self.assertEqual( + parse_method(FakeAPI.thing_create, expanders), { + 'name': 'thing_create', + 'desc': {'text': ['create command doc']}, + 'method': FakeAPI.thing_create, + 'arguments': [ + {'name': 'name', 'type': 'str', 'desc': ['the name']}, + {'name': 'value1', 'type': 'str', 'default': "'hi'", 'desc': ['the first value']}, + {'name': 'value2', 'type': 'int', 'default': 9, 'desc': ['the second value']}, + ], + 'returns': { + 'type': 'str', + 'desc': ['thing name'] + } + } + ) + self.assertEqual( + parse_method(FakeAPI.thing_list, expanders), { + 'name': 'thing_list', + 'desc': {'text': ['list command doc']}, + 'method': FakeAPI.thing_list, + 'arguments': [ + {'name': 'value1', 'type': 'str', 'desc': ['the first value']}, + {'name': 'value2', 'type': 'int', 'desc': ['the second value']}, + {'name': 'value3', 'type': 'bool', 'default': False, 'desc': ['a bool', 'multi-line']}, + {'name': 'page', 'type': 'int', 'desc': ['page to return during paginating']}, + {'name': 'page_size', 'type': 'int', 'desc': ['number of items on page during pagination']} + ], + 'returns': { + 'type': 'Paginated[Wallet]', + 'desc': ['list of wallets'], + 'json': { + 'page': 'Page number of the current items.', + 'page_size': 'Number of items to show on a page.', + 'total_pages': 'Total number of pages.', + 'total_items': 'Total number of items.', + 'items': [ + {'id': 'wallet_id', 'name': 'optional wallet name'} + ] + }, + } + } + ) + self.assertEqual( + parse_method(FakeAPI.not_grouped, expanders), { + 'name': 'not_grouped', + 'desc': { + 'text': ['group command doc'], + 'usage': [' not_grouped [--foo]'], + 'options': [' --foo : (bool) blah'], + 'returns': [' (str) blah'] + }, + 'method': FakeAPI.not_grouped, + 'arguments': [], + 'returns': {'desc': ['some string'], 'type': 'str'} + } + ) + + def test_get_api_definitions(self): + defs = get_api_definitions(FakeAPI) + self.assertEqual({'groups', 'commands'}, set(defs)) + self.assertEqual(defs['groups'], {'thing': 'thing doc'}) + self.assertEqual(defs['commands']['thing_create']['group'], 'thing') + self.assertEqual(defs['commands']['thing_create']['name'], 'create') + self.assertEqual(defs['commands']['thing_list']['group'], 'thing') + self.assertEqual(defs['commands']['thing_list']['name'], 'list') + self.assertEqual(defs['commands']['not_grouped']['name'], 'not_grouped') + self.assertNotIn('group', defs['commands']['not_grouped']) + + +class TestGenerator(TestCase): + maxDiff = None + + def test_generate_options(self): + expanders = get_expanders() + self.assertEqual( + generate_options(parse_method(FakeAPI.thing_list, expanders), indent=' '), [ + ' --value1= : (str) the first value', + ' --value2= : (int) the second value', + ' --value3 : (bool) a bool multi-line [default: False]', + ' --page= : (int) page to return during paginating', + ' --page_size= : (int) number of items on page during pagination' + ] + )