From 53c5134591c53efdd222a9fa3c86a83cf7359ae0 Mon Sep 17 00:00:00 2001 From: hackrush Date: Tue, 6 Mar 2018 03:45:52 +0530 Subject: [PATCH] Consolidated all 3 scripts into 1, better error messages --- CHANGELOG.md | 49 +--------- scripts/gen_api_docs.py | 105 --------------------- scripts/gen_cli_docs.py | 86 ----------------- scripts/gen_docs.py | 204 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 189 insertions(+), 255 deletions(-) delete mode 100644 scripts/gen_api_docs.py delete mode 100644 scripts/gen_cli_docs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ea8d158..be70701b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,22 +13,6 @@ at anytime. * ### Fixed - * - * improper parsing of arguments to CLI settings_set (https://github.com/lbryio/lbry/issues/930) - * unnecessarily verbose exchange rate error (https://github.com/lbryio/lbry/issues/984) - * value error due to a race condition when saving to the claim cache (https://github.com/lbryio/lbry/issues/1013) - * being unable to re-download updated content (https://github.com/lbryio/lbry/issues/951) - * sending error messages for failed api requests - * file manager startup being slow when handling thousands of files - * handling decryption error for blobs encrypted with an invalid key - * handling stream with no data blob (https://github.com/lbryio/lbry/issues/905) - * fetching the external ip - * `blob_list` returning an error with --uri parameter and incorrectly returning `[]` for streams where blobs are known (https://github.com/lbryio/lbry/issues/895) - * `get` failing with a non-useful error message when given a uri for a channel claim - * exception checking in several wallet unit tests - * daemon not erring properly for non-numeric values being passed to the `bid` parameter for the `publish` method - * `publish` command to allow updating claims with a `bid` amount higher than the wallet balance, so long as the amount is less than the wallet balance plus the bid amount of the claim being updated (https://github.com/lbryio/lbry/issues/748) - * incorrect `blob_num` for the stream terminator blob, which would result in creating invalid streams. Such invalid streams are detected on startup and are automatically removed (https://github.com/lbryio/lbry/issues/1124) * fixed the inconsistencies in API and CLI docstrings * @@ -40,30 +24,11 @@ at anytime. * * ### Added - * link to instructions on how to change the default peer port - * `lbrynet-console`, a tool to run or connect to lbrynet-daemon and launch an interactive python console with the api functions built in. - * `--conf` CLI flag to specify an alternate config file - * `peer_port`, `disable_max_key_fee`, `auto_renew_claim_height_delta`, `blockchain_name`, and `lbryum_servers` to configurable settings - * `wallet_unlock` command (available during startup to unlock an encrypted wallet) - * support for wallet encryption via new commands `wallet_decrypt` and `wallet_encrypt` - * `channel_import`, `channel_export`, and `claim_renew` commands - * `blob_availability` and `stream_availability` commands for debugging download issues - * a new startup stage to indicate if the daemon is waiting for the `wallet_unlock` command. - * `abandon_info` dictionary (containing `claim_name`, `claim_id`, `address`, `amount`, `balance_delta` and `nout`) for claims, supports, and updates returned by `transaction_list` - * `permanent_url` string to `channel_list_mine`, `claim_list`, `claim_show`, `resolve` and `resolve_name` (see lbryio/lbryum#203) - * `is_mine` boolean to `channel_list` results - * `txid`, `nout`, `channel_claim_id`, `channel_claim_name`, `status`, `blobs_completed`, and `blobs_in_stream` fields to file objects returned by `file_list` and `get` - * `txid`, `nout`, `channel_claim_id`, and `channel_claim_name` filters for `file` commands (`file_list`, `file_set_status`, `file_reflect`, and `file_delete`) - * unit tests for `SQLiteStorage` and updated old tests for relevant changes (https://github.com/lbryio/lbry/issues/1088) * scripts to autogenerate documentation - * - -### Added - * * ### Removed - * + * short(single dashed) arguments for `lbrynet-cli` * @@ -139,18 +104,6 @@ at anytime. * old storage classes used by the file manager, wallet, and blob manager * old `.db` database files from the data directory - * `seccure` and `gmpy` dependencies - * support for positional arguments in cli `settings_set`. Now only accepts settings changes in the form `--setting_key=value` - * `auto_re_reflect` setting from the conf file, use the `reflect_uploads` setting instead - * `name` argument for `claim_show` command - * `message` response field in file objects returned by `file_list` and `get` - * `include_tip_info` argument from `transaction_list`, which will now always include tip information. - * old and unused UI related code - * unnecessary `TempBlobManager` class - * old storage classes used by the file manager, wallet, and blob manager - * old `.db` database files from the data directory - * short(single dashed) arguments for `lbrynet-cli` - ## [0.18.0] - 2017-11-08 ### Fixed * Fixed amount of close nodes to add to list in case of extension to neighbouring k-buckets diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py deleted file mode 100644 index 701aad90e..000000000 --- a/scripts/gen_api_docs.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -# Generate docs: python gen_api_docs.py -# See docs: pip install mkdocs; mkdocs serve -# Push docs: mkdocs build - -import inspect -import sys -import re -import os.path as op -from tabulate import tabulate -from lbrynet.daemon.Daemon import Daemon - -INDENT = " " -REQD_CMD_REGEX = r"\(.*?=<(?P.*?)>\)" -OPT_CMD_REGEX = r"\[.*?=<(?P.*?)>\]" -CMD_REGEX = r"--.*?(?P.*?)[=,\s,<]" -DOCS_DIR = "docs_build" - - -def _tabulate_options(_options_docstr, method, reqd_matches, opt_matches): - _option_list = [] - for line in _options_docstr.splitlines(): - if (line.strip().startswith("--")): - # separates command name and description - parts = line.split(":", 1) - - # checks whether the command is optional or required - # and remove the cli type formatting and convert to - # api style formatitng - match = re.findall(CMD_REGEX, parts[0]) - - if match[0] not in reqd_matches: - parts[0] = "'" + match[0] + "' (optional)" - else: - parts[0] = "'" + match[0] + "'" - - # separates command type(in brackets) and description - new_parts = parts[1].lstrip().split(" ", 1) - else: - parts = [line] - - # len will be 2 when there's cmd name and description - if len(parts) == 2: - _option_list.append([parts[0], ":", new_parts[0], new_parts[1]]) - # len will be 1 when there's continuation of multiline description in the next line - # check `blob_announce`'s `stream_hash` command - elif len(parts) == 1: - _option_list.append([None, None, None, parts[0]]) - else: - print "Error: Ill formatted doc string for {}".format(method) - print "Error causing line: {}".format(line) - - # tabulate to make the options look pretty - _options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain") - - # tabulate to make the options look pretty - _options_docstr = "" - for line in _options_docstr_no_indent.splitlines(): - _options_docstr += INDENT + line + '\n' - - return _options_docstr - - -def _doc(obj): - docstr = (inspect.getdoc(obj) or '').strip() - - try: - _desc, _docstr_after_desc = docstr.split("Usage:", 1) - _usage_docstr, _docstr_after_options = _docstr_after_desc.split("Options:", 1) - _options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1) - except(ValueError): - print "Error: Ill formatted doc string for {}".format(obj) - return "Error!" - - opt_matches = re.findall(OPT_CMD_REGEX, _usage_docstr) - reqd_matches = re.findall(REQD_CMD_REGEX, _usage_docstr) - - _options_docstr = _tabulate_options(_options_docstr.strip(), obj, reqd_matches, opt_matches) - - docstr = _desc + \ - "Args:\n" + \ - _options_docstr + \ - "\nReturns:" + \ - _returns_docstr - - return docstr - - -def main(): - curdir = op.dirname(op.realpath(__file__)) - api_doc_path = op.realpath(op.join(curdir, '..', DOCS_DIR, 'index.md')) - - docs = '' - for method_name in sorted(Daemon.callable_methods.keys()): - method = Daemon.callable_methods[method_name] - docs += '## ' + method_name + "\n\n```text\n" + _doc(method) + "\n```\n\n" - - docs = "# LBRY JSON-RPC API Documentation\n\n" + docs - with open(api_doc_path, 'w+') as f: - f.write(docs) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/gen_cli_docs.py b/scripts/gen_cli_docs.py deleted file mode 100644 index 7aaa0954b..000000000 --- a/scripts/gen_cli_docs.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- - -# Generate docs: python gen_cli_docs.py -# See docs: pip install mkdocs; mkdocs serve -# Push docs: mkdocs build - -import inspect -import os.path as op -import sys -from tabulate import tabulate -from lbrynet.daemon.Daemon import Daemon - -INDENT = " " -DOCS_DIR = "docs_build" - - -def _tabulate_options(_options_docstr, method): - _option_list = [] - for line in _options_docstr.splitlines(): - if (line.strip().startswith("--")): - # separates command name and description - parts = line.split(":", 1) - # separates command type(in brackets) and description - new_parts = parts[1].lstrip().split(" ", 1) - else: - parts = [line] - - # len will be 2 when there's cmd name and description - if len(parts) == 2: - _option_list.append([parts[0], ":", new_parts[0], new_parts[1]]) - # len will be 1 when there's continuation of multiline description in the next line - # check `blob_announce`'s `stream_hash` command - elif len(parts) == 1: - _option_list.append([None, None, None, parts[0]]) - else: - print "Error: Ill formatted doc string for {}".format(method) - print "Error causing line: {}".format(line) - - # tabulate to make the options look pretty - _options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain") - - # Indent the options properly - _options_docstr = "" - for line in _options_docstr_no_indent.splitlines(): - _options_docstr += INDENT + line + '\n' - - return _options_docstr - - -def _doc(obj): - docstr = (inspect.getdoc(obj) or '').strip() - - try: - _usage_docstr, _docstr_after_options = docstr.split("Options:", 1) - _options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1) - except(ValueError): - print "Error: Ill formatted doc string for {}".format(obj) - return "Error!" - - _options_docstr = _tabulate_options(_options_docstr.strip(), obj) - - docstr = _usage_docstr + \ - "\nOptions:\n" + \ - _options_docstr + \ - "\nReturns:" + \ - _returns_docstr - - return docstr - - -def main(): - curdir = op.dirname(op.realpath(__file__)) - cli_doc_path = op.realpath(op.join(curdir, '..', DOCS_DIR, 'cli.md')) - - docs = '' - for method_name in sorted(Daemon.callable_methods.keys()): - method = Daemon.callable_methods[method_name] - docs += '## ' + method_name + "\n\n```text\n" + _doc(method) + "\n```\n\n" - - docs = "# LBRY Command Line Documentation\n\n" + docs - with open(cli_doc_path, 'w+') as f: - f.write(docs) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/gen_docs.py b/scripts/gen_docs.py index 1a076d299..57db05c23 100644 --- a/scripts/gen_docs.py +++ b/scripts/gen_docs.py @@ -1,25 +1,197 @@ -import os.path as op +# -*- coding: utf-8 -*- + +# Generate docs: python gen_api_docs.py +# See docs: pip install mkdocs; mkdocs serve +# Push docs: mkdocs build + +import re +import inspect import subprocess +import os +from lbrynet.daemon.Daemon import Daemon try: - import mkdocs + import mkdocs except ImportError: - raise ImportError("mkdocs is not installed") + raise ImportError("mkdocs is not installed") try: - import tabulate + from tabulate import tabulate except ImportError: - raise ImportError("tabulate is not installed") + raise ImportError("tabulate is not installed") -try: - import gen_cli_docs - import gen_api_docs -except ImportError: - raise ImportError("Probably not inside the lbry's virtual environment or daemon not installed") +INDENT = " " +REQD_CMD_REGEX = r"\(.*?=<(?P.*?)>\)" +OPT_CMD_REGEX = r"\[.*?=<(?P.*?)>\]" +CMD_REGEX = r"--.*?(?P.*?)[=,\s,<]" +DOCS_DIR = "docs_build" -gen_cli_docs.main() -gen_api_docs.main() -cwd = op.dirname(op.realpath(__file__)) -cwd = op.realpath(op.join(cwd, "..")) -proc = subprocess.Popen("exec mkdocs build", cwd=cwd, shell=True) -proc.kill() + +def _cli_tabulate_options(_options_docstr, method): + _option_list = [] + for line in _options_docstr.splitlines(): + if (line.strip().startswith("--")): + # separates command name and description + parts = line.split(":", 1) + # separates command type(in brackets) and description + new_parts = parts[1].lstrip().split(" ", 1) + else: + parts = [line] + + # len will be 2 when there's cmd name and description + if len(parts) == 2: + _option_list.append([parts[0], ":", new_parts[0], new_parts[1]]) + # len will be 1 when there's continuation of multiline description in the next line + # check `blob_announce`'s `stream_hash` command + elif len(parts) == 1: + _option_list.append([None, None, None, parts[0]]) + else: + print "Error: Ill formatted doc string for {}".format(method) + print "Error causing line: {}".format(line) + + # tabulate to make the options look pretty + _options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain") + + # Indent the options properly + _options_docstr = "" + for line in _options_docstr_no_indent.splitlines(): + _options_docstr += INDENT + line + '\n' + + return _options_docstr + + +def _api_tabulate_options(_options_docstr, method, reqd_matches, opt_matches): + _option_list = [] + for line in _options_docstr.splitlines(): + if (line.strip().startswith("--")): + # separates command name and description + parts = line.split(":", 1) + + # checks whether the command is optional or required + # and remove the cli type formatting and convert to + # api style formatitng + match = re.findall(CMD_REGEX, parts[0]) + + if match[0] not in reqd_matches: + parts[0] = "'" + match[0] + "'" + else: + parts[0] = "'" + match[0] + "' (required)" + + # separates command type(in brackets) and description + new_parts = parts[1].lstrip().split(" ", 1) + else: + parts = [line] + + # len will be 2 when there's cmd name and description + if len(parts) == 2: + _option_list.append([parts[0], ":", new_parts[0], new_parts[1]]) + # len will be 1 when there's continuation of multiline description in the next line + # check `blob_announce`'s `stream_hash` command + elif len(parts) == 1: + _option_list.append([None, None, None, parts[0]]) + else: + print "Error: Ill formatted doc string for {}".format(method) + print "Error causing line: {}".format(line) + + # tabulate to make the options look pretty + _options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain") + + # tabulate to make the options look pretty + _options_docstr = "" + for line in _options_docstr_no_indent.splitlines(): + _options_docstr += INDENT + line + '\n' + + return _options_docstr + + +def _cli_doc(obj): + docstr = (inspect.getdoc(obj) or '').strip() + + try: + _usage_docstr, _docstr_after_options = docstr.split("Options:", 1) + _options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1) + except(ValueError): + print "Error: Ill formatted doc string for {}".format(obj) + print "Please ensure that the docstring has all the three headings i.e. \"Usage:\"" + print "\"Options:\" and \"Returns:\" exactly as specified, including the colon" + return "Error!" + + try: + _options_docstr = _cli_tabulate_options(_options_docstr.strip(), obj) + except Exception as e: + print "Please make sure that the individual options are properly formatted" + print "It should be strictly of the format:" + print "--command_name= : (type) desc" + print e.message + + docstr = _usage_docstr + \ + "\nOptions:\n" + \ + _options_docstr + \ + "\nReturns:" + \ + _returns_docstr + + return docstr + + +def _api_doc(obj): + docstr = (inspect.getdoc(obj) or '').strip() + + try: + _desc, _docstr_after_desc = docstr.split("Usage:", 1) + _usage_docstr, _docstr_after_options = _docstr_after_desc.split("Options:", 1) + _options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1) + except(ValueError): + print "Error: Ill formatted doc string for {}".format(obj) + print "Please ensure that the docstring has all the three headings i.e. \"Usage:\"" + print "\"Options:\" and \"Returns:\" exactly as specified, including the colon" + return "Error!" + + opt_matches = re.findall(OPT_CMD_REGEX, _usage_docstr) + reqd_matches = re.findall(REQD_CMD_REGEX, _usage_docstr) + + try: + _options_docstr = _api_tabulate_options(_options_docstr.strip(), obj, reqd_matches, opt_matches) + except Exception as e: + print "Please make sure that the individual options are properly formatted" + print "It should be strictly of the format:" + print "--command_name= : (type) desc" + print e.message + + docstr = _desc + \ + "Args:\n" + \ + _options_docstr + \ + "\nReturns:" + \ + _returns_docstr + + return docstr + + +def main(): + curdir = os.path.dirname(os.path.realpath(__file__)) + api_doc_path = os.path.realpath(os.path.join(curdir, '..', DOCS_DIR, 'index.md')) + cli_doc_path = os.path.realpath(os.path.join(curdir, '..', DOCS_DIR, 'cli.md')) + + _api_docs = '' + _cli_docs = '' + for method_name in sorted(Daemon.callable_methods.keys()): + method = Daemon.callable_methods[method_name] + _api_docs += '## ' + method_name + "\n\n```text\n" + _api_doc(method) + "\n```\n\n" + _cli_docs += '## ' + method_name + "\n\n```text\n" + _cli_doc(method) + "\n```\n\n" + + _api_docs = "# LBRY JSON-RPC API Documentation\n\n" + _api_docs + with open(api_doc_path, 'w+') as f: + f.write(_api_docs) + + _cli_docs = "# LBRY JSON-RPC API Documentation\n\n" + _cli_docs + with open(cli_doc_path, 'w+') as f: + f.write(_cli_docs) + + +if __name__ == '__main__': + cwd = os.path.dirname(os.path.realpath(__file__)) + cwd = os.path.realpath(os.path.join(cwd, "..")) + directory = os.path.join(cwd, "docs_build") + if not os.path.exists(directory): + os.makedirs(directory) + proc = subprocess.Popen("exec mkdocs build", cwd=cwd, shell=True) + proc.kill()