From f0d7ea4cc66913e494f1365f2f385eff7aaaeb7d Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Tue, 31 Dec 2019 14:52:57 -0500 Subject: [PATCH] updated files and scripts post torba merge --- .gitlab-ci.yml | 18 +- .travis.yml | 17 +- Makefile | 7 +- lbry/lbry/__init__.py | 2 +- lbry/lbry/conf.py | 2 +- lbry/lbry/constants.py | 2 + lbry/lbry/crypto/__init__.py | 0 lbry/lbry/crypto/base58.py | 86 ++ lbry/lbry/crypto/crypt.py | 69 ++ lbry/lbry/crypto/hash.py | 47 ++ lbry/lbry/crypto/util.py | 13 + lbry/lbry/extras/daemon/Daemon.py | 4 +- lbry/lbry/extras/daemon/comment_client.py | 4 +- .../extras/daemon/json_response_encoder.py | 4 +- lbry/lbry/extras/daemon/storage.py | 2 +- lbry/lbry/extras/system_info.py | 3 +- lbry/lbry/schema/attrs.py | 4 +- lbry/lbry/testcase.py | 7 +- lbry/lbry/wallet/account.py | 2 +- lbry/lbry/wallet/claim_proofs.py | 2 +- lbry/lbry/wallet/client/baseaccount.py | 13 +- lbry/lbry/wallet/client/basedatabase.py | 4 +- lbry/lbry/wallet/client/baseheader.py | 4 +- lbry/lbry/wallet/client/baseledger.py | 21 +- lbry/lbry/wallet/client/basemanager.py | 4 +- lbry/lbry/wallet/client/basenetwork.py | 9 +- lbry/lbry/wallet/client/basescript.py | 4 +- lbry/lbry/wallet/client/basetransaction.py | 17 +- lbry/lbry/wallet/client/bip32.py | 14 +- lbry/lbry/wallet/client/coinselection.py | 2 +- lbry/lbry/wallet/client/errors.py | 6 - lbry/lbry/wallet/client/hash.py | 211 +---- lbry/lbry/wallet/client/mnemonic.py | 4 +- lbry/lbry/wallet/client/util.py | 15 +- lbry/lbry/wallet/client/wallet.py | 4 +- lbry/lbry/wallet/database.py | 2 +- lbry/lbry/wallet/dewies.py | 2 +- lbry/lbry/wallet/header.py | 6 +- lbry/lbry/wallet/ledger.py | 4 +- lbry/lbry/wallet/manager.py | 6 +- lbry/lbry/wallet/network.py | 5 +- lbry/lbry/wallet/orchstr8/node.py | 12 +- lbry/lbry/wallet/orchstr8/service.py | 2 +- lbry/lbry/wallet/rpc/session.py | 2 +- lbry/lbry/wallet/script.py | 4 +- lbry/lbry/wallet/server/block_processor.py | 58 +- lbry/lbry/wallet/server/coin.py | 54 +- lbry/lbry/wallet/server/daemon.py | 197 +---- .../lbry/wallet/server/db/full_text_search.py | 2 +- lbry/lbry/wallet/server/db/reader.py | 2 +- lbry/lbry/wallet/server/db/writer.py | 6 +- lbry/lbry/wallet/server/env.py | 6 +- lbry/lbry/wallet/server/hash.py | 2 +- lbry/lbry/wallet/server/history.py | 6 +- lbry/lbry/wallet/server/leveldb.py | 12 +- lbry/lbry/wallet/server/mempool.py | 6 +- lbry/lbry/wallet/server/merkle.py | 2 +- lbry/lbry/wallet/server/peer.py | 4 +- lbry/lbry/wallet/server/peers.py | 8 +- lbry/lbry/wallet/server/script.py | 2 +- lbry/lbry/wallet/server/server.py | 6 +- lbry/lbry/wallet/server/session.py | 744 +++++++----------- lbry/lbry/wallet/server/storage.py | 2 +- lbry/lbry/wallet/server/text.py | 2 +- lbry/lbry/wallet/server/tx.py | 6 +- lbry/lbry/wallet/testcase.py | 21 +- lbry/lbry/wallet/transaction.py | 5 +- .../integration/test_account_commands.py | 4 - .../test_blockchain_reorganization.py | 2 +- lbry/tests/integration/test_claim_commands.py | 2 +- lbry/tests/integration/test_cli.py | 2 +- lbry/tests/integration/test_dht.py | 2 +- .../integration/test_exchange_rate_manager.py | 2 +- lbry/tests/integration/test_file_commands.py | 1 - .../test_internal_transaction_api.py | 2 +- lbry/tests/integration/test_network.py | 22 +- .../tests/integration/test_resolve_command.py | 3 +- lbry/tests/integration/test_sync.py | 4 +- lbry/tests/integration/test_transactions.py | 28 +- .../tests/integration/test_wallet_commands.py | 4 +- .../test_wallet_server_sessions.py | 8 +- lbry/tests/unit/wallet/server/test_sqldb.py | 2 +- lbry/tests/unit/wallet/test_account.py | 84 +- .../tests/unit/wallet/test_bcd_data_stream.py | 2 +- lbry/tests/unit/wallet/test_bip32.py | 10 +- lbry/tests/unit/wallet/test_claim_proofs.py | 2 +- lbry/tests/unit/wallet/test_coinselection.py | 10 +- lbry/tests/unit/wallet/test_database.py | 40 +- lbry/tests/unit/wallet/test_hash.py | 4 +- lbry/tests/unit/wallet/test_headers.py | 233 ++---- lbry/tests/unit/wallet/test_ledger.py | 46 +- lbry/tests/unit/wallet/test_mnemonic.py | 2 +- lbry/tests/unit/wallet/test_schema_signing.py | 6 +- lbry/tests/unit/wallet/test_script.py | 11 +- .../unit/wallet/test_stream_controller.py | 6 +- lbry/tests/unit/wallet/test_transaction.py | 34 +- lbry/tests/unit/wallet/test_utils.py | 4 +- lbry/tests/unit/wallet/test_wallet.py | 40 +- 98 files changed, 927 insertions(+), 1520 deletions(-) create mode 100644 lbry/lbry/constants.py create mode 100644 lbry/lbry/crypto/__init__.py create mode 100644 lbry/lbry/crypto/base58.py create mode 100644 lbry/lbry/crypto/crypt.py create mode 100644 lbry/lbry/crypto/hash.py create mode 100644 lbry/lbry/crypto/util.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af977c46b..3178bc67a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,32 +31,18 @@ test:lint: - make install tools - make lint -test:lbry-unit: +test:unit: stage: test script: - make install tools - cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit -test:lbry-integ: +test:integration: stage: test script: - pip install coverage tox-travis - cd lbry && tox -test:torba-unit: - stage: test - script: - - pip install coverage tox-travis - - cd torba/tests - - tox -e py37-unit - -test:torba-integ: - stage: test - script: - - pip install coverage tox-travis - - cd torba/tests - - tox -e py37-integration-torba.coin.bitcoinsegwit - test:json-api: stage: test script: diff --git a/.travis.yml b/.travis.yml index 109137037..edb2d3431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ jobs: script: make lint - stage: test - name: "LBRY Unit Tests" + name: "Unit Tests" install: - make install tools script: @@ -20,7 +20,7 @@ jobs: after_success: - coverage combine lbry/ - - name: "LBRY Integration Tests" + - name: "Integration Tests" install: - pip install coverage tox-travis - sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp @@ -28,19 +28,6 @@ jobs: after_success: - coverage combine lbry - - &torba-tests - name: "Torba Unit Tests" - env: TESTTYPE=unit - install: - - pip install coverage tox-travis - script: cd torba && tox - after_success: - - coverage combine torba/tests - - - <<: *torba-tests - name: "Torba Integration Tests" - env: TESTTYPE=integration - - name: "Run Examples" install: - make install tools diff --git a/Makefile b/Makefile index 58f284607..88e0cf77d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ install: --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ --global-option=build --global-option=--enable --global-option=fts5 - cd torba && pip install -e . cd lbry && pip install -e . tools: @@ -13,13 +12,11 @@ tools: pip install coverage astroid pylint lint: - cd lbry && pylint lbry - cd torba && pylint --rcfile=setup.cfg torba - cd torba && mypy --ignore-missing-imports torba + cd lbry && pylint --rcfile=setup.cfg lbry + cd lbry && mypy --ignore-missing-imports lbry test: cd lbry && tox - cd torba && tox idea: mkdir -p .idea diff --git a/lbry/lbry/__init__.py b/lbry/lbry/__init__.py index cdcae53b6..0b7dd6bc3 100644 --- a/lbry/lbry/__init__.py +++ b/lbry/lbry/__init__.py @@ -1,4 +1,4 @@ __name__ = "lbry" __version__ = "0.51.2" -version = tuple(__version__.split('.')) +version = tuple(map(int, __version__.split('.'))) diff --git a/lbry/lbry/conf.py b/lbry/lbry/conf.py index 1339be73e..e98f3262f 100644 --- a/lbry/lbry/conf.py +++ b/lbry/lbry/conf.py @@ -9,7 +9,7 @@ from contextlib import contextmanager from appdirs import user_data_dir, user_config_dir from lbry.error import InvalidCurrencyError from lbry.dht import constants -from torba.client.coinselection import STRATEGIES +from lbry.wallet.client.coinselection import STRATEGIES log = logging.getLogger(__name__) diff --git a/lbry/lbry/constants.py b/lbry/lbry/constants.py new file mode 100644 index 000000000..1c0072d2b --- /dev/null +++ b/lbry/lbry/constants.py @@ -0,0 +1,2 @@ +CENT = 1000000 +COIN = 100*CENT diff --git a/lbry/lbry/crypto/__init__.py b/lbry/lbry/crypto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbry/lbry/crypto/base58.py b/lbry/lbry/crypto/base58.py new file mode 100644 index 000000000..3c363aa92 --- /dev/null +++ b/lbry/lbry/crypto/base58.py @@ -0,0 +1,86 @@ +from lbry.crypto.hash import double_sha256 +from lbry.crypto.util import bytes_to_int, int_to_bytes + + +class Base58Error(Exception): + """ Exception used for Base58 errors. """ + + +class Base58: + """ Class providing base 58 functionality. """ + + chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + assert len(chars) == 58 + char_map = {c: n for n, c in enumerate(chars)} + + @classmethod + def char_value(cls, c): + val = cls.char_map.get(c) + if val is None: + raise Base58Error(f'invalid base 58 character "{c}"') + return val + + @classmethod + def decode(cls, txt): + """ Decodes txt into a big-endian bytearray. """ + if isinstance(txt, memoryview): + txt = str(txt) + + if isinstance(txt, bytes): + txt = txt.decode() + + if not isinstance(txt, str): + raise TypeError('a string is required') + + if not txt: + raise Base58Error('string cannot be empty') + + value = 0 + for c in txt: + value = value * 58 + cls.char_value(c) + + result = int_to_bytes(value) + + # Prepend leading zero bytes if necessary + count = 0 + for c in txt: + if c != '1': + break + count += 1 + if count: + result = bytes((0,)) * count + result + + return result + + @classmethod + def encode(cls, be_bytes): + """Converts a big-endian bytearray into a base58 string.""" + value = bytes_to_int(be_bytes) + + txt = '' + while value: + value, mod = divmod(value, 58) + txt += cls.chars[mod] + + for byte in be_bytes: + if byte != 0: + break + txt += '1' + + return txt[::-1] + + @classmethod + def decode_check(cls, txt, hash_fn=double_sha256): + """ Decodes a Base58Check-encoded string to a payload. The version prefixes it. """ + be_bytes = cls.decode(txt) + result, check = be_bytes[:-4], be_bytes[-4:] + if check != hash_fn(result)[:4]: + raise Base58Error(f'invalid base 58 checksum for {txt}') + return result + + @classmethod + def encode_check(cls, payload, hash_fn=double_sha256): + """ Encodes a payload bytearray (which includes the version byte(s)) + into a Base58Check string.""" + be_bytes = payload + hash_fn(payload)[:4] + return cls.encode(be_bytes) diff --git a/lbry/lbry/crypto/crypt.py b/lbry/lbry/crypto/crypt.py new file mode 100644 index 000000000..382c3b4d4 --- /dev/null +++ b/lbry/lbry/crypto/crypt.py @@ -0,0 +1,69 @@ +import os +import base64 +import typing +from cryptography.hazmat.primitives.kdf.scrypt import Scrypt +from cryptography.hazmat.primitives.ciphers import Cipher, modes +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.padding import PKCS7 +from cryptography.hazmat.backends import default_backend + +from lbry.error import InvalidPasswordError +from lbry.crypto.hash import double_sha256 + + +def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str: + if init_vector is not None: + assert len(init_vector) == 16 + else: + init_vector = os.urandom(16) + key = double_sha256(secret.encode()) + encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() + padder = PKCS7(AES.block_size).padder() + padded_data = padder.update(value.encode()) + padder.finalize() + encrypted_data = encryptor.update(padded_data) + encryptor.finalize() + return base64.b64encode(init_vector + encrypted_data).decode() + + +def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]: + try: + data = base64.b64decode(value.encode()) + key = double_sha256(secret.encode()) + init_vector, data = data[:16], data[16:] + decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() + unpadder = PKCS7(AES.block_size).unpadder() + result = unpadder.update(decryptor.update(data)) + unpadder.finalize() + return result.decode(), init_vector + except ValueError as e: + if e.args[0] == 'Invalid padding bytes.': + raise InvalidPasswordError() + raise + + +def better_aes_encrypt(secret: str, value: bytes) -> bytes: + init_vector = os.urandom(16) + key = scrypt(secret.encode(), salt=init_vector) + encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() + padder = PKCS7(AES.block_size).padder() + padded_data = padder.update(value) + padder.finalize() + encrypted_data = encryptor.update(padded_data) + encryptor.finalize() + return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data) + + +def better_aes_decrypt(secret: str, value: bytes) -> bytes: + try: + data = base64.b64decode(value) + _, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4) + init_vector, data = data[:16], data[16:] + key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p)) + decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() + unpadder = PKCS7(AES.block_size).unpadder() + return unpadder.update(decryptor.update(data)) + unpadder.finalize() + except ValueError as e: + if e.args[0] == 'Invalid padding bytes.': + raise InvalidPasswordError() + raise + + +def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1): + kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend()) + return kdf.derive(passphrase) diff --git a/lbry/lbry/crypto/hash.py b/lbry/lbry/crypto/hash.py new file mode 100644 index 000000000..4f2eb728c --- /dev/null +++ b/lbry/lbry/crypto/hash.py @@ -0,0 +1,47 @@ +import hashlib +import hmac +from binascii import hexlify, unhexlify + + +def sha256(x): + """ Simple wrapper of hashlib sha256. """ + return hashlib.sha256(x).digest() + + +def sha512(x): + """ Simple wrapper of hashlib sha512. """ + return hashlib.sha512(x).digest() + + +def ripemd160(x): + """ Simple wrapper of hashlib ripemd160. """ + h = hashlib.new('ripemd160') + h.update(x) + return h.digest() + + +def double_sha256(x): + """ SHA-256 of SHA-256, as used extensively in bitcoin. """ + return sha256(sha256(x)) + + +def hmac_sha512(key, msg): + """ Use SHA-512 to provide an HMAC. """ + return hmac.new(key, msg, hashlib.sha512).digest() + + +def hash160(x): + """ RIPEMD-160 of SHA-256. + Used to make bitcoin addresses from pubkeys. """ + return ripemd160(sha256(x)) + + +def hash_to_hex_str(x): + """ Convert a big-endian binary hash to displayed hex string. + Display form of a binary hash is reversed and converted to hex. """ + return hexlify(reversed(x)) + + +def hex_str_to_hash(x): + """ Convert a displayed hex string to a binary hash. """ + return reversed(unhexlify(x)) diff --git a/lbry/lbry/crypto/util.py b/lbry/lbry/crypto/util.py new file mode 100644 index 000000000..bc641faa5 --- /dev/null +++ b/lbry/lbry/crypto/util.py @@ -0,0 +1,13 @@ +from binascii import unhexlify, hexlify + + +def bytes_to_int(be_bytes): + """ Interprets a big-endian sequence of bytes as an integer. """ + return int(hexlify(be_bytes), 16) + + +def int_to_bytes(value): + """ Converts an integer to a big-endian sequence of bytes. """ + length = (value.bit_length() + 7) // 8 + s = '%x' % value + return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2)) diff --git a/lbry/lbry/extras/daemon/Daemon.py b/lbry/lbry/extras/daemon/Daemon.py index eb6666ae1..6707c5746 100644 --- a/lbry/lbry/extras/daemon/Daemon.py +++ b/lbry/lbry/extras/daemon/Daemon.py @@ -17,8 +17,8 @@ from traceback import format_exc from aiohttp import web from functools import wraps, partial from google.protobuf.message import DecodeError -from torba.client.wallet import Wallet, ENCRYPT_ON_DISK -from torba.client.baseaccount import SingleKey, HierarchicalDeterministic +from lbry.wallet.client.wallet import Wallet, ENCRYPT_ON_DISK +from lbry.wallet.client.baseaccount import SingleKey, HierarchicalDeterministic from lbry import utils from lbry.conf import Config, Setting, NOT_SET diff --git a/lbry/lbry/extras/daemon/comment_client.py b/lbry/lbry/extras/daemon/comment_client.py index 26b2b3b18..5556d130d 100644 --- a/lbry/lbry/extras/daemon/comment_client.py +++ b/lbry/lbry/extras/daemon/comment_client.py @@ -3,9 +3,9 @@ import time import hashlib import binascii -from lbry import utils import ecdsa -from torba.client.hash import sha256 +from lbry import utils +from lbry.crypto.hash import sha256 from lbry.wallet.transaction import Output log = logging.getLogger(__name__) diff --git a/lbry/lbry/extras/daemon/json_response_encoder.py b/lbry/lbry/extras/daemon/json_response_encoder.py index 81054d9c7..c6b944acd 100644 --- a/lbry/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/lbry/extras/daemon/json_response_encoder.py @@ -6,8 +6,8 @@ from json import JSONEncoder from google.protobuf.message import DecodeError -from torba.client.wallet import Wallet -from torba.client.bip32 import PubKey +from lbry.wallet.client.wallet import Wallet +from lbry.wallet.client.bip32 import PubKey from lbry.schema.claim import Claim from lbry.wallet.ledger import MainNetLedger, Account from lbry.wallet.transaction import Transaction, Output diff --git a/lbry/lbry/extras/daemon/storage.py b/lbry/lbry/extras/daemon/storage.py index 0c8eeb02f..86be3b67f 100644 --- a/lbry/lbry/extras/daemon/storage.py +++ b/lbry/lbry/extras/daemon/storage.py @@ -5,7 +5,7 @@ import typing import asyncio import binascii import time -from torba.client.basedatabase import SQLiteMixin +from lbry.wallet.client.basedatabase import SQLiteMixin from lbry.conf import Config from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies from lbry.wallet.transaction import Transaction diff --git a/lbry/lbry/extras/system_info.py b/lbry/lbry/extras/system_info.py index 9b36cde17..704113b44 100644 --- a/lbry/lbry/extras/system_info.py +++ b/lbry/lbry/extras/system_info.py @@ -3,7 +3,6 @@ import os import logging.handlers from lbry import build_type, __version__ as lbrynet_version -from torba import __version__ as torba_version log = logging.getLogger(__name__) @@ -19,7 +18,7 @@ def get_platform() -> dict: "os_release": platform.release(), "os_system": os_system, "lbrynet_version": lbrynet_version, - "torba_version": torba_version, + "version": lbrynet_version, "build": build_type.BUILD, # CI server sets this during build step } if p["os_system"] == "Linux": diff --git a/lbry/lbry/schema/attrs.py b/lbry/lbry/schema/attrs.py index eed77505f..69f81f451 100644 --- a/lbry/lbry/schema/attrs.py +++ b/lbry/lbry/schema/attrs.py @@ -8,8 +8,8 @@ from decimal import Decimal, ROUND_UP from binascii import hexlify, unhexlify from google.protobuf.json_format import MessageToDict -from torba.client.hash import Base58 -from torba.client.constants import COIN +from lbry.crypto.base58 import Base58 +from lbry.constants import COIN from lbry.schema.mime_types import guess_media_type from lbry.schema.base import Metadata, BaseMessageList diff --git a/lbry/lbry/testcase.py b/lbry/lbry/testcase.py index 0d6355a35..01e6604eb 100644 --- a/lbry/lbry/testcase.py +++ b/lbry/lbry/testcase.py @@ -7,13 +7,10 @@ from time import time from binascii import unhexlify from functools import partial -from torba.testcase import IntegrationTestCase, WalletNode - -import lbry.wallet +from lbry.wallet.testcase import IntegrationTestCase, WalletNode from lbry.conf import Config from lbry.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty -from lbry.wallet import LbryWalletManager from lbry.wallet.account import Account from lbry.wallet.transaction import Transaction from lbry.extras.daemon.Components import Component, WalletComponent @@ -73,8 +70,6 @@ class ExchangeRateManagerComponent(Component): class CommandTestCase(IntegrationTestCase): - LEDGER = lbry.wallet - MANAGER = LbryWalletManager VERBOSITY = logging.WARN blob_lru_cache_size = 0 diff --git a/lbry/lbry/wallet/account.py b/lbry/lbry/wallet/account.py index 5ff6e40b6..4bed8eaa2 100644 --- a/lbry/lbry/wallet/account.py +++ b/lbry/lbry/wallet/account.py @@ -7,7 +7,7 @@ from string import hexdigits import ecdsa from lbry.wallet.constants import CLAIM_TYPES, TXO_TYPES -from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic +from lbry.wallet.client.baseaccount import BaseAccount, HierarchicalDeterministic log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/claim_proofs.py b/lbry/lbry/wallet/claim_proofs.py index 263807a50..5ebd4f216 100644 --- a/lbry/lbry/wallet/claim_proofs.py +++ b/lbry/lbry/wallet/claim_proofs.py @@ -1,6 +1,6 @@ import struct import binascii -from torba.client.hash import double_sha256 +from lbry.wallet.client.hash import double_sha256 class InvalidProofError(Exception): diff --git a/lbry/lbry/wallet/client/baseaccount.py b/lbry/lbry/wallet/client/baseaccount.py index 1abf4e7c3..26c3451b7 100644 --- a/lbry/lbry/wallet/client/baseaccount.py +++ b/lbry/lbry/wallet/client/baseaccount.py @@ -6,14 +6,15 @@ import random import typing from typing import Dict, Tuple, Type, Optional, Any, List -from torba.client.mnemonic import Mnemonic -from torba.client.bip32 import PrivateKey, PubKey, from_extended_key_string -from torba.client.hash import aes_encrypt, aes_decrypt, sha256 -from torba.client.constants import COIN -from torba.client.errors import InvalidPasswordError +from lbry.crypto.hash import sha256 +from lbry.crypto.crypt import aes_encrypt, aes_decrypt +from lbry.wallet.client.bip32 import PrivateKey, PubKey, from_extended_key_string +from lbry.wallet.client.mnemonic import Mnemonic +from lbry.wallet.client.constants import COIN +from lbry.error import InvalidPasswordError if typing.TYPE_CHECKING: - from torba.client import baseledger, wallet as basewallet + from lbry.wallet.client import baseledger, wallet as basewallet class AddressManager: diff --git a/lbry/lbry/wallet/client/basedatabase.py b/lbry/lbry/wallet/client/basedatabase.py index 9355970b2..7b607c47b 100644 --- a/lbry/lbry/wallet/client/basedatabase.py +++ b/lbry/lbry/wallet/client/basedatabase.py @@ -7,8 +7,8 @@ from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, import sqlite3 -from torba.client.basetransaction import BaseTransaction, TXRefImmutable -from torba.client.bip32 import PubKey +from lbry.wallet.client.basetransaction import BaseTransaction, TXRefImmutable +from lbry.wallet.client.bip32 import PubKey log = logging.getLogger(__name__) sqlite3.enable_callback_tracebacks(True) diff --git a/lbry/lbry/wallet/client/baseheader.py b/lbry/lbry/wallet/client/baseheader.py index 338c5ec8a..6450804e6 100644 --- a/lbry/lbry/wallet/client/baseheader.py +++ b/lbry/lbry/wallet/client/baseheader.py @@ -7,8 +7,8 @@ from io import BytesIO from typing import Optional, Iterator, Tuple from binascii import hexlify -from torba.client.util import ArithUint256 -from torba.client.hash import double_sha256 +from lbry.wallet.client.util import ArithUint256 +from lbry.crypto.hash import double_sha256 log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/client/baseledger.py b/lbry/lbry/wallet/client/baseledger.py index 68b5a2562..1f8e993e0 100644 --- a/lbry/lbry/wallet/client/baseledger.py +++ b/lbry/lbry/wallet/client/baseledger.py @@ -12,16 +12,17 @@ from operator import itemgetter from collections import namedtuple import pylru -from torba.client.basetransaction import BaseTransaction -from torba.tasks import TaskGroup -from torba.client import baseaccount, basenetwork, basetransaction -from torba.client.basedatabase import BaseDatabase -from torba.client.baseheader import BaseHeaders -from torba.client.coinselection import CoinSelector -from torba.client.constants import COIN, NULL_HASH32 -from torba.stream import StreamController -from torba.client.hash import hash160, double_sha256, sha256, Base58 -from torba.client.bip32 import PubKey, PrivateKey +from lbry.wallet.client.basetransaction import BaseTransaction +from lbry.wallet.tasks import TaskGroup +from lbry.wallet.client import baseaccount, basenetwork, basetransaction +from lbry.wallet.client.basedatabase import BaseDatabase +from lbry.wallet.client.baseheader import BaseHeaders +from lbry.wallet.client.coinselection import CoinSelector +from lbry.wallet.client.constants import COIN, NULL_HASH32 +from lbry.wallet.stream import StreamController +from lbry.crypto.hash import hash160, double_sha256, sha256 +from lbry.crypto.base58 import Base58 +from lbry.wallet.client.bip32 import PubKey, PrivateKey log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/client/basemanager.py b/lbry/lbry/wallet/client/basemanager.py index 22ad2c56d..d4a4a6ef7 100644 --- a/lbry/lbry/wallet/client/basemanager.py +++ b/lbry/lbry/wallet/client/basemanager.py @@ -2,8 +2,8 @@ import asyncio import logging from typing import Type, MutableSequence, MutableMapping, Optional -from torba.client.baseledger import BaseLedger, LedgerRegistry -from torba.client.wallet import Wallet, WalletStorage +from lbry.wallet.client.baseledger import BaseLedger, LedgerRegistry +from lbry.wallet.client.wallet import Wallet, WalletStorage log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/client/basenetwork.py b/lbry/lbry/wallet/client/basenetwork.py index 5a45dc848..7fa4d0251 100644 --- a/lbry/lbry/wallet/client/basenetwork.py +++ b/lbry/lbry/wallet/client/basenetwork.py @@ -4,10 +4,9 @@ from operator import itemgetter from typing import Dict, Optional, Tuple from time import perf_counter -from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError - -from torba import __version__ -from torba.stream import StreamController +import lbry +from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError +from lbry.wallet.stream import StreamController log = logging.getLogger(__name__) @@ -115,7 +114,7 @@ class ClientSession(BaseClientSession): async def ensure_server_version(self, required=None, timeout=3): required = required or self.network.PROTOCOL_VERSION return await asyncio.wait_for( - self.send_request('server.version', [__version__, required]), timeout=timeout + self.send_request('server.version', [lbry.__version__, required]), timeout=timeout ) async def create_connection(self, timeout=6): diff --git a/lbry/lbry/wallet/client/basescript.py b/lbry/lbry/wallet/client/basescript.py index 0f4f61748..6299bd909 100644 --- a/lbry/lbry/wallet/client/basescript.py +++ b/lbry/lbry/wallet/client/basescript.py @@ -3,8 +3,8 @@ from binascii import hexlify from collections import namedtuple from typing import List -from torba.client.bcd_data_stream import BCDataStream -from torba.client.util import subclass_tuple +from lbry.wallet.client.bcd_data_stream import BCDataStream +from lbry.wallet.client.util import subclass_tuple # bitcoin opcodes OP_0 = 0x00 diff --git a/lbry/lbry/wallet/client/basetransaction.py b/lbry/lbry/wallet/client/basetransaction.py index 8c10bb7f8..7b38b2e52 100644 --- a/lbry/lbry/wallet/client/basetransaction.py +++ b/lbry/lbry/wallet/client/basetransaction.py @@ -3,16 +3,17 @@ import typing from typing import List, Iterable, Optional, Tuple from binascii import hexlify -from torba.client.basescript import BaseInputScript, BaseOutputScript -from torba.client.baseaccount import BaseAccount -from torba.client.constants import COIN, NULL_HASH32 -from torba.client.bcd_data_stream import BCDataStream -from torba.client.hash import sha256, TXRef, TXRefImmutable -from torba.client.util import ReadOnlyList -from torba.client.errors import InsufficientFundsError +from lbry.crypto.hash import sha256 +from lbry.wallet.client.basescript import BaseInputScript, BaseOutputScript +from lbry.wallet.client.baseaccount import BaseAccount +from lbry.wallet.client.constants import COIN, NULL_HASH32 +from lbry.wallet.client.bcd_data_stream import BCDataStream +from lbry.wallet.client.hash import TXRef, TXRefImmutable +from lbry.wallet.client.util import ReadOnlyList +from lbry.wallet.client.errors import InsufficientFundsError if typing.TYPE_CHECKING: - from torba.client import baseledger, wallet as basewallet + from lbry.wallet.client import baseledger, wallet as basewallet log = logging.getLogger() diff --git a/lbry/lbry/wallet/client/bip32.py b/lbry/lbry/wallet/client/bip32.py index 31490bbb6..fc9a590d1 100644 --- a/lbry/lbry/wallet/client/bip32.py +++ b/lbry/lbry/wallet/client/bip32.py @@ -1,16 +1,8 @@ -# Copyright (c) 2017, Neil Booth -# Copyright (c) 2018, LBRY Inc. -# -# All rights reserved. -# -# See the file "LICENCE" for information about the copyright -# and warranty status of this software. - -""" Logic for BIP32 Hierarchical Key Derivation. """ from coincurve import PublicKey, PrivateKey as _PrivateKey -from torba.client.hash import Base58, hmac_sha512, hash160, double_sha256 -from torba.client.util import cachedproperty +from lbry.crypto.hash import hmac_sha512, hash160, double_sha256 +from lbry.crypto.base58 import Base58 +from lbry.wallet.client.util import cachedproperty class DerivationError(Exception): diff --git a/lbry/lbry/wallet/client/coinselection.py b/lbry/lbry/wallet/client/coinselection.py index 87656ab3c..253d31a21 100644 --- a/lbry/lbry/wallet/client/coinselection.py +++ b/lbry/lbry/wallet/client/coinselection.py @@ -1,7 +1,7 @@ from random import Random from typing import List -from torba.client import basetransaction +from lbry.wallet.client import basetransaction MAXIMUM_TRIES = 100000 diff --git a/lbry/lbry/wallet/client/errors.py b/lbry/lbry/wallet/client/errors.py index 0fbed62c0..cd290cf05 100644 --- a/lbry/lbry/wallet/client/errors.py +++ b/lbry/lbry/wallet/client/errors.py @@ -1,8 +1,2 @@ -class InvalidPasswordError(Exception): - - def __init__(self): - super().__init__("Password is invalid.") - - class InsufficientFundsError(Exception): pass diff --git a/lbry/lbry/wallet/client/hash.py b/lbry/lbry/wallet/client/hash.py index eb23647c7..4fdbf306a 100644 --- a/lbry/lbry/wallet/client/hash.py +++ b/lbry/lbry/wallet/client/hash.py @@ -1,28 +1,5 @@ -# Copyright (c) 2016-2017, Neil Booth -# Copyright (c) 2018, LBRY Inc. -# -# All rights reserved. -# -# See the file "LICENCE" for information about the copyright -# and warranty status of this software. - -""" Cryptography hash functions and related classes. """ - -import os -import base64 -import hashlib -import hmac -import typing from binascii import hexlify, unhexlify -from cryptography.hazmat.primitives.kdf.scrypt import Scrypt -from cryptography.hazmat.primitives.ciphers import Cipher, modes -from cryptography.hazmat.primitives.ciphers.algorithms import AES -from cryptography.hazmat.primitives.padding import PKCS7 -from cryptography.hazmat.backends import default_backend - -from torba.client.util import bytes_to_int, int_to_bytes -from torba.client.constants import NULL_HASH32 -from torba.client.errors import InvalidPasswordError +from lbry.wallet.client.constants import NULL_HASH32 class TXRef: @@ -77,189 +54,3 @@ class TXRefImmutable(TXRef): @property def height(self): return self._height - - -def sha256(x): - """ Simple wrapper of hashlib sha256. """ - return hashlib.sha256(x).digest() - - -def sha512(x): - """ Simple wrapper of hashlib sha512. """ - return hashlib.sha512(x).digest() - - -def ripemd160(x): - """ Simple wrapper of hashlib ripemd160. """ - h = hashlib.new('ripemd160') - h.update(x) - return h.digest() - - -def double_sha256(x): - """ SHA-256 of SHA-256, as used extensively in bitcoin. """ - return sha256(sha256(x)) - - -def hmac_sha512(key, msg): - """ Use SHA-512 to provide an HMAC. """ - return hmac.new(key, msg, hashlib.sha512).digest() - - -def hash160(x): - """ RIPEMD-160 of SHA-256. - Used to make bitcoin addresses from pubkeys. """ - return ripemd160(sha256(x)) - - -def hash_to_hex_str(x): - """ Convert a big-endian binary hash to displayed hex string. - Display form of a binary hash is reversed and converted to hex. """ - return hexlify(reversed(x)) - - -def hex_str_to_hash(x): - """ Convert a displayed hex string to a binary hash. """ - return reversed(unhexlify(x)) - - -def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str: - if init_vector is not None: - assert len(init_vector) == 16 - else: - init_vector = os.urandom(16) - key = double_sha256(secret.encode()) - encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() - padder = PKCS7(AES.block_size).padder() - padded_data = padder.update(value.encode()) + padder.finalize() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - return base64.b64encode(init_vector + encrypted_data).decode() - - -def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]: - try: - data = base64.b64decode(value.encode()) - key = double_sha256(secret.encode()) - init_vector, data = data[:16], data[16:] - decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() - unpadder = PKCS7(AES.block_size).unpadder() - result = unpadder.update(decryptor.update(data)) + unpadder.finalize() - return result.decode(), init_vector - except ValueError as e: - if e.args[0] == 'Invalid padding bytes.': - raise InvalidPasswordError() - raise - - -def better_aes_encrypt(secret: str, value: bytes) -> bytes: - init_vector = os.urandom(16) - key = scrypt(secret.encode(), salt=init_vector) - encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor() - padder = PKCS7(AES.block_size).padder() - padded_data = padder.update(value) + padder.finalize() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data) - - -def better_aes_decrypt(secret: str, value: bytes) -> bytes: - try: - data = base64.b64decode(value) - _, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4) - init_vector, data = data[:16], data[16:] - key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p)) - decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor() - unpadder = PKCS7(AES.block_size).unpadder() - return unpadder.update(decryptor.update(data)) + unpadder.finalize() - except ValueError as e: - if e.args[0] == 'Invalid padding bytes.': - raise InvalidPasswordError() - raise - - -def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1): - kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend()) - return kdf.derive(passphrase) - - -class Base58Error(Exception): - """ Exception used for Base58 errors. """ - - -class Base58: - """ Class providing base 58 functionality. """ - - chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - assert len(chars) == 58 - char_map = {c: n for n, c in enumerate(chars)} - - @classmethod - def char_value(cls, c): - val = cls.char_map.get(c) - if val is None: - raise Base58Error(f'invalid base 58 character "{c}"') - return val - - @classmethod - def decode(cls, txt): - """ Decodes txt into a big-endian bytearray. """ - if isinstance(txt, memoryview): - txt = str(txt) - - if isinstance(txt, bytes): - txt = txt.decode() - - if not isinstance(txt, str): - raise TypeError('a string is required') - - if not txt: - raise Base58Error('string cannot be empty') - - value = 0 - for c in txt: - value = value * 58 + cls.char_value(c) - - result = int_to_bytes(value) - - # Prepend leading zero bytes if necessary - count = 0 - for c in txt: - if c != '1': - break - count += 1 - if count: - result = bytes((0,)) * count + result - - return result - - @classmethod - def encode(cls, be_bytes): - """Converts a big-endian bytearray into a base58 string.""" - value = bytes_to_int(be_bytes) - - txt = '' - while value: - value, mod = divmod(value, 58) - txt += cls.chars[mod] - - for byte in be_bytes: - if byte != 0: - break - txt += '1' - - return txt[::-1] - - @classmethod - def decode_check(cls, txt, hash_fn=double_sha256): - """ Decodes a Base58Check-encoded string to a payload. The version prefixes it. """ - be_bytes = cls.decode(txt) - result, check = be_bytes[:-4], be_bytes[-4:] - if check != hash_fn(result)[:4]: - raise Base58Error(f'invalid base 58 checksum for {txt}') - return result - - @classmethod - def encode_check(cls, payload, hash_fn=double_sha256): - """ Encodes a payload bytearray (which includes the version byte(s)) - into a Base58Check string.""" - be_bytes = payload + hash_fn(payload)[:4] - return cls.encode(be_bytes) diff --git a/lbry/lbry/wallet/client/mnemonic.py b/lbry/lbry/wallet/client/mnemonic.py index c78bbac74..1fcc9afa5 100644 --- a/lbry/lbry/wallet/client/mnemonic.py +++ b/lbry/lbry/wallet/client/mnemonic.py @@ -12,8 +12,8 @@ from secrets import randbelow import pbkdf2 -from torba.client.hash import hmac_sha512 -from torba.client.words import english +from lbry.crypto.hash import hmac_sha512 +from lbry.wallet.client.words import english # The hash of the mnemonic seed must begin with this SEED_PREFIX = b'01' # Standard wallet diff --git a/lbry/lbry/wallet/client/util.py b/lbry/lbry/wallet/client/util.py index f57963a89..f30b4f532 100644 --- a/lbry/lbry/wallet/client/util.py +++ b/lbry/lbry/wallet/client/util.py @@ -1,7 +1,6 @@ import re -from binascii import unhexlify, hexlify from typing import TypeVar, Sequence, Optional -from torba.client.constants import COIN +from lbry.wallet.client.constants import COIN def coins_to_satoshis(coins): @@ -53,18 +52,6 @@ class cachedproperty: return value -def bytes_to_int(be_bytes): - """ Interprets a big-endian sequence of bytes as an integer. """ - return int(hexlify(be_bytes), 16) - - -def int_to_bytes(value): - """ Converts an integer to a big-endian sequence of bytes. """ - length = (value.bit_length() + 7) // 8 - s = '%x' % value - return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2)) - - class ArithUint256: # https://github.com/bitcoin/bitcoin/blob/master/src/arith_uint256.cpp diff --git a/lbry/lbry/wallet/client/wallet.py b/lbry/lbry/wallet/client/wallet.py index cb23d69e2..18e9a72d0 100644 --- a/lbry/lbry/wallet/client/wallet.py +++ b/lbry/lbry/wallet/client/wallet.py @@ -9,10 +9,10 @@ from typing import List, Sequence, MutableSequence, Optional from collections import UserDict from hashlib import sha256 from operator import attrgetter -from torba.client.hash import better_aes_encrypt, better_aes_decrypt +from lbry.crypto.crypt import better_aes_encrypt, better_aes_decrypt if typing.TYPE_CHECKING: - from torba.client import basemanager, baseaccount, baseledger + from lbry.wallet.client import basemanager, baseaccount, baseledger log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/database.py b/lbry/lbry/wallet/database.py index 456a9293b..948e063bb 100644 --- a/lbry/lbry/wallet/database.py +++ b/lbry/lbry/wallet/database.py @@ -1,6 +1,6 @@ from typing import List -from torba.client.basedatabase import BaseDatabase +from lbry.wallet.client.basedatabase import BaseDatabase from lbry.wallet.transaction import Output from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES diff --git a/lbry/lbry/wallet/dewies.py b/lbry/lbry/wallet/dewies.py index 8d27fd728..9562d0ee5 100644 --- a/lbry/lbry/wallet/dewies.py +++ b/lbry/lbry/wallet/dewies.py @@ -1,5 +1,5 @@ import textwrap -from torba.client.util import coins_to_satoshis, satoshis_to_coins +from lbry.wallet.client.util import coins_to_satoshis, satoshis_to_coins def lbc_to_dewies(lbc: str) -> int: diff --git a/lbry/lbry/wallet/header.py b/lbry/lbry/wallet/header.py index 162edd98e..bba22db3f 100644 --- a/lbry/lbry/wallet/header.py +++ b/lbry/lbry/wallet/header.py @@ -2,9 +2,9 @@ import struct from typing import Optional from binascii import hexlify, unhexlify -from torba.client.baseheader import BaseHeaders -from torba.client.util import ArithUint256 -from torba.client.hash import sha512, double_sha256, ripemd160 +from lbry.crypto.hash import sha512, double_sha256, ripemd160 +from lbry.wallet.client.baseheader import BaseHeaders +from lbry.wallet.client.util import ArithUint256 class Headers(BaseHeaders): diff --git a/lbry/lbry/wallet/ledger.py b/lbry/lbry/wallet/ledger.py index 85ee0ec0d..1e11d2b16 100644 --- a/lbry/lbry/wallet/ledger.py +++ b/lbry/lbry/wallet/ledger.py @@ -6,8 +6,8 @@ from typing import Tuple, List from datetime import datetime import pylru -from torba.client.baseledger import BaseLedger, TransactionEvent -from torba.client.baseaccount import SingleKey +from lbry.wallet.client.baseledger import BaseLedger, TransactionEvent +from lbry.wallet.client.baseaccount import SingleKey from lbry.schema.result import Outputs from lbry.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc diff --git a/lbry/lbry/wallet/manager.py b/lbry/lbry/wallet/manager.py index 8b454d38e..883a7d9a9 100644 --- a/lbry/lbry/wallet/manager.py +++ b/lbry/lbry/wallet/manager.py @@ -6,9 +6,9 @@ from binascii import unhexlify from typing import Optional, List from decimal import Decimal -from torba.client.basemanager import BaseWalletManager -from torba.client.wallet import ENCRYPT_ON_DISK -from torba.rpc.jsonrpc import CodeMessageError +from lbry.wallet.client.basemanager import BaseWalletManager +from lbry.wallet.client.wallet import ENCRYPT_ON_DISK +from lbry.wallet.rpc.jsonrpc import CodeMessageError from lbry.error import KeyFeeAboveMaxAllowedError from lbry.wallet.dewies import dewies_to_lbc diff --git a/lbry/lbry/wallet/network.py b/lbry/lbry/wallet/network.py index 416478a8d..170885e4f 100644 --- a/lbry/lbry/wallet/network.py +++ b/lbry/lbry/wallet/network.py @@ -1,8 +1,9 @@ -from torba.client.basenetwork import BaseNetwork +import lbry +from lbry.wallet.client.basenetwork import BaseNetwork class Network(BaseNetwork): - PROTOCOL_VERSION = '2.0' + PROTOCOL_VERSION = lbry.__version__ def get_claims_by_ids(self, claim_ids): return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids) diff --git a/lbry/lbry/wallet/orchstr8/node.py b/lbry/lbry/wallet/orchstr8/node.py index 3cb963669..c31d7c27c 100644 --- a/lbry/lbry/wallet/orchstr8/node.py +++ b/lbry/lbry/wallet/orchstr8/node.py @@ -12,12 +12,12 @@ from binascii import hexlify from typing import Type, Optional import urllib.request -from torba.server.server import Server -from torba.server.env import Env -from torba.client.wallet import Wallet -from torba.client.baseledger import BaseLedger, BlockHeightEvent -from torba.client.basemanager import BaseWalletManager -from torba.client.baseaccount import BaseAccount +from lbry.wallet.server.server import Server +from lbry.wallet.server.env import Env +from lbry.wallet.client.wallet import Wallet +from lbry.wallet.client.baseledger import BaseLedger, BlockHeightEvent +from lbry.wallet.client.basemanager import BaseWalletManager +from lbry.wallet.client.baseaccount import BaseAccount log = logging.getLogger(__name__) diff --git a/lbry/lbry/wallet/orchstr8/service.py b/lbry/lbry/wallet/orchstr8/service.py index 3c368c457..39aa571c8 100644 --- a/lbry/lbry/wallet/orchstr8/service.py +++ b/lbry/lbry/wallet/orchstr8/service.py @@ -3,7 +3,7 @@ import logging from aiohttp.web import Application, WebSocketResponse, json_response from aiohttp.http_websocket import WSMsgType, WSCloseCode -from torba.client.util import satoshis_to_coins +from lbry.wallet.client.util import satoshis_to_coins from .node import Conductor, set_logging diff --git a/lbry/lbry/wallet/rpc/session.py b/lbry/lbry/wallet/rpc/session.py index 1b93633f0..678c265cd 100644 --- a/lbry/lbry/wallet/rpc/session.py +++ b/lbry/lbry/wallet/rpc/session.py @@ -34,7 +34,7 @@ import logging import time from contextlib import suppress -from torba.tasks import TaskGroup +from lbry.wallet.tasks import TaskGroup from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification from .jsonrpc import RPCError, ProtocolError diff --git a/lbry/lbry/wallet/script.py b/lbry/lbry/wallet/script.py index 02f1bfda0..4c0ca9509 100644 --- a/lbry/lbry/wallet/script.py +++ b/lbry/lbry/wallet/script.py @@ -1,5 +1,5 @@ -from torba.client.basescript import BaseInputScript, BaseOutputScript, Template -from torba.client.basescript import PUSH_SINGLE, PUSH_INTEGER, OP_DROP, OP_2DROP, PUSH_SUBSCRIPT, OP_VERIFY +from lbry.wallet.client.basescript import BaseInputScript, BaseOutputScript, Template +from lbry.wallet.client.basescript import PUSH_SINGLE, PUSH_INTEGER, OP_DROP, OP_2DROP, PUSH_SUBSCRIPT, OP_VERIFY class InputScript(BaseInputScript): diff --git a/lbry/lbry/wallet/server/block_processor.py b/lbry/lbry/wallet/server/block_processor.py index 703dcbf25..510ec882a 100644 --- a/lbry/lbry/wallet/server/block_processor.py +++ b/lbry/lbry/wallet/server/block_processor.py @@ -1,16 +1,14 @@ import time - -from lbry.schema.claim import Claim -from lbry.wallet.server.db.writer import SQLDB - import asyncio from struct import pack, unpack -import torba -from torba.server.daemon import DaemonError -from torba.server.hash import hash_to_hex_str, HASHX_LEN -from torba.server.util import chunks, class_logger -from torba.server.db import FlushData +import lbry +from lbry.schema.claim import Claim +from lbry.wallet.server.db.writer import SQLDB +from lbry.wallet.server.daemon import DaemonError +from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN +from lbry.wallet.server.util import chunks, class_logger +from lbry.wallet.server.leveldb import FlushData class Prefetcher: @@ -612,7 +610,7 @@ class BlockProcessor: self.db.first_sync = False await self.flush(True) if first_sync: - self.logger.info(f'{torba.__version__} synced to ' + self.logger.info(f'{lbry.__version__} synced to ' f'height {self.height:,d}') # Reopen for serving await self.db.open_for_serving() @@ -667,46 +665,6 @@ class BlockProcessor: return False -class DecredBlockProcessor(BlockProcessor): - async def calc_reorg_range(self, count): - start, count = await super().calc_reorg_range(count) - if start > 0: - # A reorg in Decred can invalidate the previous block - start -= 1 - count += 1 - return start, count - - -class NamecoinBlockProcessor(BlockProcessor): - def advance_txs(self, txs): - result = super().advance_txs(txs) - - tx_num = self.tx_count - len(txs) - script_name_hashX = self.coin.name_hashX_from_script - update_touched = self.touched.update - hashXs_by_tx = [] - append_hashXs = hashXs_by_tx.append - - for tx, tx_hash in txs: - hashXs = [] - append_hashX = hashXs.append - - # Add the new UTXOs and associate them with the name script - for idx, txout in enumerate(tx.outputs): - # Get the hashX of the name script. Ignore non-name scripts. - hashX = script_name_hashX(txout.pk_script) - if hashX: - append_hashX(hashX) - - append_hashXs(hashXs) - update_touched(hashXs) - tx_num += 1 - - self.db.history.add_unflushed(hashXs_by_tx, self.tx_count - len(txs)) - - return result - - class Timer: def __init__(self, name): diff --git a/lbry/lbry/wallet/server/coin.py b/lbry/lbry/wallet/server/coin.py index d709477f5..f1663769b 100644 --- a/lbry/lbry/wallet/server/coin.py +++ b/lbry/lbry/wallet/server/coin.py @@ -1,35 +1,21 @@ -import struct -from hashlib import sha256 - -from torba.server.script import ScriptPubKey, OpCodes -from torba.server.util import cachedproperty -from torba.server.hash import hash_to_hex_str, HASHX_LEN -from torba.server.tx import DeserializerSegWit - -from lbry.wallet.script import OutputScript -from .session import LBRYElectrumX, LBRYSessionManager -from .block_processor import LBRYBlockProcessor -from .daemon import LBCDaemon -from .db.writer import LBRYDB -from collections import namedtuple - import re import struct -from decimal import Decimal +from typing import List from hashlib import sha256 -from functools import partial -import base64 -from typing import Type, List +from decimal import Decimal +from collections import namedtuple -import torba.server.util as util -from torba.server.hash import Base58, hash160, double_sha256, hash_to_hex_str -from torba.server.hash import HASHX_LEN, hex_str_to_hash -from torba.server.script import ScriptPubKey, OpCodes -import torba.server.tx as lib_tx -import torba.server.block_processor as block_proc -from torba.server.db import DB -import torba.server.daemon as daemon -from torba.server.session import ElectrumX, DashElectrumX, SessionManager +import lbry.wallet.server.tx as lib_tx +from lbry.wallet.script import OutputScript +from lbry.wallet.server.tx import DeserializerSegWit +from lbry.wallet.server.util import cachedproperty, subclasses +from lbry.wallet.server.hash import Base58, hash160, double_sha256, hash_to_hex_str, HASHX_LEN +from lbry.wallet.server.daemon import Daemon, LBCDaemon +from lbry.wallet.server.script import ScriptPubKey, OpCodes +from lbry.wallet.server.leveldb import DB +from lbry.wallet.server.session import LBRYElectrumX, LBRYSessionManager +from lbry.wallet.server.db.writer import LBRYDB +from lbry.wallet.server.block_processor import LBRYBlockProcessor Block = namedtuple("Block", "raw header transactions") @@ -50,11 +36,11 @@ class Coin: CHUNK_SIZE = 2016 BASIC_HEADER_SIZE = 80 STATIC_BLOCK_HEADERS = True - SESSIONCLS = ElectrumX + SESSIONCLS = LBRYElectrumX DESERIALIZER = lib_tx.Deserializer - DAEMON = daemon.Daemon - BLOCK_PROCESSOR = block_proc.BlockProcessor - SESSION_MANAGER = SessionManager + DAEMON = Daemon + BLOCK_PROCESSOR = LBRYBlockProcessor + SESSION_MANAGER = LBRYSessionManager DB = DB HEADER_VALUES = [ 'version', 'prev_block_hash', 'merkle_root', 'timestamp', 'bits', 'nonce' @@ -75,7 +61,7 @@ class Coin: Raise an exception if unrecognised.""" req_attrs = ['TX_COUNT', 'TX_COUNT_HEIGHT', 'TX_PER_BLOCK'] - for coin in util.subclasses(Coin): + for coin in subclasses(Coin): if (coin.NAME.lower() == name.lower() and coin.NET.lower() == net.lower()): coin_req_attrs = req_attrs.copy() @@ -125,7 +111,7 @@ class Coin: def lookup_xverbytes(verbytes): """Return a (is_xpub, coin_class) pair given xpub/xprv verbytes.""" # Order means BTC testnet will override NMC testnet - for coin in util.subclasses(Coin): + for coin in subclasses(Coin): if verbytes == coin.XPUB_VERBYTES: return True, coin if verbytes == coin.XPRV_VERBYTES: diff --git a/lbry/lbry/wallet/server/daemon.py b/lbry/lbry/wallet/server/daemon.py index d46cb6ec9..90600af8d 100644 --- a/lbry/lbry/wallet/server/daemon.py +++ b/lbry/lbry/wallet/server/daemon.py @@ -1,22 +1,14 @@ -from functools import wraps - -from torba.rpc.jsonrpc import RPCError - import asyncio import itertools import json import time -from calendar import timegm -from struct import pack -from time import strptime +from functools import wraps import aiohttp -from torba.server.util import hex_to_bytes, class_logger, \ - unpack_le_uint16_from, pack_varint -from torba.server.hash import hex_str_to_hash, hash_to_hex_str -from torba.server.tx import DeserializerDecred -from torba.rpc import JSONRPC +from lbry.wallet.rpc.jsonrpc import RPCError +from lbry.wallet.server.util import hex_to_bytes, class_logger +from lbry.wallet.rpc import JSONRPC class DaemonError(Exception): @@ -280,187 +272,6 @@ class Daemon: return self._height -class DashDaemon(Daemon): - - async def masternode_broadcast(self, params): - """Broadcast a transaction to the network.""" - return await self._send_single('masternodebroadcast', params) - - async def masternode_list(self, params): - """Return the masternode status.""" - return await self._send_single('masternodelist', params) - - -class FakeEstimateFeeDaemon(Daemon): - """Daemon that simulates estimatefee and relayfee RPC calls. Coin that - wants to use this daemon must define ESTIMATE_FEE & RELAY_FEE""" - - async def estimatefee(self, block_count): - """Return the fee estimate for the given parameters.""" - return self.coin.ESTIMATE_FEE - - async def relayfee(self): - """The minimum fee a low-priority tx must pay in order to be accepted - to the daemon's memory pool.""" - return self.coin.RELAY_FEE - - -class LegacyRPCDaemon(Daemon): - """Handles connections to a daemon at the given URL. - - This class is useful for daemons that don't have the new 'getblock' - RPC call that returns the block in hex, the workaround is to manually - recreate the block bytes. The recreated block bytes may not be the exact - as in the underlying blockchain but it is good enough for our indexing - purposes.""" - - async def raw_blocks(self, hex_hashes): - """Return the raw binary blocks with the given hex hashes.""" - params_iterable = ((h, ) for h in hex_hashes) - block_info = await self._send_vector('getblock', params_iterable) - - blocks = [] - for i in block_info: - raw_block = await self.make_raw_block(i) - blocks.append(raw_block) - - # Convert hex string to bytes - return blocks - - async def make_raw_header(self, b): - pbh = b.get('previousblockhash') - if pbh is None: - pbh = '0' * 64 - return b''.join([ - pack(' 0: - transactions = await self.getrawtransactions(b.get('tx'), False) - - raw_block = header - num_txs = len(transactions) - if num_txs > 0: - raw_block += pack_varint(num_txs) - raw_block += b''.join(transactions) - else: - raw_block += b'\x00' - - return raw_block - - def timestamp_safe(self, t): - if isinstance(t, int): - return t - return timegm(strptime(t, "%Y-%m-%d %H:%M:%S %Z")) - - -class DecredDaemon(Daemon): - async def raw_blocks(self, hex_hashes): - """Return the raw binary blocks with the given hex hashes.""" - - params_iterable = ((h, False) for h in hex_hashes) - blocks = await self._send_vector('getblock', params_iterable) - - raw_blocks = [] - valid_tx_tree = {} - for block in blocks: - # Convert to bytes from hex - raw_block = hex_to_bytes(block) - raw_blocks.append(raw_block) - # Check if previous block is valid - prev = self.prev_hex_hash(raw_block) - votebits = unpack_le_uint16_from(raw_block[100:102])[0] - valid_tx_tree[prev] = self.is_valid_tx_tree(votebits) - - processed_raw_blocks = [] - for hash, raw_block in zip(hex_hashes, raw_blocks): - if hash in valid_tx_tree: - is_valid = valid_tx_tree[hash] - else: - # Do something complicated to figure out if this block is valid - header = await self._send_single('getblockheader', (hash, )) - if 'nextblockhash' not in header: - raise DaemonError(f'Could not find next block for {hash}') - next_hash = header['nextblockhash'] - next_header = await self._send_single('getblockheader', - (next_hash, )) - is_valid = self.is_valid_tx_tree(next_header['votebits']) - - if is_valid: - processed_raw_blocks.append(raw_block) - else: - # If this block is invalid remove the normal transactions - self.logger.info(f'block {hash} is invalidated') - processed_raw_blocks.append(self.strip_tx_tree(raw_block)) - - return processed_raw_blocks - - @staticmethod - def prev_hex_hash(raw_block): - return hash_to_hex_str(raw_block[4:36]) - - @staticmethod - def is_valid_tx_tree(votebits): - # Check if previous block was invalidated. - return bool(votebits & (1 << 0) != 0) - - def strip_tx_tree(self, raw_block): - c = self.coin - assert issubclass(c.DESERIALIZER, DeserializerDecred) - d = c.DESERIALIZER(raw_block, start=c.BASIC_HEADER_SIZE) - d.read_tx_tree() # Skip normal transactions - # Create a fake block without any normal transactions - return raw_block[:c.BASIC_HEADER_SIZE] + b'\x00' + raw_block[d.cursor:] - - async def height(self): - height = await super().height() - if height > 0: - # Lie about the daemon height as the current tip can be invalidated - height -= 1 - self._height = height - return height - - async def mempool_hashes(self): - mempool = await super().mempool_hashes() - # Add current tip transactions to the 'fake' mempool. - real_height = await self._send_single('getblockcount') - tip_hash = await self._send_single('getblockhash', (real_height,)) - tip = await self.deserialised_block(tip_hash) - # Add normal transactions except coinbase - mempool += tip['tx'][1:] - # Add stake transactions if applicable - mempool += tip.get('stx', []) - return mempool - - def client_session(self): - # FIXME allow self signed certificates - connector = aiohttp.TCPConnector(verify_ssl=False) - return aiohttp.ClientSession(connector=connector) - - -class PreLegacyRPCDaemon(LegacyRPCDaemon): - """Handles connections to a daemon at the given URL. - - This class is useful for daemons that don't have the new 'getblock' - RPC call that returns the block in hex, and need the False parameter - for the getblock""" - - async def deserialised_block(self, hex_hash): - """Return the deserialised block with the given hex hash.""" - return await self._send_single('getblock', (hex_hash, False)) - - def handles_errors(decorated_function): @wraps(decorated_function) async def wrapper(*args, **kwargs): diff --git a/lbry/lbry/wallet/server/db/full_text_search.py b/lbry/lbry/wallet/server/db/full_text_search.py index c553fc1b2..3a88775c7 100644 --- a/lbry/lbry/wallet/server/db/full_text_search.py +++ b/lbry/lbry/wallet/server/db/full_text_search.py @@ -1,4 +1,4 @@ -from torba.client.basedatabase import constraints_to_sql +from lbry.wallet.client.basedatabase import constraints_to_sql CREATE_FULL_TEXT_SEARCH = """ create virtual table if not exists search using fts5( diff --git a/lbry/lbry/wallet/server/db/reader.py b/lbry/lbry/wallet/server/db/reader.py index 6d0f0e3c9..0d8858d80 100644 --- a/lbry/lbry/wallet/server/db/reader.py +++ b/lbry/lbry/wallet/server/db/reader.py @@ -10,7 +10,7 @@ from contextvars import ContextVar from functools import wraps from dataclasses import dataclass -from torba.client.basedatabase import query, interpolate +from lbry.wallet.client.basedatabase import query, interpolate from lbry.schema.url import URL, normalize_name from lbry.schema.tags import clean_tags diff --git a/lbry/lbry/wallet/server/db/writer.py b/lbry/lbry/wallet/server/db/writer.py index 779ebffe0..6dca73c22 100644 --- a/lbry/lbry/wallet/server/db/writer.py +++ b/lbry/lbry/wallet/server/db/writer.py @@ -5,9 +5,9 @@ from itertools import chain from decimal import Decimal from collections import namedtuple -from torba.server.db import DB -from torba.server.util import class_logger -from torba.client.basedatabase import query, constraints_to_sql +from lbry.wallet.server.leveldb import DB +from lbry.wallet.server.util import class_logger +from lbry.wallet.client.basedatabase import query, constraints_to_sql from lbry.schema.tags import clean_tags from lbry.schema.mime_types import guess_stream_type diff --git a/lbry/lbry/wallet/server/env.py b/lbry/lbry/wallet/server/env.py index c406da84a..c9ee607a6 100644 --- a/lbry/lbry/wallet/server/env.py +++ b/lbry/lbry/wallet/server/env.py @@ -12,9 +12,9 @@ from os import environ from collections import namedtuple from ipaddress import ip_address -from torba.server.util import class_logger -from torba.server.coins import Coin -import torba.server.util as lib_util +from lbry.wallet.server.util import class_logger +from lbry.wallet.server.coin import Coin +import lbry.wallet.server.util as lib_util NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix') diff --git a/lbry/lbry/wallet/server/hash.py b/lbry/lbry/wallet/server/hash.py index d09d522af..a1a5d8068 100644 --- a/lbry/lbry/wallet/server/hash.py +++ b/lbry/lbry/wallet/server/hash.py @@ -29,7 +29,7 @@ import hashlib import hmac -from torba.server.util import bytes_to_int, int_to_bytes, hex_to_bytes +from lbry.wallet.server.util import bytes_to_int, int_to_bytes, hex_to_bytes _sha256 = hashlib.sha256 _sha512 = hashlib.sha512 diff --git a/lbry/lbry/wallet/server/history.py b/lbry/lbry/wallet/server/history.py index 405355fbb..f810a1045 100644 --- a/lbry/lbry/wallet/server/history.py +++ b/lbry/lbry/wallet/server/history.py @@ -15,9 +15,9 @@ import time from collections import defaultdict from functools import partial -from torba.server import util -from torba.server.util import pack_be_uint16, unpack_be_uint16_from -from torba.server.hash import hash_to_hex_str, HASHX_LEN +from lbry.wallet.server import util +from lbry.wallet.server.util import pack_be_uint16, unpack_be_uint16_from +from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN class History: diff --git a/lbry/lbry/wallet/server/leveldb.py b/lbry/lbry/wallet/server/leveldb.py index 2b7421218..6f5ff5b46 100644 --- a/lbry/lbry/wallet/server/leveldb.py +++ b/lbry/lbry/wallet/server/leveldb.py @@ -23,12 +23,12 @@ from struct import pack, unpack import attr -from torba.server import util -from torba.server.hash import hash_to_hex_str, HASHX_LEN -from torba.server.merkle import Merkle, MerkleCache -from torba.server.util import formatted_time -from torba.server.storage import db_class -from torba.server.history import History +from lbry.wallet.server import util +from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN +from lbry.wallet.server.merkle import Merkle, MerkleCache +from lbry.wallet.server.util import formatted_time +from lbry.wallet.server.storage import db_class +from lbry.wallet.server.history import History UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value") diff --git a/lbry/lbry/wallet/server/mempool.py b/lbry/lbry/wallet/server/mempool.py index f1c162c3a..de959379e 100644 --- a/lbry/lbry/wallet/server/mempool.py +++ b/lbry/lbry/wallet/server/mempool.py @@ -15,9 +15,9 @@ from collections import defaultdict import attr -from torba.server.hash import hash_to_hex_str, hex_str_to_hash -from torba.server.util import class_logger, chunks -from torba.server.db import UTXO +from lbry.wallet.server.hash import hash_to_hex_str, hex_str_to_hash +from lbry.wallet.server.util import class_logger, chunks +from lbry.wallet.server.leveldb import UTXO @attr.s(slots=True) diff --git a/lbry/lbry/wallet/server/merkle.py b/lbry/lbry/wallet/server/merkle.py index 8cfa89c68..174e77b8e 100644 --- a/lbry/lbry/wallet/server/merkle.py +++ b/lbry/lbry/wallet/server/merkle.py @@ -29,7 +29,7 @@ from asyncio import Event from math import ceil, log -from torba.server.hash import double_sha256 +from lbry.wallet.server.hash import double_sha256 class Merkle: diff --git a/lbry/lbry/wallet/server/peer.py b/lbry/lbry/wallet/server/peer.py index 79a1459ae..078fda9e4 100644 --- a/lbry/lbry/wallet/server/peer.py +++ b/lbry/lbry/wallet/server/peer.py @@ -27,8 +27,8 @@ from ipaddress import ip_address -from torba.server import util -from torba.server.util import cachedproperty +from lbry.wallet.server import util +from lbry.wallet.server.util import cachedproperty from typing import Dict diff --git a/lbry/lbry/wallet/server/peers.py b/lbry/lbry/wallet/server/peers.py index 3b3e08a35..f1407339b 100644 --- a/lbry/lbry/wallet/server/peers.py +++ b/lbry/lbry/wallet/server/peers.py @@ -16,13 +16,13 @@ import typing from asyncio import Event, sleep from collections import defaultdict, Counter -from torba.tasks import TaskGroup -from torba.rpc import ( +from lbry.wallet.tasks import TaskGroup +from lbry.wallet.rpc import ( Connector, RPCSession, SOCKSProxy, Notification, handler_invocation, SOCKSError, RPCError ) -from torba.server.peer import Peer -from torba.server.util import class_logger, protocol_tuple +from lbry.wallet.server.peer import Peer +from lbry.wallet.server.util import class_logger, protocol_tuple PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4) STALE_SECS = 24 * 3600 diff --git a/lbry/lbry/wallet/server/script.py b/lbry/lbry/wallet/server/script.py index c7ca81306..ce8d5e5a1 100644 --- a/lbry/lbry/wallet/server/script.py +++ b/lbry/lbry/wallet/server/script.py @@ -29,7 +29,7 @@ from collections import namedtuple -from torba.server.util import unpack_le_uint16_from, unpack_le_uint32_from, \ +from lbry.wallet.server.util import unpack_le_uint16_from, unpack_le_uint32_from, \ pack_le_uint16, pack_le_uint32 diff --git a/lbry/lbry/wallet/server/server.py b/lbry/lbry/wallet/server/server.py index 691b84055..f5cf8c68c 100644 --- a/lbry/lbry/wallet/server/server.py +++ b/lbry/lbry/wallet/server/server.py @@ -3,8 +3,8 @@ import logging import asyncio from concurrent.futures.thread import ThreadPoolExecutor -import torba -from torba.server.mempool import MemPool, MemPoolAPI +import lbry +from lbry.wallet.server.mempool import MemPool, MemPoolAPI class Notifications: @@ -91,7 +91,7 @@ class Server: async def start(self): env = self.env min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() - self.log.info(f'software version: {torba.__version__}') + self.log.info(f'software version: {lbry.__version__}') self.log.info(f'supported protocol versions: {min_str}-{max_str}') self.log.info(f'event loop policy: {env.loop_policy}') self.log.info(f'reorg limit is {env.reorg_limit:,d} blocks') diff --git a/lbry/lbry/wallet/server/session.py b/lbry/lbry/wallet/server/session.py index 611f6658d..3ac62bdea 100644 --- a/lbry/lbry/wallet/server/session.py +++ b/lbry/lbry/wallet/server/session.py @@ -1,65 +1,55 @@ import os +import ssl import math import time +import json +import zlib +import pylru import base64 +import codecs +import typing import asyncio import logging +import itertools +import collections + +from asyncio import Event, sleep +from collections import defaultdict +from functools import partial + from binascii import hexlify from pylru import lrucache from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor -from torba.rpc.jsonrpc import RPCError, JSONRPC -from torba.server.session import ElectrumX, SessionManager -from torba.server import util - +import lbry from lbry.wallet.server.block_processor import LBRYBlockProcessor from lbry.wallet.server.db.writer import LBRYDB from lbry.wallet.server.db import reader from lbry.wallet.server.websocket import AdminWebSocket from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics -from lbry import __version__ as sdk_version -import base64 -import collections -import asyncio -import codecs -import datetime -import itertools -import json -import os -import zlib - -import pylru -import ssl -import time -import typing -from asyncio import Event, sleep -from collections import defaultdict -from functools import partial - -import torba -from torba.rpc import ( +from lbry.wallet.rpc import ( RPCSession, JSONRPCAutoDetect, JSONRPCConnection, - handler_invocation, RPCError, Request + handler_invocation, RPCError, Request, JSONRPC ) -from torba.server import text -from torba.server import util -from torba.server.hash import (sha256, hash_to_hex_str, hex_str_to_hash, - HASHX_LEN, Base58Error) -from torba.server.daemon import DaemonError -from torba.server.peers import PeerManager +from lbry.wallet.server import text +from lbry.wallet.server import util +from lbry.wallet.server.hash import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, Base58Error +from lbry.wallet.server.daemon import DaemonError +from lbry.wallet.server.peers import PeerManager if typing.TYPE_CHECKING: - from torba.server.env import Env - from torba.server.db import DB - from torba.server.block_processor import BlockProcessor - from torba.server.mempool import MemPool - from torba.server.daemon import Daemon + from lbry.wallet.server.env import Env + from lbry.wallet.server.leveldb import DB + from lbry.wallet.server.block_processor import BlockProcessor + from lbry.wallet.server.mempool import MemPool + from lbry.wallet.server.daemon import Daemon BAD_REQUEST = 1 DAEMON_ERROR = 2 log = logging.getLogger(__name__) + def scripthash_to_hashX(scripthash: str) -> bytes: try: bin_hash = hex_str_to_hash(scripthash) @@ -149,10 +139,6 @@ class SessionManager: self.notified_height: typing.Optional[int] = None # Cache some idea of room to avoid recounting on each subscription self.subs_room = 0 - # Masternode stuff only for such coins - if issubclass(env.coin.SESSIONCLS, DashElectrumX): - self.mn_cache_height = 0 - self.mn_cache = [] # type: ignore self.session_event = Event() @@ -335,7 +321,7 @@ class SessionManager: 'subs': self._sub_count(), 'txs_sent': self.txs_sent, 'uptime': util.formatted_time(time.time() - self.start_time), - 'version': torba.__version__, + 'version': lbry.__version__, } def _session_data(self, for_log): @@ -733,11 +719,67 @@ class SessionBase(RPCSession): return await coro -class ElectrumX(SessionBase): +class LBRYSessionManager(SessionManager): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.query_executor = None + self.websocket = None + self.metrics = ServerLoadData() + self.metrics_loop = None + self.running = False + if self.env.websocket_host is not None and self.env.websocket_port is not None: + self.websocket = AdminWebSocket(self) + self.search_cache = self.bp.search_cache + self.search_cache['search'] = lrucache(10000) + self.search_cache['resolve'] = lrucache(10000) + + async def process_metrics(self): + while self.running: + data = self.metrics.to_json_and_reset({ + 'sessions': self.session_count(), + 'height': self.db.db_height, + }) + if self.websocket is not None: + self.websocket.send_message(data) + await asyncio.sleep(1) + + async def start_other(self): + self.running = True + path = os.path.join(self.env.db_dir, 'claims.db') + args = dict( + initializer=reader.initializer, + initargs=(self.logger, path, self.env.coin.NET, self.env.database_query_timeout, + self.env.track_metrics) + ) + if self.env.max_query_workers is not None and self.env.max_query_workers == 0: + self.query_executor = ThreadPoolExecutor(max_workers=1, **args) + else: + self.query_executor = ProcessPoolExecutor( + max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args + ) + if self.websocket is not None: + await self.websocket.start() + if self.env.track_metrics: + self.metrics_loop = asyncio.create_task(self.process_metrics()) + + async def stop_other(self): + self.running = False + if self.env.track_metrics: + self.metrics_loop.cancel() + if self.websocket is not None: + await self.websocket.stop() + self.query_executor.shutdown() + + +class LBRYElectrumX(SessionBase): """A TCP server that handles incoming Electrum connections.""" - PROTOCOL_MIN = (1, 1) - PROTOCOL_MAX = (1, 4) + PROTOCOL_MIN = lbry.version + PROTOCOL_MAX = (1, 0) + max_errors = math.inf # don't disconnect people for errors! let them happen... + session_mgr: LBRYSessionManager + version = lbry.__version__ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -748,6 +790,13 @@ class ElectrumX(SessionBase): self.sv_seen = False self.mempool_statuses = {} self.set_request_handlers(self.PROTOCOL_MIN) + # fixme: this is a rebase hack, we need to go through ChainState instead later + self.daemon = self.session_mgr.daemon + self.bp: LBRYBlockProcessor = self.session_mgr.bp + self.db: LBRYDB = self.bp.db + # space separated list of channel URIs used for filtering bad content + filtering_channels = self.env.default('FILTERING_CHANNELS_IDS', '') + self.filtering_channels_ids = list(filter(None, filtering_channels.split(' '))) @classmethod def protocol_min_max_strings(cls): @@ -825,6 +874,169 @@ class ElectrumX(SessionBase): es = '' if len(changed) == 1 else 'es' self.logger.info(f'notified of {len(changed):,d} address{es}') + def get_metrics_or_placeholder_for_api(self, query_name): + """ Do not hold on to a reference to the metrics + returned by this method past an `await` or + you may be working with a stale metrics object. + """ + if self.env.track_metrics: + return self.session_mgr.metrics.for_api(query_name) + else: + return APICallMetrics(query_name) + + async def run_in_executor(self, query_name, func, kwargs): + start = time.perf_counter() + try: + result = await asyncio.get_running_loop().run_in_executor( + self.session_mgr.query_executor, func, kwargs + ) + except asyncio.CancelledError: + raise + except reader.SQLiteInterruptedError as error: + metrics = self.get_metrics_or_placeholder_for_api(query_name) + metrics.query_interrupt(start, error.metrics) + raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out') + except reader.SQLiteOperationalError as error: + metrics = self.get_metrics_or_placeholder_for_api(query_name) + metrics.query_error(start, error.metrics) + raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute') + except Exception: + log.exception("dear devs, please handle this exception better") + metrics = self.get_metrics_or_placeholder_for_api(query_name) + metrics.query_error(start, {}) + raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error') + + if self.env.track_metrics: + metrics = self.get_metrics_or_placeholder_for_api(query_name) + (result, metrics_data) = result + metrics.query_response(start, metrics_data) + + return base64.b64encode(result).decode() + + async def run_and_cache_query(self, query_name, function, kwargs): + metrics = self.get_metrics_or_placeholder_for_api(query_name) + metrics.start() + cache = self.session_mgr.search_cache[query_name] + cache_key = str(kwargs) + cache_item = cache.get(cache_key) + if cache_item is None: + cache_item = cache[cache_key] = ResultCacheItem() + elif cache_item.result is not None: + metrics.cache_response() + return cache_item.result + async with cache_item.lock: + if cache_item.result is None: + cache_item.result = await self.run_in_executor( + query_name, function, kwargs + ) + else: + metrics = self.get_metrics_or_placeholder_for_api(query_name) + metrics.cache_response() + return cache_item.result + + async def claimtrie_search(self, **kwargs): + if kwargs: + kwargs.setdefault('blocklist_channel_ids', []).extend(self.filtering_channels_ids) + return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs) + + async def claimtrie_resolve(self, *urls): + if urls: + return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls) + + async def get_server_height(self): + return self.bp.height + + async def transaction_get_height(self, tx_hash): + self.assert_tx_hash(tx_hash) + transaction_info = await self.daemon.getrawtransaction(tx_hash, True) + if transaction_info and 'hex' in transaction_info and 'confirmations' in transaction_info: + # an unconfirmed transaction from lbrycrdd will not have a 'confirmations' field + return (self.db.db_height - transaction_info['confirmations']) + 1 + elif transaction_info and 'hex' in transaction_info: + return -1 + return None + + async def claimtrie_getclaimsbyids(self, *claim_ids): + claims = await self.batched_formatted_claims_from_daemon(claim_ids) + return dict(zip(claim_ids, claims)) + + async def batched_formatted_claims_from_daemon(self, claim_ids): + claims = await self.daemon.getclaimsbyids(claim_ids) + result = [] + for claim in claims: + if claim and claim.get('value'): + result.append(self.format_claim_from_daemon(claim)) + return result + + def format_claim_from_daemon(self, claim, name=None): + """Changes the returned claim data to the format expected by lbry and adds missing fields.""" + + if not claim: + return {} + + # this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd + # it will be fixed after the lbrycrd upstream merge to v17 is done + # it originated as a fear of terminals not supporting unicode. alas, they all do + + if 'name' in claim: + name = claim['name'].encode('ISO-8859-1').decode() + info = self.db.sql.get_claims(claim_id=claim['claimId']) + if not info: + # raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id)) + return {} + address = info.address.decode() + # fixme: temporary + #supports = self.format_supports_from_daemon(claim.get('supports', [])) + supports = [] + + amount = get_from_possible_keys(claim, 'amount', 'nAmount') + height = get_from_possible_keys(claim, 'height', 'nHeight') + effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount') + valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight') + + result = { + "name": name, + "claim_id": claim['claimId'], + "txid": claim['txid'], + "nout": claim['n'], + "amount": amount, + "depth": self.db.db_height - height + 1, + "height": height, + "value": hexlify(claim['value'].encode('ISO-8859-1')).decode(), + "address": address, # from index + "supports": supports, + "effective_amount": effective_amount, + "valid_at_height": valid_at_height + } + if 'claim_sequence' in claim: + # TODO: ensure that lbrycrd #209 fills in this value + result['claim_sequence'] = claim['claim_sequence'] + else: + result['claim_sequence'] = -1 + if 'normalized_name' in claim: + result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode() + return result + + def assert_tx_hash(self, value): + '''Raise an RPCError if the value is not a valid transaction + hash.''' + try: + if len(util.hex_to_bytes(value)) == 32: + return + except Exception: + pass + raise RPCError(1, f'{value} should be a transaction hash') + + def assert_claim_id(self, value): + '''Raise an RPCError if the value is not a valid claim id + hash.''' + try: + if len(util.hex_to_bytes(value)) == 20: + return + except Exception: + pass + raise RPCError(1, f'{value} should be a claim id hash') + async def subscribe_headers_result(self): """The result of a header subscription or notification.""" return self.session_mgr.hsub_results[self.subscribe_headers_raw] @@ -1264,7 +1476,6 @@ class ElectrumX(SessionBase): def set_request_handlers(self, ptuple): self.protocol_tuple = ptuple - handlers = { 'blockchain.block.get_chunk': self.block_get_chunk, 'blockchain.block.get_header': self.block_get_header, @@ -1284,41 +1495,22 @@ class ElectrumX(SessionBase): 'server.features': self.server_features_async, 'server.peers.subscribe': self.peers_subscribe, 'server.version': self.server_version, + 'blockchain.transaction.get_height': self.transaction_get_height, + 'blockchain.claimtrie.search': self.claimtrie_search, + 'blockchain.claimtrie.resolve': self.claimtrie_resolve, + 'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids, + 'blockchain.block.get_server_height': self.get_server_height, + 'mempool.get_fee_histogram': self.mempool.compact_fee_histogram, + 'blockchain.block.headers': self.block_headers, + 'server.ping': self.ping, + 'blockchain.headers.subscribe': self.headers_subscribe_False, + 'blockchain.address.get_balance': self.address_get_balance, + 'blockchain.address.get_history': self.address_get_history, + 'blockchain.address.get_mempool': self.address_get_mempool, + 'blockchain.address.listunspent': self.address_listunspent, + 'blockchain.address.subscribe': self.address_subscribe, + 'blockchain.address.unsubscribe': self.address_unsubscribe, } - - if ptuple >= (1, 2): - # New handler as of 1.2 - handlers.update({ - 'mempool.get_fee_histogram': - self.mempool.compact_fee_histogram, - 'blockchain.block.headers': self.block_headers, - 'server.ping': self.ping, - }) - - if ptuple >= (1, 4): - handlers.update({ - 'blockchain.block.header': self.block_header, - 'blockchain.block.headers': self.block_headers, - 'blockchain.headers.subscribe': self.headers_subscribe, - 'blockchain.transaction.id_from_pos': - self.transaction_id_from_pos, - }) - elif ptuple >= (1, 3): - handlers.update({ - 'blockchain.block.header': self.block_header_13, - 'blockchain.headers.subscribe': self.headers_subscribe_True, - }) - else: - handlers.update({ - 'blockchain.headers.subscribe': self.headers_subscribe_False, - 'blockchain.address.get_balance': self.address_get_balance, - 'blockchain.address.get_history': self.address_get_history, - 'blockchain.address.get_mempool': self.address_get_mempool, - 'blockchain.address.listunspent': self.address_listunspent, - 'blockchain.address.subscribe': self.address_subscribe, - 'blockchain.address.unsubscribe': self.address_unsubscribe, - }) - self.request_handlers = handlers @@ -1334,153 +1526,6 @@ class LocalRPC(SessionBase): return 'RPC' -class DashElectrumX(ElectrumX): - """A TCP server that handles incoming Electrum Dash connections.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.mns = set() - - def set_request_handlers(self, ptuple): - super().set_request_handlers(ptuple) - self.request_handlers.update({ - 'masternode.announce.broadcast': - self.masternode_announce_broadcast, - 'masternode.subscribe': self.masternode_subscribe, - 'masternode.list': self.masternode_list - }) - - async def notify(self, touched, height_changed): - """Notify the client about changes in masternode list.""" - await super().notify(touched, height_changed) - for mn in self.mns: - status = await self.daemon_request('masternode_list', - ['status', mn]) - await self.send_notification('masternode.subscribe', - [mn, status.get(mn)]) - - # Masternode command handlers - async def masternode_announce_broadcast(self, signmnb): - """Pass through the masternode announce message to be broadcast - by the daemon. - - signmnb: signed masternode broadcast message.""" - try: - return await self.daemon_request('masternode_broadcast', - ['relay', signmnb]) - except DaemonError as e: - error, = e.args - message = error['message'] - self.logger.info(f'masternode_broadcast: {message}') - raise RPCError(BAD_REQUEST, 'the masternode broadcast was ' - f'rejected.\n\n{message}\n[{signmnb}]') - - async def masternode_subscribe(self, collateral): - """Returns the status of masternode. - - collateral: masternode collateral. - """ - result = await self.daemon_request('masternode_list', - ['status', collateral]) - if result is not None: - self.mns.add(collateral) - return result.get(collateral) - return None - - async def masternode_list(self, payees): - """ - Returns the list of masternodes. - - payees: a list of masternode payee addresses. - """ - if not isinstance(payees, list): - raise RPCError(BAD_REQUEST, 'expected a list of payees') - - def get_masternode_payment_queue(mns): - """Returns the calculated position in the payment queue for all the - valid masterernodes in the given mns list. - - mns: a list of masternodes information. - """ - now = int(datetime.datetime.utcnow().strftime("%s")) - mn_queue = [] - - # Only ENABLED masternodes are considered for the list. - for line in mns: - mnstat = mns[line].split() - if mnstat[0] == 'ENABLED': - # if last paid time == 0 - if int(mnstat[5]) == 0: - # use active seconds - mnstat.append(int(mnstat[4])) - else: - # now minus last paid - delta = now - int(mnstat[5]) - # if > active seconds, use active seconds - if delta >= int(mnstat[4]): - mnstat.append(int(mnstat[4])) - # use active seconds - else: - mnstat.append(delta) - mn_queue.append(mnstat) - mn_queue = sorted(mn_queue, key=lambda x: x[8], reverse=True) - return mn_queue - - def get_payment_position(payment_queue, address): - """ - Returns the position of the payment list for the given address. - - payment_queue: position in the payment queue for the masternode. - address: masternode payee address. - """ - position = -1 - for pos, mn in enumerate(payment_queue, start=1): - if mn[2] == address: - position = pos - break - return position - - # Accordingly with the masternode payment queue, a custom list - # with the masternode information including the payment - # position is returned. - cache = self.session_mgr.mn_cache - if not cache or self.session_mgr.mn_cache_height != self.db.db_height: - full_mn_list = await self.daemon_request('masternode_list', - ['full']) - mn_payment_queue = get_masternode_payment_queue(full_mn_list) - mn_payment_count = len(mn_payment_queue) - mn_list = [] - for key, value in full_mn_list.items(): - mn_data = value.split() - mn_info = {} - mn_info['vin'] = key - mn_info['status'] = mn_data[0] - mn_info['protocol'] = mn_data[1] - mn_info['payee'] = mn_data[2] - mn_info['lastseen'] = mn_data[3] - mn_info['activeseconds'] = mn_data[4] - mn_info['lastpaidtime'] = mn_data[5] - mn_info['lastpaidblock'] = mn_data[6] - mn_info['ip'] = mn_data[7] - mn_info['paymentposition'] = get_payment_position( - mn_payment_queue, mn_info['payee']) - mn_info['inselection'] = ( - mn_info['paymentposition'] < mn_payment_count // 10) - balance = await self.address_get_balance(mn_info['payee']) - mn_info['balance'] = (sum(balance.values()) - / self.coin.VALUE_PER_COIN) - mn_list.append(mn_info) - cache.clear() - cache.extend(mn_list) - self.session_mgr.mn_cache_height = self.db.db_height - - # If payees is an empty list the whole masternode list is returned - if payees: - return [mn for mn in cache if mn['payee'] in payees] - else: - return cache - - class ResultCacheItem: __slots__ = '_result', 'lock', 'has_result' @@ -1500,251 +1545,6 @@ class ResultCacheItem: self.has_result.set() -class LBRYSessionManager(SessionManager): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.query_executor = None - self.websocket = None - self.metrics = ServerLoadData() - self.metrics_loop = None - self.running = False - if self.env.websocket_host is not None and self.env.websocket_port is not None: - self.websocket = AdminWebSocket(self) - self.search_cache = self.bp.search_cache - self.search_cache['search'] = lrucache(10000) - self.search_cache['resolve'] = lrucache(10000) - - async def process_metrics(self): - while self.running: - data = self.metrics.to_json_and_reset({ - 'sessions': self.session_count(), - 'height': self.db.db_height, - }) - if self.websocket is not None: - self.websocket.send_message(data) - await asyncio.sleep(1) - - async def start_other(self): - self.running = True - path = os.path.join(self.env.db_dir, 'claims.db') - args = dict( - initializer=reader.initializer, - initargs=(self.logger, path, self.env.coin.NET, self.env.database_query_timeout, - self.env.track_metrics) - ) - if self.env.max_query_workers is not None and self.env.max_query_workers == 0: - self.query_executor = ThreadPoolExecutor(max_workers=1, **args) - else: - self.query_executor = ProcessPoolExecutor( - max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args - ) - if self.websocket is not None: - await self.websocket.start() - if self.env.track_metrics: - self.metrics_loop = asyncio.create_task(self.process_metrics()) - - async def stop_other(self): - self.running = False - if self.env.track_metrics: - self.metrics_loop.cancel() - if self.websocket is not None: - await self.websocket.stop() - self.query_executor.shutdown() - - -class LBRYElectrumX(ElectrumX): - PROTOCOL_MIN = (2, 0) # v0.48 is backwards incompatible, forking a new protocol - PROTOCOL_MAX = (2, 0) - max_errors = math.inf # don't disconnect people for errors! let them happen... - session_mgr: LBRYSessionManager - version = sdk_version - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # fixme: this is a rebase hack, we need to go through ChainState instead later - self.daemon = self.session_mgr.daemon - self.bp: LBRYBlockProcessor = self.session_mgr.bp - self.db: LBRYDB = self.bp.db - # space separated list of channel URIs used for filtering bad content - filtering_channels = self.env.default('FILTERING_CHANNELS_IDS', '') - self.filtering_channels_ids = list(filter(None, filtering_channels.split(' '))) - - def set_request_handlers(self, ptuple): - super().set_request_handlers((1, 2) if ptuple == (2, 0) else ptuple) # our "2.0" is electrumx 1.2 - handlers = { - 'blockchain.transaction.get_height': self.transaction_get_height, - 'blockchain.claimtrie.search': self.claimtrie_search, - 'blockchain.claimtrie.resolve': self.claimtrie_resolve, - 'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids, - 'blockchain.block.get_server_height': self.get_server_height, - } - self.request_handlers.update(handlers) - - def get_metrics_or_placeholder_for_api(self, query_name): - """ Do not hold on to a reference to the metrics - returned by this method past an `await` or - you may be working with a stale metrics object. - """ - if self.env.track_metrics: - return self.session_mgr.metrics.for_api(query_name) - else: - return APICallMetrics(query_name) - - async def run_in_executor(self, query_name, func, kwargs): - start = time.perf_counter() - try: - result = await asyncio.get_running_loop().run_in_executor( - self.session_mgr.query_executor, func, kwargs - ) - except asyncio.CancelledError: - raise - except reader.SQLiteInterruptedError as error: - metrics = self.get_metrics_or_placeholder_for_api(query_name) - metrics.query_interrupt(start, error.metrics) - raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out') - except reader.SQLiteOperationalError as error: - metrics = self.get_metrics_or_placeholder_for_api(query_name) - metrics.query_error(start, error.metrics) - raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute') - except Exception: - log.exception("dear devs, please handle this exception better") - metrics = self.get_metrics_or_placeholder_for_api(query_name) - metrics.query_error(start, {}) - raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error') - - if self.env.track_metrics: - metrics = self.get_metrics_or_placeholder_for_api(query_name) - (result, metrics_data) = result - metrics.query_response(start, metrics_data) - - return base64.b64encode(result).decode() - - async def run_and_cache_query(self, query_name, function, kwargs): - metrics = self.get_metrics_or_placeholder_for_api(query_name) - metrics.start() - cache = self.session_mgr.search_cache[query_name] - cache_key = str(kwargs) - cache_item = cache.get(cache_key) - if cache_item is None: - cache_item = cache[cache_key] = ResultCacheItem() - elif cache_item.result is not None: - metrics.cache_response() - return cache_item.result - async with cache_item.lock: - if cache_item.result is None: - cache_item.result = await self.run_in_executor( - query_name, function, kwargs - ) - else: - metrics = self.get_metrics_or_placeholder_for_api(query_name) - metrics.cache_response() - return cache_item.result - - async def claimtrie_search(self, **kwargs): - if kwargs: - kwargs.setdefault('blocklist_channel_ids', []).extend(self.filtering_channels_ids) - return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs) - - async def claimtrie_resolve(self, *urls): - if urls: - return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls) - - async def get_server_height(self): - return self.bp.height - - async def transaction_get_height(self, tx_hash): - self.assert_tx_hash(tx_hash) - transaction_info = await self.daemon.getrawtransaction(tx_hash, True) - if transaction_info and 'hex' in transaction_info and 'confirmations' in transaction_info: - # an unconfirmed transaction from lbrycrdd will not have a 'confirmations' field - return (self.db.db_height - transaction_info['confirmations']) + 1 - elif transaction_info and 'hex' in transaction_info: - return -1 - return None - - async def claimtrie_getclaimsbyids(self, *claim_ids): - claims = await self.batched_formatted_claims_from_daemon(claim_ids) - return dict(zip(claim_ids, claims)) - - async def batched_formatted_claims_from_daemon(self, claim_ids): - claims = await self.daemon.getclaimsbyids(claim_ids) - result = [] - for claim in claims: - if claim and claim.get('value'): - result.append(self.format_claim_from_daemon(claim)) - return result - - def format_claim_from_daemon(self, claim, name=None): - """Changes the returned claim data to the format expected by lbry and adds missing fields.""" - - if not claim: - return {} - - # this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd - # it will be fixed after the lbrycrd upstream merge to v17 is done - # it originated as a fear of terminals not supporting unicode. alas, they all do - - if 'name' in claim: - name = claim['name'].encode('ISO-8859-1').decode() - info = self.db.sql.get_claims(claim_id=claim['claimId']) - if not info: - # raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id)) - return {} - address = info.address.decode() - # fixme: temporary - #supports = self.format_supports_from_daemon(claim.get('supports', [])) - supports = [] - - amount = get_from_possible_keys(claim, 'amount', 'nAmount') - height = get_from_possible_keys(claim, 'height', 'nHeight') - effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount') - valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight') - - result = { - "name": name, - "claim_id": claim['claimId'], - "txid": claim['txid'], - "nout": claim['n'], - "amount": amount, - "depth": self.db.db_height - height + 1, - "height": height, - "value": hexlify(claim['value'].encode('ISO-8859-1')).decode(), - "address": address, # from index - "supports": supports, - "effective_amount": effective_amount, - "valid_at_height": valid_at_height - } - if 'claim_sequence' in claim: - # TODO: ensure that lbrycrd #209 fills in this value - result['claim_sequence'] = claim['claim_sequence'] - else: - result['claim_sequence'] = -1 - if 'normalized_name' in claim: - result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode() - return result - - def assert_tx_hash(self, value): - '''Raise an RPCError if the value is not a valid transaction - hash.''' - try: - if len(util.hex_to_bytes(value)) == 32: - return - except Exception: - pass - raise RPCError(1, f'{value} should be a transaction hash') - - def assert_claim_id(self, value): - '''Raise an RPCError if the value is not a valid claim id - hash.''' - try: - if len(util.hex_to_bytes(value)) == 20: - return - except Exception: - pass - raise RPCError(1, f'{value} should be a claim id hash') - - def get_from_possible_keys(dictionary, *keys): for key in keys: if key in dictionary: diff --git a/lbry/lbry/wallet/server/storage.py b/lbry/lbry/wallet/server/storage.py index 35dd8efe7..5e7db97dd 100644 --- a/lbry/lbry/wallet/server/storage.py +++ b/lbry/lbry/wallet/server/storage.py @@ -10,7 +10,7 @@ import os from functools import partial -from torba.server import util +from lbry.wallet.server import util def db_class(db_dir, name): diff --git a/lbry/lbry/wallet/server/text.py b/lbry/lbry/wallet/server/text.py index b34319fac..4919b0c01 100644 --- a/lbry/lbry/wallet/server/text.py +++ b/lbry/lbry/wallet/server/text.py @@ -1,6 +1,6 @@ import time -from torba.server import util +from lbry.wallet.server import util def sessions_lines(data): diff --git a/lbry/lbry/wallet/server/tx.py b/lbry/lbry/wallet/server/tx.py index 13aee32ef..411162155 100644 --- a/lbry/lbry/wallet/server/tx.py +++ b/lbry/lbry/wallet/server/tx.py @@ -29,9 +29,9 @@ from collections import namedtuple -from torba.server.hash import sha256, double_sha256, hash_to_hex_str -from torba.server.script import OpCodes -from torba.server.util import ( +from lbry.wallet.server.hash import sha256, double_sha256, hash_to_hex_str +from lbry.wallet.server.script import OpCodes +from lbry.wallet.server.util import ( unpack_le_int32_from, unpack_le_int64_from, unpack_le_uint16_from, unpack_le_uint32_from, unpack_le_uint64_from, pack_le_int32, pack_varint, pack_le_uint32, pack_le_int64, pack_varbytes, diff --git a/lbry/lbry/wallet/testcase.py b/lbry/lbry/wallet/testcase.py index 3e5ee15c2..e48711c7b 100644 --- a/lbry/lbry/wallet/testcase.py +++ b/lbry/lbry/wallet/testcase.py @@ -6,13 +6,16 @@ from asyncio.runners import _cancel_all_tasks # type: ignore import unittest from unittest.case import _Outcome from typing import Optional -from torba.orchstr8 import Conductor -from torba.orchstr8.node import BlockchainNode, WalletNode -from torba.client.baseledger import BaseLedger -from torba.client.baseaccount import BaseAccount -from torba.client.basemanager import BaseWalletManager -from torba.client.wallet import Wallet -from torba.client.util import satoshis_to_coins + +import lbry.wallet +from lbry.wallet.orchstr8 import Conductor +from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode +from lbry.wallet.client.baseledger import BaseLedger +from lbry.wallet.client.baseaccount import BaseAccount +from lbry.wallet.client.basemanager import BaseWalletManager +from lbry.wallet.client.wallet import Wallet +from lbry.wallet.client.util import satoshis_to_coins +from lbry.wallet import LbryWalletManager class ColorHandler(logging.StreamHandler): @@ -187,8 +190,8 @@ class AdvanceTimeTestCase(AsyncioTestCase): class IntegrationTestCase(AsyncioTestCase): SEED = None - LEDGER = None - MANAGER = None + LEDGER = lbry.wallet + MANAGER = LbryWalletManager ENABLE_SEGWIT = False VERBOSITY = logging.WARN diff --git a/lbry/lbry/wallet/transaction.py b/lbry/lbry/wallet/transaction.py index 819271239..74ec9bdd2 100644 --- a/lbry/lbry/wallet/transaction.py +++ b/lbry/lbry/wallet/transaction.py @@ -11,8 +11,9 @@ from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives.asymmetric.utils import Prehashed from cryptography.exceptions import InvalidSignature -from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput, ReadOnlyList -from torba.client.hash import hash160, sha256, Base58 +from lbry.crypto.base58 import Base58 +from lbry.crypto.hash import hash160, sha256 +from lbry.wallet.client.basetransaction import BaseTransaction, BaseInput, BaseOutput, ReadOnlyList from lbry.schema.claim import Claim from lbry.schema.purchase import Purchase from lbry.schema.url import normalize_name diff --git a/lbry/tests/integration/test_account_commands.py b/lbry/tests/integration/test_account_commands.py index 7de54015e..09d57a100 100644 --- a/lbry/tests/integration/test_account_commands.py +++ b/lbry/tests/integration/test_account_commands.py @@ -1,9 +1,5 @@ -import asyncio -import sqlite3 from lbry.testcase import CommandTestCase -from torba.client.basedatabase import SQLiteMixin from lbry.wallet.dewies import dewies_to_lbc -from lbry.wallet.account import Account def extract(d, keys): diff --git a/lbry/tests/integration/test_blockchain_reorganization.py b/lbry/tests/integration/test_blockchain_reorganization.py index 4f0d1587d..2205aa84f 100644 --- a/lbry/tests/integration/test_blockchain_reorganization.py +++ b/lbry/tests/integration/test_blockchain_reorganization.py @@ -1,5 +1,5 @@ import logging -from torba.testcase import IntegrationTestCase +from lbry.wallet.testcase import IntegrationTestCase class BlockchainReorganizationTests(IntegrationTestCase): diff --git a/lbry/tests/integration/test_claim_commands.py b/lbry/tests/integration/test_claim_commands.py index 59f7c1bf1..c70455822 100644 --- a/lbry/tests/integration/test_claim_commands.py +++ b/lbry/tests/integration/test_claim_commands.py @@ -5,7 +5,7 @@ from binascii import unhexlify from urllib.request import urlopen -from torba.client.errors import InsufficientFundsError +from lbry.wallet.client.errors import InsufficientFundsError from lbry.extras.daemon.Daemon import DEFAULT_PAGE_SIZE from lbry.testcase import CommandTestCase diff --git a/lbry/tests/integration/test_cli.py b/lbry/tests/integration/test_cli.py index d3f287995..7638711fa 100644 --- a/lbry/tests/integration/test_cli.py +++ b/lbry/tests/integration/test_cli.py @@ -1,6 +1,6 @@ import contextlib from io import StringIO -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase from lbry.conf import Config from lbry.extras import cli diff --git a/lbry/tests/integration/test_dht.py b/lbry/tests/integration/test_dht.py index 761a3e315..e6412de5c 100644 --- a/lbry/tests/integration/test_dht.py +++ b/lbry/tests/integration/test_dht.py @@ -7,7 +7,7 @@ from lbry.dht import constants from lbry.dht.node import Node from lbry.dht import peer as dht_peer from lbry.dht.peer import PeerManager, make_kademlia_peer -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase class DHTIntegrationTest(AsyncioTestCase): diff --git a/lbry/tests/integration/test_exchange_rate_manager.py b/lbry/tests/integration/test_exchange_rate_manager.py index b4d9af54c..d0c489967 100644 --- a/lbry/tests/integration/test_exchange_rate_manager.py +++ b/lbry/tests/integration/test_exchange_rate_manager.py @@ -1,5 +1,5 @@ from decimal import Decimal -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateManager diff --git a/lbry/tests/integration/test_file_commands.py b/lbry/tests/integration/test_file_commands.py index e558e57fe..cfd14ef0a 100644 --- a/lbry/tests/integration/test_file_commands.py +++ b/lbry/tests/integration/test_file_commands.py @@ -3,7 +3,6 @@ import os from binascii import hexlify from lbry.testcase import CommandTestCase -from lbry.blob_exchange.downloader import BlobDownloader class FileCommands(CommandTestCase): diff --git a/lbry/tests/integration/test_internal_transaction_api.py b/lbry/tests/integration/test_internal_transaction_api.py index 356d420bb..1f016f3b9 100644 --- a/lbry/tests/integration/test_internal_transaction_api.py +++ b/lbry/tests/integration/test_internal_transaction_api.py @@ -1,6 +1,6 @@ import asyncio -from torba.testcase import IntegrationTestCase +from lbry.wallet.testcase import IntegrationTestCase import lbry.wallet from lbry.schema.claim import Claim diff --git a/lbry/tests/integration/test_network.py b/lbry/tests/integration/test_network.py index ab288c48f..1e8bbdca9 100644 --- a/lbry/tests/integration/test_network.py +++ b/lbry/tests/integration/test_network.py @@ -1,12 +1,14 @@ import logging import os import asyncio + +import lbry from unittest.mock import Mock -from torba.client.basenetwork import BaseNetwork -from torba.orchstr8.node import SPVNode -from torba.rpc import RPCSession -from torba.testcase import IntegrationTestCase, AsyncioTestCase +from lbry.wallet.client.basenetwork import BaseNetwork +from lbry.wallet.orchstr8.node import SPVNode +from lbry.wallet.rpc import RPCSession +from lbry.wallet.testcase import IntegrationTestCase, AsyncioTestCase class NetworkTests(IntegrationTestCase): @@ -22,13 +24,13 @@ class NetworkTests(IntegrationTestCase): 'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH, 'hash_function': 'sha256', 'hosts': {}, - 'protocol_max': '1.4', - 'protocol_min': '1.1', + 'protocol_max': '1.0', + 'protocol_min': lbry.__version__, 'pruning': None, 'description': '', 'payment_address': '', 'daily_fee': 0, - 'server_version': '0.5.7'}, await self.ledger.network.get_server_features()) + 'server_version': lbry.__version__}, await self.ledger.network.get_server_features()) await self.conductor.spv_node.stop() address = (await self.account.get_addresses(limit=1))[0] os.environ.update({ @@ -41,13 +43,13 @@ class NetworkTests(IntegrationTestCase): 'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH, 'hash_function': 'sha256', 'hosts': {}, - 'protocol_max': '1.4', - 'protocol_min': '1.1', + 'protocol_max': '1.0', + 'protocol_min': lbry.__version__, 'pruning': None, 'description': 'Fastest server in the west.', 'payment_address': address, 'daily_fee': 42, - 'server_version': '0.5.7'}, await self.ledger.network.get_server_features()) + 'server_version': lbry.__version__}, await self.ledger.network.get_server_features()) class ReconnectTests(IntegrationTestCase): diff --git a/lbry/tests/integration/test_resolve_command.py b/lbry/tests/integration/test_resolve_command.py index fe5d6f80a..a9f27bc8e 100644 --- a/lbry/tests/integration/test_resolve_command.py +++ b/lbry/tests/integration/test_resolve_command.py @@ -1,12 +1,11 @@ import asyncio import json import hashlib -from unittest import skip from binascii import hexlify, unhexlify from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction, Output from lbry.schema.compat import OldClaimMessage -from torba.client.hash import sha256, Base58 +from lbry.wallet.client.hash import sha256, Base58 class BaseResolveTestCase(CommandTestCase): diff --git a/lbry/tests/integration/test_sync.py b/lbry/tests/integration/test_sync.py index 6e5c56381..1ba24ac96 100644 --- a/lbry/tests/integration/test_sync.py +++ b/lbry/tests/integration/test_sync.py @@ -1,7 +1,7 @@ import asyncio import logging -from torba.testcase import IntegrationTestCase, WalletNode -from torba.client.constants import CENT +from lbry.testcase import IntegrationTestCase, WalletNode +from lbry.constants import CENT class SyncTests(IntegrationTestCase): diff --git a/lbry/tests/integration/test_transactions.py b/lbry/tests/integration/test_transactions.py index 00730ccca..be96ae9be 100644 --- a/lbry/tests/integration/test_transactions.py +++ b/lbry/tests/integration/test_transactions.py @@ -2,16 +2,14 @@ import logging import asyncio import random from itertools import chain -from random import shuffle -from torba.testcase import IntegrationTestCase -from torba.client.util import satoshis_to_coins, coins_to_satoshis +from lbry.wallet.transaction import Transaction, Output, Input +from lbry.wallet.testcase import IntegrationTestCase +from lbry.wallet.client.util import satoshis_to_coins, coins_to_satoshis class BasicTransactionTests(IntegrationTestCase): - VERBOSITY = logging.WARN - async def test_variety_of_transactions_and_longish_history(self): await self.blockchain.generate(300) await self.assertBalance(self.account, '0.0') @@ -41,9 +39,9 @@ class BasicTransactionTests(IntegrationTestCase): # spend from each of the first 10 addresses to the subsequent 10 addresses txs = [] for address in addresses[10:20]: - txs.append(await self.ledger.transaction_class.create( + txs.append(await Transaction.create( [], - [self.ledger.transaction_class.output_class.pay_pubkey_hash( + [Output.pay_pubkey_hash( coins_to_satoshis('1.0'), self.ledger.address_to_hash160(address) )], [self.account], self.account @@ -66,9 +64,9 @@ class BasicTransactionTests(IntegrationTestCase): self.assertEqual(30, await self.account.get_utxo_count()) # spend all 30 UTXOs into a a 199 coin UTXO and change - tx = await self.ledger.transaction_class.create( + tx = await Transaction.create( [], - [self.ledger.transaction_class.output_class.pay_pubkey_hash( + [Output.pay_pubkey_hash( coins_to_satoshis('199.0'), self.ledger.address_to_hash160(addresses[-1]) )], [self.account], self.account @@ -99,9 +97,9 @@ class BasicTransactionTests(IntegrationTestCase): await self.assertBalance(account2, '0.0') address2 = await account2.receiving.get_or_create_usable_address() - tx = await self.ledger.transaction_class.create( + tx = await Transaction.create( [], - [self.ledger.transaction_class.output_class.pay_pubkey_hash( + [Output.pay_pubkey_hash( coins_to_satoshis('2.0'), self.ledger.address_to_hash160(address2) )], [account1], account1 @@ -115,8 +113,8 @@ class BasicTransactionTests(IntegrationTestCase): await self.assertBalance(account2, '2.0') utxos = await self.account.get_utxos() - tx = await self.ledger.transaction_class.create( - [self.ledger.transaction_class.input_class.spend(utxos[0])], + tx = await Transaction.create( + [Input.spend(utxos[0])], [], [account1], account1 ) @@ -159,8 +157,8 @@ class BasicTransactionTests(IntegrationTestCase): utxos = await self.account.get_utxos() txs = [] for utxo in utxos: - tx = await self.ledger.transaction_class.create( - [self.ledger.transaction_class.input_class.spend(utxo)], + tx = await Transaction.create( + [Input.spend(utxo)], [], [self.account], self.account ) diff --git a/lbry/tests/integration/test_wallet_commands.py b/lbry/tests/integration/test_wallet_commands.py index b39bb549a..a34197d84 100644 --- a/lbry/tests/integration/test_wallet_commands.py +++ b/lbry/tests/integration/test_wallet_commands.py @@ -1,8 +1,8 @@ import asyncio import json -from torba.client.wallet import ENCRYPT_ON_DISK -from torba.client.errors import InvalidPasswordError +from lbry.wallet.client.wallet import ENCRYPT_ON_DISK +from lbry.error import InvalidPasswordError from lbry.testcase import CommandTestCase from lbry.wallet.dewies import dict_values_to_lbc diff --git a/lbry/tests/integration/test_wallet_server_sessions.py b/lbry/tests/integration/test_wallet_server_sessions.py index e7ed17702..664fa025e 100644 --- a/lbry/tests/integration/test_wallet_server_sessions.py +++ b/lbry/tests/integration/test_wallet_server_sessions.py @@ -1,9 +1,9 @@ import asyncio +import lbry import lbry.wallet -from lbry import __version__ as sdk_version -from torba.client.basenetwork import ClientSession -from torba.testcase import IntegrationTestCase +from lbry.wallet.client.basenetwork import ClientSession +from lbry.testcase import IntegrationTestCase class TestSessions(IntegrationTestCase): @@ -33,7 +33,7 @@ class TestSessions(IntegrationTestCase): async def test_proper_version(self): info = await self.ledger.network.get_server_features() - self.assertEqual(sdk_version, info['server_version']) + self.assertEqual(lbry.__version__, info['server_version']) async def test_client_errors(self): # Goal is ensuring thsoe are raised and not trapped accidentally diff --git a/lbry/tests/unit/wallet/server/test_sqldb.py b/lbry/tests/unit/wallet/server/test_sqldb.py index c5f3f29c3..e7b103b55 100644 --- a/lbry/tests/unit/wallet/server/test_sqldb.py +++ b/lbry/tests/unit/wallet/server/test_sqldb.py @@ -3,7 +3,7 @@ import ecdsa import hashlib import logging from binascii import hexlify -from torba.client.constants import COIN, NULL_HASH32 +from lbry.wallet.client.constants import COIN, NULL_HASH32 from lbry.schema.claim import Claim from lbry.wallet.server.db import reader, writer diff --git a/lbry/tests/unit/wallet/test_account.py b/lbry/tests/unit/wallet/test_account.py index 065d43ab9..1c6c243d6 100644 --- a/lbry/tests/unit/wallet/test_account.py +++ b/lbry/tests/unit/wallet/test_account.py @@ -1,9 +1,10 @@ -from torba.testcase import AsyncioTestCase -from torba.client.wallet import Wallet - +from binascii import hexlify +from lbry.wallet.testcase import AsyncioTestCase +from lbry.wallet.client.wallet import Wallet from lbry.wallet.ledger import MainNetLedger, WalletDatabase from lbry.wallet.header import Headers from lbry.wallet.account import Account +from lbry.wallet.client.baseaccount import SingleKey, HierarchicalDeterministic class TestAccount(AsyncioTestCase): @@ -41,13 +42,14 @@ class TestAccount(AsyncioTestCase): self.assertEqual(len(addresses), 6) async def test_generate_keys_over_batch_threshold_saves_it_properly(self): - async with self.account.receiving.address_generator_lock: - await self.account.receiving._generate_keys(0, 200) - records = await self.account.receiving.get_address_records() + account = Account.generate(self.ledger, Wallet(), 'lbryum') + async with account.receiving.address_generator_lock: + await account.receiving._generate_keys(0, 200) + records = await account.receiving.get_address_records() self.assertEqual(len(records), 201) async def test_ensure_address_gap(self): - account = self.account + account = Account.generate(self.ledger, Wallet(), 'lbryum') self.assertIsInstance(account.receiving, HierarchicalDeterministic) @@ -86,7 +88,7 @@ class TestAccount(AsyncioTestCase): self.assertEqual(len(new_keys), 20) async def test_get_or_create_usable_address(self): - account = self.account + account = Account.generate(self.ledger, Wallet(), 'lbryum') keys = await account.receiving.get_addresses() self.assertEqual(len(keys), 0) @@ -158,9 +160,9 @@ class TestAccount(AsyncioTestCase): await account.ensure_address_gap() addresses = await account.receiving.get_addresses() - self.assertEqual(len(addresses), 5) + self.assertEqual(len(addresses), 17) addresses = await account.change.get_addresses() - self.assertEqual(len(addresses), 5) + self.assertEqual(len(addresses), 10) account_data['ledger'] = 'lbc_mainnet' self.assertDictEqual(account_data, account.to_dict()) @@ -202,7 +204,7 @@ class TestAccount(AsyncioTestCase): 'change': {'gap': 5, 'maximum_uses_per_address': 2} } } - account = self.ledger.account_class.from_dict(self.ledger, Wallet(), account_data) + account = Account.from_dict(self.ledger, Wallet(), account_data) self.assertEqual(account.name, 'My Account') self.assertEqual(account.modified_on, 123.456) @@ -234,13 +236,12 @@ class TestAccount(AsyncioTestCase): class TestSingleKeyAccount(AsyncioTestCase): async def asyncSetUp(self): - self.ledger = ledger_class({ - 'db': ledger_class.database_class(':memory:'), - 'headers': ledger_class.headers_class(':memory:'), + self.ledger = MainNetLedger({ + 'db': WalletDatabase(':memory:'), + 'headers': Headers(':memory:') }) await self.ledger.db.open() - self.account = self.ledger.account_class.generate( - self.ledger, Wallet(), "torba", {'name': 'single-address'}) + self.account = Account.generate(self.ledger, Wallet(), "torba", {'name': 'single-address'}) async def asyncTearDown(self): await self.ledger.db.close() @@ -336,13 +337,13 @@ class TestSingleKeyAccount(AsyncioTestCase): ) self.assertEqual( account.private_key.extended_key_string(), - 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp' - '5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', + 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' + 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', ) self.assertEqual( account.public_key.extended_key_string(), - 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' - 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', + 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EM' + 'mDgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', ) address = await account.receiving.ensure_address_gap() self.assertEqual(address[0], account.public_key.address) @@ -352,8 +353,8 @@ class TestSingleKeyAccount(AsyncioTestCase): ) self.assertEqual( private_key.extended_key_string(), - 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp' - '5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', + 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' + 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', ) invalid_key = await self.ledger.get_private_key_for_address( @@ -363,7 +364,7 @@ class TestSingleKeyAccount(AsyncioTestCase): self.assertEqual( hexlify(private_key.wif()), - b'1c92caa0ef99bfd5e2ceb73b66da8cd726a9370be8c368d448a322f3c5b23aaab901' + b'1cef6c80310b1bcbcfa3176ea809ac840f48cda634c475d402e6bd68d5bb3827d601' ) async def test_load_and_save_account(self): @@ -374,16 +375,15 @@ class TestSingleKeyAccount(AsyncioTestCase): "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, - 'private_key': - 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp' - '5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', - 'public_key': - 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' - 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', - 'address_generator': {'name': 'single-address'} + 'private_key': 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' + 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', + 'public_key': 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EM' + 'mDgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', + 'address_generator': {'name': 'single-address'}, + 'certificates': {} } - account = self.ledger.account_class.from_dict(self.ledger, Wallet(), account_data) + account = Account.from_dict(self.ledger, Wallet(), account_data) await account.ensure_address_gap() @@ -393,7 +393,7 @@ class TestSingleKeyAccount(AsyncioTestCase): self.assertEqual(len(addresses), 1) self.maxDiff = None - account_data['ledger'] = 'btc_mainnet' + account_data['ledger'] = 'lbc_mainnet' self.assertDictEqual(account_data, account.to_dict()) @@ -407,8 +407,8 @@ class AccountEncryptionTests(AsyncioTestCase): "h absent", 'encrypted': False, 'private_key': - 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp' - '5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', + 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7DRNLEo' + 'B8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', @@ -422,9 +422,9 @@ class AccountEncryptionTests(AsyncioTestCase): "bIkZ//trah9AIkmrc/ZvNkC0Q==", 'encrypted': True, 'private_key': - 'MDAwMDAwMDAwMDAwMDAwMLkWikOLScA/ZxlFSGU7dl//7Q/1gS9h7vqQyrd8DX+' - 'jwcp7SwlJ1mkMwuraUaWLq9/LxiaGmqJBUZ50p77YVZbDycaCN1unBr1/i1q6RP' - 'Ob2MNCaG8nyjxZhQai+V/2JmJ+UnFMp3nHany7F8/Hr0g=', + 'MDAwMDAwMDAwMDAwMDAwMLkWikOLScA/ZxlFSGU7dl8pqVjgdpu1S3MWQF3IJ5H' + 'OXPAQcgnhHldVq98uP7Q8JqSWOv1p4gpxGSYnA4w5Gbuh0aUD4hmV70m7nVTj7T' + '15+Pu30DCspndru59pee/S+mShoK68q7t7r32leaVIfzw=', 'public_key': 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6mKUMJFc7' 'UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', @@ -432,13 +432,13 @@ class AccountEncryptionTests(AsyncioTestCase): } async def asyncSetUp(self): - self.ledger = ledger_class({ - 'db': ledger_class.database_class(':memory:'), - 'headers': ledger_class.headers_class(':memory:'), + self.ledger = MainNetLedger({ + 'db': WalletDatabase(':memory:'), + 'headers': Headers(':memory:') }) def test_encrypt_wallet(self): - account = self.ledger.account_class.from_dict(self.ledger, Wallet(), self.unencrypted_account) + account = Account.from_dict(self.ledger, Wallet(), self.unencrypted_account) account.init_vectors = { 'seed': self.init_vector, 'private_key': self.init_vector @@ -468,7 +468,7 @@ class AccountEncryptionTests(AsyncioTestCase): self.assertFalse(account.encrypted) def test_decrypt_wallet(self): - account = self.ledger.account_class.from_dict(self.ledger, Wallet(), self.encrypted_account) + account = Account.from_dict(self.ledger, Wallet(), self.encrypted_account) self.assertTrue(account.encrypted) account.decrypt(self.password) diff --git a/lbry/tests/unit/wallet/test_bcd_data_stream.py b/lbry/tests/unit/wallet/test_bcd_data_stream.py index 846c5dfde..5e38c0b3d 100644 --- a/lbry/tests/unit/wallet/test_bcd_data_stream.py +++ b/lbry/tests/unit/wallet/test_bcd_data_stream.py @@ -1,6 +1,6 @@ import unittest -from torba.client.bcd_data_stream import BCDataStream +from lbry.wallet.client.bcd_data_stream import BCDataStream class TestBCDataStream(unittest.TestCase): diff --git a/lbry/tests/unit/wallet/test_bip32.py b/lbry/tests/unit/wallet/test_bip32.py index 0547d730a..55a72577c 100644 --- a/lbry/tests/unit/wallet/test_bip32.py +++ b/lbry/tests/unit/wallet/test_bip32.py @@ -1,10 +1,10 @@ from binascii import unhexlify, hexlify -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase -from client_tests.unit.key_fixtures import expected_ids, expected_privkeys, expected_hardened_privkeys -from torba.client.bip32 import PubKey, PrivateKey, from_extended_key_string -from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class +from tests.unit.wallet.key_fixtures import expected_ids, expected_privkeys, expected_hardened_privkeys +from lbry.wallet.client.bip32 import PubKey, PrivateKey, from_extended_key_string +from lbry.wallet import MainNetLedger as ledger_class class BIP32Tests(AsyncioTestCase): @@ -60,7 +60,7 @@ class BIP32Tests(AsyncioTestCase): self.assertEqual( ec_point[1], 86198965946979720220333266272536217633917099472454294641561154971209433250106 ) - self.assertEqual(private_key.address(), '1GVM5dEhThbiyCZ9gqBZBv6p9whga7MTXo' ) + self.assertEqual('bUDcmraBp2zCV3QWmVVeQaEgepbs1b2gC9', private_key.address()) with self.assertRaisesRegex(ValueError, 'invalid BIP32 private key child number'): private_key.child(-1) self.assertIsInstance(private_key.child(PrivateKey.HARDENED), PrivateKey) diff --git a/lbry/tests/unit/wallet/test_claim_proofs.py b/lbry/tests/unit/wallet/test_claim_proofs.py index 4485e41be..f8fcca887 100644 --- a/lbry/tests/unit/wallet/test_claim_proofs.py +++ b/lbry/tests/unit/wallet/test_claim_proofs.py @@ -2,7 +2,7 @@ import unittest from binascii import hexlify, unhexlify from lbry.wallet.claim_proofs import get_hash_for_outpoint, verify_proof -from torba.client.hash import double_sha256 +from lbry.wallet.client.hash import double_sha256 class ClaimProofsTestCase(unittest.TestCase): diff --git a/lbry/tests/unit/wallet/test_coinselection.py b/lbry/tests/unit/wallet/test_coinselection.py index 41691a42b..31874ea20 100644 --- a/lbry/tests/unit/wallet/test_coinselection.py +++ b/lbry/tests/unit/wallet/test_coinselection.py @@ -1,12 +1,12 @@ from types import GeneratorType -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase -from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class -from torba.client.coinselection import CoinSelector, MAXIMUM_TRIES -from torba.client.constants import CENT +from lbry.wallet import MainNetLedger as ledger_class +from lbry.wallet.client.coinselection import CoinSelector, MAXIMUM_TRIES +from lbry.wallet.client.constants import CENT -from client_tests.unit.test_transaction import get_output as utxo +from tests.unit.wallet.test_transaction import get_output as utxo NULL_HASH = b'\x00'*32 diff --git a/lbry/tests/unit/wallet/test_database.py b/lbry/tests/unit/wallet/test_database.py index 69a7082b7..85bf41b50 100644 --- a/lbry/tests/unit/wallet/test_database.py +++ b/lbry/tests/unit/wallet/test_database.py @@ -6,15 +6,15 @@ import tempfile import asyncio from concurrent.futures.thread import ThreadPoolExecutor -from torba.client.wallet import Wallet -from torba.client.constants import COIN -from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class -from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite -from torba.client.hash import sha256 +from lbry.wallet import MainNetLedger +from lbry.wallet.transaction import Transaction +from lbry.wallet.client.wallet import Wallet +from lbry.wallet.client.constants import COIN +from lbry.wallet.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite +from lbry.wallet.client.hash import sha256 +from lbry.wallet.testcase import AsyncioTestCase -from torba.testcase import AsyncioTestCase - -from client_tests.unit.test_transaction import get_output, NULL_HASH +from tests.unit.wallet.test_transaction import get_output, NULL_HASH class TestAIOSQLite(AsyncioTestCase): @@ -195,9 +195,9 @@ class TestQueryBuilder(unittest.TestCase): class TestQueries(AsyncioTestCase): async def asyncSetUp(self): - self.ledger = ledger_class({ - 'db': ledger_class.database_class(':memory:'), - 'headers': ledger_class.headers_class(':memory:'), + self.ledger = MainNetLedger({ + 'db': MainNetLedger.database_class(':memory:'), + 'headers': MainNetLedger.headers_class(':memory:') }) self.wallet = Wallet() await self.ledger.db.open() @@ -212,8 +212,8 @@ class TestQueries(AsyncioTestCase): async def create_tx_from_nothing(self, my_account, height): to_address = await my_account.receiving.get_or_create_usable_address() - to_hash = ledger_class.address_to_hash160(to_address) - tx = ledger_class.transaction_class(height=height, is_verified=True) \ + to_hash = MainNetLedger.address_to_hash160(to_address) + tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(self.txo(1, sha256(str(height).encode())))]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) @@ -224,8 +224,8 @@ class TestQueries(AsyncioTestCase): from_hash = txo.script.values['pubkey_hash'] from_address = self.ledger.hash160_to_address(from_hash) to_address = await to_account.receiving.get_or_create_usable_address() - to_hash = ledger_class.address_to_hash160(to_address) - tx = ledger_class.transaction_class(height=height, is_verified=True) \ + to_hash = MainNetLedger.address_to_hash160(to_address) + tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(txo)]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) @@ -237,7 +237,7 @@ class TestQueries(AsyncioTestCase): from_hash = txo.script.values['pubkey_hash'] from_address = self.ledger.hash160_to_address(from_hash) to_hash = NULL_HASH - tx = ledger_class.transaction_class(height=height, is_verified=True) \ + tx = Transaction(height=height, is_verified=True) \ .add_inputs([self.txi(txo)]) \ .add_outputs([self.txo(1, to_hash)]) await self.ledger.db.insert_transaction(tx) @@ -248,7 +248,7 @@ class TestQueries(AsyncioTestCase): return get_output(int(amount*COIN), address) def txi(self, txo): - return ledger_class.transaction_class.input_class.spend(txo) + return Transaction.input_class.spend(txo) async def test_large_tx_doesnt_hit_variable_limits(self): # SQLite is usually compiled with 999 variables limit: https://www.sqlite.org/limits.html @@ -408,9 +408,9 @@ class TestUpgrade(AsyncioTestCase): return [col[0] for col in conn.execute(sql).fetchall()] async def test_reset_on_version_change(self): - self.ledger = ledger_class({ - 'db': ledger_class.database_class(self.path), - 'headers': ledger_class.headers_class(':memory:'), + self.ledger = MainNetLedger({ + 'db': MainNetLedger.database_class(self.path), + 'headers': MainNetLedger.headers_class(':memory:') }) # initial open, pre-version enabled db diff --git a/lbry/tests/unit/wallet/test_hash.py b/lbry/tests/unit/wallet/test_hash.py index 0ec74c2b4..30239a33f 100644 --- a/lbry/tests/unit/wallet/test_hash.py +++ b/lbry/tests/unit/wallet/test_hash.py @@ -1,6 +1,6 @@ from unittest import TestCase, mock -from torba.client.hash import aes_decrypt, aes_encrypt, better_aes_decrypt, better_aes_encrypt -from torba.client.errors import InvalidPasswordError +from lbry.wallet.client.hash import aes_decrypt, aes_encrypt, better_aes_decrypt, better_aes_encrypt +from lbry.wallet.client.errors import InvalidPasswordError class TestAESEncryptDecrypt(TestCase): diff --git a/lbry/tests/unit/wallet/test_headers.py b/lbry/tests/unit/wallet/test_headers.py index 57358421a..63c7233f0 100644 --- a/lbry/tests/unit/wallet/test_headers.py +++ b/lbry/tests/unit/wallet/test_headers.py @@ -1,169 +1,16 @@ -import asyncio import os +import asyncio import tempfile -from binascii import hexlify - -from torba.client.hash import sha256 -from torba.testcase import AsyncioTestCase - -from torba.coin.bitcoinsegwit import MainHeaders -from binascii import unhexlify - -from torba.testcase import AsyncioTestCase -from torba.client.util import ArithUint256 +from binascii import hexlify, unhexlify +from lbry.wallet.client.hash import sha256 +from lbry.wallet.client.util import ArithUint256 +from lbry.wallet.testcase import AsyncioTestCase from lbry.wallet.ledger import Headers def block_bytes(blocks): - return blocks * MainHeaders.header_size - - -class BitcoinHeadersTestCase(AsyncioTestCase): - HEADER_FILE = 'bitcoin_headers' - RETARGET_BLOCK = 32256 # difficulty: 1 -> 1.18 - - def setUp(self): - self.maxDiff = None - self.header_file_name = os.path.join(os.path.dirname(__file__), self.HEADER_FILE) - - def get_bytes(self, upto: int = -1, after: int = 0) -> bytes: - with open(self.header_file_name, 'rb') as headers: - headers.seek(after, os.SEEK_SET) - return headers.read(upto) - - async def get_headers(self, upto: int = -1): - h = MainHeaders(':memory:') - h.io.write(self.get_bytes(upto)) - return h - - -class BasicHeadersTests(BitcoinHeadersTestCase): - - async def test_serialization(self): - h = await self.get_headers() - self.assertDictEqual(h[0], { - 'bits': 486604799, - 'block_height': 0, - 'merkle_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', - 'nonce': 2083236893, - 'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000', - 'timestamp': 1231006505, - 'version': 1 - }) - self.assertDictEqual(h[self.RETARGET_BLOCK-1], { - 'bits': 486604799, - 'block_height': 32255, - 'merkle_root': b'89b4f223789e40b5b475af6483bb05bceda54059e17d2053334b358f6bb310ac', - 'nonce': 312762301, - 'prev_block_hash': b'000000006baebaa74cecde6c6787c26ee0a616a3c333261bff36653babdac149', - 'timestamp': 1262152739, - 'version': 1 - }) - self.assertDictEqual(h[self.RETARGET_BLOCK], { - 'bits': 486594666, - 'block_height': 32256, - 'merkle_root': b'64b5e5f5a262f47af443a0120609206a3305877693edfe03e994f20a024ab627', - 'nonce': 121087187, - 'prev_block_hash': b'00000000984f962134a7291e3693075ae03e521f0ee33378ec30a334d860034b', - 'timestamp': 1262153464, - 'version': 1 - }) - self.assertDictEqual(h[self.RETARGET_BLOCK+1], { - 'bits': 486594666, - 'block_height': 32257, - 'merkle_root': b'4d1488981f08b3037878193297dbac701a2054e0f803d4424fe6a4d763d62334', - 'nonce': 274675219, - 'prev_block_hash': b'000000004f2886a170adb7204cb0c7a824217dd24d11a74423d564c4e0904967', - 'timestamp': 1262154352, - 'version': 1 - }) - self.assertEqual( - h.serialize(h[0]), - h.get_raw_header(0) - ) - self.assertEqual( - h.serialize(h[self.RETARGET_BLOCK]), - h.get_raw_header(self.RETARGET_BLOCK) - ) - - async def test_connect_from_genesis_to_3000_past_first_chunk_at_2016(self): - headers = MainHeaders(':memory:') - self.assertEqual(headers.height, -1) - await headers.connect(0, self.get_bytes(block_bytes(3001))) - self.assertEqual(headers.height, 3000) - - async def test_connect_9_blocks_passing_a_retarget_at_32256(self): - retarget = block_bytes(self.RETARGET_BLOCK-5) - headers = await self.get_headers(upto=retarget) - remainder = self.get_bytes(after=retarget) - self.assertEqual(headers.height, 32250) - await headers.connect(len(headers), remainder) - self.assertEqual(headers.height, 32259) - - async def test_bounds(self): - headers = MainHeaders(':memory:') - await headers.connect(0, self.get_bytes(block_bytes(3001))) - self.assertEqual(headers.height, 3000) - with self.assertRaises(IndexError): - _ = headers[3001] - with self.assertRaises(IndexError): - _ = headers[-1] - self.assertIsNotNone(headers[3000]) - self.assertIsNotNone(headers[0]) - - async def test_repair(self): - headers = MainHeaders(':memory:') - await headers.connect(0, self.get_bytes(block_bytes(3001))) - self.assertEqual(headers.height, 3000) - await headers.repair() - self.assertEqual(headers.height, 3000) - # corrupt the middle of it - headers.io.seek(block_bytes(1500)) - headers.io.write(b"wtf") - await headers.repair() - self.assertEqual(headers.height, 1499) - self.assertEqual(len(headers), 1500) - # corrupt by appending - headers.io.seek(block_bytes(len(headers))) - headers.io.write(b"appending") - await headers.repair() - self.assertEqual(headers.height, 1499) - await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500))) - self.assertEqual(headers.height, 3000) - - async def test_checkpointed_writer(self): - headers = MainHeaders(':memory:') - headers.checkpoint = 100, hexlify(sha256(self.get_bytes(block_bytes(100)))) - genblocks = lambda start, end: self.get_bytes(block_bytes(end - start), block_bytes(start)) - async with headers.checkpointed_connector() as buff: - buff.write(genblocks(0, 10)) - self.assertEqual(len(headers), 10) - async with headers.checkpointed_connector() as buff: - buff.write(genblocks(10, 100)) - self.assertEqual(len(headers), 100) - headers = MainHeaders(':memory:') - async with headers.checkpointed_connector() as buff: - buff.write(genblocks(0, 300)) - self.assertEqual(len(headers), 300) - - async def test_concurrency(self): - BLOCKS = 30 - headers_temporary_file = tempfile.mktemp() - headers = MainHeaders(headers_temporary_file) - await headers.open() - self.addCleanup(os.remove, headers_temporary_file) - async def writer(): - for block_index in range(BLOCKS): - await headers.connect(block_index, self.get_bytes(block_bytes(block_index + 1), block_bytes(block_index))) - async def reader(): - for block_index in range(BLOCKS): - while len(headers) < block_index: - await asyncio.sleep(0.000001) - assert headers[block_index]['block_height'] == block_index - reader_task = asyncio.create_task(reader()) - await writer() - await reader_task + return blocks * Headers.header_size class TestHeaders(AsyncioTestCase): @@ -201,9 +48,9 @@ class TestHeaders(AsyncioTestCase): async def test_connect_from_middle(self): h = Headers(':memory:') - h.io.write(HEADERS[:10*Headers.header_size]) + h.io.write(HEADERS[:block_bytes(10)]) self.assertEqual(h.height, 9) - await h.connect(len(h), HEADERS[10*Headers.header_size:20*Headers.header_size]) + await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)]) self.assertEqual(h.height, 19) def test_target_calculation(self): @@ -260,6 +107,70 @@ class TestHeaders(AsyncioTestCase): b"74044747b7c1ff867eb09a84d026b02d8dc539fb6adcec3536f3dfa9266495d9" ) + async def test_bounds(self): + headers = Headers(':memory:') + await headers.connect(0, HEADERS) + self.assertEqual(19, headers.height) + with self.assertRaises(IndexError): + _ = headers[3001] + with self.assertRaises(IndexError): + _ = headers[-1] + self.assertIsNotNone(headers[19]) + self.assertIsNotNone(headers[0]) + + async def test_repair(self): + headers = Headers(':memory:') + await headers.connect(0, HEADERS[:block_bytes(11)]) + self.assertEqual(10, headers.height) + await headers.repair() + self.assertEqual(10, headers.height) + # corrupt the middle of it + headers.io.seek(block_bytes(8)) + headers.io.write(b"wtf") + await headers.repair() + self.assertEqual(7, headers.height) + self.assertEqual(8, len(headers)) + # corrupt by appending + headers.io.seek(block_bytes(len(headers))) + headers.io.write(b"appending") + await headers.repair() + self.assertEqual(7, headers.height) + await headers.connect(len(headers), HEADERS[block_bytes(8):]) + self.assertEqual(19, headers.height) + + async def test_checkpointed_writer(self): + headers = Headers(':memory:') + getblocks = lambda start, end: HEADERS[block_bytes(start):block_bytes(end)] + headers.checkpoint = 10, hexlify(sha256(getblocks(10, 11))) + async with headers.checkpointed_connector() as buff: + buff.write(getblocks(0, 10)) + self.assertEqual(len(headers), 10) + async with headers.checkpointed_connector() as buff: + buff.write(getblocks(10, 19)) + self.assertEqual(len(headers), 19) + headers = Headers(':memory:') + async with headers.checkpointed_connector() as buff: + buff.write(getblocks(0, 19)) + self.assertEqual(len(headers), 19) + + async def test_concurrency(self): + BLOCKS = 19 + headers_temporary_file = tempfile.mktemp() + headers = Headers(headers_temporary_file) + await headers.open() + self.addCleanup(os.remove, headers_temporary_file) + async def writer(): + for block_index in range(BLOCKS): + await headers.connect(block_index, HEADERS[block_bytes(block_index):block_bytes(block_index + 1)]) + async def reader(): + for block_index in range(BLOCKS): + while len(headers) < block_index: + await asyncio.sleep(0.000001) + assert headers[block_index]['block_height'] == block_index + reader_task = asyncio.create_task(reader()) + await writer() + await reader_task + HEADERS = unhexlify( b'010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e4' diff --git a/lbry/tests/unit/wallet/test_ledger.py b/lbry/tests/unit/wallet/test_ledger.py index 47fb9edd0..a764fff8a 100644 --- a/lbry/tests/unit/wallet/test_ledger.py +++ b/lbry/tests/unit/wallet/test_ledger.py @@ -1,18 +1,15 @@ import os from binascii import hexlify -from torba.coin.bitcoinsegwit import MainNetLedger -from torba.client.wallet import Wallet - -from client_tests.unit.test_transaction import get_transaction, get_output -from client_tests.unit.test_headers import BitcoinHeadersTestCase, block_bytes -from torba.testcase import AsyncioTestCase -from torba.client.wallet import Wallet - +from lbry.wallet.testcase import AsyncioTestCase +from lbry.wallet.client.wallet import Wallet from lbry.wallet.account import Account from lbry.wallet.transaction import Transaction, Output, Input from lbry.wallet.ledger import MainNetLedger +from tests.unit.wallet.test_transaction import get_transaction, get_output +from tests.unit.wallet.test_headers import HEADERS, block_bytes + class MockNetwork: @@ -61,6 +58,7 @@ class LedgerTestCase(AsyncioTestCase): 'nonce': 2083236893, 'prev_block_hash': b'0000000000000000000000000000000000000000000000000000000000000000', 'timestamp': 1231006505, + 'claim_trie_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', 'version': 1 } header.update(kwargs) @@ -144,37 +142,37 @@ class BlockchainReorganizationTests(LedgerTestCase): async def test_1_block_reorganization(self): self.ledger.network = MocHeaderNetwork({ - 20: {'height': 20, 'count': 5, 'hex': hexlify( - self.get_bytes(after=block_bytes(20), upto=block_bytes(5)) + 10: {'height': 10, 'count': 5, 'hex': hexlify( + HEADERS[block_bytes(10):block_bytes(15)] )}, - 25: {'height': 25, 'count': 0, 'hex': b''} + 15: {'height': 15, 'count': 0, 'hex': b''} }) headers = self.ledger.headers - await headers.connect(0, self.get_bytes(upto=block_bytes(20))) + await headers.connect(0, HEADERS[:block_bytes(10)]) self.add_header(block_height=len(headers)) - self.assertEqual(headers.height, 20) + self.assertEqual(10, headers.height) await self.ledger.receive_header([{ - 'height': 21, 'hex': hexlify(self.make_header(block_height=21)) + 'height': 11, 'hex': hexlify(self.make_header(block_height=11)) }]) async def test_3_block_reorganization(self): self.ledger.network = MocHeaderNetwork({ - 20: {'height': 20, 'count': 5, 'hex': hexlify( - self.get_bytes(after=block_bytes(20), upto=block_bytes(5)) + 10: {'height': 10, 'count': 5, 'hex': hexlify( + HEADERS[block_bytes(10):block_bytes(15)] )}, - 21: {'height': 21, 'count': 1, 'hex': hexlify(self.make_header(block_height=21))}, - 22: {'height': 22, 'count': 1, 'hex': hexlify(self.make_header(block_height=22))}, - 25: {'height': 25, 'count': 0, 'hex': b''} + 11: {'height': 11, 'count': 1, 'hex': hexlify(self.make_header(block_height=11))}, + 12: {'height': 12, 'count': 1, 'hex': hexlify(self.make_header(block_height=12))}, + 15: {'height': 15, 'count': 0, 'hex': b''} }) headers = self.ledger.headers - await headers.connect(0, self.get_bytes(upto=block_bytes(20))) + await headers.connect(0, HEADERS[:block_bytes(10)]) self.add_header(block_height=len(headers)) self.add_header(block_height=len(headers)) self.add_header(block_height=len(headers)) - self.assertEqual(headers.height, 22) - await self.ledger.receive_header(({ - 'height': 23, 'hex': hexlify(self.make_header(block_height=23)) - },)) + self.assertEqual(headers.height, 12) + await self.ledger.receive_header([{ + 'height': 13, 'hex': hexlify(self.make_header(block_height=13)) + }]) class BasicAccountingTests(LedgerTestCase): diff --git a/lbry/tests/unit/wallet/test_mnemonic.py b/lbry/tests/unit/wallet/test_mnemonic.py index e700a7582..26c640a96 100644 --- a/lbry/tests/unit/wallet/test_mnemonic.py +++ b/lbry/tests/unit/wallet/test_mnemonic.py @@ -1,7 +1,7 @@ import unittest from binascii import hexlify -from torba.client.mnemonic import Mnemonic +from lbry.wallet.client.mnemonic import Mnemonic class TestMnemonic(unittest.TestCase): diff --git a/lbry/tests/unit/wallet/test_schema_signing.py b/lbry/tests/unit/wallet/test_schema_signing.py index 3d4cd5af4..e5a8d3026 100644 --- a/lbry/tests/unit/wallet/test_schema_signing.py +++ b/lbry/tests/unit/wallet/test_schema_signing.py @@ -1,9 +1,7 @@ from binascii import unhexlify -from cryptography.exceptions import InvalidSignature - -from torba.testcase import AsyncioTestCase -from torba.client.constants import CENT, NULL_HASH32 +from lbry.wallet.testcase import AsyncioTestCase +from lbry.wallet.client.constants import CENT, NULL_HASH32 from lbry.wallet.ledger import MainNetLedger from lbry.wallet.transaction import Transaction, Input, Output diff --git a/lbry/tests/unit/wallet/test_script.py b/lbry/tests/unit/wallet/test_script.py index 0bf449ca3..c0956ae0f 100644 --- a/lbry/tests/unit/wallet/test_script.py +++ b/lbry/tests/unit/wallet/test_script.py @@ -1,14 +1,11 @@ -import unittest -from binascii import hexlify, unhexlify - from lbry.wallet.script import OutputScript import unittest from binascii import hexlify, unhexlify -from torba.client.bcd_data_stream import BCDataStream -from torba.client.basescript import Template, ParseError, tokenize, push_data -from torba.client.basescript import PUSH_SINGLE, PUSH_INTEGER, PUSH_MANY, OP_HASH160, OP_EQUAL -from torba.client.basescript import BaseInputScript, BaseOutputScript +from lbry.wallet.client.bcd_data_stream import BCDataStream +from lbry.wallet.client.basescript import Template, ParseError, tokenize, push_data +from lbry.wallet.client.basescript import PUSH_SINGLE, PUSH_INTEGER, PUSH_MANY, OP_HASH160, OP_EQUAL +from lbry.wallet.client.basescript import BaseInputScript, BaseOutputScript def parse(opcodes, source): diff --git a/lbry/tests/unit/wallet/test_stream_controller.py b/lbry/tests/unit/wallet/test_stream_controller.py index 496975536..59f534b4b 100644 --- a/lbry/tests/unit/wallet/test_stream_controller.py +++ b/lbry/tests/unit/wallet/test_stream_controller.py @@ -1,6 +1,6 @@ -from torba.stream import StreamController -from torba.tasks import TaskGroup -from torba.testcase import AsyncioTestCase +from lbry.wallet.stream import StreamController +from lbry.wallet.tasks import TaskGroup +from lbry.wallet.testcase import AsyncioTestCase class StreamControllerTestCase(AsyncioTestCase): diff --git a/lbry/tests/unit/wallet/test_transaction.py b/lbry/tests/unit/wallet/test_transaction.py index 03e0163e7..d58574dd2 100644 --- a/lbry/tests/unit/wallet/test_transaction.py +++ b/lbry/tests/unit/wallet/test_transaction.py @@ -1,26 +1,28 @@ import unittest from binascii import hexlify, unhexlify +from itertools import cycle -from torba.testcase import AsyncioTestCase -from torba.client.constants import CENT, COIN, NULL_HASH32 -from torba.client.wallet import Wallet +from lbry.wallet.testcase import AsyncioTestCase +from lbry.wallet.client.constants import CENT, COIN, NULL_HASH32 +from lbry.wallet.client.wallet import Wallet from lbry.wallet.ledger import MainNetLedger from lbry.wallet.transaction import Transaction, Output, Input +NULL_HASH = b'\x00'*32 FEE_PER_BYTE = 50 FEE_PER_CHAR = 200000 -def get_output(amount=CENT, pubkey_hash=NULL_HASH32): - return Transaction() \ +def get_output(amount=CENT, pubkey_hash=NULL_HASH32, height=-2): + return Transaction(height=height) \ .add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \ .outputs[0] -def get_input(): - return Input.spend(get_output()) +def get_input(amount=CENT, pubkey_hash=NULL_HASH): + return Input.spend(get_output(amount, pubkey_hash)) def get_transaction(txo=None): @@ -95,7 +97,7 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): _ = tx.net_account_balance def test_paying_from_my_account_to_other_account(self): - tx = ledger_class.transaction_class() \ + tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) @@ -105,7 +107,7 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): self.assertEqual(tx.net_account_balance, -200*CENT) def test_paying_from_other_account_to_my_account(self): - tx = ledger_class.transaction_class() \ + tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) @@ -115,7 +117,7 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase): self.assertEqual(tx.net_account_balance, 190*CENT) def test_paying_from_my_account_to_my_account(self): - tx = ledger_class.transaction_class() \ + tx = Transaction() \ .add_inputs([get_input(300*CENT)]) \ .add_outputs([get_output(190*CENT, NULL_HASH), get_output(100*CENT, NULL_HASH)]) @@ -303,9 +305,9 @@ class TestTransactionSigning(AsyncioTestCase): class TransactionIOBalancing(AsyncioTestCase): async def asyncSetUp(self): - self.ledger = ledger_class({ - 'db': ledger_class.database_class(':memory:'), - 'headers': ledger_class.headers_class(':memory:'), + self.ledger = MainNetLedger({ + 'db': MainNetLedger.database_class(':memory:'), + 'headers': MainNetLedger.headers_class(':memory:') }) await self.ledger.db.open() self.account = self.ledger.account_class.from_dict( @@ -326,15 +328,15 @@ class TransactionIOBalancing(AsyncioTestCase): return get_output(int(amount*COIN), address or next(self.hash_cycler)) def txi(self, txo): - return ledger_class.transaction_class.input_class.spend(txo) + return Transaction.input_class.spend(txo) def tx(self, inputs, outputs): - return ledger_class.transaction_class.create(inputs, outputs, [self.account], self.account) + return Transaction.create(inputs, outputs, [self.account], self.account) async def create_utxos(self, amounts): utxos = [self.txo(amount) for amount in amounts] - self.funding_tx = ledger_class.transaction_class(is_verified=True) \ + self.funding_tx = Transaction(is_verified=True) \ .add_inputs([self.txi(self.txo(sum(amounts)+0.1))]) \ .add_outputs(utxos) diff --git a/lbry/tests/unit/wallet/test_utils.py b/lbry/tests/unit/wallet/test_utils.py index 3d9e19d0a..687371308 100644 --- a/lbry/tests/unit/wallet/test_utils.py +++ b/lbry/tests/unit/wallet/test_utils.py @@ -1,7 +1,7 @@ import unittest -from torba.client.util import ArithUint256 -from torba.client.util import coins_to_satoshis as c2s, satoshis_to_coins as s2c +from lbry.wallet.client.util import ArithUint256 +from lbry.wallet.client.util import coins_to_satoshis as c2s, satoshis_to_coins as s2c class TestCoinValueParsing(unittest.TestCase): diff --git a/lbry/tests/unit/wallet/test_wallet.py b/lbry/tests/unit/wallet/test_wallet.py index 7bf88c5ff..f2dedbbd4 100644 --- a/lbry/tests/unit/wallet/test_wallet.py +++ b/lbry/tests/unit/wallet/test_wallet.py @@ -2,12 +2,11 @@ import tempfile from binascii import hexlify from unittest import TestCase, mock -from torba.testcase import AsyncioTestCase +from lbry.wallet.testcase import AsyncioTestCase -from torba.coin.bitcoinsegwit import MainNetLedger as BTCLedger -from torba.coin.bitcoincash import MainNetLedger as BCHLedger -from torba.client.basemanager import BaseWalletManager -from torba.client.wallet import Wallet, WalletStorage, TimestampedPreferences +from lbry.wallet.ledger import MainNetLedger, RegTestLedger +from lbry.wallet.client.basemanager import BaseWalletManager +from lbry.wallet.client.wallet import Wallet, WalletStorage, TimestampedPreferences class TestWalletCreation(AsyncioTestCase): @@ -15,17 +14,17 @@ class TestWalletCreation(AsyncioTestCase): async def asyncSetUp(self): self.manager = BaseWalletManager() config = {'data_path': '/tmp/wallet'} - self.btc_ledger = self.manager.get_or_create_ledger(BTCLedger.get_id(), config) - self.bch_ledger = self.manager.get_or_create_ledger(BCHLedger.get_id(), config) + self.main_ledger = self.manager.get_or_create_ledger(MainNetLedger.get_id(), config) + self.test_ledger = self.manager.get_or_create_ledger(RegTestLedger.get_id(), config) def test_create_wallet_and_accounts(self): wallet = Wallet() self.assertEqual(wallet.name, 'Wallet') self.assertListEqual(wallet.accounts, []) - account1 = wallet.generate_account(self.btc_ledger) - wallet.generate_account(self.btc_ledger) - wallet.generate_account(self.bch_ledger) + account1 = wallet.generate_account(self.main_ledger) + wallet.generate_account(self.main_ledger) + wallet.generate_account(self.test_ledger) self.assertEqual(wallet.default_account, account1) self.assertEqual(len(wallet.accounts), 3) @@ -36,19 +35,20 @@ class TestWalletCreation(AsyncioTestCase): 'preferences': {}, 'accounts': [ { + 'certificates': {}, 'name': 'An Account', - 'ledger': 'btc_mainnet', + 'ledger': 'lbc_mainnet', 'modified_on': 123.456, 'seed': "carbon smart garage balance margin twelve chest sword toast envelope bottom stomac" "h absent", 'encrypted': False, 'private_key': - 'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3Z' - 'T4vYymkp5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna', + 'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7' + 'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe', 'public_key': - 'xpub661MyMwAqRbcFwwe67Bfjd53h5WXmKm6tqfBJZZH3pQLoy8Nb6' - 'mKUMJFc7UbpVNzmwFPN2evn3YHnig1pkKVYcvCV8owTd2yAcEkJfCX53g', + 'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm' + 'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9', 'address_generator': { 'name': 'deterministic-chain', 'receiving': {'gap': 17, 'maximum_uses_per_address': 3}, @@ -62,11 +62,11 @@ class TestWalletCreation(AsyncioTestCase): wallet = Wallet.from_storage(storage, self.manager) self.assertEqual(wallet.name, 'Main Wallet') self.assertEqual( - hexlify(wallet.hash), b'1bd61fbe18875cb7828c466022af576104ed861c8a1fdb1dadf5e39417a68483' + hexlify(wallet.hash), b'a75913d2e7339c1a9ac0c89d621a4e10fd3a40dc3560dc01f4cf4ada0a0b05b8' ) self.assertEqual(len(wallet.accounts), 1) account = wallet.default_account - self.assertIsInstance(account, BTCLedger.account_class) + self.assertIsInstance(account, MainNetLedger.account_class) self.maxDiff = None self.assertDictEqual(wallet_dict, wallet.to_dict()) @@ -77,7 +77,7 @@ class TestWalletCreation(AsyncioTestCase): def test_read_write(self): manager = BaseWalletManager() config = {'data_path': '/tmp/wallet'} - ledger = manager.get_or_create_ledger(BTCLedger.get_id(), config) + ledger = manager.get_or_create_ledger(MainNetLedger.get_id(), config) with tempfile.NamedTemporaryFile(suffix='.json') as wallet_file: wallet_file.write(b'{"version": 1}') @@ -98,11 +98,11 @@ class TestWalletCreation(AsyncioTestCase): wallet1 = Wallet() wallet1.preferences['one'] = 1 wallet1.preferences['conflict'] = 1 - wallet1.generate_account(self.btc_ledger) + wallet1.generate_account(self.main_ledger) wallet2 = Wallet() wallet2.preferences['two'] = 2 wallet2.preferences['conflict'] = 2 # will be more recent - wallet2.generate_account(self.btc_ledger) + wallet2.generate_account(self.main_ledger) self.assertEqual(len(wallet1.accounts), 1) self.assertEqual(wallet1.preferences, {'one': 1, 'conflict': 1})