diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index d4afab105..58a08780a 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,7 +1,10 @@ import logging +from lbrynet.custom_logger import install_logger + __name__ = "lbrynet" __version__ = "0.30.1rc1" version = tuple(__version__.split('.')) +install_logger() logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lbrynet/blob/EncryptedFileCreator.py b/lbrynet/blob/EncryptedFileCreator.py index 7d1fec30a..d8ee1868d 100644 --- a/lbrynet/blob/EncryptedFileCreator.py +++ b/lbrynet/blob/EncryptedFileCreator.py @@ -2,8 +2,8 @@ Utilities for turning plain files into LBRY Files. """ -import logging import os +import logging from binascii import hexlify from twisted.internet import defer diff --git a/lbrynet/blob/EncryptedFileDownloader.py b/lbrynet/blob/EncryptedFileDownloader.py index 12664a0c6..108bb2f80 100644 --- a/lbrynet/blob/EncryptedFileDownloader.py +++ b/lbrynet/blob/EncryptedFileDownloader.py @@ -5,10 +5,10 @@ import logging from binascii import hexlify, unhexlify from twisted.internet import defer -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader -from lbrynet.p2p.utils import short_hash +from lbrynet.utils import short_hash from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport diff --git a/lbrynet/blob/EncryptedFileManager.py b/lbrynet/blob/EncryptedFileManager.py index f7d138db3..902ecae88 100644 --- a/lbrynet/blob/EncryptedFileManager.py +++ b/lbrynet/blob/EncryptedFileManager.py @@ -7,14 +7,14 @@ from binascii import hexlify, unhexlify from twisted.internet import defer, task, reactor from twisted.python.failure import Failure +from lbrynet import conf from lbrynet.extras.reflector.reupload import reflect_file from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError -from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call -from lbrynet.extras.daemon import conf +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call log = logging.getLogger(__name__) diff --git a/lbrynet/blob/__init__.py b/lbrynet/blob/__init__.py index 3c5de8fa9..e69de29bb 100644 --- a/lbrynet/blob/__init__.py +++ b/lbrynet/blob/__init__.py @@ -1,4 +0,0 @@ -from .blob_file import BlobFile -from .creator import BlobFileCreator -from .writer import HashBlobWriter -from .reader import HashBlobReader diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 0046b6c5a..728962971 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -1,10 +1,10 @@ -import logging import os +import logging from twisted.internet import defer, threads from twisted.web.client import FileBodyProducer from twisted.python.failure import Failure +from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError -from lbrynet.p2p.utils import is_valid_blobhash from lbrynet.blob.writer import HashBlobWriter from lbrynet.blob.reader import HashBlobReader @@ -12,6 +12,24 @@ log = logging.getLogger(__name__) MAX_BLOB_SIZE = 2 * 2 ** 20 +# digest_size is in bytes, and blob hashes are hex encoded +blobhash_length = get_lbry_hash_obj().digest_size * 2 + + +def is_valid_hashcharacter(char): + return char in "0123456789abcdef" + + +def is_valid_blobhash(blobhash): + """Checks whether the blobhash is the correct length and contains only + valid characters (0-9, a-f) + + @param blobhash: string, the blobhash to check + + @return: True/False + """ + return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash) + class BlobFile: """ diff --git a/lbrynet/blob/client/CryptStreamDownloader.py b/lbrynet/blob/client/CryptStreamDownloader.py index ffc0844cf..ae14b0e6d 100644 --- a/lbrynet/blob/client/CryptStreamDownloader.py +++ b/lbrynet/blob/client/CryptStreamDownloader.py @@ -1,12 +1,13 @@ -from binascii import unhexlify import logging +from binascii import unhexlify +from twisted.internet import defer +from twisted.python.failure import Failure + from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.p2p.client.DownloadManager import DownloadManager from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler -from twisted.internet import defer -from twisted.python.failure import Failure log = logging.getLogger(__name__) diff --git a/lbrynet/blob/client/EncryptedFileDownloader.py b/lbrynet/blob/client/EncryptedFileDownloader.py index 735ea3bf4..48e7b6565 100644 --- a/lbrynet/blob/client/EncryptedFileDownloader.py +++ b/lbrynet/blob/client/EncryptedFileDownloader.py @@ -2,13 +2,13 @@ import os import logging import traceback from binascii import hexlify, unhexlify +from twisted.internet import defer, threads from lbrynet.p2p.StreamDescriptor import save_sd_info from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.p2p.Error import FileOpenError from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler -from twisted.internet import defer, threads log = logging.getLogger(__name__) diff --git a/lbrynet/blob/creator.py b/lbrynet/blob/creator.py index 04b42961b..af9dc213f 100644 --- a/lbrynet/blob/creator.py +++ b/lbrynet/blob/creator.py @@ -3,7 +3,7 @@ import logging from io import BytesIO from twisted.internet import defer from twisted.web.client import FileBodyProducer -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) diff --git a/lbrynet/blob/writer.py b/lbrynet/blob/writer.py index 5332d3533..c66e3b037 100644 --- a/lbrynet/blob/writer.py +++ b/lbrynet/blob/writer.py @@ -2,7 +2,7 @@ import logging from io import BytesIO from twisted.python.failure import Failure from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/conf.py b/lbrynet/conf.py similarity index 99% rename from lbrynet/extras/daemon/conf.py rename to lbrynet/conf.py index 311ba64cc..757aced9b 100644 --- a/lbrynet/extras/daemon/conf.py +++ b/lbrynet/conf.py @@ -7,7 +7,7 @@ import sys import yaml import envparse from appdirs import user_data_dir, user_config_dir -from lbrynet.p2p import utils +from lbrynet import utils from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError from lbrynet.androidhelpers.paths import ( android_internal_storage_dir, @@ -15,7 +15,7 @@ from lbrynet.androidhelpers.paths import ( ) try: - from .winpaths import get_path, FOLDERID, UserHandle + from lbrynet.winpaths import get_path, FOLDERID, UserHandle except (ImportError, ValueError, NameError): # Android platform: NameError: name 'c_wchar' is not defined pass diff --git a/lbrynet/p2p/cryptoutils.py b/lbrynet/cryptoutils.py similarity index 100% rename from lbrynet/p2p/cryptoutils.py rename to lbrynet/cryptoutils.py diff --git a/lbrynet/custom_logger.py b/lbrynet/custom_logger.py index bd608f1cd..8e3499e42 100644 --- a/lbrynet/custom_logger.py +++ b/lbrynet/custom_logger.py @@ -102,5 +102,8 @@ class Logger(logging.Logger): self._log(TRACE, msg, args, **kwargs) -logging.setLoggerClass(Logger) -logging.addLevelName(TRACE, 'TRACE') +def install_logger(): + current = logging.getLoggerClass() + if current is not Logger: + logging.setLoggerClass(Logger) + logging.addLevelName(TRACE, 'TRACE') diff --git a/lbrynet/p2p/call_later_manager.py b/lbrynet/dht/call_later_manager.py similarity index 100% rename from lbrynet/p2p/call_later_manager.py rename to lbrynet/dht/call_later_manager.py diff --git a/lbrynet/dht/datastore.py b/lbrynet/dht/datastore.py index 2ae0f393d..12be982f8 100644 --- a/lbrynet/dht/datastore.py +++ b/lbrynet/dht/datastore.py @@ -1,5 +1,5 @@ from collections import UserDict -from . import constants +from lbrynet.dht import constants class DictDataStore(UserDict): diff --git a/lbrynet/dht/encoding.py b/lbrynet/dht/encoding.py index f31bd119f..4eb4d0764 100644 --- a/lbrynet/dht/encoding.py +++ b/lbrynet/dht/encoding.py @@ -1,4 +1,4 @@ -from .error import DecodeError +from lbrynet.dht.error import DecodeError def bencode(data): diff --git a/lbrynet/dht/iterativefind.py b/lbrynet/dht/iterativefind.py index 765c548dc..a2be6b09b 100644 --- a/lbrynet/dht/iterativefind.py +++ b/lbrynet/dht/iterativefind.py @@ -1,8 +1,8 @@ import logging from twisted.internet import defer -from .distance import Distance -from .error import TimeoutError -from . import constants +from lbrynet.dht.distance import Distance +from lbrynet.dht.error import TimeoutError +from lbrynet.dht import constants log = logging.getLogger(__name__) diff --git a/lbrynet/dht/kbucket.py b/lbrynet/dht/kbucket.py index 7fffb4ce7..fa3f19d2e 100644 --- a/lbrynet/dht/kbucket.py +++ b/lbrynet/dht/kbucket.py @@ -1,8 +1,8 @@ import logging -from . import constants -from .distance import Distance -from .error import BucketFull +from lbrynet.dht import constants +from lbrynet.dht.distance import Distance +from lbrynet.dht.error import BucketFull log = logging.getLogger(__name__) diff --git a/lbrynet/dht/msgformat.py b/lbrynet/dht/msgformat.py index fc4381d1c..5e4c59d79 100644 --- a/lbrynet/dht/msgformat.py +++ b/lbrynet/dht/msgformat.py @@ -7,7 +7,7 @@ # The docstrings in this module contain epytext markup; API documentation # may be created by processing this file with epydoc: http://epydoc.sf.net -from . import msgtypes +from lbrynet.dht import msgtypes class MessageTranslator: diff --git a/lbrynet/dht/msgtypes.py b/lbrynet/dht/msgtypes.py index 8bf8f3227..907c1a8e7 100644 --- a/lbrynet/dht/msgtypes.py +++ b/lbrynet/dht/msgtypes.py @@ -7,8 +7,8 @@ # The docstrings in this module contain epytext markup; API documentation # may be created by processing this file with epydoc: http://epydoc.sf.net -from lbrynet.p2p.utils import generate_id -from . import constants +from lbrynet.utils import generate_id +from lbrynet.dht import constants class Message: diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 287cae8fb..fc5618d05 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -5,17 +5,12 @@ from functools import reduce from twisted.internet import defer, error, task -from lbrynet.p2p.utils import generate_id, DeferredDict -from lbrynet.p2p.call_later_manager import CallLaterManager -from lbrynet.p2p.PeerManager import PeerManager -from .error import TimeoutError -from . import constants -from . import routingtable -from . import datastore -from . import protocol -from .peerfinder import DHTPeerFinder -from .contact import ContactManager -from .iterativefind import iterativeFind +from lbrynet.utils import generate_id, DeferredDict +from lbrynet.dht.call_later_manager import CallLaterManager +from lbrynet.dht.error import TimeoutError +from lbrynet.dht import constants, routingtable, datastore, protocol +from lbrynet.dht.contact import ContactManager +from lbrynet.dht.iterativefind import iterativeFind log = logging.getLogger(__name__) @@ -83,8 +78,8 @@ class Node(MockKademliaHelper): def __init__(self, node_id=None, udpPort=4000, dataStore=None, routingTableClass=None, networkProtocol=None, externalIP=None, peerPort=3333, listenUDP=None, - callLater=None, resolve=None, clock=None, peer_finder=None, - peer_manager=None, interface='', externalUDPPort=None): + callLater=None, resolve=None, clock=None, + interface='', externalUDPPort=None): """ @param dataStore: The data store to use. This must be class inheriting from the C{DataStore} interface (or providing the @@ -124,20 +119,13 @@ class Node(MockKademliaHelper): else: self._routingTable = routingTableClass(self.node_id, self.clock.seconds) - # Initialize this node's network access mechanisms - if networkProtocol is None: - self._protocol = protocol.KademliaProtocol(self) - else: - self._protocol = networkProtocol - # Initialize the data storage mechanism used by this node + self._protocol = networkProtocol or protocol.KademliaProtocol(self) self.token_secret = self._generateID() self.old_token_secret = None self.externalIP = externalIP self.peerPort = peerPort self.externalUDPPort = externalUDPPort or self.port self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds) - self.peer_manager = peer_manager or PeerManager() - self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager) self._join_deferred = None #def __del__(self): @@ -255,7 +243,7 @@ class Node(MockKademliaHelper): yield _iterative_join() @defer.inlineCallbacks - def start(self, known_node_addresses=None): + def start(self, known_node_addresses=None, block_on_join=False): """ Causes the Node to attempt to join the DHT network by contacting the known DHT nodes. This can be called multiple times if the previous attempt has failed or if the Node has lost all the contacts. @@ -270,8 +258,11 @@ class Node(MockKademliaHelper): self.start_listening() yield self._protocol._listening # TODO: Refresh all k-buckets further away than this node's closest neighbour - yield self.joinNetwork(known_node_addresses or []) - self.start_looping_calls() + d = self.joinNetwork(known_node_addresses or []) + d.addCallback(lambda _: self.start_looping_calls()) + d.addCallback(lambda _: log.info("Joined the dht")) + if block_on_join: + yield d def start_looping_calls(self): self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 18d55977a..6504baea3 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -3,12 +3,8 @@ import errno from binascii import hexlify from twisted.internet import protocol, defer -from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected - -from . import constants -from . import encoding -from . import msgtypes -from . import msgformat +from lbrynet.dht import constants, encoding, msgformat, msgtypes +from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected log = logging.getLogger(__name__) diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index 7417e3970..dc8ff8089 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -6,14 +6,13 @@ # may be created by processing this file with epydoc: http://epydoc.sf.net import random -from binascii import unhexlify - -from twisted.internet import defer -from . import constants -from . import kbucket -from .error import TimeoutError -from .distance import Distance import logging +from binascii import unhexlify +from twisted.internet import defer + +from lbrynet.dht import constants, kbucket +from lbrynet.dht.error import TimeoutError +from lbrynet.dht.distance import Distance log = logging.getLogger(__name__) diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 3ebaf7b8a..1d45ddaee 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -24,22 +24,25 @@ from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.daemon.DaemonControl import start as daemon_main from lbrynet.extras.daemon.DaemonConsole import main as daemon_console from lbrynet.extras.daemon.auth.client import LBRYAPIClient -from lbrynet.p2p.system_info import get_platform +from lbrynet.extras.system_info import get_platform async def execute_command(method, params, conf_path=None): # this check if the daemon is running or not + api = None try: api = await LBRYAPIClient.get_client(conf_path) await api.status() except (ClientConnectorError, ConnectionError): - await api.session.close() + if api: + await api.session.close() print("Could not connect to daemon. Are you sure it's running?") return 1 # this actually executes the method + resp = await api.call(method, params) + try: - resp = await api.call(method, params) await api.session.close() print(json.dumps(resp["result"], indent=2)) except KeyError: @@ -137,7 +140,7 @@ def main(argv=None): elif method in ['version', '--version', '-v']: print("{lbrynet_name} {lbrynet_version}".format( - lbrynet_name=lbrynet_name, **get_platform(get_ip=False) + lbrynet_name=lbrynet_name, **get_platform() )) return 0 diff --git a/lbrynet/extras/daemon/Component.py b/lbrynet/extras/daemon/Component.py index 03f03ddf5..8de53a6ae 100644 --- a/lbrynet/extras/daemon/Component.py +++ b/lbrynet/extras/daemon/Component.py @@ -1,7 +1,7 @@ import logging from twisted.internet import defer from twisted._threads import AlreadyQuit -from .ComponentManager import ComponentManager +from lbrynet.extras.daemon.ComponentManager import ComponentManager log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/ComponentManager.py b/lbrynet/extras/daemon/ComponentManager.py index 087a446ee..9af07fd8d 100644 --- a/lbrynet/extras/daemon/ComponentManager.py +++ b/lbrynet/extras/daemon/ComponentManager.py @@ -1,7 +1,8 @@ import logging from twisted.internet import defer - from lbrynet.p2p.Error import ComponentStartConditionNotMet +from lbrynet.extras.daemon.PeerManager import PeerManager +from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder log = logging.getLogger(__name__) @@ -33,13 +34,15 @@ class RequiredCondition(metaclass=RequiredConditionType): class ComponentManager: default_component_classes = {} - def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components): + def __init__(self, reactor=None, analytics_manager=None, skip_components=None, + peer_manager=None, peer_finder=None, **override_components): self.skip_components = skip_components or [] - self.reactor = reactor self.component_classes = {} self.components = set() self.analytics_manager = analytics_manager + self.peer_manager = peer_manager or PeerManager() + self.peer_finder = peer_finder or DHTPeerFinder(self) for component_name, component_class in self.default_component_classes.items(): if component_name in override_components: @@ -114,10 +117,10 @@ class ComponentManager: :return: (defer.Deferred) """ - for component_name, cb in callbacks.items(): if component_name not in self.component_classes: - raise NameError("unknown component: %s" % component_name) + if component_name not in self.skip_components: + raise NameError("unknown component: %s" % component_name) if not callable(cb): raise ValueError("%s is not callable" % cb) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index eada5c554..9751adb78 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -2,35 +2,39 @@ import os import asyncio import logging import treq +import json import math import binascii from hashlib import sha256 from types import SimpleNamespace from twisted.internet import defer, threads, reactor, error, task -import lbrynet.schema + from aioupnp import __version__ as aioupnp_version from aioupnp.upnp import UPnP from aioupnp.fault import UPnPError -from lbrynet.extras.daemon import conf -from lbrynet.p2p.utils import DeferredDict + +import lbrynet.schema +from lbrynet import conf + +from lbrynet.blob.EncryptedFileManager import EncryptedFileManager +from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory +from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier +from lbrynet.dht.node import Node +from lbrynet.extras.daemon.Component import Component +from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager +from lbrynet.extras.daemon.storage import SQLiteStorage +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer +from lbrynet.extras.reflector.server.server import ReflectorServerFactory +from lbrynet.extras.wallet import LbryWalletManager +from lbrynet.extras.wallet import Network +from lbrynet.utils import DeferredDict, generate_id from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType -from lbrynet.extras.wallet import LbryWalletManager -from lbrynet.extras.wallet import Network from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory -from .Component import Component -from .ExchangeRateManager import ExchangeRateManager -from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht import node, hashannouncer -from lbrynet.blob.EncryptedFileManager import EncryptedFileManager -from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory -from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier -from lbrynet.extras.reflector import ServerFactory as reflector_server_factory -from lbrynet.p2p.utils import generate_id log = logging.getLogger(__name__) @@ -42,7 +46,6 @@ HEADERS_COMPONENT = "blockchain_headers" WALLET_COMPONENT = "wallet" DHT_COMPONENT = "dht" HASH_ANNOUNCER_COMPONENT = "hash_announcer" -STREAM_IDENTIFIER_COMPONENT = "stream_identifier" FILE_MANAGER_COMPONENT = "file_manager" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" REFLECTOR_COMPONENT = "reflector" @@ -94,10 +97,19 @@ class ConfigSettings: return conf.settings.node_id @staticmethod - def get_external_ip(): - from lbrynet.p2p.system_info import get_platform - platform = get_platform(get_ip=True) - return platform['ip'] + @defer.inlineCallbacks + def get_external_ip(): # used if upnp is disabled or non-functioning + try: + buf = [] + response = yield treq.get("https://api.lbry.io/ip") + yield treq.collect(response, buf.append) + parsed = json.loads(b"".join(buf).decode()) + if parsed['success']: + return parsed['data']['ip'] + return + except Exception as err: + return + # Shorthand for common ConfigSettings methods @@ -366,7 +378,7 @@ class WalletComponent(Component): class BlobComponent(Component): component_name = BLOB_COMPONENT - depends_on = [DATABASE_COMPONENT, DHT_COMPONENT] + depends_on = [DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -378,8 +390,12 @@ class BlobComponent(Component): def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) - dht_node = self.component_manager.get_component(DHT_COMPONENT) - self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore) + datastore = None + if DHT_COMPONENT not in self.component_manager.skip_components: + dht_node = self.component_manager.get_component(DHT_COMPONENT) + if dht_node: + datastore = dht_node._dataStore + self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore) return self.blob_manager.setup() def stop(self): @@ -424,20 +440,22 @@ class DHTComponent(Component): node_id = CS.get_node_id() if node_id is None: node_id = generate_id() + external_ip = self.upnp_component.external_ip + if not external_ip: + log.warning("UPnP component failed to get external ip") + external_ip = yield CS.get_external_ip() + if not external_ip: + log.warning("failed to get external ip") - self.dht_node = node.Node( + self.dht_node = Node( node_id=node_id, udpPort=GCS('dht_node_port'), externalUDPPort=self.external_udp_port, - externalIP=self.upnp_component.external_ip, + externalIP=external_ip, peerPort=self.external_peer_port ) - self.dht_node.start_listening() - yield self.dht_node._protocol._listening - d = self.dht_node.joinNetwork(GCS('known_dht_nodes')) - d.addCallback(lambda _: self.dht_node.start_looping_calls()) - d.addCallback(lambda _: log.info("Joined the dht")) + yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False) log.info("Started the dht") @defer.inlineCallbacks @@ -461,7 +479,7 @@ class HashAnnouncerComponent(Component): def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) dht_node = self.component_manager.get_component(DHT_COMPONENT) - self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage) + self.hash_announcer = DHTHashAnnouncer(dht_node, storage) yield self.hash_announcer.start() @defer.inlineCallbacks @@ -494,41 +512,6 @@ class RateLimiterComponent(Component): return defer.succeed(None) -class StreamIdentifierComponent(Component): - component_name = STREAM_IDENTIFIER_COMPONENT - depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] - - def __init__(self, component_manager): - super().__init__(component_manager) - self.sd_identifier = StreamDescriptorIdentifier() - - @property - def component(self): - return self.sd_identifier - - @defer.inlineCallbacks - def start(self): - dht_node = self.component_manager.get_component(DHT_COMPONENT) - rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT) - blob_manager = self.component_manager.get_component(BLOB_COMPONENT) - storage = self.component_manager.get_component(DATABASE_COMPONENT) - wallet = self.component_manager.get_component(WALLET_COMPONENT) - - add_lbry_file_to_sd_identifier(self.sd_identifier) - file_saver_factory = EncryptedFileSaverFactory( - dht_node.peer_finder, - rate_limiter, - blob_manager, - storage, - wallet, - GCS('download_directory') - ) - yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory) - - def stop(self): - pass - - class PaymentRateComponent(Component): component_name = PAYMENT_RATE_COMPONENT @@ -549,8 +532,8 @@ class PaymentRateComponent(Component): class FileManagerComponent(Component): component_name = FILE_MANAGER_COMPONENT - depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, - STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT] + depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, + PAYMENT_RATE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -569,15 +552,26 @@ class FileManagerComponent(Component): @defer.inlineCallbacks def start(self): - dht_node = self.component_manager.get_component(DHT_COMPONENT) rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT) wallet = self.component_manager.get_component(WALLET_COMPONENT) - sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT) + + sd_identifier = StreamDescriptorIdentifier() + add_lbry_file_to_sd_identifier(sd_identifier) + file_saver_factory = EncryptedFileSaverFactory( + self.component_manager.peer_finder, + rate_limiter, + blob_manager, + storage, + wallet, + GCS('download_directory') + ) + yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory) + payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT) log.info('Starting the file manager') - self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet, + self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier) yield self.file_manager.setup() log.info('Done setting up file manager') @@ -589,7 +583,7 @@ class FileManagerComponent(Component): class PeerProtocolServerComponent(Component): component_name = PEER_PROTOCOL_SERVER_COMPONENT - depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, + depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, PAYMENT_RATE_COMPONENT] def __init__(self, component_manager): @@ -618,7 +612,7 @@ class PeerProtocolServerComponent(Component): } server_factory = ServerProtocolFactory( self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers, - self.component_manager.get_component(DHT_COMPONENT).peer_manager + self.component_manager.peer_manager ) try: @@ -642,7 +636,7 @@ class PeerProtocolServerComponent(Component): class ReflectorComponent(Component): component_name = REFLECTOR_COMPONENT - depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT] + depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -656,10 +650,9 @@ class ReflectorComponent(Component): @defer.inlineCallbacks def start(self): log.info("Starting reflector server") - dht_node = self.component_manager.get_component(DHT_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT) file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT) - reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager) + reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager) try: self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory) log.info('Started reflector on port %s', self.reflector_server_port) @@ -695,10 +688,12 @@ class UPnPComponent(Component): @defer.inlineCallbacks def _setup_redirects(self): - upnp_redirects = yield DeferredDict({ - "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), - "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) - }) + d = {} + if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: + d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) + if DHT_COMPONENT not in self.component_manager.skip_components: + d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")) + upnp_redirects = yield DeferredDict(d) self.upnp_redirects.update(upnp_redirects) @defer.inlineCallbacks @@ -710,33 +705,41 @@ class UPnPComponent(Component): log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string) except Exception as err: log.warning("upnp discovery failed: %s", err) - return + self.upnp = None # update the external ip - try: - external_ip = yield from_future(self.upnp.get_external_ip()) - if external_ip == "0.0.0.0": - log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback") - external_ip = CS.get_external_ip() - if self.external_ip and self.external_ip != external_ip: - log.info("external ip changed from %s to %s", self.external_ip, external_ip) - elif not self.external_ip: - log.info("got external ip: %s", external_ip) - self.external_ip = external_ip - except (asyncio.TimeoutError, UPnPError): - pass - - if not self.upnp_redirects: # setup missing redirects + external_ip = None + if self.upnp: try: - upnp_redirects = yield DeferredDict({ - "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), - "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) - }) + external_ip = yield from_future(self.upnp.get_external_ip()) + if external_ip != "0.0.0.0": + log.info("got external ip from UPnP: %s", external_ip) + except (asyncio.TimeoutError, UPnPError): + pass + + if external_ip == "0.0.0.0" or not external_ip: + log.warning("unable to get external ip from UPnP, checking lbry.io fallback") + external_ip = yield CS.get_external_ip() + if self.external_ip and self.external_ip != external_ip: + log.info("external ip changed from %s to %s", self.external_ip, external_ip) + self.external_ip = external_ip + assert self.external_ip is not None # TODO: handle going/starting offline + + if not self.upnp_redirects and self.upnp: # setup missing redirects + try: + log.info("add UPnP port mappings") + d = {} + if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: + d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) + if DHT_COMPONENT not in self.component_manager.skip_components: + d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")) + upnp_redirects = yield DeferredDict(d) + log.info("set up redirects: %s", upnp_redirects) self.upnp_redirects.update(upnp_redirects) except (asyncio.TimeoutError, UPnPError): self.upnp = None return self._maintain_redirects() - else: # check existing redirects are still active + elif self.upnp: # check existing redirects are still active found = set() mappings = yield from_future(self.upnp.get_redirects()) for mapping in mappings: @@ -744,7 +747,7 @@ class UPnPComponent(Component): if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]: if mapping['NewInternalClient'] == self.upnp.lan_address: found.add(proto) - if 'UDP' not in found: + if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components: try: udp_port = yield from_future( self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port") @@ -753,7 +756,7 @@ class UPnPComponent(Component): log.info("refreshed upnp redirect for dht port: %i", udp_port) except (asyncio.TimeoutError, UPnPError): del self.upnp_redirects['UDP'] - if 'TCP' not in found: + if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: try: tcp_port = yield from_future( self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port") @@ -762,22 +765,28 @@ class UPnPComponent(Component): log.info("refreshed upnp redirect for peer port: %i", tcp_port) except (asyncio.TimeoutError, UPnPError): del self.upnp_redirects['TCP'] - if 'TCP' in self.upnp_redirects and 'UDP' in self.upnp_redirects: - log.debug("upnp redirects are still active") + if ('TCP' in self.upnp_redirects + and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and ( + 'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components): + if self.upnp_redirects: + log.debug("upnp redirects are still active") @defer.inlineCallbacks def start(self): + log.info("detecting external ip") if not self.use_upnp: - self.external_ip = CS.get_external_ip() + self.external_ip = yield CS.get_external_ip() return success = False yield self._maintain_redirects() if self.upnp: - if not self.upnp_redirects: + if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in + (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]): log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info) else: success = True - log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) + if self.upnp_redirects: + log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) else: log.error("failed to setup upnp") self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status()) diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index ef22fd3b6..cdd64458e 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -16,6 +16,27 @@ from twisted.python.failure import Failure from torba.client.baseaccount import SingleKey, HierarchicalDeterministic +from lbrynet import conf, utils, __version__ +from lbrynet.dht.error import TimeoutError +from lbrynet.extras import system_info +from lbrynet.extras.reflector import reupload +from lbrynet.extras.daemon.Components import d2f, f2d +from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT +from lbrynet.extras.daemon.Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT +from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT +from lbrynet.extras.daemon.ComponentManager import RequiredCondition +from lbrynet.extras.daemon.Downloader import GetStream +from lbrynet.extras.daemon.Publisher import Publisher +from lbrynet.extras.daemon.auth.server import AuthJSONRPCServer +from lbrynet.extras.wallet import LbryWalletManager +from lbrynet.extras.wallet.account import Account as LBCAccount +from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies +from lbrynet.p2p.StreamDescriptor import download_sd_blob +from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError, DownloadDataTimeout, DownloadSDTimeout +from lbrynet.p2p.Error import NullFundsError, NegativeFundsError, ResolveError +from lbrynet.p2p.Peer import Peer +from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader +from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.schema.claim import ClaimDict from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError, DecodeError @@ -23,32 +44,6 @@ from lbrynet.schema.validator import validate_claim_id from lbrynet.schema.address import decode_address from lbrynet.schema.decode import smart_decode -# TODO: importing this when internet is disabled raises a socket.gaierror -from lbrynet.p2p.system_info import get_lbrynet_version -from lbrynet.extras.daemon import conf -from lbrynet.extras.reflector import reupload -from .Components import d2f, f2d -from .Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT -from .Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT -from .Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT -from .ComponentManager import RequiredCondition -from .Downloader import GetStream -from .Publisher import Publisher -from .auth.server import AuthJSONRPCServer -from lbrynet.p2p import utils, system_info -from lbrynet.p2p.StreamDescriptor import download_sd_blob -from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError -from lbrynet.p2p.Error import DownloadDataTimeout, DownloadSDTimeout -from lbrynet.p2p.Error import NullFundsError, NegativeFundsError -from lbrynet.p2p.Error import ResolveError -from lbrynet.dht.error import TimeoutError -from lbrynet.p2p.Peer import Peer -from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader -from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader -from lbrynet.extras.wallet import LbryWalletManager -from lbrynet.extras.wallet.account import Account as LBCAccount -from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies - log = logging.getLogger(__name__) requires = AuthJSONRPCServer.requires @@ -210,7 +205,6 @@ class Daemon(AuthJSONRPCServer): DATABASE_COMPONENT: "storage", DHT_COMPONENT: "dht_node", WALLET_COMPONENT: "wallet_manager", - STREAM_IDENTIFIER_COMPONENT: "sd_identifier", FILE_MANAGER_COMPONENT: "file_manager", EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager", PAYMENT_RATE_COMPONENT: "payment_rate_manager", @@ -220,7 +214,7 @@ class Daemon(AuthJSONRPCServer): } def __init__(self, analytics_manager=None, component_manager=None): - to_skip = list(conf.settings['components_to_skip']) + to_skip = conf.settings['components_to_skip'] if 'reflector' not in to_skip and not conf.settings['run_reflector_server']: to_skip.append('reflector') looping_calls = { @@ -241,7 +235,6 @@ class Daemon(AuthJSONRPCServer): self.storage = None self.dht_node = None self.wallet_manager: LbryWalletManager = None - self.sd_identifier = None self.file_manager = None self.exchange_rate_manager = None self.payment_rate_manager = None @@ -305,7 +298,7 @@ class Daemon(AuthJSONRPCServer): rate_manager = rate_manager or self.payment_rate_manager timeout = timeout or 30 downloader = StandaloneBlobDownloader( - blob_hash, self.blob_manager, self.dht_node.peer_finder, self.rate_limiter, + blob_hash, self.blob_manager, self.component_manager.peer_finder, self.rate_limiter, rate_manager, self.wallet_manager, timeout ) return downloader.download() @@ -372,8 +365,8 @@ class Daemon(AuthJSONRPCServer): self.analytics_manager.send_download_started(download_id, name, claim_dict) self.analytics_manager.send_new_download_start(download_id, name, claim_dict) self.streams[sd_hash] = GetStream( - self.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager, - self.dht_node.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage, + self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager, + self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage, conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'], timeout ) @@ -432,7 +425,7 @@ class Daemon(AuthJSONRPCServer): if blob: return self.blob_manager.get_blob(blob[0]) return download_sd_blob( - sd_hash.decode(), self.blob_manager, self.dht_node.peer_finder, self.rate_limiter, + sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.wallet_manager, timeout=conf.settings['peer_search_timeout'], download_mirrors=conf.settings['download_mirrors'] ) @@ -450,7 +443,7 @@ class Daemon(AuthJSONRPCServer): Get total stream size in bytes from a sd blob """ - d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob) + d = self.file_manager.sd_identifier.get_metadata_for_sd_blob(sd_blob) d.addCallback(lambda metadata: metadata.validator.info_to_show()) d.addCallback(lambda info: int(dict(info)['stream_size'])) return d @@ -468,9 +461,7 @@ class Daemon(AuthJSONRPCServer): """ Calculate estimated LBC cost for a stream given its size in bytes """ - cost = self._get_est_cost_from_stream_size(size) - resolved = await self.wallet_manager.resolve(uri) if uri in resolved and 'claim' in resolved[uri]: @@ -535,7 +526,6 @@ class Daemon(AuthJSONRPCServer): sd blob will be downloaded to determine the stream size """ - if size is not None: return self.get_est_cost_using_known_size(uri, size) return self.get_est_cost_from_uri(uri) @@ -819,7 +809,7 @@ class Daemon(AuthJSONRPCServer): 'ip': (str) remote ip, if available, 'lbrynet_version': (str) lbrynet_version, 'lbryum_version': (str) lbryum_version, - 'lbrynet.schema_version': (str) lbrynet.schema_version, + 'lbryschema_version': (str) lbryschema_version, 'os_release': (str) os release string 'os_system': (str) os name 'platform': (str) platform string @@ -851,7 +841,7 @@ class Daemon(AuthJSONRPCServer): message, conf.settings.installation_id, platform_name, - get_lbrynet_version() + __version__ ) return self._render_response(True) @@ -1776,8 +1766,8 @@ class Daemon(AuthJSONRPCServer): results[resolved_uri] = resolved[resolved_uri] return results - @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, - DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, + @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, conditions=[WALLET_IS_UNLOCKED]) async def jsonrpc_get(self, uri, file_name=None, timeout=None): """ @@ -1962,7 +1952,7 @@ class Daemon(AuthJSONRPCServer): response = yield self._render_response(result) defer.returnValue(response) - @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, conditions=[WALLET_IS_UNLOCKED]) def jsonrpc_stream_cost_estimate(self, uri, size=None): diff --git a/lbrynet/extras/daemon/DaemonConsole.py b/lbrynet/extras/daemon/DaemonConsole.py index 5a7d6caa0..d3a1c1d67 100644 --- a/lbrynet/extras/daemon/DaemonConsole.py +++ b/lbrynet/extras/daemon/DaemonConsole.py @@ -6,11 +6,10 @@ import logging.handlers from twisted.internet import defer, reactor, threads from aiohttp import client_exceptions -from lbrynet.extras.daemon import analytics, conf -from lbrynet.p2p import utils -from lbrynet.p2p import log_support -from .auth.client import LBRYAPIClient -from .Daemon import Daemon +from lbrynet import utils, conf, log_support +from lbrynet.extras.daemon import analytics +from lbrynet.extras.daemon.auth.client import LBRYAPIClient +from lbrynet.extras.daemon.Daemon import Daemon log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py index 09fa07cf8..0ad0158fc 100644 --- a/lbrynet/extras/daemon/DaemonControl.py +++ b/lbrynet/extras/daemon/DaemonControl.py @@ -7,16 +7,14 @@ if 'win' in sys.platform: import certifi os.environ['SSL_CERT_FILE'] = certifi.where() -from lbrynet.p2p import log_support import argparse import logging.handlers from twisted.internet import reactor - -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils, system_info -from .Daemon import Daemon +from lbrynet import utils, conf, log_support +from lbrynet.extras import system_info +from lbrynet.extras.daemon.Daemon import Daemon log = logging.getLogger(__name__) @@ -54,7 +52,7 @@ def start(argv=None, conf_path=None): conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,)) if args.version: - version = system_info.get_platform(get_ip=False) + version = system_info.get_platform() version['installation_id'] = conf.settings.installation_id print(utils.json_dumps_pretty(version)) return diff --git a/lbrynet/extras/daemon/Downloader.py b/lbrynet/extras/daemon/Downloader.py index fe036b3cd..f4badd0ad 100644 --- a/lbrynet/extras/daemon/Downloader.py +++ b/lbrynet/extras/daemon/Downloader.py @@ -3,17 +3,17 @@ import os from twisted.internet import defer from twisted.internet.task import LoopingCall -from .Components import f2d +from lbrynet import conf from lbrynet.schema.fee import Fee from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout -from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory -from lbrynet.extras.daemon import conf from torba.client.constants import COIN from lbrynet.extras.wallet.dewies import dewies_to_lbc +from lbrynet.extras.daemon.Components import f2d INITIALIZING_CODE = 'initializing' DOWNLOAD_METADATA_CODE = 'downloading_metadata' diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/extras/daemon/HashAnnouncer.py similarity index 98% rename from lbrynet/dht/hashannouncer.py rename to lbrynet/extras/daemon/HashAnnouncer.py index d1cc26d85..819f890fa 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/extras/daemon/HashAnnouncer.py @@ -2,8 +2,7 @@ import binascii import logging from twisted.internet import defer, task -from lbrynet.p2p import utils -from lbrynet.extras.daemon import conf +from lbrynet import utils, conf log = logging.getLogger(__name__) diff --git a/lbrynet/dht/peerfinder.py b/lbrynet/extras/daemon/PeerFinder.py similarity index 60% rename from lbrynet/dht/peerfinder.py rename to lbrynet/extras/daemon/PeerFinder.py index 7d5eea159..2934b13f5 100644 --- a/lbrynet/dht/peerfinder.py +++ b/lbrynet/extras/daemon/PeerFinder.py @@ -2,7 +2,7 @@ import binascii import logging from twisted.internet import defer -from lbrynet.extras.daemon import conf +from lbrynet import conf log = logging.getLogger(__name__) @@ -18,13 +18,12 @@ class DHTPeerFinder(DummyPeerFinder): """This class finds peers which have announced to the DHT that they have certain blobs""" #implements(IPeerFinder) - def __init__(self, dht_node, peer_manager): + def __init__(self, component_manager): """ - dht_node - an instance of dht.Node class - peer_manager - an instance of PeerManager class + component_manager - an instance of ComponentManager """ - self.dht_node = dht_node - self.peer_manager = peer_manager + self.component_manager = component_manager + self.peer_manager = component_manager.peer_manager self.peers = {} self._ongoing_searchs = {} @@ -39,19 +38,30 @@ class DHTPeerFinder(DummyPeerFinder): Returns: list of peers for the blob """ - self.peers.setdefault(blob_hash, {(self.dht_node.externalIP, self.dht_node.peerPort,)}) + if "dht" in self.component_manager.skip_components: + return defer.succeed([]) + if not self.component_manager.all_components_running("dht"): + return defer.succeed([]) + dht_node = self.component_manager.get_component("dht") + + self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)}) if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called: - self._ongoing_searchs[blob_hash] = self._execute_peer_search(blob_hash, timeout) - peers = set(self._filter_self(blob_hash) if filter_self else self.peers[blob_hash]) + self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout) + + def _filter_self(blob_hash): + my_host, my_port = dht_node.externalIP, dht_node.peerPort + return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)} + + peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash]) return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers]) @defer.inlineCallbacks - def _execute_peer_search(self, blob_hash, timeout): + def _execute_peer_search(self, dht_node, blob_hash, timeout): bin_hash = binascii.unhexlify(blob_hash) - finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash]) + finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash]) timeout = timeout or conf.settings['peer_search_timeout'] if timeout: - finished_deferred.addTimeout(timeout, self.dht_node.clock) + finished_deferred.addTimeout(timeout, dht_node.clock) try: peer_list = yield finished_deferred self.peers[blob_hash].update({(host, port) for _, host, port in peer_list}) @@ -59,7 +69,3 @@ class DHTPeerFinder(DummyPeerFinder): log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout) finally: del self._ongoing_searchs[blob_hash] - - def _filter_self(self, blob_hash): - my_host, my_port = self.dht_node.externalIP, self.dht_node.peerPort - return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)} diff --git a/lbrynet/p2p/PeerManager.py b/lbrynet/extras/daemon/PeerManager.py similarity index 100% rename from lbrynet/p2p/PeerManager.py rename to lbrynet/extras/daemon/PeerManager.py diff --git a/lbrynet/extras/daemon/__init__.py b/lbrynet/extras/daemon/__init__.py index 7d3f2be07..9fb57d0d9 100644 --- a/lbrynet/extras/daemon/__init__.py +++ b/lbrynet/extras/daemon/__init__.py @@ -1 +1 @@ -from . import Components # register Component classes +from lbrynet.extras.daemon import Components # register Component classes diff --git a/lbrynet/extras/daemon/analytics.py b/lbrynet/extras/daemon/analytics.py index 2fe911b2f..0208d6757 100644 --- a/lbrynet/extras/daemon/analytics.py +++ b/lbrynet/extras/daemon/analytics.py @@ -4,8 +4,8 @@ import logging import treq from twisted.internet import defer, task -from lbrynet.extras.daemon import conf -from lbrynet.p2p import looping_call_manager, utils, system_info +from lbrynet import conf, utils +from lbrynet.extras import looping_call_manager, system_info # Things We Track SERVER_STARTUP = 'Server Startup' @@ -136,8 +136,8 @@ class Manager: def _get_looping_calls(self): return [ - ('send_heartbeat', self._send_heartbeat, 60), - ('update_tracked_metrics', self._update_tracked_metrics, 300), + ('send_heartbeat', self._send_heartbeat, 300), + ('update_tracked_metrics', self._update_tracked_metrics, 600), ] def _setup_looping_calls(self): diff --git a/lbrynet/extras/daemon/auth/auth.py b/lbrynet/extras/daemon/auth/auth.py index 5fb5cd2e7..8e350589a 100644 --- a/lbrynet/extras/daemon/auth/auth.py +++ b/lbrynet/extras/daemon/auth/auth.py @@ -3,7 +3,7 @@ from zope.interface import implementer from twisted.cred import portal, checkers, credentials, error as cred_error from twisted.internet import defer from twisted.web import resource -from .keyring import Keyring +from lbrynet.extras.daemon.auth.keyring import Keyring log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/client.py b/lbrynet/extras/daemon/auth/client.py index b34e61e50..50b0def8b 100644 --- a/lbrynet/extras/daemon/auth/client.py +++ b/lbrynet/extras/daemon/auth/client.py @@ -3,8 +3,8 @@ import aiohttp import logging from urllib.parse import urlparse -from lbrynet.extras.daemon import conf -from .keyring import Keyring, APIKey +from lbrynet import conf +from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey log = logging.getLogger(__name__) USER_AGENT = "AuthServiceProxy/0.1" diff --git a/lbrynet/extras/daemon/auth/factory.py b/lbrynet/extras/daemon/auth/factory.py index e9bb29375..163d49fe0 100644 --- a/lbrynet/extras/daemon/auth/factory.py +++ b/lbrynet/extras/daemon/auth/factory.py @@ -3,9 +3,9 @@ import logging from twisted.web import server, guard, resource from twisted.cred import portal -from lbrynet.extras.daemon import conf -from .auth import PasswordChecker, HttpPasswordRealm -from ..auth.keyring import Keyring +from lbrynet import conf +from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm +from lbrynet.extras.daemon.auth.keyring import Keyring log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/keyring.py b/lbrynet/extras/daemon/auth/keyring.py index 01ac81f16..eb4f7011a 100644 --- a/lbrynet/extras/daemon/auth/keyring.py +++ b/lbrynet/extras/daemon/auth/keyring.py @@ -3,6 +3,7 @@ import datetime import hmac import hashlib import base58 +import logging from OpenSSL.crypto import FILETYPE_PEM from ssl import create_default_context, SSLContext from cryptography.hazmat.backends import default_backend @@ -12,8 +13,8 @@ from cryptography.x509.name import NameOID, NameAttribute from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization from twisted.internet import ssl -import logging -from lbrynet.extras.daemon import conf + +from lbrynet import conf log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/server.py b/lbrynet/extras/daemon/auth/server.py index 3e94231db..82f814cf8 100644 --- a/lbrynet/extras/daemon/auth/server.py +++ b/lbrynet/extras/daemon/auth/server.py @@ -4,7 +4,6 @@ from six.moves.urllib import parse as urlparse import json import inspect import signal - from functools import wraps from twisted.web import server from twisted.internet import defer @@ -14,16 +13,17 @@ from twisted.internet.error import ConnectionDone, ConnectionLost from txjsonrpc import jsonrpclib from traceback import format_exc -from lbrynet.extras.daemon import analytics, conf +from lbrynet import conf, utils +from lbrynet.extras.daemon import analytics from lbrynet.p2p.Error import InvalidAuthenticationToken -from lbrynet.p2p import utils from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet -from lbrynet.p2p.looping_call_manager import LoopingCallManager +from lbrynet.extras.looping_call_manager import LoopingCallManager from lbrynet.extras.daemon.ComponentManager import ComponentManager -from .keyring import APIKey, Keyring -from .undecorated import undecorated -from .factory import AuthJSONRPCResource +from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring +from lbrynet.extras.daemon.auth.undecorated import undecorated +from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder + log = logging.getLogger(__name__) EMPTY_PARAMS = [{}] diff --git a/lbrynet/extras/daemon/migrator/migrate5to6.py b/lbrynet/extras/daemon/migrator/migrate5to6.py index 367039ab7..d8dcacdb5 100644 --- a/lbrynet/extras/daemon/migrator/migrate5to6.py +++ b/lbrynet/extras/daemon/migrator/migrate5to6.py @@ -2,8 +2,8 @@ import sqlite3 import os import json import logging +from lbrynet import conf from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/storage.py b/lbrynet/extras/daemon/storage.py index 48742034b..30c2d9330 100644 --- a/lbrynet/extras/daemon/storage.py +++ b/lbrynet/extras/daemon/storage.py @@ -6,13 +6,13 @@ from binascii import hexlify, unhexlify from decimal import Decimal from twisted.internet import defer, task, threads from twisted.enterprise import adbapi +from torba.client.constants import COIN +from lbrynet import conf from lbrynet.schema.claim import ClaimDict from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf from lbrynet.blob.CryptBlob import CryptBlobInfo from lbrynet.dht.constants import dataExpireTimeout -from torba.client.constants import COIN log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/looping_call_manager.py b/lbrynet/extras/looping_call_manager.py similarity index 100% rename from lbrynet/p2p/looping_call_manager.py rename to lbrynet/extras/looping_call_manager.py diff --git a/lbrynet/extras/reflector/__init__.py b/lbrynet/extras/reflector/__init__.py index d05ef4820..56f9bfdf6 100644 --- a/lbrynet/extras/reflector/__init__.py +++ b/lbrynet/extras/reflector/__init__.py @@ -63,6 +63,3 @@ If the transfer was not successful (False), the blob is re-added to the needed_b Blob requests continue for each of the blobs the client has queued to send, when completed the client disconnects. """ -from .server.server import ReflectorServerFactory as ServerFactory -from .client.client import EncryptedFileReflectorClientFactory as ClientFactory -from .client.blob import BlobReflectorClientFactory as BlobClientFactory diff --git a/lbrynet/extras/reflector/reupload.py b/lbrynet/extras/reflector/reupload.py index 5eca291d5..1a6567d27 100644 --- a/lbrynet/extras/reflector/reupload.py +++ b/lbrynet/extras/reflector/reupload.py @@ -1,8 +1,9 @@ import random from twisted.internet import reactor, defer -from lbrynet.extras.daemon import conf -from . import ClientFactory, BlobClientFactory +from lbrynet import conf +from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory +from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory def _is_ip(host): @@ -26,7 +27,7 @@ def resolve(host): @defer.inlineCallbacks def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server): reflector_address, reflector_port = reflector_server[0], reflector_server[1] - factory = ClientFactory(blob_manager, stream_hash, sd_hash) + factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash) ip = yield resolve(reflector_address) yield reactor.connectTCP(ip, reflector_port, factory) result = yield factory.finished_deferred @@ -40,7 +41,7 @@ def _reflect_file(lbry_file, reflector_server): @defer.inlineCallbacks def _reflect_blobs(blob_manager, blob_hashes, reflector_server): reflector_address, reflector_port = reflector_server[0], reflector_server[1] - factory = BlobClientFactory(blob_manager, blob_hashes) + factory = BlobReflectorClientFactory(blob_manager, blob_hashes) ip = yield resolve(reflector_address) yield reactor.connectTCP(ip, reflector_port, factory) result = yield factory.finished_deferred diff --git a/lbrynet/extras/reflector/server/server.py b/lbrynet/extras/reflector/server/server.py index 14bf2f889..f51869e38 100644 --- a/lbrynet/extras/reflector/server/server.py +++ b/lbrynet/extras/reflector/server/server.py @@ -3,7 +3,7 @@ import json from twisted.python import failure from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ServerFactory -from lbrynet.p2p.utils import is_valid_blobhash +from lbrynet.blob.blob_file import is_valid_blobhash from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader from lbrynet.p2p.StreamDescriptor import save_sd_info diff --git a/lbrynet/extras/system_info.py b/lbrynet/extras/system_info.py new file mode 100644 index 000000000..63fa3b065 --- /dev/null +++ b/lbrynet/extras/system_info.py @@ -0,0 +1,30 @@ +import platform +import os +import logging.handlers + +from lbrynet.schema import __version__ as schema_version +from lbrynet import build_type, __version__ as lbrynet_version + +log = logging.getLogger(__name__) + + +def get_platform() -> dict: + p = { + "processor": platform.processor(), + "python_version": platform.python_version(), + "platform": platform.platform(), + "os_release": platform.release(), + "os_system": platform.system(), + "lbrynet_version": lbrynet_version, + "lbryschema_version": schema_version, + "build": build_type.BUILD, # CI server sets this during build step + } + if p["os_system"] == "Linux": + try: + import distro + p["distro"] = distro.info() + p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown') + except ModuleNotFoundError: + pass + + return p diff --git a/lbrynet/extras/wallet/__init__.py b/lbrynet/extras/wallet/__init__.py index 80205e427..b2ee7b8a5 100644 --- a/lbrynet/extras/wallet/__init__.py +++ b/lbrynet/extras/wallet/__init__.py @@ -6,6 +6,6 @@ __node_url__ = ( ) __spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest' -from .ledger import MainNetLedger, RegTestLedger -from .manager import LbryWalletManager -from .network import Network +from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger +from lbrynet.extras.wallet.manager import LbryWalletManager +from lbrynet.extras.wallet.network import Network diff --git a/lbrynet/extras/wallet/ledger.py b/lbrynet/extras/wallet/ledger.py index dcddad4f7..7245ae8c4 100644 --- a/lbrynet/extras/wallet/ledger.py +++ b/lbrynet/extras/wallet/ledger.py @@ -2,17 +2,16 @@ import asyncio import logging from binascii import unhexlify -from lbrynet.extras.wallet.dewies import dewies_to_lbc +from torba.client.baseledger import BaseLedger from lbrynet.schema.error import URIParseError from lbrynet.schema.uri import parse_lbry_uri -from torba.client.baseledger import BaseLedger - -from .resolve import Resolver -from .account import Account -from .network import Network -from .database import WalletDatabase -from .transaction import Transaction -from .header import Headers, UnvalidatedHeaders +from lbrynet.extras.wallet.dewies import dewies_to_lbc +from lbrynet.extras.wallet.resolve import Resolver +from lbrynet.extras.wallet.account import Account +from lbrynet.extras.wallet.network import Network +from lbrynet.extras.wallet.database import WalletDatabase +from lbrynet.extras.wallet.transaction import Transaction +from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/manager.py b/lbrynet/extras/wallet/manager.py index 8e11cc1bd..692651e43 100644 --- a/lbrynet/extras/wallet/manager.py +++ b/lbrynet/extras/wallet/manager.py @@ -14,11 +14,11 @@ from torba.client.basemanager import BaseWalletManager from lbrynet.schema.claim import ClaimDict -from .ledger import MainNetLedger -from .account import BaseAccount, generate_certificate -from .transaction import Transaction -from .database import WalletDatabase -from .dewies import dewies_to_lbc +from lbrynet.extras.wallet.ledger import MainNetLedger +from lbrynet.extras.wallet.account import BaseAccount, generate_certificate +from lbrynet.extras.wallet.transaction import Transaction +from lbrynet.extras.wallet.database import WalletDatabase +from lbrynet.extras.wallet.dewies import dewies_to_lbc log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/resolve.py b/lbrynet/extras/wallet/resolve.py index b3195c394..6f5fef78f 100644 --- a/lbrynet/extras/wallet/resolve.py +++ b/lbrynet/extras/wallet/resolve.py @@ -9,8 +9,7 @@ from lbrynet.schema.claim import ClaimDict from lbrynet.schema.decode import smart_decode from lbrynet.schema.error import DecodeError from lbrynet.schema.uri import parse_lbry_uri - -from .claim_proofs import verify_proof, InvalidProofError +from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/server/block_processor.py b/lbrynet/extras/wallet/server/block_processor.py index 9937f4298..eae699d67 100644 --- a/lbrynet/extras/wallet/server/block_processor.py +++ b/lbrynet/extras/wallet/server/block_processor.py @@ -9,7 +9,7 @@ from lbrynet.schema.proto.claim_pb2 import Claim from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.decode import smart_decode -from .model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport +from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport class LBRYBlockProcessor(BlockProcessor): diff --git a/lbrynet/extras/wallet/server/coin.py b/lbrynet/extras/wallet/server/coin.py index 7d991565a..de5d8ef1e 100644 --- a/lbrynet/extras/wallet/server/coin.py +++ b/lbrynet/extras/wallet/server/coin.py @@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str, HASHX_LEN from hashlib import sha256 from torba.server.coins import Coin, CoinError -from .opcodes import decode_claim_script, opcodes as lbry_opcodes +from lbrynet.extras.wallet.server.opcodes import decode_claim_script, opcodes as lbry_opcodes class LBC(Coin): diff --git a/lbrynet/extras/wallet/server/db.py b/lbrynet/extras/wallet/server/db.py index 1bcb891bc..de2ad15b2 100644 --- a/lbrynet/extras/wallet/server/db.py +++ b/lbrynet/extras/wallet/server/db.py @@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str from torba.server.db import DB -from .model import ClaimInfo +from lbrynet.extras.wallet.server.model import ClaimInfo class LBRYDB(DB): diff --git a/lbrynet/extras/wallet/server/opcodes.py b/lbrynet/extras/wallet/server/opcodes.py index 2b697a064..8e17f5a25 100644 --- a/lbrynet/extras/wallet/server/opcodes.py +++ b/lbrynet/extras/wallet/server/opcodes.py @@ -1,6 +1,6 @@ import struct from torba.server.enum import Enumeration -from .model import NameClaim, ClaimSupport, ClaimUpdate +from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate # TODO: Take this to lbryschema (it's also on lbryum and lbryum-server) diff --git a/lbrynet/extras/wallet/server/session.py b/lbrynet/extras/wallet/server/session.py index 037e23817..50313da7e 100644 --- a/lbrynet/extras/wallet/server/session.py +++ b/lbrynet/extras/wallet/server/session.py @@ -8,8 +8,8 @@ from torba.server import util from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError -from .block_processor import LBRYBlockProcessor -from .db import LBRYDB +from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor +from lbrynet.extras.wallet.server.db import LBRYDB class LBRYElectrumX(ElectrumX): diff --git a/lbrynet/extras/wallet/server/tx.py b/lbrynet/extras/wallet/server/tx.py index 4d24c4088..eba24d751 100644 --- a/lbrynet/extras/wallet/server/tx.py +++ b/lbrynet/extras/wallet/server/tx.py @@ -1,6 +1,6 @@ from torba.server.tx import Deserializer -from .opcodes import decode_claim_script -from .model import TxClaimOutput, LBRYTx +from lbrynet.extras.wallet.server.opcodes import decode_claim_script +from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx class LBRYDeserializer(Deserializer): diff --git a/lbrynet/extras/wallet/transaction.py b/lbrynet/extras/wallet/transaction.py index 55856d7cf..fea2fff52 100644 --- a/lbrynet/extras/wallet/transaction.py +++ b/lbrynet/extras/wallet/transaction.py @@ -2,13 +2,12 @@ import struct from binascii import hexlify, unhexlify from typing import List, Iterable, Optional -from lbrynet.schema.decode import smart_decode -from .account import Account from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput from torba.client.hash import hash160 - +from lbrynet.schema.decode import smart_decode from lbrynet.schema.claim import ClaimDict -from .script import InputScript, OutputScript +from lbrynet.extras.wallet.account import Account +from lbrynet.extras.wallet.script import InputScript, OutputScript class Input(BaseInput): diff --git a/lbrynet/p2p/log_support.py b/lbrynet/log_support.py similarity index 99% rename from lbrynet/p2p/log_support.py rename to lbrynet/log_support.py index 1eed6659a..2b2803f5b 100644 --- a/lbrynet/p2p/log_support.py +++ b/lbrynet/log_support.py @@ -3,14 +3,11 @@ import logging import logging.handlers import sys import traceback - import treq -from twisted.internet import defer import twisted.python.log - +from twisted.internet import defer from lbrynet import __version__ as lbrynet_version, build_type -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet import utils, conf class HTTPSHandler(logging.Handler): diff --git a/lbrynet/p2p/BlobAvailability.py b/lbrynet/p2p/BlobAvailability.py index 36fb92a27..0ddb7072e 100644 --- a/lbrynet/p2p/BlobAvailability.py +++ b/lbrynet/p2p/BlobAvailability.py @@ -1,10 +1,9 @@ import logging import random import time - +from decimal import Decimal from twisted.internet import defer from twisted.internet.task import LoopingCall -from decimal import Decimal log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/HTTPBlobDownloader.py b/lbrynet/p2p/HTTPBlobDownloader.py index 8619737fa..28e6e2a4b 100644 --- a/lbrynet/p2p/HTTPBlobDownloader.py +++ b/lbrynet/p2p/HTTPBlobDownloader.py @@ -1,10 +1,11 @@ -from random import choice import logging +import treq +from random import choice from twisted.internet import defer, task from twisted.internet.error import ConnectingCancelledError from twisted.web._newclient import ResponseNeverReceived -import treq -from lbrynet.p2p.utils import DeferredDict + +from lbrynet.utils import DeferredDict from lbrynet.p2p.Error import DownloadCanceledError log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/PaymentRateManager.py b/lbrynet/p2p/PaymentRateManager.py index fcf7bbd54..7eb8b142c 100644 --- a/lbrynet/p2p/PaymentRateManager.py +++ b/lbrynet/p2p/PaymentRateManager.py @@ -1,6 +1,6 @@ -from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy -from lbrynet.extras.daemon import conf from decimal import Decimal +from lbrynet import conf +from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy class BasePaymentRateManager: diff --git a/lbrynet/p2p/Peer.py b/lbrynet/p2p/Peer.py index f06ce806d..bd52e0da3 100644 --- a/lbrynet/p2p/Peer.py +++ b/lbrynet/p2p/Peer.py @@ -1,6 +1,6 @@ import datetime from collections import defaultdict -from lbrynet.p2p import utils +from lbrynet import utils # Do not create this object except through PeerManager class Peer: diff --git a/lbrynet/p2p/PriceModel.py b/lbrynet/p2p/PriceModel.py index 14d7f0c34..4d065d47a 100644 --- a/lbrynet/p2p/PriceModel.py +++ b/lbrynet/p2p/PriceModel.py @@ -1,6 +1,5 @@ from decimal import Decimal - -from lbrynet.extras.daemon import conf +from lbrynet import conf def get_default_price_model(blob_tracker, base_price, **kwargs): diff --git a/lbrynet/p2p/SinglePeerDownloader.py b/lbrynet/p2p/SinglePeerDownloader.py index ab1eee31f..682939427 100644 --- a/lbrynet/p2p/SinglePeerDownloader.py +++ b/lbrynet/p2p/SinglePeerDownloader.py @@ -4,7 +4,7 @@ import tempfile from twisted.internet import defer, threads, reactor -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager @@ -12,7 +12,7 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht.peerfinder import DummyPeerFinder +from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/Strategy.py b/lbrynet/p2p/Strategy.py index 3477985ca..da478fb67 100644 --- a/lbrynet/p2p/Strategy.py +++ b/lbrynet/p2p/Strategy.py @@ -1,5 +1,5 @@ from decimal import Decimal -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.Offer import Offer from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice @@ -133,7 +133,6 @@ class BasicAvailabilityWeightedStrategy(Strategy): class OnlyFreeStrategy(Strategy): - def __init__(self, *args, **kwargs): price_model = ZeroPrice() super().__init__(price_model, 0.0, 0.0, True) diff --git a/lbrynet/p2p/StreamDescriptor.py b/lbrynet/p2p/StreamDescriptor.py index 9e8849dcb..9ef1048d3 100644 --- a/lbrynet/p2p/StreamDescriptor.py +++ b/lbrynet/p2p/StreamDescriptor.py @@ -1,11 +1,11 @@ -from binascii import unhexlify import string -from collections import defaultdict import json import logging - +from collections import defaultdict +from binascii import unhexlify from twisted.internet import threads, defer -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj + +from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader diff --git a/lbrynet/p2p/__init__.py b/lbrynet/p2p/__init__.py index df7d37558..6ac1f3432 100644 --- a/lbrynet/p2p/__init__.py +++ b/lbrynet/p2p/__init__.py @@ -5,5 +5,3 @@ This includes classes for connecting to other peers and downloading blobs from t connections from peers and responding to their requests, managing locally stored blobs, sending and receiving payments, and locating peers in the DHT. """ - -from lbrynet import custom_logger diff --git a/lbrynet/p2p/client/ClientProtocol.py b/lbrynet/p2p/client/ClientProtocol.py index 20568dc97..0f8167b79 100644 --- a/lbrynet/p2p/client/ClientProtocol.py +++ b/lbrynet/p2p/client/ClientProtocol.py @@ -5,8 +5,7 @@ from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ClientFactory from twisted.protocols.policies import TimeoutMixin from twisted.python import failure -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet import conf, utils from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError from lbrynet.p2p.Error import RequestCanceledError diff --git a/lbrynet/p2p/client/ConnectionManager.py b/lbrynet/p2p/client/ConnectionManager.py index f56fb0382..af299d167 100644 --- a/lbrynet/p2p/client/ConnectionManager.py +++ b/lbrynet/p2p/client/ConnectionManager.py @@ -1,10 +1,9 @@ import random import logging from twisted.internet import defer, reactor -from lbrynet.extras.daemon import conf +from lbrynet import utils, conf from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory from lbrynet.p2p.Error import InsufficientFundsError -from lbrynet.p2p import utils log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/client/StandaloneBlobDownloader.py b/lbrynet/p2p/client/StandaloneBlobDownloader.py index 08aa0b386..c9434c98c 100644 --- a/lbrynet/p2p/client/StandaloneBlobDownloader.py +++ b/lbrynet/p2p/client/StandaloneBlobDownloader.py @@ -4,7 +4,8 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.p2p.client.DownloadManager import DownloadManager from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout -from lbrynet.p2p.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call +from lbrynet.blob.blob_file import is_valid_blobhash +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call from twisted.python.failure import Failure from twisted.internet import defer from twisted.internet.task import LoopingCall diff --git a/lbrynet/p2p/system_info.py b/lbrynet/p2p/system_info.py deleted file mode 100644 index f8d0f42de..000000000 --- a/lbrynet/p2p/system_info.py +++ /dev/null @@ -1,59 +0,0 @@ -import platform -import json -import subprocess -import os - -from six.moves.urllib import request -from six.moves.urllib.error import URLError -from lbrynet.schema import __version__ as schema_version -from lbrynet import build_type, __version__ as lbrynet_version -from lbrynet.extras.daemon.conf import ROOT_DIR -import logging.handlers - -log = logging.getLogger(__name__) - - -def get_lbrynet_version() -> str: - if build_type.BUILD == "dev": - try: - with open(os.devnull, 'w') as devnull: - git_dir = ROOT_DIR + '/.git' - return subprocess.check_output( - ['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'], - stderr=devnull - ).decode().strip().lstrip('v') - except (subprocess.CalledProcessError, OSError): - log.debug("failed to get version from git") - return lbrynet_version - - -def get_platform(get_ip: bool = True) -> dict: - p = { - "processor": platform.processor(), - "python_version": platform.python_version(), - "platform": platform.platform(), - "os_release": platform.release(), - "os_system": platform.system(), - "lbrynet_version": get_lbrynet_version(), - "lbryschema_version": schema_version, - "build": build_type.BUILD, # CI server sets this during build step - } - if p["os_system"] == "Linux": - try: - import distro - p["distro"] = distro.info() - p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown') - except ModuleNotFoundError: - pass - - # TODO: remove this from get_platform and add a get_external_ip function using treq - if get_ip: - try: - response = json.loads(request.urlopen("https://api.lbry.io/ip").read()) - if not response['success']: - raise URLError("failed to get external ip") - p['ip'] = response['data']['ip'] - except (URLError, AssertionError): - p['ip'] = "Could not determine IP" - - return p diff --git a/lbrynet/schema/decode.py b/lbrynet/schema/decode.py index 95aa18a5b..e5be724e4 100644 --- a/lbrynet/schema/decode.py +++ b/lbrynet/schema/decode.py @@ -1,13 +1,12 @@ import json import binascii - +from google.protobuf import json_format # pylint: disable=no-name-in-module import six from lbrynet.schema.error import DecodeError, InvalidAddress from lbrynet.schema.legacy.migrate import migrate as schema_migrator from lbrynet.schema.claim import ClaimDict -from google.protobuf import json_format # pylint: disable=no-name-in-module def migrate_json_claim_value(decoded_json): diff --git a/lbrynet/schema/signature/flags.py b/lbrynet/schema/signature/flags.py deleted file mode 100644 index 259be0aff..000000000 --- a/lbrynet/schema/signature/flags.py +++ /dev/null @@ -1,41 +0,0 @@ -class SignatureSerializationFlag: - UNSIGNED = 0 - ''' - Format: - - or (legacy) - - ''' - ECDSA_LEGACY = 1 - ''' - Old claim format, which carried the signature inside the protobuf. Requires serializing back the claim with - signature stripped out for validation. This process requires knowledge on how a claim is serialized, thus requires - old fixed protobuf schema to work. - - Format: - - Curves: NIST256p, NIST384p, SECP256k1 - Signature content: `r` and `s` in each half of the 64 or 96 bytes (depends on curve) - Signed payload: - 1. Claim transaction output address (raw, decoded using base58) - 2. Stripped out claim protobuf serialization (without the signature) - 3. Certificate claim id (binary, not in network byte order) - ''' - ECDSA_SECP256K1 = 2 - ''' - Format: - - Curve: SECP256K1 - Signature content: 64 bytes total, each half represents `r` and `s` - Signed payload: - 1. raw claim name as bytes - 2. Claim transaction output address (raw, decoded using base58) - 3. Binary payload, independent of serialization (everything after the signature last byte) - 4. Certificate claim id, not in network byte order. - - A certificate can be signed as well, but this serialization model is unaware of content type or protobuf format. - ''' - @classmethod - def is_flag_valid(cls, flag): - # todo: use python 3 enum when fully ported, but not worth now as its an extra dependency for py2 - return 0 <= flag <= 2 \ No newline at end of file diff --git a/lbrynet/schema/signature/serializer.py b/lbrynet/schema/signature/serializer.py deleted file mode 100644 index 7ddcb2277..000000000 --- a/lbrynet/schema/signature/serializer.py +++ /dev/null @@ -1,11 +0,0 @@ -import struct -from collections import namedtuple -from .flags import SignatureSerializationFlag - - -class Signature(namedtuple("Signature", "flags signature certificate_id")): - def deserialize(cls, payload): - flag = struct.unpack(" pkg_resources.parse_version(b) diff --git a/lbrynet/extras/daemon/winpaths.py b/lbrynet/winpaths.py similarity index 100% rename from lbrynet/extras/daemon/winpaths.py rename to lbrynet/winpaths.py diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py index 2b83dce3e..3993ffaeb 100644 --- a/scripts/encrypt_blob.py +++ b/scripts/encrypt_blob.py @@ -9,7 +9,7 @@ from twisted.web.client import FileBodyProducer from lbrynet import conf from lbrynet.core import log_support -from lbrynet.dht.hashannouncer import DummyHashAnnouncer +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer from lbrynet.core.BlobManager import DiskBlobManager from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator diff --git a/tests/functional/dht/test_store.py b/tests/functional/dht/test_store.py index f3e81b332..b45eef3d7 100644 --- a/tests/functional/dht/test_store.py +++ b/tests/functional/dht/test_store.py @@ -3,7 +3,7 @@ from binascii import hexlify from twisted.internet import defer from lbrynet.dht import constants -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from .dht_test_environment import TestKademliaBase import logging diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 0cbead0c7..606c2b372 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -2,13 +2,13 @@ import os from hashlib import md5 from twisted.internet import defer, reactor from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.BlobManager import DiskBlobManager -from lbrynet.p2p.PeerManager import PeerManager +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory @@ -18,7 +18,7 @@ from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir FakeNode = mocks.Node FakeWallet = mocks.Wallet diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index c6b4128ac..6c2ff7285 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -4,8 +4,11 @@ from binascii import hexlify from twisted.internet import defer, error from twisted.trial import unittest from lbrynet.p2p.StreamDescriptor import get_sd_info -from lbrynet.extras import reflector -from lbrynet.p2p import BlobManager, PeerManager +from lbrynet.extras.reflector.server.server import ReflectorServerFactory +from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory +from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory +from lbrynet.extras.daemon.PeerManager import PeerManager +from lbrynet.p2p import BlobManager from lbrynet.p2p import StreamDescriptor from lbrynet.blob import EncryptedFileCreator from lbrynet.blob.EncryptedFileManager import EncryptedFileManager @@ -13,7 +16,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir class TestReflector(unittest.TestCase): @@ -25,7 +28,7 @@ class TestReflector(unittest.TestCase): self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir() prm = OnlyFreePaymentsManager() wallet = mocks.Wallet() - peer_manager = PeerManager.PeerManager() + peer_manager = PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) self.server_storage = SQLiteStorage(self.server_db_dir) self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage) @@ -95,7 +98,7 @@ class TestReflector(unittest.TestCase): return d def start_server(): - server_factory = reflector.ServerFactory(peer_manager, self.server_blob_manager, + server_factory = ReflectorServerFactory(peer_manager, self.server_blob_manager, self.server_lbry_file_manager) from twisted.internet import reactor port = 8943 @@ -179,7 +182,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(): - factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) + factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) from twisted.internet import reactor reactor.connectTCP('localhost', self.port, factory) @@ -208,7 +211,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -247,7 +250,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -301,7 +304,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server_as_blobs(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -312,7 +315,7 @@ class TestReflector(unittest.TestCase): return factory.finished_deferred def send_to_server_as_stream(result): - factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) + factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) from twisted.internet import reactor reactor.connectTCP('localhost', self.port, factory) diff --git a/tests/functional/test_streamify.py b/tests/functional/test_streamify.py index a044624b7..52008aeb9 100644 --- a/tests/functional/test_streamify.py +++ b/tests/functional/test_streamify.py @@ -7,8 +7,8 @@ from twisted.internet import defer, threads from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.StreamDescriptor import get_sd_info -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import DummyRateLimiter +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.blob.EncryptedFileCreator import create_lbry_file diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 855411cf0..d2861ec87 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -3,10 +3,10 @@ from twisted.trial import unittest from io import StringIO from twisted.internet import defer -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.extras import cli from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \ - DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, \ + DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \ PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, \ RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT from lbrynet.extras.daemon.Daemon import Daemon @@ -35,7 +35,7 @@ class CLIIntegrationTest(unittest.TestCase): def setUp(self): skip = [ DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, - DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, + DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT ] diff --git a/tests/integration/wallet/test_commands.py b/tests/integration/wallet/test_commands.py index 65e095515..0bdc2cf3c 100644 --- a/tests/integration/wallet/test_commands.py +++ b/tests/integration/wallet/test_commands.py @@ -15,7 +15,7 @@ from torba.testcase import IntegrationTestCase as BaseIntegrationTestCase import lbrynet.schema lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest' -from lbrynet.extras.daemon import conf as lbry_conf +from lbrynet import conf as lbry_conf from lbrynet.dht.node import Node from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.wallet import LbryWalletManager diff --git a/tests/mocks.py b/tests/mocks.py index 8e9be2294..47ec2e583 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -7,13 +7,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization from twisted.internet import defer from twisted.python.failure import Failure - +from lbrynet import conf from lbrynet.p2p.client.ClientRequest import ClientRequest from lbrynet.p2p.Error import RequestCanceledError from lbrynet.p2p import BlobAvailability from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.dht.node import Node as RealNode -from lbrynet.extras.daemon import ExchangeRateManager as ERM, conf +from lbrynet.extras.daemon import ExchangeRateManager as ERM KB = 2**10 PUBLIC_EXPONENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html diff --git a/tests/util.py b/tests/test_utils.py similarity index 91% rename from tests/util.py rename to tests/test_utils.py index 8b65b7033..c03de4745 100644 --- a/tests/util.py +++ b/tests/test_utils.py @@ -32,13 +32,14 @@ def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP): patcher.start().return_value = iso_time test_case.addCleanup(patcher.stop) - patcher = mock.patch('lbrynet.p2p.utils.now') + patcher = mock.patch('lbrynet.utils.now') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) - patcher = mock.patch('lbrynet.p2p.utils.utcnow') + patcher = mock.patch('lbrynet.utils.utcnow') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) + def is_android(): return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way diff --git a/tests/unit/components/test_Component_Manager.py b/tests/unit/components/test_Component_Manager.py index 15feeb78b..782711f43 100644 --- a/tests/unit/components/test_Component_Manager.py +++ b/tests/unit/components/test_Component_Manager.py @@ -2,7 +2,7 @@ from twisted.internet.task import Clock from twisted.trial import unittest from lbrynet.extras.daemon.ComponentManager import ComponentManager -from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, STREAM_IDENTIFIER_COMPONENT +from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT @@ -13,6 +13,7 @@ from tests import mocks class TestComponentManager(unittest.TestCase): def setUp(self): mocks.mock_conf_settings(self) + self.default_components_sort = [ [ Components.HeadersComponent, @@ -23,19 +24,14 @@ class TestComponentManager(unittest.TestCase): Components.UPnPComponent ], [ + Components.BlobComponent, Components.DHTComponent, Components.WalletComponent ], [ - Components.BlobComponent, - Components.HashAnnouncerComponent - ], - [ - Components.PeerProtocolServerComponent, - Components.StreamIdentifierComponent - ], - [ - Components.FileManagerComponent + Components.FileManagerComponent, + Components.HashAnnouncerComponent, + Components.PeerProtocolServerComponent ], [ Components.ReflectorComponent @@ -48,7 +44,6 @@ class TestComponentManager(unittest.TestCase): def test_sort_components(self): stages = self.component_manager.sort_components() - for stage_list, sorted_stage_list in zip(stages, self.default_components_sort): self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list) @@ -101,7 +96,7 @@ class TestComponentManagerProperStart(unittest.TestCase): self.reactor = Clock() mocks.mock_conf_settings(self) self.component_manager = ComponentManager( - skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, + skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT], diff --git a/tests/unit/core/client/test_ConnectionManager.py b/tests/unit/core/client/test_ConnectionManager.py index b8c0b301d..7a9d95069 100644 --- a/tests/unit/core/client/test_ConnectionManager.py +++ b/tests/unit/core/client/test_ConnectionManager.py @@ -1,17 +1,16 @@ +from twisted.trial.unittest import TestCase +from twisted.internet import defer, reactor, task +from twisted.internet.task import deferLater +from twisted.internet.protocol import ServerFactory + +from lbrynet import conf, utils from lbrynet.p2p.client.ClientRequest import ClientRequest from lbrynet.p2p.server.ServerProtocol import ServerProtocol from lbrynet.p2p.client.ClientProtocol import ClientProtocol from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.Peer import Peer -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.Error import NoResponseError - -from twisted.trial.unittest import TestCase -from twisted.internet import defer, reactor, task -from twisted.internet.task import deferLater -from twisted.internet.protocol import ServerFactory -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet.extras.daemon.PeerManager import PeerManager PEER_PORT = 5551 LOCAL_HOST = '127.0.0.1' diff --git a/tests/unit/core/test_BlobManager.py b/tests/unit/core/test_BlobManager.py index 590cb6225..70338ff1f 100644 --- a/tests/unit/core/test_BlobManager.py +++ b/tests/unit/core/test_BlobManager.py @@ -6,12 +6,12 @@ import string from twisted.trial import unittest from twisted.internet import defer -from tests.util import random_lbry_hash +from tests.test_utils import random_lbry_hash from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.Peer import Peer -from lbrynet.extras.daemon import conf -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet import conf +from lbrynet.cryptoutils import get_lbry_hash_obj class BlobManagerTest(unittest.TestCase): diff --git a/tests/unit/core/test_HTTPBlobDownloader.py b/tests/unit/core/test_HTTPBlobDownloader.py index 40fb98452..3916e6ba8 100644 --- a/tests/unit/core/test_HTTPBlobDownloader.py +++ b/tests/unit/core/test_HTTPBlobDownloader.py @@ -3,9 +3,9 @@ from mock import MagicMock from twisted.trial import unittest from twisted.internet import defer -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir class HTTPBlobDownloaderTest(unittest.TestCase): diff --git a/tests/unit/core/test_HashBlob.py b/tests/unit/core/test_HashBlob.py index 5e7ab86ff..da8b3f5bc 100644 --- a/tests/unit/core/test_HashBlob.py +++ b/tests/unit/core/test_HashBlob.py @@ -1,7 +1,7 @@ -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/unit/core/test_Wallet.py b/tests/unit/core/test_Wallet.py index 3083be6f4..d5fa5e193 100644 --- a/tests/unit/core/test_Wallet.py +++ b/tests/unit/core/test_Wallet.py @@ -5,9 +5,6 @@ from decimal import Decimal from twisted.trial import unittest from twisted.internet import defer from lbrynet.p2p.Error import InsufficientFundsError -#from lbrynet.p2p.Wallet import LBRYumWallet, ReservedPoints -#from lbryum.commands import Commands -#from lbryum.simple_config import SimpleConfig from lbrynet.schema.claim import ClaimDict test_metadata = { diff --git a/tests/unit/core/test_utils.py b/tests/unit/core/test_utils.py index c4f1289e2..bfb5f8b86 100644 --- a/tests/unit/core/test_utils.py +++ b/tests/unit/core/test_utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from lbrynet.p2p import utils +from lbrynet import utils from twisted.trial import unittest diff --git a/tests/unit/database/test_SQLiteStorage.py b/tests/unit/database/test_SQLiteStorage.py index 318a05a8e..e8794bbb7 100644 --- a/tests/unit/database/test_SQLiteStorage.py +++ b/tests/unit/database/test_SQLiteStorage.py @@ -5,10 +5,10 @@ import logging from copy import deepcopy from twisted.internet import defer from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader -from tests.util import random_lbry_hash +from tests.test_utils import random_lbry_hash log = logging.getLogger() diff --git a/tests/unit/dht/test_contact.py b/tests/unit/dht/test_contact.py index 3543854d9..2721a8fe7 100644 --- a/tests/unit/dht/test_contact.py +++ b/tests/unit/dht/test_contact.py @@ -1,7 +1,7 @@ from binascii import hexlify from twisted.internet import task from twisted.trial import unittest -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from lbrynet.dht.contact import ContactManager from lbrynet.dht import constants diff --git a/tests/unit/dht/test_hash_announcer.py b/tests/unit/dht/test_hash_announcer.py index 7919a01e1..5d7179712 100644 --- a/tests/unit/dht/test_hash_announcer.py +++ b/tests/unit/dht/test_hash_announcer.py @@ -1,8 +1,8 @@ from twisted.trial import unittest from twisted.internet import defer, task -from lbrynet.p2p import utils -from lbrynet.dht.hashannouncer import DHTHashAnnouncer -from tests.util import random_lbry_hash +from lbrynet import utils +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer +from tests.test_utils import random_lbry_hash from tests.mocks import mock_conf_settings diff --git a/tests/unit/dht/test_kbucket.py b/tests/unit/dht/test_kbucket.py index b746afa7b..360dee32d 100644 --- a/tests/unit/dht/test_kbucket.py +++ b/tests/unit/dht/test_kbucket.py @@ -6,7 +6,7 @@ from twisted.trial import unittest import struct -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from lbrynet.dht import kbucket from lbrynet.dht.contact import ContactManager from lbrynet.dht import constants diff --git a/tests/unit/dht/test_node.py b/tests/unit/dht/test_node.py index a82cdb803..983975235 100644 --- a/tests/unit/dht/test_node.py +++ b/tests/unit/dht/test_node.py @@ -5,7 +5,7 @@ from twisted.trial import unittest from twisted.internet import defer from lbrynet.dht.node import Node from lbrynet.dht import constants -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id class NodeIDTest(unittest.TestCase): diff --git a/tests/unit/dht/test_routingtable.py b/tests/unit/dht/test_routingtable.py index 37131b53f..1c7985cc8 100644 --- a/tests/unit/dht/test_routingtable.py +++ b/tests/unit/dht/test_routingtable.py @@ -6,7 +6,7 @@ from lbrynet.dht import constants from lbrynet.dht.routingtable import TreeRoutingTable from lbrynet.dht.contact import ContactManager from lbrynet.dht.distance import Distance -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id class FakeRPCProtocol: diff --git a/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 8266f29bf..36d6db299 100644 --- a/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -3,10 +3,10 @@ from twisted.trial import unittest from twisted.internet import defer from cryptography.hazmat.primitives.ciphers.algorithms import AES +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.p2p.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.BlobManager import DiskBlobManager -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.extras.daemon.storage import SQLiteStorage @@ -14,7 +14,7 @@ from lbrynet.blob import EncryptedFileCreator from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.p2p.StreamDescriptor import JSONBytesEncoder from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir FakeNode = mocks.Node diff --git a/tests/unit/lbrynet_daemon/auth/test_server.py b/tests/unit/lbrynet_daemon/auth/test_server.py index 69b1a12be..849a2634c 100644 --- a/tests/unit/lbrynet_daemon/auth/test_server.py +++ b/tests/unit/lbrynet_daemon/auth/test_server.py @@ -1,7 +1,7 @@ import mock from twisted.internet import reactor from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.extras.daemon.auth import server from tests.mocks import mock_conf_settings diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index fed651391..fd8b6d5ce 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -7,12 +7,11 @@ from twisted.internet import defer from twisted.trial import unittest from faker import Faker - +from lbrynet import conf from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.extras.daemon.ComponentManager import ComponentManager -from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, STREAM_IDENTIFIER_COMPONENT +from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT from lbrynet.extras.daemon.Components import f2d from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, BLOB_COMPONENT from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT @@ -23,29 +22,24 @@ from lbrynet.extras.wallet import LbryWalletManager from torba.client.wallet import Wallet from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager -from tests import util +from tests import test_utils from tests.mocks import mock_conf_settings, FakeNetwork, FakeFileManager from tests.mocks import ExchangeRateManager as DummyExchangeRateManager from tests.mocks import BTCLBCFeed, USDBTCFeed -from tests.util import is_android - - -import logging -logging.getLogger("lbryum").setLevel(logging.WARNING) +from tests.test_utils import is_android def get_test_daemon(data_rate=None, generous=True, with_fee=False): if data_rate is None: data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1] - rates = { - 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, - 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} + 'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, + 'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2} } component_manager = ComponentManager( skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT, - STREAM_IDENTIFIER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, RATE_LIMITER_COMPONENT], file_manager=FakeFileManager ) @@ -89,7 +83,7 @@ class TestCostEst(unittest.TestCase): def setUp(self): mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) @defer.inlineCallbacks def test_fee_and_generous_data(self): @@ -134,7 +128,7 @@ class TestJsonRpc(unittest.TestCase): return None mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) self.test_daemon = get_test_daemon() self.test_daemon.wallet_manager.is_first_run = False self.test_daemon.wallet_manager.get_best_blockhash = noop @@ -156,7 +150,7 @@ class TestFileListSorting(unittest.TestCase): def setUp(self): mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) self.faker = Faker('en_US') self.faker.seed(129) # contains 3 same points paid (5.9) self.test_daemon = get_test_daemon() diff --git a/tests/unit/lbrynet_daemon/test_Downloader.py b/tests/unit/lbrynet_daemon/test_Downloader.py index f930d2d5e..aaf600b5e 100644 --- a/tests/unit/lbrynet_daemon/test_Downloader.py +++ b/tests/unit/lbrynet_daemon/test_Downloader.py @@ -12,7 +12,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.extras.daemon import Downloader from lbrynet.extras.daemon import ExchangeRateManager from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht.peerfinder import DummyPeerFinder +from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.extras.wallet import LbryWalletManager diff --git a/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py b/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py index b3f2f1b3e..651d3c35e 100644 --- a/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py +++ b/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py @@ -3,7 +3,7 @@ from lbrynet.extras.daemon import ExchangeRateManager from lbrynet.p2p.Error import InvalidExchangeRateResponse from twisted.trial import unittest from twisted.internet import defer -from tests import util +from tests import test_utils from tests.mocks import ExchangeRateManager as DummyExchangeRateManager from tests.mocks import BTCLBCFeed, USDBTCFeed @@ -32,18 +32,18 @@ class FeeFormatTest(unittest.TestCase): class ExchangeRateTest(unittest.TestCase): def setUp(self): - util.resetTime(self) + test_utils.resetTime(self) def test_invalid_rates(self): with self.assertRaises(ValueError): - ExchangeRateManager.ExchangeRate('USDBTC', 0, util.DEFAULT_ISO_TIME) + ExchangeRateManager.ExchangeRate('USDBTC', 0, test_utils.DEFAULT_ISO_TIME) with self.assertRaises(ValueError): - ExchangeRateManager.ExchangeRate('USDBTC', -1, util.DEFAULT_ISO_TIME) + ExchangeRateManager.ExchangeRate('USDBTC', -1, test_utils.DEFAULT_ISO_TIME) class FeeTest(unittest.TestCase): def setUp(self): - util.resetTime(self) + test_utils.resetTime(self) def test_fee_converts_to_lbc(self): fee = Fee({ @@ -53,8 +53,8 @@ class FeeTest(unittest.TestCase): }) rates = { - 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, - 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} + 'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, + 'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2} } market_feeds = [BTCLBCFeed(), USDBTCFeed()] @@ -71,7 +71,7 @@ class FeeTest(unittest.TestCase): }) rates = { - 'BTCLBC': {'spot': 1.0, 'ts': util.DEFAULT_ISO_TIME + 1}, + 'BTCLBC': {'spot': 1.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, } market_feeds = [BTCLBCFeed()] manager = DummyExchangeRateManager(market_feeds, rates) diff --git a/tests/unit/lbrynet_daemon/test_claims_comparator.py b/tests/unit/lbrynet_daemon/test_claims_comparator.py index afd246900..a1d6dd0aa 100644 --- a/tests/unit/lbrynet_daemon/test_claims_comparator.py +++ b/tests/unit/lbrynet_daemon/test_claims_comparator.py @@ -1,4 +1,4 @@ -import unittest +from twisted.trial import unittest from lbrynet.extras.daemon.Daemon import sort_claim_results diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index d4cbf96a1..772b3ae09 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -3,7 +3,7 @@ from io import StringIO from twisted.trial import unittest from lbrynet.extras.cli import normalize_value, main -from lbrynet.p2p.system_info import get_platform +from lbrynet.extras.system_info import get_platform class CLITest(unittest.TestCase): @@ -67,7 +67,7 @@ class CLITest(unittest.TestCase): main(['version']) self.assertEqual( actual_output.getvalue().strip(), - "lbrynet {lbrynet_version}".format(**get_platform(get_ip=False)) + "lbrynet {lbrynet_version}".format(**get_platform()) ) def test_invalid_command(self): diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index 8958d64d6..5ab939b6d 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -4,7 +4,7 @@ import sys import tempfile from unittest import skipIf from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.Error import InvalidCurrencyError diff --git a/tests/unit/test_customLogger.py b/tests/unit/test_customLogger.py index 051b20185..77ad3b8e4 100644 --- a/tests/unit/test_customLogger.py +++ b/tests/unit/test_customLogger.py @@ -1,16 +1,14 @@ from io import StringIO import logging - import mock -import unittest +from unittest import skipIf from twisted.internet import defer -from twisted import trial - +from twisted.trial import unittest from lbrynet import custom_logger -from tests.util import is_android +from tests.test_utils import is_android -class TestLogger(trial.unittest.TestCase): +class TestLogger(unittest.TestCase): def raiseError(self): raise Exception('terrible things happened') @@ -28,7 +26,7 @@ class TestLogger(trial.unittest.TestCase): handler.setFormatter(logging.Formatter("%(filename)s:%(lineno)d - %(message)s")) self.log.addHandler(handler) - @unittest.skipIf(is_android(), + @skipIf(is_android(), 'Test cannot pass on Android because the tests package is compiled ' 'which results in a different method call stack') def test_can_log_failure(self): @@ -36,7 +34,7 @@ class TestLogger(trial.unittest.TestCase): return self.stream.getvalue().split('\n') # the line number could change if this file gets refactored - expected_first_line = 'test_customLogger.py:20 - My message: terrible things happened' + expected_first_line = 'test_customLogger.py:18 - My message: terrible things happened' # testing the entirety of the message is futile as the # traceback will depend on the system the test is being run on