diff --git a/lbrynet/daemon/Components.py b/lbrynet/daemon/Components.py index 6dee54317..593135034 100644 --- a/lbrynet/daemon/Components.py +++ b/lbrynet/daemon/Components.py @@ -3,8 +3,8 @@ import logging from hashlib import sha256 import treq import math +import binascii from twisted.internet import defer, threads, reactor, error -from txupnp.fault import UPnPError from txupnp.upnp import UPnP from lbryum.simple_config import SimpleConfig from lbryum.constants import HEADERS_URL, HEADER_SIZE @@ -183,9 +183,7 @@ class HeadersComponent(Component): return self def get_status(self): - if self._downloading_headers is None: - return {} - return { + return {} if not self._downloading_headers else { 'downloading_headers': self._downloading_headers, 'download_progress': self._headers_progress_percent } @@ -314,17 +312,16 @@ class WalletComponent(Component): @defer.inlineCallbacks def get_status(self): - if not self.wallet: - return - local_height = self.wallet.network.get_local_height() - remote_height = self.wallet.network.get_server_height() - best_hash = yield self.wallet.get_best_blockhash() - defer.returnValue({ - 'blocks': local_height, - 'blocks_behind': remote_height - local_height, - 'best_blockhash': best_hash, - 'is_encrypted': self.wallet.wallet.use_encryption - }) + if self.wallet: + local_height = self.wallet.network.get_local_height() + remote_height = self.wallet.network.get_server_height() + best_hash = yield self.wallet.get_best_blockhash() + defer.returnValue({ + 'blocks': local_height, + 'blocks_behind': remote_height - local_height, + 'best_blockhash': best_hash, + 'is_encrypted': self.wallet.wallet.use_encryption + }) @defer.inlineCallbacks def start(self): @@ -362,9 +359,8 @@ class BlobComponent(Component): @defer.inlineCallbacks def get_status(self): - if not self.blob_manager: - count = 0 - else: + count = 0 + if self.blob_manager: count = yield self.blob_manager.storage.count_finished_blobs() defer.returnValue({ 'finished_blobs': count @@ -388,7 +384,7 @@ class DHTComponent(Component): def get_status(self): return { - 'node_id': CS.get_node_id().encode('hex'), + 'node_id': binascii.hexlify(CS.get_node_id()), 'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts) } diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 677839aa9..0805a37ba 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -325,11 +325,12 @@ class Daemon(AuthJSONRPCServer): else: download_id = utils.random_string() self.analytics_manager.send_download_started(download_id, name, claim_dict) - self.streams[sd_hash] = GetStream(self.sd_identifier, self.wallet, self.exchange_rate_manager, - self.blob_manager, self.dht_node.peer_finder, self.rate_limiter, - self.payment_rate_manager, self.storage, conf.settings['max_key_fee'], - conf.settings['disable_max_key_fee'], conf.settings['data_rate'], - timeout) + self.streams[sd_hash] = GetStream( + self.sd_identifier, self.wallet, self.exchange_rate_manager, self.blob_manager, + self.dht_node.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage, + conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'], + timeout + ) try: lbry_file, finished_deferred = yield self.streams[sd_hash].start( claim_dict, name, txid, nout, file_name @@ -355,8 +356,9 @@ class Daemon(AuthJSONRPCServer): @defer.inlineCallbacks def _publish_stream(self, name, bid, claim_dict, file_path=None, certificate_id=None, claim_address=None, change_address=None): - publisher = Publisher(self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, - self.wallet, certificate_id) + publisher = Publisher( + self.blob_manager, self.payment_rate_manager, self.storage, self.file_manager, self.wallet, certificate_id + ) parse_lbry_uri(name) if not file_path: stream_hash = yield self.storage.get_stream_hash_for_sd_hash( @@ -2552,8 +2554,7 @@ class Daemon(AuthJSONRPCServer): } timeout = timeout or 30 - blob = yield self._download_blob(blob_hash, rate_manager=self.payment_rate_manager, - timeout=timeout) + blob = yield self._download_blob(blob_hash, rate_manager=self.payment_rate_manager, timeout=timeout) if encoding and encoding in decoders: blob_file = blob.open_for_reading() result = decoders[encoding](blob_file.read()) diff --git a/lbrynet/daemon/Downloader.py b/lbrynet/daemon/Downloader.py index 83063b5e4..e554e9455 100644 --- a/lbrynet/daemon/Downloader.py +++ b/lbrynet/daemon/Downloader.py @@ -177,9 +177,10 @@ class GetStream(object): @defer.inlineCallbacks def _download_sd_blob(self): - sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, - self.payment_rate_manager, self.wallet, self.timeout, - conf.settings['download_mirrors']) + sd_blob = yield download_sd_blob( + self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager, + self.wallet, self.timeout, conf.settings['download_mirrors'] + ) defer.returnValue(sd_blob) @defer.inlineCallbacks diff --git a/lbrynet/daemon/Publisher.py b/lbrynet/daemon/Publisher.py index fd8dad73b..b64adebfe 100644 --- a/lbrynet/daemon/Publisher.py +++ b/lbrynet/daemon/Publisher.py @@ -32,8 +32,10 @@ class Publisher(object): file_name = os.path.basename(file_path) with file_utils.get_read_handle(file_path) as read_handle: - self.lbry_file = yield create_lbry_file(self.blob_manager, self.storage, self.payment_rate_manager, - self.lbry_file_manager, file_name, read_handle) + self.lbry_file = yield create_lbry_file( + self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name, + read_handle + ) if 'source' not in claim_dict['stream']: claim_dict['stream']['source'] = {} @@ -44,8 +46,9 @@ class Publisher(object): claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address) # check if we have a file already for this claim (if this is a publish update with a new stream) - old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(claim_out['claim_id'], - self.lbry_file.stream_hash) + old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id( + claim_out['claim_id'], self.lbry_file.stream_hash + ) if old_stream_hashes: for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes, list(self.lbry_file_manager.lbry_files)): @@ -62,8 +65,9 @@ class Publisher(object): """Make a claim without creating a lbry file""" claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address) if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have - yield self.storage.save_content_claim(stream_hash, "%s:%i" % (claim_out['txid'], - claim_out['nout'])) + yield self.storage.save_content_claim( + stream_hash, "%s:%i" % (claim_out['txid'], claim_out['nout']) + ) self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0] defer.returnValue(claim_out) diff --git a/lbrynet/tests/functional/test_misc.py b/lbrynet/tests/functional/test_misc.py index eb100fbf0..a86a38f69 100644 --- a/lbrynet/tests/functional/test_misc.py +++ b/lbrynet/tests/functional/test_misc.py @@ -149,9 +149,13 @@ class TestTransfer(unittest.TestCase): @defer.inlineCallbacks def test_lbry_transfer(self): - sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.prm, self.wallet) + sd_blob = yield download_sd_blob( + self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.prm, self.wallet + ) metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob) - downloader = yield metadata.factories[0].make_downloader(metadata, self.prm.min_blob_data_payment_rate, self.prm, self.db_dir, download_mirrors=None) + downloader = yield metadata.factories[0].make_downloader( + metadata, self.prm.min_blob_data_payment_rate, self.prm, self.db_dir, download_mirrors=None + ) yield downloader.start() with open(os.path.join(self.db_dir, 'test_file')) as f: hashsum = md5() diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 6cc87053b..efa5b4f8a 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -21,26 +21,22 @@ class TestReflector(unittest.TestCase): mocks.mock_conf_settings(self) self.server_db_dir, self.server_blob_dir = mk_db_and_blob_dir() self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir() - prm = OnlyFreePaymentsManager() wallet = mocks.Wallet() peer_manager = PeerManager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) - self.server_storage = SQLiteStorage(self.server_db_dir) self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage) - self.client_storage = SQLiteStorage(self.client_db_dir) self.client_blob_manager = BlobManager.DiskBlobManager(self.client_blob_dir, self.client_storage) - - self.server_lbry_file_manager = EncryptedFileManager(peer_finder, DummyRateLimiter(), - self.server_blob_manager, wallet, prm, - self.server_storage, - StreamDescriptor.StreamDescriptorIdentifier()) - self.client_lbry_file_manager = EncryptedFileManager(peer_finder, DummyRateLimiter(), - self.client_blob_manager, wallet, prm, - self.client_storage, - StreamDescriptor.StreamDescriptorIdentifier()) + self.server_lbry_file_manager = EncryptedFileManager( + peer_finder, DummyRateLimiter(), self.server_blob_manager, wallet, prm, self.server_storage, + StreamDescriptor.StreamDescriptorIdentifier() + ) + self.client_lbry_file_manager = EncryptedFileManager( + peer_finder, DummyRateLimiter(), self.client_blob_manager, wallet, prm, self.client_storage, + StreamDescriptor.StreamDescriptorIdentifier() + ) self.expected_blobs = [ ( diff --git a/lbrynet/tests/functional/test_streamify.py b/lbrynet/tests/functional/test_streamify.py index e8fcbaf67..ddea87547 100644 --- a/lbrynet/tests/functional/test_streamify.py +++ b/lbrynet/tests/functional/test_streamify.py @@ -45,8 +45,10 @@ class TestStreamify(TestCase): self.storage = SQLiteStorage(self.db_dir) self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore) self.prm = OnlyFreePaymentsManager() - self.lbry_file_manager = EncryptedFileManager(self.peer_finder, self.rate_limiter, self.blob_manager, - self.wallet, self.prm, self.storage, self.sd_identifier) + self.lbry_file_manager = EncryptedFileManager( + self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage, + self.sd_identifier + ) d = self.storage.setup() d.addCallback(lambda _: self.lbry_file_manager.setup()) return d @@ -80,8 +82,10 @@ class TestStreamify(TestCase): def create_stream(): test_file = GenFile(5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) - d = create_lbry_file(self.blob_manager, self.storage, self.prm, self.lbry_file_manager, "test_file", - test_file, key="0123456701234567", iv_generator=iv_generator()) + d = create_lbry_file( + self.blob_manager, self.storage, self.prm, self.lbry_file_manager, "test_file", test_file, + key="0123456701234567", iv_generator=iv_generator() + ) d.addCallback(lambda lbry_file: lbry_file.stream_hash) return d diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 17c018855..3716587f3 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -456,6 +456,10 @@ class FakeFileManager(FakeComponent): def stop(self): pass + def get_status(self): + return {} + + create_stream_sd_file = { 'stream_name': '746573745f66696c65', 'blobs': [ diff --git a/lbrynet/tests/unit/components/test_Component_Manager.py b/lbrynet/tests/unit/components/test_Component_Manager.py index 2b7fb9798..6b35d0aba 100644 --- a/lbrynet/tests/unit/components/test_Component_Manager.py +++ b/lbrynet/tests/unit/components/test_Component_Manager.py @@ -5,7 +5,7 @@ from lbrynet.daemon.ComponentManager import ComponentManager from lbrynet.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, STREAM_IDENTIFIER_COMPONENT from lbrynet.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT from lbrynet.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT -from lbrynet.daemon.Components import BLOB_COMPONENT, RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT +from lbrynet.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT from lbrynet.daemon import Components from lbrynet.tests import mocks diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 29dfddc9b..2c5e671ba 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -63,8 +63,9 @@ class CreateEncryptedFileTest(unittest.TestCase): def create_file(self, filename): handle = mocks.GenFile(3*MB, '1') key = '2' * (AES.block_size / 8) - out = yield EncryptedFileCreator.create_lbry_file(self.blob_manager, self.storage, self.prm, self.lbry_file_manager, filename, handle, - key, iv_generator()) + out = yield EncryptedFileCreator.create_lbry_file( + self.blob_manager, self.storage, self.prm, self.lbry_file_manager, filename, handle, key, iv_generator() + ) defer.returnValue(out) @defer.inlineCallbacks diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py index b80d4b029..f8925b0b6 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py @@ -15,8 +15,9 @@ from lbrynet.core import Wallet from lbrynet.database.storage import SQLiteStorage from lbrynet.daemon.ComponentManager import ComponentManager from lbrynet.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, STREAM_IDENTIFIER_COMPONENT -from lbrynet.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT +from lbrynet.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, BLOB_COMPONENT from lbrynet.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT +from lbrynet.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, FILE_MANAGER_COMPONENT from lbrynet.daemon.Daemon import Daemon as LBRYDaemon from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager @@ -39,7 +40,14 @@ def get_test_daemon(data_rate=None, generous=True, with_fee=False): 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} } - daemon = LBRYDaemon() + component_manager = ComponentManager( + skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, + PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT, + STREAM_IDENTIFIER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + HEADERS_COMPONENT, RATE_LIMITER_COMPONENT], + file_manager=FakeFileManager + ) + daemon = LBRYDaemon(component_manager=component_manager) daemon.payment_rate_manager = OnlyFreePaymentsManager() daemon.wallet = mock.Mock(spec=Wallet.LBRYumWallet) daemon.wallet.wallet = mock.Mock(spec=NewWallet) @@ -48,6 +56,7 @@ def get_test_daemon(data_rate=None, generous=True, with_fee=False): daemon.storage = mock.Mock(spec=SQLiteStorage) market_feeds = [BTCLBCFeed(), USDBTCFeed()] daemon.exchange_rate_manager = DummyExchangeRateManager(market_feeds, rates) + daemon.file_manager = component_manager.get_component(FILE_MANAGER_COMPONENT) metadata = { "author": "fake author", @@ -117,10 +126,6 @@ class TestJsonRpc(unittest.TestCase): mock_conf_settings(self) util.resetTime(self) self.test_daemon = get_test_daemon() - for component in self.test_daemon.component_manager.components: - if component.component_name == "file_manager": - component._running = True - self.test_daemon.wallet.is_first_run = False self.test_daemon.wallet.get_best_blockhash = noop @@ -144,17 +149,7 @@ class TestFileListSorting(unittest.TestCase): self.faker = Faker('en_US') self.faker.seed(66410) self.test_daemon = get_test_daemon() - component_manager = ComponentManager( - skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, - PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT, - STREAM_IDENTIFIER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT], - file_manager=FakeFileManager - ) - component_manager.setup() - self.test_daemon.component_manager = component_manager - self.test_daemon.file_manager = component_manager.get_component("file_manager") self.test_daemon.file_manager.lbry_files = self._get_fake_lbry_files() - # Pre-sorted lists of prices and file names in ascending order produced by # faker with seed 66410. This seed was chosen becacuse it produces 3 results # 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria. @@ -165,6 +160,7 @@ class TestFileListSorting(unittest.TestCase): self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales', 'lucasfowler', 'peggytorres', 'qmitchell', 'trevoranderson', 'xmitchell', 'zhangsusan'] + return self.test_daemon.component_manager.setup() def test_sort_by_points_paid_no_direction_specified(self): sort_options = ['points_paid'] diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_Downloader.py b/lbrynet/tests/unit/lbrynet_daemon/test_Downloader.py index 7e62e9eaf..a70771c9b 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_Downloader.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_Downloader.py @@ -75,9 +75,10 @@ class GetStreamTests(unittest.TestCase): max_key_fee = {'currency': "LBC", 'amount': 10, 'address': ''} disable_max_key_fee = False data_rate = {'currency': "LBC", 'amount': 0, 'address': ''} - getstream = Downloader.GetStream(sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, - DummyRateLimiter(), prm, storage, max_key_fee, disable_max_key_fee, - timeout=3, data_rate=data_rate) + getstream = Downloader.GetStream( + sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, DummyRateLimiter(), prm, + storage, max_key_fee, disable_max_key_fee, timeout=3, data_rate=data_rate + ) getstream.pay_key_fee_called = False self.clock = task.Clock() @@ -98,7 +99,6 @@ class GetStreamTests(unittest.TestCase): with self.assertRaises(AttributeError): yield getstream.start(stream_info, name, "deadbeef" * 12, 0) - @defer.inlineCallbacks def test_sd_blob_download_timeout(self): """