From 57117baab599ae9a781d2a66b701422c13c0d5f7 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 10 Nov 2017 12:54:33 -0500 Subject: [PATCH 1/9] remove ManagedEncryptedFileDownloader.load_file_attributes() move it to EncryptedFileDownloader.set_stream_info() --- lbrynet/daemon/Publisher.py | 5 ++-- .../file_manager/EncryptedFileDownloader.py | 23 ++++-------------- .../client/EncryptedFileDownloader.py | 24 ++++++++++--------- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/lbrynet/daemon/Publisher.py b/lbrynet/daemon/Publisher.py index 2d9d2eeb6..01ed7a065 100644 --- a/lbrynet/daemon/Publisher.py +++ b/lbrynet/daemon/Publisher.py @@ -35,9 +35,9 @@ class Publisher(object): stream_hash = yield create_lbry_file(self.session, self.lbry_file_manager, file_name, read_handle) prm = self.session.payment_rate_manager - self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, prm) sd_hash = yield publish_sd_blob(self.lbry_file_manager.stream_info_manager, - self.session.blob_manager, self.lbry_file.stream_hash) + self.session.blob_manager, stream_hash) + self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, prm) if 'source' not in claim_dict['stream']: claim_dict['stream']['source'] = {} claim_dict['stream']['source']['source'] = sd_hash @@ -47,7 +47,6 @@ class Publisher(object): claim_out = yield self.make_claim(name, bid, claim_dict, claim_address, change_address) self.lbry_file.completed = True - yield self.lbry_file.load_file_attributes(sd_hash) yield self.lbry_file.save_status() defer.returnValue(claim_out) diff --git a/lbrynet/file_manager/EncryptedFileDownloader.py b/lbrynet/file_manager/EncryptedFileDownloader.py index c1fbb043e..6dcc34ba7 100644 --- a/lbrynet/file_manager/EncryptedFileDownloader.py +++ b/lbrynet/file_manager/EncryptedFileDownloader.py @@ -7,7 +7,6 @@ from zope.interface import implements from twisted.internet import defer from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager -from lbrynet.core.Error import NoSuchStreamHash from lbrynet.core.utils import short_hash from lbrynet.core.StreamDescriptor import StreamMetadata from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver @@ -35,18 +34,21 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): STATUS_RUNNING = "running" STATUS_STOPPED = "stopped" STATUS_FINISHED = "finished" - + """ + These are started by EncryptedFileManager, aka, file_manager + """ def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name=None): + EncryptedFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager, payment_rate_manager, wallet, download_directory, file_name) - self.sd_hash = None + self.rowid = rowid self.lbry_file_manager = lbry_file_manager self._saving_status = False @@ -57,7 +59,6 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): @defer.inlineCallbacks def restore(self): - yield self.load_file_attributes() status = yield self.lbry_file_manager.get_lbry_file_status(self) log_status(self.file_name, self.sd_hash, status) @@ -101,23 +102,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): defer.returnValue(EncryptedFileStatusReport(self.file_name, num_blobs_completed, num_blobs_known, status)) - @defer.inlineCallbacks - def load_file_attributes(self, sd_hash=None): - if not sd_hash: - sd_hash = yield self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash) - if sd_hash: - self.sd_hash = sd_hash[0] - else: - raise NoSuchStreamHash(self.stream_hash) - else: - self.sd_hash = sd_hash - - defer.returnValue(None) - @defer.inlineCallbacks def _start(self): yield EncryptedFileSaver._start(self) - yield self.load_file_attributes() status = yield self._save_status() log_status(self.file_name, self.sd_hash, status) defer.returnValue(status) diff --git a/lbrynet/lbry_file/client/EncryptedFileDownloader.py b/lbrynet/lbry_file/client/EncryptedFileDownloader.py index 735c8027e..9445ccaa4 100644 --- a/lbrynet/lbry_file/client/EncryptedFileDownloader.py +++ b/lbrynet/lbry_file/client/EncryptedFileDownloader.py @@ -6,6 +6,7 @@ from lbrynet.lbry_file.StreamDescriptor import save_sd_info from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager from lbrynet.core.StreamDescriptor import StreamMetadata +from lbrynet.core.Error import NoSuchStreamHash from lbrynet.interfaces import IStreamDownloaderFactory from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler import os @@ -28,21 +29,22 @@ class EncryptedFileDownloader(CryptStreamDownloader): self.stream_info_manager = stream_info_manager self.suggested_file_name = None self._calculated_total_bytes = None + self.sd_hash = None + @defer.inlineCallbacks def set_stream_info(self): if self.key is None: - d = self.stream_info_manager.get_stream_info(self.stream_hash) + out = yield self.stream_info_manager.get_stream_info(self.stream_hash) + key, stream_name, suggested_file_name = out + self.key = binascii.unhexlify(key) + self.stream_name = binascii.unhexlify(stream_name) + self.suggested_file_name = binascii.unhexlify(suggested_file_name) - def set_stream_info(stream_info): - key, stream_name, suggested_file_name = stream_info - self.key = binascii.unhexlify(key) - self.stream_name = binascii.unhexlify(stream_name) - self.suggested_file_name = binascii.unhexlify(suggested_file_name) - - d.addCallback(set_stream_info) - return d - else: - return defer.succeed(True) + out = yield self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash) + if out: + self.sd_hash = out[0] + else: + raise NoSuchStreamHash(self.stream_hash) def delete_data(self): d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash) From 6218df2a92c7d961160bc744069e6d24fceda427 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Fri, 10 Nov 2017 16:37:51 -0500 Subject: [PATCH 2/9] make payment_rate_manager in add_lbry_file() an optional argument, initialize it from Session if not specified --- lbrynet/daemon/Publisher.py | 3 +-- lbrynet/file_manager/EncryptedFileManager.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lbrynet/daemon/Publisher.py b/lbrynet/daemon/Publisher.py index 01ed7a065..f4e6e5501 100644 --- a/lbrynet/daemon/Publisher.py +++ b/lbrynet/daemon/Publisher.py @@ -34,10 +34,9 @@ class Publisher(object): with file_utils.get_read_handle(file_path) as read_handle: stream_hash = yield create_lbry_file(self.session, self.lbry_file_manager, file_name, read_handle) - prm = self.session.payment_rate_manager sd_hash = yield publish_sd_blob(self.lbry_file_manager.stream_info_manager, self.session.blob_manager, stream_hash) - self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash, prm) + self.lbry_file = yield self.lbry_file_manager.add_lbry_file(stream_hash) if 'source' not in claim_dict['stream']: claim_dict['stream']['source'] = {} claim_dict['stream']['source']['source'] = sd_hash diff --git a/lbrynet/file_manager/EncryptedFileManager.py b/lbrynet/file_manager/EncryptedFileManager.py index fd96aa8f0..517e33f12 100644 --- a/lbrynet/file_manager/EncryptedFileManager.py +++ b/lbrynet/file_manager/EncryptedFileManager.py @@ -183,8 +183,10 @@ class EncryptedFileManager(object): yield self._stop_lbry_file(lbry_file) @defer.inlineCallbacks - def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, + def add_lbry_file(self, stream_hash, payment_rate_manager=None, blob_data_rate=None, download_directory=None, file_name=None): + if not payment_rate_manager: + payment_rate_manager = self.sesion.payment_rate_manager rowid = yield self._save_lbry_file(stream_hash, blob_data_rate) lbry_file = yield self.start_lbry_file(rowid, stream_hash, payment_rate_manager, blob_data_rate, download_directory, From ab667015c4309d11e0cc07aa2dadfcc8d201f439 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 15 Nov 2017 09:12:31 -0500 Subject: [PATCH 3/9] add lbry_file_manager to reflector server and add files to it --- lbrynet/daemon/Daemon.py | 3 ++- lbrynet/file_manager/EncryptedFileManager.py | 2 +- lbrynet/reflector/server/server.py | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index f01bb524c..1a4f65f09 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -314,7 +314,8 @@ class Daemon(AuthJSONRPCServer): reflector_factory = reflector_server_factory( self.session.peer_manager, self.session.blob_manager, - self.stream_info_manager + self.stream_info_manager, + self.lbry_file_manager ) try: self.reflector_server_port = reactor.listenTCP(self.reflector_port, diff --git a/lbrynet/file_manager/EncryptedFileManager.py b/lbrynet/file_manager/EncryptedFileManager.py index 517e33f12..9d88d891c 100644 --- a/lbrynet/file_manager/EncryptedFileManager.py +++ b/lbrynet/file_manager/EncryptedFileManager.py @@ -186,7 +186,7 @@ class EncryptedFileManager(object): def add_lbry_file(self, stream_hash, payment_rate_manager=None, blob_data_rate=None, download_directory=None, file_name=None): if not payment_rate_manager: - payment_rate_manager = self.sesion.payment_rate_manager + payment_rate_manager = self.session.payment_rate_manager rowid = yield self._save_lbry_file(stream_hash, blob_data_rate) lbry_file = yield self.start_lbry_file(rowid, stream_hash, payment_rate_manager, blob_data_rate, download_directory, diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index d7ddc4018..1651eb6b0 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -33,6 +33,7 @@ class ReflectorServer(Protocol): self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) self.blob_manager = self.factory.blob_manager self.stream_info_manager = self.factory.stream_info_manager + self.lbry_file_manager = self.factory.lbry_file_manager self.protocol_version = self.factory.protocol_version self.received_handshake = False self.peer_version = None @@ -107,6 +108,7 @@ class ReflectorServer(Protocol): yield save_sd_info(self.stream_info_manager, sd_info) yield self.stream_info_manager.save_sd_blob_hash_to_stream(sd_info['stream_hash'], blob.blob_hash) + yield self.lbry_file_manager.add_lbry_file(sd_info['stream_hash']) should_announce = True # if we already have the head blob, set it to be announced now that we know it's @@ -399,10 +401,11 @@ class ReflectorServer(Protocol): class ReflectorServerFactory(ServerFactory): protocol = ReflectorServer - def __init__(self, peer_manager, blob_manager, stream_info_manager): + def __init__(self, peer_manager, blob_manager, stream_info_manager, lbry_file_manager): self.peer_manager = peer_manager self.blob_manager = blob_manager self.stream_info_manager = stream_info_manager + self.lbry_file_manager = lbry_file_manager self.protocol_version = REFLECTOR_V2 def buildProtocol(self, addr): From ae74d80c8a4bd6ee7b1b61d847c8ac848eac6b4f Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 15 Nov 2017 09:24:12 -0500 Subject: [PATCH 4/9] add lbry_file_manager to reflector server in functional reflector test, make sure file is added to lbry_file_manager when reflected --- lbrynet/tests/functional/test_reflector.py | 41 ++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index d252986a2..6fc50de7b 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -52,7 +52,7 @@ class TestReflector(unittest.TestCase): 1015056 ), ] - + ## Setup reflector client classes ## self.db_dir, self.blob_dir = mk_db_and_blob_dir() self.session = Session.Session( conf.settings['data_rate'], @@ -75,19 +75,39 @@ class TestReflector(unittest.TestCase): self.lbry_file_manager = EncryptedFileManager.EncryptedFileManager( self.session, self.stream_info_manager, sd_identifier) + ## Setup reflector server classes ## self.server_db_dir, self.server_blob_dir = mk_db_and_blob_dir() + self.server_session = Session.Session( + conf.settings['data_rate'], + db_dir=self.server_db_dir, + node_id="abcd", + peer_finder=peer_finder, + hash_announcer=hash_announcer, + blob_dir=self.server_blob_dir, + peer_port=5553, + use_upnp=False, + wallet=wallet, + blob_tracker_class=mocks.BlobAvailabilityTracker, + external_ip="127.0.0.1" + ) + self.server_blob_manager = BlobManager.DiskBlobManager( hash_announcer, self.server_blob_dir, self.server_db_dir) self.server_stream_info_manager = \ EncryptedFileMetadataManager.DBEncryptedFileMetadataManager(self.server_db_dir) + self.server_lbry_file_manager = EncryptedFileManager.EncryptedFileManager( + self.server_session, self.server_stream_info_manager, + sd_identifier) d = self.session.setup() d.addCallback(lambda _: self.stream_info_manager.setup()) d.addCallback(lambda _: EncryptedFileOptions.add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) + d.addCallback(lambda _: self.server_session.setup()) d.addCallback(lambda _: self.server_blob_manager.setup()) d.addCallback(lambda _: self.server_stream_info_manager.setup()) + d.addCallback(lambda _: self.server_lbry_file_manager.setup()) def verify_equal(sd_info): self.assertEqual(mocks.create_stream_sd_file, sd_info) @@ -123,7 +143,8 @@ class TestReflector(unittest.TestCase): def start_server(): server_factory = reflector.ServerFactory( - peer_manager, self.server_blob_manager, self.server_stream_info_manager) + peer_manager, self.server_blob_manager, self.server_stream_info_manager, + self.server_lbry_file_manager) from twisted.internet import reactor port = 8943 while self.reflector_port is None: @@ -140,14 +161,24 @@ class TestReflector(unittest.TestCase): def take_down_env(self): d = defer.succeed(True) + ## Close client classes ## if self.lbry_file_manager is not None: d.addCallback(lambda _: self.lbry_file_manager.stop()) if self.session is not None: d.addCallback(lambda _: self.session.shut_down()) if self.stream_info_manager is not None: d.addCallback(lambda _: self.stream_info_manager.stop()) + + ## Close server classes ## if self.server_blob_manager is not None: d.addCallback(lambda _: self.server_blob_manager.stop()) + if self.server_lbry_file_manager is not None: + d.addCallback(lambda _: self.server_lbry_file_manager.stop()) + if self.server_session is not None: + d.addCallback(lambda _: self.server_session.shut_down()) + if self.server_stream_info_manager is not None: + d.addCallback(lambda _: self.server_stream_info_manager.stop()) + if self.reflector_port is not None: d.addCallback(lambda _: self.reflector_port.stopListening()) @@ -186,6 +217,12 @@ class TestReflector(unittest.TestCase): expected_sd_hash = self.expected_blobs[-1][0] self.assertEqual(self.sd_hash, sd_hashes[0]) + # check lbry file manager has the file + files = yield self.server_lbry_file_manager.lbry_files + self.assertEqual(1, len(files)) + self.assertEqual(self.sd_hash, files[0].sd_hash) + self.assertEqual('test_file', files[0].file_name) + # check should_announce blobs on blob_manager blob_hashes = yield self.server_blob_manager._get_all_should_announce_blob_hashes() self.assertEqual(2, len(blob_hashes)) From 4971ddda78c1447935b811fc2aa015f039b5489f Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 15 Nov 2017 10:19:01 -0500 Subject: [PATCH 5/9] No need to check for None in reflector test tear down --- lbrynet/tests/functional/test_reflector.py | 24 ++++++++-------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 6fc50de7b..df67299ce 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -162,25 +162,17 @@ class TestReflector(unittest.TestCase): def take_down_env(self): d = defer.succeed(True) ## Close client classes ## - if self.lbry_file_manager is not None: - d.addCallback(lambda _: self.lbry_file_manager.stop()) - if self.session is not None: - d.addCallback(lambda _: self.session.shut_down()) - if self.stream_info_manager is not None: - d.addCallback(lambda _: self.stream_info_manager.stop()) + d.addCallback(lambda _: self.lbry_file_manager.stop()) + d.addCallback(lambda _: self.session.shut_down()) + d.addCallback(lambda _: self.stream_info_manager.stop()) ## Close server classes ## - if self.server_blob_manager is not None: - d.addCallback(lambda _: self.server_blob_manager.stop()) - if self.server_lbry_file_manager is not None: - d.addCallback(lambda _: self.server_lbry_file_manager.stop()) - if self.server_session is not None: - d.addCallback(lambda _: self.server_session.shut_down()) - if self.server_stream_info_manager is not None: - d.addCallback(lambda _: self.server_stream_info_manager.stop()) + d.addCallback(lambda _: self.server_blob_manager.stop()) + d.addCallback(lambda _: self.server_lbry_file_manager.stop()) + d.addCallback(lambda _: self.server_session.shut_down()) + d.addCallback(lambda _: self.server_stream_info_manager.stop()) - if self.reflector_port is not None: - d.addCallback(lambda _: self.reflector_port.stopListening()) + d.addCallback(lambda _: self.reflector_port.stopListening()) def delete_test_env(): try: From 95289952b55cc915b5ef2771c0fc79f0f1d51ff0 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 15 Nov 2017 10:19:57 -0500 Subject: [PATCH 6/9] No need to set up DummmyRateLimiter for reflector test --- lbrynet/tests/functional/test_reflector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index df67299ce..b58c1e455 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -6,7 +6,6 @@ from lbrynet import lbry_file from lbrynet import reflector from lbrynet.core import BlobManager from lbrynet.core import PeerManager -from lbrynet.core import RateLimiter from lbrynet.core import Session from lbrynet.core import StreamDescriptor from lbrynet.lbry_file import EncryptedFileMetadataManager @@ -32,7 +31,6 @@ class TestReflector(unittest.TestCase): peer_manager = PeerManager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) hash_announcer = mocks.Announcer() - rate_limiter = RateLimiter.DummyRateLimiter() sd_identifier = StreamDescriptor.StreamDescriptorIdentifier() self.expected_blobs = [ @@ -63,7 +61,6 @@ class TestReflector(unittest.TestCase): blob_dir=self.blob_dir, peer_port=5553, use_upnp=False, - rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=mocks.BlobAvailabilityTracker, external_ip="127.0.0.1" From 1d3deb37b16ecbd6dcb626d6bf11ecfd7085dd7f Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 4 Dec 2017 13:13:40 -0500 Subject: [PATCH 7/9] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f5f2b610..a7028f12d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ at anytime. * Removed claim related filter arguments `name`, `claim_id`, and `outpoint` from `file_list`, `file_delete`, `file_set_status`, and `file_reflect` * Removed unused files * Removed old and unused UI related code + * Removed claim information from lbry file internals ## [0.18.0] - 2017-11-08 From 63c6d3580d1d660a00b81567100d92910cc1f42b Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Tue, 5 Dec 2017 12:47:27 -0500 Subject: [PATCH 8/9] do not yield on add_lbry_file() when obtaining sd blob as reflector server --- lbrynet/reflector/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 1651eb6b0..995c35141 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -108,7 +108,7 @@ class ReflectorServer(Protocol): yield save_sd_info(self.stream_info_manager, sd_info) yield self.stream_info_manager.save_sd_blob_hash_to_stream(sd_info['stream_hash'], blob.blob_hash) - yield self.lbry_file_manager.add_lbry_file(sd_info['stream_hash']) + self.lbry_file_manager.add_lbry_file(sd_info['stream_hash']) should_announce = True # if we already have the head blob, set it to be announced now that we know it's From 915ab668f5d34009cd6cf64683ce7bc9e943c8e0 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Tue, 5 Dec 2017 13:12:29 -0500 Subject: [PATCH 9/9] add some tests to make sure file status after reflector server download (make sure in stopped state, and have correct number of blobs) --- lbrynet/tests/functional/test_reflector.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index b58c1e455..16a948358 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -212,6 +212,12 @@ class TestReflector(unittest.TestCase): self.assertEqual(self.sd_hash, files[0].sd_hash) self.assertEqual('test_file', files[0].file_name) + status = yield files[0].status() + self.assertEqual('stopped', status.running_status) + num_blobs = len(self.expected_blobs) -1 # subtract sd hash + self.assertEqual(num_blobs, status.num_completed) + self.assertEqual(num_blobs, status.num_known) + # check should_announce blobs on blob_manager blob_hashes = yield self.server_blob_manager._get_all_should_announce_blob_hashes() self.assertEqual(2, len(blob_hashes))