diff --git a/CHANGELOG.md b/CHANGELOG.md index 64524f2a8..d436a7750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ at anytime. ## [0.8.3] - 2017-02-15 ### Fixed + * Get lbry files with pending claims * Add better logging to help track down [#478](https://github.com/lbryio/lbry/issues/478) * Catch UnknownNameErrors when resolving a name. [#479](https://github.com/lbryio/lbry/issues/479) diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index 36e20151a..79a982740 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -84,6 +84,12 @@ class NoSuchStreamHashError(Exception): pass +class NoSuchSDHash(Exception): + """ + Raised if sd hash is not known + """ + + class InvalidBlobHashError(Exception): pass diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index fe1190d0e..76d542b36 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -105,3 +105,7 @@ def setup_certs_for_windows(): def random_string(length=10, chars=string.ascii_lowercase): return ''.join([random.choice(chars) for _ in range(length)]) + + +def short_hash(hash_str): + return hash_str[:6] diff --git a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py index 356cb02df..3b3f27a39 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileDownloader.py +++ b/lbrynet/lbryfilemanager/EncryptedFileDownloader.py @@ -7,6 +7,8 @@ from zope.interface import implements from twisted.internet import defer from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager +from lbrynet.core.Error import NoSuchSDHash, NoSuchStreamHashError +from lbrynet.core.utils import short_hash from lbrynet.core.StreamDescriptor import StreamMetadata from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileSaver from lbrynet.lbryfile.client.EncryptedFileDownloader import EncryptedFileDownloader @@ -17,6 +19,18 @@ from lbrynet.lbryfile.StreamDescriptor import save_sd_info log = logging.getLogger(__name__) +def log_status(uri, sd_hash, status): + if status == ManagedEncryptedFileDownloader.STATUS_RUNNING: + status_string = "running" + elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED: + status_string = "stopped" + elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED: + status_string = "finished" + else: + status_string = "unknown" + log.info("lbry://%s (%s) is %s", uri, short_hash(sd_hash), status_string) + + class ManagedEncryptedFileDownloader(EncryptedFileSaver): STATUS_RUNNING = "running" STATUS_STOPPED = "stopped" @@ -47,17 +61,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): @defer.inlineCallbacks def restore(self): - sd_hash = yield self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash) - if sd_hash: - self.sd_hash = sd_hash[0] - else: - raise Exception("No sd hash for stream hash %s" % self.stream_hash) - claim_metadata = yield self.wallet.get_claim_metadata_for_sd_hash(self.sd_hash) - if claim_metadata is None: - raise Exception("A claim doesn't exist for sd %s" % self.sd_hash) - self.uri, self.txid, self.nout = claim_metadata - self.claim_id = yield self.wallet.get_claimid(self.uri, self.txid, self.nout) + yield self.load_file_attributes() + status = yield self.lbry_file_manager.get_lbry_file_status(self) + log_status(self.uri, self.sd_hash, status) + if status == ManagedEncryptedFileDownloader.STATUS_RUNNING: # start returns self.finished_deferred # which fires when we've finished downloading the file @@ -73,7 +81,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): @defer.inlineCallbacks def stop(self, err=None, change_status=True): - log.debug('Stopping download for %s', self.sd_hash) + log.debug('Stopping download for %s', short_hash(self.sd_hash)) # EncryptedFileSaver deletes metadata when it's stopped. We don't want that here. yield EncryptedFileDownloader.stop(self, err=err) if change_status is True: @@ -98,20 +106,29 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): num_blobs_known, status)) @defer.inlineCallbacks - def _start(self): - log.info('Starting Downloader for %s', self.stream_hash) - yield EncryptedFileSaver._start(self) + def load_file_attributes(self): sd_hash = yield self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash) - if len(sd_hash): + if sd_hash: self.sd_hash = sd_hash[0] - maybe_metadata = yield self.wallet.get_claim_metadata_for_sd_hash(self.sd_hash) - if maybe_metadata: - name, txid, nout = maybe_metadata - self.uri = name - self.txid = txid - self.nout = nout + else: + raise NoSuchStreamHashError(self.stream_hash) + stream_metadata = yield self.wallet.get_claim_metadata_for_sd_hash(self.sd_hash) + if stream_metadata: + name, txid, nout = stream_metadata + self.uri = name + self.txid = txid + self.nout = nout + else: + raise NoSuchSDHash(self.sd_hash) + self.claim_id = yield self.wallet.get_claimid(self.uri, self.txid, self.nout) + defer.returnValue(None) + + @defer.inlineCallbacks + def _start(self): + yield EncryptedFileSaver._start(self) + yield self.load_file_attributes() status = yield self._save_status() - log.info('Set Downloader status for %s to %s', self.stream_hash, status) + log_status(self.uri, self.sd_hash, status) defer.returnValue(status) def _get_finished_deferred_callback_value(self): @@ -133,6 +150,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver): self._saving_status = False defer.returnValue(status) + def save_status(self): + return self._save_status() + def _get_progress_manager(self, download_manager): return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager) diff --git a/lbrynet/lbrynet_daemon/Daemon.py b/lbrynet/lbrynet_daemon/Daemon.py index 852cfe602..07d9c0f3f 100644 --- a/lbrynet/lbrynet_daemon/Daemon.py +++ b/lbrynet/lbrynet_daemon/Daemon.py @@ -45,7 +45,7 @@ from lbrynet.core.Wallet import LBRYumWallet, SqliteStorage from lbrynet.core.looping_call_manager import LoopingCallManager from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.core.Error import InsufficientFundsError, UnknownNameError +from lbrynet.core.Error import InsufficientFundsError, UnknownNameError, NoSuchSDHash log = logging.getLogger(__name__) @@ -794,7 +794,6 @@ class Daemon(AuthJSONRPCServer): d = reupload.reflect_stream(publisher.lbry_file) d.addCallbacks(lambda _: log.info("Reflected new publication to lbry://%s", name), log.exception) - log.info("Success! Published to lbry://%s txid: %s nout: %d", name, claim_out['txid'], claim_out['nout']) yield self._add_to_pending_claims(claim_out, name) @@ -957,34 +956,53 @@ class Daemon(AuthJSONRPCServer): return self.get_est_cost_using_known_size(name, size) return self.get_est_cost_from_name(name) + def _find_lbry_file_by_uri(self, uri): + for lbry_file in self.lbry_file_manager.lbry_files: + if uri == lbry_file.uri: + return lbry_file + raise UnknownNameError(uri) + + def _find_lbry_file_by_sd_hash(self, sd_hash): + for lbry_file in self.lbry_file_manager.lbry_files: + if lbry_file.sd_hash == sd_hash: + return lbry_file + raise NoSuchSDHash(sd_hash) + + def _find_lbry_file_by_file_name(self, file_name): + for lbry_file in self.lbry_file_manager.lbry_files: + if lbry_file.file_name == file_name: + return lbry_file + raise Exception("File %s not found" % file_name) + + @defer.inlineCallbacks def _get_lbry_file_by_uri(self, name): - def _get_file(stream_info): - sd = stream_info['sources']['lbry_sd_hash'] - - for l in self.lbry_file_manager.lbry_files: - if l.sd_hash == sd: - return defer.succeed(l) - return defer.succeed(None) - - d = self._resolve_name(name) - d.addCallback(_get_file) - - return d + try: + stream_info = yield self._resolve_name(name) + sd_hash = stream_info['sources']['lbry_sd_hash'] + lbry_file = yield self._get_lbry_file_by_sd_hash(sd_hash) + except (UnknownNameError, NoSuchSDHash): + lbry_file = yield self._find_lbry_file_by_uri(name) + defer.returnValue(lbry_file) + @defer.inlineCallbacks def _get_lbry_file_by_sd_hash(self, sd_hash): - for l in self.lbry_file_manager.lbry_files: - if l.sd_hash == sd_hash: - return defer.succeed(l) - return defer.succeed(None) + lbry_file = yield self._find_lbry_file_by_sd_hash(sd_hash) + defer.returnValue(lbry_file) + @defer.inlineCallbacks def _get_lbry_file_by_file_name(self, file_name): - for l in self.lbry_file_manager.lbry_files: - if l.file_name == file_name: - return defer.succeed(l) - return defer.succeed(None) + lbry_file = yield self._get_lbry_file_by_file_name(file_name) + defer.returnValue(lbry_file) + @defer.inlineCallbacks def _get_lbry_file(self, search_by, val, return_json=True): - return _GetFileHelper(self, search_by, val, return_json).retrieve_file() + helper = _GetFileHelper(self, search_by, val, return_json) + try: + lbry_file = yield helper.retrieve_file() + defer.returnValue(lbry_file) + except Exception as err: + # TODO: do something with the error, don't return None when a file isn't found + defer.returnValue(None) def _get_lbry_files(self): def safe_get(sd_hash): diff --git a/lbrynet/lbrynet_daemon/Publisher.py b/lbrynet/lbrynet_daemon/Publisher.py index ad7078714..a0f390d3c 100644 --- a/lbrynet/lbrynet_daemon/Publisher.py +++ b/lbrynet/lbrynet_daemon/Publisher.py @@ -48,6 +48,9 @@ class Publisher(object): metadata['content_type'] = get_content_type(file_path) metadata['ver'] = Metadata.current_version claim_out = yield self.make_claim(name, bid, metadata) + self.lbry_file.completed = True + yield self.lbry_file.load_file_attributes() + yield self.lbry_file.save_status() defer.returnValue(claim_out) @defer.inlineCallbacks diff --git a/tests/functional/test_streamify.py b/tests/functional/test_streamify.py index c0073ce54..4543459ea 100644 --- a/tests/functional/test_streamify.py +++ b/tests/functional/test_streamify.py @@ -12,6 +12,7 @@ from lbrynet.lbryfile.EncryptedFileMetadataManager import DBEncryptedFileMetadat from lbrynet.lbryfilemanager.EncryptedFileManager import EncryptedFileManager from lbrynet.core.Session import Session from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier +from lbrynet.lbryfile import publish_sd_blob from lbrynet.lbryfilemanager.EncryptedFileCreator import create_lbry_file from lbrynet.lbryfile.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from lbrynet.lbryfile.StreamDescriptor import get_sd_info @@ -158,11 +159,13 @@ class TestStreamify(TestCase): d.addCallback(lambda _: check_md5_sum()) return d + @defer.inlineCallbacks def create_stream(): test_file = GenFile(53209343, b''.join([chr(i + 5) for i in xrange(0, 64, 6)])) - return create_lbry_file( - self.session, self.lbry_file_manager, "test_file", test_file, - suggested_file_name="test_file") + stream_hash = yield create_lbry_file(self.session, self.lbry_file_manager, "test_file", + test_file, suggested_file_name="test_file") + yield publish_sd_blob(self.stream_info_manager, self.session.blob_manager, stream_hash) + defer.returnValue(stream_hash) d = self.session.setup() d.addCallback(lambda _: self.stream_info_manager.setup()) diff --git a/tests/mocks.py b/tests/mocks.py index e1cbd9518..8e3ed945a 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -80,6 +80,9 @@ class Wallet(object): def get_claim_metadata_for_sd_hash(self, sd_hash): return "fakeuri", "faketxid", "fakenout" + def get_claimid(self, name, txid=None, nout=None): + return "fake claimid" + class PeerFinder(object): def __init__(self, start_port, peer_manager, num_peers):