From d06c670a128899b0cbbbadec56b250c9e46fb462 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 4 Nov 2016 11:34:42 -0500 Subject: [PATCH 1/4] Allow publishing with unicode filenames encodes unicode into a string before hashing. Also adds tests to verify that the fix works and that normal files can be made into EncryptedFiles --- .../lbryfilemanager/EncryptedFileCreator.py | 20 +++++-- tests/unit/lbryfilemanager/__init__.py | 0 .../test_EncryptedFileCreator.py | 59 +++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 tests/unit/lbryfilemanager/__init__.py create mode 100644 tests/unit/lbryfilemanager/test_EncryptedFileCreator.py diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index c9feaccf3..516259c82 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -37,9 +37,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator): def _save_stream_info(self): stream_info_manager = self.lbry_file_manager.stream_info_manager - d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name), - binascii.hexlify(self.key), - binascii.hexlify(self.suggested_file_name), + d = stream_info_manager.save_stream(self.stream_hash, hexlify(self.name), + hexlify(self.key), + hexlify(self.suggested_file_name), self.blob_infos) return d @@ -68,9 +68,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator): def _make_stream_hash(self): hashsum = get_lbry_hash_obj() - hashsum.update(binascii.hexlify(self.name)) - hashsum.update(binascii.hexlify(self.key)) - hashsum.update(binascii.hexlify(self.suggested_file_name)) + hashsum.update(hexlify(self.name)) + hashsum.update(hexlify(self.key)) + hashsum.update(hexlify(self.suggested_file_name)) hashsum.update(self._get_blobs_hashsum()) self.stream_hash = hashsum.hexdigest() @@ -155,3 +155,11 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non d = lbry_file_creator.setup() d.addCallback(lambda _: start_stream()) return d + + +def hexlify(str_or_unicode): + if isinstance(str_or_unicode, unicode): + strng = str_or_unicode.encode('utf-8') + else: + strng = str_or_unicode + return binascii.hexlify(strng) diff --git a/tests/unit/lbryfilemanager/__init__.py b/tests/unit/lbryfilemanager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py new file mode 100644 index 000000000..ae2efccc2 --- /dev/null +++ b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import shutil +import tempfile + +from Crypto.Cipher import AES +import mock +from twisted.trial import unittest + +from lbrynet.core import BlobManager +from lbrynet.core import Session +from lbrynet.core.server import DHTHashAnnouncer +from lbrynet.lbryfilemanager import EncryptedFileCreator +from lbrynet.lbryfilemanager import EncryptedFileManager + +from tests import mocks + + +MB = 2**20 + + +def iv_generator(): + while True: + yield '3' * AES.block_size + + +class CreateEncryptedFileTest(unittest.TestCase): + timeout = 5 + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def create_file(self, filename): + session = mock.Mock(spec=Session.Session)(None, None) + hash_announcer = mock.Mock(spec=DHTHashAnnouncer.DHTHashAnnouncer)(None, None) + session.blob_manager = BlobManager.TempBlobManager(hash_announcer) + session.db_dir = self.tmp_dir + manager = mock.Mock(spec=EncryptedFileManager.EncryptedFileManager)() + handle = mocks.GenFile(3*MB, '1') + key = '2'*AES.block_size + return EncryptedFileCreator.create_lbry_file( + session, manager, filename, handle, key, iv_generator()) + + def test_can_create_file(self): + expected_stream_hash = ('41e6b247d923d191b154fb6f1b8529d6ddd6a73d65c357b1acb7' + '42dd83151fb66393a7709e9f346260a4f4db6de10c25') + filename = 'test.file' + d = self.create_file(filename) + d.addCallback(self.assertEqual, expected_stream_hash) + return d + + def test_can_create_file_with_unicode_filename(self): + expected_stream_hash = ('d1da4258f3ce12edb91d7e8e160d091d3ab1432c2e55a6352dce0' + '2fd5adb86fe144e93e110075b5865fff8617776c6c0') + filename = u'☃.file' + d = self.create_file(filename) + d.addCallback(self.assertEqual, expected_stream_hash) + return d From 0036685ecee7c1444ce23c04ec404c59b65cd0e0 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 4 Nov 2016 11:36:43 -0500 Subject: [PATCH 2/4] use more readable KB, MB --- lbrynet/conf.py | 12 +++++++----- tests/mocks.py | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 807f3cb0e..a3ccad693 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -14,6 +14,8 @@ log = logging.getLogger(__name__) LINUX = 1 DARWIN = 2 WINDOWS = 3 +KB = 2**10 +MB = 2**20 if sys.platform.startswith("darwin"): @@ -185,13 +187,13 @@ class AdjustableSettings(Setting): class ApplicationSettings(Setting): """Settings that are constants and shouldn't be overriden""" def __init__(self): - self.MAX_HANDSHAKE_SIZE = 2**16 - self.MAX_REQUEST_SIZE = 2**16 - self.MAX_BLOB_REQUEST_SIZE = 2**16 - self.MAX_RESPONSE_INFO_SIZE = 2**16 + self.MAX_HANDSHAKE_SIZE = 64*KB + self.MAX_REQUEST_SIZE = 64*KB + self.MAX_BLOB_REQUEST_SIZE = 64*KB + self.MAX_RESPONSE_INFO_SIZE = 64*KB self.MAX_BLOB_INFOS_TO_REQUEST = 20 self.BLOBFILES_DIR = "blobfiles" - self.BLOB_SIZE = 2**21 + self.BLOB_SIZE = 2*MB self.LOG_FILE_NAME = "lbrynet.log" self.LOG_POST_URL = "https://lbry.io/log-upload" self.CRYPTSD_FILE_EXTENSION = ".cryptsd" diff --git a/tests/mocks.py b/tests/mocks.py index c63827932..55c3ed45e 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -8,6 +8,8 @@ from lbrynet.core import PTCWallet from lbrynet.core import BlobAvailability +KB = 2**10 + class Node(object): def __init__(self, *args, **kwargs): pass @@ -129,12 +131,12 @@ class GenFile(io.RawIOBase): def readall(self): return self.read() - def _generate_chunk(self, n=2**10): - output = self.pattern[self.last_offset:self.last_offset + n] - n_left = n - len(output) + def _generate_chunk(self, size=KB): + output = self.pattern[self.last_offset:self.last_offset + size] + n_left = size - len(output) whole_patterns = n_left / len(self.pattern) output += self.pattern * whole_patterns - self.last_offset = n - len(output) + self.last_offset = size - len(output) output += self.pattern[:self.last_offset] return output From 43b2cd1433422ff96ba1424a4515db7344bff0ff Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 4 Nov 2016 11:39:52 -0500 Subject: [PATCH 3/4] cleanup whitespace and long-lines --- lbrynet/core/StreamCreator.py | 4 +--- lbrynet/cryptstream/CryptStreamCreator.py | 20 ++++++++-------- .../lbryfilemanager/EncryptedFileCreator.py | 23 ++++++++++--------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/lbrynet/core/StreamCreator.py b/lbrynet/core/StreamCreator.py index e0b2dab83..cc579865d 100644 --- a/lbrynet/core/StreamCreator.py +++ b/lbrynet/core/StreamCreator.py @@ -65,12 +65,10 @@ class StreamCreator(object): pass def write(self, data): - from twisted.internet import reactor - self._write(data) if self.stopped is False and self.streaming is False: reactor.callLater(0, self.producer.resumeProducing) def _write(self, data): - pass \ No newline at end of file + pass diff --git a/lbrynet/cryptstream/CryptStreamCreator.py b/lbrynet/cryptstream/CryptStreamCreator.py index 61d4d0270..9b2aa1770 100644 --- a/lbrynet/cryptstream/CryptStreamCreator.py +++ b/lbrynet/cryptstream/CryptStreamCreator.py @@ -18,22 +18,23 @@ log = logging.getLogger(__name__) class CryptStreamCreator(StreamCreator): """Create a new stream with blobs encrypted by a symmetric cipher. - Each blob is encrypted with the same key, but each blob has its own initialization vector - which is associated with the blob when the blob is associated with the stream.""" + Each blob is encrypted with the same key, but each blob has its + own initialization vector which is associated with the blob when + the blob is associated with the stream. + """ def __init__(self, blob_manager, name=None, key=None, iv_generator=None): - """ - @param blob_manager: Object that stores and provides access to blobs. + """@param blob_manager: Object that stores and provides access to blobs. @type blob_manager: BlobManager @param name: the name of the stream, which will be presented to the user @type name: string - @param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will - be generated. + @param key: the raw AES key which will be used to encrypt the + blobs. If None, a random key will be generated. @type key: string - @param iv_generator: a generator which yields initialization vectors for the blobs. Will be called - once for each blob. + @param iv_generator: a generator which yields initialization + vectors for the blobs. Will be called once for each blob. @type iv_generator: a generator function which yields strings @return: None @@ -75,7 +76,6 @@ class CryptStreamCreator(StreamCreator): return d def _write(self, data): - def close_blob(blob): d = blob.close() d.addCallback(self._blob_finished) @@ -94,4 +94,4 @@ class CryptStreamCreator(StreamCreator): self.current_blob = None def _get_blob_maker(self, iv, blob_creator): - return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator) \ No newline at end of file + return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator) diff --git a/lbrynet/lbryfilemanager/EncryptedFileCreator.py b/lbrynet/lbryfilemanager/EncryptedFileCreator.py index 516259c82..77a168b96 100644 --- a/lbrynet/lbryfilemanager/EncryptedFileCreator.py +++ b/lbrynet/lbryfilemanager/EncryptedFileCreator.py @@ -24,10 +24,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator): key=None, iv_generator=None, suggested_file_name=None): CryptStreamCreator.__init__(self, blob_manager, name, key, iv_generator) self.lbry_file_manager = lbry_file_manager - if suggested_file_name is None: - self.suggested_file_name = name - else: - self.suggested_file_name = suggested_file_name + self.suggested_file_name = suggested_file_name or name self.stream_hash = None self.blob_infos = [] @@ -82,8 +79,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator): def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None, iv_generator=None, suggested_file_name=None): - """ - Turn a plain file into an LBRY File. + """Turn a plain file into an LBRY File. An LBRY File is a collection of encrypted blobs of data and the metadata that binds them together which, when decrypted and put back together according to the metadata, results @@ -113,8 +109,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non be generated. @type key: string - @param iv_generator: a generator which yields initialization vectors for the blobs. Will be called - once for each blob. + @param iv_generator: a generator which yields initialization + vectors for the blobs. Will be called once for each blob. @type iv_generator: a generator function which yields strings @param suggested_file_name: what the file should be called when the LBRY File is saved to disk. @@ -130,7 +126,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non def make_stream_desc_file(stream_hash): log.debug("creating the stream descriptor file") - descriptor_file_path = os.path.join(session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION) + descriptor_file_path = os.path.join( + session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION) descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path) d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True) @@ -141,8 +138,12 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non base_file_name = os.path.basename(file_name) - lbry_file_creator = EncryptedFileStreamCreator(session.blob_manager, lbry_file_manager, base_file_name, - key, iv_generator, suggested_file_name) + lbry_file_creator = EncryptedFileStreamCreator( + session.blob_manager, + lbry_file_manager, + base_file_name, key, + iv_generator, + suggested_file_name) def start_stream(): file_sender = FileSender() From b68a1838f288901f109260df27b3a60932dde3c5 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 4 Nov 2016 11:40:23 -0500 Subject: [PATCH 4/4] eliminate unused return value --- lbrynet/cryptstream/CryptStreamCreator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/cryptstream/CryptStreamCreator.py b/lbrynet/cryptstream/CryptStreamCreator.py index 9b2aa1770..5993fc2f8 100644 --- a/lbrynet/cryptstream/CryptStreamCreator.py +++ b/lbrynet/cryptstream/CryptStreamCreator.py @@ -73,7 +73,6 @@ class CryptStreamCreator(StreamCreator): d.addCallback(self._blob_finished) self.finished_deferreds.append(d) log.debug("called close on final blob, returning from make_final_blob") - return d def _write(self, data): def close_blob(blob):