Merge pull request #243 from lbryio/unicode-publish

Unicode publish

lbrynet/conf.py

@@ -14,6 +14,8 @@ log = logging.getLogger(__name__)
 LINUX = 1
 DARWIN = 2
 WINDOWS = 3
+KB = 2**10
+MB = 2**20
 
 
 if sys.platform.startswith("darwin"):
@@ -185,13 +187,13 @@ class AdjustableSettings(Setting):
 class ApplicationSettings(Setting):
     """Settings that are constants and shouldn't be overriden"""
     def __init__(self):
-        self.MAX_HANDSHAKE_SIZE = 2**16
+        self.MAX_HANDSHAKE_SIZE = 64*KB
-        self.MAX_REQUEST_SIZE = 2**16
+        self.MAX_REQUEST_SIZE = 64*KB
-        self.MAX_BLOB_REQUEST_SIZE = 2**16
+        self.MAX_BLOB_REQUEST_SIZE = 64*KB
-        self.MAX_RESPONSE_INFO_SIZE = 2**16
+        self.MAX_RESPONSE_INFO_SIZE = 64*KB
         self.MAX_BLOB_INFOS_TO_REQUEST = 20
         self.BLOBFILES_DIR = "blobfiles"
-        self.BLOB_SIZE = 2**21
+        self.BLOB_SIZE = 2*MB
         self.LOG_FILE_NAME = "lbrynet.log"
         self.LOG_POST_URL = "https://lbry.io/log-upload"
         self.CRYPTSD_FILE_EXTENSION = ".cryptsd"

lbrynet/core/StreamCreator.py

@@ -65,12 +65,10 @@ class StreamCreator(object):
         pass
 
     def write(self, data):
-
         from twisted.internet import reactor
-
         self._write(data)
         if self.stopped is False and self.streaming is False:
             reactor.callLater(0, self.producer.resumeProducing)
 
     def _write(self, data):
-        pass
+        pass

lbrynet/cryptstream/CryptStreamCreator.py

@@ -18,22 +18,23 @@ log = logging.getLogger(__name__)
 class CryptStreamCreator(StreamCreator):
     """Create a new stream with blobs encrypted by a symmetric cipher.
 
-    Each blob is encrypted with the same key, but each blob has its own initialization vector
+    Each blob is encrypted with the same key, but each blob has its
-    which is associated with the blob when the blob is associated with the stream."""
+    own initialization vector which is associated with the blob when
+    the blob is associated with the stream.
+    """
     def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
-        """
+        """@param blob_manager: Object that stores and provides access to blobs.
-        @param blob_manager: Object that stores and provides access to blobs.
         @type blob_manager: BlobManager
 
         @param name: the name of the stream, which will be presented to the user
         @type name: string
 
-        @param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
+        @param key: the raw AES key which will be used to encrypt the
-            be generated.
+            blobs. If None, a random key will be generated.
         @type key: string
 
-        @param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
+        @param iv_generator: a generator which yields initialization
-            once for each blob.
+            vectors for the blobs. Will be called once for each blob.
         @type iv_generator: a generator function which yields strings
 
         @return: None
@@ -72,10 +73,8 @@ class CryptStreamCreator(StreamCreator):
         d.addCallback(self._blob_finished)
         self.finished_deferreds.append(d)
         log.debug("called close on final blob, returning from make_final_blob")
-        return d
 
     def _write(self, data):
-
         def close_blob(blob):
             d = blob.close()
             d.addCallback(self._blob_finished)
@@ -94,4 +93,4 @@ class CryptStreamCreator(StreamCreator):
                 self.current_blob = None
 
     def _get_blob_maker(self, iv, blob_creator):
-        return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
+        return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)

lbrynet/lbryfilemanager/EncryptedFileCreator.py

@@ -24,10 +24,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
                  key=None, iv_generator=None, suggested_file_name=None):
         CryptStreamCreator.__init__(self, blob_manager, name, key, iv_generator)
         self.lbry_file_manager = lbry_file_manager
-        if suggested_file_name is None:
+        self.suggested_file_name = suggested_file_name or name
-            self.suggested_file_name = name
-        else:
-            self.suggested_file_name = suggested_file_name
         self.stream_hash = None
         self.blob_infos = []
 
@@ -37,9 +34,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
 
     def _save_stream_info(self):
         stream_info_manager = self.lbry_file_manager.stream_info_manager
-        d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
+        d = stream_info_manager.save_stream(self.stream_hash, hexlify(self.name),
-                                            binascii.hexlify(self.key),
+                                            hexlify(self.key),
-                                            binascii.hexlify(self.suggested_file_name),
+                                            hexlify(self.suggested_file_name),
                                             self.blob_infos)
         return d
 
@@ -68,9 +65,9 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
 
     def _make_stream_hash(self):
         hashsum = get_lbry_hash_obj()
-        hashsum.update(binascii.hexlify(self.name))
+        hashsum.update(hexlify(self.name))
-        hashsum.update(binascii.hexlify(self.key))
+        hashsum.update(hexlify(self.key))
-        hashsum.update(binascii.hexlify(self.suggested_file_name))
+        hashsum.update(hexlify(self.suggested_file_name))
         hashsum.update(self._get_blobs_hashsum())
         self.stream_hash = hashsum.hexdigest()
 
@@ -82,8 +79,7 @@ class EncryptedFileStreamCreator(CryptStreamCreator):
 
 def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=None,
                      iv_generator=None, suggested_file_name=None):
-    """
+    """Turn a plain file into an LBRY File.
-    Turn a plain file into an LBRY File.
 
     An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
     together which, when decrypted and put back together according to the metadata, results
@@ -113,8 +109,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
         be generated.
     @type key: string
 
-    @param iv_generator: a generator which yields initialization vectors for the blobs. Will be called
+    @param iv_generator: a generator which yields initialization
-        once for each blob.
+        vectors for the blobs. Will be called once for each blob.
     @type iv_generator: a generator function which yields strings
 
     @param suggested_file_name: what the file should be called when the LBRY File is saved to disk.
@@ -130,7 +126,8 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
 
     def make_stream_desc_file(stream_hash):
         log.debug("creating the stream descriptor file")
-        descriptor_file_path = os.path.join(session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
+        descriptor_file_path = os.path.join(
+            session.db_dir, file_name + settings.CRYPTSD_FILE_EXTENSION)
         descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
 
         d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
@@ -141,8 +138,12 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
 
     base_file_name = os.path.basename(file_name)
 
-    lbry_file_creator = EncryptedFileStreamCreator(session.blob_manager, lbry_file_manager, base_file_name,
+    lbry_file_creator = EncryptedFileStreamCreator(
-                                              key, iv_generator, suggested_file_name)
+        session.blob_manager,
+        lbry_file_manager,
+        base_file_name, key,
+        iv_generator,
+        suggested_file_name)
 
     def start_stream():
         file_sender = FileSender()
@@ -155,3 +156,11 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
     d = lbry_file_creator.setup()
     d.addCallback(lambda _: start_stream())
     return d
+
+
+def hexlify(str_or_unicode):
+    if isinstance(str_or_unicode, unicode):
+        strng = str_or_unicode.encode('utf-8')
+    else:
+        strng = str_or_unicode
+    return binascii.hexlify(strng)

tests/mocks.py

@@ -8,6 +8,8 @@ from lbrynet.core import PTCWallet
 from lbrynet.core import BlobAvailability
 
 
+KB = 2**10
+
 class Node(object):
     def __init__(self, *args, **kwargs):
         pass
@@ -129,12 +131,12 @@ class GenFile(io.RawIOBase):
     def readall(self):
         return self.read()
 
-    def _generate_chunk(self, n=2**10):
+    def _generate_chunk(self, size=KB):
-        output = self.pattern[self.last_offset:self.last_offset + n]
+        output = self.pattern[self.last_offset:self.last_offset + size]
-        n_left = n - len(output)
+        n_left = size - len(output)
         whole_patterns = n_left / len(self.pattern)
         output += self.pattern * whole_patterns
-        self.last_offset = n - len(output)
+        self.last_offset = size - len(output)
         output += self.pattern[:self.last_offset]
         return output
 

tests/unit/lbryfilemanager/test_EncryptedFileCreator.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+import shutil
+import tempfile
+
+from Crypto.Cipher import AES
+import mock
+from twisted.trial import unittest
+
+from lbrynet.core import BlobManager
+from lbrynet.core import Session
+from lbrynet.core.server import DHTHashAnnouncer
+from lbrynet.lbryfilemanager import EncryptedFileCreator
+from lbrynet.lbryfilemanager import EncryptedFileManager
+
+from tests import mocks
+
+
+MB = 2**20
+
+
+def iv_generator():
+    while True:
+        yield '3' * AES.block_size
+
+
+class CreateEncryptedFileTest(unittest.TestCase):
+    timeout = 5
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def create_file(self, filename):
+        session = mock.Mock(spec=Session.Session)(None, None)
+        hash_announcer = mock.Mock(spec=DHTHashAnnouncer.DHTHashAnnouncer)(None, None)
+        session.blob_manager = BlobManager.TempBlobManager(hash_announcer)
+        session.db_dir = self.tmp_dir
+        manager = mock.Mock(spec=EncryptedFileManager.EncryptedFileManager)()
+        handle = mocks.GenFile(3*MB, '1')
+        key = '2'*AES.block_size
+        return EncryptedFileCreator.create_lbry_file(
+            session, manager, filename, handle, key, iv_generator())
+
+    def test_can_create_file(self):
+        expected_stream_hash = ('41e6b247d923d191b154fb6f1b8529d6ddd6a73d65c357b1acb7'
+                                '42dd83151fb66393a7709e9f346260a4f4db6de10c25')
+        filename = 'test.file'
+        d = self.create_file(filename)
+        d.addCallback(self.assertEqual, expected_stream_hash)
+        return d
+
+    def test_can_create_file_with_unicode_filename(self):
+        expected_stream_hash = ('d1da4258f3ce12edb91d7e8e160d091d3ab1432c2e55a6352dce0'
+                                '2fd5adb86fe144e93e110075b5865fff8617776c6c0')
+        filename = u'☃.file'
+        d = self.create_file(filename)
+        d.addCallback(self.assertEqual, expected_stream_hash)
+        return d