Profile get using rpc calls

scripts/profile_get.py

@@ -1,127 +1,18 @@
-# Script to download the sd_blobs of the front page
-# Author: hackrush(hackrush@lbry.io)
-import asyncio
-
-from twisted.internet import asyncioreactor
-asyncioreactor.install()
-
 import argparse
+import asyncio
+import aiohttp
 import keyring
-import logging
-import tempfile
 import time
-import treq
-import typing
-import shutil
 
-from binascii import unhexlify
+from aiohttp import ClientConnectorError
-from twisted.internet import defer, reactor
+from lbrynet import conf
-
+from lbrynet.extras.daemon.auth.client import UnAuthAPIClient
-from lbrynet import conf, log_support
-from lbrynet.p2p.Peer import Peer
-from lbrynet.p2p.BlobManager import DiskBlobManager
-from lbrynet.p2p import SinglePeerDownloader
-from lbrynet.extras.daemon.ComponentManager import ComponentManager
-from lbrynet.extras.daemon.Components import DatabaseComponent
-from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
-from lbrynet.extras.daemon.storage import SQLiteStorage
-
-log = logging.getLogger()
-
-component_manager = None
 
 
-def f2d(future):
+def kill_loop():
-    return defer.Deferred.fromFuture(asyncio.ensure_future(future))
+    loop = asyncio.get_event_loop()
-
+    loop.stop()
-
+    # loop.close()
-class FakeAnalytics:
-    @property
-    def is_started(self):
-        return True
-
-    def send_server_startup_success(self):
-        pass
-
-    def send_server_startup(self):
-        pass
-
-    def shutdown(self):
-        pass
-
-    def send_upnp_setup_success_fail(self, success, status):
-        pass
-
-
-class TempDatabaseComponent(DatabaseComponent):
-    @defer.inlineCallbacks
-    def start(self):
-        self.storage = SQLiteStorage(tempfile.tempdir)
-        yield self.storage.setup()
-
-    @defer.inlineCallbacks
-    def stop(self):
-        yield self.storage.stop()
-        self.storage = None
-
-
-class MultiplePeerFinder(DummyPeerFinder):
-    def __init__(self, peer):
-        super().__init__()
-        # This is just a dud!! It has absolutely no use
-        self.peer = peer
-
-    @defer.inlineCallbacks
-    def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
-        dht = component_manager.get_component('dht')
-        peers = yield dht.iterativeFindValue(unhexlify(blob_hash))
-        peers = [Peer(host, port) for node_id, host, port in peers]
-        return peers
-
-
-# monkeypatching custom peer finder
-SinglePeerDownloader.SinglePeerFinder = MultiplePeerFinder
-
-
-@defer.inlineCallbacks
-def download_blob_for_uri(blob_hash: typing.Text):
-    tmp_blob_manager = DiskBlobManager(tempfile.tempdir, component_manager.get_component('database'))
-
-    downloader = SinglePeerDownloader.SinglePeerDownloader()
-    downloader.setup(component_manager.get_component('wallet'))
-
-    peer = Peer(None, None)  # required for the log statements in SinglePeerDownloader
-    result = yield downloader.download_blob_from_peer(peer, 180, blob_hash, tmp_blob_manager)
-    tmp_blob_manager.stop()
-    return result
-
-
-async def get_sd_hash_from_uri(uri: typing.Text):
-    wallet = component_manager.get_component("wallet")
-    resolved = await wallet.resolve(uri)
-    sd_hash = resolved[uri]["claim"]["value"]["stream"]["source"]["source"]
-    print(sd_hash)
-    return sd_hash
-
-
-@defer.inlineCallbacks
-def benchmark_performance(uris: list, output_path: str) -> dict:
-    results = dict()
-    for uri in uris:
-        sd_hash = yield f2d(get_sd_hash_from_uri(uri))
-
-        start = time.time()
-        was_download_successful = yield download_blob_for_uri(sd_hash)
-        end = time.time()
-
-        if was_download_successful:
-            results[uri] = end - start
-        else:
-            results[uri] = "Could not download"
-
-        print(results[uri], uri, file=open(output_path, "a"))
-
-    return results
 
 
 def extract_uris(response):
@@ -133,85 +24,89 @@ def extract_uris(response):
     return uris
 
 
-@defer.inlineCallbacks
+async def get_frontpage_uris():
-def get_frontpage_uris():
     kr = keyring.get_keyring()
     c = kr.get_preferred_collection()
     lbry_keyring = None
     for col in c.get_all_items():
         if col.get_label() == "LBRY/auth_token":
             lbry_keyring = col
-    lbry_keyring = lbry_keyring.get_secret().decode("ascii")
+            break
-    response = yield treq.get("https://api.lbry.io/file/list_homepage?auth_token={}".format(lbry_keyring))
-    if response.code != 200:
-        log.error("API returned non 200 code!!")
-        reactor.callLater(0, reactor.stop)
 
-    body = yield response.json()
+    if lbry_keyring is None:
+        print("An auth token is needed to fetch the front page uris")
+        print("To generate the auth token, run the LBRY app at least once")
+        print("Then run the script again")
+
+    lbry_keyring = lbry_keyring.get_secret().decode("ascii")
+
+    session = aiohttp.ClientSession()
+    response = await session.get("https://api.lbry.io/file/list_homepage?auth_token={}".format(lbry_keyring))
+    if response.status != 200:
+        print("API returned non 200 code!!")
+        await session.close()
+        kill_loop()
+
+    body = await response.json()
+    await session.close()
     uris = extract_uris(body['data']['Uris'])
     return uris
 
 
-@defer.inlineCallbacks
+async def main():
-def main(output_path):
+    uris = await get_frontpage_uris()
-    global component_manager
+    api = await UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
-    yield component_manager.setup()
-    yield component_manager.get_component('dht')._join_deferred
-    uris = yield get_frontpage_uris()
-    # uris = [
-    #     "linux-kernel-patch-replaces-naughty#e4fc36ac921970b3d15138781d697841dfb745f7",
-    #     "linux-thursday-dec-1-2018-linux-y#062615ab5b974f5536b7e3d47731e7c74279ea23",
-    #     "what"
-    # ]
-    results = yield benchmark_performance(uris)
-    yield component_manager.stop()
-    shutil.rmtree(tempfile.tempdir)
 
-    max_len = len(max(uris, key=len))
+    try:
-    _sum = num = 0
+        await api.status()
-    for result, value in results.items():
+    except (ClientConnectorError, ConnectionError):
-        if not isinstance(value, str):
+        await api.session.close()
-            _sum += value
+        kill_loop()
-            num += 1
+        print("Could not connect to daemon. Are you sure it's running?")
-        print("{0:>{1:d}s}: {2}".format(result, max_len, value))
+        return 1
 
-    avg = _sum / num
+    results = dict()
-    print("Average for {0:d} downloaded files is: {1:f}".format(num, avg))
-    could_not_download = len(uris) - num
-    smiley = ":-)" if could_not_download == 0 else ":-("
-    print("{0:d} files could not be downloaded {1:s}".format(could_not_download, smiley))
 
-    reactor.callLater(0, reactor.stop)
+    # uris = ["what", "holi", "aweqwfq"]
+    _sum = 0
+    downloaded = len(uris)
+
+    for uri in uris:
+        start = time.time()
+        resp = await api.call("get", {"uri": uri})
+        end = time.time()
+
+        await api.call("file_delete", {"delete_from_download_dir": True,
+                                       "delete_all": True,
+                                       "claim_name": uri
+                                       })
+
+        time_taken = end - start
+        results[uri] = time_taken
+        _sum += time_taken
+
+        if resp.get('error'):
+            results[uri] = "Could not download"
+            downloaded -= 1
+            _sum -= time_taken
+
+        print(results[uri], uri)
+
+    avg = _sum / downloaded
+    print()
+    print("Average time taken:", avg)
+    print("Downloaded {} Not Downloaded {}".format(downloaded, len(uris) - downloaded))
+
+    await api.session.close()
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--wallet_path", help="Enter None to use a temporary directory")
+    parser.add_argument("--data_dir")
-    parser.add_argument("--output_path")
+    parser.add_argument("--wallet_dir")
+    parser.add_argument("--download_directory")
     args = parser.parse_args()
-    wallet_path = args.wallet_path
-    output_path = args.output_path
 
-    tempfile.tempdir = tempfile.mkdtemp()
+    conf.initialize_settings(data_dir=args.data_dir, wallet_dir=args.wallet_dir, download_dir=args.download_directory)
-    if wallet_path == "None":
+    loop = asyncio.get_event_loop()
-        wallet_path = tempfile.tempdir
+    loop.run_until_complete(main())
-
-    log_support.configure_console(level='INFO')
-    log_support.configure_twisted()
-
-    conf.initialize_settings()
-    conf.settings.set('download_directory', tempfile.tempdir)
-    conf.settings.set('lbryum_wallet_dir', wallet_path)
-    conf.settings.set('data_dir', tempfile.tempdir)
-    conf.settings.set('use_upnp', False)
-
-    skip_components = ["blob_manager", "hash_announcer", "file_manager", "peer_protocol_server", "reflector",
-                       "exchange_rate_manager", "rate_limiter", "payment_rate_manager"]
-    component_manager = ComponentManager(
-        analytics_manager=FakeAnalytics,
-        skip_components=skip_components,
-        database=TempDatabaseComponent
-    )
-
-    main(output_path)
-    reactor.run()