From 2dd0221711f14dbd95706fcd658d810aecf20639 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 21 Sep 2018 12:38:57 -0400 Subject: [PATCH] [API] support ssl, add `use_https` setting --- lbrynet/conf.py | 14 +- lbrynet/daemon/Daemon.py | 4 +- lbrynet/daemon/auth/auth.py | 28 +--- lbrynet/daemon/auth/client.py | 35 +++-- lbrynet/daemon/auth/factory.py | 32 +++-- lbrynet/daemon/auth/keyring.py | 130 ++++++++++++++++++ lbrynet/daemon/auth/server.py | 30 ++-- lbrynet/daemon/auth/util.py | 92 ------------- .../core/client/test_ConnectionManager.py | 4 +- tests/unit/lbrynet_daemon/auth/test_server.py | 37 +++++ 10 files changed, 242 insertions(+), 164 deletions(-) create mode 100644 lbrynet/daemon/auth/keyring.py delete mode 100644 lbrynet/daemon/auth/util.py diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 6b926aff6..10d486b95 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -278,6 +278,7 @@ ADJUSTABLE_SETTINGS = { 'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY 'peer_search_timeout': (int, 60), 'use_auth_http': (bool, False), + 'use_https': (bool, False), 'use_upnp': (bool, True), 'use_keyring': (bool, False), 'wallet': (str, LBRYUM_WALLET), @@ -573,11 +574,14 @@ class Config: """ return os.path.join(self.ensure_data_dir(), self['LOG_FILE_NAME']) - def get_api_connection_string(self): - return 'http://%s:%i/%s' % (self['api_host'], self['api_port'], self['API_ADDRESS']) - - def get_ui_address(self): - return 'http://%s:%i' % (self['api_host'], self['api_port']) + def get_api_connection_string(self, user: str = None, password: str = None) -> str: + return 'http%s://%s%s:%i/%s' % ( + "" if not self['use_https'] else "s", + "" if not (user and password) else "%s:%s@" % (user, password), + self['api_host'], + self['api_port'], + self['API_ADDRESS'] + ) def get_db_revision_filename(self): return os.path.join(self.ensure_data_dir(), self['DB_REVISION_FILE_NAME']) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 0623102c8..6e9974666 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -215,8 +215,8 @@ class Daemon(AuthJSONRPCServer): Checker.INTERNET_CONNECTION[1]) } AuthJSONRPCServer.__init__(self, analytics_manager=analytics_manager, component_manager=component_manager, - use_authentication=conf.settings['use_auth_http'], to_skip=to_skip, - looping_calls=looping_calls) + use_authentication=conf.settings['use_auth_http'], + use_https=conf.settings['use_https'], to_skip=to_skip, looping_calls=looping_calls) self.is_first_run = is_first_run() # TODO: move this to a component diff --git a/lbrynet/daemon/auth/auth.py b/lbrynet/daemon/auth/auth.py index 104d75887..5fb5cd2e7 100644 --- a/lbrynet/daemon/auth/auth.py +++ b/lbrynet/daemon/auth/auth.py @@ -3,7 +3,7 @@ from zope.interface import implementer from twisted.cred import portal, checkers, credentials, error as cred_error from twisted.internet import defer from twisted.web import resource -from lbrynet.daemon.auth.util import load_api_keys +from .keyring import Keyring log = logging.getLogger(__name__) @@ -24,29 +24,11 @@ class HttpPasswordRealm: class PasswordChecker: credentialInterfaces = (credentials.IUsernamePassword,) - def __init__(self, passwords): - self.passwords = passwords - - @classmethod - def load_file(cls, key_path): - keys = load_api_keys(key_path) - return cls.load(keys) - - @classmethod - def load(cls, password_dict): - passwords = {key: password_dict[key].secret for key in password_dict} - log.info("Loaded %i api key(s)", len(passwords)) - return cls(passwords) + def __init__(self, keyring: Keyring): + self.api_key = keyring.api_key def requestAvatarId(self, creds): - password_dict_bytes = {} - for api in self.passwords: - password_dict_bytes.update({api.encode(): self.passwords[api].encode()}) - - if creds.username in password_dict_bytes: - pw = password_dict_bytes.get(creds.username) - pw_match = creds.checkPassword(pw) - if pw_match: - return defer.succeed(creds.username) + if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode(): + return defer.succeed(creds.username) log.warning('Incorrect username or password') return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password')) diff --git a/lbrynet/daemon/auth/client.py b/lbrynet/daemon/auth/client.py index 669d75e11..e6b3c63b6 100644 --- a/lbrynet/daemon/auth/client.py +++ b/lbrynet/daemon/auth/client.py @@ -1,18 +1,17 @@ -import os import json import aiohttp import logging from urllib.parse import urlparse from lbrynet import conf -from lbrynet.daemon.auth.util import load_api_keys, APIKey, API_KEY_NAME, get_auth_message +from lbrynet.daemon.auth.keyring import Keyring, APIKey log = logging.getLogger(__name__) USER_AGENT = "AuthServiceProxy/0.1" +TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION" TWISTED_SESSION = "TWISTED_SESSION" LBRY_SECRET = "LBRY_SECRET" HTTP_TIMEOUT = 30 -SCHEME = "http" class JSONRPCException(Exception): @@ -26,7 +25,6 @@ class UnAuthAPIClient: self.host = host self.port = port self.session = session - self.scheme = SCHEME def __getattr__(self, method): async def f(*args, **kwargs): @@ -39,12 +37,15 @@ class UnAuthAPIClient: url_fragment = urlparse(url) host = url_fragment.hostname port = url_fragment.port - session = aiohttp.ClientSession() + connector = aiohttp.TCPConnector( + ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context + ) + session = aiohttp.ClientSession(connector=connector) return cls(host, port, session) async def call(self, method, params=None): message = {'method': method, 'params': params} - async with self.session.get('{}://{}:{}'.format(self.scheme, self.host, self.port), json=message) as resp: + async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp: return await resp.json() @@ -76,7 +77,7 @@ class AuthAPIClient: 'params': params, 'id': self.__id_count } - to_auth = get_auth_message(pre_auth_post_data) + to_auth = json.dumps(pre_auth_post_data, sort_keys=True) auth_msg = self.__api_key.get_hmac(to_auth).decode() pre_auth_post_data.update({'hmac': auth_msg}) post_data = json.dumps(pre_auth_post_data) @@ -100,14 +101,11 @@ class AuthAPIClient: @classmethod async def get_client(cls, key_name=None): - api_key_name = key_name or API_KEY_NAME + api_key_name = key_name or "api" + keyring = Keyring.load_from_disk() - pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") - keys = load_api_keys(pw_path) - api_key = keys.get(api_key_name, False) - - login_url = "http://{}:{}@{}:{}".format(api_key_name, api_key.secret, conf.settings['api_host'], - conf.settings['api_port']) + api_key = keyring.api_key + login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret) url = urlparse(login_url) headers = { @@ -115,14 +113,13 @@ class AuthAPIClient: 'User-Agent': USER_AGENT, 'Content-type': 'application/json' } - - session = aiohttp.ClientSession() + connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context) + session = aiohttp.ClientSession(connector=connector) async with session.post(login_url, headers=headers) as r: cookies = r.cookies - - uid = cookies.get(TWISTED_SESSION).value - api_key = APIKey.new(seed=uid.encode()) + uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value + api_key = APIKey.create(seed=uid.encode()) return cls(api_key, session, cookies, url, login_url) diff --git a/lbrynet/daemon/auth/factory.py b/lbrynet/daemon/auth/factory.py index 86da0bbb1..549b9bd68 100644 --- a/lbrynet/daemon/auth/factory.py +++ b/lbrynet/daemon/auth/factory.py @@ -1,16 +1,28 @@ import logging -import os from twisted.web import server, guard, resource from twisted.cred import portal from lbrynet import conf from .auth import PasswordChecker, HttpPasswordRealm -from .util import initialize_api_key_file +from ..auth.keyring import Keyring log = logging.getLogger(__name__) +class HTTPJSONRPCFactory(server.Site): + def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs): + super().__init__(resource, requestFactory=requestFactory, *args, **kwargs) + self.use_ssl = False + + +class HTTPSJSONRPCFactory(server.Site): + def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs): + super().__init__(resource, requestFactory=requestFactory, *args, **kwargs) + self.options = keyring.private_certificate.options() + self.use_ssl = True + + class AuthJSONRPCResource(resource.Resource): def __init__(self, protocol): resource.Resource.__init__(self) @@ -22,17 +34,17 @@ class AuthJSONRPCResource(resource.Resource): request.setHeader('expires', '0') return self if name == '' else resource.Resource.getChild(self, name, request) - def getServerFactory(self): - if conf.settings['use_auth_http']: + def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site: + factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory + if use_authentication: log.info("Using authenticated API") - pw_path = os.path.join(conf.settings['data_dir'], ".api_keys") - initialize_api_key_file(pw_path) - checker = PasswordChecker.load_file(pw_path) + checker = PasswordChecker(keyring) realm = HttpPasswordRealm(self) portal_to_realm = portal.Portal(realm, [checker, ]) - factory = guard.BasicCredentialFactory('Login to lbrynet api') - root = guard.HTTPAuthSessionWrapper(portal_to_realm, [factory, ]) + root = guard.HTTPAuthSessionWrapper( + portal_to_realm, [guard.BasicCredentialFactory('Login to lbrynet api'), ] + ) else: log.info("Using non-authenticated API") root = self - return server.Site(root) + return factory_class(root, keyring) diff --git a/lbrynet/daemon/auth/keyring.py b/lbrynet/daemon/auth/keyring.py new file mode 100644 index 000000000..91d9290a1 --- /dev/null +++ b/lbrynet/daemon/auth/keyring.py @@ -0,0 +1,130 @@ +import os +import datetime +import hmac +import hashlib +import base58 +from OpenSSL.crypto import FILETYPE_PEM +from ssl import create_default_context, SSLContext +from cryptography.hazmat.backends import default_backend +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.x509.name import NameOID, NameAttribute +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from twisted.internet import ssl +import logging +from lbrynet import conf + +log = logging.getLogger(__name__) + + +def sha(x: bytes) -> str: + h = hashlib.sha256(x).digest() + return base58.b58encode(h).decode() + + +def generate_key(x: bytes = None) -> str: + if not x: + return sha(os.urandom(256)) + else: + return sha(x) + + +class APIKey: + def __init__(self, secret: str, name: str): + self.secret = secret + self.name = name + + @classmethod + def create(cls, seed=None, name=None): + secret = generate_key(seed) + return APIKey(secret, name) + + def _raw_key(self) -> str: + return base58.b58decode(self.secret) + + def get_hmac(self, message) -> str: + decoded_key = self._raw_key() + signature = hmac.new(decoded_key, message.encode(), hashlib.sha256) + return base58.b58encode(signature.digest()) + + def compare_hmac(self, message, token) -> bool: + decoded_token = base58.b58decode(token) + target = base58.b58decode(self.get_hmac(message)) + + try: + if len(decoded_token) != len(target): + return False + return hmac.compare_digest(decoded_token, target) + except: + return False + + +class Keyring: + encoding = serialization.Encoding.PEM + filetype = FILETYPE_PEM + + def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None): + self.api_key: APIKey = api_key + self.public_certificate: str = public_certificate + self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate + self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate) + + @classmethod + def load_from_disk(cls): + api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token') + api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem') + if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path): + return + with open(api_key_path, 'rb') as f: + api_key = APIKey(f.read().decode(), "api") + with open(api_ssl_cert_path, 'rb') as f: + public_cert = f.read().decode() + return cls(api_key, public_cert) + + @classmethod + def generate_and_save(cls): + dns = conf.settings['api_host'] + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + subject = issuer = x509.Name([ + NameAttribute(NameOID.COUNTRY_NAME, "US"), + NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"), + NameAttribute(NameOID.COMMON_NAME, "LBRY API"), + ]) + alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)]) + certificate = x509.CertificateBuilder( + subject_name=subject, + issuer_name=issuer, + public_key=private_key.public_key(), + serial_number=x509.random_serial_number(), + not_valid_before=datetime.datetime.utcnow(), + not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365), + extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)] + ).sign(private_key, hashes.SHA256(), default_backend()) + public_certificate = certificate.public_bytes(cls.encoding).decode() + private_certificate = ssl.PrivateCertificate.load( + public_certificate, + ssl.KeyPair.load( + private_key.private_bytes( + encoding=cls.encoding, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ).decode(), + cls.filetype + ), + cls.filetype + ) + + auth_token = APIKey.create(seed=None, name="api") + + with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f: + f.write(auth_token.secret.encode()) + + with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f: + f.write(public_certificate.encode()) + + return cls(auth_token, public_certificate, private_certificate) diff --git a/lbrynet/daemon/auth/server.py b/lbrynet/daemon/auth/server.py index 259baf9fd..34632e947 100644 --- a/lbrynet/daemon/auth/server.py +++ b/lbrynet/daemon/auth/server.py @@ -18,13 +18,14 @@ from lbrynet.core import utils from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet from lbrynet.core.looping_call_manager import LoopingCallManager from lbrynet.daemon.ComponentManager import ComponentManager -from .util import APIKey, get_auth_message, LBRY_SECRET +from .keyring import APIKey, Keyring from .undecorated import undecorated from .factory import AuthJSONRPCResource from lbrynet.daemon.json_response_encoder import JSONResponseEncoder log = logging.getLogger(__name__) EMPTY_PARAMS = [{}] +LBRY_SECRET = "LBRY_SECRET" class JSONRPCError: @@ -186,8 +187,8 @@ class AuthJSONRPCServer(AuthorizedBase): allowed_during_startup = [] component_attributes = {} - def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, to_skip=None, - looping_calls=None, reactor=None): + def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None, + to_skip=None, looping_calls=None, reactor=None): if not reactor: from twisted.internet import reactor self.analytics_manager = analytics_manager or analytics.Manager.new_instance() @@ -199,11 +200,13 @@ class AuthJSONRPCServer(AuthorizedBase): self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()}) self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()} self._use_authentication = use_authentication or conf.settings['use_auth_http'] + self._use_https = use_https or conf.settings['use_https'] self.listening_port = None self._component_setup_deferred = None self.announced_startup = False self.sessions = {} self.server = None + self.keyring = Keyring.generate_and_save() @defer.inlineCallbacks def start_listening(self): @@ -211,9 +214,16 @@ class AuthJSONRPCServer(AuthorizedBase): try: self.server = self.get_server_factory() - self.listening_port = reactor.listenTCP( - conf.settings['api_port'], self.server, interface=conf.settings['api_host'] - ) + if self.server.use_ssl: + log.info("Using SSL") + self.listening_port = reactor.listenSSL( + conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host'] + ) + else: + log.info("Not using SSL") + self.listening_port = reactor.listenTCP( + conf.settings['api_port'], self.server, interface=conf.settings['api_host'] + ) log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port']) yield self.setup() self.analytics_manager.send_server_startup_success() @@ -274,7 +284,7 @@ class AuthJSONRPCServer(AuthorizedBase): return d def get_server_factory(self): - return AuthJSONRPCResource(self).getServerFactory() + return AuthJSONRPCResource(self).getServerFactory(self.keyring, self._use_authentication, self._use_https) def _set_headers(self, request, data, update_secret=False): if conf.settings['allowed_origin']: @@ -460,7 +470,7 @@ class AuthJSONRPCServer(AuthorizedBase): @return: secret """ log.info("Started new api session") - token = APIKey.new(seed=session_id) + token = APIKey.create(seed=session_id) self.sessions.update({session_id: token}) def _unregister_user_session(self, session_id): @@ -565,13 +575,13 @@ class AuthJSONRPCServer(AuthorizedBase): def _verify_token(self, session_id, message, token): if token is None: raise InvalidAuthenticationToken('Authentication token not found') - to_auth = get_auth_message(message) + to_auth = json.dumps(message, sort_keys=True) api_key = self.sessions.get(session_id) if not api_key.compare_hmac(to_auth, token): raise InvalidAuthenticationToken('Invalid authentication token') def _update_session_secret(self, session_id): - self.sessions.update({session_id: APIKey.new(name=session_id)}) + self.sessions.update({session_id: APIKey.create(name=session_id)}) def _callback_render(self, result, request, id_, auth_required=False): try: diff --git a/lbrynet/daemon/auth/util.py b/lbrynet/daemon/auth/util.py deleted file mode 100644 index 9c860e479..000000000 --- a/lbrynet/daemon/auth/util.py +++ /dev/null @@ -1,92 +0,0 @@ -import base58 -import hmac -import hashlib -import yaml -import os -import json -import logging - -log = logging.getLogger(__name__) - -API_KEY_NAME = "api" -LBRY_SECRET = "LBRY_SECRET" - - -def sha(x: bytes) -> bytes: - h = hashlib.sha256(x).digest() - return base58.b58encode(h) - - -def generate_key(x: bytes = None) -> bytes: - if x is None: - return sha(os.urandom(256)) - else: - return sha(x) - - -class APIKey: - def __init__(self, secret, name, expiration=None): - self.secret = secret - self.name = name - self.expiration = expiration - - @classmethod - def new(cls, seed=None, name=None, expiration=None): - secret = generate_key(seed) - key_name = name if name else sha(secret) - return APIKey(secret, key_name, expiration) - - def _raw_key(self): - return base58.b58decode(self.secret) - - def get_hmac(self, message): - decoded_key = self._raw_key() - signature = hmac.new(decoded_key, message.encode(), hashlib.sha256) - return base58.b58encode(signature.digest()) - - def compare_hmac(self, message, token): - decoded_token = base58.b58decode(token) - target = base58.b58decode(self.get_hmac(message)) - - try: - if len(decoded_token) != len(target): - return False - return hmac.compare_digest(decoded_token, target) - except: - return False - - -def load_api_keys(path): - if not os.path.isfile(path): - raise Exception("Invalid api key path") - - with open(path, "r") as f: - data = yaml.load(f.read()) - - keys_for_return = {} - for key_name in data: - key = data[key_name] - secret = key['secret'].decode() - expiration = key['expiration'] - keys_for_return.update({key_name: APIKey(secret, key_name, expiration)}) - return keys_for_return - - -def save_api_keys(keys, path): - with open(path, "w") as f: - key_dict = {keys[key_name].name: {'secret': keys[key_name].secret, - 'expiration': keys[key_name].expiration} - for key_name in keys} - data = yaml.safe_dump(key_dict) - f.write(data) - - -def initialize_api_key_file(key_path): - keys = {} - new_api_key = APIKey.new(name=API_KEY_NAME) - keys.update({new_api_key.name: new_api_key}) - save_api_keys(keys, key_path) - - -def get_auth_message(message_dict): - return json.dumps(message_dict, sort_keys=True) diff --git a/tests/unit/core/client/test_ConnectionManager.py b/tests/unit/core/client/test_ConnectionManager.py index dc745b32f..ab9f28a38 100644 --- a/tests/unit/core/client/test_ConnectionManager.py +++ b/tests/unit/core/client/test_ConnectionManager.py @@ -1,5 +1,3 @@ -from unittest import skip - from lbrynet.core.client.ClientRequest import ClientRequest from lbrynet.core.server.ServerProtocol import ServerProtocol from lbrynet.core.client.ClientProtocol import ClientProtocol @@ -116,8 +114,8 @@ class MocServerProtocolFactory(ServerFactory): self.peer_manager = PeerManager() -@skip('times out, needs to be refactored to work with py3') class TestIntegrationConnectionManager(TestCase): + skip = 'times out, needs to be refactored to work with py3' def setUp(self): diff --git a/tests/unit/lbrynet_daemon/auth/test_server.py b/tests/unit/lbrynet_daemon/auth/test_server.py index 38ebe0f5c..d28bb8ad4 100644 --- a/tests/unit/lbrynet_daemon/auth/test_server.py +++ b/tests/unit/lbrynet_daemon/auth/test_server.py @@ -1,4 +1,5 @@ import mock +from twisted.internet import defer, reactor from twisted.trial import unittest from lbrynet import conf from lbrynet.daemon.auth import server @@ -14,6 +15,42 @@ class AuthJSONRPCServerTest(unittest.TestCase): conf.initialize_settings(False) self.server = server.AuthJSONRPCServer(True, use_authentication=False) + def test_listen_auth_https(self): + self.server._use_https = True + self.server._use_authentication = True + factory = self.server.get_server_factory() + listening_port = reactor.listenSSL( + conf.settings['api_port'], factory, factory.options, interface="localhost" + ) + listening_port.stopListening() + + def test_listen_no_auth_https(self): + self.server._use_https = True + self.server._use_authentication = False + factory = self.server.get_server_factory() + listening_port = reactor.listenSSL( + conf.settings['api_port'], factory, factory.options, interface="localhost" + ) + listening_port.stopListening() + + def test_listen_auth_http(self): + self.server._use_https = False + self.server._use_authentication = True + factory = self.server.get_server_factory() + listening_port = reactor.listenTCP( + conf.settings['api_port'], factory, interface="localhost" + ) + listening_port.stopListening() + + def test_listen_no_auth_http(self): + self.server._use_https = False + self.server._use_authentication = False + factory = self.server.get_server_factory() + listening_port = reactor.listenTCP( + conf.settings['api_port'], factory, interface="localhost" + ) + listening_port.stopListening() + def test_get_server_port(self): self.assertSequenceEqual( ('example.com', 80), self.server.get_server_port('http://example.com'))