lbry-sdk/lbry/wallet/server/storage.py

# Copyright (c) 2016-2017, the ElectrumX authors
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.

"""Backend database abstraction."""

import os
from functools import partial

from lbry.wallet.server import util


def db_class(db_dir, name):
    """Returns a DB engine class."""
    for db_class in util.subclasses(Storage):
        if db_class.__name__.lower() == name.lower():
            db_class.import_module()
            return partial(db_class, db_dir)
    raise RuntimeError(f'unrecognised DB engine "{name}"')


class Storage:
    """Abstract base class of the DB backend abstraction."""

    def __init__(self, db_dir, name, for_sync, read_only=False):
        self.db_dir = db_dir
        self.is_new = not os.path.exists(os.path.join(db_dir, name))
        self.for_sync = for_sync or self.is_new
        self.open(name, create=self.is_new, read_only=read_only)

    @classmethod
    def import_module(cls):
        """Import the DB engine module."""
        raise NotImplementedError

    def open(self, name, create, read_only=False):
        """Open an existing database or create a new one."""
        raise NotImplementedError

    def close(self):
        """Close an existing database."""
        raise NotImplementedError

    def get(self, key):
        raise NotImplementedError

    def put(self, key, value):
        raise NotImplementedError

    def write_batch(self):
        """Return a context manager that provides `put` and `delete`.

        Changes should only be committed when the context manager
        closes without an exception.
        """
        raise NotImplementedError

    def iterator(self, prefix=b'', reverse=False):
        """Return an iterator that yields (key, value) pairs from the
        database sorted by key.

        If `prefix` is set, only keys starting with `prefix` will be
        included.  If `reverse` is True the items are returned in
        reverse order.
        """
        raise NotImplementedError


class LevelDB(Storage):
    """LevelDB database engine."""

    @classmethod
    def import_module(cls):
        import plyvel
        cls.module = plyvel

    def open(self, name, create, read_only=False):
        mof = 10000
        path = os.path.join(self.db_dir, name)
        # Use snappy compression (the default)
        self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
        self.close = self.db.close
        self.get = self.db.get
        self.put = self.db.put
        self.iterator = self.db.iterator
        self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)


class RocksDB(Storage):
    """RocksDB database engine."""

    @classmethod
    def import_module(cls):
        import rocksdb
        cls.module = rocksdb

    def open(self, name, create, read_only=False):
        mof = 10000
        path = os.path.join(self.db_dir, name)
        # Use snappy compression (the default)
        options = self.module.Options(create_if_missing=create,
                                      use_fsync=True,
                                      target_file_size_base=33554432,
                                      max_open_files=mof)
        self.db = self.module.DB(path, options, read_only=read_only)
        self.get = self.db.get
        self.put = self.db.put
        self.multi_get = self.db.multi_get

    def close(self):
        # PyRocksDB doesn't provide a close method; hopefully this is enough
        self.db = self.get = self.put = None
        import gc
        gc.collect()

    def write_batch(self):
        return RocksDBWriteBatch(self.db)

    def iterator(self, **kwargs):
        return RocksDBIterator(self.db, **kwargs)


class RocksDBWriteBatch:
    """A write batch for RocksDB."""

    def __init__(self, db):
        self.batch = RocksDB.module.WriteBatch()
        self.db = db

    def __enter__(self):
        return self.batch

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not exc_val:
            self.db.write(self.batch)


class RocksDBIterator:
    """An iterator for RocksDB."""

    __slots__ = [
        'start',
        'prefix',
        'stop',
        'iterator',
        'include_key',
        'include_value',
        'prev',
        'reverse'
    ]

    def __init__(self, db, prefix=None, start=None, stop=None, include_key=True, include_value=True, reverse=False):
        assert (start is None and stop is None) or (prefix is None), 'cannot use start/stop and prefix'
        self.start = start
        self.prefix = prefix
        self.stop = stop
        self.iterator = db.iteritems() if not reverse else reversed(db.iteritems())
        if prefix is not None:
            self.iterator.seek(prefix)
        elif start is not None:
            self.iterator.seek(start)
        self.include_key = include_key
        self.include_value = include_value
        self.prev = None
        self.reverse = reverse

    def __iter__(self):
        return self

    def __next__(self):
        if None not in (self.stop, self.prev) and self.prev.startswith(self.stop):
            raise StopIteration
        self.prev, v = next(self.iterator)
        prev = self.prev
        if self.prefix is not None and not prev.startswith(self.prefix):
            raise StopIteration
        if self.include_key and self.include_value:
            return prev, v
        elif self.include_key:
            return prev
        return v