add caching blob store

store/caching.go

@@ -0,0 +1,71 @@
+package store
+
+import (
+	"github.com/lbryio/lbry.go/extras/errors"
+)
+
+// CachingBlobStore combines two stores, typically a local and a remote store, to improve performance.
+// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
+// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
+type CachingBlobStore struct {
+	origin, cache BlobStore
+}
+
+// NewCachingBlobStore makes a new caching disk store and returns a pointer to it.
+func NewCachingBlobStore(origin, cache BlobStore) *CachingBlobStore {
+	return &CachingBlobStore{origin: origin, cache: cache}
+}
+
+// Has checks the cache and then the origin for a hash. It returns true if either store has it.
+func (c *CachingBlobStore) Has(hash string) (bool, error) {
+	has, err := c.cache.Has(hash)
+	if has || err != nil {
+		return has, err
+	}
+	return c.origin.Has(hash)
+}
+
+// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
+// from the origin, it is also stored in the cache.
+func (c *CachingBlobStore) Get(hash string) ([]byte, error) {
+	blob, err := c.cache.Get(hash)
+	if err == nil || !errors.Is(err, ErrBlobNotFound) {
+		return blob, err
+	}
+
+	blob, err = c.origin.Get(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	err = c.cache.Put(hash, blob)
+
+	return blob, err
+}
+
+// Put stores the blob in the origin and the cache
+func (c *CachingBlobStore) Put(hash string, blob []byte) error {
+	err := c.origin.Put(hash, blob)
+	if err != nil {
+		return err
+	}
+	return c.cache.Put(hash, blob)
+}
+
+// PutSD stores the sd blob in the origin and the cache
+func (c *CachingBlobStore) PutSD(hash string, blob []byte) error {
+	err := c.origin.PutSD(hash, blob)
+	if err != nil {
+		return err
+	}
+	return c.cache.PutSD(hash, blob)
+}
+
+// Delete deletes the blob from the origin and the cache
+func (c *CachingBlobStore) Delete(hash string) error {
+	err := c.origin.Delete(hash)
+	if err != nil {
+		return err
+	}
+	return c.cache.Delete(hash)
+}

store/caching_test.go

@@ -0,0 +1,73 @@
+package store
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestCachingBlobStore_Put(t *testing.T) {
+	origin := &MemoryBlobStore{}
+	cache := &MemoryBlobStore{}
+	s := NewCachingBlobStore(origin, cache)
+
+	b := []byte("this is a blob of stuff")
+	hash := "hash"
+
+	err := s.Put(hash, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	has, err := origin.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("failed to store blob in origin")
+	}
+
+	has, err = cache.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("failed to store blob in cache")
+	}
+}
+
+func TestCachingBlobStore_CacheMiss(t *testing.T) {
+	origin := &MemoryBlobStore{}
+	cache := &MemoryBlobStore{}
+	s := NewCachingBlobStore(origin, cache)
+
+	b := []byte("this is a blob of stuff")
+	hash := "hash"
+	err := origin.Put(hash, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	res, err := s.Get(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(b, res) {
+		t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
+	}
+
+	has, err := cache.Has(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !has {
+		t.Errorf("Get() did not copy blob to cache")
+	}
+
+	res, err = cache.Get(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(b, res) {
+		t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
+	}
+}

store/store.go

@@ -16,10 +16,11 @@ type BlobStore interface {
 	Delete(hash string) error
 }
 
+// Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.
 type Blocklister interface {
 	// Block deletes the blob and prevents it from being uploaded in the future
 	Block(hash string) error
-	// Wants returns false if the hash exists or is blocked, true otherwise
+	// Wants returns false if the hash exists in store or is blocked, true otherwise
 	Wants(hash string) (bool, error)
 }