From af2742c34f6fb3b2201d1c2809d9e078d50192c9 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 18 Nov 2020 05:03:02 +0100 Subject: [PATCH] update quic don't wait for a blob to be written to disk before sending it downstream don't wait for the disk store to be walked before starting everything up --- go.mod | 12 ++++++------ go.sum | 13 +++++++------ store/caching.go | 13 ++++++++++--- store/lru.go | 13 +++++++------ store/speedwalk/speedwalk.go | 3 +-- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 0911e81..b98b2ad 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/lbryio/reflector.go replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 -//replace github.com/lbryio/lbry.go/v2 => ../lbry.go - require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/aws/aws-sdk-go v1.16.11 @@ -14,6 +12,7 @@ require ( github.com/golang/protobuf v1.4.2 github.com/google/btree v1.0.0 // indirect github.com/google/gops v0.3.7 + github.com/google/martian v2.1.0+incompatible github.com/gorilla/mux v1.7.4 github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/golang-lru v0.5.4 @@ -24,9 +23,9 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/lbryio/chainquery v1.9.0 github.com/lbryio/lbry.go v1.1.2 // indirect - github.com/lbryio/lbry.go/v2 v2.7.2-0.20210402181800-ec4d2cb37ce9 - github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 - github.com/lucas-clemente/quic-go v0.18.1 + github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 + github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec + github.com/lucas-clemente/quic-go v0.19.1 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.2 @@ -34,9 +33,10 @@ require ( github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.4.0 github.com/volatiletech/null v8.0.0+incompatible go.uber.org/atomic v1.5.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect diff --git a/go.sum b/go.sum index a46d5f3..9140bdb 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,7 @@ github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+u github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw= github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -216,8 +217,8 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiV github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M= github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= -github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4= +github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= @@ -226,12 +227,12 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0= -github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= +github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg= -github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= +github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/store/caching.go b/store/caching.go index 0a06395..c8e7d45 100644 --- a/store/caching.go +++ b/store/caching.go @@ -5,6 +5,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/stream" + log "github.com/sirupsen/logrus" "github.com/lbryio/reflector.go/internal/metrics" ) @@ -62,9 +63,15 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) { if err != nil { return nil, err } - - err = c.cache.Put(hash, blob) - return blob, err + // there is no need to wait for the blob to be stored before we return it + // TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS + go func() { + err = c.cache.Put(hash, blob) + if err != nil { + log.Errorf("error saving blob to underlying cache: %s", err.Error()) + } + }() + return blob, nil } // Put stores the blob in the origin and the cache diff --git a/store/lru.go b/store/lru.go index 4ef4908..6d06aa6 100644 --- a/store/lru.go +++ b/store/lru.go @@ -30,13 +30,14 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore { store: store, lru: lru, } - - if lstr, ok := store.(lister); ok { - err = l.loadExisting(lstr, maxItems) - if err != nil { - panic(err) // TODO: what should happen here? panic? return nil? just keep going? + go func() { + if lstr, ok := store.(lister); ok { + err = l.loadExisting(lstr, maxItems) + if err != nil { + panic(err) // TODO: what should happen here? panic? return nil? just keep going? + } } - } + }() return l } diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go index e2563ba..23230df 100644 --- a/store/speedwalk/speedwalk.go +++ b/store/speedwalk/speedwalk.go @@ -60,7 +60,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) { walkerWG.Done() goroutineLimiter <- struct{}{} }() - err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{ Unsorted: true, // faster this way Callback: func(osPathname string, de *godirwalk.Dirent) error { @@ -84,6 +83,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) { close(pathChan) pathWG.Wait() - + logrus.Infoln("loaded LRU") return paths, nil }