diff --git a/gcs/builder/builder.go b/gcs/builder/builder.go index 7a2e5e0..63afad3 100644 --- a/gcs/builder/builder.go +++ b/gcs/builder/builder.go @@ -55,8 +55,8 @@ func DeriveKey(keyHash *chainhash.Hash) [gcs.KeySize]byte { } // OutPointToFilterEntry is a utility function that derives a filter entry from -// a wire.OutPoint in a standardized way for use with both building and querying -// filters. +// a wire.OutPoint in a standardized way for use with both building and +// querying filters. func OutPointToFilterEntry(outpoint wire.OutPoint) []byte { // Size of the hash plus size of int32 index data := make([]byte, chainhash.HashSize+4) @@ -118,7 +118,7 @@ func (b *GCSBuilder) SetP(p uint8) *GCSBuilder { // Preallocate sets the estimated filter size after calling Builder() to reduce // the probability of memory reallocations. If the builder has already had data -// added to it, SetN has no effect. +// added to it, Preallocate has no effect. func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder { // Do nothing if the builder's already errored out. if b.err != nil { @@ -128,6 +128,7 @@ func (b *GCSBuilder) Preallocate(n uint32) *GCSBuilder { if len(b.data) == 0 { b.data = make([][]byte, 0, n) } + return b } @@ -157,8 +158,8 @@ func (b *GCSBuilder) AddEntries(data [][]byte) *GCSBuilder { return b } -// AddOutPoint adds a wire.OutPoint to the list of entries to be included in the -// GCS filter when it's built. +// AddOutPoint adds a wire.OutPoint to the list of entries to be included in +// the GCS filter when it's built. func (b *GCSBuilder) AddOutPoint(outpoint wire.OutPoint) *GCSBuilder { // Do nothing if the builder's already errored out. if b.err != nil { @@ -181,7 +182,7 @@ func (b *GCSBuilder) AddHash(hash *chainhash.Hash) *GCSBuilder { // AddScript adds all the data pushed in the script serialized as the passed // []byte to the list of entries to be included in the GCS filter when it's -// built. T +// built. func (b *GCSBuilder) AddScript(script []byte) *GCSBuilder { // Do nothing if the builder's already errored out. if b.err != nil { @@ -204,16 +205,16 @@ func (b *GCSBuilder) Build() (*gcs.Filter, error) { return gcs.BuildGCSFilter(b.p, b.key, b.data) } -// WithKeyPN creates a GCSBuilder with specified key and the passed -// probability and estimated filter size. +// WithKeyPN creates a GCSBuilder with specified key and the passed probability +// and estimated filter size. func WithKeyPN(key [gcs.KeySize]byte, p uint8, n uint32) *GCSBuilder { b := GCSBuilder{} return b.SetKey(key).SetP(p).Preallocate(n) } -// WithKeyP creates a GCSBuilder with specified key and the passed -// probability. Estimated filter size is set to zero, which means more -// reallocations are done when building the filter. +// WithKeyP creates a GCSBuilder with specified key and the passed probability. +// Estimated filter size is set to zero, which means more reallocations are +// done when building the filter. func WithKeyP(key [gcs.KeySize]byte, p uint8) *GCSBuilder { return WithKeyPN(key, p, 0) } @@ -246,8 +247,8 @@ func WithKeyHash(keyHash *chainhash.Hash) *GCSBuilder { return WithKeyHashPN(keyHash, DefaultP, 0) } -// WithRandomKeyPN creates a GCSBuilder with a cryptographically random -// key and the passed probability and estimated filter size. +// WithRandomKeyPN creates a GCSBuilder with a cryptographically random key and +// the passed probability and estimated filter size. func WithRandomKeyPN(p uint8, n uint32) *GCSBuilder { key, err := RandomKey() if err != nil { @@ -257,44 +258,63 @@ func WithRandomKeyPN(p uint8, n uint32) *GCSBuilder { return WithKeyPN(key, p, n) } -// WithRandomKeyP creates a GCSBuilder with a cryptographically random -// key and the passed probability. Estimated filter size is set to zero, which -// means more reallocations are done when building the filter. +// WithRandomKeyP creates a GCSBuilder with a cryptographically random key and +// the passed probability. Estimated filter size is set to zero, which means +// more reallocations are done when building the filter. func WithRandomKeyP(p uint8) *GCSBuilder { return WithRandomKeyPN(p, 0) } -// WithRandomKey creates a GCSBuilder with a cryptographically random -// key. Probability is set to 20 (2^-20 collision probability). Estimated -// filter size is set to zero, which means more reallocations are done when +// WithRandomKey creates a GCSBuilder with a cryptographically random key. +// Probability is set to 20 (2^-20 collision probability). Estimated filter +// size is set to zero, which means more reallocations are done when // building the filter. func WithRandomKey() *GCSBuilder { return WithRandomKeyPN(DefaultP, 0) } -// BuildBasicFilter builds a basic GCS filter from a block. +// BuildBasicFilter builds a basic GCS filter from a block. A basic GCS filter +// will contain all the previous outpoints spent within a block, as well as the +// data pushes within all the outputs created within a block. func BuildBasicFilter(block *wire.MsgBlock) (*gcs.Filter, error) { blockHash := block.BlockHash() b := WithKeyHash(&blockHash) + + // If the filter had an issue with the specified key, then we force it + // to bubble up here by calling the Key() function. _, err := b.Key() if err != nil { return nil, err } + + // In order to build a basic filter, we'll range over the entire block, + // adding the outpoint data as well as the data pushes within the + // pkScript. for i, tx := range block.Transactions { // Skip the inputs for the coinbase transaction if i != 0 { + // Each each txin, we'll add a serialized version of + // the txid:index to the filters data slices. for _, txIn := range tx.TxIn { b.AddOutPoint(txIn.PreviousOutPoint) } } + + // For each output in a transaction, we'll add each of the + // individual data pushes within the script. for _, txOut := range tx.TxOut { b.AddScript(txOut.PkScript) } } + return b.Build() } -// BuildExtFilter builds an extended GCS filter from a block. +// BuildExtFilter builds an extended GCS filter from a block. An extended +// filter supplements a regular basic filter by include all the _witness_ data +// found within a block. This includes all the data pushes within any signature +// scripts as well as each element of an input's witness stack. Additionally, +// the _hashes_ of each transaction are also inserted into the filter. func BuildExtFilter(block *wire.MsgBlock) (*gcs.Filter, error) { blockHash := block.BlockHash() b := WithKeyHash(&blockHash) @@ -312,6 +332,7 @@ func BuildExtFilter(block *wire.MsgBlock) (*gcs.Filter, error) { } } } + return b.Build() } @@ -326,8 +347,14 @@ func GetFilterHash(filter *gcs.Filter) chainhash.Hash { func MakeHeaderForFilter(filter *gcs.Filter, prevHeader chainhash.Hash) chainhash.Hash { filterTip := make([]byte, 2*chainhash.HashSize) filterHash := GetFilterHash(filter) + + // In the buffer we created above we'll compute hash || prevHash as an + // intermediate value. copy(filterTip, filterHash[:]) copy(filterTip[chainhash.HashSize:], prevHeader[:]) + + // The final filter hash is the double-sha256 of the hash computed + // above. hash1 := chainhash.HashH(filterTip) return chainhash.HashH(hash1[:]) } diff --git a/gcs/gcs.go b/gcs/gcs.go index ccf73e3..46890c8 100644 --- a/gcs/gcs.go +++ b/gcs/gcs.go @@ -30,19 +30,18 @@ var ( ) const ( - //KeySize is the size of the byte array required for key material for + // KeySize is the size of the byte array required for key material for // the SipHash keyed hash function. KeySize = 16 ) -// Filter describes an immutable filter that can be built from -// a set of data elements, serialized, deserialized, and queried -// in a thread-safe manner. The serialized form is compressed as -// a Golomb Coded Set (GCS), but does not include N or P to allow -// the user to encode the metadata separately if necessary. The -// hash function used is SipHash, a keyed function; the key used -// in building the filter is required in order to match filter -// values and is not included in the serialized form. +// Filter describes an immutable filter that can be built from a set of data +// elements, serialized, deserialized, and queried in a thread-safe manner. The +// serialized form is compressed as a Golomb Coded Set (GCS), but does not +// include N or P to allow the user to encode the metadata separately if +// necessary. The hash function used is SipHash, a keyed function; the key used +// in building the filter is required in order to match filter values and is +// not included in the serialized form. type Filter struct { n uint32 p uint8 @@ -54,9 +53,7 @@ type Filter struct { // BuildGCSFilter builds a new GCS filter with the collision probability of // `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of // the set. -func BuildGCSFilter(P uint8, key [KeySize]byte, - data [][]byte) (*Filter, error) { - +func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) { // Some initial parameter checks: make sure we have data from which to // build the filter, and make sure our parameters will fit the hash // function we're using. @@ -97,10 +94,12 @@ func BuildGCSFilter(P uint8, key [KeySize]byte, // Calculate the difference between this value and the last, // modulo P. remainder = (v - lastValue) % f.modulusP + // Calculate the difference between this value and the last, // divided by P. value = (v - lastValue - remainder) / f.modulusP lastValue = v + // Write the P multiple into the bitstream in unary; the // average should be around 1 (2 bits - 0b10). for value > 0 { @@ -108,6 +107,7 @@ func BuildGCSFilter(P uint8, key [KeySize]byte, value-- } b.WriteBit(false) + // Write the remainder as a big-endian integer with enough bits // to represent the appropriate collision probability. b.WriteBits(remainder, int(f.p)) @@ -115,11 +115,12 @@ func BuildGCSFilter(P uint8, key [KeySize]byte, // Copy the bitstream into the filter object and return the object. f.filterData = b.Bytes() + return &f, nil } -// FromBytes deserializes a GCS filter from a known N, P, and serialized -// filter as returned by Bytes(). +// FromBytes deserializes a GCS filter from a known N, P, and serialized filter +// as returned by Bytes(). func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) { // Basic sanity check. @@ -138,6 +139,7 @@ func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) { // Copy the filter. f.filterData = make([]byte, len(d)) copy(f.filterData, d) + return f, nil } @@ -206,8 +208,8 @@ func (f *Filter) N() uint32 { return f.n } -// Match checks whether a []byte value is likely (within collision -// probability) to be a member of the set represented by the filter. +// Match checks whether a []byte value is likely (within collision probability) +// to be a member of the set represented by the filter. func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) { // Create a filter bitstream. @@ -220,8 +222,9 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) { // Go through the search filter and look for the desired value. var lastValue uint64 for lastValue < term { - // Read the difference between previous and new value - // from bitstream. + + // Read the difference between previous and new value from + // bitstream. value, err := f.readFullUint64(b) if err != nil { if err == io.EOF { @@ -229,19 +232,22 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) { } return false, err } + // Add the previous value to it. value += lastValue if value == term { return true, nil } + lastValue = value } + return false, nil } -// MatchAny returns checks whether any []byte value is likely (within -// collision probability) to be a member of the set represented by the -// filter faster than calling Match() for each value individually. +// MatchAny returns checks whether any []byte value is likely (within collision +// probability) to be a member of the set represented by the filter faster than +// calling Match() for each value individually. func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) { // Basic sanity check. @@ -262,7 +268,8 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) { sort.Sort(values) // Zip down the filters, comparing values until we either run out of - // values to compare in one of the filters or we reach a matching value. + // values to compare in one of the filters or we reach a matching + // value. var lastValue1, lastValue2 uint64 lastValue2 = values[0] i := 1 @@ -292,13 +299,14 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) { lastValue1 += value } } - // If we've made it this far, an element matched between filters so - // we return true. + + // If we've made it this far, an element matched between filters so we + // return true. return true, nil } -// readFullUint64 reads a value represented by the sum of a unary multiple -// of the filter's P modulus (`2**P`) and a big-endian P-bit remainder. +// readFullUint64 reads a value represented by the sum of a unary multiple of +// the filter's P modulus (`2**P`) and a big-endian P-bit remainder. func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) { var v uint64 diff --git a/gcs/gcs_test.go b/gcs/gcs_test.go index de31bd8..ed62de4 100644 --- a/gcs/gcs_test.go +++ b/gcs/gcs_test.go @@ -22,8 +22,8 @@ var ( P = uint8(20) // Filters are conserved between tests but we must define with an - // interface which functions we're testing because the gcsFilter - // type isn't exported + // interface which functions we're testing because the gcsFilter type + // isn't exported filter, filter2, filter3, filter4, filter5 *gcs.Filter // We need to use the same key for building and querying the filters @@ -73,8 +73,8 @@ var ( ) // TestGCSFilterBuild builds a test filter with a randomized key. For Bitcoin -// use, deterministic filter generation is desired. Therefore, a -// key that's derived deterministically would be required. +// use, deterministic filter generation is desired. Therefore, a key that's +// derived deterministically would be required. func TestGCSFilterBuild(t *testing.T) { for i := 0; i < gcs.KeySize; i += 4 { binary.BigEndian.PutUint32(key[i:], rand.Uint32()) @@ -105,8 +105,8 @@ func TestGCSFilterCopy(t *testing.T) { } } -// TestGCSFilterMetadata checks that the filter metadata is built and -// copied correctly. +// TestGCSFilterMetadata checks that the filter metadata is built and copied +// correctly. func TestGCSFilterMetadata(t *testing.T) { if filter.P() != P { t.Fatal("P not correctly stored in filter metadata") @@ -213,8 +213,8 @@ func TestGCSFilterMatch(t *testing.T) { } } -// TestGCSFilterMatchAny checks that both the built and copied filters match -// a list correctly, logging any false positives without failing on them. +// TestGCSFilterMatchAny checks that both the built and copied filters match a +// list correctly, logging any false positives without failing on them. func TestGCSFilterMatchAny(t *testing.T) { match, err := filter.MatchAny(key, contents2) if err != nil { diff --git a/gcs/gcsbench_test.go b/gcs/gcsbench_test.go index bd22246..65581b3 100644 --- a/gcs/gcsbench_test.go +++ b/gcs/gcsbench_test.go @@ -41,7 +41,8 @@ func BenchmarkGCSFilterMatch(b *testing.B) { } } -// BenchmarkGCSFilterMatchAny benchmarks querying a filter for a list of values. +// BenchmarkGCSFilterMatchAny benchmarks querying a filter for a list of +// values. func BenchmarkGCSFilterMatchAny(b *testing.B) { for i := 0; i < b.N; i++ { filter.MatchAny(key, contents2) diff --git a/gcs/uint64slice.go b/gcs/uint64slice.go index 9629701..34fd2f0 100644 --- a/gcs/uint64slice.go +++ b/gcs/uint64slice.go @@ -5,8 +5,8 @@ package gcs -// uint64slice is a package-local utility class that allows us to use Go's -// sort package to sort a []uint64 by implementing sort.Interface. +// uint64slice is a package-local utility class that allows us to use Go's sort +// package to sort a []uint64 by implementing sort.Interface. type uint64Slice []uint64 // Len returns the length of the slice. @@ -14,8 +14,8 @@ func (p uint64Slice) Len() int { return len(p) } -// Less returns true when the ith element is smaller than the jth element -// of the slice, and returns false otherwise. +// Less returns true when the ith element is smaller than the jth element of +// the slice, and returns false otherwise. func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }