diff --git a/fs/du_unix.go b/fs/du_unix.go index d960b23..c34316b 100644 --- a/fs/du_unix.go +++ b/fs/du_unix.go @@ -20,10 +20,13 @@ package fs import ( "context" + "encoding/binary" "fmt" "os" "path/filepath" "syscall" + + "github.com/bits-and-blooms/bloom/v3" ) // blocksUnitSize is the unit used by `st_blocks` in `stat` in bytes. @@ -58,12 +61,46 @@ func fileInfoStat(path string, fi os.FileInfo) (*syscall.Stat_t, error) { return stat, nil } +type InodeStore struct { + filter *bloom.BloomFilter + real map[inode]struct{} +} + +func NewInodeStore(expectedItems uint, falsePositiveRate float64) *InodeStore { + return &InodeStore{ + filter: bloom.NewWithEstimates(expectedItems, falsePositiveRate), + real: make(map[inode]struct{}, expectedItems), + } +} + +func (s *InodeStore) Add(i inode) { + b := i.MarshalToBuf() + s.filter.Add(b) + s.real[i] = struct{}{} +} + +func (s *InodeStore) Exists(i inode) bool { + b := i.MarshalToBuf() + if !s.filter.Test(b) { + return false + } + _, ok := s.real[i] + return ok +} + +func (i inode) MarshalToBuf() []byte { + var arr [16]byte + binary.BigEndian.PutUint64(arr[0:8], i.dev) + binary.BigEndian.PutUint64(arr[8:16], i.ino) + return arr[:] +} + func diskUsage(ctx context.Context, roots ...string) (Usage, error) { var ( size int64 inodes = map[inode]struct{}{} // expensive! ) - + store := NewInodeStore(500000, 0.01) for _, root := range roots { if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -81,11 +118,10 @@ func diskUsage(ctx context.Context, roots ...string) (Usage, error) { return err } inoKey := newInode(stat) - if _, ok := inodes[inoKey]; !ok { - inodes[inoKey] = struct{}{} + if ok := store.Exists(inoKey); !ok { + store.Add(inoKey) size += stat.Blocks * blocksUnitSize } - return nil }); err != nil { return Usage{}, err diff --git a/go.mod b/go.mod index 555605d..da76c77 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/Microsoft/go-winio v0.6.2 + github.com/bits-and-blooms/bloom/v3 v3.7.1 github.com/containerd/log v0.1.0 github.com/opencontainers/go-digest v1.0.0 golang.org/x/sync v0.8.0 @@ -11,4 +12,7 @@ require ( google.golang.org/protobuf v1.35.1 ) -require github.com/sirupsen/logrus v1.9.3 // indirect +require ( + github.com/bits-and-blooms/bitset v1.24.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect +) diff --git a/go.sum b/go.sum index 4cd8137..258be34 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= +github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bloom/v3 v3.7.1 h1:WXovk4TRKZttAMJfoQx6K2DM0zNIt8w+c67UqO+etV0= +github.com/bits-and-blooms/bloom/v3 v3.7.1/go.mod h1:rZzYLLje2dfzXfAkJNxQQHsKurAyK55KUnL43Euk0hU= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,6 +21,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/vendor/github.com/bits-and-blooms/bitset/.gitignore b/vendor/github.com/bits-and-blooms/bitset/.gitignore new file mode 100644 index 0000000..5c204d2 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/.gitignore @@ -0,0 +1,26 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +target diff --git a/vendor/github.com/bits-and-blooms/bitset/.travis.yml b/vendor/github.com/bits-and-blooms/bitset/.travis.yml new file mode 100644 index 0000000..094aa5c --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/.travis.yml @@ -0,0 +1,37 @@ +language: go + +sudo: false + +branches: + except: + - release + +branches: + only: + - master + - travis + +go: + - "1.11.x" + - tip + +matrix: + allow_failures: + - go: tip + +before_install: + - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; + - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; + - go get github.com/mattn/goveralls + +before_script: + - make deps + +script: + - make qa + +after_failure: + - cat ./target/test/report.xml + +after_success: + - if [ "$TRAVIS_GO_VERSION" = "1.11.1" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; diff --git a/vendor/github.com/bits-and-blooms/bitset/LICENSE b/vendor/github.com/bits-and-blooms/bitset/LICENSE new file mode 100644 index 0000000..59cab8a --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md new file mode 100644 index 0000000..599982f --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/README.md @@ -0,0 +1,176 @@ +# bitset + +*Go language library to map between non-negative integers and boolean values* + +[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) +[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc) + + +This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems: + +* [beego](https://github.com/beego/beego) +* [CubeFS](https://github.com/cubefs/cubefs) +* [Amazon EKS Distro](https://github.com/aws/eks-distro) +* [sourcegraph](https://github.com/sourcegraph/sourcegraph-public-snapshot) +* [torrent](https://github.com/anacrolix/torrent) + + +## Description + +Package bitset implements bitsets, a mapping between non-negative integers and boolean values. +It should be more efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing individual integers. + +But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk automatically, but `Shrink` and `Compact` methods are available. On creation, a hint can be given for the number of bits that will be used. + +Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. + +### Example use: + +```go +package main + +import ( + "fmt" + "math/rand" + + "github.com/bits-and-blooms/bitset" +) + +func main() { + fmt.Printf("Hello from BitSet!\n") + var b bitset.BitSet + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` + +If you have Go 1.23 or better, you can iterate over the set bits like so: + +```go +for i := range b.EachSet() {} +``` + + + +Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc + +## Serialization + + +You may serialize a bitset safely and portably to a stream +of bytes as follows: +```Go + const length = 9585 + const oneEvery = 97 + bs := bitset.New(length) + // Add some bits + for i := uint(0); i < length; i += oneEvery { + bs = bs.Set(i) + } + + var buf bytes.Buffer + n, err := bs.WriteTo(&buf) + if err != nil { + // failure + } + // Here n == buf.Len() +``` +You can later deserialize the result as follows: + +```Go + // Read back from buf + bs = bitset.New() + n, err = bs.ReadFrom(&buf) + if err != nil { + // error + } + // n is the number of bytes read +``` + +The `ReadFrom` function attempts to read the data into the existing +BitSet instance, to minimize memory allocations. + + +*Performance tip*: +When reading and writing to a file or a network connection, you may get better performance by +wrapping your streams with `bufio` instances. + +E.g., +```Go + f, err := os.Create("myfile") + w := bufio.NewWriter(f) +``` +```Go + f, err := os.Open("myfile") + r := bufio.NewReader(f) +``` + +## Memory Usage + +The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](https://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). + +The `roaring` library allows you to go back and forth between compressed Roaring bitmaps and the conventional bitset instances: +```Go + mybitset := roaringbitmap.ToBitSet() + newroaringbitmap := roaring.FromBitSet(mybitset) +``` + + +### Goroutine safety + +In general, it's not safe to access the same BitSet using different goroutines--they are unsynchronized for performance. + +Should you want to access a BitSet from more than one goroutine, you should provide synchronization. Typically this is done by using channels to pass the *BitSet around (in Go style; so there is only ever one owner), or by using `sync.Mutex` to serialize operations on BitSets. + +## Installation + +```bash +go get github.com/bits-and-blooms/bitset +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +## Running all tests + +Before committing the code, please check if it passes tests, has adequate coverage, etc. +```bash +go test +go test -cover +``` + +## Stars + + +[![Star History Chart](https://api.star-history.com/svg?repos=bits-and-blooms/bitset&type=Date)](https://www.star-history.com/#bits-and-blooms/bitset&Date) + +## Further reading + +

Mastering Programming: From Testing to Performance in Go

+
diff --git a/vendor/github.com/bits-and-blooms/bitset/SECURITY.md b/vendor/github.com/bits-and-blooms/bitset/SECURITY.md new file mode 100644 index 0000000..f888420 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +You can report privately a vulnerability by email at daniel@lemire.me (current maintainer). diff --git a/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml b/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml new file mode 100644 index 0000000..f9b2959 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml @@ -0,0 +1,39 @@ +# Go +# Build your Go project. +# Add steps that test, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/go + +trigger: +- master + +pool: + vmImage: 'Ubuntu-16.04' + +variables: + GOBIN: '$(GOPATH)/bin' # Go binaries path + GOROOT: '/usr/local/go1.11' # Go installation path + GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path + modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code + +steps: +- script: | + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + shopt -s extglob + shopt -s dotglob + mv !(gopath) '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + displayName: 'Set up the Go workspace' + +- script: | + go version + go get -v -t -d ./... + if [ -f Gopkg.toml ]; then + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + dep ensure + fi + go build -v . + workingDirectory: '$(modulePath)' + displayName: 'Get dependencies, then build' diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go new file mode 100644 index 0000000..c8d8ddb --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/bitset.go @@ -0,0 +1,1767 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. +*/ +package bitset + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math/bits" + "strconv" +) + +// the wordSize of a bit set +const wordSize = 64 + +// the wordSize of a bit set in bytes +const wordBytes = wordSize / 8 + +// wordMask is wordSize-1, used for bit indexing in a word +const wordMask = wordSize - 1 + +// log2WordSize is lg(wordSize) +const log2WordSize = 6 + +// allBits has every bit set +const allBits uint64 = 0xffffffffffffffff + +// default binary BigEndian +var binaryOrder binary.ByteOrder = binary.BigEndian + +// default json encoding base64.URLEncoding +var base64Encoding = base64.URLEncoding + +// Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) +func Base64StdEncoding() { base64Encoding = base64.StdEncoding } + +// LittleEndian sets Marshal/Unmarshal Binary as Little Endian (Default: binary.BigEndian) +func LittleEndian() { binaryOrder = binary.LittleEndian } + +// BigEndian sets Marshal/Unmarshal Binary as Big Endian (Default: binary.BigEndian) +func BigEndian() { binaryOrder = binary.BigEndian } + +// BinaryOrder returns the current binary order, see also LittleEndian() +// and BigEndian() to change the order. +func BinaryOrder() binary.ByteOrder { return binaryOrder } + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet struct { + length uint + set []uint64 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet) safeSet() []uint64 { + if b.set == nil { + b.set = make([]uint64, wordsNeeded(0)) + } + return b.set +} + +// SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance +func (b *BitSet) SetBitsetFrom(buf []uint64) { + b.length = uint(len(buf)) * 64 + b.set = buf +} + +// From is a constructor used to create a BitSet from an array of words +func From(buf []uint64) *BitSet { + return FromWithLength(uint(len(buf))*64, buf) +} + +// FromWithLength constructs from an array of words and length in bits. +// This function is for advanced users, most users should prefer +// the From function. +// As a user of FromWithLength, you are responsible for ensuring +// that the length is correct: your slice should have length at +// least (length+63)/64 in 64-bit words. +func FromWithLength(length uint, set []uint64) *BitSet { + if len(set) < wordsNeeded(length) { + panic("BitSet.FromWithLength: slice is too short") + } + return &BitSet{length, set} +} + +// Bytes returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +// +// Deprecated: Bytes is deprecated. Use [BitSet.Words] instead. +func (b *BitSet) Bytes() []uint64 { + return b.set +} + +// Words returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +func (b *BitSet) Words() []uint64 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > (Cap() - wordMask) { + return int(Cap() >> log2WordSize) + } + return int((i + wordMask) >> log2WordSize) +} + +// wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. +// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing BitSet). +func wordsNeededUnbound(i uint) int { + return (int(i) + wordMask) >> log2WordSize +} + +// wordsIndex calculates the index of words in a `uint64` +func wordsIndex(i uint) uint { + return i & wordMask +} + +// New creates a new BitSet with a hint that length bits will be required. +// The memory usage is at least length/8 bytes. +// In case of allocation failure, the function will return a BitSet with zero +// capacity. +func New(length uint) (bset *BitSet) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet{ + 0, + make([]uint64, 0), + } + } + }() + + bset = &BitSet{ + length, + make([]uint64, wordsNeeded(length)), + } + + return bset +} + +// MustNew creates a new BitSet with the given length bits. +// It panics if length exceeds the possible capacity or by a lack of memory. +func MustNew(length uint) (bset *BitSet) { + if length >= Cap() { + panic("You are exceeding the capacity") + } + + return &BitSet{ + length, + make([]uint64, wordsNeeded(length)), // may panic on lack of memory + } +} + +// Cap returns the total possible capacity, or number of bits +// that can be stored in the BitSet theoretically. Under 32-bit system, +// it is 4294967295 and under 64-bit system, it is 18446744073709551615. +// Note that this is further limited by the maximum allocation size in Go, +// and your available memory, as any Go data structure. +func Cap() uint { + return ^uint(0) +} + +// Len returns the number of bits in the BitSet. +// Note that it differ from Count function. +func (b *BitSet) Len() uint { + return b.length +} + +// extendSet adds additional words to incorporate new bits if needed +func (b *BitSet) extendSet(i uint) { + if i >= Cap() { + panic("You are exceeding the capacity") + } + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint64, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 +} + +// Test whether bit i is set. +func (b *BitSet) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<> log2WordSize) + subWordIndex := wordsIndex(i) + + // The word that the index falls within, shifted so the index is at bit 0 + var firstWord, secondWord uint64 + if firstWordIndex < len(b.set) { + firstWord = b.set[firstWordIndex] >> subWordIndex + } + + // The next word, masked to only include the necessary bits and shifted to cover the + // top of the word + if (firstWordIndex + 1) < len(b.set) { + secondWord = b.set[firstWordIndex+1] << uint64(wordSize-subWordIndex) + } + + return firstWord | secondWord +} + +// Set bit i to 1, the capacity of the bitset is automatically +// increased accordingly. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +// The memory usage is at least slightly over i/8 bytes. +func (b *BitSet) Set(i uint) *BitSet { + if i >= b.length { // if we need more bits, make 'em + b.extendSet(i) + } + b.set[i>>log2WordSize] |= 1 << wordsIndex(i) + return b +} + +// Clear bit i to 0. This never cause a memory allocation. It is always safe. +func (b *BitSet) Clear(i uint) *BitSet { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << wordsIndex(i) + return b +} + +// SetTo sets bit i to value. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet) SetTo(i uint, value bool) *BitSet { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet) Flip(i uint) *BitSet { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << wordsIndex(i) + return b +} + +// FlipRange bit in [start, end). +// Warning: using a very large value for 'end' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet) FlipRange(start, end uint) *BitSet { + if start >= end { + return b + } + + if end-1 >= b.length { // if we need more bits, make 'em + b.extendSet(end - 1) + } + + startWord := int(start >> log2WordSize) + endWord := int(end >> log2WordSize) + + // b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) + // e.g: + // start = 71, + // startWord = 1 + // wordsIndex(start) = 71 % 64 = 7 + // (^uint64(0) << 7) = 0b111111....11110000000 + // + // mask = ^(^uint64(0) << 7) = 0b000000....00001111111 + // + // flips the first 7 bits in b.set[1] and + // in the range loop, the b.set[1] gets again flipped + // so the two expressions flip results in a flip + // in b.set[1] from [7,63] + // + // handle startWord special, get's reflipped in range loop + b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) + + for idx := range b.set[startWord:endWord] { + b.set[startWord+idx] = ^b.set[startWord+idx] + } + + // handle endWord special + // e.g. + // end = 135 + // endWord = 2 + // + // wordsIndex(-7) = 57 + // see the golang spec: + // "For unsigned integer values, the operations +, -, *, and << are computed + // modulo 2n, where n is the bit width of the unsigned integer's type." + // + // mask = ^uint64(0) >> 57 = 0b00000....0001111111 + // + // flips in b.set[2] from [0,7] + // + // is end at word boundary? + if idx := wordsIndex(-end); idx != 0 { + b.set[endWord] ^= ^uint64(0) >> wordsIndex(idx) + } + + return b +} + +// Shrink shrinks BitSet so that the provided value is the last possible +// set value. It clears all bits > the provided index and reduces the size +// and length of the set. +// +// Note that the parameter value is not the new length in bits: it is the +// maximal value that can be stored in the bitset after the function call. +// The new length in bits is the parameter value + 1. Thus it is not possible +// to use this function to set the length to 0, the minimal value of the length +// after this function call is 1. +// +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. +func (b *BitSet) Shrink(lastbitindex uint) *BitSet { + length := lastbitindex + 1 + idx := wordsNeeded(length) + if idx > len(b.set) { + return b + } + shrunk := make([]uint64, idx) + copy(shrunk, b.set[:idx]) + b.set = shrunk + b.length = length + lastWordUsedBits := length % 64 + if lastWordUsedBits != 0 { + b.set[idx-1] &= allBits >> uint64(64-wordsIndex(lastWordUsedBits)) + } + return b +} + +// Compact shrinks BitSet to so that we preserve all set bits, while minimizing +// memory usage. Compact calls Shrink. +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. +func (b *BitSet) Compact() *BitSet { + idx := len(b.set) - 1 + for ; idx >= 0 && b.set[idx] == 0; idx-- { + } + newlength := uint((idx + 1) << log2WordSize) + if newlength >= b.length { + return b // nothing to do + } + if newlength > 0 { + return b.Shrink(newlength - 1) + } + // We preserve one word + return b.Shrink(63) +} + +// InsertAt takes an index which indicates where a bit should be +// inserted. Then it shifts all the bits in the set to the left by 1, starting +// from the given index position, and sets the index position to 0. +// +// Depending on the size of your BitSet, and where you are inserting the new entry, +// this method could be extremely slow and in some cases might cause the entire BitSet +// to be recopied. +func (b *BitSet) InsertAt(idx uint) *BitSet { + insertAtElement := idx >> log2WordSize + + // if length of set is a multiple of wordSize we need to allocate more space first + if b.isLenExactMultiple() { + b.set = append(b.set, uint64(0)) + } + + var i uint + for i = uint(len(b.set) - 1); i > insertAtElement; i-- { + // all elements above the position where we want to insert can simply by shifted + b.set[i] <<= 1 + + // we take the most significant bit of the previous element and set it as + // the least significant bit of the current element + b.set[i] |= (b.set[i-1] & 0x8000000000000000) >> 63 + } + + // generate a mask to extract the data that we need to shift left + // within the element where we insert a bit + dataMask := uint64(1)< 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// DeleteAt deletes the bit at the given index position from +// within the bitset +// All the bits residing on the left of the deleted bit get +// shifted right by 1 +// The running time of this operation may potentially be +// relatively slow, O(length) +func (b *BitSet) DeleteAt(i uint) *BitSet { + // the index of the slice element where we'll delete a bit + deleteAtElement := i >> log2WordSize + + // generate a mask for the data that needs to be shifted right + // within that slice element that gets modified + dataMask := ^((uint64(1) << wordsIndex(i)) - 1) + + // extract the data that we'll shift right from the slice element + data := b.set[deleteAtElement] & dataMask + + // set the masked area to 0 while leaving the rest as it is + b.set[deleteAtElement] &= ^dataMask + + // shift the previously extracted data to the right and then + // set it in the previously masked area + b.set[deleteAtElement] |= (data >> 1) & dataMask + + // loop over all the consecutive slice elements to copy each + // lowest bit into the highest position of the previous element, + // then shift the entire content to the right by 1 + for i := int(deleteAtElement) + 1; i < len(b.set); i++ { + b.set[i-1] |= (b.set[i] & 1) << 63 + b.set[i] >>= 1 + } + + b.length = b.length - 1 + + return b +} + +// AppendTo appends all set bits to buf and returns the (maybe extended) buf. +// In case of allocation failure, the function will panic. +// +// See also [BitSet.AsSlice] and [BitSet.NextSetMany]. +func (b *BitSet) AppendTo(buf []uint) []uint { + // In theory, we could overflow uint, but in practice, we will not. + for idx, word := range b.set { + for word != 0 { + // In theory idx<> log2WordSize) + if x >= len(b.set) { + return 0, false + } + + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + if word != 0 { + return i + uint(bits.TrailingZeros64(word)), true + } + + // process the following full words until next bit is set + // x < len(b.set), no out-of-bounds panic in following slice expression + x++ + for idx, word := range b.set[x:] { + if word != 0 { + return uint((x+idx)< 0; j, buffer = bitmap.NextSetMany(j,buffer) { +// for k := range buffer { +// do something with buffer[k] +// } +// j += 1 +// } +// +// It is possible to retrieve all set bits as follow: +// +// indices := make([]uint, bitmap.Count()) +// bitmap.NextSetMany(0, indices) +// +// It is also possible to retrieve all set bits with [BitSet.AppendTo] +// or [BitSet.AsSlice]. +// +// However if Count() is large, it might be preferable to +// use several calls to NextSetMany for memory reasons. +func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { + // In theory, we could overflow uint, but in practice, we will not. + capacity := cap(buffer) + result := buffer[:capacity] + + x := int(i >> log2WordSize) + if x >= len(b.set) || capacity == 0 { + return 0, result[:0] + } + + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + + size := 0 + for word != 0 { + result[size] = i + uint(bits.TrailingZeros64(word)) + + size++ + if size == capacity { + return result[size-1], result[:size] + } + + // clear the rightmost set bit + word &= word - 1 + } + + // process the following full words + // x < len(b.set), no out-of-bounds panic in following slice expression + x++ + for idx, word := range b.set[x:] { + for word != 0 { + result[size] = uint((x+idx)< 0 { + return result[size-1], result[:size] + } + return 0, result[:0] +} + +// NextClear returns the next clear bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are set) +func (b *BitSet) NextClear(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + + // process first (maybe partial) word + word := b.set[x] + word = word >> wordsIndex(i) + wordAll := allBits >> wordsIndex(i) + + index := i + uint(bits.TrailingZeros64(^word)) + if word != wordAll && index < b.length { + return index, true + } + + // process the following full words until next bit is cleared + // x < len(b.set), no out-of-bounds panic in following slice expression + x++ + for idx, word := range b.set[x:] { + if word != allBits { + index = uint((x+idx)*wordSize + bits.TrailingZeros64(^word)) + if index < b.length { + return index, true + } + } + } + + return 0, false +} + +// PreviousSet returns the previous set bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are clear) +func (b *BitSet) PreviousSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + if word != 0 { + return uint(x<> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Flip all bits and find the highest one bit + word = ^word + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + word = ^word + if word != 0 { + return uint(x< b.wordCount() { + l = b.wordCount() + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the difference +func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() + } + cnt := uint64(0) + if l > 0 { + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + } + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) InPlaceDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() + } + if l <= 0 { + return + } + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + for i := 0; i < l; i++ { + data[i] &^= cmpData[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +// In case of allocation failure, the function will return an empty BitSet. +func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the intersection +func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + if b.length == 0 || compare.length == 0 { + return 0 + } + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet) InPlaceIntersection(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() + } + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + + for i := 0; i < l; i++ { + data[i] &= cmpData[i] + } + } + if l >= 0 { + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + } + if compare.length > 0 { + if compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + } +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet) Union(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet) UnionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntOrSlice(b.set, compare.set) + } + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet) InPlaceUnion(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() + } + if compare.length > 0 && compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + + for i := 0; i < l; i++ { + data[i] |= cmpData[i] + } + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntXorSlice(b.set, compare.set) + } + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() + } + if compare.length > 0 && compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + for i := 0; i < l; i++ { + data[i] ^= cmpData[i] + } + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet) isLenExactMultiple() bool { + return wordsIndex(b.length) == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet) cleanLastWord() { + if !b.isLenExactMultiple() { + b.set[len(b.set)-1] &= allBits >> (wordSize - wordsIndex(b.length)) + } +} + +// Complement computes the (local) complement of a bitset (up to length bits) +// In case of allocation failure, the function will return an empty BitSet. +func (b *BitSet) Complement() (result *BitSet) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet) IsSuperSet(other *BitSet) bool { + l := other.wordCount() + if b.wordCount() < l { + l = b.wordCount() + } + for i, word := range other.set[:l] { + if b.set[i]&word != word { + return false + } + } + return popcntSlice(other.set[l:]) == 0 +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits. Following the usual convention in Go, +// the least significant bits are printed last (index 0 is at the end of the string). +// This is useful for debugging and testing. It is not suitable for serialization. +func (b *BitSet) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return buffer.String() +} + +// BinaryStorageSize returns the binary storage requirements (see WriteTo) in bytes. +func (b *BitSet) BinaryStorageSize() int { + return wordBytes + wordBytes*b.wordCount() +} + +func readUint64Array(reader io.Reader, data []uint64) error { + length := len(data) + bufferSize := 128 + buffer := make([]byte, bufferSize*wordBytes) + for i := 0; i < length; i += bufferSize { + end := i + bufferSize + if end > length { + end = length + buffer = buffer[:wordBytes*(end-i)] + } + chunk := data[i:end] + if _, err := io.ReadFull(reader, buffer); err != nil { + return err + } + for i := range chunk { + chunk[i] = uint64(binaryOrder.Uint64(buffer[8*i:])) + } + } + return nil +} + +func writeUint64Array(writer io.Writer, data []uint64) error { + bufferSize := 128 + buffer := make([]byte, bufferSize*wordBytes) + for i := 0; i < len(data); i += bufferSize { + end := i + bufferSize + if end > len(data) { + end = len(data) + buffer = buffer[:wordBytes*(end-i)] + } + chunk := data[i:end] + for i, x := range chunk { + binaryOrder.PutUint64(buffer[8*i:], x) + } + _, err := writer.Write(buffer) + if err != nil { + return err + } + } + return nil +} + +// WriteTo writes a BitSet to a stream. The format is: +// 1. uint64 length +// 2. []uint64 set +// The length is the number of bits in the BitSet. +// +// The set is a slice of uint64s containing between length and length + 63 bits. +// It is interpreted as a big-endian array of uint64s by default (see BinaryOrder()) +// meaning that the first 8 bits are stored at byte index 7, the next 8 bits are stored +// at byte index 6... the bits 64 to 71 are stored at byte index 8, etc. +// If you change the binary order, you need to do so for both reading and writing. +// We recommend using the default binary order. +// +// Upon success, the number of bytes written is returned. +// +// Performance: if this function is used to write to a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Writer. +// E.g., +// +// f, err := os.Create("myfile") +// w := bufio.NewWriter(f) +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + // Write length + err := binary.Write(stream, binaryOrder, &length) + if err != nil { + // Upon failure, we do not guarantee that we + // return the number of bytes written. + return int64(0), err + } + err = writeUint64Array(stream, b.set[:b.wordCount()]) + if err != nil { + // Upon failure, we do not guarantee that we + // return the number of bytes written. + return int64(wordBytes), err + } + return int64(b.BinaryStorageSize()), nil +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +// The format is: +// 1. uint64 length +// 2. []uint64 set +// See WriteTo for details. +// Upon success, the number of bytes read is returned. +// If the current BitSet is not large enough to hold the data, +// it is extended. In case of error, the BitSet is either +// left unchanged or made empty if the error occurs too late +// to preserve the content. +// +// Performance: if this function is used to read from a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Reader. +// E.g., +// +// f, err := os.Open("myfile") +// r := bufio.NewReader(f) +func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + err := binary.Read(stream, binaryOrder, &length) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, err + } + newlength := uint(length) + + if uint64(newlength) != length { + return 0, errors.New("unmarshalling error: type mismatch") + } + nWords := wordsNeeded(uint(newlength)) + if cap(b.set) >= nWords { + b.set = b.set[:nWords] + } else { + b.set = make([]uint64, nWords) + } + + b.length = newlength + + err = readUint64Array(stream, b.set) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + // We do not want to leave the BitSet partially filled as + // it is error prone. + b.set = b.set[:0] + b.length = 0 + return 0, err + } + + return int64(b.BinaryStorageSize()), nil +} + +// MarshalBinary encodes a BitSet into a binary form and returns the result. +// Please see WriteTo for details. +func (b *BitSet) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + _, err := b.WriteTo(&buf) + if err != nil { + return []byte{}, err + } + + return buf.Bytes(), err +} + +// UnmarshalBinary decodes the binary form generated by MarshalBinary. +// Please see WriteTo for details. +func (b *BitSet) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + _, err := b.ReadFrom(buf) + return err +} + +// MarshalJSON marshals a BitSet as a JSON structure +func (b BitSet) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) + _, err := b.WriteTo(buffer) + if err != nil { + return nil, err + } + + // URLEncode all bytes + return json.Marshal(base64Encoding.EncodeToString(buffer.Bytes())) +} + +// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON +func (b *BitSet) UnmarshalJSON(data []byte) error { + // Unmarshal as string + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + // URLDecode string + buf, err := base64Encoding.DecodeString(s) + if err != nil { + return err + } + + _, err = b.ReadFrom(bytes.NewReader(buf)) + return err +} + +// Rank returns the number of set bits up to and including the index +// that are set in the bitset. +// See https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics +func (b *BitSet) Rank(index uint) (rank uint) { + index++ // Rank is up to and including + + // needed more than once + length := len(b.set) + + // TODO: built-in min requires go1.21 or later + // idx := min(int(index>>6), len(b.set)) + idx := int(index >> 6) + if idx > length { + idx = length + } + + // sum up the popcounts until idx ... + // TODO: cannot range over idx (...): requires go1.22 or later + // for j := range idx { + for j := 0; j < idx; j++ { + if w := b.set[j]; w != 0 { + rank += uint(bits.OnesCount64(w)) + } + } + + // ... plus partial word at idx, + // make Rank inlineable and faster in the end + // don't test index&63 != 0, just add, less branching + if idx < length { + rank += uint(bits.OnesCount64(b.set[idx] << (64 - index&63))) + } + + return +} + +// Select returns the index of the jth set bit, where j is the argument. +// The caller is responsible to ensure that 0 <= j < Count(): when j is +// out of range, the function returns the length of the bitset (b.length). +// +// Note that this function differs in convention from the Rank function which +// returns 1 when ranking the smallest value. We follow the conventional +// textbook definition of Select and Rank. +func (b *BitSet) Select(index uint) uint { + leftover := index + for idx, word := range b.set { + w := uint(bits.OnesCount64(word)) + if w > leftover { + return uint(idx)*64 + select64(word, leftover) + } + leftover -= w + } + return b.length +} + +// top detects the top bit set +func (b *BitSet) top() (uint, bool) { + for idx := len(b.set) - 1; idx >= 0; idx-- { + if word := b.set[idx]; word != 0 { + return uint(idx<= b.length { + b.length = top + bits + 1 + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just add pages + copy(dst[pages:nsize], b.set) + } else { + if pad+shift >= wordSize { + dst[idx+pages+1] = b.set[idx] >> (wordSize - shift) + } + + for i := int(idx); i >= 0; i-- { + if i > 0 { + dst[i+int(pages)] = (b.set[i] << shift) | (b.set[i-1] >> (wordSize - shift)) + } else { + dst[i+int(pages)] = b.set[i] << shift + } + } + } + + // zeroing extra pages + for i := 0; i < int(pages); i++ { + dst[i] = 0 + } + + b.set = dst +} + +// ShiftRight shifts the bitset like >> operation would do. +func (b *BitSet) ShiftRight(bits uint) { + panicIfNull(b) + + if bits == 0 { + return + } + + top, ok := b.top() + if !ok { + return + } + + if bits > top { + b.set = make([]uint64, wordsNeeded(b.length)) + return + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just clear pages + b.set = b.set[pages:] + b.length -= pages * wordSize + } else { + for i := 0; i <= int(idx-pages); i++ { + if i < int(idx-pages) { + b.set[i] = (b.set[i+int(pages)] >> shift) | (b.set[i+int(pages)+1] << (wordSize - shift)) + } else { + b.set[i] = b.set[i+int(pages)] >> shift + } + } + + if pad < shift { + b.set[int(idx-pages)] = 0 + } + } + + for i := int(idx-pages) + 1; i <= int(idx); i++ { + b.set[i] = 0 + } +} + +// OnesBetween returns the number of set bits in the range [from, to). +// The range is inclusive of 'from' and exclusive of 'to'. +// Returns 0 if from >= to. +func (b *BitSet) OnesBetween(from, to uint) uint { + panicIfNull(b) + + if from >= to { + return 0 + } + + // Calculate indices and masks for the starting and ending words + startWord := from >> log2WordSize // Divide by wordSize + endWord := to >> log2WordSize + startOffset := from & wordMask // Mod wordSize + endOffset := to & wordMask + + // Case 1: Bits lie within a single word + if startWord == endWord { + // Create mask for bits between from and to + mask := uint64((1<= startOffset + count = uint(bits.OnesCount64(b.set[startWord] & startMask)) + + // 2b: Count all bits in complete words between start and end + if endWord > startWord+1 { + count += uint(popcntSlice(b.set[startWord+1 : endWord])) + } + + // 2c: Count bits in last word (from start of word to endOffset) + if endOffset > 0 { + endMask := uint64(1<> log2WordSize + bitOffset := outPos & wordMask + + // Write extracted bits, handling word boundary crossing + dst.set[wordIdx] |= extracted << bitOffset + if bitOffset+bitsExtracted > wordSize { + dst.set[wordIdx+1] = extracted >> (wordSize - bitOffset) + } + + outPos += bitsExtracted + } +} + +// Deposit creates a new BitSet and deposits bits according to a mask. +// See DepositTo for details. +func (b *BitSet) Deposit(mask *BitSet) *BitSet { + dst := New(mask.length) + b.DepositTo(mask, dst) + return dst +} + +// DepositTo spreads bits from a compacted form in the BitSet into positions +// specified by mask in dst. This is the inverse operation of Extract. +// +// For example, if mask has bits set at positions 1,4,5, then DepositTo will +// take consecutive bits 0,1,2 from the source BitSet and place them into +// positions 1,4,5 in the destination BitSet. +func (b *BitSet) DepositTo(mask *BitSet, dst *BitSet) { + panicIfNull(b) + panicIfNull(mask) + panicIfNull(dst) + + if len(dst.set) == 0 || len(mask.set) == 0 || len(b.set) == 0 { + return + } + + inPos := uint(0) + length := len(mask.set) + if len(dst.set) < length { + length = len(dst.set) + } + + // Process each word + for i := 0; i < length; i++ { + if mask.set[i] == 0 { + continue // Skip words with no bits to deposit + } + + // Calculate source word index + wordIdx := inPos >> log2WordSize + if wordIdx >= uint(len(b.set)) { + break // No more source bits available + } + + // Get source bits, handling word boundary crossing + sourceBits := b.set[wordIdx] + bitOffset := inPos & wordMask + if wordIdx+1 < uint(len(b.set)) && bitOffset != 0 { + // Combine bits from current and next word + sourceBits = (sourceBits >> bitOffset) | + (b.set[wordIdx+1] << (wordSize - bitOffset)) + } else { + sourceBits >>= bitOffset + } + + // Deposit bits according to mask + dst.set[i] = (dst.set[i] &^ mask.set[i]) | pdep(sourceBits, mask.set[i]) + inPos += uint(bits.OnesCount64(mask.set[i])) + } +} + +//go:generate go run cmd/pextgen/main.go -pkg=bitset + +func pext(w, m uint64) (result uint64) { + var outPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + b := uint8(w >> shift) + mask := uint8(m >> shift) + + extracted := pextLUT[b][mask] + bits := popLUT[mask] + + result |= uint64(extracted) << outPos + outPos += uint(bits) + } + + return result +} + +func pdep(w, m uint64) (result uint64) { + var inPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + mask := uint8(m >> shift) + bits := popLUT[mask] + + // Get the bits we'll deposit from the source + b := uint8(w >> inPos) + + // Deposit them according to the mask for this byte + deposited := pdepLUT[b][mask] + + // Add to result + result |= uint64(deposited) << shift + inPos += uint(bits) + } + + return result +} diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go new file mode 100644 index 0000000..79bf8a0 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go @@ -0,0 +1,23 @@ +//go:build go1.23 +// +build go1.23 + +package bitset + +import ( + "iter" + "math/bits" +) + +func (b *BitSet) EachSet() iter.Seq[uint] { + return func(yield func(uint) bool) { + for wordIndex, word := range b.set { + idx := 0 + for trail := bits.TrailingZeros64(word); trail != 64; trail = bits.TrailingZeros64(word >> idx) { + if !yield(uint(wordIndex<= len(s) > 0. +func popcntAndSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE + for i := range s { + cnt += uint64(bits.OnesCount64(s[i] & m[i])) + } + return +} + +// popcntOrSlice computes the population count of the OR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntOrSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE + for i := range s { + cnt += uint64(bits.OnesCount64(s[i] | m[i])) + } + return +} + +// popcntXorSlice computes the population count of the XOR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntXorSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE + for i := range s { + cnt += uint64(bits.OnesCount64(s[i] ^ m[i])) + } + return +} diff --git a/vendor/github.com/bits-and-blooms/bitset/select.go b/vendor/github.com/bits-and-blooms/bitset/select.go new file mode 100644 index 0000000..a43c6bd --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/select.go @@ -0,0 +1,47 @@ +package bitset + +import "math/bits" + +func select64(w uint64, j uint) uint { + seen := 0 + // Divide 64bit + part := w & 0xFFFFFFFF + n := uint(bits.OnesCount64(part)) + if n <= j { + part = w >> 32 + seen += 32 + j -= n + } + ww := part + + // Divide 32bit + part = ww & 0xFFFF + + n = uint(bits.OnesCount64(part)) + if n <= j { + part = ww >> 16 + seen += 16 + j -= n + } + ww = part + + // Divide 16bit + part = ww & 0xFF + n = uint(bits.OnesCount64(part)) + if n <= j { + part = ww >> 8 + seen += 8 + j -= n + } + ww = part + + // Lookup in final byte + counter := 0 + for ; counter < 8; counter++ { + j -= uint((ww >> counter) & 1) + if j+1 == 0 { + break + } + } + return uint(seen + counter) +} diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/.gitignore b/vendor/github.com/bits-and-blooms/bloom/v3/.gitignore new file mode 100644 index 0000000..e719739 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/.gitignore @@ -0,0 +1,27 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +target +.idea diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/.travis.yml b/vendor/github.com/bits-and-blooms/bloom/v3/.travis.yml new file mode 100644 index 0000000..7b8fd30 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/.travis.yml @@ -0,0 +1,38 @@ +language: go + +sudo: false + +branches: + except: + - release + +branches: + only: + - master + - develop + - travis + +go: + - 1.8 + - tip + +matrix: + allow_failures: + - go: tip + +before_install: + - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; + - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; + - go get github.com/mattn/goveralls + +before_script: + - make deps + +script: + - make qa + +after_failure: + - cat ./target/test/report.xml + +after_success: + - if [ "$TRAVIS_GO_VERSION" = "1.8" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/LICENSE b/vendor/github.com/bits-and-blooms/bloom/v3/LICENSE new file mode 100644 index 0000000..3b9d36a --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/Makefile b/vendor/github.com/bits-and-blooms/bloom/v3/Makefile new file mode 100644 index 0000000..0fcbdcb --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/Makefile @@ -0,0 +1,197 @@ +# MAKEFILE +# +# @author Nicola Asuni +# @link https://github.com/bits-and-blooms/bloom +# ------------------------------------------------------------------------------ + +# List special make targets that are not associated with files +.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke + +# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS). +SHELL=/bin/bash + +# CVS path (path to the parent dir containing the project) +CVSPATH=github.com/bits-and-blooms + +# Project owner +OWNER=bits-and-blooms + +# Project vendor +VENDOR=bits-and-blooms + +# Project name +PROJECT=bloom + +# Project version +VERSION=$(shell cat VERSION) + +# Name of RPM or DEB package +PKGNAME=${VENDOR}-${PROJECT} + +# Current directory +CURRENTDIR=$(shell pwd) + +# GO lang path +ifneq ($(GOPATH),) + ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),) + # the defined GOPATH is not valid + GOPATH= + endif +endif +ifeq ($(GOPATH),) + # extract the GOPATH + GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR))) +endif + +# --- MAKE TARGETS --- + +# Display general help about this command +help: + @echo "" + @echo "$(PROJECT) Makefile." + @echo "GOPATH=$(GOPATH)" + @echo "The following commands are available:" + @echo "" + @echo " make qa : Run all the tests" + @echo " make test : Run the unit tests" + @echo "" + @echo " make format : Format the source code" + @echo " make fmtcheck : Check if the source code has been formatted" + @echo " make vet : Check for suspicious constructs" + @echo " make lint : Check for style errors" + @echo " make coverage : Generate the coverage report" + @echo " make cyclo : Generate the cyclomatic complexity report" + @echo " make ineffassign : Detect ineffectual assignments" + @echo " make misspell : Detect commonly misspelled words in source files" + @echo " make structcheck : Find unused struct fields" + @echo " make varcheck : Find unused global variables and constants" + @echo " make errcheck : Check that error return values are used" + @echo " make gosimple : Suggest code simplifications" + @echo " make astscan : GO AST scanner" + @echo "" + @echo " make docs : Generate source code documentation" + @echo "" + @echo " make deps : Get the dependencies" + @echo " make clean : Remove any build artifact" + @echo " make nuke : Deletes any intermediate file" + @echo "" + +# Alias for help target +all: help + +# Run the unit tests +test: + @mkdir -p target/test + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go test \ + -covermode=atomic \ + -bench=. \ + -race \ + -cpuprofile=target/report/cpu.out \ + -memprofile=target/report/mem.out \ + -mutexprofile=target/report/mutex.out \ + -coverprofile=target/report/coverage.out \ + -v ./... | \ + tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \ + test $${PIPESTATUS[0]} -eq 0 + +# Format the source code +format: + @find . -type f -name "*.go" -exec gofmt -s -w {} \; + +# Check if the source code has been formatted +fmtcheck: + @mkdir -p target + @find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff + @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } + +# Check for syntax errors +vet: + GOPATH=$(GOPATH) go vet . + +# Check for style errors +lint: + GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint . + +# Generate the coverage report +coverage: + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go tool cover -html=target/report/coverage.out -o target/report/coverage.html + +# Report cyclomatic complexity +cyclo: + @mkdir -p target/report + GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect ineffectual assignments +ineffassign: + @mkdir -p target/report + GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect commonly misspelled words in source files +misspell: + @mkdir -p target/report + GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Find unused struct fields +structcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt + +# Find unused global variables and constants +varcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt + +# Check that error return values are used +errcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt + +# Suggest code simplifications +gosimple: + @mkdir -p target/report + GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt + +# AST scanner +astscan: + @mkdir -p target/report + GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Generate source docs +docs: + @mkdir -p target/docs + nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 & + wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060` + @echo ''${PKGNAME}' Documentation ...' > target/docs/index.html + +# Alias to run all quality-assurance checks +qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan + +# --- INSTALL --- + +# Get the dependencies +deps: + GOPATH=$(GOPATH) go get ./... + GOPATH=$(GOPATH) go get github.com/golang/lint/golint + GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report + GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov + GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo + GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign + GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck + GOPATH=$(GOPATH) go get github.com/kisielk/errcheck + GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple + GOPATH=$(GOPATH) go get github.com/securego/gosec + +# Remove any build artifact +clean: + GOPATH=$(GOPATH) go clean ./... + +# Deletes any intermediate file +nuke: + rm -rf ./target + GOPATH=$(GOPATH) go clean -i ./... diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/README.md b/vendor/github.com/bits-and-blooms/bloom/v3/README.md new file mode 100644 index 0000000..435266b --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/README.md @@ -0,0 +1,176 @@ +Bloom filters +------------- +[![Test](https://github.com/bits-and-blooms/bloom/actions/workflows/test.yml/badge.svg)](https://github.com/bits-and-blooms/bloom/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/bits-and-blooms/bloom)](https://goreportcard.com/report/github.com/bits-and-blooms/bloom) +[![Go Reference](https://pkg.go.dev/badge/github.com/bits-and-blooms/bloom.svg)](https://pkg.go.dev/github.com/bits-and-blooms/bloom/v3) + +This library is used by popular systems such as [Milvus](https://github.com/milvus-io/milvus) and [beego](https://github.com/beego/Beego). + +A Bloom filter is a concise/compressed representation of a set, where the main +requirement is to make membership queries; _i.e._, whether an item is a +member of a set. A Bloom filter will always correctly report the presence +of an element in the set when the element is indeed present. A Bloom filter +can use much less storage than the original set, but it allows for some 'false positives': +it may sometimes report that an element is in the set whereas it is not. + +When you construct, you need to know how many elements you have (the desired capacity), and what is the desired false positive rate you are willing to tolerate. A common false-positive rate is 1%. The +lower the false-positive rate, the more memory you are going to require. Similarly, the higher the +capacity, the more memory you will use. +You may construct the Bloom filter capable of receiving 1 million elements with a false-positive +rate of 1% in the following manner. + +```Go + filter := bloom.NewWithEstimates(1000000, 0.01) +``` + +You should call `NewWithEstimates` conservatively: if you specify a number of elements that it is +too small, the false-positive bound might be exceeded. A Bloom filter is not a dynamic data structure: +you must know ahead of time what your desired capacity is. + +Our implementation accepts keys for setting and testing as `[]byte`. Thus, to +add a string item, `"Love"`: + +```Go + filter.Add([]byte("Love")) +``` + +Similarly, to test if `"Love"` is in bloom: + +```Go + if filter.Test([]byte("Love")) +``` + +For numerical data, we recommend that you look into the encoding/binary library. But, for example, to add a `uint32` to the filter: + +```Go + i := uint32(100) + n1 := make([]byte, 4) + binary.BigEndian.PutUint32(n1, i) + filter.Add(n1) +``` + +Godoc documentation: https://pkg.go.dev/github.com/bits-and-blooms/bloom/v3 + + +## Installation + +```bash +go get -u github.com/bits-and-blooms/bloom/v3 +``` + +## Verifying the False Positive Rate + + +Sometimes, the actual false positive rate may differ (slightly) from the +theoretical false positive rate. We have a function to estimate the false positive rate of a +Bloom filter with _m_ bits and _k_ hashing functions for a set of size _n_: + +```Go + if bloom.EstimateFalsePositiveRate(20*n, 5, n) > 0.001 ... +``` + +You can use it to validate the computed m, k parameters: + +```Go + m, k := bloom.EstimateParameters(n, fp) + ActualfpRate := bloom.EstimateFalsePositiveRate(m, k, n) +``` + +or + +```Go + f := bloom.NewWithEstimates(n, fp) + ActualfpRate := bloom.EstimateFalsePositiveRate(f.m, f.k, n) +``` + +You would expect `ActualfpRate` to be close to the desired false-positive rate `fp` in these cases. + +The `EstimateFalsePositiveRate` function creates a temporary Bloom filter. It is +also relatively expensive and only meant for validation. + +## Serialization + +You can read and write the Bloom filters as follows: + + +```Go + f := New(1000, 4) + var buf bytes.Buffer + bytesWritten, err := f.WriteTo(&buf) + if err != nil { + t.Fatal(err.Error()) + } + var g BloomFilter + bytesRead, err := g.ReadFrom(&buf) + if err != nil { + t.Fatal(err.Error()) + } + if bytesRead != bytesWritten { + t.Errorf("read unexpected number of bytes %d != %d", bytesRead, bytesWritten) + } +``` + +*Performance tip*: +When reading and writing to a file or a network connection, you may get better performance by +wrapping your streams with `bufio` instances. + +E.g., +```Go + f, err := os.Create("myfile") + w := bufio.NewWriter(f) +``` +```Go + f, err := os.Open("myfile") + r := bufio.NewReader(f) +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +This project includes a Makefile that allows you to test and build the project with simple commands. +To see all available options: +```bash +make help +``` + +## Running all tests + +Before committing the code, please check if it passes all tests using (note: this will install some dependencies): +```bash +make deps +make qa +``` + +## Design + +A Bloom filter has two parameters: _m_, the number of bits used in storage, and _k_, the number of hashing functions on elements of the set. (The actual hashing functions are important, too, but this is not a parameter for this implementation). A Bloom filter is backed by a [BitSet](https://github.com/bits-and-blooms/bitset); a key is represented in the filter by setting the bits at each value of the hashing functions (modulo _m_). Set membership is done by _testing_ whether the bits at each value of the hashing functions (again, modulo _m_) are set. If so, the item is in the set. If the item is actually in the set, a Bloom filter will never fail (the true positive rate is 1.0); but it is susceptible to false positives. The art is to choose _k_ and _m_ correctly. + +In this implementation, the hashing functions used is [murmurhash](github.com/twmb/murmur3), a non-cryptographic hashing function. + + +Given the particular hashing scheme, it's best to be empirical about this. Note +that estimating the FP rate will clear the Bloom filter. + + + + +### Goroutine safety + +In general, it not safe to access +the same filter using different goroutines--they are +unsynchronized for performance. Should you want to access +a filter from more than one goroutine, you should +provide synchronization. Typically this is done by using channels (in Go style; so there is only ever one owner), +or by using `sync.Mutex` to serialize operations. Exceptionally, you may access the same filter from different +goroutines if you never modify the content of the filter. + +## Stars + + +[![Star History Chart](https://api.star-history.com/svg?repos=bits-and-blooms/bloom&type=Date)](https://www.star-history.com/#bits-and-blooms/bloom&Date) + +## Further reading + +

Mastering Programming: From Testing to Performance in Go

+
diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/SECURITY.md b/vendor/github.com/bits-and-blooms/bloom/v3/SECURITY.md new file mode 100644 index 0000000..f888420 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +You can report privately a vulnerability by email at daniel@lemire.me (current maintainer). diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/bloom.go b/vendor/github.com/bits-and-blooms/bloom/v3/bloom.go new file mode 100644 index 0000000..89dbe24 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/bloom.go @@ -0,0 +1,453 @@ +/* +Package bloom provides data structures and methods for creating Bloom filters. + +A Bloom filter is a representation of a set of _n_ items, where the main +requirement is to make membership queries; _i.e._, whether an item is a +member of a set. + +A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large +multiple of the cardinality of the set to represent) and _k_, the number of hashing +functions on elements of the set. (The actual hashing functions are important, too, +but this is not a parameter for this implementation). A Bloom filter is backed by +a BitSet; a key is represented in the filter by setting the bits at each value of the +hashing functions (modulo _m_). Set membership is done by _testing_ whether the +bits at each value of the hashing functions (again, modulo _m_) are set. If so, +the item is in the set. If the item is actually in the set, a Bloom filter will +never fail (the true positive rate is 1.0); but it is susceptible to false +positives. The art is to choose _k_ and _m_ correctly. + +In this implementation, the hashing functions used is murmurhash, +a non-cryptographic hashing function. + +This implementation accepts keys for setting as testing as []byte. Thus, to +add a string item, "Love": + + uint n = 1000 + filter := bloom.New(20*n, 5) // load of 20, 5 keys + filter.Add([]byte("Love")) + +Similarly, to test if "Love" is in bloom: + + if filter.Test([]byte("Love")) + +For numeric data, I recommend that you look into the binary/encoding library. But, +for example, to add a uint32 to the filter: + + i := uint32(100) + n1 := make([]byte,4) + binary.BigEndian.PutUint32(n1,i) + f.Add(n1) + +Finally, there is a method to estimate the false positive rate of a +Bloom filter with _m_ bits and _k_ hashing functions for a set of size _n_: + + if bloom.EstimateFalsePositiveRate(20*n, 5, n) > 0.001 ... + +You can use it to validate the computed m, k parameters: + + m, k := bloom.EstimateParameters(n, fp) + ActualfpRate := bloom.EstimateFalsePositiveRate(m, k, n) + +or + + f := bloom.NewWithEstimates(n, fp) + ActualfpRate := bloom.EstimateFalsePositiveRate(f.m, f.k, n) + +You would expect ActualfpRate to be close to the desired fp in these cases. + +The EstimateFalsePositiveRate function creates a temporary Bloom filter. It is +also relatively expensive and only meant for validation. +*/ +package bloom + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + + "github.com/bits-and-blooms/bitset" +) + +// A BloomFilter is a representation of a set of _n_ items, where the main +// requirement is to make membership queries; _i.e._, whether an item is a +// member of a set. +type BloomFilter struct { + m uint + k uint + b *bitset.BitSet +} + +func max(x, y uint) uint { + if x > y { + return x + } + return y +} + +// New creates a new Bloom filter with _m_ bits and _k_ hashing functions +// We force _m_ and _k_ to be at least one to avoid panics. +func New(m uint, k uint) *BloomFilter { + return &BloomFilter{max(1, m), max(1, k), bitset.New(m)} +} + +// From creates a new Bloom filter with len(_data_) * 64 bits and _k_ hashing +// functions. The data slice is not going to be reset. +func From(data []uint64, k uint) *BloomFilter { + m := uint(len(data) * 64) + return FromWithM(data, m, k) +} + +// FromWithM creates a new Bloom filter with _m_ length, _k_ hashing functions. +// The data slice is not going to be reset. +func FromWithM(data []uint64, m, k uint) *BloomFilter { + return &BloomFilter{m, k, bitset.From(data)} +} + +// baseHashes returns the four hash values of data that are used to create k +// hashes +func baseHashes(data []byte) [4]uint64 { + var d digest128 // murmur hashing + hash1, hash2, hash3, hash4 := d.sum256(data) + return [4]uint64{ + hash1, hash2, hash3, hash4, + } +} + +// location returns the ith hashed location using the four base hash values +func location(h [4]uint64, i uint) uint64 { + ii := uint64(i) + return h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)] +} + +// location returns the ith hashed location using the four base hash values +func (f *BloomFilter) location(h [4]uint64, i uint) uint { + return uint(location(h, i) % uint64(f.m)) +} + +// EstimateParameters estimates requirements for m and k. +// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go +// used with permission. +func EstimateParameters(n uint, p float64) (m uint, k uint) { + m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2))) + k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n))) + return +} + +// NewWithEstimates creates a new Bloom filter for about n items with fp +// false positive rate +func NewWithEstimates(n uint, fp float64) *BloomFilter { + m, k := EstimateParameters(n, fp) + return New(m, k) +} + +// Cap returns the capacity, _m_, of a Bloom filter +func (f *BloomFilter) Cap() uint { + return f.m +} + +// K returns the number of hash functions used in the BloomFilter +func (f *BloomFilter) K() uint { + return f.k +} + +// BitSet returns the underlying bitset for this filter. +func (f *BloomFilter) BitSet() *bitset.BitSet { + return f.b +} + +// Add data to the Bloom Filter. Returns the filter (allows chaining) +func (f *BloomFilter) Add(data []byte) *BloomFilter { + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + f.b.Set(f.location(h, i)) + } + return f +} + +// Merge the data from two Bloom Filters. +func (f *BloomFilter) Merge(g *BloomFilter) error { + // Make sure the m's and k's are the same, otherwise merging has no real use. + if f.m != g.m { + return fmt.Errorf("m's don't match: %d != %d", f.m, g.m) + } + + if f.k != g.k { + return fmt.Errorf("k's don't match: %d != %d", f.m, g.m) + } + + f.b.InPlaceUnion(g.b) + return nil +} + +// Copy creates a copy of a Bloom filter. +func (f *BloomFilter) Copy() *BloomFilter { + fc := New(f.m, f.k) + fc.Merge(f) // #nosec + return fc +} + +// AddString to the Bloom Filter. Returns the filter (allows chaining) +func (f *BloomFilter) AddString(data string) *BloomFilter { + return f.Add([]byte(data)) +} + +// Test returns true if the data is in the BloomFilter, false otherwise. +// If true, the result might be a false positive. If false, the data +// is definitely not in the set. +func (f *BloomFilter) Test(data []byte) bool { + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + if !f.b.Test(f.location(h, i)) { + return false + } + } + return true +} + +// TestString returns true if the string is in the BloomFilter, false otherwise. +// If true, the result might be a false positive. If false, the data +// is definitely not in the set. +func (f *BloomFilter) TestString(data string) bool { + return f.Test([]byte(data)) +} + +// TestLocations returns true if all locations are set in the BloomFilter, false +// otherwise. +func (f *BloomFilter) TestLocations(locs []uint64) bool { + for i := 0; i < len(locs); i++ { + if !f.b.Test(uint(locs[i] % uint64(f.m))) { + return false + } + } + return true +} + +// TestAndAdd is equivalent to calling Test(data) then Add(data). +// The filter is written to unconditionnally: even if the element is present, +// the corresponding bits are still set. See also TestOrAdd. +// Returns the result of Test. +func (f *BloomFilter) TestAndAdd(data []byte) bool { + present := true + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + l := f.location(h, i) + if !f.b.Test(l) { + present = false + } + f.b.Set(l) + } + return present +} + +// TestAndAddString is the equivalent to calling Test(string) then Add(string). +// The filter is written to unconditionnally: even if the string is present, +// the corresponding bits are still set. See also TestOrAdd. +// Returns the result of Test. +func (f *BloomFilter) TestAndAddString(data string) bool { + return f.TestAndAdd([]byte(data)) +} + +// TestOrAdd is equivalent to calling Test(data) then if not present Add(data). +// If the element is already in the filter, then the filter is unchanged. +// Returns the result of Test. +func (f *BloomFilter) TestOrAdd(data []byte) bool { + present := true + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + l := f.location(h, i) + if !f.b.Test(l) { + present = false + f.b.Set(l) + } + } + return present +} + +// TestOrAddString is the equivalent to calling Test(string) then if not present Add(string). +// If the string is already in the filter, then the filter is unchanged. +// Returns the result of Test. +func (f *BloomFilter) TestOrAddString(data string) bool { + return f.TestOrAdd([]byte(data)) +} + +// ClearAll clears all the data in a Bloom filter, removing all keys +func (f *BloomFilter) ClearAll() *BloomFilter { + f.b.ClearAll() + return f +} + +// EstimateFalsePositiveRate returns, for a BloomFilter of m bits +// and k hash functions, an estimation of the false positive rate when +// +// storing n entries. This is an empirical, relatively slow +// +// test using integers as keys. +// This function is useful to validate the implementation. +func EstimateFalsePositiveRate(m, k, n uint) (fpRate float64) { + rounds := uint32(100000) + // We construct a new filter. + f := New(m, k) + n1 := make([]byte, 4) + // We populate the filter with n values. + for i := uint32(0); i < uint32(n); i++ { + binary.BigEndian.PutUint32(n1, i) + f.Add(n1) + } + fp := 0 + // test for number of rounds + for i := uint32(0); i < rounds; i++ { + binary.BigEndian.PutUint32(n1, i+uint32(n)+1) + if f.Test(n1) { + fp++ + } + } + fpRate = float64(fp) / (float64(rounds)) + return +} + +// Approximating the number of items +// https://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter +func (f *BloomFilter) ApproximatedSize() uint32 { + x := float64(f.b.Count()) + m := float64(f.Cap()) + k := float64(f.K()) + size := -1 * m / k * math.Log(1-x/m) / math.Log(math.E) + return uint32(math.Floor(size + 0.5)) // round +} + +// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct. +type bloomFilterJSON struct { + M uint `json:"m"` + K uint `json:"k"` + B *bitset.BitSet `json:"b"` +} + +// MarshalJSON implements json.Marshaler interface. +func (f BloomFilter) MarshalJSON() ([]byte, error) { + return json.Marshal(bloomFilterJSON{f.m, f.k, f.b}) +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (f *BloomFilter) UnmarshalJSON(data []byte) error { + var j bloomFilterJSON + err := json.Unmarshal(data, &j) + if err != nil { + return err + } + f.m = j.M + f.k = j.K + f.b = j.B + return nil +} + +// WriteTo writes a binary representation of the BloomFilter to an i/o stream. +// It returns the number of bytes written. +// +// Performance: if this function is used to write to a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Writer. +// E.g., +// +// f, err := os.Create("myfile") +// w := bufio.NewWriter(f) +func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) { + err := binary.Write(stream, binary.BigEndian, uint64(f.m)) + if err != nil { + return 0, err + } + err = binary.Write(stream, binary.BigEndian, uint64(f.k)) + if err != nil { + return 0, err + } + numBytes, err := f.b.WriteTo(stream) + return numBytes + int64(2*binary.Size(uint64(0))), err +} + +// ReadFrom reads a binary representation of the BloomFilter (such as might +// have been written by WriteTo()) from an i/o stream. It returns the number +// of bytes read. +// +// Performance: if this function is used to read from a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Reader. +// E.g., +// +// f, err := os.Open("myfile") +// r := bufio.NewReader(f) +func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) { + var m, k uint64 + err := binary.Read(stream, binary.BigEndian, &m) + if err != nil { + return 0, err + } + err = binary.Read(stream, binary.BigEndian, &k) + if err != nil { + return 0, err + } + b := &bitset.BitSet{} + numBytes, err := b.ReadFrom(stream) + if err != nil { + return 0, err + } + f.m = uint(m) + f.k = uint(k) + f.b = b + return numBytes + int64(2*binary.Size(uint64(0))), nil +} + +// GobEncode implements gob.GobEncoder interface. +func (f *BloomFilter) GobEncode() ([]byte, error) { + var buf bytes.Buffer + _, err := f.WriteTo(&buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// GobDecode implements gob.GobDecoder interface. +func (f *BloomFilter) GobDecode(data []byte) error { + buf := bytes.NewBuffer(data) + _, err := f.ReadFrom(buf) + + return err +} + +// MarshalBinary implements binary.BinaryMarshaler interface. +func (f *BloomFilter) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + _, err := f.WriteTo(&buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// UnmarshalBinary implements binary.BinaryUnmarshaler interface. +func (f *BloomFilter) UnmarshalBinary(data []byte) error { + buf := bytes.NewBuffer(data) + _, err := f.ReadFrom(buf) + + return err +} + +// Equal tests for the equality of two Bloom filters +func (f *BloomFilter) Equal(g *BloomFilter) bool { + return f.m == g.m && f.k == g.k && f.b.Equal(g.b) +} + +// Locations returns a list of hash locations representing a data item. +func Locations(data []byte, k uint) []uint64 { + locs := make([]uint64, k) + + // calculate locations + h := baseHashes(data) + for i := uint(0); i < k; i++ { + locs[i] = location(h, i) + } + + return locs +} diff --git a/vendor/github.com/bits-and-blooms/bloom/v3/murmur.go b/vendor/github.com/bits-and-blooms/bloom/v3/murmur.go new file mode 100644 index 0000000..c93b1ba --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bloom/v3/murmur.go @@ -0,0 +1,289 @@ +/* +The bloom library relied on the excellent murmur library +by Sébastien Paolacci. Unfortunately, it involved some heap +allocation. We want to avoid any heap allocation whatsoever +in the hashing process. To preserve backward compatibility, we roll +our own hashing functions. They are designed to be strictly equivalent +to Paolacci's implementation. + +License on original code: + + +Copyright 2013, Sébastien Paolacci. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package bloom + +import ( + "encoding/binary" + "math/bits" + "unsafe" +) + +const ( + c1_128 = 0x87c37b91114253d5 + c2_128 = 0x4cf5ad432745937f + block_size = 16 +) + +// digest128 represents a partial evaluation of a 128 bites hash. +type digest128 struct { + h1 uint64 // Unfinalized running hash part 1. + h2 uint64 // Unfinalized running hash part 2. +} + +// bmix will hash blocks (16 bytes) +func (d *digest128) bmix(p []byte) { + nblocks := len(p) / block_size + for i := 0; i < nblocks; i++ { + b := (*[16]byte)(unsafe.Pointer(&p[i*block_size])) + k1, k2 := binary.LittleEndian.Uint64(b[:8]), binary.LittleEndian.Uint64(b[8:]) + d.bmix_words(k1, k2) + } +} + +// bmix_words will hash two 64-bit words (16 bytes) +func (d *digest128) bmix_words(k1, k2 uint64) { + h1, h2 := d.h1, d.h2 + + k1 *= c1_128 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2_128 + h1 ^= k1 + + h1 = bits.RotateLeft64(h1, 27) + h1 += h2 + h1 = h1*5 + 0x52dce729 + + k2 *= c2_128 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1_128 + h2 ^= k2 + + h2 = bits.RotateLeft64(h2, 31) + h2 += h1 + h2 = h2*5 + 0x38495ab5 + d.h1, d.h2 = h1, h2 +} + +// sum128 computers two 64-bit hash value. It is assumed that +// bmix was first called on the data to process complete blocks +// of 16 bytes. The 'tail' is a slice representing the 'tail' (leftover +// elements, fewer than 16). If pad_tail is true, we make it seem like +// there is an extra element with value 1 appended to the tail. +// The length parameter represents the full length of the data (including +// the blocks of 16 bytes, and, if pad_tail is true, an extra byte). +func (d *digest128) sum128(pad_tail bool, length uint, tail []byte) (h1, h2 uint64) { + h1, h2 = d.h1, d.h2 + + var k1, k2 uint64 + if pad_tail { + switch (len(tail) + 1) & 15 { + case 15: + k2 ^= uint64(1) << 48 + break + case 14: + k2 ^= uint64(1) << 40 + break + case 13: + k2 ^= uint64(1) << 32 + break + case 12: + k2 ^= uint64(1) << 24 + break + case 11: + k2 ^= uint64(1) << 16 + break + case 10: + k2 ^= uint64(1) << 8 + break + case 9: + k2 ^= uint64(1) << 0 + + k2 *= c2_128 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1_128 + h2 ^= k2 + + break + + case 8: + k1 ^= uint64(1) << 56 + break + case 7: + k1 ^= uint64(1) << 48 + break + case 6: + k1 ^= uint64(1) << 40 + break + case 5: + k1 ^= uint64(1) << 32 + break + case 4: + k1 ^= uint64(1) << 24 + break + case 3: + k1 ^= uint64(1) << 16 + break + case 2: + k1 ^= uint64(1) << 8 + break + case 1: + k1 ^= uint64(1) << 0 + k1 *= c1_128 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2_128 + h1 ^= k1 + } + + } + switch len(tail) & 15 { + case 15: + k2 ^= uint64(tail[14]) << 48 + fallthrough + case 14: + k2 ^= uint64(tail[13]) << 40 + fallthrough + case 13: + k2 ^= uint64(tail[12]) << 32 + fallthrough + case 12: + k2 ^= uint64(tail[11]) << 24 + fallthrough + case 11: + k2 ^= uint64(tail[10]) << 16 + fallthrough + case 10: + k2 ^= uint64(tail[9]) << 8 + fallthrough + case 9: + k2 ^= uint64(tail[8]) << 0 + + k2 *= c2_128 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1_128 + h2 ^= k2 + + fallthrough + + case 8: + k1 ^= uint64(tail[7]) << 56 + fallthrough + case 7: + k1 ^= uint64(tail[6]) << 48 + fallthrough + case 6: + k1 ^= uint64(tail[5]) << 40 + fallthrough + case 5: + k1 ^= uint64(tail[4]) << 32 + fallthrough + case 4: + k1 ^= uint64(tail[3]) << 24 + fallthrough + case 3: + k1 ^= uint64(tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint64(tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint64(tail[0]) << 0 + k1 *= c1_128 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2_128 + h1 ^= k1 + } + + h1 ^= uint64(length) + h2 ^= uint64(length) + + h1 += h2 + h2 += h1 + + h1 = fmix64(h1) + h2 = fmix64(h2) + + h1 += h2 + h2 += h1 + + return h1, h2 +} + +func fmix64(k uint64) uint64 { + k ^= k >> 33 + k *= 0xff51afd7ed558ccd + k ^= k >> 33 + k *= 0xc4ceb9fe1a85ec53 + k ^= k >> 33 + return k +} + +// sum256 will compute 4 64-bit hash values from the input. +// It is designed to never allocate memory on the heap. So it +// works without any byte buffer whatsoever. +// It is designed to be strictly equivalent to +// +// a1 := []byte{1} +// hasher := murmur3.New128() +// hasher.Write(data) // #nosec +// v1, v2 := hasher.Sum128() +// hasher.Write(a1) // #nosec +// v3, v4 := hasher.Sum128() +// +// See TestHashRandom. +func (d *digest128) sum256(data []byte) (hash1, hash2, hash3, hash4 uint64) { + // We always start from zero. + d.h1, d.h2 = 0, 0 + // Process as many bytes as possible. + d.bmix(data) + // We have enough to compute the first two 64-bit numbers + length := uint(len(data)) + tail_length := length % block_size + tail := data[length-tail_length:] + hash1, hash2 = d.sum128(false, length, tail) + // Next we want to 'virtually' append 1 to the input, but, + // we do not want to append to an actual array!!! + if tail_length+1 == block_size { + // We are left with no tail!!! + word1 := binary.LittleEndian.Uint64(tail[:8]) + word2 := uint64(binary.LittleEndian.Uint32(tail[8 : 8+4])) + word2 = word2 | (uint64(tail[12]) << 32) | (uint64(tail[13]) << 40) | (uint64(tail[14]) << 48) + // We append 1. + word2 = word2 | (uint64(1) << 56) + // We process the resulting 2 words. + d.bmix_words(word1, word2) + tail := data[length:] // empty slice, deliberate. + hash3, hash4 = d.sum128(false, length+1, tail) + } else { + // We still have a tail (fewer than 15 bytes) but we + // need to append '1' to it. + hash3, hash4 = d.sum128(true, length+1, tail) + } + + return hash1, hash2, hash3, hash4 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c6943f5..c3a704a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -5,6 +5,12 @@ github.com/Microsoft/go-winio/internal/fs github.com/Microsoft/go-winio/internal/socket github.com/Microsoft/go-winio/internal/stringbuffer github.com/Microsoft/go-winio/pkg/guid +# github.com/bits-and-blooms/bitset v1.24.2 +## explicit; go 1.16 +github.com/bits-and-blooms/bitset +# github.com/bits-and-blooms/bloom/v3 v3.7.1 +## explicit; go 1.16 +github.com/bits-and-blooms/bloom/v3 # github.com/containerd/log v0.1.0 ## explicit; go 1.20 github.com/containerd/log