// Copyright 2013 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package bloom
import (
"crypto/rand"
"strings"
"testing"
"github.com/cockroachdb/pebble/internal/base"
"github.com/stretchr/testify/require"
)
func (f tableFilter) String() string {
var buf strings.Builder
for i, x := range f {
if i > 0 {
if i%8 == 0 {
buf.WriteString("\n")
} else {
buf.WriteString(" ")
}
}
for j := uint(0); j < 8; j++ {
if x&(1<<(7-j)) != 0 {
buf.WriteString("1")
} else {
buf.WriteString(".")
}
}
}
buf.WriteString("\n")
return buf.String()
}
func newTableFilter(bitsPerKey int, keys ...[]byte) tableFilter {
w := FilterPolicy(bitsPerKey).NewWriter(base.TableFilter)
for _, key := range keys {
w.AddKey(key)
}
return tableFilter(w.Finish(nil))
}
func TestSmallBloomFilter(t *testing.T) {
f := newTableFilter(10, []byte("hello"), []byte("world"))
// The magic expected string comes from running RocksDB's util/bloom_test.cc:FullBloomTest.FullSmall.
want := `
........ ........ ........ .......1 ........ ........ ........ ........
........ .1...... ........ .1...... ........ ........ ........ ........
...1.... ........ ........ ........ ........ ........ ........ ........
........ ........ ........ ........ ........ ........ ........ ...1....
........ ........ ........ ........ .....1.. ........ ........ ........
.......1 ........ ........ ........ ........ ........ .1...... ........
........ ........ ........ ........ ........ ...1.... ........ ........
.......1 ........ ........ ........ .1...1.. ........ ........ ........
.....11. .......1 ........ ........ ........
`
want = strings.TrimLeft(want, "\n")
require.EqualValues(t, want, f.String())
m := map[string]bool{
"hello": true,
"world": true,
"x": false,
"foo": false,
}
for k, want := range m {
require.EqualValues(t, want, f.MayContain([]byte(k)))
}
}
func TestBloomFilter(t *testing.T) {
nextLength := func(x int) int {
if x < 10 {
return x + 1
}
if x < 100 {
return x + 10
}
if x < 1000 {
return x + 100
}
return x + 1000
}
le32 := func(i int) []byte {
b := make([]byte, 4)
b[0] = uint8(uint32(i) >> 0)
b[1] = uint8(uint32(i) >> 8)
b[2] = uint8(uint32(i) >> 16)
b[3] = uint8(uint32(i) >> 24)
return b
}
nMediocreFilters, nGoodFilters := 0, 0
loop:
for length := 1; length <= 10000; length = nextLength(length) {
keys := make([][]byte, 0, length)
for i := 0; i < length; i++ {
keys = append(keys, le32(i))
}
f := newTableFilter(10, keys...)
// The size of the table bloom filter is measured in multiples of the
// cache line size. The '+2' contribution captures the rounding up in the
// length division plus preferring an odd number of cache lines. As such,
// this formula isn't exact, but the exact formula is hard to read.
maxLen := 5 + ((length*10)/cacheLineBits+2)*cacheLineSize
if len(f) > maxLen {
t.Errorf("length=%d: len(f)=%d > max len %d", length, len(f), maxLen)
continue
}
// All added keys must match.
for _, key := range keys {
if !f.MayContain(key) {
t.Errorf("length=%d: did not contain key %q", length, key)
continue loop
}
}
// Check false positive rate.
nFalsePositive := 0
for i := 0; i < 10000; i++ {
if f.MayContain(le32(1e9 + i)) {
nFalsePositive++
}
}
if nFalsePositive > 0.02*10000 {
t.Errorf("length=%d: %d false positives in 10000", length, nFalsePositive)
continue
}
if nFalsePositive > 0.0125*10000 {
nMediocreFilters++
} else {
nGoodFilters++
}
}
if nMediocreFilters > nGoodFilters/5 {
t.Errorf("%d mediocre filters but only %d good filters", nMediocreFilters, nGoodFilters)
}
}
func TestHash(t *testing.T) {
testCases := []struct {
s string
expected uint32
}{
// The magic expected numbers come from RocksDB's util/hash_test.cc:TestHash.
{"", 3164544308},
{"\x08", 422599524},
{"\x17", 3168152998},
{"\x9a", 3195034349},
{"\x1c", 2651681383},
{"\x4d\x76", 2447836956},
{"\x52\xd5", 3854228105},
{"\x91\xf7", 31066776},
{"\xd6\x27", 1806091603},
{"\x30\x46\x0b", 3808221797},
{"\x56\xdc\xd6", 2157698265},
{"\xd4\x52\x33", 1721992661},
{"\x6a\xb5\xf4", 2469105222},
{"\x67\x53\x81\x1c", 118283265},
{"\x69\xb8\xc0\x88", 3416318611},
{"\x1e\x84\xaf\x2d", 3315003572},
{"\x46\xdc\x54\xbe", 447346355},
{"\xd0\x7a\x6e\xea\x56", 4255445370},
{"\x86\x83\xd5\xa4\xd8", 2390603402},
{"\xb7\x46\xbb\x77\xce", 2048907743},
{"\x6c\xa8\xbc\xe5\x99", 2177978500},
{"\x5c\x5e\xe1\xa0\x73\x81", 1036846008},
{"\x08\x5d\x73\x1c\xe5\x2e", 229980482},
{"\x42\xfb\xf2\x52\xb4\x10", 3655585422},
{"\x73\xe1\xff\x56\x9c\xce", 3502708029},
{"\x5c\xbe\x97\x75\x54\x9a\x52", 815120748},
{"\x16\x82\x39\x49\x88\x2b\x36", 3056033698},
{"\x59\x77\xf0\xa7\x24\xf4\x78", 587205227},
{"\xd3\xa5\x7c\x0e\xc0\x02\x07", 2030937252},
{"\x31\x1b\x98\x75\x96\x22\xd3\x9a", 469635402},
{"\x38\xd6\xf7\x28\x20\xb4\x8a\xe9", 3530274698},
{"\xbb\x18\x5d\xf4\x12\x03\xf7\x99", 1974545809},
{"\x80\xd4\x3b\x3b\xae\x22\xa2\x78", 3563570120},
{"\x1a\xb5\xd0\xfe\xab\xc3\x61\xb2\x99", 2706087434},
{"\x8e\x4a\xc3\x18\x20\x2f\x06\xe6\x3c", 1534654151},
{"\xb6\xc0\xdd\x05\x3f\xc4\x86\x4c\xef", 2355554696},
{"\x9a\x5f\x78\x0d\xaf\x50\xe1\x1f\x55", 1400800912},
{"\x22\x6f\x39\x1f\xf8\xdd\x4f\x52\x17\x94", 3420325137},
{"\x32\x89\x2a\x75\x48\x3a\x4a\x02\x69\xdd", 3427803584},
{"\x06\x92\x5c\xf4\x88\x0e\x7e\x68\x38\x3e", 1152407945},
{"\xbd\x2c\x63\x38\xbf\xe9\x78\xb7\xbf\x15", 3382479516},
}
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
require.EqualValues(t, tc.expected, hash([]byte(tc.s)))
})
}
}
func BenchmarkBloomFilter(b *testing.B) {
const keyLen = 128
const numKeys = 1024
keys := make([][]byte, numKeys)
for i := range keys {
keys[i] = make([]byte, keyLen)
_, _ = rand.Read(keys[i])
}
b.ResetTimer()
policy := FilterPolicy(10)
for i := 0; i < b.N; i++ {
w := policy.NewWriter(base.TableFilter)
for _, key := range keys {
w.AddKey(key)
}
w.Finish(nil)
}
}