From 943e80e26c82d56753b4a96e6a27d05c3bdea52e Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 21 Oct 2015 17:00:09 +0200 Subject: [PATCH 1/5] Make benchmarks more realistic --- lib/db/benchmark_test.go | 199 +++++++++++++++++++++++++++++++++++++++ lib/db/set_test.go | 187 ------------------------------------ 2 files changed, 199 insertions(+), 187 deletions(-) create mode 100644 lib/db/benchmark_test.go diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go new file mode 100644 index 000000000..777b042ab --- /dev/null +++ b/lib/db/benchmark_test.go @@ -0,0 +1,199 @@ +// Copyright (C) 2015 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package db_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +var files, oneFile, firstHalf, secondHalf []protocol.FileInfo +var fs *db.FileSet + +func init() { + for i := 0; i < 1000; i++ { + files = append(files, protocol.FileInfo{ + Name: fmt.Sprintf("file%d", i), + Version: protocol.Vector{{ID: myID, Value: 1000}}, + Blocks: genBlocks(i), + }) + } + + middle := len(files) / 2 + firstHalf = files[:middle] + secondHalf = files[middle:] + oneFile = firstHalf[middle-1 : middle] + + ldb, _ := tempDB() + fs = db.NewFileSet("test", ldb) + fs.Replace(remoteDevice0, files) + fs.Replace(protocol.LocalDeviceID, firstHalf) +} + +func genBlocks(n int) []protocol.BlockInfo { + b := make([]protocol.BlockInfo, n) + for i := range b { + h := make([]byte, 32) + for j := range h { + h[j] = byte(i + j) + } + b[i].Size = int32(i) + b[i].Hash = h + } + return b +} + +func tempDB() (*leveldb.DB, string) { + dir, err := ioutil.TempDir("", "syncthing") + if err != nil { + panic(err) + } + db, err := leveldb.OpenFile(filepath.Join(dir, "db"), &opt.Options{OpenFilesCacheCapacity: 100}) + if err != nil { + panic(err) + } + return db, dir +} + +func BenchmarkReplaceAll(b *testing.B) { + ldb, dir := tempDB() + defer func() { + ldb.Close() + os.RemoveAll(dir) + }() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m := db.NewFileSet("test", ldb) + m.Replace(protocol.LocalDeviceID, files) + } + + b.ReportAllocs() +} + +func BenchmarkUpdateOneChanged(b *testing.B) { + changed := make([]protocol.FileInfo, 1) + changed[0] = oneFile[0] + changed[0].Version = changed[0].Version.Update(myID) + changed[0].Blocks = genBlocks(len(changed[0].Blocks)) + + for i := 0; i < b.N; i++ { + if i%1 == 0 { + fs.Update(protocol.LocalDeviceID, changed) + } else { + fs.Update(protocol.LocalDeviceID, oneFile) + } + } + + b.ReportAllocs() +} + +func BenchmarkUpdateOneUnchanged(b *testing.B) { + for i := 0; i < b.N; i++ { + fs.Update(protocol.LocalDeviceID, oneFile) + } + + b.ReportAllocs() +} + +func BenchmarkNeedHalf(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(secondHalf) { + b.Errorf("wrong length %d != %d", count, len(secondHalf)) + } + } + + b.ReportAllocs() +} + +func BenchmarkHave(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(firstHalf) { + b.Errorf("wrong length %d != %d", count, len(firstHalf)) + } + } + + b.ReportAllocs() +} + +func BenchmarkGlobal(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithGlobal(func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(files) { + b.Errorf("wrong length %d != %d", count, len(files)) + } + } + + b.ReportAllocs() +} + +func BenchmarkNeedHalfTruncated(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(secondHalf) { + b.Errorf("wrong length %d != %d", count, len(secondHalf)) + } + } + + b.ReportAllocs() +} + +func BenchmarkHaveTruncated(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(firstHalf) { + b.Errorf("wrong length %d != %d", count, len(firstHalf)) + } + } + + b.ReportAllocs() +} + +func BenchmarkGlobalTruncated(b *testing.B) { + for i := 0; i < b.N; i++ { + count := 0 + fs.WithGlobalTruncated(func(fi db.FileIntf) bool { + count++ + return true + }) + if count != len(files) { + b.Errorf("wrong length %d != %d", count, len(files)) + } + } + + b.ReportAllocs() +} diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 64e12603d..45f5837a1 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -28,19 +28,6 @@ func init() { const myID = 1 -func genBlocks(n int) []protocol.BlockInfo { - b := make([]protocol.BlockInfo, n) - for i := range b { - h := make([]byte, 32) - for j := range h { - h[j] = byte(i + j) - } - b[i].Size = int32(i) - b[i].Hash = h - } - return b -} - func globalList(s *db.FileSet) []protocol.FileInfo { var fs []protocol.FileInfo s.WithGlobal(func(fi db.FileIntf) bool { @@ -417,180 +404,6 @@ func TestInvalidAvailability(t *testing.T) { t.Error("Incorrect availability for 'none':", av) } } -func Benchmark10kReplace(b *testing.B) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - - var local []protocol.FileInfo - for i := 0; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - m := db.NewFileSet("test", ldb) - m.Replace(protocol.LocalDeviceID, local) - } -} - -func Benchmark10kUpdateChg(b *testing.B) { - var remote []protocol.FileInfo - for i := 0; i < 10000; i++ { - remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - - m := db.NewFileSet("test", ldb) - m.Replace(remoteDevice0, remote) - - var local []protocol.FileInfo - for i := 0; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - m.Replace(protocol.LocalDeviceID, local) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - for j := range local { - local[j].Version = local[j].Version.Update(myID) - } - b.StartTimer() - m.Update(protocol.LocalDeviceID, local) - } -} - -func Benchmark10kUpdateSme(b *testing.B) { - var remote []protocol.FileInfo - for i := 0; i < 10000; i++ { - remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - m := db.NewFileSet("test", ldb) - m.Replace(remoteDevice0, remote) - - var local []protocol.FileInfo - for i := 0; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - m.Replace(protocol.LocalDeviceID, local) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - m.Update(protocol.LocalDeviceID, local) - } -} - -func Benchmark10kNeed2k(b *testing.B) { - var remote []protocol.FileInfo - for i := 0; i < 10000; i++ { - remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - - m := db.NewFileSet("test", ldb) - m.Replace(remoteDevice0, remote) - - var local []protocol.FileInfo - for i := 0; i < 8000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - for i := 8000; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}}) - } - - m.Replace(protocol.LocalDeviceID, local) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - fs := needList(m, protocol.LocalDeviceID) - if l := len(fs); l != 2000 { - b.Errorf("wrong length %d != 2k", l) - } - } -} - -func Benchmark10kHaveFullList(b *testing.B) { - var remote []protocol.FileInfo - for i := 0; i < 10000; i++ { - remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - - m := db.NewFileSet("test", ldb) - m.Replace(remoteDevice0, remote) - - var local []protocol.FileInfo - for i := 0; i < 2000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - for i := 2000; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}}) - } - - m.Replace(protocol.LocalDeviceID, local) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - fs := haveList(m, protocol.LocalDeviceID) - if l := len(fs); l != 10000 { - b.Errorf("wrong length %d != 10k", l) - } - } -} - -func Benchmark10kGlobal(b *testing.B) { - var remote []protocol.FileInfo - for i := 0; i < 10000; i++ { - remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - - m := db.NewFileSet("test", ldb) - m.Replace(remoteDevice0, remote) - - var local []protocol.FileInfo - for i := 0; i < 2000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}}) - } - for i := 2000; i < 10000; i++ { - local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}}) - } - - m.Replace(protocol.LocalDeviceID, local) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - fs := globalList(m) - if l := len(fs); l != 10000 { - b.Errorf("wrong length %d != 10k", l) - } - } -} func TestGlobalReset(t *testing.T) { ldb, err := leveldb.Open(storage.NewMemStorage(), nil) From 0c0c69f0cf5c0045ef4f191e50a8ebc8568ec11b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 21 Oct 2015 22:47:31 +0200 Subject: [PATCH 2/5] The GC runs are legacy and slows things down quite a bit benchmark old ns/op new ns/op delta BenchmarkReplaceAll-8 2942370526 2866418930 -2.58% BenchmarkUpdateOneChanged-8 7402489 226635 -96.94% BenchmarkUpdateOneUnchanged-8 7298777 229090 -96.86% BenchmarkNeedHalf-8 113608416 104483393 -8.03% BenchmarkHave-8 29834263 29288220 -1.83% BenchmarkGlobal-8 162773699 159269126 -2.15% BenchmarkNeedHalfTruncated-8 111943400 108235000 -3.31% BenchmarkHaveTruncated-8 29490369 28945489 -1.85% BenchmarkGlobalTruncated-8 165841081 149355833 -9.94% benchmark old allocs new allocs delta BenchmarkReplaceAll-8 1054942 1054944 +0.00% BenchmarkUpdateOneChanged-8 1149 1135 -1.22% BenchmarkUpdateOneUnchanged-8 1135 1135 +0.00% BenchmarkNeedHalf-8 374774 374777 +0.00% BenchmarkHave-8 151995 151995 +0.00% BenchmarkGlobal-8 530042 530063 +0.00% BenchmarkNeedHalfTruncated-8 374697 374699 +0.00% BenchmarkHaveTruncated-8 151834 151834 +0.00% BenchmarkGlobalTruncated-8 530050 530021 -0.01% benchmark old bytes new bytes delta BenchmarkReplaceAll-8 5074294728 5074297112 +0.00% BenchmarkUpdateOneChanged-8 141048 135097 -4.22% BenchmarkUpdateOneUnchanged-8 134976 134976 +0.00% BenchmarkNeedHalf-8 44734813 44759436 +0.06% BenchmarkHave-8 11911634 11911138 -0.00% BenchmarkGlobal-8 80436854 81609867 +1.46% BenchmarkNeedHalfTruncated-8 46514673 46588024 +0.16% BenchmarkHaveTruncated-8 11348357 11348354 -0.00% BenchmarkGlobalTruncated-8 81730740 79485168 -2.75% --- lib/db/leveldb.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index 151f3ff31..159069e4e 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -12,7 +12,6 @@ package db import ( "bytes" "fmt" - "runtime" "sort" "github.com/syncthing/syncthing/lib/protocol" @@ -170,8 +169,6 @@ func globalKeyFolder(key []byte) []byte { type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 { - runtime.GC() - sort.Sort(fileList(fs)) // sort list on name, same as in the database start := deviceKey(folder, device, nil) // before all folder/device files @@ -306,8 +303,6 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, l } func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 { - runtime.GC() - batch := new(leveldb.Batch) l.Debugf("new batch %p", batch) snap, err := db.GetSnapshot() @@ -601,8 +596,6 @@ func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterat } func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f FileInfoTruncated) bool) { - runtime.GC() - start := deviceKey(folder, nil, nil) // before all folder/device files limit := deviceKey(folder, protocol.LocalDeviceID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files snap, err := db.GetSnapshot() @@ -706,8 +699,6 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf, } func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iterator) { - runtime.GC() - snap, err := db.GetSnapshot() if err != nil { panic(err) @@ -787,8 +778,6 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID { } func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) { - runtime.GC() - start := globalKey(folder, nil) limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff}) snap, err := db.GetSnapshot() @@ -887,8 +876,6 @@ nextFile: } func ldbListFolders(db *leveldb.DB) []string { - runtime.GC() - snap, err := db.GetSnapshot() if err != nil { panic(err) @@ -920,8 +907,6 @@ func ldbListFolders(db *leveldb.DB) []string { } func ldbDropFolder(db *leveldb.DB, folder []byte) { - runtime.GC() - snap, err := db.GetSnapshot() if err != nil { panic(err) @@ -966,8 +951,6 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { } func ldbCheckGlobals(db *leveldb.DB, folder []byte, globalSize *sizeTracker) { - defer runtime.GC() - snap, err := db.GetSnapshot() if err != nil { panic(err) From 0d9a04c7135395b358ec85c31d1322aca7748138 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 21 Oct 2015 22:55:40 +0200 Subject: [PATCH 3/5] Reuse blockkey, speeds up large Update and Replace calls benchmark old ns/op new ns/op delta BenchmarkReplaceAll-8 2866418930 2880834572 +0.50% BenchmarkUpdateOneChanged-8 226635 236596 +4.40% BenchmarkUpdateOneUnchanged-8 229090 227326 -0.77% BenchmarkNeedHalf-8 104483393 105151538 +0.64% BenchmarkHave-8 29288220 28827492 -1.57% BenchmarkGlobal-8 159269126 150768724 -5.34% BenchmarkNeedHalfTruncated-8 108235000 104434216 -3.51% BenchmarkHaveTruncated-8 28945489 27860093 -3.75% BenchmarkGlobalTruncated-8 149355833 149972888 +0.41% benchmark old allocs new allocs delta BenchmarkReplaceAll-8 1054944 555451 -47.35% BenchmarkUpdateOneChanged-8 1135 1135 +0.00% BenchmarkUpdateOneUnchanged-8 1135 1135 +0.00% BenchmarkNeedHalf-8 374777 374779 +0.00% BenchmarkHave-8 151995 151996 +0.00% BenchmarkGlobal-8 530063 530066 +0.00% BenchmarkNeedHalfTruncated-8 374699 374702 +0.00% BenchmarkHaveTruncated-8 151834 151834 +0.00% BenchmarkGlobalTruncated-8 530021 530049 +0.01% benchmark old bytes new bytes delta BenchmarkReplaceAll-8 5074297112 5018351912 -1.10% BenchmarkUpdateOneChanged-8 135097 135085 -0.01% BenchmarkUpdateOneUnchanged-8 134976 134976 +0.00% BenchmarkNeedHalf-8 44759436 44769400 +0.02% BenchmarkHave-8 11911138 11930612 +0.16% BenchmarkGlobal-8 81609867 81523668 -0.11% BenchmarkNeedHalfTruncated-8 46588024 46692342 +0.22% BenchmarkHaveTruncated-8 11348354 11348357 +0.00% BenchmarkGlobalTruncated-8 79485168 81843956 +2.97% --- lib/db/blockmap.go | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/db/blockmap.go b/lib/db/blockmap.go index 41354d348..3e572e227 100644 --- a/lib/db/blockmap.go +++ b/lib/db/blockmap.go @@ -42,6 +42,7 @@ func NewBlockMap(db *leveldb.DB, folder string) *BlockMap { func (m *BlockMap) Add(files []protocol.FileInfo) error { batch := new(leveldb.Batch) buf := make([]byte, 4) + var key []byte for _, file := range files { if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() { continue @@ -49,7 +50,8 @@ func (m *BlockMap) Add(files []protocol.FileInfo) error { for i, block := range file.Blocks { binary.BigEndian.PutUint32(buf, uint32(i)) - batch.Put(m.blockKey(block.Hash, file.Name), buf) + key = m.blockKeyInto(key, block.Hash, file.Name) + batch.Put(key, buf) } } return m.db.Write(batch, nil) @@ -59,6 +61,7 @@ func (m *BlockMap) Add(files []protocol.FileInfo) error { func (m *BlockMap) Update(files []protocol.FileInfo) error { batch := new(leveldb.Batch) buf := make([]byte, 4) + var key []byte for _, file := range files { if file.IsDirectory() { continue @@ -66,14 +69,16 @@ func (m *BlockMap) Update(files []protocol.FileInfo) error { if file.IsDeleted() || file.IsInvalid() { for _, block := range file.Blocks { - batch.Delete(m.blockKey(block.Hash, file.Name)) + key = m.blockKeyInto(key, block.Hash, file.Name) + batch.Delete(key) } continue } for i, block := range file.Blocks { binary.BigEndian.PutUint32(buf, uint32(i)) - batch.Put(m.blockKey(block.Hash, file.Name), buf) + key = m.blockKeyInto(key, block.Hash, file.Name) + batch.Put(key, buf) } } return m.db.Write(batch, nil) @@ -82,9 +87,11 @@ func (m *BlockMap) Update(files []protocol.FileInfo) error { // Discard block map state, removing the given files func (m *BlockMap) Discard(files []protocol.FileInfo) error { batch := new(leveldb.Batch) + var key []byte for _, file := range files { for _, block := range file.Blocks { - batch.Delete(m.blockKey(block.Hash, file.Name)) + key = m.blockKeyInto(key, block.Hash, file.Name) + batch.Delete(key) } } return m.db.Write(batch, nil) @@ -93,7 +100,7 @@ func (m *BlockMap) Discard(files []protocol.FileInfo) error { // Drop block map, removing all entries related to this block map from the db. func (m *BlockMap) Drop() error { batch := new(leveldb.Batch) - iter := m.db.NewIterator(util.BytesPrefix(m.blockKey(nil, "")[:1+64]), nil) + iter := m.db.NewIterator(util.BytesPrefix(m.blockKeyInto(nil, nil, "")[:1+64]), nil) defer iter.Release() for iter.Next() { batch.Delete(iter.Key()) @@ -104,8 +111,8 @@ func (m *BlockMap) Drop() error { return m.db.Write(batch, nil) } -func (m *BlockMap) blockKey(hash []byte, file string) []byte { - return toBlockKey(hash, m.folder, file) +func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte { + return blockKeyInto(o, hash, m.folder, file) } type BlockFinder struct { @@ -134,8 +141,9 @@ func (f *BlockFinder) String() string { // reason. The iterator finally returns the result, whether or not a // satisfying block was eventually found. func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool { + var key []byte for _, folder := range folders { - key := toBlockKey(hash, folder, "") + key = blockKeyInto(key, hash, folder, "") iter := f.db.NewIterator(util.BytesPrefix(key), nil) defer iter.Release() @@ -157,8 +165,8 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b binary.BigEndian.PutUint32(buf, uint32(index)) batch := new(leveldb.Batch) - batch.Delete(toBlockKey(oldHash, folder, file)) - batch.Put(toBlockKey(newHash, folder, file), buf) + batch.Delete(blockKeyInto(nil, oldHash, folder, file)) + batch.Put(blockKeyInto(nil, newHash, folder, file), buf) return f.db.Write(batch, nil) } @@ -167,8 +175,13 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b // folder (64 bytes) // block hash (32 bytes) // file name (variable size) -func toBlockKey(hash []byte, folder, file string) []byte { - o := make([]byte, 1+64+32+len(file)) +func blockKeyInto(o, hash []byte, folder, file string) []byte { + reqLen := 1 + 64 + 32 + len(file) + if cap(o) < reqLen { + o = make([]byte, reqLen) + } else { + o = o[:reqLen] + } o[0] = KeyTypeBlock copy(o[1:], []byte(folder)) copy(o[1+64:], []byte(hash)) From 918ef4dff88d89f90a93a848347e1a1eef914b02 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 21 Oct 2015 23:03:42 +0200 Subject: [PATCH 4/5] Use batches in blockmap, speeds up and reduces memory usage on large Replace and Update ops benchmark old ns/op new ns/op delta BenchmarkReplaceAll-8 2880834572 1868198122 -35.15% BenchmarkUpdateOneChanged-8 236596 231852 -2.01% BenchmarkUpdateOneUnchanged-8 227326 230624 +1.45% BenchmarkNeedHalf-8 105151538 104601744 -0.52% BenchmarkHave-8 28827492 29102480 +0.95% BenchmarkGlobal-8 150768724 150547687 -0.15% BenchmarkNeedHalfTruncated-8 104434216 102471355 -1.88% BenchmarkHaveTruncated-8 27860093 28758368 +3.22% BenchmarkGlobalTruncated-8 149972888 151192913 +0.81% benchmark old allocs new allocs delta BenchmarkReplaceAll-8 555451 555577 +0.02% BenchmarkUpdateOneChanged-8 1135 1135 +0.00% BenchmarkUpdateOneUnchanged-8 1135 1135 +0.00% BenchmarkNeedHalf-8 374779 374780 +0.00% BenchmarkHave-8 151996 151992 -0.00% BenchmarkGlobal-8 530066 530033 -0.01% BenchmarkNeedHalfTruncated-8 374702 374699 -0.00% BenchmarkHaveTruncated-8 151834 151834 +0.00% BenchmarkGlobalTruncated-8 530049 530037 -0.00% benchmark old bytes new bytes delta BenchmarkReplaceAll-8 5018351912 1765116216 -64.83% BenchmarkUpdateOneChanged-8 135085 135085 +0.00% BenchmarkUpdateOneUnchanged-8 134976 134976 +0.00% BenchmarkNeedHalf-8 44769400 44758752 -0.02% BenchmarkHave-8 11930612 11845052 -0.72% BenchmarkGlobal-8 81523668 80431136 -1.34% BenchmarkNeedHalfTruncated-8 46692342 46526459 -0.36% BenchmarkHaveTruncated-8 11348357 11348357 +0.00% BenchmarkGlobalTruncated-8 81843956 80977672 -1.06% --- lib/db/blockmap.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/db/blockmap.go b/lib/db/blockmap.go index 3e572e227..a4b8abc57 100644 --- a/lib/db/blockmap.go +++ b/lib/db/blockmap.go @@ -26,6 +26,8 @@ import ( var blockFinder *BlockFinder +const maxBatchSize = 256 << 10 + type BlockMap struct { db *leveldb.DB folder string @@ -44,6 +46,13 @@ func (m *BlockMap) Add(files []protocol.FileInfo) error { buf := make([]byte, 4) var key []byte for _, file := range files { + if batch.Len() > maxBatchSize { + if err := m.db.Write(batch, nil); err != nil { + return err + } + batch.Reset() + } + if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() { continue } @@ -63,6 +72,13 @@ func (m *BlockMap) Update(files []protocol.FileInfo) error { buf := make([]byte, 4) var key []byte for _, file := range files { + if batch.Len() > maxBatchSize { + if err := m.db.Write(batch, nil); err != nil { + return err + } + batch.Reset() + } + if file.IsDirectory() { continue } @@ -89,6 +105,13 @@ func (m *BlockMap) Discard(files []protocol.FileInfo) error { batch := new(leveldb.Batch) var key []byte for _, file := range files { + if batch.Len() > maxBatchSize { + if err := m.db.Write(batch, nil); err != nil { + return err + } + batch.Reset() + } + for _, block := range file.Blocks { key = m.blockKeyInto(key, block.Hash, file.Name) batch.Delete(key) @@ -103,6 +126,13 @@ func (m *BlockMap) Drop() error { iter := m.db.NewIterator(util.BytesPrefix(m.blockKeyInto(nil, nil, "")[:1+64]), nil) defer iter.Release() for iter.Next() { + if batch.Len() > maxBatchSize { + if err := m.db.Write(batch, nil); err != nil { + return err + } + batch.Reset() + } + batch.Delete(iter.Key()) } if iter.Error() != nil { From c1591a5efdcb363b4a7e4b2422c61c3703ceb6d2 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 21 Oct 2015 23:19:26 +0200 Subject: [PATCH 5/5] Only run benchmarks with -tags benchmark Avoids creating temp database and stuff on a normal test run --- lib/db/benchmark_test.go | 15 ++------------- lib/db/set_test.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go index 777b042ab..69b3155e3 100644 --- a/lib/db/benchmark_test.go +++ b/lib/db/benchmark_test.go @@ -4,6 +4,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. +// +build benchmark + package db_test import ( @@ -42,19 +44,6 @@ func init() { fs.Replace(protocol.LocalDeviceID, firstHalf) } -func genBlocks(n int) []protocol.BlockInfo { - b := make([]protocol.BlockInfo, n) - for i := range b { - h := make([]byte, 32) - for j := range h { - h[j] = byte(i + j) - } - b[i].Size = int32(i) - b[i].Hash = h - } - return b -} - func tempDB() (*leveldb.DB, string) { dir, err := ioutil.TempDir("", "syncthing") if err != nil { diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 45f5837a1..00e83159f 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -28,6 +28,19 @@ func init() { const myID = 1 +func genBlocks(n int) []protocol.BlockInfo { + b := make([]protocol.BlockInfo, n) + for i := range b { + h := make([]byte, 32) + for j := range h { + h[j] = byte(i + j) + } + b[i].Size = int32(i) + b[i].Hash = h + } + return b +} + func globalList(s *db.FileSet) []protocol.FileInfo { var fs []protocol.FileInfo s.WithGlobal(func(fi db.FileIntf) bool {