From 9d1e2d9f46dbeab84e4b385948b90b40161151c1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 7 Feb 2015 10:52:42 +0000 Subject: [PATCH 1/3] Add /rest/tree API call --- cmd/syncthing/gui.go | 19 ++ internal/db/leveldb.go | 6 +- internal/db/set.go | 11 +- internal/model/model.go | 63 ++++++ internal/model/model_test.go | 423 +++++++++++++++++++++++++++++++++++ 5 files changed, 516 insertions(+), 6 deletions(-) diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 4d4c703ac..bbdf5ded8 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -130,6 +130,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro getRestMux.HandleFunc("/rest/system", restGetSystem) getRestMux.HandleFunc("/rest/upgrade", restGetUpgrade) getRestMux.HandleFunc("/rest/version", restGetVersion) + getRestMux.HandleFunc("/rest/tree", withModel(m, restGetTree)) getRestMux.HandleFunc("/rest/stats/device", withModel(m, restGetDeviceStats)) getRestMux.HandleFunc("/rest/stats/folder", withModel(m, restGetFolderStats)) @@ -262,6 +263,24 @@ func restGetVersion(w http.ResponseWriter, r *http.Request) { }) } +func restGetTree(m *model.Model, w http.ResponseWriter, r *http.Request) { + qs := r.URL.Query() + folder := qs.Get("folder") + prefix := qs.Get("prefix") + dirsonly := qs.Get("dirsonly") != "" + + levels, err := strconv.Atoi(qs.Get("levels")) + if err != nil { + levels = -1 + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + tree := m.GlobalDirectoryTree(folder, prefix, levels, dirsonly) + + json.NewEncoder(w).Encode(tree) +} + func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) { var qs = r.URL.Query() var folder = qs.Get("folder") diff --git a/internal/db/leveldb.go b/internal/db/leveldb.go index 1b02840fc..b530a2c85 100644 --- a/internal/db/leveldb.go +++ b/internal/db/leveldb.go @@ -709,11 +709,9 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf, return fi, true } -func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) { +func ldbWithGlobal(db *leveldb.DB, folder, prefix []byte, truncate bool, fn Iterator) { runtime.GC() - start := globalKey(folder, nil) - limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff}) snap, err := db.GetSnapshot() if err != nil { panic(err) @@ -728,7 +726,7 @@ func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) { snap.Release() }() - dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil) + dbi := snap.NewIterator(util.BytesPrefix(globalKey(folder, prefix)), nil) defer dbi.Release() for dbi.Next() { diff --git a/internal/db/set.go b/internal/db/set.go index 794df4bd0..a43e6b582 100644 --- a/internal/db/set.go +++ b/internal/db/set.go @@ -172,14 +172,21 @@ func (s *FileSet) WithGlobal(fn Iterator) { if debug { l.Debugf("%s WithGlobal()", s.folder) } - ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn)) + ldbWithGlobal(s.db, []byte(s.folder), nil, false, nativeFileIterator(fn)) } func (s *FileSet) WithGlobalTruncated(fn Iterator) { if debug { l.Debugf("%s WithGlobalTruncated()", s.folder) } - ldbWithGlobal(s.db, []byte(s.folder), true, nativeFileIterator(fn)) + ldbWithGlobal(s.db, []byte(s.folder), nil, true, nativeFileIterator(fn)) +} + +func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) { + if debug { + l.Debugf("%s WithPrefixedGlobalTruncated()", s.folder, prefix) + } + ldbWithGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)) } func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) { diff --git a/internal/model/model.go b/internal/model/model.go index 83f289f0c..9721f8ccc 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -1417,6 +1417,69 @@ func (m *Model) RemoteLocalVersion(folder string) int64 { return ver } +func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} { + m.fmut.RLock() + files, ok := m.folderFiles[folder] + m.fmut.RUnlock() + if !ok { + return nil + } + + output := make(map[string]interface{}) + sep := string(filepath.Separator) + prefix = osutil.NativeFilename(prefix) + + if prefix != "" && !strings.HasSuffix(prefix, sep) { + prefix = prefix + sep + } + + files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { + f := fi.(db.FileInfoTruncated) + + if f.IsInvalid() || f.IsDeleted() || f.Name == prefix { + return true + } + + f.Name = strings.Replace(f.Name, prefix, "", 1) + + var dir, base string + if f.IsDirectory() && !f.IsSymlink() { + dir = f.Name + } else { + dir = filepath.Dir(f.Name) + base = filepath.Base(f.Name) + } + + if levels > -1 && strings.Count(f.Name, sep) > levels { + return true + } + + last := output + if dir != "." { + for _, path := range strings.Split(dir, sep) { + directory, ok := last[path] + if !ok { + newdir := make(map[string]interface{}) + last[path] = newdir + last = newdir + } else { + last = directory.(map[string]interface{}) + } + } + } + + if !dirsonly && base != "" { + last[base] = []int64{ + f.Modified, f.Size(), + } + } + + return true + }) + + return output +} + func (m *Model) availability(folder, file string) []protocol.DeviceID { // Acquire this lock first, as the value returned from foldersFiles can // get heavily modified on Close() diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 06298420e..68da7f47a 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -17,8 +17,11 @@ package model import ( "bytes" + "encoding/json" "fmt" "os" + "path/filepath" + "reflect" "testing" "time" @@ -580,3 +583,423 @@ func TestRefuseUnknownBits(t *testing.T) { t.Error("Valid file not found or name mismatch", ok, f) } } + +func TestGlobalDirectoryTree(t *testing.T) { + fcfg := config.FolderConfiguration{ + ID: "default", + Path: "testdata", + Devices: []config.FolderDeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + cfg := config.Configuration{ + Folders: []config.FolderConfiguration{fcfg}, + Devices: []config.DeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) + m.AddFolder(fcfg) + + b := func(isfile bool, path ...string) protocol.FileInfo { + var flags uint32 = protocol.FlagDirectory + blocks := []protocol.BlockInfo{} + if isfile { + flags = 0 + blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}} + } + return protocol.FileInfo{ + Name: filepath.Join(path...), + Flags: flags, + Modified: 0x666, + Blocks: blocks, + } + } + + filedata := []int64{0x666, 0xa} + + testdata := []protocol.FileInfo{ + b(false, "another"), + b(false, "another", "directory"), + b(true, "another", "directory", "afile"), + b(false, "another", "directory", "with"), + b(false, "another", "directory", "with", "a"), + b(true, "another", "directory", "with", "a", "file"), + b(true, "another", "directory", "with", "file"), + b(true, "another", "file"), + + b(false, "other"), + b(false, "other", "rand"), + b(false, "other", "random"), + b(false, "other", "random", "dir"), + b(false, "other", "random", "dirx"), + b(false, "other", "randomx"), + + b(false, "some"), + b(false, "some", "directory"), + b(false, "some", "directory", "with"), + b(false, "some", "directory", "with", "a"), + b(true, "some", "directory", "with", "a", "file"), + + b(true, "rootfile"), + } + expectedResult := map[string]interface{}{ + "another": map[string]interface{}{ + "directory": map[string]interface{}{ + "afile": filedata, + "with": map[string]interface{}{ + "a": map[string]interface{}{ + "file": filedata, + }, + "file": filedata, + }, + }, + "file": filedata, + }, + "other": map[string]interface{}{ + "rand": map[string]interface{}{}, + "random": map[string]interface{}{ + "dir": map[string]interface{}{}, + "dirx": map[string]interface{}{}, + }, + "randomx": map[string]interface{}{}, + }, + "some": map[string]interface{}{ + "directory": map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{ + "file": filedata, + }, + }, + }, + }, + "rootfile": filedata, + } + + mm := func(data interface{}) string { + bytes, err := json.Marshal(data) + if err != nil { + panic(err) + } + return string(bytes) + } + + m.Index(device1, "default", testdata) + + result := m.GlobalDirectoryTree("default", "", -1, false) + + if !reflect.DeepEqual(result, expectedResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult)) + } + + result = m.GlobalDirectoryTree("default", "another", -1, false) + + if !reflect.DeepEqual(result, expectedResult["another"]) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult["another"])) + } + + result = m.GlobalDirectoryTree("default", "", 0, false) + currentResult := map[string]interface{}{ + "another": map[string]interface{}{}, + "other": map[string]interface{}{}, + "some": map[string]interface{}{}, + "rootfile": filedata, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "", 1, false) + currentResult = map[string]interface{}{ + "another": map[string]interface{}{ + "directory": map[string]interface{}{}, + "file": filedata, + }, + "other": map[string]interface{}{ + "rand": map[string]interface{}{}, + "random": map[string]interface{}{}, + "randomx": map[string]interface{}{}, + }, + "some": map[string]interface{}{ + "directory": map[string]interface{}{}, + }, + "rootfile": filedata, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "", -1, true) + currentResult = map[string]interface{}{ + "another": map[string]interface{}{ + "directory": map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{}, + }, + }, + }, + "other": map[string]interface{}{ + "rand": map[string]interface{}{}, + "random": map[string]interface{}{ + "dir": map[string]interface{}{}, + "dirx": map[string]interface{}{}, + }, + "randomx": map[string]interface{}{}, + }, + "some": map[string]interface{}{ + "directory": map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{}, + }, + }, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "", 1, true) + currentResult = map[string]interface{}{ + "another": map[string]interface{}{ + "directory": map[string]interface{}{}, + }, + "other": map[string]interface{}{ + "rand": map[string]interface{}{}, + "random": map[string]interface{}{}, + "randomx": map[string]interface{}{}, + }, + "some": map[string]interface{}{ + "directory": map[string]interface{}{}, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "another", 0, false) + currentResult = map[string]interface{}{ + "directory": map[string]interface{}{}, + "file": filedata, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "some/directory", 0, false) + currentResult = map[string]interface{}{ + "with": map[string]interface{}{}, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "some/directory", 1, false) + currentResult = map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{}, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "some/directory", 2, false) + currentResult = map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{ + "file": filedata, + }, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "another", -1, true) + currentResult = map[string]interface{}{ + "directory": map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{}, + }, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + // No prefix matching! + result = m.GlobalDirectoryTree("default", "som", -1, false) + currentResult = map[string]interface{}{} + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } +} + +func TestGlobalDirectorySelfFixing(t *testing.T) { + fcfg := config.FolderConfiguration{ + ID: "default", + Path: "testdata", + Devices: []config.FolderDeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + cfg := config.Configuration{ + Folders: []config.FolderConfiguration{fcfg}, + Devices: []config.DeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) + m.AddFolder(fcfg) + + b := func(isfile bool, path ...string) protocol.FileInfo { + var flags uint32 = protocol.FlagDirectory + blocks := []protocol.BlockInfo{} + if isfile { + flags = 0 + blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}} + } + return protocol.FileInfo{ + Name: filepath.Join(path...), + Flags: flags, + Modified: 0x666, + Blocks: blocks, + } + } + + filedata := []int64{0x666, 0xa} + + testdata := []protocol.FileInfo{ + b(true, "another", "directory", "afile"), + b(true, "another", "directory", "with", "a", "file"), + b(true, "another", "directory", "with", "file"), + + b(false, "other", "random", "dirx"), + b(false, "other", "randomx"), + + b(false, "some", "directory", "with", "x"), + b(true, "some", "directory", "with", "a", "file"), + + b(false, "this", "is", "a", "deep", "invalid", "directory"), + + b(true, "xthis", "is", "a", "deep", "invalid", "file"), + } + expectedResult := map[string]interface{}{ + "another": map[string]interface{}{ + "directory": map[string]interface{}{ + "afile": filedata, + "with": map[string]interface{}{ + "a": map[string]interface{}{ + "file": filedata, + }, + "file": filedata, + }, + }, + }, + "other": map[string]interface{}{ + "random": map[string]interface{}{ + "dirx": map[string]interface{}{}, + }, + "randomx": map[string]interface{}{}, + }, + "some": map[string]interface{}{ + "directory": map[string]interface{}{ + "with": map[string]interface{}{ + "a": map[string]interface{}{ + "file": filedata, + }, + "x": map[string]interface{}{}, + }, + }, + }, + "this": map[string]interface{}{ + "is": map[string]interface{}{ + "a": map[string]interface{}{ + "deep": map[string]interface{}{ + "invalid": map[string]interface{}{ + "directory": map[string]interface{}{}, + }, + }, + }, + }, + }, + "xthis": map[string]interface{}{ + "is": map[string]interface{}{ + "a": map[string]interface{}{ + "deep": map[string]interface{}{ + "invalid": map[string]interface{}{ + "file": filedata, + }, + }, + }, + }, + }, + } + + mm := func(data interface{}) string { + bytes, err := json.Marshal(data) + if err != nil { + panic(err) + } + return string(bytes) + } + + m.Index(device1, "default", testdata) + + result := m.GlobalDirectoryTree("default", "", -1, false) + + if !reflect.DeepEqual(result, expectedResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(expectedResult)) + } + + result = m.GlobalDirectoryTree("default", "xthis/is/a/deep", -1, false) + currentResult := map[string]interface{}{ + "invalid": map[string]interface{}{ + "file": filedata, + }, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + result = m.GlobalDirectoryTree("default", "xthis/is/a/deep", -1, true) + currentResult = map[string]interface{}{ + "invalid": map[string]interface{}{}, + } + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } + + // !!! This is actually BAD, because we don't have enough level allowance + // to accept this file, hence the tree is left unbuilt !!! + result = m.GlobalDirectoryTree("default", "xthis", 1, false) + currentResult = map[string]interface{}{} + + if !reflect.DeepEqual(result, currentResult) { + t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) + } +} From fb649e95255bd58288c88e5ba45b33aac73ff0f4 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 4 Mar 2015 23:33:48 +0000 Subject: [PATCH 2/3] Fix benchmarks, cleanup tests --- internal/model/model_test.go | 124 ++++++++++++---------------------- internal/model/puller_test.go | 45 +++++------- 2 files changed, 62 insertions(+), 107 deletions(-) diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 68da7f47a..43e1a25f5 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -19,9 +19,11 @@ import ( "bytes" "encoding/json" "fmt" + "math/rand" "os" "path/filepath" "reflect" + "strconv" "testing" "time" @@ -32,10 +34,31 @@ import ( ) var device1, device2 protocol.DeviceID +var defaultConfig *config.Wrapper +var defaultFolderConfig config.FolderConfiguration func init() { device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") + + defaultFolderConfig = config.FolderConfiguration{ + ID: "default", + Path: "testdata", + Devices: []config.FolderDeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + _defaultConfig := config.Configuration{ + Folders: []config.FolderConfiguration{defaultFolderConfig}, + Devices: []config.DeviceConfiguration{ + { + DeviceID: device1, + }, + }, + } + defaultConfig = config.Wrap("/tmp/test", _defaultConfig) } var testDataExpected = map[string]protocol.FileInfo{ @@ -72,10 +95,10 @@ func init() { func TestRequest(t *testing.T) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) // device1 shares default, but device2 doesn't - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata", Devices: []config.FolderDeviceConfiguration{{DeviceID: device1}}}) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") // Existing, shared file @@ -158,8 +181,8 @@ func genFiles(n int) []protocol.FileInfo { func BenchmarkIndex10000(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) @@ -171,8 +194,8 @@ func BenchmarkIndex10000(b *testing.B) { func BenchmarkIndex00100(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(100) @@ -184,8 +207,8 @@ func BenchmarkIndex00100(b *testing.B) { func BenchmarkIndexUpdate10000f10000(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) m.Index(device1, "default", files) @@ -198,8 +221,8 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) { func BenchmarkIndexUpdate10000f00100(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) m.Index(device1, "default", files) @@ -213,8 +236,8 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) { func BenchmarkIndexUpdate10000f00001(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) m.Index(device1, "default", files) @@ -271,8 +294,8 @@ func (FakeConnection) Statistics() protocol.Statistics { func BenchmarkRequest(b *testing.B) { db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(nil, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") const n = 1000 @@ -454,12 +477,8 @@ func TestIgnores(t *testing.T) { } db, _ := leveldb.Open(storage.NewMemStorage(), nil) - fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"} - cfg := config.Wrap("/tmp", config.Configuration{ - Folders: []config.FolderConfiguration{fcfg}, - }) - m := NewModel(cfg, "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) expected := []string{ ".*", @@ -530,27 +549,9 @@ func TestIgnores(t *testing.T) { } func TestRefuseUnknownBits(t *testing.T) { - fcfg := config.FolderConfiguration{ - ID: "default", - Path: "testdata", - Devices: []config.FolderDeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } - cfg := config.Configuration{ - Folders: []config.FolderConfiguration{fcfg}, - Devices: []config.DeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } - db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) m.ScanFolder("default") m.Index(device1, "default", []protocol.FileInfo{ @@ -585,27 +586,9 @@ func TestRefuseUnknownBits(t *testing.T) { } func TestGlobalDirectoryTree(t *testing.T) { - fcfg := config.FolderConfiguration{ - ID: "default", - Path: "testdata", - Devices: []config.FolderDeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } - cfg := config.Configuration{ - Folders: []config.FolderConfiguration{fcfg}, - Devices: []config.DeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } - db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) b := func(isfile bool, path ...string) protocol.FileInfo { var flags uint32 = protocol.FlagDirectory @@ -852,27 +835,10 @@ func TestGlobalDirectoryTree(t *testing.T) { } func TestGlobalDirectorySelfFixing(t *testing.T) { - fcfg := config.FolderConfiguration{ - ID: "default", - Path: "testdata", - Devices: []config.FolderDeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } - cfg := config.Configuration{ - Folders: []config.FolderConfiguration{fcfg}, - Devices: []config.DeviceConfiguration{ - { - DeviceID: device1, - }, - }, - } db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) b := func(isfile bool, path ...string) protocol.FileInfo { var flags uint32 = protocol.FlagDirectory diff --git a/internal/model/puller_test.go b/internal/model/puller_test.go index d20283ee1..ca8bd0a61 100644 --- a/internal/model/puller_test.go +++ b/internal/model/puller_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/syncthing/protocol" - "github.com/syncthing/syncthing/internal/config" "github.com/syncthing/syncthing/internal/scanner" "github.com/syndtr/goleveldb/leveldb" @@ -77,8 +76,8 @@ func TestHandleFile(t *testing.T) { requiredFile.Blocks = blocks[1:] db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) // Update index m.updateLocal("default", existingFile) @@ -131,8 +130,8 @@ func TestHandleFileWithTemp(t *testing.T) { requiredFile.Blocks = blocks[1:] db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) // Update index m.updateLocal("default", existingFile) @@ -190,12 +189,9 @@ func TestCopierFinder(t *testing.T) { requiredFile.Blocks = blocks[1:] requiredFile.Name = "file2" - fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"} - cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}} - db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) // Update index m.updateLocal("default", existingFile) @@ -268,12 +264,9 @@ func TestCopierCleanup(t *testing.T) { return true } - fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"} - cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}} - db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) // Create a file file := protocol.FileInfo{ @@ -320,12 +313,9 @@ func TestCopierCleanup(t *testing.T) { // Make sure that the copier routine hashes the content when asked, and pulls // if it fails to find the block. func TestLastResortPulling(t *testing.T) { - fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"} - cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}} - db, _ := leveldb.Open(storage.NewMemStorage(), nil) - m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db) - m.AddFolder(fcfg) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) // Add a file to index (with the incorrect block representation, as content // doesn't actually match the block list) @@ -396,11 +386,11 @@ func TestDeregisterOnFailInCopy(t *testing.T) { defer os.Remove("testdata/" + defTempNamer.TempName("filex")) db, _ := leveldb.Open(storage.NewMemStorage(), nil) - cw := config.Wrap("/tmp/test", config.Configuration{}) - m := NewModel(cw, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) - emitter := NewProgressEmitter(cw) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) + + emitter := NewProgressEmitter(defaultConfig) go emitter.Serve() p := Puller{ @@ -484,11 +474,10 @@ func TestDeregisterOnFailInPull(t *testing.T) { defer os.Remove("testdata/" + defTempNamer.TempName("filex")) db, _ := leveldb.Open(storage.NewMemStorage(), nil) - cw := config.Wrap("/tmp/test", config.Configuration{}) - m := NewModel(cw, "device", "syncthing", "dev", db) - m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"}) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) - emitter := NewProgressEmitter(cw) + emitter := NewProgressEmitter(defaultConfig) go emitter.Serve() p := Puller{ From bf3e249237beb0cff620335785e8880a3fa4f0d4 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 4 Mar 2015 23:34:03 +0000 Subject: [PATCH 3/3] Add GlobalDirectoryTree benchmarks --- internal/model/model_test.go | 86 ++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 43e1a25f5..39b18da10 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -969,3 +969,89 @@ func TestGlobalDirectorySelfFixing(t *testing.T) { t.Errorf("Does not match:\n%s\n%s", mm(result), mm(currentResult)) } } + +func genDeepFiles(n, d int) []protocol.FileInfo { + rand.Seed(int64(n)) + files := make([]protocol.FileInfo, n) + t := time.Now().Unix() + for i := 0; i < n; i++ { + path := "" + for i := 0; i <= d; i++ { + path = filepath.Join(path, strconv.Itoa(rand.Int())) + } + + sofar := "" + for _, path := range filepath.SplitList(path) { + sofar = filepath.Join(sofar, path) + files[i] = protocol.FileInfo{ + Name: sofar, + } + i++ + } + + files[i].Modified = t + files[i].Blocks = []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}} + } + + return files +} + +func BenchmarkTree_10000_50(b *testing.B) { + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) + m.ScanFolder("default") + files := genDeepFiles(10000, 50) + + m.Index(device1, "default", files) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.GlobalDirectoryTree("default", "", -1, false) + } +} + +func BenchmarkTree_10000_10(b *testing.B) { + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) + m.ScanFolder("default") + files := genDeepFiles(10000, 10) + + m.Index(device1, "default", files) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.GlobalDirectoryTree("default", "", -1, false) + } +} + +func BenchmarkTree_00100_50(b *testing.B) { + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) + m.ScanFolder("default") + files := genDeepFiles(100, 50) + + m.Index(device1, "default", files) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.GlobalDirectoryTree("default", "", -1, false) + } +} + +func BenchmarkTree_00100_10(b *testing.B) { + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + m := NewModel(defaultConfig, "device", "syncthing", "dev", db) + m.AddFolder(defaultFolderConfig) + m.ScanFolder("default") + files := genDeepFiles(100, 10) + + m.Index(device1, "default", files) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.GlobalDirectoryTree("default", "", -1, false) + } +}