From a94951becdd7bc2de1cee34d603db3aa92e0fba7 Mon Sep 17 00:00:00 2001 From: Simon Frei Date: Mon, 11 May 2020 15:07:06 +0200 Subject: [PATCH] lib/db, lib/model: Keep need stats in metadata (ref #5899) (#6413) --- lib/db/benchmark_test.go | 83 +++++- lib/db/db_test.go | 72 +++++ lib/db/lowlevel.go | 19 ++ lib/db/meta.go | 66 ++++- lib/db/meta_test.go | 6 + lib/db/schemaupdater.go | 96 ++++++- lib/db/set.go | 19 +- lib/db/set_test.go | 56 ++++ lib/db/structs.go | 6 + lib/db/structs.pb.go | 130 +++++---- lib/db/structs.proto | 1 + lib/db/testdata/v1.4.0-updateTo10.json | 24 ++ lib/db/transactions.go | 370 +++++++++++++++++-------- lib/db/util_test.go | 34 +++ lib/model/devicedownloadstate.go | 33 +++ lib/model/folder_summary.go | 2 +- lib/model/model.go | 47 +--- lib/model/sentdownloadstate.go | 7 + lib/model/testutils_test.go | 2 +- lib/protocol/bep.pb.go | 255 +++++++++-------- lib/protocol/bep.proto | 1 + 21 files changed, 971 insertions(+), 358 deletions(-) create mode 100644 lib/db/testdata/v1.4.0-updateTo10.json diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go index 9d3482bd6..328df2bc8 100644 --- a/lib/db/benchmark_test.go +++ b/lib/db/benchmark_test.go @@ -16,7 +16,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -var files, oneFile, firstHalf, secondHalf []protocol.FileInfo +var files, filesUpdated, oneFile, firstHalf, secondHalf, changed100, unchanged100 []protocol.FileInfo func lazyInitBenchFiles() { if files != nil { @@ -36,6 +36,12 @@ func lazyInitBenchFiles() { firstHalf = files[:middle] secondHalf = files[middle:] oneFile = firstHalf[middle-1 : middle] + + unchanged100 := files[100:200] + changed100 := append([]protocol.FileInfo{}, unchanged100...) + for i := range changed100 { + changed100[i].Version = changed100[i].Version.Copy().Update(myID) + } } func getBenchFileSet() (*db.Lowlevel, *db.FileSet) { @@ -86,18 +92,43 @@ func BenchmarkUpdate100Changed(b *testing.B) { ldb, benchS := getBenchFileSet() defer ldb.Close() - unchanged := files[100:200] - changed := append([]protocol.FileInfo{}, unchanged...) - for i := range changed { - changed[i].Version = changed[i].Version.Copy().Update(myID) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i%2 == 0 { + benchS.Update(protocol.LocalDeviceID, changed100) + } else { + benchS.Update(protocol.LocalDeviceID, unchanged100) + } } + b.ReportAllocs() +} + +func setup10Remotes(benchS *db.FileSet) { + idBase := remoteDevice1.String()[1:] + first := 'J' + for i := 0; i < 10; i++ { + id, _ := protocol.DeviceIDFromString(fmt.Sprintf("%v%s", first+rune(i), idBase)) + if i%2 == 0 { + benchS.Update(id, changed100) + } else { + benchS.Update(id, unchanged100) + } + } +} + +func BenchmarkUpdate100Changed10Remotes(b *testing.B) { + ldb, benchS := getBenchFileSet() + defer ldb.Close() + + setup10Remotes(benchS) + b.ResetTimer() for i := 0; i < b.N; i++ { if i%2 == 0 { - benchS.Update(protocol.LocalDeviceID, changed) + benchS.Update(protocol.LocalDeviceID, changed100) } else { - benchS.Update(protocol.LocalDeviceID, unchanged) + benchS.Update(protocol.LocalDeviceID, unchanged100) } } @@ -108,18 +139,28 @@ func BenchmarkUpdate100ChangedRemote(b *testing.B) { ldb, benchS := getBenchFileSet() defer ldb.Close() - unchanged := files[100:200] - changed := append([]protocol.FileInfo{}, unchanged...) - for i := range changed { - changed[i].Version = changed[i].Version.Copy().Update(myID) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i%2 == 0 { + benchS.Update(remoteDevice0, changed100) + } else { + benchS.Update(remoteDevice0, unchanged100) + } } + b.ReportAllocs() +} + +func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) { + ldb, benchS := getBenchFileSet() + defer ldb.Close() + b.ResetTimer() for i := 0; i < b.N; i++ { if i%2 == 0 { - benchS.Update(remoteDevice0, changed) + benchS.Update(remoteDevice0, changed100) } else { - benchS.Update(remoteDevice0, unchanged) + benchS.Update(remoteDevice0, unchanged100) } } @@ -287,3 +328,19 @@ func BenchmarkGlobalTruncated(b *testing.B) { b.ReportAllocs() } + +func BenchmarkNeedCount(b *testing.B) { + ldb, benchS := getBenchFileSet() + defer ldb.Close() + + benchS.Update(protocol.LocalDeviceID, changed100) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + snap := benchS.Snapshot() + _ = snap.NeedSize(protocol.LocalDeviceID) + snap.Release() + } + + b.ReportAllocs() +} diff --git a/lib/db/db_test.go b/lib/db/db_test.go index 8efb052fc..8b7b8df83 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -481,3 +481,75 @@ func TestCheckGlobals(t *testing.T) { t.Error("Expected key missing error, got", err) } } + +func TestUpdateTo10(t *testing.T) { + ldb, err := openJSONS("./testdata/v1.4.0-updateTo10.json") + if err != nil { + t.Fatal(err) + } + db := NewLowlevel(ldb) + defer db.Close() + + UpdateSchema(db) + + folder := "test" + + meta := db.getMetaAndCheck(folder) + + empty := Counts{} + + c := meta.Counts(protocol.LocalDeviceID, needFlag) + if c.Files != 1 { + t.Error("Expected 1 needed file locally, got", c.Files) + } + c.Files = 0 + if c.Deleted != 1 { + t.Error("Expected 1 needed deletion locally, got", c.Deleted) + } + c.Deleted = 0 + if !c.Equal(empty) { + t.Error("Expected all counts to be zero, got", c) + } + c = meta.Counts(remoteDevice0, needFlag) + if !c.Equal(empty) { + t.Error("Expected all counts to be zero, got", c) + } + + trans, err := db.newReadOnlyTransaction() + if err != nil { + t.Fatal(err) + } + defer trans.Release() + // a + vl, err := trans.getGlobalVersions(nil, []byte(folder), []byte("a")) + if err != nil { + t.Fatal(err) + } + for _, v := range vl.Versions { + if !v.Deleted { + t.Error("Unexpected undeleted global version for a") + } + } + // b + vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("b")) + if err != nil { + t.Fatal(err) + } + if !vl.Versions[0].Deleted { + t.Error("vl.Versions[0] not deleted for b") + } + if vl.Versions[1].Deleted { + t.Error("vl.Versions[1] deleted for b") + } + // c + vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("c")) + if err != nil { + t.Fatal(err) + } + if vl.Versions[0].Deleted { + t.Error("vl.Versions[0] deleted for c") + } + if !vl.Versions[1].Deleted { + t.Error("vl.Versions[1] not deleted for c") + } +} diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index 12a4c38d1..d7fc8d4c8 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -748,6 +748,25 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) { return nil, err } + meta.emptyNeeded(protocol.LocalDeviceID) + err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f FileIntf) bool { + meta.addNeeded(protocol.LocalDeviceID, f) + return true + }) + if err != nil { + return nil, err + } + for _, device := range meta.devices() { + meta.emptyNeeded(device) + err = t.withNeed([]byte(folder), device[:], true, func(f FileIntf) bool { + meta.addNeeded(device, f) + return true + }) + if err != nil { + return nil, err + } + } + meta.SetCreated() if err := meta.toDB(t, []byte(folder)); err != nil { return nil, err diff --git a/lib/db/meta.go b/lib/db/meta.go index 0225d8925..b5632f9b2 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -32,6 +32,8 @@ type metaKey struct { flag uint32 } +const needFlag uint32 = 1 << 31 // Last bit, as early ones are local flags + func newMetadataTracker() *metadataTracker { return &metadataTracker{ mut: sync.NewRWMutex(), @@ -116,10 +118,26 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts idx = len(m.counts.Counts) m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flag}) m.indexes[key] = idx + if flag == needFlag { + // Initially a new device needs everything, except deletes + m.counts.Counts[idx] = m.allNeededCounts(dev) + } } return &m.counts.Counts[idx] } +// allNeeded makes sure there is a counts in case the device needs everything. +func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts { + counts := Counts{} + if idx, ok := m.indexes[metaKey{protocol.GlobalDeviceID, 0}]; ok { + counts = m.counts.Counts[idx] + counts.Deleted = 0 // Don't need deletes if having nothing + } + counts.DeviceID = dev[:] + counts.LocalFlags = needFlag + return counts +} + // addFile adds a file to the counts, adjusting the sequence number as // appropriate func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { @@ -146,6 +164,30 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { } } +// emptyNeeded makes sure there is a zero counts in case the device needs nothing. +func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) { + m.mut.Lock() + defer m.mut.Unlock() + + m.dirty = true + + m.indexes[metaKey{dev, needFlag}] = len(m.counts.Counts) + m.counts.Counts = append(m.counts.Counts, Counts{ + DeviceID: dev[:], + LocalFlags: needFlag, + }) +} + +// addNeeded adds a file to the needed counts +func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f FileIntf) { + m.mut.Lock() + defer m.mut.Unlock() + + m.dirty = true + + m.addFileLocked(dev, needFlag, f) +} + func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 { m.mut.Lock() defer m.mut.Unlock() @@ -200,6 +242,16 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { } } +// removeNeeded removes a file from the needed counts +func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) { + m.mut.Lock() + defer m.mut.Unlock() + + m.dirty = true + + m.removeFileLocked(dev, needFlag, f) +} + func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) { cp := m.countsPtr(dev, flag) @@ -277,21 +329,17 @@ func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts { idx, ok := m.indexes[metaKey{dev, flag}] if !ok { + if flag == needFlag { + // If there's nothing about a device in the index yet, + // it needs everything. + return m.allNeededCounts(dev) + } return Counts{} } return m.counts.Counts[idx] } -// Counts returns the counts for the given device ID and flag. `flag` should -// be zero or have exactly one bit set. -func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { - m.mut.RLock() - defer m.mut.RUnlock() - - return m.countsMap.Counts(dev, flag) -} - // Snapshot returns a copy of the metadata for reading. func (m *metadataTracker) Snapshot() *countsMap { m.mut.RLock() diff --git a/lib/db/meta_test.go b/lib/db/meta_test.go index fca86fd01..4e8d83b1c 100644 --- a/lib/db/meta_test.go +++ b/lib/db/meta_test.go @@ -175,3 +175,9 @@ func TestRecalcMeta(t *testing.T) { t.Fatalf("Wrong fixed global byte count, %d != 3000", gs.Bytes) } } + +func TestMetaKeyCollisions(t *testing.T) { + if protocol.LocalAllFlags&needFlag != 0 { + t.Error("Collision between need flag and protocol local file flags") + } +} diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 2cd5084db..e68e733c2 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -22,9 +22,10 @@ import ( // 6: v0.14.50 // 7: v0.14.53 // 8-9: v1.4.0 +// 10: v1.6.0 const ( - dbVersion = 9 - dbMinSyncthingVersion = "v1.4.0" + dbVersion = 10 + dbMinSyncthingVersion = "v1.6.0" ) type databaseDowngradeError struct { @@ -85,6 +86,7 @@ func (db *schemaUpdater) updateSchema() error { {6, db.updateSchema5to6}, {7, db.updateSchema6to7}, {9, db.updateSchemato9}, + {10, db.updateSchemato10}, } for _, m := range migrations { @@ -154,8 +156,10 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { if err != nil { return err } + // Purposely pass nil file name to remove from global list, + // but don't touch meta and needs buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil) - if err != nil { + if err != nil && err != errEntryFromGlobalMissing { return err } if err := t.Delete(dbi.Key()); err != nil { @@ -278,7 +282,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error { if ok { v = haveFile.FileVersion() } - if !need(f, ok, v) { + fv := FileVersion{ + Version: f.FileVersion(), + Invalid: f.IsInvalid(), + Deleted: f.IsDeleted(), + } + if !need(fv, ok, v) { return true } nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName())) @@ -390,7 +399,6 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { var delErr error err := t.withNeedLocal(folder, false, func(f FileIntf) bool { name := []byte(f.FileName()) - global := f.(protocol.FileInfo) gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name) if delErr != nil { return false @@ -413,7 +421,13 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { // so lets not act on it. return true } - if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(global, haveLocalFV, localFV.Version) { + globalFV := FileVersion{ + Version: f.FileVersion(), + Invalid: f.IsInvalid(), + Deleted: f.IsDeleted(), + } + + if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) { key, err := t.keyer.GenerateNeedFileKey(nk, folder, name) if err != nil { delErr = err @@ -484,3 +498,73 @@ func (db *schemaUpdater) updateSchemato9(prev int) error { return t.Commit() } + +func (db *schemaUpdater) updateSchemato10(_ int) error { + t, err := db.newReadWriteTransaction() + if err != nil { + return err + } + defer t.close() + + var buf []byte + + for _, folderStr := range db.ListFolders() { + folder := []byte(folderStr) + + buf, err = t.keyer.GenerateGlobalVersionKey(buf, folder, nil) + if err != nil { + return err + } + buf = globalVersionKey(buf).WithoutName() + dbi, err := t.NewPrefixIterator(buf) + if err != nil { + return err + } + defer dbi.Release() + + for dbi.Next() { + var vl VersionList + if err := vl.Unmarshal(dbi.Value()); err != nil { + return err + } + + changed := false + name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) + + for i, fv := range vl.Versions { + buf, err = t.keyer.GenerateDeviceFileKey(buf, folder, fv.Device, name) + if err != nil { + return err + } + f, ok, err := t.getFileTrunc(buf, true) + if !ok { + return errEntryFromGlobalMissing + } + if err != nil { + return err + } + if f.IsDeleted() { + vl.Versions[i].Deleted = true + changed = true + } + } + + if changed { + if err := t.Put(dbi.Key(), mustMarshal(&vl)); err != nil { + return err + } + if err := t.Checkpoint(); err != nil { + return err + } + } + } + dbi.Release() + } + + // Trigger metadata recalc + if err := t.deleteKeyPrefix([]byte{KeyTypeFolderMeta}); err != nil { + return err + } + + return t.Commit() +} diff --git a/lib/db/set.go b/lib/db/set.go index 2728981bf..69276b0e6 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -314,23 +314,8 @@ func (s *Snapshot) GlobalSize() Counts { return global.Add(recvOnlyChanged) } -func (s *Snapshot) NeedSize() Counts { - var result Counts - s.WithNeedTruncated(protocol.LocalDeviceID, func(f FileIntf) bool { - switch { - case f.IsDeleted(): - result.Deleted++ - case f.IsDirectory(): - result.Directories++ - case f.IsSymlink(): - result.Symlinks++ - default: - result.Files++ - result.Bytes += f.FileSize() - } - return true - }) - return result +func (s *Snapshot) NeedSize(device protocol.DeviceID) Counts { + return s.meta.Counts(device, needFlag) } // LocalChangedFiles returns a paginated list of files that were changed locally. diff --git a/lib/db/set_test.go b/lib/db/set_test.go index e9563dadc..ae7f22d00 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -296,6 +296,8 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Need incorrect (local);\n A: %v !=\n E: %v", n, expectedLocalNeed) } + checkNeed(t, m, protocol.LocalDeviceID, expectedLocalNeed) + n = fileList(needList(m, remoteDevice0)) sort.Sort(n) @@ -303,6 +305,8 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Need incorrect (remote);\n A: %v !=\n E: %v", n, expectedRemoteNeed) } + checkNeed(t, m, remoteDevice0, expectedRemoteNeed) + snap := m.Snapshot() defer snap.Release() f, ok := snap.Get(protocol.LocalDeviceID, "b") @@ -427,6 +431,8 @@ func TestGlobalSet(t *testing.T) { if fmt.Sprint(n) != fmt.Sprint(expectedSecRemoteNeed) { t.Errorf("Need incorrect (secRemote);\n A: %v !=\n E: %v", n, expectedSecRemoteNeed) } + + checkNeed(t, m, remoteDevice1, expectedSecRemoteNeed) } func TestNeedWithInvalid(t *testing.T) { @@ -465,6 +471,8 @@ func TestNeedWithInvalid(t *testing.T) { if fmt.Sprint(need) != fmt.Sprint(expectedNeed) { t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed) } + + checkNeed(t, s, protocol.LocalDeviceID, expectedNeed) } func TestUpdateToInvalid(t *testing.T) { @@ -642,6 +650,8 @@ func TestNeed(t *testing.T) { if fmt.Sprint(need) != fmt.Sprint(shouldNeed) { t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed) } + + checkNeed(t, m, protocol.LocalDeviceID, shouldNeed) } func TestSequence(t *testing.T) { @@ -761,6 +771,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) { if fmt.Sprint(need) != fmt.Sprint(total) { t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total) } + checkNeed(t, s, protocol.LocalDeviceID, total) global := fileList(globalList(s)) if fmt.Sprint(global) != fmt.Sprint(total) { @@ -1095,10 +1106,12 @@ func TestMoveGlobalBack(t *testing.T) { } else if !need[0].IsEquivalent(remote0Have[0], 0) { t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0]) } + checkNeed(t, s, protocol.LocalDeviceID, remote0Have[:1]) if need := needList(s, remoteDevice0); len(need) != 0 { t.Error("Expected no need for remote 0, got", need) } + checkNeed(t, s, remoteDevice0, nil) ls := localSize(s) if haveBytes := localHave[0].Size; ls.Bytes != haveBytes { @@ -1121,10 +1134,12 @@ func TestMoveGlobalBack(t *testing.T) { } else if !need[0].IsEquivalent(localHave[0], 0) { t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0]) } + checkNeed(t, s, remoteDevice0, localHave[:1]) if need := needList(s, protocol.LocalDeviceID); len(need) != 0 { t.Error("Expected no local need, got", need) } + checkNeed(t, s, protocol.LocalDeviceID, nil) ls = localSize(s) if haveBytes := localHave[0].Size; ls.Bytes != haveBytes { @@ -1158,6 +1173,7 @@ func TestIssue5007(t *testing.T) { } else if !need[0].IsEquivalent(fs[0], 0) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } + checkNeed(t, s, protocol.LocalDeviceID, fs[:1]) fs[0].LocalFlags = protocol.FlagLocalIgnored s.Update(protocol.LocalDeviceID, fs) @@ -1165,6 +1181,7 @@ func TestIssue5007(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 0 { t.Fatal("Expected no local need, got", need) } + checkNeed(t, s, protocol.LocalDeviceID, nil) } // TestNeedDeleted checks that a file that doesn't exist locally isn't needed @@ -1184,6 +1201,7 @@ func TestNeedDeleted(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 0 { t.Fatal("Expected no local need, got", need) } + checkNeed(t, s, protocol.LocalDeviceID, nil) fs[0].Deleted = false fs[0].Version = fs[0].Version.Update(remoteDevice0.Short()) @@ -1194,6 +1212,7 @@ func TestNeedDeleted(t *testing.T) { } else if !need[0].IsEquivalent(fs[0], 0) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } + checkNeed(t, s, protocol.LocalDeviceID, fs[:1]) fs[0].Deleted = true fs[0].Version = fs[0].Version.Update(remoteDevice0.Short()) @@ -1202,6 +1221,7 @@ func TestNeedDeleted(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 0 { t.Fatal("Expected no local need, got", need) } + checkNeed(t, s, protocol.LocalDeviceID, nil) } func TestReceiveOnlyAccounting(t *testing.T) { @@ -1338,6 +1358,7 @@ func TestNeedAfterUnignore(t *testing.T) { } else if !need[0].IsEquivalent(remote, 0) { t.Fatalf("Got %v, expected %v", need[0], remote) } + checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{remote}) } func TestRemoteInvalidNotAccounted(t *testing.T) { @@ -1384,6 +1405,7 @@ func TestNeedWithNewerInvalid(t *testing.T) { if !need[0].IsEquivalent(file, 0) { t.Fatalf("Got needed file %v, expected %v", need[0], file) } + checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file}) // rem1 sends an invalid file with increased version inv := file @@ -1399,6 +1421,7 @@ func TestNeedWithNewerInvalid(t *testing.T) { if !need[0].IsEquivalent(file, 0) { t.Fatalf("Got needed file %v, expected %v", need[0], file) } + checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file}) } func TestNeedAfterDeviceRemove(t *testing.T) { @@ -1425,6 +1448,7 @@ func TestNeedAfterDeviceRemove(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 0 { t.Fatal("Expected no local need, got", need) } + checkNeed(t, s, protocol.LocalDeviceID, nil) } func TestCaseSensitive(t *testing.T) { @@ -1608,8 +1632,40 @@ func globalSize(fs *db.FileSet) db.Counts { return snap.GlobalSize() } +func needSize(fs *db.FileSet, id protocol.DeviceID) db.Counts { + snap := fs.Snapshot() + defer snap.Release() + return snap.NeedSize(id) +} + func receiveOnlyChangedSize(fs *db.FileSet) db.Counts { snap := fs.Snapshot() defer snap.Release() return snap.ReceiveOnlyChangedSize() } + +func filesToCounts(files []protocol.FileInfo) db.Counts { + cp := db.Counts{} + for _, f := range files { + switch { + case f.IsDeleted(): + cp.Deleted++ + case f.IsDirectory() && !f.IsSymlink(): + cp.Directories++ + case f.IsSymlink(): + cp.Symlinks++ + default: + cp.Files++ + } + cp.Bytes += f.FileSize() + } + return cp +} + +func checkNeed(t testing.TB, s *db.FileSet, dev protocol.DeviceID, expected []protocol.FileInfo) { + t.Helper() + counts := needSize(s, dev) + if exp := filesToCounts(expected); !exp.Equal(counts) { + t.Errorf("Count incorrect (%v): expected %v, got %v", dev, exp, counts) + } +} diff --git a/lib/db/structs.go b/lib/db/structs.go index 0fad0cbe3..8c8c60c18 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -187,6 +187,11 @@ func (c Counts) TotalItems() int32 { return c.Files + c.Directories + c.Symlinks + c.Deleted } +// Equal compares the numbers only, not sequence/dev/flags. +func (c Counts) Equal(o Counts) bool { + return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes +} + func (vl VersionList) String() string { var b bytes.Buffer var id protocol.DeviceID @@ -212,6 +217,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t re Device: device, Version: file.Version, Invalid: file.IsInvalid(), + Deleted: file.IsDeleted(), } i := 0 if nv.Invalid { diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index 312856cff..7b829d188 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -29,6 +29,7 @@ type FileVersion struct { Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` Invalid bool `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"` + Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` } func (m *FileVersion) Reset() { *m = FileVersion{} } @@ -327,54 +328,54 @@ func init() { func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) } var fileDescriptor_e774e8f5f348d14d = []byte{ - // 743 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44, - 0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08, - 0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5, - 0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01, - 0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6, - 0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17, - 0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96, - 0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79, - 0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84, - 0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb, - 0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70, - 0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20, - 0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8, - 0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d, - 0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f, - 0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a, - 0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e, - 0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f, - 0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7, - 0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12, - 0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35, - 0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e, - 0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26, - 0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28, - 0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40, - 0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69, - 0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6, - 0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33, - 0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3, - 0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66, - 0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5, - 0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2, - 0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54, - 0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e, - 0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29, - 0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb, - 0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84, - 0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4, - 0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5, - 0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13, - 0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19, - 0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23, - 0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4, - 0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c, - 0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74, - 0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8, - 0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00, + // 747 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xbf, 0x8f, 0xe2, 0x46, + 0x14, 0xc6, 0x07, 0x66, 0xe1, 0x19, 0xc8, 0xdd, 0xe4, 0xb4, 0xb2, 0x90, 0x62, 0x2c, 0xa2, 0x93, + 0xac, 0x14, 0x90, 0xdb, 0xeb, 0x12, 0x29, 0x85, 0x73, 0x5a, 0x05, 0x29, 0xca, 0x45, 0xc3, 0xe9, + 0xaa, 0x48, 0xc8, 0x3f, 0x06, 0x18, 0xad, 0xf1, 0x10, 0xcf, 0xb0, 0x2b, 0x6f, 0x97, 0x3a, 0x4d, + 0xca, 0x94, 0xfb, 0xe7, 0x6c, 0xb9, 0x65, 0x94, 0x02, 0x25, 0x90, 0x22, 0x7f, 0x46, 0x34, 0x33, + 0xb6, 0xf1, 0x6e, 0x93, 0xeb, 0xde, 0xf7, 0xbd, 0x07, 0xef, 0xc7, 0xf7, 0x79, 0xa0, 0xcf, 0x45, + 0xb6, 0x8b, 0x04, 0x9f, 0x6c, 0x33, 0x26, 0x18, 0x7a, 0x16, 0x87, 0xc3, 0xcf, 0x33, 0xb2, 0x65, + 0x7c, 0xaa, 0x88, 0x70, 0xb7, 0x9c, 0xae, 0xd8, 0x8a, 0x29, 0xa0, 0x22, 0x5d, 0x38, 0x3c, 0x4f, + 0x68, 0xa8, 0x4b, 0x22, 0x96, 0x4c, 0x43, 0xb2, 0xd5, 0xfc, 0xf8, 0x57, 0x03, 0xac, 0x4b, 0x9a, + 0x90, 0x0f, 0x24, 0xe3, 0x94, 0xa5, 0xe8, 0x4b, 0x38, 0xbb, 0xd6, 0xa1, 0x6d, 0xb8, 0x86, 0x67, + 0x5d, 0x3c, 0x9f, 0x94, 0xbf, 0x9a, 0x7c, 0x20, 0x91, 0x60, 0x99, 0xdf, 0xba, 0xdf, 0x8f, 0x1a, + 0xb8, 0x2c, 0x43, 0xe7, 0xd0, 0x8e, 0xc9, 0x35, 0x8d, 0x88, 0xfd, 0xcc, 0x35, 0xbc, 0x1e, 0x2e, + 0x10, 0xb2, 0xe1, 0x8c, 0xa6, 0xd7, 0x41, 0x42, 0x63, 0xbb, 0xe9, 0x1a, 0x5e, 0x07, 0x97, 0x50, + 0x66, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xa5, 0x33, 0x05, 0x1c, 0x5f, 0x82, 0x55, 0x0c, 0xf2, + 0x3d, 0xe5, 0x02, 0xbd, 0x86, 0x4e, 0xd1, 0x85, 0xdb, 0x86, 0xdb, 0xf4, 0xac, 0x8b, 0x4f, 0x26, + 0x71, 0x38, 0xa9, 0xcd, 0x5b, 0x0c, 0x53, 0x95, 0x7d, 0xd5, 0xfa, 0xfd, 0x6e, 0xd4, 0x18, 0xff, + 0x62, 0xc2, 0x0b, 0x59, 0x35, 0x4b, 0x97, 0xec, 0x7d, 0xb6, 0x4b, 0xa3, 0x40, 0x90, 0x18, 0x21, + 0x68, 0xa5, 0xc1, 0x86, 0xa8, 0xc5, 0xba, 0x58, 0xc5, 0x92, 0xe3, 0xf4, 0x96, 0xa8, 0x11, 0x9b, + 0x58, 0xc5, 0xe8, 0x33, 0x80, 0x0d, 0x8b, 0xe9, 0x92, 0x92, 0x78, 0xc1, 0x6d, 0x53, 0x65, 0xba, + 0x25, 0x33, 0x47, 0x3f, 0x81, 0x55, 0xa5, 0xc3, 0xdc, 0xee, 0xb9, 0x86, 0xd7, 0xf2, 0xbf, 0x96, + 0x73, 0xfc, 0xb9, 0x1f, 0xbd, 0x59, 0x51, 0xb1, 0xde, 0x85, 0x93, 0x88, 0x6d, 0xa6, 0x3c, 0x4f, + 0x23, 0xb1, 0xa6, 0xe9, 0xaa, 0x16, 0xd5, 0x65, 0x98, 0xcc, 0xd7, 0x2c, 0x13, 0xb3, 0xb7, 0xb8, + 0x6a, 0xe7, 0xe7, 0x75, 0x01, 0xba, 0x1f, 0x27, 0xc0, 0x10, 0x3a, 0x9c, 0xfc, 0xbc, 0x23, 0x69, + 0x44, 0x6c, 0x50, 0xc3, 0x56, 0x18, 0xbd, 0x82, 0x01, 0xcf, 0x37, 0x09, 0x4d, 0xaf, 0x16, 0x22, + 0xc8, 0x56, 0x44, 0xd8, 0x2f, 0xd4, 0xf2, 0xfd, 0x82, 0x7d, 0xaf, 0x48, 0x34, 0x02, 0x2b, 0x4c, + 0x58, 0x74, 0xc5, 0x17, 0xeb, 0x80, 0xaf, 0x6d, 0xa4, 0x84, 0x04, 0x4d, 0x7d, 0x17, 0xf0, 0x35, + 0xfa, 0x02, 0x5a, 0x22, 0xdf, 0x6a, 0x89, 0x07, 0x17, 0xe7, 0xa7, 0x91, 0xaa, 0x2b, 0xe7, 0x5b, + 0x82, 0x55, 0x0d, 0x72, 0xc1, 0xda, 0x92, 0x6c, 0x43, 0xb9, 0x16, 0x4e, 0x4a, 0xdc, 0xc7, 0x75, + 0x4a, 0xb6, 0xab, 0x2e, 0x98, 0x72, 0xdb, 0x72, 0x0d, 0xcf, 0x3c, 0x1d, 0xe1, 0x07, 0x8e, 0xa6, + 0xa0, 0x9b, 0x2f, 0x94, 0x36, 0x7d, 0x99, 0xf7, 0x9f, 0x1f, 0xf6, 0xa3, 0x1e, 0x0e, 0x6e, 0x7c, + 0x99, 0x98, 0xd3, 0x5b, 0x82, 0xbb, 0x61, 0x19, 0xca, 0x9e, 0x09, 0x8b, 0x82, 0x64, 0xb1, 0x4c, + 0x82, 0x15, 0xb7, 0xff, 0x3d, 0x53, 0x4d, 0x41, 0x71, 0x97, 0x92, 0xaa, 0x9b, 0xae, 0xfd, 0xc8, + 0x74, 0xc8, 0x3b, 0x19, 0x55, 0xfe, 0xac, 0xe3, 0x0f, 0x0e, 0xfb, 0x11, 0xe0, 0xe0, 0x66, 0xa6, + 0xd9, 0x93, 0x71, 0x5f, 0xc1, 0x20, 0x65, 0x8b, 0xfa, 0x72, 0x1d, 0xf5, 0x57, 0xfd, 0x94, 0xfd, + 0x78, 0x22, 0x0b, 0x0f, 0x7e, 0x03, 0x5d, 0x35, 0x6a, 0xe1, 0xe4, 0xb6, 0x02, 0xa5, 0x8f, 0x3f, + 0x3d, 0x5d, 0x50, 0xf1, 0xf2, 0x84, 0x85, 0xae, 0x45, 0xe1, 0xf8, 0x35, 0x0c, 0xfc, 0x4a, 0x80, + 0x77, 0x69, 0x92, 0xff, 0xaf, 0x4a, 0xe3, 0x7f, 0x0c, 0x68, 0x7f, 0xcb, 0x76, 0xa9, 0xe0, 0xe8, + 0x25, 0x98, 0x4b, 0x9a, 0x10, 0xae, 0xcc, 0x6e, 0x62, 0x0d, 0xe4, 0x99, 0x62, 0x9a, 0x29, 0x17, + 0x51, 0xc2, 0x95, 0x9a, 0x26, 0xae, 0x53, 0xca, 0x4c, 0xda, 0x1a, 0x5c, 0x7d, 0x13, 0x26, 0xae, + 0xf0, 0xd3, 0xef, 0xd6, 0x3c, 0x9d, 0xf0, 0x25, 0x98, 0x61, 0x2e, 0x48, 0xf9, 0xb1, 0x68, 0xf0, + 0xc8, 0x98, 0xed, 0x27, 0xc6, 0x1c, 0x42, 0x47, 0xbf, 0x13, 0xb3, 0xb7, 0xca, 0x92, 0x3d, 0x5c, + 0x61, 0xe4, 0x40, 0x4d, 0x38, 0xb5, 0xe6, 0x23, 0x29, 0xc7, 0xef, 0xa0, 0xab, 0xb7, 0x9c, 0x13, + 0x81, 0x3c, 0x68, 0x47, 0x0a, 0x14, 0x97, 0x05, 0xf9, 0x42, 0xe8, 0x74, 0x79, 0x50, 0x9d, 0x97, + 0xe3, 0x47, 0x19, 0x91, 0x2f, 0x81, 0x5a, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0xf7, 0x7f, 0x3b, 0x8d, + 0xfb, 0x83, 0x63, 0x3c, 0x1c, 0x1c, 0xe3, 0xaf, 0x83, 0xd3, 0xf8, 0xed, 0xe8, 0x34, 0xee, 0x8e, + 0x8e, 0xf1, 0x70, 0x74, 0x1a, 0x7f, 0x1c, 0x9d, 0x46, 0xd8, 0x56, 0x72, 0xbd, 0xf9, 0x2f, 0x00, + 0x00, 0xff, 0xff, 0x8a, 0x67, 0x08, 0x85, 0x7f, 0x05, 0x00, 0x00, } func (m *FileVersion) Marshal() (dAtA []byte, err error) { @@ -397,6 +398,16 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Deleted { + i-- + if m.Deleted { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if m.Invalid { i-- if m.Invalid { @@ -805,6 +816,9 @@ func (m *FileVersion) ProtoSize() (n int) { if m.Invalid { n += 2 } + if m.Deleted { + n += 2 + } return n } @@ -1084,6 +1098,26 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error { } } m.Invalid = bool(v != 0) + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Deleted = bool(v != 0) default: iNdEx = preIndex skippy, err := skipStructs(dAtA[iNdEx:]) diff --git a/lib/db/structs.proto b/lib/db/structs.proto index 1acbaaee3..0f99e70e8 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -16,6 +16,7 @@ message FileVersion { protocol.Vector version = 1 [(gogoproto.nullable) = false]; bytes device = 2; bool invalid = 3; + bool deleted = 4; } message VersionList { diff --git a/lib/db/testdata/v1.4.0-updateTo10.json b/lib/db/testdata/v1.4.0-updateTo10.json new file mode 100644 index 000000000..68bd96c86 --- /dev/null +++ b/lib/db/testdata/v1.4.0-updateTo10.json @@ -0,0 +1,24 @@ +{"k":"AAAAAAAAAAABYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="} +{"k":"AAAAAAAAAAABYg==","v":"CgFiSgcKBQgBEOgHUAKCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gkgEgC8XkepY1E4woWwAAyi81YItXr5CMuwY6mfvf2iLupTo="} +{"k":"AAAAAAAAAAABYw==","v":"CgFjMAFKBwoFCAEQ6AdQAw=="} +{"k":"AAAAAAAAAAACYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="} +{"k":"AAAAAAAAAAACYg==","v":"CgFiMAFKFQoFCAEQ6AcKDAi5vtz687f5kQIQAVACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIJIBIAvF5HqWNROMKFsAAMovNWCLV6+QjLsGOpn739oi7qU6"} +{"k":"AAAAAAAAAAACYw==","v":"CgFjShUKBQgBEOgHCgwIub7c+vO3+ZECEAFQA4IBIhogAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh+SASBjDc0pZsQzZpESVEi7sltP9BKknHMtssirwbhYG9cQ3Q=="} +{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCisKBwoFCAEQ6AcSIP//////////////////////////////////////////"} +{"k":"AQAAAABi","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="} +{"k":"AQAAAABj","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="} +{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2I=","v":"AAAAAA=="} +{"k":"AgAAAAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIGI=","v":"AAAAAQ=="} +{"k":"BgAAAAAAAAAA","v":"dGVzdA=="} +{"k":"BwAAAAAAAAAA","v":""} +{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="} +{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="} +{"k":"CQAAAAA=","v":"CikIASACMAOKASD//////////////////////////////////////////wonCAEgAooBIPj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4CikIASACMAOKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLRDE7Jrik+mdhhY="} +{"k":"CmRiTWluU3luY3RoaW5nVmVyc2lvbg==","v":"djEuNC4w"} +{"k":"CmRiVmVyc2lvbg==","v":"AAAAAAAAAAk="} +{"k":"Cmxhc3RJbmRpcmVjdEdDVGltZQ==","v":"AAAAAF6yy/Q="} +{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABYQ=="} +{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYg=="} +{"k":"CwAAAAAAAAAAAAAAAw==","v":"AAAAAAAAAAABYw=="} +{"k":"DAAAAABi","v":""} +{"k":"DAAAAABj","v":""} diff --git a/lib/db/transactions.go b/lib/db/transactions.go index c78dab096..1c40eada6 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -142,6 +142,9 @@ func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate boo } else if err != nil { return nil, nil, false, err } + if len(vl.Versions) == 0 { + return nil, nil, false, nil + } keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file) if err != nil { @@ -348,10 +351,14 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn return err } + globalFV := vl.Versions[0] haveFV, have := vl.Get(device) + if !need(globalFV, have, haveFV.Version) { + continue + } name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) - dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name) + dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, globalFV.Device, name) if err != nil { return err } @@ -362,10 +369,7 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn if !ok { return errEntryFromGlobalMissing } - if !need(gf, have, haveFV.Version) { - continue - } - l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, vl.Versions[0].Version, vl.Versions[0].Device) + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, globalFV.Version, globalFV.Device) if !fn(gf) { return dbi.Error() } @@ -473,7 +477,9 @@ func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo, truncat // file. If the device is already present in the list, the version is updated. // If the file does not have an entry in the global list, it is created. func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) ([]byte, bool, error) { - l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.IsInvalid()) + deviceID := protocol.DeviceIDFromBytes(device) + + l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, deviceID, file.Name, file.Version, file.IsInvalid()) fl, err := t.getGlobalVersionsByKey(gk) if err != nil && !backend.IsNotFound(err) { @@ -487,110 +493,214 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi name := []byte(file.Name) - var global FileIntf - if insertedAt == 0 { - // Inserted a new newest version - global = file - } else { - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name) - if err != nil { - return nil, false, err - } - new, ok, err := t.getFileTrunc(keyBuf, true) - if err != nil || !ok { - return keyBuf, false, err - } - global = new - } - - // Fixup the list of files we need. - keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, fl, global) - if err != nil { - return nil, false, err - } - - if removedAt != 0 && insertedAt != 0 { - l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) - if err := t.Put(gk, mustMarshal(&fl)); err != nil { - return nil, false, err - } - return keyBuf, true, nil - } - - // Remove the old global from the global size counter - var oldGlobalFV FileVersion - if removedAt == 0 { - oldGlobalFV = removedFV - } else if len(fl.Versions) > 1 { - // The previous newest version is now at index 1 - oldGlobalFV = fl.Versions[1] - } - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name) - if err != nil { - return nil, false, err - } - oldFile, ok, err := t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, false, err - } - if ok { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - meta.removeFile(protocol.GlobalDeviceID, oldFile) - } - - // Add the new global to the global size counter - meta.addFile(protocol.GlobalDeviceID, global) - l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) if err := t.Put(gk, mustMarshal(&fl)); err != nil { return nil, false, err } + // Only load those from db if actually needed + + var gotGlobal, gotOldGlobal bool + var global, oldGlobal FileIntf + + globalFV := fl.Versions[0] + var oldGlobalFV FileVersion + haveOldGlobal := false + + globalUnaffected := removedAt != 0 && insertedAt != 0 + if globalUnaffected { + oldGlobalFV = globalFV + haveOldGlobal = true + } else { + if removedAt == 0 { + oldGlobalFV = removedFV + haveOldGlobal = true + } else if len(fl.Versions) > 1 { + // The previous newest version is now at index 1 + oldGlobalFV = fl.Versions[1] + haveOldGlobal = true + } + } + + // Check the need of the device that was updated + // Must happen before updating global meta: If this is the first + // item from this device, it will be initialized with the global state. + + needBefore := false + if haveOldGlobal { + needBefore = need(oldGlobalFV, removedAt >= 0, removedFV.Version) + } + needNow := need(globalFV, true, fl.Versions[insertedAt].Version) + if needBefore { + if !gotOldGlobal { + if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil { + return nil, false, err + } + gotOldGlobal = true + } + meta.removeNeeded(deviceID, oldGlobal) + if !needNow && bytes.Equal(device, protocol.LocalDeviceID[:]) { + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, false); err != nil { + return nil, false, err + } + } + } + if needNow { + if !gotGlobal { + if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil { + return nil, false, err + } + gotGlobal = true + } + meta.addNeeded(deviceID, global) + if !needBefore && bytes.Equal(device, protocol.LocalDeviceID[:]) { + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, true); err != nil { + return nil, false, err + } + } + } + + // Update global size counter if necessary + // Necessary here means the first item in the global list was changed, + // even if both new and old are invalid, due to potential change in + // LocalFlags. + + if !globalUnaffected { + if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil { + return nil, false, err + } + gotGlobal = true + if haveOldGlobal { + if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil { + return nil, false, err + } + gotOldGlobal = true + // Remove the old global from the global size counter + meta.removeFile(protocol.GlobalDeviceID, oldGlobal) + } + + // Add the new global to the global size counter + meta.addFile(protocol.GlobalDeviceID, global) + } + + if globalUnaffected { + // Neither the global state nor the needs of any devices, except + // the one updated, changed. + return keyBuf, true, nil + } + + // If global changed, but both the new and old are invalid, noone needed + // the file before and now -> nothing to do. + if global.IsInvalid() && (!haveOldGlobal || oldGlobal.IsInvalid()) { + return keyBuf, true, nil + } + + // check for local (if not already done before) + if !bytes.Equal(device, protocol.LocalDeviceID[:]) { + localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:]) + needBefore := false + if haveOldGlobal { + needBefore = need(oldGlobalFV, haveLocal, localFV.Version) + } + needNow := need(globalFV, haveLocal, localFV.Version) + if needBefore { + meta.removeNeeded(protocol.LocalDeviceID, oldGlobal) + if !needNow { + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, false); err != nil { + return nil, false, err + } + } + } + if need(globalFV, haveLocal, localFV.Version) { + meta.addNeeded(protocol.LocalDeviceID, global) + if !needBefore { + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, true); err != nil { + return nil, false, err + } + } + } + } + + for _, dev := range meta.devices() { + if bytes.Equal(dev[:], device) { + // Already handled above + continue + } + fv, have := fl.Get(dev[:]) + if haveOldGlobal && need(oldGlobalFV, have, fv.Version) { + meta.removeNeeded(dev, oldGlobal) + } + if need(globalFV, have, fv.Version) { + meta.addNeeded(dev, global) + } + } + return keyBuf, true, nil } -// updateLocalNeed checks whether the given file is still needed on the local -// device according to the version list and global FileInfo given and updates -// the db accordingly. -func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, fl VersionList, global FileIntf) ([]byte, error) { +func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, insertedAt int, fl VersionList) (FileIntf, error) { + if insertedAt == 0 { + // Inserted a new newest version + return file, nil + } + var err error + keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name) + if err != nil { + return nil, err + } + global, ok, err := t.getFileTrunc(keyBuf, true) + if err != nil { + return nil, err + } + if !ok { + return nil, errEntryFromGlobalMissing + } + return global, nil +} + +func (t readWriteTransaction) updateGlobalGetOldGlobal(keyBuf, folder, name []byte, oldGlobalFV FileVersion) (FileIntf, error) { + var err error + keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name) + if err != nil { + return nil, err + } + oldGlobal, ok, err := t.getFileTrunc(keyBuf, true) + if err != nil { + return nil, err + } + if !ok { + return nil, errEntryFromGlobalMissing + } + return oldGlobal, nil +} + +func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add bool) ([]byte, error) { var err error keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, name) if err != nil { return nil, err } - _, err = t.Get(keyBuf) - if err != nil && !backend.IsNotFound(err) { - return nil, err - } - hasNeeded := err == nil - if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(global, haveLocalFV, localFV.Version) { - if !hasNeeded { - l.Debugf("local need insert; folder=%q, name=%q", folder, name) - if err := t.Put(keyBuf, nil); err != nil { - return nil, err - } - } - } else if hasNeeded { + if add { + l.Debugf("local need insert; folder=%q, name=%q", folder, name) + err = t.Put(keyBuf, nil) + } else { l.Debugf("local need delete; folder=%q, name=%q", folder, name) - if err := t.Delete(keyBuf); err != nil { - return nil, err - } + err = t.Delete(keyBuf) } - return keyBuf, nil + return keyBuf, err } -func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool { +func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool { // We never need an invalid file. - if global.IsInvalid() { + if global.Invalid { return false } // We don't need a deleted file if we don't have it. - if global.IsDeleted() && !haveLocal { + if global.Deleted && !haveLocal { return false } // We don't need the global file if we already have the same version. - if haveLocal && localVersion.GreaterEqual(global.FileVersion()) { + if haveLocal && localVersion.GreaterEqual(global.Version) { return false } return true @@ -600,7 +710,9 @@ func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool { // given file. If the version list is empty after this, the file entry is // removed entirely. func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) { - l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file) + deviceID := protocol.DeviceIDFromBytes(device) + + l.Debugf("remove from global; folder=%q device=%v file=%q", folder, deviceID, file) fl, err := t.getGlobalVersionsByKey(gk) if backend.IsNotFound(err) { @@ -611,57 +723,79 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte return nil, err } - fl, _, removedAt := fl.pop(device) + fl, removedFV, removedAt := fl.pop(device) if removedAt == -1 { // There is no version for the given device return keyBuf, nil } - if removedAt == 0 { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, device, file) - if err != nil { + if removedAt != 0 { + l.Debugf("new global after remove: %v", fl) + if err := t.Put(gk, mustMarshal(&fl)); err != nil { return nil, err } - if f, ok, err := t.getFileTrunc(keyBuf, true); err != nil { + } + + keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, device, file) + if err != nil { + return nil, err + } + f, ok, err := t.getFileTrunc(keyBuf, true) + if err != nil { + return nil, err + } else if !ok { + return nil, errEntryFromGlobalMissing + } + meta.removeFile(protocol.GlobalDeviceID, f) + + if fv, have := fl.Get(protocol.LocalDeviceID[:]); need(removedFV, have, fv.Version) { + meta.removeNeeded(protocol.LocalDeviceID, f) + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, false); err != nil { return nil, err - } else if ok { - meta.removeFile(protocol.GlobalDeviceID, f) + } + } + for _, dev := range meta.devices() { + if bytes.Equal(dev[:], device) { + continue + } + if fv, have := fl.Get(dev[:]); need(removedFV, have, fv.Version) { + meta.removeNeeded(deviceID, f) } } if len(fl.Versions) == 0 { - keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, file) - if err != nil { - return nil, err - } - if err := t.Delete(keyBuf); err != nil { - return nil, err - } if err := t.Delete(gk); err != nil { return nil, err } return keyBuf, nil } - if removedAt == 0 { - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file) - if err != nil { - return nil, err + globalFV := fl.Versions[0] + keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, globalFV.Device, file) + if err != nil { + return nil, err + } + global, ok, err := t.getFileTrunc(keyBuf, true) + if err != nil { + return nil, err + } + if !ok { + return nil, errEntryFromGlobalMissing + } + meta.addFile(protocol.GlobalDeviceID, global) + + if !globalFV.Invalid { + if fv, have := fl.Get(protocol.LocalDeviceID[:]); need(globalFV, have, fv.Version) { + meta.addNeeded(deviceID, global) + if keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, true); err != nil { + return nil, err + } } - global, ok, err := t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, err + for _, dev := range meta.devices() { + if fv, have := fl.Get(dev[:]); need(globalFV, have, fv.Version) { + meta.addNeeded(deviceID, global) + } } - if !ok { - return nil, errEntryFromGlobalMissing - } - keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, fl, global) - if err != nil { - return nil, err - } - meta.addFile(protocol.GlobalDeviceID, global) } l.Debugf("new global after remove: %v", fl) diff --git a/lib/db/util_test.go b/lib/db/util_test.go index e7e5d5413..6cf71581b 100644 --- a/lib/db/util_test.go +++ b/lib/db/util_test.go @@ -10,8 +10,11 @@ import ( "encoding/json" "io" "os" + // "testing" "github.com/syncthing/syncthing/lib/db/backend" + // "github.com/syncthing/syncthing/lib/fs" + // "github.com/syncthing/syncthing/lib/protocol" ) // writeJSONS serializes the database to a JSON stream that can be checked @@ -114,3 +117,34 @@ func openJSONS(file string) (backend.Backend, error) { // } // writeJSONS(os.Stdout, db.DB) // } + +// func TestGenerateUpdateTo10(t *testing.T) { +// db := NewLowlevel(backend.OpenMemory()) +// defer db.Close() + +// if err := UpdateSchema(db); err != nil { +// t.Fatal(err) +// } + +// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) + +// files := []protocol.FileInfo{ +// {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 1}, +// {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2), Sequence: 2}, +// {Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 3}, +// } +// fs.Update(protocol.LocalDeviceID, files) +// files[1].Version = files[1].Version.Update(remoteDevice0.Short()) +// files[1].Deleted = true +// files[2].Version = files[2].Version.Update(remoteDevice0.Short()) +// files[2].Blocks = genBlocks(1) +// files[2].Deleted = false +// fs.Update(remoteDevice0, files) + +// fd, err := os.Create("./testdata/v1.4.0-updateTo10.json") +// if err != nil { +// panic(err) +// } +// defer fd.Close() +// writeJSONS(fd, db) +// } diff --git a/lib/model/devicedownloadstate.go b/lib/model/devicedownloadstate.go index b94f8548d..0d1249b3d 100644 --- a/lib/model/devicedownloadstate.go +++ b/lib/model/devicedownloadstate.go @@ -18,6 +18,7 @@ import ( type deviceFolderFileDownloadState struct { blockIndexes []int32 version protocol.Vector + blockSize int } // deviceFolderDownloadState holds current download state of all files that @@ -62,10 +63,12 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre local = deviceFolderFileDownloadState{ blockIndexes: update.BlockIndexes, version: update.Version, + blockSize: int(update.BlockSize), } } else if !local.version.Equal(update.Version) { local.blockIndexes = append(local.blockIndexes[:0], update.BlockIndexes...) local.version = update.Version + local.blockSize = int(update.BlockSize) } else { local.blockIndexes = append(local.blockIndexes, update.BlockIndexes...) } @@ -74,6 +77,20 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre } } +func (p *deviceFolderDownloadState) BytesDownloaded() int64 { + var res int64 + for _, state := range p.files { + // BlockSize is a new field introduced in 1.4.1, thus a fallback + // is required (will potentially underrepresent downloaded bytes). + if state.blockSize != 0 { + res += int64(len(state.blockIndexes) * state.blockSize) + } else { + res += int64(len(state.blockIndexes) * protocol.MinBlockSize) + } + } + return res +} + // GetBlockCounts returns a map filename -> number of blocks downloaded. func (p *deviceFolderDownloadState) GetBlockCounts() map[string]int { p.mut.RLock() @@ -150,6 +167,22 @@ func (t *deviceDownloadState) GetBlockCounts(folder string) map[string]int { return nil } +func (t *deviceDownloadState) BytesDownloaded(folder string) int64 { + if t == nil { + return 0 + } + + t.mut.RLock() + defer t.mut.RUnlock() + + for name, state := range t.folders { + if name == folder { + return state.BytesDownloaded() + } + } + return 0 +} + func newDeviceDownloadState() *deviceDownloadState { return &deviceDownloadState{ mut: sync.NewRWMutex(), diff --git a/lib/model/folder_summary.go b/lib/model/folder_summary.go index 8450983bd..eb11f6779 100644 --- a/lib/model/folder_summary.go +++ b/lib/model/folder_summary.go @@ -86,7 +86,7 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e if snap, err = c.model.DBSnapshot(folder); err == nil { global = snap.GlobalSize() local = snap.LocalSize() - need = snap.NeedSize() + need = snap.NeedSize(protocol.LocalDeviceID) ro = snap.ReceiveOnlyChangedSize() ourSeq = snap.Sequence(protocol.LocalDeviceID) remoteSeq = snap.Sequence(protocol.GlobalDeviceID) diff --git a/lib/model/model.go b/lib/model/model.go index 25d4793fd..7c4e94ed5 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -714,9 +714,9 @@ func (m *model) FolderStatistics() (map[string]stats.FolderStatistics, error) { type FolderCompletion struct { CompletionPct float64 NeedBytes int64 - NeedItems int64 GlobalBytes int64 - NeedDeletes int64 + NeedItems int32 + NeedDeletes int32 } // Map returns the members as a map, e.g. used in api to serialize as Json. @@ -752,52 +752,35 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple } m.pmut.RLock() - counts := m.deviceDownloads[device].GetBlockCounts(folder) + downloaded := m.deviceDownloads[device].BytesDownloaded(folder) m.pmut.RUnlock() - var need, items, fileNeed, downloaded, deletes int64 - snap.WithNeedTruncated(device, func(f db.FileIntf) bool { - ft := f.(db.FileInfoTruncated) + need := snap.NeedSize(device) + need.Bytes -= downloaded + // This might might be more than it really is, because some blocks can be of a smaller size. + if need.Bytes < 0 { + need.Bytes = 0 + } - // If the file is deleted, we account it only in the deleted column. - if ft.Deleted { - deletes++ - return true - } - - // This might might be more than it really is, because some blocks can be of a smaller size. - downloaded = int64(counts[ft.Name]) * int64(ft.BlockSize()) - - fileNeed = ft.FileSize() - downloaded - if fileNeed < 0 { - fileNeed = 0 - } - - need += fileNeed - items++ - - return true - }) - - needRatio := float64(need) / float64(tot) + needRatio := float64(need.Bytes) / float64(tot) completionPct := 100 * (1 - needRatio) // If the completion is 100% but there are deletes we need to handle, // drop it down a notch. Hack for consumers that look only at the // percentage (our own GUI does the same calculation as here on its own // and needs the same fixup). - if need == 0 && deletes > 0 { + if need.Bytes == 0 && need.Deleted > 0 { completionPct = 95 // chosen by fair dice roll } - l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio) + l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need.Bytes, tot, needRatio) return FolderCompletion{ CompletionPct: completionPct, - NeedBytes: need, - NeedItems: items, + NeedBytes: need.Bytes, + NeedItems: need.Files + need.Directories + need.Symlinks, GlobalBytes: tot, - NeedDeletes: deletes, + NeedDeletes: need.Deleted, } } diff --git a/lib/model/sentdownloadstate.go b/lib/model/sentdownloadstate.go index d1b7fb92f..821e395a3 100644 --- a/lib/model/sentdownloadstate.go +++ b/lib/model/sentdownloadstate.go @@ -19,6 +19,7 @@ type sentFolderFileDownloadState struct { version protocol.Vector updated time.Time created time.Time + blockSize int } // sentFolderDownloadState represents a state of what we've announced as available @@ -43,6 +44,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco pullerVersion := puller.file.Version pullerBlockIndexesUpdated := puller.AvailableUpdated() pullerCreated := puller.created + pullerBlockSize := int32(puller.file.BlockSize()) localFile, ok := s.files[name] @@ -55,6 +57,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco updated: pullerBlockIndexesUpdated, version: pullerVersion, created: pullerCreated, + blockSize: int(pullerBlockSize), } updates = append(updates, protocol.FileDownloadProgressUpdate{ @@ -62,6 +65,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco Version: pullerVersion, UpdateType: protocol.UpdateTypeAppend, BlockIndexes: pullerBlockIndexes, + BlockSize: pullerBlockSize, }) } continue @@ -86,11 +90,13 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco Version: pullerVersion, UpdateType: protocol.UpdateTypeAppend, BlockIndexes: pullerBlockIndexes, + BlockSize: pullerBlockSize, }) localFile.blockIndexes = pullerBlockIndexes localFile.updated = pullerBlockIndexesUpdated localFile.version = pullerVersion localFile.created = pullerCreated + localFile.blockSize = int(pullerBlockSize) continue } @@ -108,6 +114,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco Version: localFile.version, UpdateType: protocol.UpdateTypeAppend, BlockIndexes: newBlocks, + BlockSize: pullerBlockSize, }) } } diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index dd0d6c6bb..878bb5d52 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -207,7 +207,7 @@ func needSize(t *testing.T, m Model, folder string) db.Counts { t.Helper() snap := dbSnapshot(t, m, folder) defer snap.Release() - return snap.NeedSize() + return snap.NeedSize(protocol.LocalDeviceID) } func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot { diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go index 03036b0fe..a8d347a5f 100644 --- a/lib/protocol/bep.pb.go +++ b/lib/protocol/bep.pb.go @@ -784,6 +784,7 @@ type FileDownloadProgressUpdate struct { Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` Version Vector `protobuf:"bytes,3,opt,name=version,proto3" json:"version"` BlockIndexes []int32 `protobuf:"varint,4,rep,name=block_indexes,json=blockIndexes,proto3" json:"block_indexes,omitempty"` + BlockSize int32 `protobuf:"varint,5,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` } func (m *FileDownloadProgressUpdate) Reset() { *m = FileDownloadProgressUpdate{} } @@ -921,121 +922,122 @@ func init() { func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) } var fileDescriptor_e3f59eb60afbbc6e = []byte{ - // 1816 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8, - 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92, - 0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47, - 0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76, - 0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14, - 0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3, - 0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13, - 0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a, - 0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda, - 0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b, - 0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43, - 0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03, - 0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96, - 0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4, - 0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a, - 0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70, - 0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1, - 0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83, - 0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2, - 0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda, - 0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f, - 0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f, - 0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47, - 0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66, - 0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc, - 0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20, - 0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68, - 0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a, - 0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45, - 0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2, - 0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66, - 0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe, - 0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2, - 0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85, - 0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e, - 0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4, - 0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8, - 0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00, - 0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4, - 0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1, - 0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff, - 0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce, - 0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb, - 0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a, - 0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c, - 0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04, - 0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a, - 0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83, - 0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb, - 0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61, - 0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7, - 0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba, - 0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f, - 0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2, - 0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53, - 0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5, - 0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce, - 0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a, - 0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b, - 0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83, - 0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22, - 0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67, - 0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b, - 0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37, - 0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59, - 0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc, - 0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c, - 0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4, + // 1825 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x73, 0xdb, 0xc6, + 0x15, 0x26, 0x48, 0xf0, 0xd7, 0x23, 0xa5, 0x40, 0x6b, 0x5b, 0x45, 0x61, 0x9b, 0x84, 0x69, 0x3b, + 0x66, 0x34, 0xa9, 0xed, 0x26, 0x69, 0x3b, 0xed, 0xb4, 0x9d, 0xe1, 0x0f, 0x48, 0xe6, 0x54, 0x26, + 0xd5, 0x25, 0xe5, 0xd4, 0x39, 0x14, 0x03, 0x11, 0x4b, 0x0a, 0x63, 0x10, 0xcb, 0x02, 0xa0, 0x64, + 0xe6, 0x4f, 0xe0, 0xa9, 0xc7, 0x5e, 0x38, 0x93, 0x99, 0x9e, 0xfa, 0x9f, 0xf8, 0xe8, 0xf6, 0xd4, + 0xe9, 0x41, 0xd3, 0xc8, 0x97, 0x1c, 0x7b, 0xe9, 0xb5, 0xed, 0xec, 0x2e, 0x40, 0x82, 0x52, 0x9c, + 0xc9, 0xa1, 0x27, 0xec, 0xbe, 0xf7, 0xed, 0x5b, 0xec, 0xf7, 0xde, 0xfb, 0x76, 0xa1, 0x78, 0x42, + 0xa6, 0x8f, 0xa7, 0x3e, 0x0d, 0x29, 0x2a, 0xf0, 0xcf, 0x90, 0xba, 0xda, 0x7d, 0x9f, 0x4c, 0x69, + 0xf0, 0x84, 0xcf, 0x4f, 0x66, 0xa3, 0x27, 0x63, 0x3a, 0xa6, 0x7c, 0xc2, 0x47, 0x02, 0x5e, 0x9b, + 0x42, 0xf6, 0x19, 0x71, 0x5d, 0x8a, 0xaa, 0x50, 0xb2, 0xc9, 0x99, 0x33, 0x24, 0xa6, 0x67, 0x4d, + 0x88, 0x2a, 0xe9, 0x52, 0xbd, 0x88, 0x41, 0x98, 0xba, 0xd6, 0x84, 0x30, 0xc0, 0xd0, 0x75, 0x88, + 0x17, 0x0a, 0x40, 0x5a, 0x00, 0x84, 0x89, 0x03, 0x1e, 0xc2, 0x76, 0x04, 0x38, 0x23, 0x7e, 0xe0, + 0x50, 0x4f, 0xcd, 0x70, 0xcc, 0x96, 0xb0, 0xbe, 0x10, 0xc6, 0x5a, 0x00, 0xb9, 0x67, 0xc4, 0xb2, + 0x89, 0x8f, 0x3e, 0x02, 0x39, 0x9c, 0x4f, 0xc5, 0x5e, 0xdb, 0x9f, 0xdc, 0x7a, 0x1c, 0xff, 0xf9, + 0xe3, 0xe7, 0x24, 0x08, 0xac, 0x31, 0x19, 0xcc, 0xa7, 0x04, 0x73, 0x08, 0xfa, 0x35, 0x94, 0x86, + 0x74, 0x32, 0xf5, 0x49, 0xc0, 0x03, 0xa7, 0xf9, 0x8a, 0x3b, 0xd7, 0x56, 0xb4, 0xd6, 0x18, 0x9c, + 0x5c, 0x50, 0x6b, 0xc0, 0x56, 0xcb, 0x9d, 0x05, 0x21, 0xf1, 0x5b, 0xd4, 0x1b, 0x39, 0x63, 0xf4, + 0x14, 0xf2, 0x23, 0xea, 0xda, 0xc4, 0x0f, 0x54, 0x49, 0xcf, 0xd4, 0x4b, 0x9f, 0x28, 0xeb, 0x60, + 0xfb, 0xdc, 0xd1, 0x94, 0xdf, 0x5c, 0x54, 0x53, 0x38, 0x86, 0xd5, 0xfe, 0x9c, 0x86, 0x9c, 0xf0, + 0xa0, 0x5d, 0x48, 0x3b, 0xb6, 0xa0, 0xa8, 0x99, 0xbb, 0xbc, 0xa8, 0xa6, 0x3b, 0x6d, 0x9c, 0x76, + 0x6c, 0x74, 0x13, 0xb2, 0xae, 0x75, 0x42, 0xdc, 0x88, 0x1c, 0x31, 0x41, 0xb7, 0xa1, 0xe8, 0x13, + 0xcb, 0x36, 0xa9, 0xe7, 0xce, 0x39, 0x25, 0x05, 0x5c, 0x60, 0x86, 0x9e, 0xe7, 0xce, 0xd1, 0x8f, + 0x00, 0x39, 0x63, 0x8f, 0xfa, 0xc4, 0x9c, 0x12, 0x7f, 0xe2, 0xf0, 0xbf, 0x0d, 0x54, 0x99, 0xa3, + 0x76, 0x84, 0xe7, 0x68, 0xed, 0x40, 0xf7, 0x61, 0x2b, 0x82, 0xdb, 0xc4, 0x25, 0x21, 0x51, 0xb3, + 0x1c, 0x59, 0x16, 0xc6, 0x36, 0xb7, 0xa1, 0xa7, 0x70, 0xd3, 0x76, 0x02, 0xeb, 0xc4, 0x25, 0x66, + 0x48, 0x26, 0x53, 0xd3, 0xf1, 0x6c, 0xf2, 0x9a, 0x04, 0x6a, 0x8e, 0x63, 0x51, 0xe4, 0x1b, 0x90, + 0xc9, 0xb4, 0x23, 0x3c, 0x68, 0x17, 0x72, 0x53, 0x6b, 0x16, 0x10, 0x5b, 0xcd, 0x73, 0x4c, 0x34, + 0x63, 0x2c, 0x89, 0x0a, 0x08, 0x54, 0xe5, 0x2a, 0x4b, 0x6d, 0xee, 0x88, 0x59, 0x8a, 0x60, 0xb5, + 0x7f, 0xa5, 0x21, 0x27, 0x3c, 0xe8, 0xc3, 0x15, 0x4b, 0xe5, 0xe6, 0x2e, 0x43, 0xfd, 0xe3, 0xa2, + 0x5a, 0x10, 0xbe, 0x4e, 0x3b, 0xc1, 0x1a, 0x02, 0x39, 0x51, 0x51, 0x7c, 0x8c, 0xee, 0x40, 0xd1, + 0xb2, 0x6d, 0x96, 0x3d, 0x12, 0xa8, 0x19, 0x3d, 0x53, 0x2f, 0xe2, 0xb5, 0x01, 0xfd, 0x6c, 0xb3, + 0x1a, 0xe4, 0xab, 0xf5, 0xf3, 0xbe, 0x32, 0x60, 0xa9, 0x18, 0x12, 0x3f, 0xaa, 0xe0, 0x2c, 0xdf, + 0xaf, 0xc0, 0x0c, 0xbc, 0x7e, 0xef, 0x41, 0x79, 0x62, 0xbd, 0x36, 0x03, 0xf2, 0x87, 0x19, 0xf1, + 0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xba, 0x1f, 0x99, 0x50, 0x05, 0xc0, 0xf1, 0x42, + 0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x16, 0xf4, 0x13, 0x28, 0x70, 0xb2, 0x4d, 0xc7, + 0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2, + 0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x4b, 0xd0, 0x82, 0x57, 0x0e, 0x4b, 0x94, 0x88, 0x14, 0x3a, 0xd4, + 0x33, 0x7d, 0x32, 0xa1, 0x67, 0x96, 0x1b, 0xa8, 0x45, 0xbe, 0x8d, 0xca, 0x10, 0x9d, 0x04, 0x00, + 0x47, 0xfe, 0x5a, 0x0f, 0xb2, 0x3c, 0x22, 0xcb, 0xa2, 0x28, 0xd6, 0xa8, 0x7b, 0xa3, 0x19, 0x7a, + 0x0c, 0xd9, 0x91, 0xe3, 0x92, 0x40, 0x4d, 0xf3, 0x1c, 0xa2, 0x44, 0xa5, 0x3b, 0x2e, 0xe9, 0x78, + 0x23, 0x1a, 0x65, 0x51, 0xc0, 0x6a, 0xc7, 0x50, 0xe2, 0x01, 0x8f, 0xa7, 0xb6, 0x15, 0x92, 0xff, + 0x5b, 0xd8, 0xff, 0xca, 0x50, 0x88, 0x3d, 0xab, 0xa4, 0x4b, 0x89, 0xa4, 0x23, 0x90, 0x03, 0xe7, + 0x4b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0xdd, 0x05, 0x98, 0x50, 0xdb, 0x19, 0x39, 0xc4, 0x36, + 0x03, 0x9e, 0xb2, 0x0c, 0x2e, 0xc6, 0x96, 0x3e, 0x7a, 0x0a, 0xa5, 0x95, 0xfb, 0x64, 0xae, 0x96, + 0x39, 0xe7, 0x1f, 0xc4, 0x9c, 0xf7, 0x4f, 0xa9, 0x1f, 0x76, 0xda, 0x78, 0x15, 0xa2, 0x39, 0x67, + 0x25, 0x1d, 0xcb, 0x13, 0x23, 0x76, 0xa3, 0xa4, 0x5f, 0x90, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1, + 0x90, 0x06, 0x85, 0x55, 0x4d, 0x00, 0xff, 0x81, 0xd5, 0x1c, 0xfd, 0x18, 0x72, 0x27, 0x2e, 0x1d, + 0xbe, 0x8a, 0xfb, 0xe3, 0xc6, 0x3a, 0x58, 0x93, 0xd9, 0x13, 0x2c, 0x44, 0x40, 0x26, 0x93, 0xc1, + 0x7c, 0xe2, 0x3a, 0xde, 0x2b, 0x33, 0xb4, 0xfc, 0x31, 0x09, 0xd5, 0x1d, 0x21, 0x93, 0x91, 0x75, + 0xc0, 0x8d, 0x4c, 0x6e, 0xc5, 0x02, 0xf3, 0xd4, 0x0a, 0x4e, 0x55, 0xc4, 0xda, 0x08, 0x83, 0x30, + 0x3d, 0xb3, 0x82, 0x53, 0xb4, 0x17, 0xa9, 0xa7, 0xd0, 0xc2, 0xdd, 0xeb, 0xec, 0x27, 0xe4, 0x53, + 0x87, 0xd2, 0x55, 0x79, 0xd9, 0xc2, 0x49, 0x13, 0xdb, 0x6e, 0x45, 0xa4, 0x17, 0xa8, 0x25, 0x5d, + 0xaa, 0x67, 0xd7, 0xbc, 0x75, 0x03, 0xf4, 0x04, 0xc4, 0xe6, 0x26, 0x4f, 0xd1, 0x16, 0xf3, 0x37, + 0x95, 0xcb, 0x8b, 0x6a, 0x19, 0x5b, 0xe7, 0xfc, 0xa8, 0x7d, 0xe7, 0x4b, 0x82, 0x8b, 0x27, 0xf1, + 0x90, 0xed, 0xe9, 0xd2, 0xa1, 0xe5, 0x9a, 0x23, 0xd7, 0x1a, 0x07, 0xea, 0x37, 0x79, 0xbe, 0x29, + 0x70, 0xdb, 0x3e, 0x33, 0x21, 0x95, 0xa9, 0x0b, 0x53, 0x2c, 0x3b, 0x92, 0xa6, 0x78, 0x8a, 0xea, + 0x90, 0x77, 0xbc, 0x33, 0xcb, 0x75, 0x22, 0x41, 0x6a, 0x6e, 0x5f, 0x5e, 0x54, 0x01, 0x5b, 0xe7, + 0x1d, 0x61, 0xc5, 0xb1, 0x9b, 0xb1, 0xe9, 0xd1, 0x0d, 0xed, 0x2c, 0xf0, 0x50, 0x5b, 0x1e, 0x4d, + 0xe8, 0xe6, 0x2f, 0xe4, 0x3f, 0x7d, 0x55, 0x4d, 0xd5, 0x3c, 0x28, 0xae, 0xb2, 0xc2, 0xaa, 0x8d, + 0x33, 0x9b, 0xe1, 0xcc, 0xf2, 0x31, 0x2b, 0x75, 0x3a, 0x1a, 0x05, 0x24, 0xe4, 0x75, 0x99, 0xc1, + 0xd1, 0x6c, 0x55, 0x99, 0x69, 0x4e, 0x8b, 0xa8, 0xcc, 0xdb, 0x50, 0x3c, 0x27, 0xd6, 0x2b, 0x91, + 0x1e, 0xc1, 0x68, 0x81, 0x19, 0x58, 0x72, 0xa2, 0xfd, 0x7e, 0x05, 0x39, 0x51, 0x52, 0xe8, 0x53, + 0x28, 0x0c, 0xe9, 0xcc, 0x0b, 0xd7, 0xf7, 0xcd, 0x4e, 0x52, 0xae, 0xb8, 0x27, 0xaa, 0x93, 0x15, + 0xb0, 0xb6, 0x0f, 0xf9, 0xc8, 0x85, 0x1e, 0xae, 0xb4, 0x54, 0x6e, 0xde, 0xba, 0x52, 0xde, 0x9b, + 0x17, 0xd0, 0x99, 0xe5, 0xce, 0xc4, 0x8f, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59, + 0xc5, 0x06, 0x61, 0xe2, 0xea, 0xca, 0x6e, 0x5c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc, + 0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x95, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c, + 0x82, 0xe5, 0x87, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x5f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4, 0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98, - 0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18, - 0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5, + 0x52, 0x2f, 0x20, 0xef, 0x3d, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c, 0xf9, 0x18, + 0x3d, 0x02, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5, 0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3, - 0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1, - 0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75, - 0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc, - 0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f, - 0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39, - 0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80, - 0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7, - 0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b, - 0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45, - 0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c, - 0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f, - 0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18, - 0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58, - 0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46, - 0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04, - 0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe, - 0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e, - 0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12, - 0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f, - 0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41, - 0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68, - 0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56, - 0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4, - 0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72, - 0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e, - 0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17, - 0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a, - 0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde, - 0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52, - 0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25, - 0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30, - 0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70, - 0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d, - 0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f, - 0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c, - 0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3, - 0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3, - 0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a, - 0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92, - 0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25, - 0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, - 0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00, + 0x1b, 0xe4, 0xbd, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0x0f, 0x36, 0xbb, 0xf1, + 0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0xff, 0x96, 0x40, 0x7b, 0x3f, 0x1a, 0x75, + 0xa0, 0x24, 0x90, 0x66, 0xe2, 0xd1, 0x54, 0xff, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc, + 0xad, 0x37, 0x6e, 0x42, 0x17, 0x33, 0xdf, 0x4f, 0x17, 0x1f, 0xc1, 0x96, 0x50, 0x84, 0xf8, 0x7d, + 0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x4c, 0xbc, 0x2e, 0xee, + 0x6e, 0x48, 0x87, 0xa8, 0x8e, 0xb5, 0x50, 0xd4, 0x72, 0x20, 0x1f, 0x39, 0xde, 0xb8, 0x56, 0x85, + 0x6c, 0xcb, 0xa5, 0x3c, 0x9f, 0x39, 0x9f, 0x58, 0x01, 0xf5, 0x62, 0x9a, 0xc5, 0x6c, 0xef, 0x6f, + 0x69, 0x28, 0x25, 0x9e, 0x86, 0xe8, 0x29, 0x6c, 0xb7, 0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, + 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b, 0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0x57, + 0x5f, 0x15, 0xb2, 0x9d, 0x6e, 0xdb, 0xf8, 0x9d, 0x22, 0x69, 0x37, 0x17, 0x4b, 0x5d, 0x49, 0x00, + 0xc5, 0x15, 0xfa, 0x31, 0x94, 0x39, 0xc0, 0x3c, 0x3e, 0x6a, 0x37, 0x06, 0x86, 0x92, 0xd6, 0xb4, + 0xc5, 0x52, 0xdf, 0xbd, 0x8a, 0x8b, 0x52, 0x72, 0x1f, 0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, + 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18, 0x77, 0xdc, 0x43, 0x28, 0x60, 0xa3, 0x7f, + 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58, 0xea, 0x37, 0x36, 0x50, 0x51, 0x11, 0xff, + 0x14, 0x76, 0xda, 0xbd, 0xcf, 0xbb, 0x87, 0xbd, 0x46, 0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, + 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04, 0xfe, 0x5a, 0x4d, 0xde, 0x05, 0xf9, 0xa8, + 0xd3, 0x3d, 0x50, 0x72, 0xda, 0x8d, 0xc5, 0x52, 0xff, 0x20, 0x01, 0x65, 0xa4, 0xb2, 0x13, 0xb7, + 0x0e, 0x7b, 0x7d, 0x43, 0xc9, 0x5f, 0x3b, 0x31, 0x27, 0x7b, 0xef, 0xf7, 0x80, 0xae, 0x3f, 0x9e, + 0xd1, 0x03, 0x90, 0xbb, 0xbd, 0xae, 0xa1, 0xa4, 0xc4, 0xf9, 0xaf, 0x23, 0xba, 0xd4, 0x23, 0xa8, + 0x06, 0x99, 0xc3, 0x2f, 0x3e, 0x53, 0x24, 0xed, 0x87, 0x8b, 0xa5, 0x7e, 0xeb, 0x3a, 0xe8, 0xf0, + 0x8b, 0xcf, 0xf6, 0x28, 0x94, 0x92, 0x81, 0x6b, 0x50, 0x78, 0x6e, 0x0c, 0x1a, 0xed, 0xc6, 0xa0, + 0xa1, 0xa4, 0xc4, 0x2f, 0xc5, 0xee, 0xe7, 0x24, 0xb4, 0x78, 0x8f, 0xde, 0x81, 0x6c, 0xd7, 0x78, + 0x61, 0x60, 0x45, 0xd2, 0x76, 0x16, 0x4b, 0x7d, 0x2b, 0x06, 0x74, 0xc9, 0x19, 0xf1, 0x51, 0x05, + 0x72, 0x8d, 0xc3, 0xcf, 0x1b, 0x2f, 0xfb, 0x4a, 0x5a, 0x43, 0x8b, 0xa5, 0xbe, 0x1d, 0xbb, 0x1b, + 0xee, 0xb9, 0x35, 0x0f, 0xf6, 0xfe, 0x23, 0x41, 0x39, 0x79, 0x05, 0xa2, 0x0a, 0xc8, 0xfb, 0x9d, + 0x43, 0x23, 0xde, 0x2e, 0xe9, 0x63, 0x63, 0x54, 0x87, 0x62, 0xbb, 0x83, 0x8d, 0xd6, 0xa0, 0x87, + 0x5f, 0xc6, 0x67, 0x49, 0x82, 0xda, 0x8e, 0xcf, 0xeb, 0x7f, 0x8e, 0x7e, 0x0e, 0xe5, 0xfe, 0xcb, + 0xe7, 0x87, 0x9d, 0xee, 0x6f, 0x4c, 0x1e, 0x31, 0xad, 0x3d, 0x5a, 0x2c, 0xf5, 0x7b, 0x1b, 0x60, + 0x32, 0xf5, 0xc9, 0xd0, 0x0a, 0x89, 0xdd, 0x17, 0xd7, 0x39, 0x73, 0x16, 0x24, 0xd4, 0x82, 0x9d, + 0x78, 0xe9, 0x7a, 0xb3, 0x8c, 0xf6, 0xf1, 0x62, 0xa9, 0x7f, 0xf8, 0x9d, 0xeb, 0x57, 0xbb, 0x17, + 0x24, 0xf4, 0x00, 0xf2, 0x51, 0x90, 0xb8, 0x92, 0x92, 0x4b, 0xa3, 0x05, 0x7b, 0x7f, 0x91, 0xa0, + 0xb8, 0x52, 0x33, 0x46, 0x78, 0xb7, 0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0xca, 0xd9, 0xa5, + 0x7c, 0x88, 0xee, 0x41, 0xfe, 0xc0, 0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20, + 0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x08, 0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8b, 0x8f, 0xce, + 0xf7, 0x4f, 0x84, 0xea, 0xcf, 0x86, 0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0x2f, 0x1a, 0x87, 0x9d, + 0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1, 0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x1d, 0xce, 0xb0, 0x7b, 0x36, + 0x54, 0xbe, 0x5b, 0xb7, 0x90, 0x0e, 0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0x7f, 0xbf, 0xf6, + 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3, 0x07, 0xc6, 0x20, 0xfe, 0xf9, 0x35, 0x62, + 0x9f, 0xb2, 0xb7, 0x54, 0xb3, 0xfe, 0xe6, 0xeb, 0x4a, 0xea, 0xed, 0xd7, 0x95, 0xd4, 0x9b, 0xcb, + 0x8a, 0xf4, 0xf6, 0xb2, 0x22, 0xfd, 0xf3, 0xb2, 0x92, 0xfa, 0xe6, 0xb2, 0x22, 0xfd, 0xf1, 0x5d, + 0x25, 0xf5, 0xd5, 0xbb, 0x8a, 0xf4, 0xf6, 0x5d, 0x25, 0xf5, 0xf7, 0x77, 0x95, 0xd4, 0x49, 0x8e, + 0x6b, 0xde, 0xa7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x75, 0x30, 0x0d, 0x51, 0x0f, 0x00, + 0x00, } func (m *Hello) Marshal() (dAtA []byte, err error) { @@ -1878,6 +1880,11 @@ func (m *FileDownloadProgressUpdate) MarshalToSizedBuffer(dAtA []byte) (int, err _ = i var l int _ = l + if m.BlockSize != 0 { + i = encodeVarintBep(dAtA, i, uint64(m.BlockSize)) + i-- + dAtA[i] = 0x28 + } if len(m.BlockIndexes) > 0 { for iNdEx := len(m.BlockIndexes) - 1; iNdEx >= 0; iNdEx-- { i = encodeVarintBep(dAtA, i, uint64(m.BlockIndexes[iNdEx])) @@ -2352,6 +2359,9 @@ func (m *FileDownloadProgressUpdate) ProtoSize() (n int) { n += 1 + sovBep(uint64(e)) } } + if m.BlockSize != 0 { + n += 1 + sovBep(uint64(m.BlockSize)) + } return n } @@ -4929,6 +4939,25 @@ func (m *FileDownloadProgressUpdate) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field BlockIndexes", wireType) } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockSize", wireType) + } + m.BlockSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBep + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BlockSize |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipBep(dAtA[iNdEx:]) diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto index 0e848006e..60344e339 100644 --- a/lib/protocol/bep.proto +++ b/lib/protocol/bep.proto @@ -190,6 +190,7 @@ message FileDownloadProgressUpdate { string name = 2; Vector version = 3 [(gogoproto.nullable) = false]; repeated int32 block_indexes = 4 [packed=false]; + int32 block_size = 5; } enum FileDownloadProgressUpdateType {