lib/db, lib/model: Keep need stats in metadata (ref #5899) (#6413)

lib/db/benchmark_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-var files, oneFile, firstHalf, secondHalf []protocol.FileInfo
+var files, filesUpdated, oneFile, firstHalf, secondHalf, changed100, unchanged100 []protocol.FileInfo
 
 func lazyInitBenchFiles() {
 	if files != nil {
@@ -36,6 +36,12 @@ func lazyInitBenchFiles() {
 	firstHalf = files[:middle]
 	secondHalf = files[middle:]
 	oneFile = firstHalf[middle-1 : middle]
+
+	unchanged100 := files[100:200]
+	changed100 := append([]protocol.FileInfo{}, unchanged100...)
+	for i := range changed100 {
+		changed100[i].Version = changed100[i].Version.Copy().Update(myID)
+	}
 }
 
 func getBenchFileSet() (*db.Lowlevel, *db.FileSet) {
@@ -86,18 +92,43 @@ func BenchmarkUpdate100Changed(b *testing.B) {
 	ldb, benchS := getBenchFileSet()
 	defer ldb.Close()
 
-	unchanged := files[100:200]
+	b.ResetTimer()
-	changed := append([]protocol.FileInfo{}, unchanged...)
+	for i := 0; i < b.N; i++ {
-	for i := range changed {
+		if i%2 == 0 {
-		changed[i].Version = changed[i].Version.Copy().Update(myID)
+			benchS.Update(protocol.LocalDeviceID, changed100)
+		} else {
+			benchS.Update(protocol.LocalDeviceID, unchanged100)
+		}
 	}
 
+	b.ReportAllocs()
+}
+
+func setup10Remotes(benchS *db.FileSet) {
+	idBase := remoteDevice1.String()[1:]
+	first := 'J'
+	for i := 0; i < 10; i++ {
+		id, _ := protocol.DeviceIDFromString(fmt.Sprintf("%v%s", first+rune(i), idBase))
+		if i%2 == 0 {
+			benchS.Update(id, changed100)
+		} else {
+			benchS.Update(id, unchanged100)
+		}
+	}
+}
+
+func BenchmarkUpdate100Changed10Remotes(b *testing.B) {
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
+
+	setup10Remotes(benchS)
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		if i%2 == 0 {
-			benchS.Update(protocol.LocalDeviceID, changed)
+			benchS.Update(protocol.LocalDeviceID, changed100)
 		} else {
-			benchS.Update(protocol.LocalDeviceID, unchanged)
+			benchS.Update(protocol.LocalDeviceID, unchanged100)
 		}
 	}
 
@@ -108,18 +139,28 @@ func BenchmarkUpdate100ChangedRemote(b *testing.B) {
 	ldb, benchS := getBenchFileSet()
 	defer ldb.Close()
 
-	unchanged := files[100:200]
+	b.ResetTimer()
-	changed := append([]protocol.FileInfo{}, unchanged...)
+	for i := 0; i < b.N; i++ {
-	for i := range changed {
+		if i%2 == 0 {
-		changed[i].Version = changed[i].Version.Copy().Update(myID)
+			benchS.Update(remoteDevice0, changed100)
+		} else {
+			benchS.Update(remoteDevice0, unchanged100)
+		}
 	}
 
+	b.ReportAllocs()
+}
+
+func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) {
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		if i%2 == 0 {
-			benchS.Update(remoteDevice0, changed)
+			benchS.Update(remoteDevice0, changed100)
 		} else {
-			benchS.Update(remoteDevice0, unchanged)
+			benchS.Update(remoteDevice0, unchanged100)
 		}
 	}
 
@@ -287,3 +328,19 @@ func BenchmarkGlobalTruncated(b *testing.B) {
 
 	b.ReportAllocs()
 }
+
+func BenchmarkNeedCount(b *testing.B) {
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
+
+	benchS.Update(protocol.LocalDeviceID, changed100)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		snap := benchS.Snapshot()
+		_ = snap.NeedSize(protocol.LocalDeviceID)
+		snap.Release()
+	}
+
+	b.ReportAllocs()
+}

lib/db/db_test.go

@@ -481,3 +481,75 @@ func TestCheckGlobals(t *testing.T) {
 		t.Error("Expected key missing error, got", err)
 	}
 }
+
+func TestUpdateTo10(t *testing.T) {
+	ldb, err := openJSONS("./testdata/v1.4.0-updateTo10.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db := NewLowlevel(ldb)
+	defer db.Close()
+
+	UpdateSchema(db)
+
+	folder := "test"
+
+	meta := db.getMetaAndCheck(folder)
+
+	empty := Counts{}
+
+	c := meta.Counts(protocol.LocalDeviceID, needFlag)
+	if c.Files != 1 {
+		t.Error("Expected 1 needed file locally, got", c.Files)
+	}
+	c.Files = 0
+	if c.Deleted != 1 {
+		t.Error("Expected 1 needed deletion locally, got", c.Deleted)
+	}
+	c.Deleted = 0
+	if !c.Equal(empty) {
+		t.Error("Expected all counts to be zero, got", c)
+	}
+	c = meta.Counts(remoteDevice0, needFlag)
+	if !c.Equal(empty) {
+		t.Error("Expected all counts to be zero, got", c)
+	}
+
+	trans, err := db.newReadOnlyTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer trans.Release()
+	// a
+	vl, err := trans.getGlobalVersions(nil, []byte(folder), []byte("a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range vl.Versions {
+		if !v.Deleted {
+			t.Error("Unexpected undeleted global version for a")
+		}
+	}
+	// b
+	vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !vl.Versions[0].Deleted {
+		t.Error("vl.Versions[0] not deleted for b")
+	}
+	if vl.Versions[1].Deleted {
+		t.Error("vl.Versions[1] deleted for b")
+	}
+	// c
+	vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("c"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if vl.Versions[0].Deleted {
+		t.Error("vl.Versions[0] deleted for c")
+	}
+	if !vl.Versions[1].Deleted {
+		t.Error("vl.Versions[1] not deleted for c")
+	}
+}

lib/db/lowlevel.go

@@ -748,6 +748,25 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
 		return nil, err
 	}
 
+	meta.emptyNeeded(protocol.LocalDeviceID)
+	err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f FileIntf) bool {
+		meta.addNeeded(protocol.LocalDeviceID, f)
+		return true
+	})
+	if err != nil {
+		return nil, err
+	}
+	for _, device := range meta.devices() {
+		meta.emptyNeeded(device)
+		err = t.withNeed([]byte(folder), device[:], true, func(f FileIntf) bool {
+			meta.addNeeded(device, f)
+			return true
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	meta.SetCreated()
 	if err := meta.toDB(t, []byte(folder)); err != nil {
 		return nil, err

lib/db/meta.go

@@ -32,6 +32,8 @@ type metaKey struct {
 	flag uint32
 }
 
+const needFlag uint32 = 1 << 31 // Last bit, as early ones are local flags
+
 func newMetadataTracker() *metadataTracker {
 	return &metadataTracker{
 		mut: sync.NewRWMutex(),
@@ -116,10 +118,26 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts
 		idx = len(m.counts.Counts)
 		m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flag})
 		m.indexes[key] = idx
+		if flag == needFlag {
+			// Initially a new device needs everything, except deletes
+			m.counts.Counts[idx] = m.allNeededCounts(dev)
+		}
 	}
 	return &m.counts.Counts[idx]
 }
 
+// allNeeded makes sure there is a counts in case the device needs everything.
+func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts {
+	counts := Counts{}
+	if idx, ok := m.indexes[metaKey{protocol.GlobalDeviceID, 0}]; ok {
+		counts = m.counts.Counts[idx]
+		counts.Deleted = 0 // Don't need deletes if having nothing
+	}
+	counts.DeviceID = dev[:]
+	counts.LocalFlags = needFlag
+	return counts
+}
+
 // addFile adds a file to the counts, adjusting the sequence number as
 // appropriate
 func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
@@ -146,6 +164,30 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
 	}
 }
 
+// emptyNeeded makes sure there is a zero counts in case the device needs nothing.
+func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	m.dirty = true
+
+	m.indexes[metaKey{dev, needFlag}] = len(m.counts.Counts)
+	m.counts.Counts = append(m.counts.Counts, Counts{
+		DeviceID:   dev[:],
+		LocalFlags: needFlag,
+	})
+}
+
+// addNeeded adds a file to the needed counts
+func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f FileIntf) {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	m.dirty = true
+
+	m.addFileLocked(dev, needFlag, f)
+}
+
 func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 {
 	m.mut.Lock()
 	defer m.mut.Unlock()
@@ -200,6 +242,16 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
 	}
 }
 
+// removeNeeded removes a file from the needed counts
+func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	m.dirty = true
+
+	m.removeFileLocked(dev, needFlag, f)
+}
+
 func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
 	cp := m.countsPtr(dev, flag)
 
@@ -277,21 +329,17 @@ func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts {
 
 	idx, ok := m.indexes[metaKey{dev, flag}]
 	if !ok {
+		if flag == needFlag {
+			// If there's nothing about a device in the index yet,
+			// it needs everything.
+			return m.allNeededCounts(dev)
+		}
 		return Counts{}
 	}
 
 	return m.counts.Counts[idx]
 }
 
-// Counts returns the counts for the given device ID and flag. `flag` should
-// be zero or have exactly one bit set.
-func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
-	m.mut.RLock()
-	defer m.mut.RUnlock()
-
-	return m.countsMap.Counts(dev, flag)
-}
-
 // Snapshot returns a copy of the metadata for reading.
 func (m *metadataTracker) Snapshot() *countsMap {
 	m.mut.RLock()

lib/db/meta_test.go

@@ -175,3 +175,9 @@ func TestRecalcMeta(t *testing.T) {
 		t.Fatalf("Wrong fixed global byte count, %d != 3000", gs.Bytes)
 	}
 }
+
+func TestMetaKeyCollisions(t *testing.T) {
+	if protocol.LocalAllFlags&needFlag != 0 {
+		t.Error("Collision between need flag and protocol local file flags")
+	}
+}

lib/db/schemaupdater.go

@@ -22,9 +22,10 @@ import (
 //   6: v0.14.50
 //   7: v0.14.53
 //   8-9: v1.4.0
+//   10: v1.6.0
 const (
-	dbVersion             = 9
+	dbVersion             = 10
-	dbMinSyncthingVersion = "v1.4.0"
+	dbMinSyncthingVersion = "v1.6.0"
 )
 
 type databaseDowngradeError struct {
@@ -85,6 +86,7 @@ func (db *schemaUpdater) updateSchema() error {
 		{6, db.updateSchema5to6},
 		{7, db.updateSchema6to7},
 		{9, db.updateSchemato9},
+		{10, db.updateSchemato10},
 	}
 
 	for _, m := range migrations {
@@ -154,8 +156,10 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if err != nil {
 				return err
 			}
+			// Purposely pass nil file name to remove from global list,
+			// but don't touch meta and needs
 			buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil)
-			if err != nil {
+			if err != nil && err != errEntryFromGlobalMissing {
 				return err
 			}
 			if err := t.Delete(dbi.Key()); err != nil {
@@ -278,7 +282,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			if ok {
 				v = haveFile.FileVersion()
 			}
-			if !need(f, ok, v) {
+			fv := FileVersion{
+				Version: f.FileVersion(),
+				Invalid: f.IsInvalid(),
+				Deleted: f.IsDeleted(),
+			}
+			if !need(fv, ok, v) {
 				return true
 			}
 			nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
@@ -390,7 +399,6 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 		var delErr error
 		err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
 			name := []byte(f.FileName())
-			global := f.(protocol.FileInfo)
 			gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if delErr != nil {
 				return false
@@ -413,7 +421,13 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				// so lets not act on it.
 				return true
 			}
-			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(global, haveLocalFV, localFV.Version) {
+			globalFV := FileVersion{
+				Version: f.FileVersion(),
+				Invalid: f.IsInvalid(),
+				Deleted: f.IsDeleted(),
+			}
+
+			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) {
 				key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
 				if err != nil {
 					delErr = err
@@ -484,3 +498,73 @@ func (db *schemaUpdater) updateSchemato9(prev int) error {
 
 	return t.Commit()
 }
+
+func (db *schemaUpdater) updateSchemato10(_ int) error {
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.close()
+
+	var buf []byte
+
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+
+		buf, err = t.keyer.GenerateGlobalVersionKey(buf, folder, nil)
+		if err != nil {
+			return err
+		}
+		buf = globalVersionKey(buf).WithoutName()
+		dbi, err := t.NewPrefixIterator(buf)
+		if err != nil {
+			return err
+		}
+		defer dbi.Release()
+
+		for dbi.Next() {
+			var vl VersionList
+			if err := vl.Unmarshal(dbi.Value()); err != nil {
+				return err
+			}
+
+			changed := false
+			name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+
+			for i, fv := range vl.Versions {
+				buf, err = t.keyer.GenerateDeviceFileKey(buf, folder, fv.Device, name)
+				if err != nil {
+					return err
+				}
+				f, ok, err := t.getFileTrunc(buf, true)
+				if !ok {
+					return errEntryFromGlobalMissing
+				}
+				if err != nil {
+					return err
+				}
+				if f.IsDeleted() {
+					vl.Versions[i].Deleted = true
+					changed = true
+				}
+			}
+
+			if changed {
+				if err := t.Put(dbi.Key(), mustMarshal(&vl)); err != nil {
+					return err
+				}
+				if err := t.Checkpoint(); err != nil {
+					return err
+				}
+			}
+		}
+		dbi.Release()
+	}
+
+	// Trigger metadata recalc
+	if err := t.deleteKeyPrefix([]byte{KeyTypeFolderMeta}); err != nil {
+		return err
+	}
+
+	return t.Commit()
+}

lib/db/set.go

@@ -314,23 +314,8 @@ func (s *Snapshot) GlobalSize() Counts {
 	return global.Add(recvOnlyChanged)
 }
 
-func (s *Snapshot) NeedSize() Counts {
+func (s *Snapshot) NeedSize(device protocol.DeviceID) Counts {
-	var result Counts
+	return s.meta.Counts(device, needFlag)
-	s.WithNeedTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
-		switch {
-		case f.IsDeleted():
-			result.Deleted++
-		case f.IsDirectory():
-			result.Directories++
-		case f.IsSymlink():
-			result.Symlinks++
-		default:
-			result.Files++
-			result.Bytes += f.FileSize()
-		}
-		return true
-	})
-	return result
 }
 
 // LocalChangedFiles returns a paginated list of files that were changed locally.

lib/db/set_test.go

@@ -296,6 +296,8 @@ func TestGlobalSet(t *testing.T) {
 			t.Errorf("Need incorrect (local);\n A: %v !=\n E: %v", n, expectedLocalNeed)
 		}
 
+		checkNeed(t, m, protocol.LocalDeviceID, expectedLocalNeed)
+
 		n = fileList(needList(m, remoteDevice0))
 		sort.Sort(n)
 
@@ -303,6 +305,8 @@ func TestGlobalSet(t *testing.T) {
 			t.Errorf("Need incorrect (remote);\n A: %v !=\n E: %v", n, expectedRemoteNeed)
 		}
 
+		checkNeed(t, m, remoteDevice0, expectedRemoteNeed)
+
 		snap := m.Snapshot()
 		defer snap.Release()
 		f, ok := snap.Get(protocol.LocalDeviceID, "b")
@@ -427,6 +431,8 @@ func TestGlobalSet(t *testing.T) {
 	if fmt.Sprint(n) != fmt.Sprint(expectedSecRemoteNeed) {
 		t.Errorf("Need incorrect (secRemote);\n A: %v !=\n E: %v", n, expectedSecRemoteNeed)
 	}
+
+	checkNeed(t, m, remoteDevice1, expectedSecRemoteNeed)
 }
 
 func TestNeedWithInvalid(t *testing.T) {
@@ -465,6 +471,8 @@ func TestNeedWithInvalid(t *testing.T) {
 	if fmt.Sprint(need) != fmt.Sprint(expectedNeed) {
 		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed)
 	}
+
+	checkNeed(t, s, protocol.LocalDeviceID, expectedNeed)
 }
 
 func TestUpdateToInvalid(t *testing.T) {
@@ -642,6 +650,8 @@ func TestNeed(t *testing.T) {
 	if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
 		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
 	}
+
+	checkNeed(t, m, protocol.LocalDeviceID, shouldNeed)
 }
 
 func TestSequence(t *testing.T) {
@@ -761,6 +771,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 	if fmt.Sprint(need) != fmt.Sprint(total) {
 		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, total)
 
 	global := fileList(globalList(s))
 	if fmt.Sprint(global) != fmt.Sprint(total) {
@@ -1095,10 +1106,12 @@ func TestMoveGlobalBack(t *testing.T) {
 	} else if !need[0].IsEquivalent(remote0Have[0], 0) {
 		t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, remote0Have[:1])
 
 	if need := needList(s, remoteDevice0); len(need) != 0 {
 		t.Error("Expected no need for remote 0, got", need)
 	}
+	checkNeed(t, s, remoteDevice0, nil)
 
 	ls := localSize(s)
 	if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
@@ -1121,10 +1134,12 @@ func TestMoveGlobalBack(t *testing.T) {
 	} else if !need[0].IsEquivalent(localHave[0], 0) {
 		t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
 	}
+	checkNeed(t, s, remoteDevice0, localHave[:1])
 
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 0 {
 		t.Error("Expected no local need, got", need)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, nil)
 
 	ls = localSize(s)
 	if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
@@ -1158,6 +1173,7 @@ func TestIssue5007(t *testing.T) {
 	} else if !need[0].IsEquivalent(fs[0], 0) {
 		t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, fs[:1])
 
 	fs[0].LocalFlags = protocol.FlagLocalIgnored
 	s.Update(protocol.LocalDeviceID, fs)
@@ -1165,6 +1181,7 @@ func TestIssue5007(t *testing.T) {
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 0 {
 		t.Fatal("Expected no local need, got", need)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, nil)
 }
 
 // TestNeedDeleted checks that a file that doesn't exist locally isn't needed
@@ -1184,6 +1201,7 @@ func TestNeedDeleted(t *testing.T) {
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 0 {
 		t.Fatal("Expected no local need, got", need)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, nil)
 
 	fs[0].Deleted = false
 	fs[0].Version = fs[0].Version.Update(remoteDevice0.Short())
@@ -1194,6 +1212,7 @@ func TestNeedDeleted(t *testing.T) {
 	} else if !need[0].IsEquivalent(fs[0], 0) {
 		t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, fs[:1])
 
 	fs[0].Deleted = true
 	fs[0].Version = fs[0].Version.Update(remoteDevice0.Short())
@@ -1202,6 +1221,7 @@ func TestNeedDeleted(t *testing.T) {
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 0 {
 		t.Fatal("Expected no local need, got", need)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, nil)
 }
 
 func TestReceiveOnlyAccounting(t *testing.T) {
@@ -1338,6 +1358,7 @@ func TestNeedAfterUnignore(t *testing.T) {
 	} else if !need[0].IsEquivalent(remote, 0) {
 		t.Fatalf("Got %v, expected %v", need[0], remote)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{remote})
 }
 
 func TestRemoteInvalidNotAccounted(t *testing.T) {
@@ -1384,6 +1405,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
 	if !need[0].IsEquivalent(file, 0) {
 		t.Fatalf("Got needed file %v, expected %v", need[0], file)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file})
 
 	// rem1 sends an invalid file with increased version
 	inv := file
@@ -1399,6 +1421,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
 	if !need[0].IsEquivalent(file, 0) {
 		t.Fatalf("Got needed file %v, expected %v", need[0], file)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file})
 }
 
 func TestNeedAfterDeviceRemove(t *testing.T) {
@@ -1425,6 +1448,7 @@ func TestNeedAfterDeviceRemove(t *testing.T) {
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 0 {
 		t.Fatal("Expected no local need, got", need)
 	}
+	checkNeed(t, s, protocol.LocalDeviceID, nil)
 }
 
 func TestCaseSensitive(t *testing.T) {
@@ -1608,8 +1632,40 @@ func globalSize(fs *db.FileSet) db.Counts {
 	return snap.GlobalSize()
 }
 
+func needSize(fs *db.FileSet, id protocol.DeviceID) db.Counts {
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.NeedSize(id)
+}
+
 func receiveOnlyChangedSize(fs *db.FileSet) db.Counts {
 	snap := fs.Snapshot()
 	defer snap.Release()
 	return snap.ReceiveOnlyChangedSize()
 }
+
+func filesToCounts(files []protocol.FileInfo) db.Counts {
+	cp := db.Counts{}
+	for _, f := range files {
+		switch {
+		case f.IsDeleted():
+			cp.Deleted++
+		case f.IsDirectory() && !f.IsSymlink():
+			cp.Directories++
+		case f.IsSymlink():
+			cp.Symlinks++
+		default:
+			cp.Files++
+		}
+		cp.Bytes += f.FileSize()
+	}
+	return cp
+}
+
+func checkNeed(t testing.TB, s *db.FileSet, dev protocol.DeviceID, expected []protocol.FileInfo) {
+	t.Helper()
+	counts := needSize(s, dev)
+	if exp := filesToCounts(expected); !exp.Equal(counts) {
+		t.Errorf("Count incorrect (%v): expected %v, got %v", dev, exp, counts)
+	}
+}

lib/db/structs.go

@@ -187,6 +187,11 @@ func (c Counts) TotalItems() int32 {
 	return c.Files + c.Directories + c.Symlinks + c.Deleted
 }
 
+// Equal compares the numbers only, not sequence/dev/flags.
+func (c Counts) Equal(o Counts) bool {
+	return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
+}
+
 func (vl VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
@@ -212,6 +217,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t re
 		Device:  device,
 		Version: file.Version,
 		Invalid: file.IsInvalid(),
+		Deleted: file.IsDeleted(),
 	}
 	i := 0
 	if nv.Invalid {

lib/db/structs.pb.go

@@ -29,6 +29,7 @@ type FileVersion struct {
 	Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
 	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
 	Invalid bool            `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	Deleted bool            `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
 }
 
 func (m *FileVersion) Reset()         { *m = FileVersion{} }
@@ -327,54 +328,54 @@ func init() {
 func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) }
 
 var fileDescriptor_e774e8f5f348d14d = []byte{
-	// 743 bytes of a gzipped FileDescriptorProto
+	// 747 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6f, 0xe3, 0x44,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xbf, 0x8f, 0xe2, 0x46,
-	0x14, 0x8e, 0x37, 0x71, 0x9a, 0x3c, 0x27, 0x61, 0x77, 0x58, 0x55, 0x56, 0x24, 0x1c, 0x2b, 0x08,
+	0x14, 0xc6, 0x07, 0x66, 0xe1, 0x19, 0xc8, 0xdd, 0xe4, 0xb4, 0xb2, 0x90, 0x62, 0x2c, 0xa2, 0x93,
-	0xc9, 0xe2, 0x90, 0xb0, 0xdd, 0x1b, 0x48, 0x1c, 0xcc, 0xaa, 0x22, 0x12, 0x62, 0xd1, 0x64, 0xd5,
+	0xac, 0x14, 0x90, 0xdb, 0xeb, 0x12, 0x29, 0x85, 0x73, 0x5a, 0x05, 0x29, 0xca, 0x45, 0xc3, 0xe9,
-	0x13, 0x52, 0xe4, 0x1f, 0x93, 0x64, 0x54, 0xc7, 0x93, 0x7a, 0x26, 0xad, 0xdc, 0x1b, 0xff, 0x01,
+	0xaa, 0x48, 0xc8, 0x3f, 0x06, 0x18, 0xad, 0xf1, 0x10, 0xcf, 0xb0, 0x2b, 0x6f, 0x97, 0x3a, 0x4d,
-	0x47, 0x8e, 0xfd, 0x73, 0x7a, 0xec, 0x11, 0x71, 0x88, 0x20, 0xe1, 0xc0, 0x9f, 0x81, 0x66, 0xc6,
+	0xca, 0x94, 0xfb, 0xe7, 0x6c, 0xb9, 0x65, 0x94, 0x02, 0x25, 0x90, 0x22, 0x7f, 0x46, 0x34, 0x33,
-	0x76, 0x5c, 0x2e, 0xec, 0x6d, 0xbe, 0xef, 0x3d, 0xfb, 0xbd, 0xf9, 0xbe, 0xcf, 0x86, 0x3e, 0x17,
+	0xb6, 0xf1, 0x6e, 0x93, 0xeb, 0xde, 0xf7, 0xbd, 0x07, 0xef, 0xc7, 0xf7, 0x79, 0xa0, 0xcf, 0x45,
-	0xd9, 0x2e, 0x12, 0x7c, 0xb2, 0xcd, 0x98, 0x60, 0xe8, 0x45, 0x1c, 0x0e, 0x3f, 0xcf, 0xc8, 0x96,
+	0xb6, 0x8b, 0x04, 0x9f, 0x6c, 0x33, 0x26, 0x18, 0x7a, 0x16, 0x87, 0xc3, 0xcf, 0x33, 0xb2, 0x65,
-	0xf1, 0xa9, 0x22, 0xc2, 0xdd, 0x72, 0xba, 0x62, 0x2b, 0xa6, 0x80, 0x3a, 0xe9, 0xc6, 0xe1, 0x79,
+	0x7c, 0xaa, 0x88, 0x70, 0xb7, 0x9c, 0xae, 0xd8, 0x8a, 0x29, 0xa0, 0x22, 0x5d, 0x38, 0x3c, 0x4f,
-	0x42, 0x43, 0xdd, 0x12, 0xb1, 0x64, 0x1a, 0x92, 0xad, 0xe6, 0xc7, 0x37, 0x60, 0x5d, 0xd2, 0x84,
+	0x68, 0xa8, 0x4b, 0x22, 0x96, 0x4c, 0x43, 0xb2, 0xd5, 0xfc, 0xf8, 0x57, 0x03, 0xac, 0x4b, 0x9a,
-	0x5c, 0x91, 0x8c, 0x53, 0x96, 0xa2, 0xaf, 0xe0, 0xec, 0x56, 0x1f, 0x6d, 0xc3, 0x35, 0x3c, 0xeb,
+	0x90, 0x0f, 0x24, 0xe3, 0x94, 0xa5, 0xe8, 0x4b, 0x38, 0xbb, 0xd6, 0xa1, 0x6d, 0xb8, 0x86, 0x67,
-	0xe2, 0xe5, 0xa4, 0x7c, 0x68, 0x72, 0x45, 0x22, 0xc1, 0x32, 0xbf, 0xf5, 0xb8, 0x1f, 0x35, 0x70,
+	0x5d, 0x3c, 0x9f, 0x94, 0xbf, 0x9a, 0x7c, 0x20, 0x91, 0x60, 0x99, 0xdf, 0xba, 0xdf, 0x8f, 0x1a,
-	0xd9, 0x86, 0xce, 0xa1, 0x1d, 0x93, 0x5b, 0x1a, 0x11, 0xfb, 0x85, 0x6b, 0x78, 0x3d, 0x5c, 0x20,
+	0xb8, 0x2c, 0x43, 0xe7, 0xd0, 0x8e, 0xc9, 0x35, 0x8d, 0x88, 0xfd, 0xcc, 0x35, 0xbc, 0x1e, 0x2e,
-	0x64, 0xc3, 0x19, 0x4d, 0x6f, 0x83, 0x84, 0xc6, 0x76, 0xd3, 0x35, 0xbc, 0x0e, 0x2e, 0xe1, 0xf8,
+	0x10, 0xb2, 0xe1, 0x8c, 0xa6, 0xd7, 0x41, 0x42, 0x63, 0xbb, 0xe9, 0x1a, 0x5e, 0x07, 0x97, 0x50,
-	0x12, 0xac, 0x62, 0xdc, 0x0f, 0x94, 0x0b, 0xf4, 0x06, 0x3a, 0xc5, 0xbb, 0xb8, 0x6d, 0xb8, 0x4d,
+	0x66, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xa5, 0x33, 0x05, 0x1c, 0x5f, 0x82, 0x55, 0x0c, 0xf2,
-	0xcf, 0xba, 0xf8, 0x64, 0x12, 0x87, 0x93, 0xda, 0x56, 0xc5, 0xc8, 0xaa, 0xed, 0xeb, 0xd6, 0x6f,
+	0x3d, 0xe5, 0x02, 0xbd, 0x86, 0x4e, 0xd1, 0x85, 0xdb, 0x86, 0xdb, 0xf4, 0xac, 0x8b, 0x4f, 0x26,
-	0x0f, 0xa3, 0xc6, 0xf8, 0x17, 0x13, 0x5e, 0xc9, 0xae, 0x59, 0xba, 0x64, 0x1f, 0xb2, 0x5d, 0x1a,
+	0x71, 0x38, 0xa9, 0xcd, 0x5b, 0x0c, 0x53, 0x95, 0x7d, 0xd5, 0xfa, 0xfd, 0x6e, 0xd4, 0x18, 0xff,
-	0x05, 0x82, 0xc4, 0x08, 0x41, 0x2b, 0x0d, 0x36, 0x44, 0xad, 0xdf, 0xc5, 0xea, 0x2c, 0x39, 0x4e,
+	0x62, 0xc2, 0x0b, 0x59, 0x35, 0x4b, 0x97, 0xec, 0x7d, 0xb6, 0x4b, 0xa3, 0x40, 0x90, 0x18, 0x21,
-	0xef, 0x89, 0x5a, 0xa4, 0x89, 0xd5, 0x19, 0x7d, 0x06, 0xb0, 0x61, 0x31, 0x5d, 0x52, 0x12, 0x2f,
+	0x68, 0xa5, 0xc1, 0x86, 0xa8, 0xc5, 0xba, 0x58, 0xc5, 0x92, 0xe3, 0xf4, 0x96, 0xa8, 0x11, 0x9b,
-	0xb8, 0x6d, 0xaa, 0x4a, 0xb7, 0x64, 0xe6, 0xe8, 0x67, 0xb0, 0xaa, 0x72, 0x98, 0xdb, 0x3d, 0xd7,
+	0x58, 0xc5, 0xe8, 0x33, 0x80, 0x0d, 0x8b, 0xe9, 0x92, 0x92, 0x78, 0xc1, 0x6d, 0x53, 0x65, 0xba,
-	0xf0, 0x5a, 0xfe, 0x37, 0x72, 0x8f, 0x3f, 0xf6, 0xa3, 0xb7, 0x2b, 0x2a, 0xd6, 0xbb, 0x70, 0x12,
+	0x25, 0x33, 0x47, 0x3f, 0x81, 0x55, 0xa5, 0xc3, 0xdc, 0xee, 0xb9, 0x86, 0xd7, 0xf2, 0xbf, 0x96,
-	0xb1, 0xcd, 0x94, 0xe7, 0x69, 0x24, 0xd6, 0x34, 0x5d, 0xd5, 0x4e, 0x75, 0xad, 0x27, 0xf3, 0x35,
+	0x73, 0xfc, 0xb9, 0x1f, 0xbd, 0x59, 0x51, 0xb1, 0xde, 0x85, 0x93, 0x88, 0x6d, 0xa6, 0x3c, 0x4f,
-	0xcb, 0xc4, 0xec, 0x1d, 0xae, 0xc6, 0xf9, 0x79, 0x5d, 0xe6, 0xee, 0xc7, 0xc9, 0x3c, 0x84, 0x0e,
+	0x23, 0xb1, 0xa6, 0xe9, 0xaa, 0x16, 0xd5, 0x65, 0x98, 0xcc, 0xd7, 0x2c, 0x13, 0xb3, 0xb7, 0xb8,
-	0x27, 0x37, 0x3b, 0x92, 0x46, 0xc4, 0x06, 0xb5, 0x6c, 0x85, 0xd1, 0x17, 0x30, 0xe0, 0xf9, 0x26,
+	0x6a, 0xe7, 0xe7, 0x75, 0x01, 0xba, 0x1f, 0x27, 0xc0, 0x10, 0x3a, 0x9c, 0xfc, 0xbc, 0x23, 0x69,
-	0xa1, 0xe9, 0xf5, 0x42, 0x04, 0xd9, 0x8a, 0x08, 0xfb, 0x95, 0xba, 0x7c, 0xbf, 0x60, 0x3f, 0x28,
+	0x44, 0x6c, 0x50, 0xc3, 0x56, 0x18, 0xbd, 0x82, 0x01, 0xcf, 0x37, 0x09, 0x4d, 0xaf, 0x16, 0x22,
-	0x12, 0x8d, 0xc0, 0x0a, 0x13, 0x16, 0x5d, 0xf3, 0xc5, 0x3a, 0xe0, 0x6b, 0x1b, 0x29, 0xbb, 0x40,
+	0xc8, 0x56, 0x44, 0xd8, 0x2f, 0xd4, 0xf2, 0xfd, 0x82, 0x7d, 0xaf, 0x48, 0x34, 0x02, 0x2b, 0x4c,
-	0x53, 0xdf, 0x07, 0x7c, 0x8d, 0xbe, 0x84, 0x96, 0xc8, 0xb7, 0xda, 0xc8, 0xc1, 0xc5, 0xf9, 0x69,
+	0x58, 0x74, 0xc5, 0x17, 0xeb, 0x80, 0xaf, 0x6d, 0xa4, 0x84, 0x04, 0x4d, 0x7d, 0x17, 0xf0, 0x35,
-	0xa5, 0x4a, 0xe5, 0x7c, 0x4b, 0xb0, 0xea, 0x41, 0x2e, 0x58, 0x5b, 0x92, 0x6d, 0x28, 0xd7, 0xc6,
+	0xfa, 0x02, 0x5a, 0x22, 0xdf, 0x6a, 0x89, 0x07, 0x17, 0xe7, 0xa7, 0x91, 0xaa, 0x2b, 0xe7, 0x5b,
-	0xb5, 0x5c, 0xc3, 0xeb, 0xe3, 0x3a, 0x25, 0xc7, 0x55, 0x0a, 0xa6, 0xdc, 0xb6, 0x5c, 0xc3, 0x33,
+	0x82, 0x55, 0x0d, 0x72, 0xc1, 0xda, 0x92, 0x6c, 0x43, 0xb9, 0x16, 0x4e, 0x4a, 0xdc, 0xc7, 0x75,
-	0x4f, 0x22, 0xfc, 0xc8, 0xd1, 0x14, 0xf4, 0xf0, 0x85, 0xf2, 0xa6, 0x2f, 0xeb, 0xfe, 0xcb, 0xc3,
+	0x4a, 0xb6, 0xab, 0x2e, 0x98, 0x72, 0xdb, 0x72, 0x0d, 0xcf, 0x3c, 0x1d, 0xe1, 0x07, 0x8e, 0xa6,
-	0x7e, 0xd4, 0xc3, 0xc1, 0x9d, 0x2f, 0x0b, 0x73, 0x7a, 0x4f, 0x70, 0x37, 0x2c, 0x8f, 0x72, 0x66,
+	0xa0, 0x9b, 0x2f, 0x94, 0x36, 0x7d, 0x99, 0xf7, 0x9f, 0x1f, 0xf6, 0xa3, 0x1e, 0x0e, 0x6e, 0x7c,
-	0xc2, 0xa2, 0x20, 0x59, 0x2c, 0x93, 0x60, 0xc5, 0xed, 0x7f, 0xce, 0xd4, 0x50, 0x50, 0xdc, 0xa5,
+	0x99, 0x98, 0xd3, 0x5b, 0x82, 0xbb, 0x61, 0x19, 0xca, 0x9e, 0x09, 0x8b, 0x82, 0x64, 0xb1, 0x4c,
-	0xa4, 0x64, 0xe8, 0x62, 0x92, 0x10, 0x41, 0x62, 0xbb, 0xad, 0x43, 0x57, 0x40, 0xe4, 0x9d, 0xe2,
+	0x82, 0x15, 0xb7, 0xff, 0x3d, 0x53, 0x4d, 0x41, 0x71, 0x97, 0x92, 0xaa, 0x9b, 0xae, 0xfd, 0xc8,
-	0x28, 0x1f, 0xeb, 0xf8, 0x83, 0xc3, 0x7e, 0x04, 0x38, 0xb8, 0x9b, 0x69, 0xb6, 0x8a, 0xa7, 0x54,
+	0x74, 0xc8, 0x3b, 0x19, 0x55, 0xfe, 0xac, 0xe3, 0x0f, 0x0e, 0xfb, 0x11, 0xe0, 0xe0, 0x66, 0xa6,
-	0x33, 0x65, 0x8b, 0xfa, 0xe5, 0x3a, 0xea, 0x55, 0xfd, 0x94, 0xfd, 0x74, 0x22, 0x8b, 0x0c, 0x7e,
+	0xd9, 0x93, 0x71, 0x5f, 0xc1, 0x20, 0x65, 0x8b, 0xfa, 0x72, 0x1d, 0xf5, 0x57, 0xfd, 0x94, 0xfd,
-	0x0b, 0x5d, 0xb5, 0x6a, 0x91, 0xe4, 0xb6, 0x02, 0x65, 0x8e, 0x3f, 0x3d, 0x29, 0xa8, 0x78, 0x29,
+	0x78, 0x22, 0x0b, 0x0f, 0x7e, 0x03, 0x5d, 0x35, 0x6a, 0xe1, 0xe4, 0xb6, 0x02, 0xa5, 0x8f, 0x3f,
-	0x61, 0xe1, 0x6b, 0xd1, 0x38, 0x7e, 0x03, 0x03, 0xbf, 0x32, 0xe0, 0x7d, 0x9a, 0xe4, 0xff, 0xeb,
+	0x3d, 0x5d, 0x50, 0xf1, 0xf2, 0x84, 0x85, 0xae, 0x45, 0xe1, 0xf8, 0x35, 0x0c, 0xfc, 0x4a, 0x80,
-	0xd2, 0xf8, 0x6f, 0x03, 0xda, 0xdf, 0xb1, 0x5d, 0x2a, 0x38, 0x7a, 0x0d, 0xe6, 0x92, 0x26, 0x84,
+	0x77, 0x69, 0x92, 0xff, 0xaf, 0x4a, 0xe3, 0x7f, 0x0c, 0x68, 0x7f, 0xcb, 0x76, 0xa9, 0xe0, 0xe8,
-	0xab, 0xb0, 0x9b, 0x58, 0x03, 0x29, 0x53, 0x4c, 0x33, 0x95, 0x22, 0x4a, 0xb8, 0x72, 0xd3, 0xc4,
+	0x25, 0x98, 0x4b, 0x9a, 0x10, 0xae, 0xcc, 0x6e, 0x62, 0x0d, 0xe4, 0x99, 0x62, 0x9a, 0x29, 0x17,
-	0x75, 0x4a, 0x85, 0x49, 0x47, 0x83, 0xab, 0x6f, 0xc2, 0xc4, 0x15, 0xae, 0x4b, 0xd8, 0x52, 0xa5,
+	0x51, 0xc2, 0x95, 0x9a, 0x26, 0xae, 0x53, 0xca, 0x4c, 0xda, 0x1a, 0x5c, 0x7d, 0x13, 0x26, 0xae,
-	0x4a, 0xc2, 0xd7, 0x60, 0x86, 0xb9, 0x20, 0xe5, 0xc7, 0xa2, 0xc1, 0xb3, 0x60, 0xb6, 0xff, 0x13,
+	0xf0, 0xd3, 0xef, 0xd6, 0x3c, 0x9d, 0xf0, 0x25, 0x98, 0x61, 0x2e, 0x48, 0xf9, 0xb1, 0x68, 0xf0,
-	0xcc, 0x21, 0x74, 0xf4, 0xdf, 0x60, 0xf6, 0x4e, 0x45, 0xb2, 0x87, 0x2b, 0x8c, 0x1c, 0xa8, 0x19,
+	0xc8, 0x98, 0xed, 0x27, 0xc6, 0x1c, 0x42, 0x47, 0xbf, 0x13, 0xb3, 0xb7, 0xca, 0x92, 0x3d, 0x5c,
-	0xa7, 0xae, 0xf9, 0xcc, 0xca, 0xf1, 0x7b, 0xe8, 0xea, 0x5b, 0xce, 0x89, 0x40, 0x1e, 0xb4, 0x23,
+	0x61, 0xe4, 0x40, 0x4d, 0x38, 0xb5, 0xe6, 0x23, 0x29, 0xc7, 0xef, 0xa0, 0xab, 0xb7, 0x9c, 0x13,
-	0x05, 0x0a, 0x65, 0x41, 0xfe, 0x21, 0x74, 0xb9, 0x14, 0x54, 0xd7, 0xe5, 0xfa, 0x51, 0x46, 0xe4,
+	0x81, 0x3c, 0x68, 0x47, 0x0a, 0x14, 0x97, 0x05, 0xf9, 0x42, 0xe8, 0x74, 0x79, 0x50, 0x9d, 0x97,
-	0x9f, 0x40, 0x5d, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0x8f, 0x7f, 0x39, 0x8d, 0xc7, 0x83, 0x63, 0x3c,
+	0xe3, 0x47, 0x19, 0x91, 0x2f, 0x81, 0x5a, 0xbc, 0x89, 0x4b, 0xe8, 0xbb, 0xf7, 0x7f, 0x3b, 0x8d,
-	0x1d, 0x1c, 0xe3, 0xcf, 0x83, 0xd3, 0xf8, 0xf5, 0xe8, 0x34, 0x1e, 0x8e, 0x8e, 0xf1, 0x74, 0x74,
+	0xfb, 0x83, 0x63, 0x3c, 0x1c, 0x1c, 0xe3, 0xaf, 0x83, 0xd3, 0xf8, 0xed, 0xe8, 0x34, 0xee, 0x8e,
-	0x1a, 0xbf, 0x1f, 0x9d, 0x46, 0xd8, 0x56, 0x76, 0xbd, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8,
+	0x8e, 0xf1, 0x70, 0x74, 0x1a, 0x7f, 0x1c, 0x9d, 0x46, 0xd8, 0x56, 0x72, 0xbd, 0xf9, 0x2f, 0x00,
-	0xd0, 0x05, 0xba, 0x64, 0x05, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x8a, 0x67, 0x08, 0x85, 0x7f, 0x05, 0x00, 0x00,
 }
 
 func (m *FileVersion) Marshal() (dAtA []byte, err error) {
@@ -397,6 +398,16 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if m.Deleted {
+		i--
+		if m.Deleted {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i--
+		dAtA[i] = 0x20
+	}
 	if m.Invalid {
 		i--
 		if m.Invalid {
@@ -805,6 +816,9 @@ func (m *FileVersion) ProtoSize() (n int) {
 	if m.Invalid {
 		n += 2
 	}
+	if m.Deleted {
+		n += 2
+	}
 	return n
 }
 
@@ -1084,6 +1098,26 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.Invalid = bool(v != 0)
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipStructs(dAtA[iNdEx:])

lib/db/structs.proto

@@ -16,6 +16,7 @@ message FileVersion {
     protocol.Vector version = 1 [(gogoproto.nullable) = false];
     bytes           device  = 2;
     bool            invalid = 3;
+    bool            deleted = 4;
 }
 
 message VersionList {

lib/db/testdata/v1.4.0-updateTo10.json

@@ -0,0 +1,24 @@
+{"k":"AAAAAAAAAAABYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="}
+{"k":"AAAAAAAAAAABYg==","v":"CgFiSgcKBQgBEOgHUAKCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gkgEgC8XkepY1E4woWwAAyi81YItXr5CMuwY6mfvf2iLupTo="}
+{"k":"AAAAAAAAAAABYw==","v":"CgFjMAFKBwoFCAEQ6AdQAw=="}
+{"k":"AAAAAAAAAAACYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="}
+{"k":"AAAAAAAAAAACYg==","v":"CgFiMAFKFQoFCAEQ6AcKDAi5vtz687f5kQIQAVACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIJIBIAvF5HqWNROMKFsAAMovNWCLV6+QjLsGOpn739oi7qU6"}
+{"k":"AAAAAAAAAAACYw==","v":"CgFjShUKBQgBEOgHCgwIub7c+vO3+ZECEAFQA4IBIhogAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh+SASBjDc0pZsQzZpESVEi7sltP9BKknHMtssirwbhYG9cQ3Q=="}
+{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
+{"k":"AQAAAABi","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="}
+{"k":"AQAAAABj","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="}
+{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2I=","v":"AAAAAA=="}
+{"k":"AgAAAAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIGI=","v":"AAAAAQ=="}
+{"k":"BgAAAAAAAAAA","v":"dGVzdA=="}
+{"k":"BwAAAAAAAAAA","v":""}
+{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
+{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="}
+{"k":"CQAAAAA=","v":"CikIASACMAOKASD//////////////////////////////////////////wonCAEgAooBIPj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4CikIASACMAOKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLRDE7Jrik+mdhhY="}
+{"k":"CmRiTWluU3luY3RoaW5nVmVyc2lvbg==","v":"djEuNC4w"}
+{"k":"CmRiVmVyc2lvbg==","v":"AAAAAAAAAAk="}
+{"k":"Cmxhc3RJbmRpcmVjdEdDVGltZQ==","v":"AAAAAF6yy/Q="}
+{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABYQ=="}
+{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYg=="}
+{"k":"CwAAAAAAAAAAAAAAAw==","v":"AAAAAAAAAAABYw=="}
+{"k":"DAAAAABi","v":""}
+{"k":"DAAAAABj","v":""}

lib/db/transactions.go

@@ -142,6 +142,9 @@ func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate boo
 	} else if err != nil {
 		return nil, nil, false, err
 	}
+	if len(vl.Versions) == 0 {
+		return nil, nil, false, nil
+	}
 
 	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
 	if err != nil {
@@ -348,10 +351,14 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
 			return err
 		}
 
+		globalFV := vl.Versions[0]
 		haveFV, have := vl.Get(device)
 
+		if !need(globalFV, have, haveFV.Version) {
+			continue
+		}
 		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
-		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
+		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, globalFV.Device, name)
 		if err != nil {
 			return err
 		}
@@ -362,10 +369,7 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
 		if !ok {
 			return errEntryFromGlobalMissing
 		}
-		if !need(gf, have, haveFV.Version) {
+		l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, globalFV.Version, globalFV.Device)
-			continue
-		}
-		l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, vl.Versions[0].Version, vl.Versions[0].Device)
 		if !fn(gf) {
 			return dbi.Error()
 		}
@@ -473,7 +477,9 @@ func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo, truncat
 // file. If the device is already present in the list, the version is updated.
 // If the file does not have an entry in the global list, it is created.
 func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) ([]byte, bool, error) {
-	l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.IsInvalid())
+	deviceID := protocol.DeviceIDFromBytes(device)
+
+	l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, deviceID, file.Name, file.Version, file.IsInvalid())
 
 	fl, err := t.getGlobalVersionsByKey(gk)
 	if err != nil && !backend.IsNotFound(err) {
@@ -487,110 +493,214 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 
 	name := []byte(file.Name)
 
-	var global FileIntf
-	if insertedAt == 0 {
-		// Inserted a new newest version
-		global = file
-	} else {
-		keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name)
-		if err != nil {
-			return nil, false, err
-		}
-		new, ok, err := t.getFileTrunc(keyBuf, true)
-		if err != nil || !ok {
-			return keyBuf, false, err
-		}
-		global = new
-	}
-
-	// Fixup the list of files we need.
-	keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, fl, global)
-	if err != nil {
-		return nil, false, err
-	}
-
-	if removedAt != 0 && insertedAt != 0 {
-		l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
-		if err := t.Put(gk, mustMarshal(&fl)); err != nil {
-			return nil, false, err
-		}
-		return keyBuf, true, nil
-	}
-
-	// Remove the old global from the global size counter
-	var oldGlobalFV FileVersion
-	if removedAt == 0 {
-		oldGlobalFV = removedFV
-	} else if len(fl.Versions) > 1 {
-		// The previous newest version is now at index 1
-		oldGlobalFV = fl.Versions[1]
-	}
-	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name)
-	if err != nil {
-		return nil, false, err
-	}
-	oldFile, ok, err := t.getFileTrunc(keyBuf, true)
-	if err != nil {
-		return nil, false, err
-	}
-	if ok {
-		// A failure to get the file here is surprising and our
-		// global size data will be incorrect until a restart...
-		meta.removeFile(protocol.GlobalDeviceID, oldFile)
-	}
-
-	// Add the new global to the global size counter
-	meta.addFile(protocol.GlobalDeviceID, global)
-
 	l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
 	if err := t.Put(gk, mustMarshal(&fl)); err != nil {
 		return nil, false, err
 	}
 
+	// Only load those from db if actually needed
+
+	var gotGlobal, gotOldGlobal bool
+	var global, oldGlobal FileIntf
+
+	globalFV := fl.Versions[0]
+	var oldGlobalFV FileVersion
+	haveOldGlobal := false
+
+	globalUnaffected := removedAt != 0 && insertedAt != 0
+	if globalUnaffected {
+		oldGlobalFV = globalFV
+		haveOldGlobal = true
+	} else {
+		if removedAt == 0 {
+			oldGlobalFV = removedFV
+			haveOldGlobal = true
+		} else if len(fl.Versions) > 1 {
+			// The previous newest version is now at index 1
+			oldGlobalFV = fl.Versions[1]
+			haveOldGlobal = true
+		}
+	}
+
+	// Check the need of the device that was updated
+	// Must happen before updating global meta: If this is the first
+	// item from this device, it will be initialized with the global state.
+
+	needBefore := false
+	if haveOldGlobal {
+		needBefore = need(oldGlobalFV, removedAt >= 0, removedFV.Version)
+	}
+	needNow := need(globalFV, true, fl.Versions[insertedAt].Version)
+	if needBefore {
+		if !gotOldGlobal {
+			if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
+				return nil, false, err
+			}
+			gotOldGlobal = true
+		}
+		meta.removeNeeded(deviceID, oldGlobal)
+		if !needNow && bytes.Equal(device, protocol.LocalDeviceID[:]) {
+			if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, false); err != nil {
+				return nil, false, err
+			}
+		}
+	}
+	if needNow {
+		if !gotGlobal {
+			if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
+				return nil, false, err
+			}
+			gotGlobal = true
+		}
+		meta.addNeeded(deviceID, global)
+		if !needBefore && bytes.Equal(device, protocol.LocalDeviceID[:]) {
+			if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, true); err != nil {
+				return nil, false, err
+			}
+		}
+	}
+
+	// Update global size counter if necessary
+	// Necessary here means the first item in the global list was changed,
+	// even if both new and old are invalid, due to potential change in
+	// LocalFlags.
+
+	if !globalUnaffected {
+		if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
+			return nil, false, err
+		}
+		gotGlobal = true
+		if haveOldGlobal {
+			if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
+				return nil, false, err
+			}
+			gotOldGlobal = true
+			// Remove the old global from the global size counter
+			meta.removeFile(protocol.GlobalDeviceID, oldGlobal)
+		}
+
+		// Add the new global to the global size counter
+		meta.addFile(protocol.GlobalDeviceID, global)
+	}
+
+	if globalUnaffected {
+		// Neither the global state nor the needs of any devices, except
+		// the one updated, changed.
+		return keyBuf, true, nil
+	}
+
+	// If global changed, but both the new and old are invalid, noone needed
+	// the file before and now -> nothing to do.
+	if global.IsInvalid() && (!haveOldGlobal || oldGlobal.IsInvalid()) {
+		return keyBuf, true, nil
+	}
+
+	// check for local (if not already done before)
+	if !bytes.Equal(device, protocol.LocalDeviceID[:]) {
+		localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:])
+		needBefore := false
+		if haveOldGlobal {
+			needBefore = need(oldGlobalFV, haveLocal, localFV.Version)
+		}
+		needNow := need(globalFV, haveLocal, localFV.Version)
+		if needBefore {
+			meta.removeNeeded(protocol.LocalDeviceID, oldGlobal)
+			if !needNow {
+				if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, false); err != nil {
+					return nil, false, err
+				}
+			}
+		}
+		if need(globalFV, haveLocal, localFV.Version) {
+			meta.addNeeded(protocol.LocalDeviceID, global)
+			if !needBefore {
+				if keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, true); err != nil {
+					return nil, false, err
+				}
+			}
+		}
+	}
+
+	for _, dev := range meta.devices() {
+		if bytes.Equal(dev[:], device) {
+			// Already handled above
+			continue
+		}
+		fv, have := fl.Get(dev[:])
+		if haveOldGlobal && need(oldGlobalFV, have, fv.Version) {
+			meta.removeNeeded(dev, oldGlobal)
+		}
+		if need(globalFV, have, fv.Version) {
+			meta.addNeeded(dev, global)
+		}
+	}
+
 	return keyBuf, true, nil
 }
 
-// updateLocalNeed checks whether the given file is still needed on the local
+func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, insertedAt int, fl VersionList) (FileIntf, error) {
-// device according to the version list and global FileInfo given and updates
+	if insertedAt == 0 {
-// the db accordingly.
+		// Inserted a new newest version
-func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, fl VersionList, global FileIntf) ([]byte, error) {
+		return file, nil
+	}
+	var err error
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name)
+	if err != nil {
+		return nil, err
+	}
+	global, ok, err := t.getFileTrunc(keyBuf, true)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, errEntryFromGlobalMissing
+	}
+	return global, nil
+}
+
+func (t readWriteTransaction) updateGlobalGetOldGlobal(keyBuf, folder, name []byte, oldGlobalFV FileVersion) (FileIntf, error) {
+	var err error
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name)
+	if err != nil {
+		return nil, err
+	}
+	oldGlobal, ok, err := t.getFileTrunc(keyBuf, true)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, errEntryFromGlobalMissing
+	}
+	return oldGlobal, nil
+}
+
+func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add bool) ([]byte, error) {
 	var err error
 	keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, name)
 	if err != nil {
 		return nil, err
 	}
-	_, err = t.Get(keyBuf)
+	if add {
-	if err != nil && !backend.IsNotFound(err) {
+		l.Debugf("local need insert; folder=%q, name=%q", folder, name)
-		return nil, err
+		err = t.Put(keyBuf, nil)
-	}
+	} else {
-	hasNeeded := err == nil
-	if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(global, haveLocalFV, localFV.Version) {
-		if !hasNeeded {
-			l.Debugf("local need insert; folder=%q, name=%q", folder, name)
-			if err := t.Put(keyBuf, nil); err != nil {
-				return nil, err
-			}
-		}
-	} else if hasNeeded {
 		l.Debugf("local need delete; folder=%q, name=%q", folder, name)
-		if err := t.Delete(keyBuf); err != nil {
+		err = t.Delete(keyBuf)
-			return nil, err
-		}
 	}
-	return keyBuf, nil
+	return keyBuf, err
 }
 
-func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool {
+func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
 	// We never need an invalid file.
-	if global.IsInvalid() {
+	if global.Invalid {
 		return false
 	}
 	// We don't need a deleted file if we don't have it.
-	if global.IsDeleted() && !haveLocal {
+	if global.Deleted && !haveLocal {
 		return false
 	}
 	// We don't need the global file if we already have the same version.
-	if haveLocal && localVersion.GreaterEqual(global.FileVersion()) {
+	if haveLocal && localVersion.GreaterEqual(global.Version) {
 		return false
 	}
 	return true
@@ -600,7 +710,9 @@ func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool {
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
 func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) {
-	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
+	deviceID := protocol.DeviceIDFromBytes(device)
+
+	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, deviceID, file)
 
 	fl, err := t.getGlobalVersionsByKey(gk)
 	if backend.IsNotFound(err) {
@@ -611,57 +723,79 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
 		return nil, err
 	}
 
-	fl, _, removedAt := fl.pop(device)
+	fl, removedFV, removedAt := fl.pop(device)
 	if removedAt == -1 {
 		// There is no version for the given device
 		return keyBuf, nil
 	}
 
-	if removedAt == 0 {
+	if removedAt != 0 {
-		// A failure to get the file here is surprising and our
+		l.Debugf("new global after remove: %v", fl)
-		// global size data will be incorrect until a restart...
+		if err := t.Put(gk, mustMarshal(&fl)); err != nil {
-		keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, device, file)
-		if err != nil {
 			return nil, err
 		}
-		if f, ok, err := t.getFileTrunc(keyBuf, true); err != nil {
+	}
+
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, device, file)
+	if err != nil {
+		return nil, err
+	}
+	f, ok, err := t.getFileTrunc(keyBuf, true)
+	if err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, errEntryFromGlobalMissing
+	}
+	meta.removeFile(protocol.GlobalDeviceID, f)
+
+	if fv, have := fl.Get(protocol.LocalDeviceID[:]); need(removedFV, have, fv.Version) {
+		meta.removeNeeded(protocol.LocalDeviceID, f)
+		if keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, false); err != nil {
 			return nil, err
-		} else if ok {
+		}
-			meta.removeFile(protocol.GlobalDeviceID, f)
+	}
+	for _, dev := range meta.devices() {
+		if bytes.Equal(dev[:], device) {
+			continue
+		}
+		if fv, have := fl.Get(dev[:]); need(removedFV, have, fv.Version) {
+			meta.removeNeeded(deviceID, f)
 		}
 	}
 
 	if len(fl.Versions) == 0 {
-		keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, file)
-		if err != nil {
-			return nil, err
-		}
-		if err := t.Delete(keyBuf); err != nil {
-			return nil, err
-		}
 		if err := t.Delete(gk); err != nil {
 			return nil, err
 		}
 		return keyBuf, nil
 	}
 
-	if removedAt == 0 {
+	globalFV := fl.Versions[0]
-		keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, globalFV.Device, file)
-		if err != nil {
+	if err != nil {
-			return nil, err
+		return nil, err
+	}
+	global, ok, err := t.getFileTrunc(keyBuf, true)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, errEntryFromGlobalMissing
+	}
+	meta.addFile(protocol.GlobalDeviceID, global)
+
+	if !globalFV.Invalid {
+		if fv, have := fl.Get(protocol.LocalDeviceID[:]); need(globalFV, have, fv.Version) {
+			meta.addNeeded(deviceID, global)
+			if keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, true); err != nil {
+				return nil, err
+			}
 		}
-		global, ok, err := t.getFileTrunc(keyBuf, true)
+		for _, dev := range meta.devices() {
-		if err != nil {
+			if fv, have := fl.Get(dev[:]); need(globalFV, have, fv.Version) {
-			return nil, err
+				meta.addNeeded(deviceID, global)
+			}
 		}
-		if !ok {
-			return nil, errEntryFromGlobalMissing
-		}
-		keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, fl, global)
-		if err != nil {
-			return nil, err
-		}
-		meta.addFile(protocol.GlobalDeviceID, global)
 	}
 
 	l.Debugf("new global after remove: %v", fl)

lib/db/util_test.go

@@ -10,8 +10,11 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	// "testing"
 
 	"github.com/syncthing/syncthing/lib/db/backend"
+	// "github.com/syncthing/syncthing/lib/fs"
+	// "github.com/syncthing/syncthing/lib/protocol"
 )
 
 // writeJSONS serializes the database to a JSON stream that can be checked
@@ -114,3 +117,34 @@ func openJSONS(file string) (backend.Backend, error) {
 // 	}
 // 	writeJSONS(os.Stdout, db.DB)
 // }
+
+// func TestGenerateUpdateTo10(t *testing.T) {
+// 	db := NewLowlevel(backend.OpenMemory())
+// 	defer db.Close()
+
+// 	if err := UpdateSchema(db); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
+
+// 	files := []protocol.FileInfo{
+// 		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 1},
+// 		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2), Sequence: 2},
+// 		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 3},
+// 	}
+// 	fs.Update(protocol.LocalDeviceID, files)
+// 	files[1].Version = files[1].Version.Update(remoteDevice0.Short())
+// 	files[1].Deleted = true
+// 	files[2].Version = files[2].Version.Update(remoteDevice0.Short())
+// 	files[2].Blocks = genBlocks(1)
+// 	files[2].Deleted = false
+// 	fs.Update(remoteDevice0, files)
+
+// 	fd, err := os.Create("./testdata/v1.4.0-updateTo10.json")
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// 	defer fd.Close()
+// 	writeJSONS(fd, db)
+// }

lib/model/devicedownloadstate.go

@@ -18,6 +18,7 @@ import (
 type deviceFolderFileDownloadState struct {
 	blockIndexes []int32
 	version      protocol.Vector
+	blockSize    int
 }
 
 // deviceFolderDownloadState holds current download state of all files that
@@ -62,10 +63,12 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre
 				local = deviceFolderFileDownloadState{
 					blockIndexes: update.BlockIndexes,
 					version:      update.Version,
+					blockSize:    int(update.BlockSize),
 				}
 			} else if !local.version.Equal(update.Version) {
 				local.blockIndexes = append(local.blockIndexes[:0], update.BlockIndexes...)
 				local.version = update.Version
+				local.blockSize = int(update.BlockSize)
 			} else {
 				local.blockIndexes = append(local.blockIndexes, update.BlockIndexes...)
 			}
@@ -74,6 +77,20 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre
 	}
 }
 
+func (p *deviceFolderDownloadState) BytesDownloaded() int64 {
+	var res int64
+	for _, state := range p.files {
+		// BlockSize is a new field introduced in 1.4.1, thus a fallback
+		// is required (will potentially underrepresent downloaded bytes).
+		if state.blockSize != 0 {
+			res += int64(len(state.blockIndexes) * state.blockSize)
+		} else {
+			res += int64(len(state.blockIndexes) * protocol.MinBlockSize)
+		}
+	}
+	return res
+}
+
 // GetBlockCounts returns a map filename -> number of blocks downloaded.
 func (p *deviceFolderDownloadState) GetBlockCounts() map[string]int {
 	p.mut.RLock()
@@ -150,6 +167,22 @@ func (t *deviceDownloadState) GetBlockCounts(folder string) map[string]int {
 	return nil
 }
 
+func (t *deviceDownloadState) BytesDownloaded(folder string) int64 {
+	if t == nil {
+		return 0
+	}
+
+	t.mut.RLock()
+	defer t.mut.RUnlock()
+
+	for name, state := range t.folders {
+		if name == folder {
+			return state.BytesDownloaded()
+		}
+	}
+	return 0
+}
+
 func newDeviceDownloadState() *deviceDownloadState {
 	return &deviceDownloadState{
 		mut:     sync.NewRWMutex(),

lib/model/folder_summary.go

@@ -86,7 +86,7 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
 		if snap, err = c.model.DBSnapshot(folder); err == nil {
 			global = snap.GlobalSize()
 			local = snap.LocalSize()
-			need = snap.NeedSize()
+			need = snap.NeedSize(protocol.LocalDeviceID)
 			ro = snap.ReceiveOnlyChangedSize()
 			ourSeq = snap.Sequence(protocol.LocalDeviceID)
 			remoteSeq = snap.Sequence(protocol.GlobalDeviceID)

lib/model/model.go

@@ -714,9 +714,9 @@ func (m *model) FolderStatistics() (map[string]stats.FolderStatistics, error) {
 type FolderCompletion struct {
 	CompletionPct float64
 	NeedBytes     int64
-	NeedItems     int64
 	GlobalBytes   int64
-	NeedDeletes   int64
+	NeedItems     int32
+	NeedDeletes   int32
 }
 
 // Map returns the members as a map, e.g. used in api to serialize as Json.
@@ -752,52 +752,35 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 	}
 
 	m.pmut.RLock()
-	counts := m.deviceDownloads[device].GetBlockCounts(folder)
+	downloaded := m.deviceDownloads[device].BytesDownloaded(folder)
 	m.pmut.RUnlock()
 
-	var need, items, fileNeed, downloaded, deletes int64
+	need := snap.NeedSize(device)
-	snap.WithNeedTruncated(device, func(f db.FileIntf) bool {
+	need.Bytes -= downloaded
-		ft := f.(db.FileInfoTruncated)
+	// This might might be more than it really is, because some blocks can be of a smaller size.
+	if need.Bytes < 0 {
+		need.Bytes = 0
+	}
 
-		// If the file is deleted, we account it only in the deleted column.
+	needRatio := float64(need.Bytes) / float64(tot)
-		if ft.Deleted {
-			deletes++
-			return true
-		}
-
-		// This might might be more than it really is, because some blocks can be of a smaller size.
-		downloaded = int64(counts[ft.Name]) * int64(ft.BlockSize())
-
-		fileNeed = ft.FileSize() - downloaded
-		if fileNeed < 0 {
-			fileNeed = 0
-		}
-
-		need += fileNeed
-		items++
-
-		return true
-	})
-
-	needRatio := float64(need) / float64(tot)
 	completionPct := 100 * (1 - needRatio)
 
 	// If the completion is 100% but there are deletes we need to handle,
 	// drop it down a notch. Hack for consumers that look only at the
 	// percentage (our own GUI does the same calculation as here on its own
 	// and needs the same fixup).
-	if need == 0 && deletes > 0 {
+	if need.Bytes == 0 && need.Deleted > 0 {
 		completionPct = 95 // chosen by fair dice roll
 	}
 
-	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)
+	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need.Bytes, tot, needRatio)
 
 	return FolderCompletion{
 		CompletionPct: completionPct,
-		NeedBytes:     need,
+		NeedBytes:     need.Bytes,
-		NeedItems:     items,
+		NeedItems:     need.Files + need.Directories + need.Symlinks,
 		GlobalBytes:   tot,
-		NeedDeletes:   deletes,
+		NeedDeletes:   need.Deleted,
 	}
 }
 

lib/model/sentdownloadstate.go

@@ -19,6 +19,7 @@ type sentFolderFileDownloadState struct {
 	version      protocol.Vector
 	updated      time.Time
 	created      time.Time
+	blockSize    int
 }
 
 // sentFolderDownloadState represents a state of what we've announced as available
@@ -43,6 +44,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 		pullerVersion := puller.file.Version
 		pullerBlockIndexesUpdated := puller.AvailableUpdated()
 		pullerCreated := puller.created
+		pullerBlockSize := int32(puller.file.BlockSize())
 
 		localFile, ok := s.files[name]
 
@@ -55,6 +57,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 					updated:      pullerBlockIndexesUpdated,
 					version:      pullerVersion,
 					created:      pullerCreated,
+					blockSize:    int(pullerBlockSize),
 				}
 
 				updates = append(updates, protocol.FileDownloadProgressUpdate{
@@ -62,6 +65,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 					Version:      pullerVersion,
 					UpdateType:   protocol.UpdateTypeAppend,
 					BlockIndexes: pullerBlockIndexes,
+					BlockSize:    pullerBlockSize,
 				})
 			}
 			continue
@@ -86,11 +90,13 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 				Version:      pullerVersion,
 				UpdateType:   protocol.UpdateTypeAppend,
 				BlockIndexes: pullerBlockIndexes,
+				BlockSize:    pullerBlockSize,
 			})
 			localFile.blockIndexes = pullerBlockIndexes
 			localFile.updated = pullerBlockIndexesUpdated
 			localFile.version = pullerVersion
 			localFile.created = pullerCreated
+			localFile.blockSize = int(pullerBlockSize)
 			continue
 		}
 
@@ -108,6 +114,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 				Version:      localFile.version,
 				UpdateType:   protocol.UpdateTypeAppend,
 				BlockIndexes: newBlocks,
+				BlockSize:    pullerBlockSize,
 			})
 		}
 	}

lib/model/testutils_test.go

@@ -207,7 +207,7 @@ func needSize(t *testing.T, m Model, folder string) db.Counts {
 	t.Helper()
 	snap := dbSnapshot(t, m, folder)
 	defer snap.Release()
-	return snap.NeedSize()
+	return snap.NeedSize(protocol.LocalDeviceID)
 }
 
 func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {

lib/protocol/bep.pb.go

@@ -784,6 +784,7 @@ type FileDownloadProgressUpdate struct {
 	Name         string                         `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
 	Version      Vector                         `protobuf:"bytes,3,opt,name=version,proto3" json:"version"`
 	BlockIndexes []int32                        `protobuf:"varint,4,rep,name=block_indexes,json=blockIndexes,proto3" json:"block_indexes,omitempty"`
+	BlockSize    int32                          `protobuf:"varint,5,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"`
 }
 
 func (m *FileDownloadProgressUpdate) Reset()         { *m = FileDownloadProgressUpdate{} }
@@ -921,121 +922,122 @@ func init() {
 func init() { proto.RegisterFile("bep.proto", fileDescriptor_e3f59eb60afbbc6e) }
 
 var fileDescriptor_e3f59eb60afbbc6e = []byte{
-	// 1816 bytes of a gzipped FileDescriptorProto
+	// 1825 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xdb, 0xc8,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x73, 0xdb, 0xc6,
-	0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xec, 0xa5, 0x27, 0x89, 0xcb, 0x32, 0x59, 0x89, 0x51, 0x92,
+	0x15, 0x26, 0x48, 0xf0, 0xd7, 0x23, 0xa5, 0x40, 0x6b, 0x5b, 0x45, 0x61, 0x9b, 0x84, 0x69, 0x3b,
-	0x8d, 0xd6, 0xd8, 0x26, 0xe9, 0xee, 0xb6, 0x45, 0x8b, 0xb6, 0x80, 0xfe, 0xd0, 0x8e, 0x50, 0x47,
+	0x66, 0x34, 0xa9, 0xed, 0x26, 0x69, 0x3b, 0xed, 0xb4, 0x9d, 0xe1, 0x0f, 0x48, 0xe6, 0x54, 0x26,
-	0x72, 0x47, 0x72, 0xb6, 0xd9, 0x43, 0x09, 0x5a, 0x1c, 0xc9, 0x44, 0x28, 0x8e, 0x4a, 0x52, 0x76,
+	0xd5, 0x25, 0xe5, 0xd4, 0x39, 0x14, 0x03, 0x11, 0x4b, 0x0a, 0x63, 0x10, 0xcb, 0x02, 0xa0, 0x64,
-	0xb4, 0x1f, 0x41, 0xa7, 0x1e, 0x7b, 0x11, 0xb0, 0x40, 0x4f, 0xfd, 0x26, 0x39, 0xa6, 0x3d, 0x14,
+	0xe6, 0x4f, 0xe0, 0xa9, 0xc7, 0x5e, 0x38, 0x93, 0x99, 0x9e, 0xfa, 0x9f, 0xf8, 0xe8, 0xf6, 0xd4,
-	0x45, 0x0f, 0x46, 0xd7, 0xb9, 0xec, 0xb1, 0x9f, 0xa0, 0x2d, 0x66, 0x86, 0x94, 0x28, 0x7b, 0xb3,
+	0xe9, 0x41, 0xd3, 0xc8, 0x97, 0x1c, 0x7b, 0xe9, 0xb5, 0xed, 0xec, 0x2e, 0x40, 0x82, 0x52, 0x9c,
-	0xc8, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x37, 0x33, 0xbf, 0xf7, 0xde, 0x6f, 0x08, 0xc5, 0x13,
+	0xc9, 0xa1, 0x27, 0xec, 0xbe, 0xf7, 0xed, 0x5b, 0xec, 0xf7, 0xde, 0xfb, 0x76, 0xa1, 0x78, 0x42,
-	0x32, 0x7d, 0x34, 0xf5, 0x69, 0x48, 0x51, 0x81, 0x7f, 0x86, 0xd4, 0xd5, 0xee, 0xf9, 0x64, 0x4a,
+	0xa6, 0x8f, 0xa7, 0x3e, 0x0d, 0x29, 0x2a, 0xf0, 0xcf, 0x90, 0xba, 0xda, 0x7d, 0x9f, 0x4c, 0x69,
-	0x83, 0xc7, 0x7c, 0x7e, 0x32, 0x1b, 0x3d, 0x1e, 0xd3, 0x31, 0xe5, 0x13, 0x3e, 0x12, 0xf0, 0xda,
+	0xf0, 0x84, 0xcf, 0x4f, 0x66, 0xa3, 0x27, 0x63, 0x3a, 0xa6, 0x7c, 0xc2, 0x47, 0x02, 0x5e, 0x9b,
-	0x14, 0xb2, 0x4f, 0x89, 0xeb, 0x52, 0x54, 0x85, 0x92, 0x4d, 0xce, 0x9c, 0x21, 0x31, 0x3d, 0x6b,
+	0x42, 0xf6, 0x19, 0x71, 0x5d, 0x8a, 0xaa, 0x50, 0xb2, 0xc9, 0x99, 0x33, 0x24, 0xa6, 0x67, 0x4d,
-	0x42, 0x54, 0x49, 0x97, 0xea, 0x45, 0x0c, 0xc2, 0xd4, 0xb5, 0x26, 0x84, 0x01, 0x86, 0xae, 0x43,
+	0x88, 0x2a, 0xe9, 0x52, 0xbd, 0x88, 0x41, 0x98, 0xba, 0xd6, 0x84, 0x30, 0xc0, 0xd0, 0x75, 0x88,
-	0xbc, 0x50, 0x00, 0xd2, 0x02, 0x20, 0x4c, 0x1c, 0xf0, 0x00, 0xb6, 0x23, 0xc0, 0x19, 0xf1, 0x03,
+	0x17, 0x0a, 0x40, 0x5a, 0x00, 0x84, 0x89, 0x03, 0x1e, 0xc2, 0x76, 0x04, 0x38, 0x23, 0x7e, 0xe0,
-	0x87, 0x7a, 0x6a, 0x86, 0x63, 0xb6, 0x84, 0xf5, 0xb9, 0x30, 0xd6, 0x02, 0xc8, 0x3d, 0x25, 0x96,
+	0x50, 0x4f, 0xcd, 0x70, 0xcc, 0x96, 0xb0, 0xbe, 0x10, 0xc6, 0x5a, 0x00, 0xb9, 0x67, 0xc4, 0xb2,
-	0x4d, 0x7c, 0xf4, 0x31, 0xc8, 0xe1, 0x7c, 0x2a, 0xf6, 0xda, 0xfe, 0xf4, 0xd6, 0xa3, 0xf8, 0xe4,
+	0x89, 0x8f, 0x3e, 0x02, 0x39, 0x9c, 0x4f, 0xc5, 0x5e, 0xdb, 0x9f, 0xdc, 0x7a, 0x1c, 0xff, 0xf9,
-	0x8f, 0x9e, 0x91, 0x20, 0xb0, 0xc6, 0x64, 0x30, 0x9f, 0x12, 0xcc, 0x21, 0xe8, 0xd7, 0x50, 0x1a,
+	0xe3, 0xe7, 0x24, 0x08, 0xac, 0x31, 0x19, 0xcc, 0xa7, 0x04, 0x73, 0x08, 0xfa, 0x35, 0x94, 0x86,
-	0xd2, 0xc9, 0xd4, 0x27, 0x01, 0x0f, 0x9c, 0xe6, 0x2b, 0xee, 0x5c, 0x5b, 0xd1, 0x5a, 0x63, 0x70,
+	0x74, 0x32, 0xf5, 0x49, 0xc0, 0x03, 0xa7, 0xf9, 0x8a, 0x3b, 0xd7, 0x56, 0xb4, 0xd6, 0x18, 0x9c,
-	0x72, 0x41, 0xad, 0x01, 0x5b, 0x2d, 0x77, 0x16, 0x84, 0xc4, 0x6f, 0x51, 0x6f, 0xe4, 0x8c, 0xd1,
+	0x5c, 0x50, 0x6b, 0xc0, 0x56, 0xcb, 0x9d, 0x05, 0x21, 0xf1, 0x5b, 0xd4, 0x1b, 0x39, 0x63, 0xf4,
-	0x13, 0xc8, 0x8f, 0xa8, 0x6b, 0x13, 0x3f, 0x50, 0x25, 0x3d, 0x53, 0x2f, 0x7d, 0xaa, 0xac, 0x83,
+	0x14, 0xf2, 0x23, 0xea, 0xda, 0xc4, 0x0f, 0x54, 0x49, 0xcf, 0xd4, 0x4b, 0x9f, 0x28, 0xeb, 0x60,
-	0xed, 0x73, 0x47, 0x53, 0x7e, 0x7d, 0x51, 0x4d, 0xe1, 0x18, 0x56, 0xfb, 0x73, 0x1a, 0x72, 0xc2,
+	0xfb, 0xdc, 0xd1, 0x94, 0xdf, 0x5c, 0x54, 0x53, 0x38, 0x86, 0xd5, 0xfe, 0x9c, 0x86, 0x9c, 0xf0,
-	0x83, 0x76, 0x21, 0xed, 0xd8, 0x82, 0xa2, 0x66, 0xee, 0xf2, 0xa2, 0x9a, 0xee, 0xb4, 0x71, 0xda,
+	0xa0, 0x5d, 0x48, 0x3b, 0xb6, 0xa0, 0xa8, 0x99, 0xbb, 0xbc, 0xa8, 0xa6, 0x3b, 0x6d, 0x9c, 0x76,
-	0xb1, 0xd1, 0x4d, 0xc8, 0xba, 0xd6, 0x09, 0x71, 0x23, 0x72, 0xc4, 0x04, 0xdd, 0x86, 0xa2, 0x4f,
+	0x6c, 0x74, 0x13, 0xb2, 0xae, 0x75, 0x42, 0xdc, 0x88, 0x1c, 0x31, 0x41, 0xb7, 0xa1, 0xe8, 0x13,
-	0x2c, 0xdb, 0xa4, 0x9e, 0x3b, 0xe7, 0x94, 0x14, 0x70, 0x81, 0x19, 0x7a, 0x9e, 0x3b, 0x47, 0x3f,
+	0xcb, 0x36, 0xa9, 0xe7, 0xce, 0x39, 0x25, 0x05, 0x5c, 0x60, 0x86, 0x9e, 0xe7, 0xce, 0xd1, 0x8f,
-	0x02, 0xe4, 0x8c, 0x3d, 0xea, 0x13, 0x73, 0x4a, 0xfc, 0x89, 0xc3, 0x4f, 0x1b, 0xa8, 0x32, 0x47,
+	0x00, 0x39, 0x63, 0x8f, 0xfa, 0xc4, 0x9c, 0x12, 0x7f, 0xe2, 0xf0, 0xbf, 0x0d, 0x54, 0x99, 0xa3,
-	0xed, 0x08, 0xcf, 0xd1, 0xda, 0x81, 0xee, 0xc1, 0x56, 0x04, 0xb7, 0x89, 0x4b, 0x42, 0xa2, 0x66,
+	0x76, 0x84, 0xe7, 0x68, 0xed, 0x40, 0xf7, 0x61, 0x2b, 0x82, 0xdb, 0xc4, 0x25, 0x21, 0x51, 0xb3,
-	0x39, 0xb2, 0x2c, 0x8c, 0x6d, 0x6e, 0x43, 0x4f, 0xe0, 0xa6, 0xed, 0x04, 0xd6, 0x89, 0x4b, 0xcc,
+	0x1c, 0x59, 0x16, 0xc6, 0x36, 0xb7, 0xa1, 0xa7, 0x70, 0xd3, 0x76, 0x02, 0xeb, 0xc4, 0x25, 0x66,
-	0x90, 0x4c, 0xa6, 0xa6, 0xe3, 0xd9, 0xe4, 0x15, 0x09, 0xd4, 0x1c, 0xc7, 0xa2, 0xc8, 0x37, 0x20,
+	0x48, 0x26, 0x53, 0xd3, 0xf1, 0x6c, 0xf2, 0x9a, 0x04, 0x6a, 0x8e, 0x63, 0x51, 0xe4, 0x1b, 0x90,
-	0x93, 0x69, 0x47, 0x78, 0xd0, 0x2e, 0xe4, 0xa6, 0xd6, 0x2c, 0x20, 0xb6, 0x9a, 0xe7, 0x98, 0x68,
+	0xc9, 0xb4, 0x23, 0x3c, 0x68, 0x17, 0x72, 0x53, 0x6b, 0x16, 0x10, 0x5b, 0xcd, 0x73, 0x4c, 0x34,
-	0xc6, 0x58, 0x12, 0x15, 0x10, 0xa8, 0xca, 0x55, 0x96, 0xda, 0xdc, 0x11, 0xb3, 0x14, 0xc1, 0x6a,
+	0x63, 0x2c, 0x89, 0x0a, 0x08, 0x54, 0xe5, 0x2a, 0x4b, 0x6d, 0xee, 0x88, 0x59, 0x8a, 0x60, 0xb5,
-	0xff, 0x4e, 0x43, 0x4e, 0x78, 0xd0, 0x47, 0x2b, 0x96, 0xca, 0xcd, 0x5d, 0x86, 0xfa, 0xe7, 0x45,
+	0x7f, 0xa5, 0x21, 0x27, 0x3c, 0xe8, 0xc3, 0x15, 0x4b, 0xe5, 0xe6, 0x2e, 0x43, 0xfd, 0xe3, 0xa2,
-	0xb5, 0x20, 0x7c, 0x9d, 0x76, 0x82, 0x35, 0x04, 0x72, 0xa2, 0xa2, 0xf8, 0x18, 0xdd, 0x81, 0xa2,
+	0x5a, 0x10, 0xbe, 0x4e, 0x3b, 0xc1, 0x1a, 0x02, 0x39, 0x51, 0x51, 0x7c, 0x8c, 0xee, 0x40, 0xd1,
-	0x65, 0xdb, 0x2c, 0x7b, 0x24, 0x50, 0x33, 0x7a, 0xa6, 0x5e, 0xc4, 0x6b, 0x03, 0xfa, 0xd9, 0x66,
+	0xb2, 0x6d, 0x96, 0x3d, 0x12, 0xa8, 0x19, 0x3d, 0x53, 0x2f, 0xe2, 0xb5, 0x01, 0xfd, 0x6c, 0xb3,
-	0x35, 0xc8, 0x57, 0xeb, 0xe7, 0x5d, 0x65, 0xc0, 0x52, 0x31, 0x24, 0x7e, 0x54, 0xc1, 0x59, 0xbe,
+	0x1a, 0xe4, 0xab, 0xf5, 0xf3, 0xbe, 0x32, 0x60, 0xa9, 0x18, 0x12, 0x3f, 0xaa, 0xe0, 0x2c, 0xdf,
-	0x5f, 0x81, 0x19, 0x78, 0xfd, 0xde, 0x85, 0xf2, 0xc4, 0x7a, 0x65, 0x06, 0xe4, 0x0f, 0x33, 0xe2,
+	0xaf, 0xc0, 0x0c, 0xbc, 0x7e, 0xef, 0x41, 0x79, 0x62, 0xbd, 0x36, 0x03, 0xf2, 0x87, 0x19, 0xf1,
-	0x0d, 0x09, 0xa7, 0x2b, 0x83, 0x4b, 0x13, 0xeb, 0x55, 0x3f, 0x32, 0xa1, 0x0a, 0x80, 0xe3, 0x85,
+	0x86, 0x84, 0xd3, 0x95, 0xc1, 0xa5, 0x89, 0xf5, 0xba, 0x1f, 0x99, 0x50, 0x05, 0xc0, 0xf1, 0x42,
-	0x3e, 0xb5, 0x67, 0x43, 0xe2, 0x47, 0x5c, 0x25, 0x2c, 0xe8, 0x27, 0x50, 0xe0, 0x64, 0x9b, 0x8e,
+	0x9f, 0xda, 0xb3, 0x21, 0xf1, 0x23, 0xae, 0x12, 0x16, 0xf4, 0x13, 0x28, 0x70, 0xb2, 0x4d, 0xc7,
-	0xad, 0x16, 0x74, 0xa9, 0x2e, 0x37, 0xb5, 0xe8, 0xe2, 0x79, 0x4e, 0x35, 0xbf, 0x77, 0x3c, 0xc4,
+	0x56, 0x0b, 0xba, 0x54, 0x97, 0x9b, 0x5a, 0x74, 0xf0, 0x3c, 0xa7, 0x9a, 0x9f, 0x3b, 0x1e, 0xe2,
-	0x79, 0x8e, 0xed, 0xd8, 0xe8, 0x97, 0xa0, 0x05, 0x2f, 0x1d, 0x96, 0x28, 0x11, 0x29, 0x74, 0xa8,
+	0x3c, 0xc7, 0x76, 0x6c, 0xf4, 0x4b, 0xd0, 0x82, 0x57, 0x0e, 0x4b, 0x94, 0x88, 0x14, 0x3a, 0xd4,
-	0x67, 0xfa, 0x64, 0x42, 0xcf, 0x2c, 0x37, 0x50, 0x8b, 0x7c, 0x1b, 0x95, 0x21, 0x3a, 0x09, 0x00,
+	0x33, 0x7d, 0x32, 0xa1, 0x67, 0x96, 0x1b, 0xa8, 0x45, 0xbe, 0x8d, 0xca, 0x10, 0x9d, 0x04, 0x00,
-	0x8e, 0xfc, 0xb5, 0x1e, 0x64, 0x79, 0x44, 0x96, 0x45, 0x51, 0xac, 0x51, 0xf7, 0x46, 0x33, 0xf4,
+	0x47, 0xfe, 0x5a, 0x0f, 0xb2, 0x3c, 0x22, 0xcb, 0xa2, 0x28, 0xd6, 0xa8, 0x7b, 0xa3, 0x19, 0x7a,
-	0x08, 0xb2, 0x23, 0xc7, 0x25, 0x81, 0x9a, 0xe6, 0x39, 0x44, 0x89, 0x4a, 0x77, 0x5c, 0xd2, 0xf1,
+	0x0c, 0xd9, 0x91, 0xe3, 0x92, 0x40, 0x4d, 0xf3, 0x1c, 0xa2, 0x44, 0xa5, 0x3b, 0x2e, 0xe9, 0x78,
-	0x46, 0x34, 0xca, 0xa2, 0x80, 0xd5, 0x8e, 0xa1, 0xc4, 0x03, 0x1e, 0x4f, 0x6d, 0x2b, 0x24, 0xff,
+	0x23, 0x1a, 0x65, 0x51, 0xc0, 0x6a, 0xc7, 0x50, 0xe2, 0x01, 0x8f, 0xa7, 0xb6, 0x15, 0x92, 0xff,
-	0xb7, 0xb0, 0xff, 0x95, 0xa1, 0x10, 0x7b, 0x56, 0x49, 0x97, 0x12, 0x49, 0x47, 0x20, 0x07, 0xce,
+	0x5b, 0xd8, 0xff, 0xca, 0x50, 0x88, 0x3d, 0xab, 0xa4, 0x4b, 0x89, 0xa4, 0x23, 0x90, 0x03, 0xe7,
-	0x57, 0x84, 0xf7, 0x48, 0x06, 0xf3, 0x31, 0xfa, 0x10, 0x60, 0x42, 0x6d, 0x67, 0xe4, 0x10, 0xdb,
+	0x4b, 0xc2, 0x7b, 0x24, 0x83, 0xf9, 0x18, 0xdd, 0x05, 0x98, 0x50, 0xdb, 0x19, 0x39, 0xc4, 0x36,
-	0x0c, 0x78, 0xca, 0x32, 0xb8, 0x18, 0x5b, 0xfa, 0xe8, 0x09, 0x94, 0x56, 0xee, 0x93, 0xb9, 0x5a,
+	0x03, 0x9e, 0xb2, 0x0c, 0x2e, 0xc6, 0x96, 0x3e, 0x7a, 0x0a, 0xa5, 0x95, 0xfb, 0x64, 0xae, 0x96,
-	0xe6, 0x9c, 0x7f, 0x10, 0x73, 0xde, 0x3f, 0xa5, 0x7e, 0xd8, 0x69, 0xe3, 0x55, 0x88, 0xe6, 0x9c,
+	0x39, 0xe7, 0x1f, 0xc4, 0x9c, 0xf7, 0x4f, 0xa9, 0x1f, 0x76, 0xda, 0x78, 0x15, 0xa2, 0x39, 0x67,
-	0x95, 0x74, 0x2c, 0x4f, 0x8c, 0xd8, 0x8d, 0x92, 0x7e, 0x4e, 0x86, 0x21, 0x5d, 0x35, 0x7e, 0x04,
+	0x25, 0x1d, 0xcb, 0x13, 0x23, 0x76, 0xa3, 0xa4, 0x5f, 0x90, 0x61, 0x48, 0x57, 0x8d, 0x1f, 0xc1,
-	0x43, 0x1a, 0x14, 0x56, 0x35, 0x01, 0xfc, 0x00, 0xab, 0x39, 0xfa, 0x31, 0xe4, 0x4e, 0x5c, 0x3a,
+	0x90, 0x06, 0x85, 0x55, 0x4d, 0x00, 0xff, 0x81, 0xd5, 0x1c, 0xfd, 0x18, 0x72, 0x27, 0x2e, 0x1d,
-	0x7c, 0x19, 0xf7, 0xc7, 0x8d, 0x75, 0xb0, 0x26, 0xb3, 0x27, 0x58, 0x88, 0x80, 0x4c, 0x26, 0x83,
+	0xbe, 0x8a, 0xfb, 0xe3, 0xc6, 0x3a, 0x58, 0x93, 0xd9, 0x13, 0x2c, 0x44, 0x40, 0x26, 0x93, 0xc1,
-	0xf9, 0xc4, 0x75, 0xbc, 0x97, 0x66, 0x68, 0xf9, 0x63, 0x12, 0xaa, 0x3b, 0x42, 0x26, 0x23, 0xeb,
+	0x7c, 0xe2, 0x3a, 0xde, 0x2b, 0x33, 0xb4, 0xfc, 0x31, 0x09, 0xd5, 0x1d, 0x21, 0x93, 0x91, 0x75,
-	0x80, 0x1b, 0x99, 0xdc, 0x8a, 0x05, 0xe6, 0xa9, 0x15, 0x9c, 0xaa, 0x88, 0xb5, 0x11, 0x06, 0x61,
+	0xc0, 0x8d, 0x4c, 0x6e, 0xc5, 0x02, 0xf3, 0xd4, 0x0a, 0x4e, 0x55, 0xc4, 0xda, 0x08, 0x83, 0x30,
-	0x7a, 0x6a, 0x05, 0xa7, 0x68, 0x2f, 0x52, 0x4f, 0xa1, 0x85, 0xbb, 0xd7, 0xd9, 0x4f, 0xc8, 0xa7,
+	0x3d, 0xb3, 0x82, 0x53, 0xb4, 0x17, 0xa9, 0xa7, 0xd0, 0xc2, 0xdd, 0xeb, 0xec, 0x27, 0xe4, 0x53,
-	0x0e, 0xa5, 0xab, 0xf2, 0xb2, 0x85, 0x93, 0x26, 0xb6, 0xdd, 0x8a, 0x48, 0x2f, 0x50, 0x4b, 0xba,
+	0x87, 0xd2, 0x55, 0x79, 0xd9, 0xc2, 0x49, 0x13, 0xdb, 0x6e, 0x45, 0xa4, 0x17, 0xa8, 0x25, 0x5d,
-	0x54, 0xcf, 0xae, 0x79, 0xeb, 0x06, 0xe8, 0x31, 0x88, 0xcd, 0x4d, 0x9e, 0xa2, 0x2d, 0xe6, 0x6f,
+	0xaa, 0x67, 0xd7, 0xbc, 0x75, 0x03, 0xf4, 0x04, 0xc4, 0xe6, 0x26, 0x4f, 0xd1, 0x16, 0xf3, 0x37,
-	0x2a, 0x97, 0x17, 0xd5, 0x32, 0xb6, 0xce, 0xf9, 0x55, 0xfb, 0xce, 0x57, 0x04, 0x17, 0x4f, 0xe2,
+	0x95, 0xcb, 0x8b, 0x6a, 0x19, 0x5b, 0xe7, 0xfc, 0xa8, 0x7d, 0xe7, 0x4b, 0x82, 0x8b, 0x27, 0xf1,
-	0x21, 0xdb, 0xd3, 0xa5, 0x43, 0xcb, 0x35, 0x47, 0xae, 0x35, 0x0e, 0xd4, 0x6f, 0xf3, 0x7c, 0x53,
+	0x90, 0xed, 0xe9, 0xd2, 0xa1, 0xe5, 0x9a, 0x23, 0xd7, 0x1a, 0x07, 0xea, 0x37, 0x79, 0xbe, 0x29,
-	0xe0, 0xb6, 0x7d, 0x66, 0x42, 0x2a, 0x53, 0x17, 0xa6, 0x58, 0x76, 0x24, 0x4d, 0xf1, 0x14, 0xd5,
+	0x70, 0xdb, 0x3e, 0x33, 0x21, 0x95, 0xa9, 0x0b, 0x53, 0x2c, 0x3b, 0x92, 0xa6, 0x78, 0x8a, 0xea,
-	0x21, 0xef, 0x78, 0x67, 0x96, 0xeb, 0x44, 0x82, 0xd4, 0xdc, 0xbe, 0xbc, 0xa8, 0x02, 0xb6, 0xce,
+	0x90, 0x77, 0xbc, 0x33, 0xcb, 0x75, 0x22, 0x41, 0x6a, 0x6e, 0x5f, 0x5e, 0x54, 0x01, 0x5b, 0xe7,
-	0x3b, 0xc2, 0x8a, 0x63, 0x37, 0x63, 0xd3, 0xa3, 0x1b, 0xda, 0x59, 0xe0, 0xa1, 0xb6, 0x3c, 0x9a,
+	0x1d, 0x61, 0xc5, 0xb1, 0x9b, 0xb1, 0xe9, 0xd1, 0x0d, 0xed, 0x2c, 0xf0, 0x50, 0x5b, 0x1e, 0x4d,
-	0xd0, 0xcd, 0x5f, 0xc8, 0x7f, 0xfa, 0xba, 0x9a, 0xaa, 0x79, 0x50, 0x5c, 0x65, 0x85, 0x55, 0x1b,
+	0xe8, 0xe6, 0x2f, 0xe4, 0x3f, 0x7d, 0x55, 0x4d, 0xd5, 0x3c, 0x28, 0xae, 0xb2, 0xc2, 0xaa, 0x8d,
-	0x67, 0x36, 0xc3, 0x99, 0xe5, 0x63, 0x56, 0xea, 0x74, 0x34, 0x0a, 0x48, 0xc8, 0xeb, 0x32, 0x83,
+	0x33, 0x9b, 0xe1, 0xcc, 0xf2, 0x31, 0x2b, 0x75, 0x3a, 0x1a, 0x05, 0x24, 0xe4, 0x75, 0x99, 0xc1,
-	0xa3, 0xd9, 0xaa, 0x32, 0xd3, 0x9c, 0x16, 0x51, 0x99, 0xb7, 0xa1, 0x78, 0x4e, 0xac, 0x97, 0x22,
+	0xd1, 0x6c, 0x55, 0x99, 0x69, 0x4e, 0x8b, 0xa8, 0xcc, 0xdb, 0x50, 0x3c, 0x27, 0xd6, 0x2b, 0x91,
-	0x3d, 0x82, 0xd1, 0x02, 0x33, 0xb0, 0xe4, 0x44, 0xfb, 0xfd, 0x0a, 0x72, 0xa2, 0xa4, 0xd0, 0x67,
+	0x1e, 0xc1, 0x68, 0x81, 0x19, 0x58, 0x72, 0xa2, 0xfd, 0x7e, 0x05, 0x39, 0x51, 0x52, 0xe8, 0x53,
-	0x50, 0x18, 0xd2, 0x99, 0x17, 0xae, 0xdf, 0x9b, 0x9d, 0xa4, 0x5c, 0x71, 0x4f, 0x54, 0x27, 0x2b,
+	0x28, 0x0c, 0xe9, 0xcc, 0x0b, 0xd7, 0xf7, 0xcd, 0x4e, 0x52, 0xae, 0xb8, 0x27, 0xaa, 0x93, 0x15,
-	0x60, 0x6d, 0x1f, 0xf2, 0x91, 0x0b, 0x3d, 0x58, 0x69, 0xa9, 0xdc, 0xbc, 0x75, 0xa5, 0xbc, 0x37,
+	0xb0, 0xb6, 0x0f, 0xf9, 0xc8, 0x85, 0x1e, 0xae, 0xb4, 0x54, 0x6e, 0xde, 0xba, 0x52, 0xde, 0x9b,
-	0x1f, 0xa0, 0x33, 0xcb, 0x9d, 0x89, 0x83, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59,
+	0x17, 0xd0, 0x99, 0xe5, 0xce, 0xc4, 0x8f, 0xca, 0x58, 0x4c, 0x6a, 0x7f, 0x95, 0x20, 0x8f, 0x59,
-	0xc5, 0x06, 0x61, 0xe2, 0xe9, 0xca, 0x6e, 0x3c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc,
+	0xc5, 0x06, 0x61, 0xe2, 0xea, 0xca, 0x6e, 0x5c, 0x5d, 0xeb, 0x26, 0x4f, 0x6f, 0x34, 0x79, 0xdc,
-	0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x93, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c,
+	0xa7, 0x99, 0x44, 0x9f, 0xae, 0x59, 0x92, 0xbf, 0x95, 0xa5, 0x6c, 0x82, 0xa5, 0x98, 0xe5, 0x5c,
-	0x82, 0xe5, 0x07, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x3f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4,
+	0x82, 0xe5, 0x87, 0xb0, 0x3d, 0xf2, 0xe9, 0x84, 0x5f, 0x4e, 0xd4, 0xb7, 0xfc, 0x79, 0xa4, 0xa4,
 	0x5b, 0xcc, 0x3a, 0x88, 0x8d, 0x9b, 0x04, 0x17, 0x36, 0x09, 0xae, 0x99, 0x50, 0xc0, 0x24, 0x98,
-	0x52, 0x2f, 0x20, 0xef, 0xbc, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0xdf, 0xa8, 0x8c, 0xf9, 0x18,
+	0x52, 0x2f, 0x20, 0xef, 0x3d, 0x13, 0x02, 0xd9, 0xb6, 0x42, 0x8b, 0x9f, 0xa8, 0x8c, 0xf9, 0x18,
-	0x3d, 0x04, 0x79, 0x48, 0x6d, 0x71, 0x9f, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5,
+	0x3d, 0x02, 0x79, 0x48, 0x6d, 0x71, 0x9e, 0xed, 0x64, 0xbb, 0x1a, 0xbe, 0x4f, 0xfd, 0x16, 0xb5,
 	0x09, 0xe6, 0x80, 0xda, 0x14, 0x94, 0x36, 0x3d, 0xf7, 0x5c, 0x6a, 0xd9, 0x47, 0x3e, 0x1d, 0xb3,
-	0x17, 0xe4, 0x9d, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0xf7, 0x37, 0xbb, 0xf1,
+	0x1b, 0xe4, 0xbd, 0x4a, 0xd8, 0x86, 0xfc, 0x8c, 0x6b, 0x65, 0xac, 0x85, 0x0f, 0x36, 0xbb, 0xf1,
-	0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0x7f, 0x97, 0x40, 0x7b, 0x37, 0x1a, 0x75,
+	0x6a, 0x20, 0x21, 0xac, 0xb1, 0xce, 0x44, 0x4b, 0x6b, 0xff, 0x96, 0x40, 0x7b, 0x3f, 0x1a, 0x75,
-	0xa0, 0x24, 0x90, 0x66, 0xe2, 0xa7, 0xa9, 0xfe, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc,
+	0xa0, 0x24, 0x90, 0x66, 0xe2, 0xd1, 0x54, 0xff, 0x3e, 0x1b, 0x71, 0x21, 0x80, 0xd9, 0x6a, 0xfc,
-	0x9d, 0x2f, 0x6e, 0x42, 0x17, 0x33, 0xef, 0xa7, 0x8b, 0x0f, 0x61, 0x4b, 0x28, 0x42, 0xfc, 0x7f,
+	0xad, 0x37, 0x6e, 0x42, 0x17, 0x33, 0xdf, 0x4f, 0x17, 0x1f, 0xc1, 0x96, 0x50, 0x84, 0xf8, 0x7d,
-	0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x8c, 0xdb, 0x6b, 0x39,
+	0x21, 0xeb, 0x99, 0x7a, 0xb6, 0x99, 0x56, 0x52, 0xb8, 0x7c, 0x22, 0xda, 0x4c, 0xbc, 0x2e, 0xee,
-	0x90, 0x8f, 0x1c, 0x6f, 0x5c, 0xab, 0x42, 0xb6, 0xe5, 0x52, 0x9e, 0xb0, 0x9c, 0x4f, 0xac, 0x80,
+	0x6e, 0x48, 0x87, 0xa8, 0x8e, 0xb5, 0x50, 0xd4, 0x72, 0x20, 0x1f, 0x39, 0xde, 0xb8, 0x56, 0x85,
-	0x7a, 0x31, 0x8f, 0x62, 0xb6, 0xf7, 0xb7, 0x34, 0x94, 0x12, 0xff, 0x7e, 0xe8, 0x09, 0x6c, 0xb7,
+	0x6c, 0xcb, 0xa5, 0x3c, 0x9f, 0x39, 0x9f, 0x58, 0x01, 0xf5, 0x62, 0x9a, 0xc5, 0x6c, 0xef, 0x6f,
-	0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad, 0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b,
+	0x69, 0x28, 0x25, 0x9e, 0x86, 0xe8, 0x29, 0x6c, 0xb7, 0x0e, 0x8f, 0xfb, 0x03, 0x03, 0x9b, 0xad,
-	0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0xdf, 0xba, 0x2a, 0x64, 0x3b, 0xdd, 0xb6, 0xf1, 0x3b, 0x45,
+	0x5e, 0x77, 0xbf, 0x73, 0xa0, 0xa4, 0xb4, 0x3b, 0x8b, 0xa5, 0xae, 0x4e, 0xd6, 0xa0, 0xcd, 0x57,
-	0xd2, 0x6e, 0x2e, 0x96, 0xba, 0x92, 0x00, 0x8a, 0x37, 0xf2, 0x13, 0x28, 0x73, 0x80, 0x79, 0x7c,
+	0x5f, 0x15, 0xb2, 0x9d, 0x6e, 0xdb, 0xf8, 0x9d, 0x22, 0x69, 0x37, 0x17, 0x4b, 0x5d, 0x49, 0x00,
-	0xd4, 0x6e, 0x0c, 0x0c, 0x25, 0xad, 0x69, 0x8b, 0xa5, 0xbe, 0x7b, 0x15, 0x17, 0x71, 0x7e, 0x0f,
+	0xc5, 0x15, 0xfa, 0x31, 0x94, 0x39, 0xc0, 0x3c, 0x3e, 0x6a, 0x37, 0x06, 0x86, 0x92, 0xd6, 0xb4,
-	0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f, 0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18,
+	0xc5, 0x52, 0xdf, 0xbd, 0x8a, 0x8b, 0x52, 0x72, 0x1f, 0xf2, 0xd8, 0xf8, 0xed, 0xb1, 0xd1, 0x1f,
-	0xb7, 0xd4, 0x03, 0x28, 0x60, 0xa3, 0x7f, 0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58,
+	0x28, 0x19, 0x6d, 0x77, 0xb1, 0xd4, 0x51, 0x02, 0x18, 0x77, 0xdc, 0x43, 0x28, 0x60, 0xa3, 0x7f,
-	0xea, 0x37, 0x36, 0x50, 0x51, 0x95, 0xfe, 0x14, 0x76, 0xda, 0xbd, 0x2f, 0xba, 0x87, 0xbd, 0x46,
+	0xd4, 0xeb, 0xf6, 0x0d, 0x45, 0xd6, 0x7e, 0xb0, 0x58, 0xea, 0x37, 0x36, 0x50, 0x51, 0x11, 0xff,
-	0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4, 0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04,
+	0x14, 0x76, 0xda, 0xbd, 0xcf, 0xbb, 0x87, 0xbd, 0x46, 0xdb, 0x3c, 0xc2, 0xbd, 0x03, 0x6c, 0xf4,
-	0xfe, 0x5a, 0xd1, 0x7d, 0x08, 0xf2, 0x51, 0xa7, 0x7b, 0xa0, 0xe4, 0xb4, 0x1b, 0x8b, 0xa5, 0xfe,
+	0xfb, 0x4a, 0x56, 0xab, 0x2e, 0x96, 0xfa, 0xed, 0x04, 0xfe, 0x5a, 0x4d, 0xde, 0x05, 0xf9, 0xa8,
-	0x41, 0x02, 0xca, 0x48, 0x65, 0x37, 0x6e, 0x1d, 0xf6, 0xfa, 0x86, 0x92, 0xbf, 0x76, 0x63, 0x4e,
+	0xd3, 0x3d, 0x50, 0x72, 0xda, 0x8d, 0xc5, 0x52, 0xff, 0x20, 0x01, 0x65, 0xa4, 0xb2, 0x13, 0xb7,
-	0xf6, 0xde, 0xef, 0x01, 0x5d, 0xff, 0x3b, 0x46, 0xf7, 0x41, 0xee, 0xf6, 0xba, 0x86, 0x92, 0x12,
+	0x0e, 0x7b, 0x7d, 0x43, 0xc9, 0x5f, 0x3b, 0x31, 0x27, 0x7b, 0xef, 0xf7, 0x80, 0xae, 0x3f, 0x9e,
-	0xf7, 0xbf, 0x8e, 0xe8, 0x52, 0x8f, 0xa0, 0x1a, 0x64, 0x0e, 0xbf, 0xfc, 0x5c, 0x91, 0xb4, 0x1f,
+	0xd1, 0x03, 0x90, 0xbb, 0xbd, 0xae, 0xa1, 0xa4, 0xc4, 0xf9, 0xaf, 0x23, 0xba, 0xd4, 0x23, 0xa8,
-	0x2e, 0x96, 0xfa, 0xad, 0xeb, 0xa0, 0xc3, 0x2f, 0x3f, 0xdf, 0xa3, 0x50, 0x4a, 0x06, 0xae, 0x41,
+	0x06, 0x99, 0xc3, 0x2f, 0x3e, 0x53, 0x24, 0xed, 0x87, 0x8b, 0xa5, 0x7e, 0xeb, 0x3a, 0xe8, 0xf0,
-	0xe1, 0x99, 0x31, 0x68, 0xb4, 0x1b, 0x83, 0x86, 0x92, 0x12, 0x47, 0x8a, 0xdd, 0xcf, 0x48, 0x68,
+	0x8b, 0xcf, 0xf6, 0x28, 0x94, 0x92, 0x81, 0x6b, 0x50, 0x78, 0x6e, 0x0c, 0x1a, 0xed, 0xc6, 0xa0,
-	0xf1, 0x26, 0xbc, 0x03, 0xd9, 0xae, 0xf1, 0xdc, 0xc0, 0x8a, 0xa4, 0xed, 0x2c, 0x96, 0xfa, 0x56,
+	0xa1, 0xa4, 0xc4, 0x2f, 0xc5, 0xee, 0xe7, 0x24, 0xb4, 0x78, 0x8f, 0xde, 0x81, 0x6c, 0xd7, 0x78,
-	0x0c, 0xe8, 0x92, 0x33, 0xe2, 0xa3, 0x0a, 0xe4, 0x1a, 0x87, 0x5f, 0x34, 0x5e, 0xf4, 0x95, 0xb4,
+	0x61, 0x60, 0x45, 0xd2, 0x76, 0x16, 0x4b, 0x7d, 0x2b, 0x06, 0x74, 0xc9, 0x19, 0xf1, 0x51, 0x05,
-	0x86, 0x16, 0x4b, 0x7d, 0x3b, 0x76, 0x37, 0xdc, 0x73, 0x6b, 0x1e, 0xec, 0xfd, 0x47, 0x82, 0x72,
+	0x72, 0x8d, 0xc3, 0xcf, 0x1b, 0x2f, 0xfb, 0x4a, 0x5a, 0x43, 0x8b, 0xa5, 0xbe, 0x1d, 0xbb, 0x1b,
-	0xf2, 0x8d, 0x43, 0x15, 0x90, 0xf7, 0x3b, 0x87, 0x46, 0xbc, 0x5d, 0xd2, 0xc7, 0xc6, 0xa8, 0x0e,
+	0xee, 0xb9, 0x35, 0x0f, 0xf6, 0xfe, 0x23, 0x41, 0x39, 0x79, 0x05, 0xa2, 0x0a, 0xc8, 0xfb, 0x9d,
-	0xc5, 0x76, 0x07, 0x1b, 0xad, 0x41, 0x0f, 0xbf, 0x88, 0xef, 0x92, 0x04, 0xb5, 0x1d, 0x9f, 0x17,
+	0x43, 0x23, 0xde, 0x2e, 0xe9, 0x63, 0x63, 0x54, 0x87, 0x62, 0xbb, 0x83, 0x8d, 0xd6, 0xa0, 0x87,
-	0xf8, 0x1c, 0xfd, 0x1c, 0xca, 0xfd, 0x17, 0xcf, 0x0e, 0x3b, 0xdd, 0xdf, 0x98, 0x3c, 0x62, 0x5a,
+	0x5f, 0xc6, 0x67, 0x49, 0x82, 0xda, 0x8e, 0xcf, 0xeb, 0x7f, 0x8e, 0x7e, 0x0e, 0xe5, 0xfe, 0xcb,
-	0x7b, 0xb8, 0x58, 0xea, 0x77, 0x37, 0xc0, 0x64, 0xea, 0x93, 0xa1, 0x15, 0x12, 0xbb, 0x2f, 0xde,
+	0xe7, 0x87, 0x9d, 0xee, 0x6f, 0x4c, 0x1e, 0x31, 0xad, 0x3d, 0x5a, 0x2c, 0xf5, 0x7b, 0x1b, 0x60,
-	0x6b, 0xe6, 0x2c, 0x48, 0xa8, 0x05, 0x3b, 0xf1, 0xd2, 0xf5, 0x66, 0x19, 0xed, 0x93, 0xc5, 0x52,
+	0x32, 0xf5, 0xc9, 0xd0, 0x0a, 0x89, 0xdd, 0x17, 0xd7, 0x39, 0x73, 0x16, 0x24, 0xd4, 0x82, 0x9d,
-	0xff, 0xe8, 0x7b, 0xd7, 0xaf, 0x76, 0x2f, 0x48, 0xe8, 0x3e, 0xe4, 0xa3, 0x20, 0x71, 0x25, 0x25,
+	0x78, 0xe9, 0x7a, 0xb3, 0x8c, 0xf6, 0xf1, 0x62, 0xa9, 0x7f, 0xf8, 0x9d, 0xeb, 0x57, 0xbb, 0x17,
-	0x97, 0x46, 0x0b, 0xf6, 0xfe, 0x22, 0x41, 0x71, 0x25, 0x57, 0x8c, 0xf0, 0x6e, 0xcf, 0x34, 0x30,
+	0x24, 0xf4, 0x00, 0xf2, 0x51, 0x90, 0xb8, 0x92, 0x92, 0x4b, 0xa3, 0x05, 0x7b, 0x7f, 0x91, 0xa0,
-	0xee, 0xe1, 0x98, 0x81, 0x95, 0xb3, 0x4b, 0xf9, 0x10, 0xdd, 0x85, 0xfc, 0x81, 0xd1, 0x35, 0x70,
+	0xb8, 0x52, 0x33, 0x46, 0x78, 0xb7, 0x67, 0x1a, 0x18, 0xf7, 0x70, 0xcc, 0xc0, 0xca, 0xd9, 0xa5,
-	0xa7, 0x15, 0x37, 0xc6, 0x0a, 0x72, 0x40, 0x3c, 0xe2, 0x3b, 0x43, 0xf4, 0x31, 0x94, 0xbb, 0x3d,
+	0x7c, 0x88, 0xee, 0x41, 0xfe, 0xc0, 0xe8, 0x1a, 0xb8, 0xd3, 0x8a, 0x1b, 0x63, 0x05, 0x39, 0x20,
-	0xb3, 0x7f, 0xdc, 0x7a, 0x1a, 0x5f, 0x9d, 0xef, 0x9f, 0x08, 0xd5, 0x9f, 0x0d, 0x4f, 0x39, 0x9f,
+	0x1e, 0xf1, 0x9d, 0x21, 0xfa, 0x08, 0xca, 0xdd, 0x9e, 0xd9, 0x3f, 0x6e, 0x3d, 0x8b, 0x8f, 0xce,
-	0x7b, 0xac, 0x87, 0x9e, 0x37, 0x0e, 0x3b, 0x6d, 0x01, 0xcd, 0x68, 0xea, 0x62, 0xa9, 0xdf, 0x5c,
+	0xf7, 0x4f, 0x84, 0xea, 0xcf, 0x86, 0xa7, 0x9c, 0xcf, 0x3d, 0xd6, 0x43, 0x2f, 0x1a, 0x87, 0x9d,
-	0x41, 0xa3, 0x47, 0x9a, 0x61, 0xf7, 0x6c, 0xa8, 0x7c, 0xbf, 0x30, 0x21, 0x1d, 0x72, 0x8d, 0xa3,
+	0xb6, 0x80, 0x66, 0x34, 0x75, 0xb1, 0xd4, 0x6f, 0xae, 0xa0, 0xd1, 0x1d, 0xce, 0xb0, 0x7b, 0x36,
-	0x23, 0xa3, 0xdb, 0x8e, 0x4f, 0xbf, 0xf6, 0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3,
+	0x54, 0xbe, 0x5b, 0xb7, 0x90, 0x0e, 0xb9, 0xc6, 0xd1, 0x91, 0xd1, 0x6d, 0xc7, 0x7f, 0xbf, 0xf6,
-	0x07, 0xc6, 0x20, 0x3e, 0xfc, 0x1a, 0xb1, 0x4f, 0xd9, 0xcf, 0x52, 0xb3, 0xfe, 0xfa, 0x9b, 0x4a,
+	0x35, 0xa6, 0x53, 0xe2, 0xd9, 0x0c, 0xb1, 0xdf, 0xc3, 0x07, 0xc6, 0x20, 0xfe, 0xf9, 0x35, 0x62,
-	0xea, 0xcd, 0x37, 0x95, 0xd4, 0xeb, 0xcb, 0x8a, 0xf4, 0xe6, 0xb2, 0x22, 0xfd, 0xeb, 0xb2, 0x92,
+	0x9f, 0xb2, 0xb7, 0x54, 0xb3, 0xfe, 0xe6, 0xeb, 0x4a, 0xea, 0xed, 0xd7, 0x95, 0xd4, 0x9b, 0xcb,
-	0xfa, 0xf6, 0xb2, 0x22, 0xfd, 0xf1, 0x6d, 0x25, 0xf5, 0xf5, 0xdb, 0x8a, 0xf4, 0xe6, 0x6d, 0x25,
+	0x8a, 0xf4, 0xf6, 0xb2, 0x22, 0xfd, 0xf3, 0xb2, 0x92, 0xfa, 0xe6, 0xb2, 0x22, 0xfd, 0xf1, 0x5d,
-	0xf5, 0x8f, 0xb7, 0x95, 0xd4, 0x49, 0x8e, 0x8b, 0xda, 0x67, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff,
+	0x25, 0xf5, 0xd5, 0xbb, 0x8a, 0xf4, 0xf6, 0x5d, 0x25, 0xf5, 0xf7, 0x77, 0x95, 0xd4, 0x49, 0x8e,
-	0x39, 0xd2, 0xc3, 0x6e, 0x32, 0x0f, 0x00, 0x00,
+	0x6b, 0xde, 0xa7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x75, 0x30, 0x0d, 0x51, 0x0f, 0x00,
+	0x00,
 }
 
 func (m *Hello) Marshal() (dAtA []byte, err error) {
@@ -1878,6 +1880,11 @@ func (m *FileDownloadProgressUpdate) MarshalToSizedBuffer(dAtA []byte) (int, err
 	_ = i
 	var l int
 	_ = l
+	if m.BlockSize != 0 {
+		i = encodeVarintBep(dAtA, i, uint64(m.BlockSize))
+		i--
+		dAtA[i] = 0x28
+	}
 	if len(m.BlockIndexes) > 0 {
 		for iNdEx := len(m.BlockIndexes) - 1; iNdEx >= 0; iNdEx-- {
 			i = encodeVarintBep(dAtA, i, uint64(m.BlockIndexes[iNdEx]))
@@ -2352,6 +2359,9 @@ func (m *FileDownloadProgressUpdate) ProtoSize() (n int) {
 			n += 1 + sovBep(uint64(e))
 		}
 	}
+	if m.BlockSize != 0 {
+		n += 1 + sovBep(uint64(m.BlockSize))
+	}
 	return n
 }
 
@@ -4929,6 +4939,25 @@ func (m *FileDownloadProgressUpdate) Unmarshal(dAtA []byte) error {
 			} else {
 				return fmt.Errorf("proto: wrong wireType = %d for field BlockIndexes", wireType)
 			}
+		case 5:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field BlockSize", wireType)
+			}
+			m.BlockSize = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBep
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.BlockSize |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipBep(dAtA[iNdEx:])

lib/protocol/bep.proto

@@ -190,6 +190,7 @@ message FileDownloadProgressUpdate {
     string                         name          = 2;
     Vector                         version       = 3 [(gogoproto.nullable) = false];
     repeated int32                 block_indexes = 4 [packed=false];
+    int32                          block_size    = 5;
 }
 
 enum FileDownloadProgressUpdateType {