Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model: Compare all items with global on scan (fixes #7740) (#7791)

This commit is contained in:
Simon Frei 2021-06-27 08:48:54 +02:00 committed by GitHub
parent 69ce121267
commit 445a82f120
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 88 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
lib/model/folder.go
View File

@ -473,16 +473,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
f.setState(FolderScanning) f.setState(FolderScanning)
f.clearScanErrors(subDirs) f.clearScanErrors(subDirs)
batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { batch := f.newScanBatch()
if err := f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
return err
}
f.updateLocalsFromScanning(fs)
return nil
})
batchAppend := f.scanSubdirsBatchAppendFunc(batch)
// Schedule a pull after scanning, but only if we actually detected any // Schedule a pull after scanning, but only if we actually detected any
// changes. // changes.
@ -494,7 +485,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
} }
}() }()
changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch, batchAppend) changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch)
changes += changesHere changes += changesHere
if err != nil { if err != nil {
return err return err
@ -513,7 +504,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// Do a scan of the database for each prefix, to check for deleted and // Do a scan of the database for each prefix, to check for deleted and
// ignored files. // ignored files.
changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch, batchAppend) changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch)
changes += changesHere changes += changesHere
if err != nil { if err != nil {
return err return err
@ -527,58 +518,57 @@ func (f *folder) scanSubdirs(subDirs []string) error {
return nil return nil
} }
type batchAppendFunc func(protocol.FileInfo, *db.Snapshot) bool type scanBatch struct {
*db.FileInfoBatch
f *folder
}
func (f *folder) scanSubdirsBatchAppendFunc(batch *db.FileInfoBatch) batchAppendFunc { func (f *folder) newScanBatch() *scanBatch {
// Resolve items which are identical with the global state. b := &scanBatch{
switch f.Type { f: f,
case config.FolderTypeReceiveOnly:
return func(fi protocol.FileInfo, snap *db.Snapshot) bool {
switch gf, ok := snap.GetGlobal(fi.Name); {
case !ok:
case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
// What we have locally is equivalent to the global file.
fi.Version = gf.Version
l.Debugf("%v scanning: Merging identical locally changed item with global", f, fi)
fallthrough
case fi.IsDeleted() && (gf.IsReceiveOnlyChanged() || gf.IsDeleted()):
// Our item is deleted and the global item is our own
// receive only file or deleted too. In the former
// case we can't delete file infos, so we just
// pretend it is a normal deleted file (nobody
// cares about that).
l.Debugf("%v scanning: Marking item as not locally changed", f, fi)
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
} }
batch.Append(fi) b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
return true if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err)
return err
} }
case config.FolderTypeReceiveEncrypted: b.f.updateLocalsFromScanning(fs)
return func(fi protocol.FileInfo, _ *db.Snapshot) bool { return nil
// This is a "virtual" parent directory of encrypted files. })
// We don't track it, but check if anything still exists return b
// within and delete it otherwise. }
if fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
if names, err := f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 { // Append adds the fileinfo to the batch for updating, and does a few checks.
f.mtimefs.Remove(fi.Name) // It returns false if the checks result in the file not going to be updated or removed.
func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool {
// Check for a "virtual" parent directory of encrypted files. We don't track
// it, but check if anything still exists within and delete it otherwise.
if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
if names, err := b.f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
b.f.mtimefs.Remove(fi.Name)
} }
return false return false
} }
// Any local change must not be sent as index entry to // Resolve receive-only items which are identical with the global state or
// remotes and show up as an error in the UI. // the global item is our own receive-only item.
fi.LocalFlags = protocol.FlagLocalReceiveOnly // Except if they are in a receive-encrypted folder and are locally added.
batch.Append(fi) // Those must never be sent in index updates and thus must retain the flag.
switch gf, ok := snap.GetGlobal(fi.Name); {
case !ok:
case gf.IsReceiveOnlyChanged():
if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted {
l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi)
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi)
fi = gf
}
b.FileInfoBatch.Append(fi)
return true return true
}
default:
return func(fi protocol.FileInfo, _ *db.Snapshot) bool {
batch.Append(fi)
return true
}
}
} }
func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) { func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (int, error) {
changes := 0 changes := 0
snap, err := f.dbSnapshot() snap, err := f.dbSnapshot()
if err != nil { if err != nil {
@ -630,7 +620,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
return changes, err return changes, err
} }
if batchAppend(res.File, snap) { if batch.Append(res.File, snap) {
changes++ changes++
} }
@ -638,7 +628,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
default: default:
if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok { if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok {
if batchAppend(nf, snap) { if batch.Append(nf, snap) {
changes++ changes++
} }
} }
@ -648,7 +638,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
return changes, nil return changes, nil
} }
func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) { func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch) (int, error) {
var toIgnore []db.FileInfoTruncated var toIgnore []db.FileInfoTruncated
ignoredParent := "" ignoredParent := ""
changes := 0 changes := 0
@ -679,7 +669,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
for _, file := range toIgnore { for _, file := range toIgnore {
l.Debugln("marking file as ignored", file) l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) { if batch.Append(nf, snap) {
changes++ changes++
} }
if err := batch.FlushIfFull(); err != nil { if err := batch.FlushIfFull(); err != nil {
@ -709,7 +699,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
l.Debugln("marking file as ignored", file) l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) { if batch.Append(nf, snap) {
changes++ changes++
} }
@ -739,25 +729,37 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
nf.Version = protocol.Vector{} nf.Version = protocol.Vector{}
} }
l.Debugln("marking file as deleted", nf) l.Debugln("marking file as deleted", nf)
if batchAppend(nf, snap) { if batch.Append(nf, snap) {
changes++ changes++
} }
case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type == config.FolderTypeReceiveOnly && len(snap.Availability(file.Name)) == 0: case file.IsDeleted() && file.IsReceiveOnlyChanged():
file.Version = protocol.Vector{} switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() {
// Our item is deleted and the global item is deleted too. We just
// pretend it is a normal deleted file (nobody cares about that).
// Except if this is a receive-encrypted folder and it
// is a locally added file. Those must never be sent
// in index updates and thus must retain the flag.
if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() {
return true
}
l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name)
file.LocalFlags &^= protocol.FlagLocalReceiveOnly file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("marking deleted item that doesn't exist anywhere as not receive-only", file) if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
if batchAppend(file.ConvertDeletedToFileInfo(), snap) {
changes++ changes++
} }
case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type != config.FolderTypeReceiveOnly: }
default:
// No need to bump the version for a file that was and is // No need to bump the version for a file that was and is
// deleted and just the folder type/local flags changed. // deleted and just the folder type/local flags changed.
file.LocalFlags &^= protocol.FlagLocalReceiveOnly file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("removing receive-only flag on deleted item", file) l.Debugln("removing receive-only flag on deleted item", file)
if batchAppend(file.ConvertDeletedToFileInfo(), snap) { if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
changes++ changes++
} }
} }
}
return true return true
}) })
@ -772,7 +774,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
for _, file := range toIgnore { for _, file := range toIgnore {
l.Debugln("marking file as ignored", f) l.Debugln("marking file as ignored", f)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batchAppend(nf, snap) { if batch.Append(nf, snap) {
changes++ changes++
} }
if iterError = batch.FlushIfFull(); iterError != nil { if iterError = batch.FlushIfFull(); iterError != nil {

4
lib/model/folder_recvenc.go
View File

@ -29,7 +29,9 @@ type receiveEncryptedFolder struct {
} }
func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)} f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return f
} }
func (f *receiveEncryptedFolder) Revert() { func (f *receiveEncryptedFolder) Revert() {