gui, lib: Fix tracking deleted locally-changed on encrypted (fixes #7715) (#7726)

This commit is contained in:
Simon Frei 2021-11-10 09:46:21 +01:00 committed by GitHub
parent dec6f80d2b
commit 591e4d8af1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 271 additions and 142 deletions

View File

@ -455,12 +455,6 @@
<a href="" ng-click="showLocalChanged(folder.id, folder.type)">{{model[folder.id].receiveOnlyTotalItems | alwaysNumber | localeNumber}} <span translate>items</span>, ~{{model[folder.id].receiveOnlyChangedBytes | binary}}B</a> <a href="" ng-click="showLocalChanged(folder.id, folder.type)">{{model[folder.id].receiveOnlyTotalItems | alwaysNumber | localeNumber}} <span translate>items</span>, ~{{model[folder.id].receiveOnlyChangedBytes | binary}}B</a>
</td> </td>
</tr> </tr>
<tr ng-if="hasReceiveEncryptedItems(folder)">
<th><span class="fas fa-fw fa-exclamation-circle"></span>&nbsp;<span translate>Locally Changed Items</span></th>
<td class="text-right">
<a href="" ng-click="showLocalChanged(folder.id, folder.type)">{{receiveEncryptedItemsCount(folder) | alwaysNumber | localeNumber}} <span translate>items</span>, ~{{model[folder.id].receiveOnlyChangedBytes | binary}}B</a>
</td>
</tr>
<tr ng-if="folder.type != 'sendreceive'"> <tr ng-if="folder.type != 'sendreceive'">
<th><span class="fas fa-fw fa-folder"></span>&nbsp;<span translate>Folder Type</span></th> <th><span class="fas fa-fw fa-folder"></span>&nbsp;<span translate>Folder Type</span></th>
<td class="text-right"> <td class="text-right">

View File

@ -931,9 +931,9 @@ angular.module('syncthing.core')
return 'faileditems'; return 'faileditems';
} }
if ($scope.hasReceiveOnlyChanged(folderCfg)) { if ($scope.hasReceiveOnlyChanged(folderCfg)) {
return 'localadditions'; if (folderCfg.type === "receiveonly") {
} return 'localadditions';
if ($scope.hasReceiveEncryptedItems(folderCfg)) { }
return 'localunencrypted'; return 'localunencrypted';
} }
if (folderCfg.devices.length <= 1) { if (folderCfg.devices.length <= 1) {
@ -2609,28 +2609,13 @@ angular.module('syncthing.core')
}; };
$scope.hasReceiveOnlyChanged = function (folderCfg) { $scope.hasReceiveOnlyChanged = function (folderCfg) {
if (!folderCfg || folderCfg.type !== "receiveonly") { if (!folderCfg || folderCfg.type !== ["receiveonly", "receiveencrypted"].indexOf(folderCfg.type) === -1) {
return false; return false;
} }
var counts = $scope.model[folderCfg.id]; var counts = $scope.model[folderCfg.id];
return counts && counts.receiveOnlyTotalItems > 0; return counts && counts.receiveOnlyTotalItems > 0;
}; };
$scope.hasReceiveEncryptedItems = function (folderCfg) {
if (!folderCfg || folderCfg.type !== "receiveencrypted") {
return false;
}
return $scope.receiveEncryptedItemsCount(folderCfg) > 0;
};
$scope.receiveEncryptedItemsCount = function (folderCfg) {
var counts = $scope.model[folderCfg.id];
if (!counts) {
return 0;
}
return counts.receiveOnlyTotalItems - counts.receiveOnlyChangedDeletes;
}
$scope.revertOverride = function () { $scope.revertOverride = function () {
$http.post( $http.post(
urlbase + "/db/" + $scope.revertOverrideParams.operation +"?folder=" urlbase + "/db/" + $scope.revertOverrideParams.operation +"?folder="

View File

@ -223,35 +223,10 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
blocksHashSame := ok && bytes.Equal(ef.BlocksHash, f.BlocksHash) blocksHashSame := ok && bytes.Equal(ef.BlocksHash, f.BlocksHash)
if ok { if ok {
if len(ef.Blocks) != 0 && !ef.IsInvalid() && ef.Size > 0 { keyBuf, err = db.removeLocalBlockAndSequenceInfo(keyBuf, folder, name, ef, !blocksHashSame, &t)
for _, block := range ef.Blocks {
keyBuf, err = db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
if err != nil {
return err
}
if err := t.Delete(keyBuf); err != nil {
return err
}
}
if !blocksHashSame {
keyBuf, err := db.keyer.GenerateBlockListMapKey(keyBuf, folder, ef.BlocksHash, name)
if err != nil {
return err
}
if err = t.Delete(keyBuf); err != nil {
return err
}
}
}
keyBuf, err = db.keyer.GenerateSequenceKey(keyBuf, folder, ef.SequenceNo())
if err != nil { if err != nil {
return err return err
} }
if err := t.Delete(keyBuf); err != nil {
return err
}
l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, ef.SequenceNo(), ef.FileName())
} }
f.Sequence = meta.nextLocalSeq() f.Sequence = meta.nextLocalSeq()
@ -314,6 +289,96 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
return t.Commit() return t.Commit()
} }
func (db *Lowlevel) removeLocalFiles(folder []byte, nameStrs []string, meta *metadataTracker) error {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
t, err := db.newReadWriteTransaction(meta.CommitHook(folder))
if err != nil {
return err
}
defer t.close()
var dk, gk, buf []byte
for _, nameStr := range nameStrs {
name := []byte(nameStr)
dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
if err != nil {
return err
}
ef, ok, err := t.getFileByKey(dk)
if err != nil {
return err
}
if !ok {
l.Debugf("remove (local); folder=%q %v: file doesn't exist", folder, nameStr)
continue
}
buf, err = db.removeLocalBlockAndSequenceInfo(buf, folder, name, ef, true, &t)
if err != nil {
return err
}
meta.removeFile(protocol.LocalDeviceID, ef)
gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
buf, err = t.removeFromGlobal(gk, buf, folder, protocol.LocalDeviceID[:], name, meta)
if err != nil {
return err
}
err = t.Delete(dk)
if err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
return t.Commit()
}
func (db *Lowlevel) removeLocalBlockAndSequenceInfo(keyBuf, folder, name []byte, ef protocol.FileInfo, removeFromBlockListMap bool, t *readWriteTransaction) ([]byte, error) {
var err error
if len(ef.Blocks) != 0 && !ef.IsInvalid() && ef.Size > 0 {
for _, block := range ef.Blocks {
keyBuf, err = db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
if err != nil {
return nil, err
}
if err := t.Delete(keyBuf); err != nil {
return nil, err
}
}
if removeFromBlockListMap {
keyBuf, err := db.keyer.GenerateBlockListMapKey(keyBuf, folder, ef.BlocksHash, name)
if err != nil {
return nil, err
}
if err = t.Delete(keyBuf); err != nil {
return nil, err
}
}
}
keyBuf, err = db.keyer.GenerateSequenceKey(keyBuf, folder, ef.SequenceNo())
if err != nil {
return nil, err
}
if err := t.Delete(keyBuf); err != nil {
return nil, err
}
l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, ef.SequenceNo(), ef.FileName())
return keyBuf, nil
}
func (db *Lowlevel) dropFolder(folder []byte) error { func (db *Lowlevel) dropFolder(folder []byte) error {
db.gcMut.RLock() db.gcMut.RLock()
defer db.gcMut.RUnlock() defer db.gcMut.RUnlock()

View File

@ -144,6 +144,22 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
} }
} }
func (s *FileSet) RemoveLocalItems(items []string) {
opStr := fmt.Sprintf("%s RemoveLocalItems([%d])", s.folder, len(items))
l.Debugf(opStr)
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
for i := range items {
items[i] = osutil.NormalizedFilename(items[i])
}
if err := s.db.removeLocalFiles([]byte(s.folder), items, s.meta); err != nil && !backend.IsClosed(err) {
fatalError(err, opStr, s.db)
}
}
type Snapshot struct { type Snapshot struct {
folder string folder string
t readOnlyTransaction t readOnlyTransaction

View File

@ -162,6 +162,7 @@ func (f *fakeConnection) sendIndexUpdate() {
func addFakeConn(m *testModel, dev protocol.DeviceID, folderID string) *fakeConnection { func addFakeConn(m *testModel, dev protocol.DeviceID, folderID string) *fakeConnection {
fc := newFakeConnection(dev, m) fc := newFakeConnection(dev, m)
fc.folder = folderID
m.AddConnection(fc, protocol.Hello{}) m.AddConnection(fc, protocol.Hello{})
m.ClusterConfig(dev, protocol.ClusterConfig{ m.ClusterConfig(dev, protocol.ClusterConfig{

View File

@ -532,16 +532,20 @@ func (f *folder) scanSubdirs(subDirs []string) error {
return nil return nil
} }
const maxToRemove = 1000
type scanBatch struct { type scanBatch struct {
*db.FileInfoBatch f *folder
f *folder updateBatch *db.FileInfoBatch
toRemove []string
} }
func (f *folder) newScanBatch() *scanBatch { func (f *folder) newScanBatch() *scanBatch {
b := &scanBatch{ b := &scanBatch{
f: f, f: f,
toRemove: make([]string, 0, maxToRemove),
} }
b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { b.updateBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
if err := b.f.getHealthErrorWithoutIgnores(); err != nil { if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err) l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err)
return err return err
@ -552,9 +556,32 @@ func (f *folder) newScanBatch() *scanBatch {
return b return b
} }
// Append adds the fileinfo to the batch for updating, and does a few checks. func (b *scanBatch) Remove(item string) {
b.toRemove = append(b.toRemove, item)
}
func (b *scanBatch) flushToRemove() {
if len(b.toRemove) > 0 {
b.f.fset.RemoveLocalItems(b.toRemove)
b.toRemove = b.toRemove[:0]
}
}
func (b *scanBatch) Flush() error {
b.flushToRemove()
return b.updateBatch.Flush()
}
func (b *scanBatch) FlushIfFull() error {
if len(b.toRemove) >= maxToRemove {
b.flushToRemove()
}
return b.updateBatch.FlushIfFull()
}
// Update adds the fileinfo to the batch for updating, and does a few checks.
// It returns false if the checks result in the file not going to be updated or removed. // It returns false if the checks result in the file not going to be updated or removed.
func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { func (b *scanBatch) Update(fi protocol.FileInfo, snap *db.Snapshot) bool {
// Check for a "virtual" parent directory of encrypted files. We don't track // Check for a "virtual" parent directory of encrypted files. We don't track
// it, but check if anything still exists within and delete it otherwise. // it, but check if anything still exists within and delete it otherwise.
if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) { if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
@ -565,20 +592,21 @@ func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool {
} }
// Resolve receive-only items which are identical with the global state or // Resolve receive-only items which are identical with the global state or
// the global item is our own receive-only item. // the global item is our own receive-only item.
// Except if they are in a receive-encrypted folder and are locally added.
// Those must never be sent in index updates and thus must retain the flag.
switch gf, ok := snap.GetGlobal(fi.Name); { switch gf, ok := snap.GetGlobal(fi.Name); {
case !ok: case !ok:
case gf.IsReceiveOnlyChanged(): case gf.IsReceiveOnlyChanged():
if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted { if fi.IsDeleted() {
l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi) // Our item is deleted and the global item is our own receive only
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly // file. No point in keeping track of that.
b.Remove(fi.Name)
return true
} }
case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly): case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi) // What we have locally is equivalent to the global file.
l.Debugf("%v scanning: Merging identical locally changed item with global", b.f, fi)
fi = gf fi = gf
} }
b.FileInfoBatch.Append(fi) b.updateBatch.Append(fi)
return true return true
} }
@ -634,7 +662,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i
return changes, err return changes, err
} }
if batch.Append(res.File, snap) { if batch.Update(res.File, snap) {
changes++ changes++
} }
@ -642,7 +670,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
default: default:
if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok { if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok {
if batch.Append(nf, snap) { if batch.Update(nf, snap) {
changes++ changes++
} }
} }
@ -683,7 +711,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
for _, file := range toIgnore { for _, file := range toIgnore {
l.Debugln("marking file as ignored", file) l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batch.Append(nf, snap) { if batch.Update(nf, snap) {
changes++ changes++
} }
if err := batch.FlushIfFull(); err != nil { if err := batch.FlushIfFull(); err != nil {
@ -713,7 +741,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
l.Debugln("marking file as ignored", file) l.Debugln("marking file as ignored", file)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batch.Append(nf, snap) { if batch.Update(nf, snap) {
changes++ changes++
} }
@ -743,24 +771,24 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
nf.Version = protocol.Vector{} nf.Version = protocol.Vector{}
} }
l.Debugln("marking file as deleted", nf) l.Debugln("marking file as deleted", nf)
if batch.Append(nf, snap) { if batch.Update(nf, snap) {
changes++ changes++
} }
case file.IsDeleted() && file.IsReceiveOnlyChanged(): case file.IsDeleted() && file.IsReceiveOnlyChanged():
switch f.Type { switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() { switch gf, ok := snap.GetGlobal(file.Name); {
case !ok:
case gf.IsReceiveOnlyChanged():
l.Debugln("removing deleted, receive-only item that is globally receive-only from db", file)
batch.Remove(file.Name)
changes++
case gf.IsDeleted():
// Our item is deleted and the global item is deleted too. We just // Our item is deleted and the global item is deleted too. We just
// pretend it is a normal deleted file (nobody cares about that). // pretend it is a normal deleted file (nobody cares about that).
// Except if this is a receive-encrypted folder and it
// is a locally added file. Those must never be sent
// in index updates and thus must retain the flag.
if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() {
return true
}
l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name) l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name)
file.LocalFlags &^= protocol.FlagLocalReceiveOnly file.LocalFlags &^= protocol.FlagLocalReceiveOnly
if batch.Append(file.ConvertDeletedToFileInfo(), snap) { if batch.Update(file.ConvertDeletedToFileInfo(), snap) {
changes++ changes++
} }
} }
@ -769,7 +797,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
// deleted and just the folder type/local flags changed. // deleted and just the folder type/local flags changed.
file.LocalFlags &^= protocol.FlagLocalReceiveOnly file.LocalFlags &^= protocol.FlagLocalReceiveOnly
l.Debugln("removing receive-only flag on deleted item", file) l.Debugln("removing receive-only flag on deleted item", file)
if batch.Append(file.ConvertDeletedToFileInfo(), snap) { if batch.Update(file.ConvertDeletedToFileInfo(), snap) {
changes++ changes++
} }
} }
@ -788,7 +816,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch
for _, file := range toIgnore { for _, file := range toIgnore {
l.Debugln("marking file as ignored", f) l.Debugln("marking file as ignored", f)
nf := file.ConvertToIgnoredFileInfo() nf := file.ConvertToIgnoredFileInfo()
if batch.Append(nf, snap) { if batch.Update(nf, snap) {
changes++ changes++
} }
if iterError = batch.FlushIfFull(); iterError != nil { if iterError = batch.FlushIfFull(); iterError != nil {

View File

@ -37,9 +37,9 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
for _, dir := range []string{".stfolder", "ignDir", "unknownDir"} { for _, dir := range []string{".stfolder", "ignDir", "unknownDir"} {
must(t, ffs.MkdirAll(dir, 0755)) must(t, ffs.MkdirAll(dir, 0755))
} }
must(t, writeFile(ffs, "ignDir/ignFile", []byte("hello\n"), 0644)) writeFilePerm(t, ffs, "ignDir/ignFile", []byte("hello\n"), 0644)
must(t, writeFile(ffs, "unknownDir/unknownFile", []byte("hello\n"), 0644)) writeFilePerm(t, ffs, "unknownDir/unknownFile", []byte("hello\n"), 0644)
must(t, writeFile(ffs, ".stignore", []byte("ignDir\n"), 0644)) writeFilePerm(t, ffs, ".stignore", []byte("ignDir\n"), 0644)
knownFiles := setupKnownFiles(t, ffs, []byte("hello\n")) knownFiles := setupKnownFiles(t, ffs, []byte("hello\n"))
@ -151,7 +151,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// Update the file. // Update the file.
newData := []byte("totally different data\n") newData := []byte("totally different data\n")
must(t, writeFile(ffs, "knownDir/knownFile", newData, 0644)) writeFilePerm(t, ffs, "knownDir/knownFile", newData, 0644)
// Rescan. // Rescan.
@ -241,8 +241,8 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Create a file and modify another // Create a file and modify another
const file = "foo" const file = "foo"
must(t, writeFile(ffs, file, []byte("hello\n"), 0644)) writeFilePerm(t, ffs, file, []byte("hello\n"), 0644)
must(t, writeFile(ffs, "knownDir/knownFile", []byte("bye\n"), 0644)) writeFilePerm(t, ffs, "knownDir/knownFile", []byte("bye\n"), 0644)
must(t, m.ScanFolder("ro")) must(t, m.ScanFolder("ro"))
@ -254,7 +254,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Remove the file again and undo the modification // Remove the file again and undo the modification
must(t, ffs.Remove(file)) must(t, ffs.Remove(file))
must(t, writeFile(ffs, "knownDir/knownFile", oldData, 0644)) writeFilePerm(t, ffs, "knownDir/knownFile", oldData, 0644)
must(t, ffs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime())) must(t, ffs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime()))
must(t, m.ScanFolder("ro")) must(t, m.ScanFolder("ro"))
@ -377,8 +377,8 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
const file = "foo" const file = "foo"
knownFile := filepath.Join("knownDir", "knownFile") knownFile := filepath.Join("knownDir", "knownFile")
must(t, writeFile(ffs, file, []byte("hello\n"), 0644)) writeFilePerm(t, ffs, file, []byte("hello\n"), 0644)
must(t, writeFile(ffs, knownFile, []byte("bye\n"), 0644)) writeFilePerm(t, ffs, knownFile, []byte("bye\n"), 0644)
must(t, m.ScanFolder("ro")) must(t, m.ScanFolder("ro"))
@ -434,7 +434,7 @@ func TestRecvOnlyRevertOwnID(t *testing.T) {
must(t, ffs.MkdirAll(".stfolder", 0755)) must(t, ffs.MkdirAll(".stfolder", 0755))
data := []byte("hello\n") data := []byte("hello\n")
name := "foo" name := "foo"
must(t, writeFile(ffs, name, data, 0644)) writeFilePerm(t, ffs, name, data, 0644)
// Make sure the file is scanned and locally changed // Make sure the file is scanned and locally changed
must(t, m.ScanFolder("ro")) must(t, m.ScanFolder("ro"))
@ -484,7 +484,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
t.Helper() t.Helper()
must(t, ffs.MkdirAll("knownDir", 0755)) must(t, ffs.MkdirAll("knownDir", 0755))
must(t, writeFile(ffs, "knownDir/knownFile", data, 0644)) writeFilePerm(t, ffs, "knownDir/knownFile", data, 0644)
t0 := time.Now().Add(-1 * time.Minute) t0 := time.Now().Add(-1 * time.Minute)
must(t, ffs.Chtimes("knownDir/knownFile", t0, t0)) must(t, ffs.Chtimes("knownDir/knownFile", t0, t0))
@ -541,18 +541,3 @@ func setupROFolder(t *testing.T) (*testModel, *receiveOnlyFolder, context.Cancel
return m, f, cancel return m, f, cancel
} }
func writeFile(fs fs.Filesystem, filename string, data []byte, perm fs.FileMode) error {
fd, err := fs.Create(filename)
if err != nil {
return err
}
_, err = fd.Write(data)
if err != nil {
return err
}
if err := fd.Close(); err != nil {
return err
}
return fs.Chmod(filename, perm)
}

View File

@ -77,12 +77,10 @@ func setupFile(filename string, blockNumbers []int) protocol.FileInfo {
} }
} }
func createFile(t *testing.T, name string, fs fs.Filesystem) protocol.FileInfo { func createEmptyFileInfo(t *testing.T, name string, fs fs.Filesystem) protocol.FileInfo {
t.Helper() t.Helper()
f, err := fs.Create(name) writeFile(t, fs, name, nil)
must(t, err)
f.Close()
fi, err := fs.Stat(name) fi, err := fs.Stat(name)
must(t, err) must(t, err)
file, err := scanner.CreateFileInfo(fi, name, fs) file, err := scanner.CreateFileInfo(fi, name, fs)
@ -913,7 +911,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
name := "foo" name := "foo"
// create local file // create local file
file := createFile(t, name, ffs) file := createEmptyFileInfo(t, name, ffs)
file.Version = protocol.Vector{}.Update(myID.Short()) file.Version = protocol.Vector{}.Update(myID.Short())
f.updateLocalsFromScanning([]protocol.FileInfo{file}) f.updateLocalsFromScanning([]protocol.FileInfo{file})
@ -945,7 +943,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
name := "foo" name := "foo"
// create local file // create local file
file := createFile(t, name, ffs) file := createEmptyFileInfo(t, name, ffs)
file.Version = protocol.Vector{}.Update(myID.Short()) file.Version = protocol.Vector{}.Update(myID.Short())
f.updateLocalsFromScanning([]protocol.FileInfo{file}) f.updateLocalsFromScanning([]protocol.FileInfo{file})
@ -983,7 +981,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
file := filepath.Join(link, "file") file := filepath.Join(link, "file")
must(t, ffs.MkdirAll(link, 0755)) must(t, ffs.MkdirAll(link, 0755))
fi := createFile(t, file, ffs) fi := createEmptyFileInfo(t, file, ffs)
f.updateLocalsFromScanning([]protocol.FileInfo{fi}) f.updateLocalsFromScanning([]protocol.FileInfo{fi})
must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file")) must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file"))
must(t, ffs.RemoveAll(link)) must(t, ffs.RemoveAll(link))
@ -1099,7 +1097,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
name := "foo" name := "foo"
contents := []byte("contents") contents := []byte("contents")
must(t, writeFile(ffs, name, contents, 0644)) writeFile(t, ffs, name, contents)
must(t, f.scanSubdirs(nil)) must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo var cur protocol.FileInfo
@ -1122,7 +1120,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
remote.Version = protocol.Vector{}.Update(device1.Short()) remote.Version = protocol.Vector{}.Update(device1.Short())
remote.Name = strings.ToUpper(cur.Name) remote.Name = strings.ToUpper(cur.Name)
temp := fs.TempName(remote.Name) temp := fs.TempName(remote.Name)
must(t, writeFile(ffs, temp, contents, 0644)) writeFile(t, ffs, temp, contents)
scanChan := make(chan string, 1) scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1) dbUpdateChan := make(chan dbUpdateJob, 1)

View File

@ -1053,7 +1053,6 @@ func (m *model) RemoteNeedFolderFiles(folder string, device protocol.DeviceID, p
func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error) { func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
m.fmut.RLock() m.fmut.RLock()
rf, ok := m.folderFiles[folder] rf, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
m.fmut.RUnlock() m.fmut.RUnlock()
if !ok { if !ok {
@ -1071,11 +1070,10 @@ func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.
} }
p := newPager(page, perpage) p := newPager(page, perpage)
recvEnc := cfg.Type == config.FolderTypeReceiveEncrypted
files := make([]db.FileInfoTruncated, 0, perpage) files := make([]db.FileInfoTruncated, 0, perpage)
snap.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { snap.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
if !f.IsReceiveOnlyChanged() || (recvEnc && f.IsDeleted()) { if !f.IsReceiveOnlyChanged() {
return true return true
} }
if p.skip() { if p.skip() {

View File

@ -299,7 +299,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
mustRemove(b, defaultFs.RemoveAll("request")) mustRemove(b, defaultFs.RemoveAll("request"))
defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }() defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }()
must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755)) must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755))
writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644) writeFile(b, defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf)
b.ResetTimer() b.ResetTimer()
@ -1489,7 +1489,7 @@ func TestIgnores(t *testing.T) {
// Assure a clean start state // Assure a clean start state
mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644) writeFile(t, defaultFs, ".stignore", []byte(".*\nquux\n"))
m := setupModel(t, defaultCfgWrapper) m := setupModel(t, defaultCfgWrapper)
defer cleanupModel(m) defer cleanupModel(m)
@ -2030,8 +2030,8 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
func TestIssue3028(t *testing.T) { func TestIssue3028(t *testing.T) {
// Create two files that we'll delete, one with a name that is a prefix of the other. // Create two files that we'll delete, one with a name that is a prefix of the other.
must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644)) writeFile(t, defaultFs, "testrm", []byte("Hello"))
must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644)) writeFile(t, defaultFs, "testrm2", []byte("Hello"))
defer func() { defer func() {
mustRemove(t, defaultFs.Remove("testrm")) mustRemove(t, defaultFs.Remove("testrm"))
mustRemove(t, defaultFs.Remove("testrm2")) mustRemove(t, defaultFs.Remove("testrm2"))
@ -3403,7 +3403,7 @@ func TestRenameSequenceOrder(t *testing.T) {
ffs := fcfg.Filesystem() ffs := fcfg.Filesystem()
for i := 0; i < numFiles; i++ { for i := 0; i < numFiles; i++ {
v := fmt.Sprintf("%d", i) v := fmt.Sprintf("%d", i)
must(t, writeFile(ffs, v, []byte(v), 0644)) writeFile(t, ffs, v, []byte(v))
} }
m.ScanFolders() m.ScanFolders()
@ -3426,7 +3426,7 @@ func TestRenameSequenceOrder(t *testing.T) {
continue continue
} }
v := fmt.Sprintf("%d", i) v := fmt.Sprintf("%d", i)
must(t, writeFile(ffs, v, []byte(v+"-new"), 0644)) writeFile(t, ffs, v, []byte(v+"-new"))
} }
// Rename // Rename
must(t, ffs.Rename("3", "17")) must(t, ffs.Rename("3", "17"))
@ -3470,7 +3470,7 @@ func TestRenameSameFile(t *testing.T) {
defer cleanupModel(m) defer cleanupModel(m)
ffs := fcfg.Filesystem() ffs := fcfg.Filesystem()
must(t, writeFile(ffs, "file", []byte("file"), 0644)) writeFile(t, ffs, "file", []byte("file"))
m.ScanFolders() m.ScanFolders()
@ -3522,8 +3522,8 @@ func TestRenameEmptyFile(t *testing.T) {
ffs := fcfg.Filesystem() ffs := fcfg.Filesystem()
must(t, writeFile(ffs, "file", []byte("data"), 0644)) writeFile(t, ffs, "file", []byte("data"))
must(t, writeFile(ffs, "empty", nil, 0644)) writeFile(t, ffs, "empty", nil)
m.ScanFolders() m.ScanFolders()
@ -3598,11 +3598,11 @@ func TestBlockListMap(t *testing.T) {
defer cleanupModel(m) defer cleanupModel(m)
ffs := fcfg.Filesystem() ffs := fcfg.Filesystem()
must(t, writeFile(ffs, "one", []byte("content"), 0644)) writeFile(t, ffs, "one", []byte("content"))
must(t, writeFile(ffs, "two", []byte("content"), 0644)) writeFile(t, ffs, "two", []byte("content"))
must(t, writeFile(ffs, "three", []byte("content"), 0644)) writeFile(t, ffs, "three", []byte("content"))
must(t, writeFile(ffs, "four", []byte("content"), 0644)) writeFile(t, ffs, "four", []byte("content"))
must(t, writeFile(ffs, "five", []byte("content"), 0644)) writeFile(t, ffs, "five", []byte("content"))
m.ScanFolders() m.ScanFolders()
@ -3631,7 +3631,7 @@ func TestBlockListMap(t *testing.T) {
// Modify // Modify
must(t, ffs.Remove("two")) must(t, ffs.Remove("two"))
must(t, writeFile(ffs, "two", []byte("mew-content"), 0644)) writeFile(t, ffs, "two", []byte("mew-content"))
// Rename // Rename
must(t, ffs.Rename("three", "new-three")) must(t, ffs.Rename("three", "new-three"))
@ -3667,7 +3667,7 @@ func TestScanRenameCaseOnly(t *testing.T) {
ffs := fcfg.Filesystem() ffs := fcfg.Filesystem()
name := "foo" name := "foo"
must(t, writeFile(ffs, name, []byte("contents"), 0644)) writeFile(t, ffs, name, []byte("contents"))
m.ScanFolders() m.ScanFolders()
@ -3791,7 +3791,7 @@ func TestScanDeletedROChangedOnSR(t *testing.T) {
name := "foo" name := "foo"
must(t, writeFile(ffs, name, []byte(name), 0644)) writeFile(t, ffs, name, []byte(name))
m.ScanFolders() m.ScanFolders()
file, ok := m.testCurrentFolderFile(fcfg.ID, name) file, ok := m.testCurrentFolderFile(fcfg.ID, name)
@ -4255,6 +4255,46 @@ func TestPendingFolder(t *testing.T) {
} }
} }
func TestDeletedNotLocallyChangedReceiveOnly(t *testing.T) {
deletedNotLocallyChanged(t, config.FolderTypeReceiveOnly)
}
func TestDeletedNotLocallyChangedReceiveEncrypted(t *testing.T) {
deletedNotLocallyChanged(t, config.FolderTypeReceiveEncrypted)
}
func deletedNotLocallyChanged(t *testing.T, ft config.FolderType) {
w, fcfg, wCancel := tmpDefaultWrapper()
tfs := fcfg.Filesystem()
fcfg.Type = ft
setFolder(t, w, fcfg)
defer wCancel()
m := setupModel(t, w)
defer cleanupModelAndRemoveDir(m, tfs.URI())
name := "foo"
writeFile(t, tfs, name, nil)
must(t, m.ScanFolder(fcfg.ID))
fi, ok, err := m.CurrentFolderFile(fcfg.ID, name)
must(t, err)
if !ok {
t.Fatal("File hasn't been added")
}
if !fi.IsReceiveOnlyChanged() {
t.Fatal("File isn't receive-only-changed")
}
must(t, tfs.Remove(name))
must(t, m.ScanFolder(fcfg.ID))
_, ok, err = m.CurrentFolderFile(fcfg.ID, name)
must(t, err)
if ok {
t.Error("Expected file to be removed from db")
}
}
func equalStringsInAnyOrder(a, b []string) bool { func equalStringsInAnyOrder(a, b []string) bool {
if len(a) != len(b) { if len(a) != len(b) {
return false return false

View File

@ -330,9 +330,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
fc.deleteFile(invDel) fc.deleteFile(invDel)
fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents) fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents)
fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents) fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents)
if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil { writeFile(t, fss, ignExisting, otherContents)
panic(err)
}
done := make(chan struct{}) done := make(chan struct{})
fc.setIndexFn(func(_ context.Context, folder string, fs []protocol.FileInfo) error { fc.setIndexFn(func(_ context.Context, folder string, fs []protocol.FileInfo) error {
@ -486,7 +484,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
payload := []byte("hello") payload := []byte("hello")
must(t, writeFile(tfs, "foo", payload, 0777)) writeFile(t, tfs, "foo", payload)
received := make(chan []protocol.FileInfo) received := make(chan []protocol.FileInfo)
fc.setIndexFn(func(_ context.Context, _ string, fs []protocol.FileInfo) error { fc.setIndexFn(func(_ context.Context, _ string, fs []protocol.FileInfo) error {
@ -526,7 +524,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
payload = []byte("bye") payload = []byte("bye")
buf = make([]byte, len(payload)) buf = make([]byte, len(payload))
must(t, writeFile(tfs, "foo", payload, 0777)) writeFile(t, tfs, "foo", payload)
_, err = m.Request(device1, "default", "foo", 0, int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false) _, err = m.Request(device1, "default", "foo", 0, int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false)
if err == nil { if err == nil {
@ -1066,9 +1064,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
return nil return nil
}) })
if err := writeFile(fss, file, contents, 0644); err != nil { writeFile(t, fss, file, contents)
panic(err)
}
m.ScanFolders() m.ScanFolders()
select { select {
@ -1430,6 +1426,14 @@ func TestRequestReceiveEncrypted(t *testing.T) {
// Simulate request from device that is untrusted too, i.e. with non-empty, but garbage hash // Simulate request from device that is untrusted too, i.e. with non-empty, but garbage hash
_, err := m.Request(device1, fcfg.ID, name, 0, 1064, 0, []byte("garbage"), 0, false) _, err := m.Request(device1, fcfg.ID, name, 0, 1064, 0, []byte("garbage"), 0, false)
must(t, err) must(t, err)
changed, err := m.LocalChangedFolderFiles(fcfg.ID, 1, 10)
must(t, err)
if l := len(changed); l != 1 {
t.Errorf("Expected one locally changed file, got %v", l)
} else if changed[0].Name != files[0].Name {
t.Errorf("Expected %v, got %v", files[0].Name, changed[0].Name)
}
} }
func TestRequestGlobalInvalidToValid(t *testing.T) { func TestRequestGlobalInvalidToValid(t *testing.T) {

View File

@ -435,3 +435,18 @@ func addDevice2(t testing.TB, w config.Wrapper, fcfg config.FolderConfiguration)
must(t, err) must(t, err)
waiter.Wait() waiter.Wait()
} }
func writeFile(t testing.TB, filesystem fs.Filesystem, name string, data []byte) {
t.Helper()
fd, err := filesystem.Create(name)
must(t, err)
defer fd.Close()
_, err = fd.Write(data)
must(t, err)
}
func writeFilePerm(t testing.TB, filesystem fs.Filesystem, name string, data []byte, perm fs.FileMode) {
t.Helper()
writeFile(t, filesystem, name, data)
must(t, filesystem.Chmod(name, perm))
}