diff --git a/go.mod b/go.mod index fcfa47508..ce27ca54c 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/prometheus/client_golang v0.9.4 github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 github.com/sasha-s/go-deadlock v0.2.0 + github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/thejerf/suture v3.0.2+incompatible diff --git a/go.sum b/go.sum index 79379b12d..21c159413 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6 h1:Apvc github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q= github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg= github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -37,6 +38,7 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d h1:IngNQgbqr5ZOU0exk395Szrvkzes9Ilk1fmJfkw7d+M= github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= @@ -132,6 +134,11 @@ github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 h1:jmLW6izPBVl github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q= +github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/gopsutil v2.19.6+incompatible h1:49/Gru26Lne9Cl3IoAVDZVM09hvkSrUodgIIsCVRwbs= +github.com/shirou/gopsutil v2.19.6+incompatible/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index ad497add9..67df39a43 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -10,6 +10,10 @@ import ( "errors" "fmt" "runtime" + "strings" + "time" + + "github.com/shirou/gopsutil/disk" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" @@ -53,8 +57,10 @@ type FolderConfiguration struct { WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash. MarkerName string `xml:"markerName" json:"markerName"` CopyOwnershipFromParent bool `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"` + RawModTimeWindowS int `xml:"modTimeWindowS" json:"modTimeWindowS"` - cachedFilesystem fs.Filesystem + cachedFilesystem fs.Filesystem + cachedModTimeWindow time.Duration DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"` DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"` @@ -111,6 +117,10 @@ func (f FolderConfiguration) Versioner() versioner.Versioner { return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params) } +func (f FolderConfiguration) ModTimeWindow() time.Duration { + return f.cachedModTimeWindow +} + func (f *FolderConfiguration) CreateMarker() error { if err := f.CheckPath(); err != ErrMarkerMissing { return err @@ -233,6 +243,24 @@ func (f *FolderConfiguration) prepare() { if f.MarkerName == "" { f.MarkerName = DefaultMarkerName } + + switch { + case f.RawModTimeWindowS > 0: + f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second + case runtime.GOOS == "android": + usage, err := disk.Usage(f.Filesystem().URI()) + if err != nil { + l.Debugf("Error detecting FS at %v on android, setting mtime window to 2s: %v", f.Path, err) + f.cachedModTimeWindow = 2 * time.Second + break + } + if strings.Contains(strings.ToLower(usage.Fstype), "fat") { + l.Debugf("Detecting FS at %v on android, found %v, thus setting mtime window to 2s", f.Path, usage.Fstype) + f.cachedModTimeWindow = 2 * time.Second + break + } + l.Debugf("Detecting FS at %v on android, found %v, thus leaving mtime window at 0", f.Path, usage.Fstype) + } } // RequiresRestartOnly returns a copy with only the attributes that require diff --git a/lib/db/db_test.go b/lib/db/db_test.go index 8c6942f67..076727a01 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -167,7 +167,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Unexpected additional file via sequence", f.FileName()) return true } - if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) { + if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, 0, true, true, 0) { found = true } else { t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) @@ -192,7 +192,7 @@ func TestUpdate0to3(t *testing.T) { } f := fi.(protocol.FileInfo) delete(need, f.Name) - if !f.IsEquivalentOptional(e, true, true, 0) { + if !f.IsEquivalentOptional(e, 0, true, true, 0) { t.Errorf("Wrong needed file, got %v, expected %v", f, e) } return true diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 8f4ce711d..74904ba53 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -905,7 +905,7 @@ func TestWithHaveSequence(t *testing.T) { i := 2 s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { - if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) { + if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) { t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) } i++ @@ -1004,7 +1004,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Error("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(remote0Have[0]) { + } else if !need[0].IsEquivalent(remote0Have[0], 0) { t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0]) } @@ -1030,7 +1030,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, remoteDevice0); len(need) != 1 { t.Error("Expected 1 need for remote 0, got", need) - } else if !need[0].IsEquivalent(localHave[0]) { + } else if !need[0].IsEquivalent(localHave[0], 0) { t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0]) } @@ -1066,7 +1066,7 @@ func TestIssue5007(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0]) { + } else if !need[0].IsEquivalent(fs[0], 0) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1101,7 +1101,7 @@ func TestNeedDeleted(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0]) { + } else if !need[0].IsEquivalent(fs[0], 0) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1243,7 +1243,7 @@ func TestNeedAfterUnignore(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected one local need, got", need) - } else if !need[0].IsEquivalent(remote) { + } else if !need[0].IsEquivalent(remote, 0) { t.Fatalf("Got %v, expected %v", need[0], remote) } } @@ -1287,7 +1287,7 @@ func TestNeedWithNewerInvalid(t *testing.T) { if len(need) != 1 { t.Fatal("Locally missing file should be needed") } - if !need[0].IsEquivalent(file) { + if !need[0].IsEquivalent(file, 0) { t.Fatalf("Got needed file %v, expected %v", need[0], file) } @@ -1302,7 +1302,7 @@ func TestNeedWithNewerInvalid(t *testing.T) { if len(need) != 1 { t.Fatal("Locally missing file should be needed regardless of invalid files") } - if !need[0].IsEquivalent(file) { + if !need[0].IsEquivalent(file, 0) { t.Fatalf("Got needed file %v, expected %v", need[0], file) } } diff --git a/lib/model/folder.go b/lib/model/folder.go index e9107d5a3..d01bfd213 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -347,6 +347,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { ShortID: f.shortID, ProgressTickIntervalS: f.ScanProgressIntervalS, LocalFlags: f.localFlags, + ModTimeWindow: f.ModTimeWindow(), }) batchFn := func(fs []protocol.FileInfo) error { @@ -365,7 +366,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { switch gf, ok := f.fset.GetGlobal(fs[i].Name); { case !ok: continue - case gf.IsEquivalentOptional(fs[i], false, false, protocol.FlagLocalReceiveOnly): + case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly): // What we have locally is equivalent to the global file. fs[i].Version = fs[i].Version.Merge(gf.Version) fallthrough diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index 671c1d958..d199eec59 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -74,7 +74,7 @@ func (f *sendOnlyFolder) pull() bool { } file := intf.(protocol.FileInfo) - if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) { + if !file.IsEquivalentOptional(curFile, f.ModTimeWindow(), f.IgnorePerms, false, 0) { return true } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 8bf22f76e..4161c33da 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -953,7 +953,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db default: var fi protocol.FileInfo if fi, err = scanner.CreateFileInfo(stat, target.Name, f.fs); err == nil { - if !fi.IsEquivalentOptional(curTarget, f.IgnorePerms, true, protocol.LocalAllFlags) { + if !fi.IsEquivalentOptional(curTarget, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) { // Target changed scanChan <- target.Name err = errModified @@ -1919,7 +1919,7 @@ func (f *sendReceiveFolder) scanIfItemChanged(stat fs.FileInfo, item protocol.Fi return errors.Wrap(err, "comparing item on disk to db") } - if !statItem.IsEquivalentOptional(item, f.IgnorePerms, true, protocol.LocalAllFlags) { + if !statItem.IsEquivalentOptional(item, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) { return errModified } diff --git a/lib/model/model_test.go b/lib/model/model_test.go index ba1e1f65e..1689df458 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -3305,6 +3305,58 @@ func TestConnCloseOnRestart(t *testing.T) { } } +func TestModTimeWindow(t *testing.T) { + w, fcfg := tmpDefaultWrapper() + tfs := fcfg.Filesystem() + fcfg.RawModTimeWindowS = 2 + w.SetFolder(fcfg) + m := setupModel(w) + defer cleanupModelAndRemoveDir(m, tfs.URI()) + + name := "foo" + + fd, err := tfs.Create(name) + must(t, err) + stat, err := fd.Stat() + must(t, err) + modTime := stat.ModTime() + fd.Close() + + m.ScanFolders() + + v := protocol.Vector{} + v = v.Update(myID.Short()) + fi, ok := m.CurrentFolderFile("default", name) + if !ok { + t.Fatal("File missing") + } + if !fi.Version.Equal(v) { + t.Fatalf("Got version %v, expected %v", fi.Version, v) + } + + err = tfs.Chtimes(name, time.Now(), modTime.Add(time.Second)) + must(t, err) + + m.ScanFolders() + + // No change due to window + fi, _ = m.CurrentFolderFile("default", name) + if !fi.Version.Equal(v) { + t.Fatalf("Got version %v, expected %v", fi.Version, v) + } + + err = tfs.Chtimes(name, time.Now(), modTime.Add(2*time.Second)) + must(t, err) + + m.ScanFolders() + + v = v.Update(myID.Short()) + fi, _ = m.CurrentFolderFile("default", name) + if !fi.Version.Equal(v) { + t.Fatalf("Got version %v, expected %v", fi.Version, v) + } +} + func TestDevicePause(t *testing.T) { sub := events.Default.Subscribe(events.DevicePaused) defer events.Default.Unsubscribe(sub) diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 550d47fbd..b19b14a19 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -158,12 +158,12 @@ func (f FileInfo) IsEmpty() bool { return f.Version.Counters == nil } -func (f FileInfo) IsEquivalent(other FileInfo) bool { - return f.isEquivalent(other, false, false, 0) +func (f FileInfo) IsEquivalent(other FileInfo, modTimeWindow time.Duration) bool { + return f.isEquivalent(other, modTimeWindow, false, false, 0) } -func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { - return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags) +func (f FileInfo) IsEquivalentOptional(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { + return f.isEquivalent(other, modTimeWindow, ignorePerms, ignoreBlocks, ignoreFlags) } // isEquivalent checks that the two file infos represent the same actual file content, @@ -175,13 +175,13 @@ func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreB // - invalid flag // - permissions, unless they are ignored // A file is not "equivalent", if it has different -// - modification time +// - modification time (difference bigger than modTimeWindow) // - size // - blocks, unless there are no blocks to compare (scanning) // A symlink is not "equivalent", if it has different // - target // A directory does not have anything specific to check. -func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { +func (f FileInfo) isEquivalent(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { if f.MustRescan() || other.MustRescan() { // These are per definition not equivalent because they don't // represent a valid state, even if both happen to have the @@ -203,7 +203,7 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo switch f.Type { case FileInfoTypeFile: - return f.Size == other.Size && f.ModTime().Equal(other.ModTime()) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks)) + return f.Size == other.Size && ModTimeEqual(f.ModTime(), other.ModTime(), modTimeWindow) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks)) case FileInfoTypeSymlink: return f.SymlinkTarget == other.SymlinkTarget case FileInfoTypeDirectory: @@ -213,6 +213,17 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo return false } +func ModTimeEqual(a, b time.Time, modTimeWindow time.Duration) bool { + if a.Equal(b) { + return true + } + diff := a.Sub(b) + if diff < 0 { + diff *= -1 + } + return diff < modTimeWindow +} + func PermsEqual(a, b uint32) bool { switch runtime.GOOS { case "windows": diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index a04c1f466..6075d719d 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -770,10 +770,10 @@ func TestIsEquivalent(t *testing.T) { continue } - if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { + if res := tc.a.isEquivalent(tc.b, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } - if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { + if res := tc.b.isEquivalent(tc.a, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } } diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 58d66c38c..92be4c6a6 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -54,6 +54,8 @@ type Config struct { ProgressTickIntervalS int // Local flags to set on scanned files LocalFlags uint32 + // Modification time is to be considered unchanged if the difference is lower. + ModTimeWindow time.Duration } type CurrentFiler interface { @@ -346,7 +348,7 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn f.RawBlockSize = int32(blockSize) if hasCurFile { - if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { + if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) { return nil } if curFile.ShouldConflict() { @@ -379,7 +381,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, f.NoPermissions = w.IgnorePerms if hasCurFile { - if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { + if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) { return nil } if curFile.ShouldConflict() { @@ -423,7 +425,7 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileIn f = w.updateFileInfo(f, curFile) if hasCurFile { - if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { + if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) { return nil } if curFile.ShouldConflict() {