2014-11-16 21:13:20 +01:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
2014-09-29 21:43:32 +02:00
|
|
|
//
|
2015-03-07 21:36:35 +01:00
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 07:52:18 +01:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2014-08-22 00:41:17 +02:00
|
|
|
|
|
|
|
package versioner
|
|
|
|
|
|
|
|
import (
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"time"
|
2014-08-22 18:16:05 +02:00
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/fs"
|
2015-08-06 11:29:25 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/sync"
|
2016-06-26 12:47:23 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/util"
|
2014-08-22 00:41:17 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
// Register the constructor for this type of versioner with the name "staggered"
|
|
|
|
Factories["staggered"] = NewStaggered
|
|
|
|
}
|
|
|
|
|
|
|
|
type Interval struct {
|
|
|
|
step int64
|
|
|
|
end int64
|
|
|
|
}
|
|
|
|
|
|
|
|
type Staggered struct {
|
|
|
|
cleanInterval int64
|
2017-08-19 16:36:56 +02:00
|
|
|
folderFs fs.Filesystem
|
|
|
|
versionsFs fs.Filesystem
|
2014-08-22 00:41:17 +02:00
|
|
|
interval [4]Interval
|
2015-04-23 00:54:31 +02:00
|
|
|
mutex sync.Mutex
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
stop chan struct{}
|
|
|
|
testCleanDone chan struct{}
|
|
|
|
}
|
2016-08-08 20:27:55 +02:00
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
|
2014-08-22 00:41:17 +02:00
|
|
|
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
|
|
|
|
if err != nil {
|
|
|
|
maxAge = 31536000 // Default: ~1 year
|
|
|
|
}
|
|
|
|
cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
|
|
|
|
if err != nil {
|
|
|
|
cleanInterval = 3600 // Default: clean once per hour
|
|
|
|
}
|
|
|
|
|
2019-04-29 00:30:16 +02:00
|
|
|
// Backwards compatibility
|
|
|
|
params["fsPath"] = params["versionsPath"]
|
|
|
|
versionsFs := fsFromParams(folderFs, params)
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
s := &Staggered{
|
2014-08-22 00:41:17 +02:00
|
|
|
cleanInterval: cleanInterval,
|
2017-08-19 16:36:56 +02:00
|
|
|
folderFs: folderFs,
|
|
|
|
versionsFs: versionsFs,
|
2014-08-22 00:41:17 +02:00
|
|
|
interval: [4]Interval{
|
2014-11-16 21:13:20 +01:00
|
|
|
{30, 3600}, // first hour -> 30 sec between versions
|
|
|
|
{3600, 86400}, // next day -> 1 h between versions
|
|
|
|
{86400, 592000}, // next 30 days -> 1 day between versions
|
|
|
|
{604800, maxAge}, // next year -> 1 week between versions
|
2014-08-22 00:41:17 +02:00
|
|
|
},
|
2015-04-23 00:54:31 +02:00
|
|
|
mutex: sync.NewMutex(),
|
2017-02-07 14:31:13 +01:00
|
|
|
stop: make(chan struct{}),
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugf("instantiated %#v", s)
|
2017-02-07 14:31:13 +01:00
|
|
|
return s
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) Serve() {
|
|
|
|
v.clean()
|
|
|
|
if v.testCleanDone != nil {
|
|
|
|
close(v.testCleanDone)
|
|
|
|
}
|
|
|
|
|
|
|
|
tck := time.NewTicker(time.Duration(v.cleanInterval) * time.Second)
|
|
|
|
defer tck.Stop()
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-tck.C:
|
|
|
|
v.clean()
|
|
|
|
case <-v.stop:
|
|
|
|
return
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2017-02-07 14:31:13 +01:00
|
|
|
}
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) Stop() {
|
|
|
|
close(v.stop)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) clean() {
|
2017-08-19 16:36:56 +02:00
|
|
|
l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
|
2014-08-22 00:41:17 +02:00
|
|
|
v.mutex.Lock()
|
|
|
|
defer v.mutex.Unlock()
|
2017-08-19 16:36:56 +02:00
|
|
|
l.Debugln("Versioner clean: Cleaning", v.versionsFs)
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
|
2015-05-03 08:20:44 +02:00
|
|
|
// There is no need to clean a nonexistent dir.
|
|
|
|
return
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
versionsPerFile := make(map[string][]string)
|
2017-11-18 16:56:53 +01:00
|
|
|
dirTracker := make(emptyDirTracker)
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2017-11-18 16:56:53 +01:00
|
|
|
walkFn := func(path string, f fs.FileInfo, err error) error {
|
2014-09-20 22:06:48 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2014-12-03 11:49:53 +01:00
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
if f.IsDir() && !f.IsSymlink() {
|
2017-11-18 16:56:53 +01:00
|
|
|
dirTracker.addDir(path)
|
|
|
|
return nil
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2017-11-18 16:56:53 +01:00
|
|
|
// Regular file, or possibly a symlink.
|
|
|
|
dirTracker.addFile(path)
|
2018-01-01 15:39:23 +01:00
|
|
|
|
|
|
|
name, _ := UntagFilename(path)
|
|
|
|
if name == "" {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-11-18 16:56:53 +01:00
|
|
|
versionsPerFile[name] = append(versionsPerFile[name], path)
|
|
|
|
|
2014-08-22 00:41:17 +02:00
|
|
|
return nil
|
2017-11-18 16:56:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := v.versionsFs.Walk(".", walkFn); err != nil {
|
2014-08-22 18:16:05 +02:00
|
|
|
l.Warnln("Versioner: error scanning versions dir", err)
|
2014-08-23 10:43:48 +02:00
|
|
|
return
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
for _, versionList := range versionsPerFile {
|
|
|
|
// List from filepath.Walk is sorted
|
|
|
|
v.expire(versionList)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-08-23 10:43:48 +02:00
|
|
|
|
2017-11-18 16:56:53 +01:00
|
|
|
dirTracker.deleteEmptyDirs(v.versionsFs)
|
2015-05-03 08:20:44 +02:00
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) expire(versions []string) {
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("Versioner: Expiring versions", versions)
|
2016-04-15 16:26:39 +02:00
|
|
|
for _, file := range v.toRemove(versions, time.Now()) {
|
2017-08-19 16:36:56 +02:00
|
|
|
if fi, err := v.versionsFs.Lstat(file); err != nil {
|
2014-12-03 11:49:53 +01:00
|
|
|
l.Warnln("versioner:", err)
|
|
|
|
continue
|
2016-04-15 16:26:39 +02:00
|
|
|
} else if fi.IsDir() {
|
2014-12-03 11:49:53 +01:00
|
|
|
l.Infof("non-file %q is named like a file version", file)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-08-19 16:36:56 +02:00
|
|
|
if err := v.versionsFs.Remove(file); err != nil {
|
2016-04-15 16:26:39 +02:00
|
|
|
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) toRemove(versions []string, now time.Time) []string {
|
2016-04-15 16:26:39 +02:00
|
|
|
var prevAge int64
|
|
|
|
firstFile := true
|
|
|
|
var remove []string
|
|
|
|
for _, file := range versions {
|
2015-08-19 17:16:31 +02:00
|
|
|
loc, _ := time.LoadLocation("Local")
|
2018-01-01 15:39:23 +01:00
|
|
|
versionTime, err := time.ParseInLocation(TimeFormat, ExtractTag(file), loc)
|
2014-12-03 11:49:53 +01:00
|
|
|
if err != nil {
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugf("Versioner: file name %q is invalid: %v", file, err)
|
2014-12-03 11:49:53 +01:00
|
|
|
continue
|
|
|
|
}
|
2016-04-15 16:26:39 +02:00
|
|
|
age := int64(now.Sub(versionTime).Seconds())
|
2014-12-03 11:49:53 +01:00
|
|
|
|
|
|
|
// If the file is older than the max age of the last interval, remove it
|
|
|
|
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("Versioner: File over maximum age -> delete ", file)
|
2017-08-19 16:36:56 +02:00
|
|
|
err = v.versionsFs.Remove(file)
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
2014-12-03 11:49:53 +01:00
|
|
|
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-12-03 11:49:53 +01:00
|
|
|
continue
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2014-12-03 11:49:53 +01:00
|
|
|
// If it's the first (oldest) file in the list we can skip the interval checks
|
|
|
|
if firstFile {
|
|
|
|
prevAge = age
|
|
|
|
firstFile = false
|
|
|
|
continue
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2014-12-03 11:49:53 +01:00
|
|
|
// Find the interval the file fits in
|
|
|
|
var usedInterval Interval
|
|
|
|
for _, usedInterval = range v.interval {
|
|
|
|
if age < usedInterval.end {
|
|
|
|
break
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-12-03 11:49:53 +01:00
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2014-12-03 11:49:53 +01:00
|
|
|
if prevAge-age < usedInterval.step {
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("too many files in step -> delete", file)
|
2016-04-15 16:26:39 +02:00
|
|
|
remove = append(remove, file)
|
2014-12-03 11:49:53 +01:00
|
|
|
continue
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-12-03 11:49:53 +01:00
|
|
|
|
|
|
|
prevAge = age
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2016-04-15 16:26:39 +02:00
|
|
|
|
|
|
|
return remove
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2015-04-28 22:32:10 +02:00
|
|
|
// Archive moves the named file away to a version archive. If this function
|
|
|
|
// returns nil, the named file does not exist any more (has been archived).
|
2017-02-07 14:31:13 +01:00
|
|
|
func (v *Staggered) Archive(filePath string) error {
|
2017-08-19 16:36:56 +02:00
|
|
|
l.Debugln("Waiting for lock on ", v.versionsFs)
|
2014-08-22 00:41:17 +02:00
|
|
|
v.mutex.Lock()
|
|
|
|
defer v.mutex.Unlock()
|
|
|
|
|
2019-04-29 00:30:16 +02:00
|
|
|
if err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
|
2014-12-08 16:36:15 +01:00
|
|
|
return err
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
file := filepath.Base(filePath)
|
2017-08-19 16:36:56 +02:00
|
|
|
inFolderPath := filepath.Dir(filePath)
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2014-11-24 10:58:57 +01:00
|
|
|
// Glob according to the new file~timestamp.ext pattern.
|
2018-01-01 15:39:23 +01:00
|
|
|
pattern := filepath.Join(inFolderPath, TagFilename(file, TimeGlob))
|
2017-08-19 16:36:56 +02:00
|
|
|
newVersions, err := v.versionsFs.Glob(pattern)
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
2015-12-15 10:04:13 +01:00
|
|
|
l.Warnln("globbing:", err, "for", pattern)
|
2014-08-22 00:41:17 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-11-24 10:58:57 +01:00
|
|
|
// Also according to the old file.ext~timestamp pattern.
|
2017-08-19 16:36:56 +02:00
|
|
|
pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob)
|
|
|
|
oldVersions, err := v.versionsFs.Glob(pattern)
|
2014-11-24 10:58:57 +01:00
|
|
|
if err != nil {
|
2015-12-15 10:04:13 +01:00
|
|
|
l.Warnln("globbing:", err, "for", pattern)
|
2014-11-24 10:58:57 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Use all the found filenames.
|
|
|
|
versions := append(oldVersions, newVersions...)
|
2016-06-26 12:47:23 +02:00
|
|
|
v.expire(util.UniqueStrings(versions))
|
2014-08-22 00:41:17 +02:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2019-04-29 00:30:16 +02:00
|
|
|
|
|
|
|
func (v *Staggered) GetVersions() (map[string][]FileVersion, error) {
|
|
|
|
return retrieveVersions(v.versionsFs)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v *Staggered) Restore(filepath string, versionTime time.Time) error {
|
|
|
|
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
|
|
|
|
}
|