2014-11-16 21:13:20 +01:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
2014-09-29 21:43:32 +02:00
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify it
|
|
|
|
// under the terms of the GNU General Public License as published by the Free
|
|
|
|
// Software Foundation, either version 3 of the License, or (at your option)
|
|
|
|
// any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful, but WITHOUT
|
|
|
|
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
// more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU General Public License along
|
|
|
|
// with this program. If not, see <http://www.gnu.org/licenses/>.
|
2014-08-22 00:41:17 +02:00
|
|
|
|
|
|
|
package versioner
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"time"
|
2014-08-22 18:16:05 +02:00
|
|
|
|
2014-09-22 21:42:11 +02:00
|
|
|
"github.com/syncthing/syncthing/internal/osutil"
|
2014-08-22 00:41:17 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
// Register the constructor for this type of versioner with the name "staggered"
|
|
|
|
Factories["staggered"] = NewStaggered
|
|
|
|
}
|
|
|
|
|
|
|
|
type Interval struct {
|
|
|
|
step int64
|
|
|
|
end int64
|
|
|
|
}
|
|
|
|
|
|
|
|
// The type holds our configuration
|
|
|
|
type Staggered struct {
|
|
|
|
versionsPath string
|
|
|
|
cleanInterval int64
|
2014-09-28 13:05:25 +02:00
|
|
|
folderPath string
|
2014-08-22 00:41:17 +02:00
|
|
|
interval [4]Interval
|
|
|
|
mutex *sync.Mutex
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if file or dir
|
|
|
|
func isFile(path string) bool {
|
2014-08-23 10:37:58 +02:00
|
|
|
fileInfo, err := os.Stat(path)
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
|
|
|
l.Infoln("versioner isFile:", err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return fileInfo.Mode().IsRegular()
|
|
|
|
}
|
|
|
|
|
2014-09-07 13:40:22 +02:00
|
|
|
// Rename versions with old version format
|
|
|
|
func (v Staggered) renameOld() {
|
|
|
|
err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
|
2014-09-11 20:14:27 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2014-09-07 13:40:22 +02:00
|
|
|
if f.Mode().IsRegular() {
|
|
|
|
versionUnix, err := strconv.ParseInt(strings.Replace(filepath.Ext(path), ".v", "", 1), 10, 0)
|
|
|
|
if err == nil {
|
|
|
|
l.Infoln("Renaming file", path, "from old to new version format")
|
|
|
|
versiondate := time.Unix(versionUnix, 0)
|
|
|
|
name := path[:len(path)-len(filepath.Ext(path))]
|
2014-11-24 10:58:57 +01:00
|
|
|
err = osutil.Rename(path, taggedFilename(name, versiondate.Format(TimeFormat)))
|
2014-09-07 13:40:22 +02:00
|
|
|
if err != nil {
|
|
|
|
l.Infoln("Error renaming to new format", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
l.Infoln("Versioner: error scanning versions dir", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-22 00:41:17 +02:00
|
|
|
// The constructor function takes a map of parameters and creates the type.
|
2014-09-28 13:00:38 +02:00
|
|
|
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
|
2014-08-22 00:41:17 +02:00
|
|
|
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
|
|
|
|
if err != nil {
|
|
|
|
maxAge = 31536000 // Default: ~1 year
|
|
|
|
}
|
|
|
|
cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
|
|
|
|
if err != nil {
|
|
|
|
cleanInterval = 3600 // Default: clean once per hour
|
|
|
|
}
|
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
// Use custom path if set, otherwise .stversions in folderPath
|
2014-08-22 00:41:17 +02:00
|
|
|
var versionsDir string
|
|
|
|
if params["versionsPath"] == "" {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("using default dir .stversions")
|
|
|
|
}
|
2014-09-28 13:00:38 +02:00
|
|
|
versionsDir = filepath.Join(folderPath, ".stversions")
|
2014-08-22 00:41:17 +02:00
|
|
|
} else {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("using dir", params["versionsPath"])
|
|
|
|
}
|
|
|
|
versionsDir = params["versionsPath"]
|
|
|
|
}
|
|
|
|
|
|
|
|
var mutex sync.Mutex
|
|
|
|
s := Staggered{
|
|
|
|
versionsPath: versionsDir,
|
|
|
|
cleanInterval: cleanInterval,
|
2014-09-28 13:05:25 +02:00
|
|
|
folderPath: folderPath,
|
2014-08-22 00:41:17 +02:00
|
|
|
interval: [4]Interval{
|
2014-11-16 21:13:20 +01:00
|
|
|
{30, 3600}, // first hour -> 30 sec between versions
|
|
|
|
{3600, 86400}, // next day -> 1 h between versions
|
|
|
|
{86400, 592000}, // next 30 days -> 1 day between versions
|
|
|
|
{604800, maxAge}, // next year -> 1 week between versions
|
2014-08-22 00:41:17 +02:00
|
|
|
},
|
|
|
|
mutex: &mutex,
|
|
|
|
}
|
|
|
|
|
|
|
|
if debug {
|
|
|
|
l.Debugf("instantiated %#v", s)
|
|
|
|
}
|
|
|
|
|
2014-09-07 13:40:22 +02:00
|
|
|
// Rename version with old version format
|
|
|
|
s.renameOld()
|
|
|
|
|
2014-08-22 00:41:17 +02:00
|
|
|
go func() {
|
|
|
|
s.clean()
|
|
|
|
for _ = range time.Tick(time.Duration(cleanInterval) * time.Second) {
|
|
|
|
s.clean()
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v Staggered) clean() {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
|
|
|
|
}
|
|
|
|
v.mutex.Lock()
|
|
|
|
defer v.mutex.Unlock()
|
|
|
|
if debug {
|
|
|
|
l.Debugln("Versioner clean: Cleaning", v.versionsPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err := os.Stat(v.versionsPath)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("creating versions dir", v.versionsPath)
|
|
|
|
}
|
|
|
|
os.MkdirAll(v.versionsPath, 0755)
|
|
|
|
osutil.HideFile(v.versionsPath)
|
|
|
|
} else {
|
2014-08-22 18:16:05 +02:00
|
|
|
l.Warnln("Versioner: can't create versions dir", err)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
versionsPerFile := make(map[string][]string)
|
|
|
|
filesPerDir := make(map[string]int)
|
2014-08-22 00:41:17 +02:00
|
|
|
|
|
|
|
err = filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
|
2014-09-20 22:06:48 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
switch mode := f.Mode(); {
|
|
|
|
case mode.IsDir():
|
2014-08-23 10:43:48 +02:00
|
|
|
filesPerDir[path] = 0
|
2014-09-04 16:48:24 +02:00
|
|
|
if path != v.versionsPath {
|
|
|
|
dir := filepath.Dir(path)
|
|
|
|
filesPerDir[dir]++
|
|
|
|
}
|
2014-08-22 00:41:17 +02:00
|
|
|
case mode.IsRegular():
|
2014-11-24 10:58:57 +01:00
|
|
|
extension := filenameTag(path)
|
2014-08-23 10:43:48 +02:00
|
|
|
dir := filepath.Dir(path)
|
2014-09-07 13:40:22 +02:00
|
|
|
name := path[:len(path)-len(extension)-1]
|
2014-08-23 10:43:48 +02:00
|
|
|
|
|
|
|
filesPerDir[dir]++
|
|
|
|
versionsPerFile[name] = append(versionsPerFile[name], path)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
2014-08-22 18:16:05 +02:00
|
|
|
l.Warnln("Versioner: error scanning versions dir", err)
|
2014-08-23 10:43:48 +02:00
|
|
|
return
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
for _, versionList := range versionsPerFile {
|
|
|
|
// List from filepath.Walk is sorted
|
|
|
|
v.expire(versionList)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-08-23 10:43:48 +02:00
|
|
|
|
|
|
|
for path, numFiles := range filesPerDir {
|
2014-09-04 16:48:24 +02:00
|
|
|
if numFiles > 0 {
|
|
|
|
continue
|
|
|
|
}
|
2014-09-07 13:40:22 +02:00
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
if path == v.versionsPath {
|
2014-08-22 00:41:17 +02:00
|
|
|
if debug {
|
2014-08-23 10:43:48 +02:00
|
|
|
l.Debugln("Cleaner: versions dir is empty, don't delete", path)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
2014-09-04 16:48:24 +02:00
|
|
|
|
2014-08-22 00:41:17 +02:00
|
|
|
if debug {
|
2014-08-23 10:43:48 +02:00
|
|
|
l.Debugln("Cleaner: deleting empty directory", path)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
2014-08-23 10:43:48 +02:00
|
|
|
err = os.Remove(path)
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
2014-08-23 10:43:48 +02:00
|
|
|
l.Warnln("Versioner: can't remove directory", path, err)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if debug {
|
|
|
|
l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
func (v Staggered) expire(versions []string) {
|
2014-08-22 00:41:17 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln("Versioner: Expiring versions", versions)
|
|
|
|
}
|
|
|
|
var prevAge int64
|
|
|
|
firstFile := true
|
|
|
|
for _, file := range versions {
|
|
|
|
if isFile(file) {
|
2014-11-24 10:58:57 +01:00
|
|
|
versionTime, err := time.Parse(TimeFormat, filenameTag(file))
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
2014-08-28 21:35:48 +02:00
|
|
|
l.Infof("Versioner: file name %q is invalid: %v", file, err)
|
2014-08-22 00:41:17 +02:00
|
|
|
continue
|
|
|
|
}
|
2014-09-07 13:40:22 +02:00
|
|
|
age := int64(time.Since(versionTime).Seconds())
|
2014-08-22 00:41:17 +02:00
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
// If the file is older than the max age of the last interval, remove it
|
2014-08-22 18:16:05 +02:00
|
|
|
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
|
2014-08-22 00:41:17 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln("Versioner: File over maximum age -> delete ", file)
|
|
|
|
}
|
|
|
|
err = os.Remove(file)
|
|
|
|
if err != nil {
|
2014-08-23 10:43:48 +02:00
|
|
|
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
// If it's the first (oldest) file in the list we can skip the interval checks
|
2014-08-22 00:41:17 +02:00
|
|
|
if firstFile {
|
|
|
|
prevAge = age
|
|
|
|
firstFile = false
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
// Find the interval the file fits in
|
|
|
|
var usedInterval Interval
|
|
|
|
for _, usedInterval = range v.interval {
|
|
|
|
if age < usedInterval.end {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-22 00:41:17 +02:00
|
|
|
if prevAge-age < usedInterval.step {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("too many files in step -> delete", file)
|
|
|
|
}
|
|
|
|
err = os.Remove(file)
|
|
|
|
if err != nil {
|
2014-08-23 10:43:48 +02:00
|
|
|
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2014-08-23 10:43:48 +02:00
|
|
|
prevAge = age
|
2014-08-22 00:41:17 +02:00
|
|
|
} else {
|
2014-08-28 21:35:48 +02:00
|
|
|
l.Infof("non-file %q is named like a file version", file)
|
2014-08-22 00:41:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Move away the named file to a version archive. If this function returns
|
|
|
|
// nil, the named file does not exist any more (has been archived).
|
|
|
|
func (v Staggered) Archive(filePath string) error {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("Waiting for lock on ", v.versionsPath)
|
|
|
|
}
|
|
|
|
v.mutex.Lock()
|
|
|
|
defer v.mutex.Unlock()
|
|
|
|
|
2014-12-06 13:26:46 +01:00
|
|
|
if _, err := os.Stat(filePath); err != nil {
|
2014-08-22 00:41:17 +02:00
|
|
|
if os.IsNotExist(err) {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("not archiving nonexistent file", filePath)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
} else {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-12-06 13:26:46 +01:00
|
|
|
if _, err := os.Stat(v.versionsPath); err != nil {
|
2014-08-22 00:41:17 +02:00
|
|
|
if os.IsNotExist(err) {
|
|
|
|
if debug {
|
|
|
|
l.Debugln("creating versions dir", v.versionsPath)
|
|
|
|
}
|
|
|
|
os.MkdirAll(v.versionsPath, 0755)
|
|
|
|
osutil.HideFile(v.versionsPath)
|
|
|
|
} else {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if debug {
|
|
|
|
l.Debugln("archiving", filePath)
|
|
|
|
}
|
|
|
|
|
|
|
|
file := filepath.Base(filePath)
|
2014-09-28 13:00:38 +02:00
|
|
|
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
dir := filepath.Join(v.versionsPath, inFolderPath)
|
2014-08-22 00:41:17 +02:00
|
|
|
err = os.MkdirAll(dir, 0755)
|
|
|
|
if err != nil && !os.IsExist(err) {
|
|
|
|
return err
|
|
|
|
}
|
2014-08-23 10:43:48 +02:00
|
|
|
|
2014-12-06 13:26:46 +01:00
|
|
|
ver := taggedFilename(file, time.Now().Format(TimeFormat))
|
2014-08-22 00:41:17 +02:00
|
|
|
dst := filepath.Join(dir, ver)
|
|
|
|
if debug {
|
|
|
|
l.Debugln("moving to", dst)
|
|
|
|
}
|
|
|
|
err = osutil.Rename(filePath, dst)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2014-11-24 10:58:57 +01:00
|
|
|
// Glob according to the new file~timestamp.ext pattern.
|
|
|
|
newVersions, err := filepath.Glob(filepath.Join(dir, taggedFilename(file, TimeGlob)))
|
2014-08-22 00:41:17 +02:00
|
|
|
if err != nil {
|
2014-11-24 10:58:57 +01:00
|
|
|
l.Warnln("globbing:", err)
|
2014-08-22 00:41:17 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-11-24 10:58:57 +01:00
|
|
|
// Also according to the old file.ext~timestamp pattern.
|
|
|
|
oldVersions, err := filepath.Glob(filepath.Join(dir, file+"~"+TimeGlob))
|
|
|
|
if err != nil {
|
|
|
|
l.Warnln("globbing:", err)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Use all the found filenames.
|
|
|
|
versions := append(oldVersions, newVersions...)
|
2014-12-02 19:53:38 +01:00
|
|
|
v.expire(uniqueSortedStrings(versions))
|
2014-08-22 00:41:17 +02:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|