Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib: Folder marker is now a folder 

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4341
LGTM: calmh
This commit is contained in:
Audrius Butkevicius 2017-09-02 05:52:38 +00:00 committed by Jakob Borg
parent 19e52a10df
commit ab132ff6fe
16 changed files with 143 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/stcompdirs/main.go
View File

@ -88,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
}
rn, _ := filepath.Rel(dir, path)
if rn == "." || rn == ".stfolder" {
if rn == "." {
return nil
}
if rn == ".stversions" {
if rn == ".stversions" || rn == ".stfolder" {
return filepath.SkipDir
}

32
lib/config/config.go
View File

@ -32,7 +32,7 @@ import (
const (
OldestHandledVersion = 10
CurrentVersion = 22
CurrentVersion = 23
MaxRescanIntervalS = 365 * 24 * 60 * 60
)
@ -323,6 +323,9 @@ func (cfg *Configuration) clean() error {
if cfg.Version == 21 {
convertV21V22(cfg)
}
if cfg.Version == 22 {
convertV22V23(cfg)
}
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool)
@ -372,6 +375,33 @@ func (cfg *Configuration) clean() error {
return nil
}
func convertV22V23(cfg *Configuration) {
permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" {
// Windows has no umask so we must chose a safer set of bits to
// begin with.
permBits = 0700
}
for i := range cfg.Folders {
fs := cfg.Folders[i].Filesystem()
// Invalid config posted, or tests.
if fs == nil {
continue
}
if stat, err := fs.Stat(".stfolder"); err == nil && !stat.IsDir() {
err = fs.Remove(".stfolder")
if err == nil {
err = fs.Mkdir(".stfolder", permBits)
}
if err != nil {
l.Fatalln("failed to upgrade folder marker:", err)
}
}
}
cfg.Version = 23
}
func convertV21V22(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic

2
lib/config/config_test.go
View File

@ -86,7 +86,7 @@ func TestDefaultValues(t *testing.T) {
func TestDeviceConfig(t *testing.T) {
for i := OldestHandledVersion; i <= CurrentVersion; i++ {
os.Remove("testdata/.stfolder")
os.RemoveAll("testdata/.stfolder")
wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
if err != nil {
t.Fatal(err)

9
lib/config/folderconfiguration.go
View File

@ -81,12 +81,17 @@ func (f FolderConfiguration) Filesystem() fs.Filesystem {
func (f *FolderConfiguration) CreateMarker() error {
if !f.HasMarker() {
permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" {
// Windows has no umask so we must chose a safer set of bits to
// begin with.
permBits = 0700
}
fs := f.Filesystem()
fd, err := fs.Create(".stfolder")
err := fs.Mkdir(".stfolder", permBits)
if err != nil {
return err
}
fd.Close()
if dir, err := fs.Open("."); err == nil {
if serr := dir.Sync(); err != nil {
l.Infof("fsync %q failed: %v", ".", serr)

16
lib/config/testdata/v23.xml vendored Normal file
View File

@ -0,0 +1,16 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

18
lib/fs/filesystem.go
View File

@ -11,6 +11,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"time"
)
@ -133,3 +134,20 @@ func NewFilesystem(fsType FilesystemType, uri string) Filesystem {
}
return fs
}
// IsInternal returns true if the file, as a path relative to the folder
// root, represents an internal file that should always be ignored. The file
// path must be clean (i.e., in canonical shortest form).
func IsInternal(file string) bool {
internals := []string{".stfolder", ".stignore", ".stversions"}
pathSep := string(PathSeparator)
for _, internal := range internals {
if file == internal {
return true
}
if strings.HasPrefix(file, internal+pathSep) {
return true
}
}
return false
}

43
lib/fs/filesystem_test.go Normal file
View File

@ -0,0 +1,43 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"path/filepath"
"testing"
)
func TestIsInternal(t *testing.T) {
cases := []struct {
file string
internal bool
}{
{".stfolder", true},
{".stignore", true},
{".stversions", true},
{".stfolder/foo", true},
{".stignore/foo", true},
{".stversions/foo", true},
{".stfolderfoo", false},
{".stignorefoo", false},
{".stversionsfoo", false},
{"foo.stfolder", false},
{"foo.stignore", false},
{"foo.stversions", false},
{"foo/.stfolder", false},
{"foo/.stignore", false},
{"foo/.stversions", false},
}
for _, tc := range cases {
res := IsInternal(filepath.FromSlash(tc.file))
if res != tc.internal {
t.Errorf("Unexpected result: IsInteral(%q): %v should be %v", tc.file, res, tc.internal)
}
}
}

2
lib/ignore/tempname.go → lib/fs/tempname.go
View File

@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package ignore
package fs
import (
"crypto/md5"

2
lib/ignore/tempname_test.go → lib/fs/tempname_test.go
View File

@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package ignore
package fs
import (
"strings"

21
lib/ignore/ignore.go
View File

@ -278,10 +278,10 @@ func (m *Matcher) clean(d time.Duration) {
// ShouldIgnore returns true when a file is temporary, internal or ignored
func (m *Matcher) ShouldIgnore(filename string) bool {
switch {
case IsTemporary(filename):
case fs.IsTemporary(filename):
return true
case IsInternal(filename):
case fs.IsInternal(filename):
return true
case m.Match(filename).IsIgnored():
@ -458,23 +458,6 @@ func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd Chan
return lines, patterns, nil
}
// IsInternal returns true if the file, as a path relative to the folder
// root, represents an internal file that should always be ignored. The file
// path must be clean (i.e., in canonical shortest form).
func IsInternal(file string) bool {
internals := []string{".stfolder", ".stignore", ".stversions"}
pathSep := string(fs.PathSeparator)
for _, internal := range internals {
if file == internal {
return true
}
if strings.HasPrefix(file, internal+pathSep) {
return true
}
}
return false
}
// WriteIgnores is a convenience function to avoid code duplication
func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error {
fd, err := osutil.CreateAtomicFilesystem(filesystem, path)

31
lib/ignore/ignore_test.go
View File

@ -837,37 +837,6 @@ func TestGobwasGlobIssue18(t *testing.T) {
}
}
func TestIsInternal(t *testing.T) {
cases := []struct {
file string
internal bool
}{
{".stfolder", true},
{".stignore", true},
{".stversions", true},
{".stfolder/foo", true},
{".stignore/foo", true},
{".stversions/foo", true},
{".stfolderfoo", false},
{".stignorefoo", false},
{".stversionsfoo", false},
{"foo.stfolder", false},
{"foo.stignore", false},
{"foo.stversions", false},
{"foo/.stfolder", false},
{"foo/.stignore", false},
{"foo/.stversions", false},
}
for _, tc := range cases {
res := IsInternal(filepath.FromSlash(tc.file))
if res != tc.internal {
t.Errorf("Unexpected result: IsInteral(%q): %v should be %v", tc.file, res, tc.internal)
}
}
}
func TestRoot(t *testing.T) {
stignore := `
!/a

8
lib/model/model.go
View File

@ -336,7 +336,7 @@ func (m *Model) RemoveFolder(folder string) {
// Delete syncthing specific files
folderCfg := m.folderCfgs[folder]
fs := folderCfg.Filesystem()
fs.Remove(".stfolder")
fs.RemoveAll(".stfolder")
m.tearDownFolderLocked(folder)
// Remove it from the database
@ -1156,7 +1156,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// acceptable relative to "folderPath" and in canonical form, so we can
// trust it.
if ignore.IsInternal(name) {
if fs.IsInternal(name) {
l.Debugf("%v REQ(in) for internal file: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf))
return protocol.ErrNoSuchFile
}
@ -1174,7 +1174,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// Only check temp files if the flag is set, and if we are set to advertise
// the temp indexes.
if fromTemporary && !folderCfg.DisableTempIndexes {
tempFn := ignore.TempName(name)
tempFn := fs.TempName(name)
if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() {
// Reject reads for anything that doesn't exist or is something
@ -2552,7 +2552,7 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
func trimUntilParentKnown(dirs []string, exists func(dir string) bool) []string {
var subs []string
for _, sub := range dirs {
for sub != "" && !ignore.IsInternal(sub) {
for sub != "" && !fs.IsInternal(sub) {
sub = filepath.Clean(sub)
parent := filepath.Dir(sub)
if parent == "." || exists(parent) {

3
lib/model/model_test.go
View File

@ -1026,7 +1026,8 @@ func changeIgnores(t *testing.T, m *Model, expected []string) {
func TestIgnores(t *testing.T) {
// Assure a clean start state
ioutil.WriteFile("testdata/.stfolder", nil, 0644)
os.RemoveAll("testdata/.stfolder")
os.MkdirAll("testdata/.stfolder", 0644)
ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
db := db.OpenMemory()

4
lib/model/rwfolder.go
View File

@ -760,7 +760,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Ma
files, _ := f.fs.DirNames(file.Name)
for _, dirFile := range files {
fullDirFile := filepath.Join(file.Name, dirFile)
if ignore.IsTemporary(dirFile) || (matcher != nil && matcher.Match(fullDirFile).IsDeletable()) {
if fs.IsTemporary(dirFile) || (matcher != nil && matcher.Match(fullDirFile).IsDeletable()) {
f.fs.RemoveAll(fullDirFile)
}
}
@ -988,7 +988,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
return
}
tempName := ignore.TempName(file.Name)
tempName := fs.TempName(file.Name)
scanner.PopulateOffsets(file.Blocks)

23
lib/model/rwfolder_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/sync"
@ -27,15 +26,15 @@ func TestMain(m *testing.M) {
// We do this to make sure that the temp file required for the tests
// does not get removed during the tests. Also set the prefix so it's
// found correctly regardless of platform.
if ignore.TempPrefix != ignore.WindowsTempPrefix {
originalPrefix := ignore.TempPrefix
ignore.TempPrefix = ignore.WindowsTempPrefix
if fs.TempPrefix != fs.WindowsTempPrefix {
originalPrefix := fs.TempPrefix
fs.TempPrefix = fs.WindowsTempPrefix
defer func() {
ignore.TempPrefix = originalPrefix
fs.TempPrefix = originalPrefix
}()
}
future := time.Now().Add(time.Hour)
err := os.Chtimes(filepath.Join("testdata", ignore.TempName("file")), future, future)
err := os.Chtimes(filepath.Join("testdata", fs.TempName("file")), future, future)
if err != nil {
panic(err)
}
@ -191,14 +190,14 @@ func TestCopierFinder(t *testing.T) {
// After dropping out blocks found locally:
// Pull: 1, 5, 6, 8
tempFile := filepath.Join("testdata", ignore.TempName("file2"))
tempFile := filepath.Join("testdata", fs.TempName("file2"))
err := os.Remove(tempFile)
if err != nil && !os.IsNotExist(err) {
t.Error(err)
}
existingBlocks := []int{0, 2, 3, 4, 0, 0, 7, 0}
existingFile := setUpFile(ignore.TempName("file"), existingBlocks)
existingFile := setUpFile(fs.TempName("file"), existingBlocks)
requiredFile := existingFile
requiredFile.Blocks = blocks[1:]
requiredFile.Name = "file2"
@ -261,7 +260,7 @@ func TestCopierFinder(t *testing.T) {
}
func TestWeakHash(t *testing.T) {
tempFile := filepath.Join("testdata", ignore.TempName("weakhash"))
tempFile := filepath.Join("testdata", fs.TempName("weakhash"))
var shift int64 = 10
var size int64 = 1 << 20
expectBlocks := int(size / protocol.BlockSize)
@ -466,12 +465,12 @@ func TestLastResortPulling(t *testing.T) {
}
(<-finisherChan).fd.Close()
os.Remove(filepath.Join("testdata", ignore.TempName("newfile")))
os.Remove(filepath.Join("testdata", fs.TempName("newfile")))
}
func TestDeregisterOnFailInCopy(t *testing.T) {
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
defer os.Remove("testdata/" + ignore.TempName("filex"))
defer os.Remove("testdata/" + fs.TempName("filex"))
db := db.OpenMemory()
@ -545,7 +544,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
func TestDeregisterOnFailInPull(t *testing.T) {
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
defer os.Remove("testdata/" + ignore.TempName("filex"))
defer os.Remove("testdata/" + fs.TempName("filex"))
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "syncthing", "dev", db, nil)

4
lib/scanner/walk.go
View File

@ -230,7 +230,7 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
return skip
}
if ignore.IsTemporary(path) {
if fs.IsTemporary(path) {
l.Debugln("temporary:", path)
if info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
w.Filesystem.Remove(path)
@ -239,7 +239,7 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
return nil
}
if ignore.IsInternal(path) {
if fs.IsInternal(path) {
l.Debugln("ignored (internal):", path)
return skip
}