Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model, lib/osutil: Verify target directory before pulling / requesting 

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3798
This commit is contained in:
Jakob Borg 2016-12-13 10:24:10 +00:00
parent 5070d52f2f
commit 3582783972
6 changed files with 432 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
lib/model/model.go
View File

@ -120,6 +120,7 @@ var (
errDevicePaused = errors.New("device is paused")
errDeviceIgnored = errors.New("device is ignored")
errNotRelative = errors.New("not a relative path")
errNotDir = errors.New("parent is not a directory")
)
// NewModel creates and starts a new model. The model starts in read-only mode,
@ -577,7 +578,7 @@ func (m *Model) NeedSize(folder string) db.Counts {
ignores := m.folderIgnores[folder]
cfg := m.folderCfgs[folder]
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if shouldIgnore(f, ignores, cfg.IgnoreDelete) {
if shouldIgnore(f, ignores, cfg.IgnoreDelete, defTempNamer) {
return true
}
@ -641,7 +642,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
ignores := m.folderIgnores[folder]
cfg := m.folderCfgs[folder]
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if shouldIgnore(f, ignores, cfg.IgnoreDelete) {
if shouldIgnore(f, ignores, cfg.IgnoreDelete, defTempNamer) {
return true
}
@ -1113,26 +1114,22 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
return protocol.ErrNoSuchFile
}
if info, err := osutil.Lstat(fn); err == nil && info.Mode()&os.ModeSymlink != 0 {
target, _, err := symlinks.Read(fn)
if err != nil {
l.Debugln("symlinks.Read:", err)
if os.IsNotExist(err) {
return protocol.ErrNoSuchFile
}
return protocol.ErrGeneric
}
if _, err := strings.NewReader(target).ReadAt(buf, offset); err != nil {
l.Debugln("symlink.Reader.ReadAt", err)
return protocol.ErrGeneric
}
return nil
if !osutil.IsDir(folderPath, filepath.Dir(name)) {
l.Debugf("%v REQ(in) for file not in dir: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf))
return protocol.ErrNoSuchFile
}
// Only check temp files if the flag is set, and if we are set to advertise
// the temp indexes.
if fromTemporary && !folderCfg.DisableTempIndexes {
tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
if info, err := osutil.Lstat(tempFn); err != nil || !info.Mode().IsRegular() {
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
return protocol.ErrNoSuchFile
}
if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
return nil
}
@ -1140,6 +1137,12 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// file has finished downloading.
}
if info, err := osutil.Lstat(fn); err != nil || !info.Mode().IsRegular() {
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
return protocol.ErrNoSuchFile
}
err = readOffsetIntoBuf(fn, offset, buf)
if os.IsNotExist(err) {
return protocol.ErrNoSuchFile
@ -2537,7 +2540,7 @@ func makeForgetUpdate(files []protocol.FileInfo) []protocol.FileDownloadProgress
}
// shouldIgnore returns true when a file should be excluded from processing
func shouldIgnore(file db.FileIntf, matcher *ignore.Matcher, ignoreDelete bool) bool {
func shouldIgnore(file db.FileIntf, matcher *ignore.Matcher, ignoreDelete bool, tmpNamer tempNamer) bool {
switch {
case ignoreDelete && file.IsDeleted():
// ignoreDelete first because it's a very cheap test so a win if it
@ -2545,6 +2548,9 @@ func shouldIgnore(file db.FileIntf, matcher *ignore.Matcher, ignoreDelete bool)
// deleted files.
return true
case tmpNamer.IsTemporary(file.FileName()):
return true
case ignore.IsInternal(file.FileName()):
return true

43
lib/model/model_test.go
View File

@ -227,6 +227,7 @@ type fakeConnection struct {
folder string
model *Model
indexFn func(string, []protocol.FileInfo)
requestFn func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
mut sync.Mutex
}
@ -271,6 +272,11 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
}
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
f.mut.Lock()
defer f.mut.Unlock()
if f.requestFn != nil {
return f.requestFn(folder, name, offset, size, hash, fromTemporary)
}
return f.fileData[name], nil
}
@ -307,24 +313,35 @@ func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.File
})
}
func (f *fakeConnection) addFile(name string, flags uint32, data []byte) {
func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileInfoType, data []byte) {
f.mut.Lock()
defer f.mut.Unlock()
blocks, _ := scanner.Blocks(bytes.NewReader(data), protocol.BlockSize, int64(len(data)), nil)
var version protocol.Vector
version.Update(f.id.Short())
version = version.Update(f.id.Short())
f.files = append(f.files, protocol.FileInfo{
Name: name,
Type: protocol.FileInfoTypeFile,
Size: int64(len(data)),
ModifiedS: time.Now().Unix(),
Permissions: flags,
Version: version,
Sequence: time.Now().UnixNano(),
Blocks: blocks,
})
if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory {
f.files = append(f.files, protocol.FileInfo{
Name: name,
Type: ftype,
Size: int64(len(data)),
ModifiedS: time.Now().Unix(),
Permissions: flags,
Version: version,
Sequence: time.Now().UnixNano(),
Blocks: blocks,
})
} else {
// Symlink
f.files = append(f.files, protocol.FileInfo{
Name: name,
Type: ftype,
Version: version,
Sequence: time.Now().UnixNano(),
SymlinkTarget: string(data),
})
}
if f.fileData == nil {
f.fileData = make(map[string][]byte)
@ -349,7 +366,7 @@ func BenchmarkRequestOut(b *testing.B) {
fc := &fakeConnection{id: device1}
for _, f := range files {
fc.addFile(f.Name, 0644, []byte("some data to return"))
fc.addFile(f.Name, 0644, protocol.FileInfoTypeFile, []byte("some data to return"))
}
m.AddConnection(fc, protocol.HelloResult{})
m.Index(device1, "default", files)

149
lib/model/requests_test.go
View File

@ -10,7 +10,10 @@ import (
"bytes"
"io/ioutil"
"os"
"runtime"
"strings"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
@ -42,7 +45,7 @@ func TestRequestSimple(t *testing.T) {
// Send an update for the test file, wait for it to sync and be reported back.
contents := []byte("test file contents\n")
fc.addFile("testfile", 0644, contents)
fc.addFile("testfile", 0644, protocol.FileInfoTypeFile, contents)
fc.sendIndexUpdate()
<-done
@ -57,6 +60,150 @@ func TestRequestSimple(t *testing.T) {
}
}
func TestSymlinkTraversalRead(t *testing.T) {
// Verify that a symlink can not be traversed for reading.
if runtime.GOOS == "windows" {
t.Skip("no symlink support on CI")
return
}
defer os.RemoveAll("_tmpfolder")
m, fc := setupModelWithConnection()
defer m.Stop()
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
done := make(chan struct{})
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
for _, f := range fs {
if f.Name == "symlink" {
close(done)
return
}
}
}
fc.mut.Unlock()
// Send an update for the symlink, wait for it to sync and be reported back.
contents := []byte("..")
fc.addFile("symlink", 0644, protocol.FileInfoTypeSymlinkDirectory, contents)
fc.sendIndexUpdate()
<-done
// Request a file by traversing the symlink
buf := make([]byte, 10)
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, false, buf)
if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
t.Error("Managed to traverse symlink")
}
}
func TestSymlinkTraversalWrite(t *testing.T) {
// Verify that a symlink can not be traversed for writing.
if runtime.GOOS == "windows" {
t.Skip("no symlink support on CI")
return
}
defer os.RemoveAll("_tmpfolder")
m, fc := setupModelWithConnection()
defer m.Stop()
// We listen for incoming index updates and trigger when we see one for
// the expected names.
done := make(chan struct{}, 1)
badReq := make(chan string, 1)
badIdx := make(chan string, 1)
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
for _, f := range fs {
if f.Name == "symlink" {
done <- struct{}{}
return
}
if strings.HasPrefix(f.Name, "symlink") {
badIdx <- f.Name
return
}
}
}
fc.requestFn = func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
if name != "symlink" && strings.HasPrefix(name, "symlink") {
badReq <- name
}
return fc.fileData[name], nil
}
fc.mut.Unlock()
// Send an update for the symlink, wait for it to sync and be reported back.
contents := []byte("..")
fc.addFile("symlink", 0644, protocol.FileInfoTypeSymlinkDirectory, contents)
fc.sendIndexUpdate()
<-done
// Send an update for things behind the symlink, wait for requests for
// blocks for any of them to come back, or index entries. Hopefully none
// of that should happen.
contents = []byte("testdata testdata\n")
fc.addFile("symlink/testfile", 0644, protocol.FileInfoTypeFile, contents)
fc.addFile("symlink/testdir", 0644, protocol.FileInfoTypeDirectory, contents)
fc.addFile("symlink/testsyml", 0644, protocol.FileInfoTypeSymlinkFile, contents)
fc.sendIndexUpdate()
select {
case name := <-badReq:
t.Fatal("Should not have requested the data for", name)
case name := <-badIdx:
t.Fatal("Should not have sent the index entry for", name)
case <-time.After(3 * time.Second):
// Unfortunately not much else to trigger on here. The puller sleep
// interval is 1s so if we didn't get any requests within two
// iterations we should be fine.
}
}
func TestRequestCreateTmpSymlink(t *testing.T) {
// Verify that the model performs a request and creates a file based on
// an incoming index update.
defer os.RemoveAll("_tmpfolder")
m, fc := setupModelWithConnection()
defer m.Stop()
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
badIdx := make(chan string)
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
for _, f := range fs {
if f.Name == ".syncthing.testlink.tmp" {
badIdx <- f.Name
return
}
}
}
fc.mut.Unlock()
// Send an update for the test file, wait for it to sync and be reported back.
fc.addFile(".syncthing.testlink.tmp", 0644, protocol.FileInfoTypeSymlinkDirectory, []byte(".."))
fc.sendIndexUpdate()
select {
case name := <-badIdx:
t.Fatal("Should not have sent the index entry for", name)
case <-time.After(3 * time.Second):
// Unfortunately not much else to trigger on here. The puller sleep
// interval is 1s so if we didn't get any requests within two
// iterations we should be fine.
}
}
func setupModelWithConnection() (*Model, *fakeConnection) {
cfg := defaultConfig.RawCopy()
cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder")

163
lib/model/rwfolder.go
View File

@ -382,23 +382,81 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
folderFiles := f.model.folderFiles[f.folderID]
f.model.fmut.RUnlock()
// !!!
// WithNeed takes a database snapshot (by necessity). By the time we've
// handled a bunch of files it might have become out of date and we might
// be attempting to sync with an old version of a file...
// !!!
changed := 0
var processDirectly []protocol.FileInfo
// Iterate the list of items that we need and sort them into piles.
// Regular files to pull goes into the file queue, everything else
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
if shouldIgnore(intf, ignores, f.ignoreDelete, defTempNamer) {
return true
}
if err := fileValid(intf); err != nil {
// The file isn't valid so we can't process it. Pretend that we
// tried and set the error for the file.
f.newError(intf.FileName(), err)
changed++
return true
}
file := intf.(protocol.FileInfo)
switch {
case file.IsDeleted():
processDirectly = append(processDirectly, file)
changed++
case file.Type == protocol.FileInfoTypeFile:
// Queue files for processing after directories and symlinks, if
// it has availability.
devices := folderFiles.Availability(file.Name)
for _, dev := range devices {
if f.model.ConnectedTo(dev) {
f.queue.Push(file.Name, file.Size, file.ModTime())
changed++
break
}
}
default:
// Directories, symlinks
processDirectly = append(processDirectly, file)
changed++
}
return true
})
// Sort the "process directly" pile by number of path components. This
// ensures that we handle parents before children.
sort.Sort(byComponentCount(processDirectly))
// Process the list.
fileDeletions := map[string]protocol.FileInfo{}
dirDeletions := []protocol.FileInfo{}
buckets := map[string][]protocol.FileInfo{}
handleItem := func(fi protocol.FileInfo) bool {
for _, fi := range processDirectly {
// Verify that the thing we are handling lives inside a directory,
// and not a symlink or empty space.
if !osutil.IsDir(f.dir, filepath.Dir(fi.Name)) {
f.newError(fi.Name, errNotDir)
continue
}
switch {
case fi.IsDeleted():
// A deleted file, directory or symlink
if fi.IsDirectory() {
// Perform directory deletions at the end, as we may have
// files to delete inside them before we get to that point.
dirDeletions = append(dirDeletions, fi)
} else {
fileDeletions[fi.Name] = fi
@ -413,63 +471,22 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
buckets[key] = append(buckets[key], df)
}
}
changed++
case fi.IsDirectory() && !fi.IsSymlink():
l.Debugln("Handling directory", fi.Name)
f.handleDir(fi)
changed++
case fi.IsSymlink():
l.Debugln("Handling symlink", fi.Name)
f.handleSymlink(fi)
changed++
default:
return false
l.Warnln(fi)
panic("unhandleable item type, can't happen")
}
return true
}
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
// Needed items are delivered sorted lexicographically. We'll handle
// directories as they come along, so parents before children. Files
// are queued and the order may be changed later.
if shouldIgnore(intf, ignores, f.ignoreDelete) {
return true
}
if err := fileValid(intf); err != nil {
// The file isn't valid so we can't process it. Pretend that we
// tried and set the error for the file.
f.newError(intf.FileName(), err)
changed++
return true
}
file := intf.(protocol.FileInfo)
l.Debugln(f, "handling", file.Name)
if !handleItem(file) {
// A new or changed file or symlink. This is the only case where
// we do stuff concurrently in the background. We only queue
// files where we are connected to at least one device that has
// the file.
devices := folderFiles.Availability(file.Name)
for _, dev := range devices {
if f.model.ConnectedTo(dev) {
f.queue.Push(file.Name, file.Size, file.ModTime())
changed++
break
}
}
}
return true
})
// Reorder the file queue according to configuration
// Now do the file queue. Reorder it according to configuration.
switch f.order {
case config.OrderRandom:
@ -486,7 +503,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
f.queue.SortNewestFirst()
}
// Process the file queue
// Process the file queue.
nextFile:
for {
@ -509,10 +526,17 @@ nextFile:
continue
}
// Handles races where an index update arrives changing what the file
// is between queueing and retrieving it from the queue, effectively
// changing how the file should be handled.
if handleItem(fi) {
if fi.IsDeleted() || fi.Type != protocol.FileInfoTypeFile {
// The item has changed type or status in the index while we
// were processing directories above.
f.queue.Done(fileName)
continue
}
// Verify that the thing we are handling lives inside a directory,
// and not a symlink or empty space.
if !osutil.IsDir(f.dir, filepath.Dir(fi.Name)) {
f.newError(fi.Name, errNotDir)
continue
}
@ -779,6 +803,7 @@ func (f *rwFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Matcher) {
f.newError(file.Name, err)
return
}
// Delete any temporary files lying around in the directory
dir, _ := os.Open(realName)
if dir != nil {
@ -1727,3 +1752,29 @@ func windowsInvalidFilename(name string) bool {
// The path must not contain any disallowed characters
return strings.ContainsAny(name, windowsDisallowedCharacters)
}
// byComponentCount sorts by the number of path components in Name, that is
// "x/y" sorts before "foo/bar/baz".
type byComponentCount []protocol.FileInfo
func (l byComponentCount) Len() int {
return len(l)
}
func (l byComponentCount) Less(a, b int) bool {
return componentCount(l[a].Name) < componentCount(l[b].Name)
}
func (l byComponentCount) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func componentCount(name string) int {
count := 0
for _, codepoint := range name {
if codepoint == os.PathSeparator {
count++
}
}
return count
}

49
lib/osutil/isdir.go Normal file
View File

@ -0,0 +1,49 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"os"
"path/filepath"
"strings"
)
// IsDir returns true if base and every path component of name up to and
// including filepath.Join(base, name) is a directory (and not a symlink or
// similar). Base and name must both be clean and name must be relative to
// base.
func IsDir(base, name string) bool {
path := base
info, err := Lstat(path)
if err != nil {
return false
}
if !info.IsDir() {
return false
}
if name == "." {
// The result of calling IsDir("some/where", filepath.Dir("foo"))
return true
}
parts := strings.Split(name, string(os.PathSeparator))
for _, part := range parts {
path = filepath.Join(path, part)
info, err := Lstat(path)
if err != nil {
return false
}
if info.Mode()&os.ModeSymlink != 0 {
return false
}
if !info.IsDir() {
return false
}
}
return true
}

75
lib/osutil/isdir_test.go Normal file
View File

@ -0,0 +1,75 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil_test
import (
"os"
"testing"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/symlinks"
)
func TestIsDir(t *testing.T) {
if !symlinks.Supported {
t.Skip("pointless test")
return
}
os.RemoveAll("testdata")
defer os.RemoveAll("testdata")
os.MkdirAll("testdata/a/b/c", 0755)
symlinks.Create("testdata/a/l", "b", symlinks.TargetDirectory)
// a/l -> b, so a/l/c should resolve by normal stat
info, err := osutil.Lstat("testdata/a/l/c")
if err != nil {
t.Fatal("unexpected error", err)
}
if !info.IsDir() {
t.Fatal("error in setup, a/l/c should be a directory")
}
cases := []struct {
name string
isDir bool
}{
// Exist
{".", true},
{"a", true},
{"a/b", true},
{"a/b/c", true},
// Don't exist
{"x", false},
{"a/x", false},
{"a/b/x", false},
{"a/x/c", false},
// Symlink or behind symlink
{"a/l", false},
{"a/l/c", false},
}
for _, tc := range cases {
if res := osutil.IsDir("testdata", tc.name); res != tc.isDir {
t.Errorf("IsDir(%q) = %v, should be %v", tc.name, res, tc.isDir)
}
}
}
var isDirResult bool
func BenchmarkIsDir(b *testing.B) {
os.RemoveAll("testdata")
defer os.RemoveAll("testdata")
os.MkdirAll("testdata/a/b/c", 0755)
for i := 0; i < b.N; i++ {
isDirResult = osutil.IsDir("testdata", "a/b/c")
}
b.ReportAllocs()
}