Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add symlink support (fixes #873)

This commit is contained in:
Audrius Butkevicius 2014-11-09 04:26:52 +00:00 committed by Jakob Borg
parent 6e88d9688b
commit c325ffd0f8
5 changed files with 179 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
internal/model/model.go
View File

@ -39,6 +39,7 @@ import (
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/stats"
"github.com/syncthing/syncthing/internal/symlinks"
"github.com/syncthing/syncthing/internal/versioner"
"github.com/syndtr/goleveldb/leveldb"
)
@ -114,6 +115,8 @@ type Model struct {
var (
ErrNoSuchFile = errors.New("no such file")
ErrInvalid = errors.New("file is invalid")
SymlinkWarning = sync.Once{}
)
// NewModel creates and starts a new model. The model starts in read-only mode,
@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
for i := 0; i < len(fs); {
lamport.Default.Tick(fs[i].Version)
if ignores != nil && ignores.Match(fs[i].Name) {
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
if debug {
l.Debugln("dropping update for ignored", fs[i])
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
}
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]
@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
for i := 0; i < len(fs); {
lamport.Default.Tick(fs[i].Version)
if ignores != nil && ignores.Match(fs[i].Name) {
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
if debug {
l.Debugln("dropping update for ignored", fs[i])
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
}
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]
@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
m.fmut.RLock()
fn := filepath.Join(m.folderCfgs[folder].Path, name)
m.fmut.RUnlock()
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
if err != nil {
return nil, err
var reader io.ReaderAt
var err error
if lf.IsSymlink() {
target, _, err := symlinks.Read(fn)
if err != nil {
return nil, err
}
reader = strings.NewReader(target)
} else {
reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
if err != nil {
return nil, err
}
defer reader.(*os.File).Close()
}
defer fd.Close()
buf := make([]byte, size)
_, err = fd.ReadAt(buf, offset)
_, err = reader.ReadAt(buf, offset)
if err != nil {
return nil, err
}
@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer = f.LocalVersion
}
if ignores != nil && ignores.Match(f.Name) {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if debug {
l.Debugln("not sending update for ignored", f)
l.Debugln("not sending update for ignored/unsupported symlink", f)
}
return true
}
@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0]
}
if ignores != nil && ignores.Match(f.Name) {
// File has been ignored. Set invalid bit.
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
// File has been ignored or an unsupported symlink. Set invalid bit.
l.Debugln("setting invalid bit on ignored", f)
nf := protocol.FileInfo{
Name: f.Name,
@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
"size": f.Size(),
})
batch = append(batch, nf)
} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
} else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
// File has been deleted
nf := protocol.FileInfo{
Name: f.Name,
@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() {
}
}
}
func symlinkInvalid(isLink bool) bool {
if !symlinks.Supported && isLink {
SymlinkWarning.Do(func() {
l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.")
})
return true
}
return false
}

56
internal/model/puller.go
View File

@ -20,6 +20,7 @@ import (
"crypto/sha256"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
@ -32,6 +33,7 @@ import (
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/symlinks"
"github.com/syncthing/syncthing/internal/versioner"
)
@ -314,14 +316,15 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo
switch {
case file.IsDeleted():
// A deleted file or directory
// A deleted file, directory or symlink
deletions = append(deletions, file)
case file.IsDirectory():
case file.IsDirectory() && !file.IsSymlink():
// A new or changed directory
p.handleDir(file)
default:
// A new or changed file. This is the only case where we do stuff
// in the background; the other three are done synchronously.
// A new or changed file or symlink. This is the only case where we
// do stuff in the background; the other three are done
// synchronously.
p.handleFile(file, copyChan, finisherChan)
}
@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
if len(curFile.Blocks) == len(file.Blocks) {
for i := range file.Blocks {
if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
goto FilesAreDifferent
}
}
if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
// We are supposed to copy the entire file, and then fetch nothing. We
// are only updating metadata, so we don't actually *need* to make the
// copy.
if debug {
l.Debugln(p, "taking shortcut on", file.Name)
}
p.shortcutFile(file)
if file.IsSymlink() {
p.shortcutSymlink(curFile, file)
} else {
p.shortcutFile(file)
}
return
}
FilesAreDifferent:
scanner.PopulateOffsets(file.Blocks)
// Figure out the absolute filenames we need once and for all
@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
p.model.updateLocal(p.folder, file)
}
// shortcutSymlink changes the symlinks type if necessery.
func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
if err != nil {
l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
return
}
p.model.updateLocal(p.folder, file)
}
// copierRoutine reads copierStates until the in channel closes and performs
// the relevant copies when possible, or passes it to the puller routine.
func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
continue
}
// If it's a symlink, the target of the symlink is inside the file.
if state.file.IsSymlink() {
content, err := ioutil.ReadFile(state.realName)
if err != nil {
l.Warnln("puller: final: reading symlink:", err)
continue
}
// Remove the file, and replace it with a symlink.
err = osutil.InWritableDir(func(path string) error {
os.Remove(path)
return symlinks.Create(path, string(content), state.file.Flags)
}, state.realName)
if err != nil {
l.Warnln("puller: final: creating symlink:", err)
continue
}
}
// Record the updated file in the index
p.model.updateLocal(p.folder, state.file)
}

2
internal/scanner/blockqueue.go
View File

@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
for f := range inbox {
if f.IsDirectory() || f.IsDeleted() {
if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
outbox <- f
continue
}

15
internal/scanner/blocks.go
View File

@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
return nil
}
// BlockEqual returns whether two slices of blocks are exactly the same hash
// and index pair wise.
func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
if len(tgt) != len(src) {
return false
}
for i, sblk := range src {
if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
return false
}
}
return true
}

81
internal/scanner/walk.go
View File

@ -27,6 +27,7 @@ import (
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/symlinks"
)
type Walker struct {
@ -131,6 +132,70 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
return nil
}
// We must perform this check, as symlinks on Windows are always
// .IsRegular or .IsDir unlike on Unix.
// Index wise symlinks are always files, regardless of what the target
// is, because symlinks carry their target path as their content.
isSymlink, _ := symlinks.IsSymlink(p)
if isSymlink {
var rval error
// If the target is a directory, do NOT descend down there.
// This will cause files to get tracked, and removing the symlink
// will as a result remove files in their real location.
// But do not SkipDir if the target is not a directory, as it will
// stop scanning the current directory.
if info.IsDir() {
rval = filepath.SkipDir
}
// We always rehash symlinks as they have no modtime or
// permissions.
// We check if they point to the old target by checking that
// their existing blocks match with the blocks in the index.
// If we don't have a filer or don't support symlinks, skip.
if w.CurrentFiler == nil || !symlinks.Supported {
return rval
}
target, flags, err := symlinks.Read(p)
flags = flags & protocol.SymlinkTypeMask
if err != nil {
if debug {
l.Debugln("readlink error:", p, err)
}
return rval
}
blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
if err != nil {
if debug {
l.Debugln("hash link error:", p, err)
}
return rval
}
cf := w.CurrentFiler.CurrentFile(rn)
if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
return rval
}
f := protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
Modified: 0,
Blocks: blocks,
}
if debug {
l.Debugln("symlink to hash:", p, f)
}
fchan <- f
return rval
}
if info.Mode().IsDir() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool {
return a&0777 == b&0777
}
}
// If the target is missing, Unix never knows what type of symlink it is
// and Windows always knows even if there is no target.
// Which means that without this special check a Unix node would be fighting
// with a Windows node about whether or not the target is known.
// Basically, if you don't know and someone else knows, just accept it.
// The fact that you don't know means you are on Unix, and on Unix you don't
// really care what the target type is. The moment you do know, and if something
// doesn't match, that will propogate throught the cluster.
func SymlinkTypeEqual(disk, index uint32) bool {
if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
return true
}
return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
}