diff --git a/internal/model/model.go b/internal/model/model.go index 5cec18ccd..73d10b604 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -39,6 +39,7 @@ import ( "github.com/syncthing/syncthing/internal/protocol" "github.com/syncthing/syncthing/internal/scanner" "github.com/syncthing/syncthing/internal/stats" + "github.com/syncthing/syncthing/internal/symlinks" "github.com/syncthing/syncthing/internal/versioner" "github.com/syndtr/goleveldb/leveldb" ) @@ -114,6 +115,8 @@ type Model struct { var ( ErrNoSuchFile = errors.New("no such file") ErrInvalid = errors.New("file is invalid") + + SymlinkWarning = sync.Once{} ) // NewModel creates and starts a new model. The model starts in read-only mode, @@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F for i := 0; i < len(fs); { lamport.Default.Tick(fs[i].Version) - if ignores != nil && ignores.Match(fs[i].Name) { + if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) { if debug { - l.Debugln("dropping update for ignored", fs[i]) + l.Debugln("dropping update for ignored/unsupported symlink", fs[i]) } fs[i] = fs[len(fs)-1] fs = fs[:len(fs)-1] @@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot for i := 0; i < len(fs); { lamport.Default.Tick(fs[i].Version) - if ignores != nil && ignores.Match(fs[i].Name) { + if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) { if debug { - l.Debugln("dropping update for ignored", fs[i]) + l.Debugln("dropping update for ignored/unsupported symlink", fs[i]) } fs[i] = fs[len(fs)-1] fs = fs[:len(fs)-1] @@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset m.fmut.RLock() fn := filepath.Join(m.folderCfgs[folder].Path, name) m.fmut.RUnlock() - fd, err := os.Open(fn) // XXX: Inefficient, should cache fd? - if err != nil { - return nil, err + + var reader io.ReaderAt + var err error + if lf.IsSymlink() { + target, _, err := symlinks.Read(fn) + if err != nil { + return nil, err + } + reader = strings.NewReader(target) + } else { + reader, err = os.Open(fn) // XXX: Inefficient, should cache fd? + if err != nil { + return nil, err + } + + defer reader.(*os.File).Close() } - defer fd.Close() buf := make([]byte, size) - _, err = fd.ReadAt(buf, offset) + _, err = reader.ReadAt(buf, offset) if err != nil { return nil, err } @@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol maxLocalVer = f.LocalVersion } - if ignores != nil && ignores.Match(f.Name) { + if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) { if debug { - l.Debugln("not sending update for ignored", f) + l.Debugln("not sending update for ignored/unsupported symlink", f) } return true } @@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error { batch = batch[:0] } - if ignores != nil && ignores.Match(f.Name) { - // File has been ignored. Set invalid bit. + if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) { + // File has been ignored or an unsupported symlink. Set invalid bit. l.Debugln("setting invalid bit on ignored", f) nf := protocol.FileInfo{ Name: f.Name, @@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error { "size": f.Size(), }) batch = append(batch, nf) - } else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) { + } else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) { // File has been deleted nf := protocol.FileInfo{ Name: f.Name, @@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() { } } } + +func symlinkInvalid(isLink bool) bool { + if !symlinks.Supported && isLink { + SymlinkWarning.Do(func() { + l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.") + }) + return true + } + return false +} diff --git a/internal/model/puller.go b/internal/model/puller.go index 3f34ef2ff..d4fbdc8cb 100644 --- a/internal/model/puller.go +++ b/internal/model/puller.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "sync" @@ -32,6 +33,7 @@ import ( "github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/protocol" "github.com/syncthing/syncthing/internal/scanner" + "github.com/syncthing/syncthing/internal/symlinks" "github.com/syncthing/syncthing/internal/versioner" ) @@ -314,14 +316,15 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo switch { case file.IsDeleted(): - // A deleted file or directory + // A deleted file, directory or symlink deletions = append(deletions, file) - case file.IsDirectory(): + case file.IsDirectory() && !file.IsSymlink(): // A new or changed directory p.handleDir(file) default: - // A new or changed file. This is the only case where we do stuff - // in the background; the other three are done synchronously. + // A new or changed file or symlink. This is the only case where we + // do stuff in the background; the other three are done + // synchronously. p.handleFile(file, copyChan, finisherChan) } @@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) { func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) { curFile := p.model.CurrentFolderFile(p.folder, file.Name) - if len(curFile.Blocks) == len(file.Blocks) { - for i := range file.Blocks { - if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) { - goto FilesAreDifferent - } - } + if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) { // We are supposed to copy the entire file, and then fetch nothing. We // are only updating metadata, so we don't actually *need* to make the // copy. if debug { l.Debugln(p, "taking shortcut on", file.Name) } - p.shortcutFile(file) + if file.IsSymlink() { + p.shortcutSymlink(curFile, file) + } else { + p.shortcutFile(file) + } return } -FilesAreDifferent: - scanner.PopulateOffsets(file.Blocks) // Figure out the absolute filenames we need once and for all @@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) { p.model.updateLocal(p.folder, file) } +// shortcutSymlink changes the symlinks type if necessery. +func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) { + err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags) + if err != nil { + l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err) + return + } + + p.model.updateLocal(p.folder, file) +} + // copierRoutine reads copierStates until the in channel closes and performs // the relevant copies when possible, or passes it to the puller routine. func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) { @@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) { continue } + // If it's a symlink, the target of the symlink is inside the file. + if state.file.IsSymlink() { + content, err := ioutil.ReadFile(state.realName) + if err != nil { + l.Warnln("puller: final: reading symlink:", err) + continue + } + + // Remove the file, and replace it with a symlink. + err = osutil.InWritableDir(func(path string) error { + os.Remove(path) + return symlinks.Create(path, string(content), state.file.Flags) + }, state.realName) + if err != nil { + l.Warnln("puller: final: creating symlink:", err) + continue + } + } + // Record the updated file in the index p.model.updateLocal(p.folder, state.file) } diff --git a/internal/scanner/blockqueue.go b/internal/scanner/blockqueue.go index cd0a0a11f..2d1a8656a 100644 --- a/internal/scanner/blockqueue.go +++ b/internal/scanner/blockqueue.go @@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) { func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) { for f := range inbox { - if f.IsDirectory() || f.IsDeleted() { + if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() { outbox <- f continue } diff --git a/internal/scanner/blocks.go b/internal/scanner/blocks.go index 825c12508..2f28b1fe4 100644 --- a/internal/scanner/blocks.go +++ b/internal/scanner/blocks.go @@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error { return nil } + +// BlockEqual returns whether two slices of blocks are exactly the same hash +// and index pair wise. +func BlocksEqual(src, tgt []protocol.BlockInfo) bool { + if len(tgt) != len(src) { + return false + } + + for i, sblk := range src { + if !bytes.Equal(sblk.Hash, tgt[i].Hash) { + return false + } + } + return true +} diff --git a/internal/scanner/walk.go b/internal/scanner/walk.go index bd3ecc4f6..f2efda59a 100644 --- a/internal/scanner/walk.go +++ b/internal/scanner/walk.go @@ -27,6 +27,7 @@ import ( "github.com/syncthing/syncthing/internal/ignore" "github.com/syncthing/syncthing/internal/lamport" "github.com/syncthing/syncthing/internal/protocol" + "github.com/syncthing/syncthing/internal/symlinks" ) type Walker struct { @@ -131,6 +132,70 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun return nil } + // We must perform this check, as symlinks on Windows are always + // .IsRegular or .IsDir unlike on Unix. + // Index wise symlinks are always files, regardless of what the target + // is, because symlinks carry their target path as their content. + isSymlink, _ := symlinks.IsSymlink(p) + if isSymlink { + var rval error + // If the target is a directory, do NOT descend down there. + // This will cause files to get tracked, and removing the symlink + // will as a result remove files in their real location. + // But do not SkipDir if the target is not a directory, as it will + // stop scanning the current directory. + if info.IsDir() { + rval = filepath.SkipDir + } + + // We always rehash symlinks as they have no modtime or + // permissions. + // We check if they point to the old target by checking that + // their existing blocks match with the blocks in the index. + // If we don't have a filer or don't support symlinks, skip. + if w.CurrentFiler == nil || !symlinks.Supported { + return rval + } + + target, flags, err := symlinks.Read(p) + flags = flags & protocol.SymlinkTypeMask + if err != nil { + if debug { + l.Debugln("readlink error:", p, err) + } + return rval + } + + blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0) + if err != nil { + if debug { + l.Debugln("hash link error:", p, err) + } + return rval + } + + cf := w.CurrentFiler.CurrentFile(rn) + if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) { + return rval + } + + f := protocol.FileInfo{ + Name: rn, + Version: lamport.Default.Tick(0), + Flags: protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666, + Modified: 0, + Blocks: blocks, + } + + if debug { + l.Debugln("symlink to hash:", p, f) + } + + fchan <- f + + return rval + } + if info.Mode().IsDir() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) @@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool { return a&0777 == b&0777 } } + +// If the target is missing, Unix never knows what type of symlink it is +// and Windows always knows even if there is no target. +// Which means that without this special check a Unix node would be fighting +// with a Windows node about whether or not the target is known. +// Basically, if you don't know and someone else knows, just accept it. +// The fact that you don't know means you are on Unix, and on Unix you don't +// really care what the target type is. The moment you do know, and if something +// doesn't match, that will propogate throught the cluster. +func SymlinkTypeEqual(disk, index uint32) bool { + if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 { + return true + } + return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask + +}