Add symlink support (fixes #873)
This commit is contained in:
parent
6e88d9688b
commit
c325ffd0f8
|
@ -39,6 +39,7 @@ import (
|
||||||
"github.com/syncthing/syncthing/internal/protocol"
|
"github.com/syncthing/syncthing/internal/protocol"
|
||||||
"github.com/syncthing/syncthing/internal/scanner"
|
"github.com/syncthing/syncthing/internal/scanner"
|
||||||
"github.com/syncthing/syncthing/internal/stats"
|
"github.com/syncthing/syncthing/internal/stats"
|
||||||
|
"github.com/syncthing/syncthing/internal/symlinks"
|
||||||
"github.com/syncthing/syncthing/internal/versioner"
|
"github.com/syncthing/syncthing/internal/versioner"
|
||||||
"github.com/syndtr/goleveldb/leveldb"
|
"github.com/syndtr/goleveldb/leveldb"
|
||||||
)
|
)
|
||||||
|
@ -114,6 +115,8 @@ type Model struct {
|
||||||
var (
|
var (
|
||||||
ErrNoSuchFile = errors.New("no such file")
|
ErrNoSuchFile = errors.New("no such file")
|
||||||
ErrInvalid = errors.New("file is invalid")
|
ErrInvalid = errors.New("file is invalid")
|
||||||
|
|
||||||
|
SymlinkWarning = sync.Once{}
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewModel creates and starts a new model. The model starts in read-only mode,
|
// NewModel creates and starts a new model. The model starts in read-only mode,
|
||||||
|
@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
|
||||||
|
|
||||||
for i := 0; i < len(fs); {
|
for i := 0; i < len(fs); {
|
||||||
lamport.Default.Tick(fs[i].Version)
|
lamport.Default.Tick(fs[i].Version)
|
||||||
if ignores != nil && ignores.Match(fs[i].Name) {
|
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
|
||||||
if debug {
|
if debug {
|
||||||
l.Debugln("dropping update for ignored", fs[i])
|
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
|
||||||
}
|
}
|
||||||
fs[i] = fs[len(fs)-1]
|
fs[i] = fs[len(fs)-1]
|
||||||
fs = fs[:len(fs)-1]
|
fs = fs[:len(fs)-1]
|
||||||
|
@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
|
||||||
|
|
||||||
for i := 0; i < len(fs); {
|
for i := 0; i < len(fs); {
|
||||||
lamport.Default.Tick(fs[i].Version)
|
lamport.Default.Tick(fs[i].Version)
|
||||||
if ignores != nil && ignores.Match(fs[i].Name) {
|
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
|
||||||
if debug {
|
if debug {
|
||||||
l.Debugln("dropping update for ignored", fs[i])
|
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
|
||||||
}
|
}
|
||||||
fs[i] = fs[len(fs)-1]
|
fs[i] = fs[len(fs)-1]
|
||||||
fs = fs[:len(fs)-1]
|
fs = fs[:len(fs)-1]
|
||||||
|
@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
|
||||||
m.fmut.RLock()
|
m.fmut.RLock()
|
||||||
fn := filepath.Join(m.folderCfgs[folder].Path, name)
|
fn := filepath.Join(m.folderCfgs[folder].Path, name)
|
||||||
m.fmut.RUnlock()
|
m.fmut.RUnlock()
|
||||||
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
|
|
||||||
|
var reader io.ReaderAt
|
||||||
|
var err error
|
||||||
|
if lf.IsSymlink() {
|
||||||
|
target, _, err := symlinks.Read(fn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer fd.Close()
|
reader = strings.NewReader(target)
|
||||||
|
} else {
|
||||||
|
reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer reader.(*os.File).Close()
|
||||||
|
}
|
||||||
|
|
||||||
buf := make([]byte, size)
|
buf := make([]byte, size)
|
||||||
_, err = fd.ReadAt(buf, offset)
|
_, err = reader.ReadAt(buf, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
|
||||||
maxLocalVer = f.LocalVersion
|
maxLocalVer = f.LocalVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
if ignores != nil && ignores.Match(f.Name) {
|
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
|
||||||
if debug {
|
if debug {
|
||||||
l.Debugln("not sending update for ignored", f)
|
l.Debugln("not sending update for ignored/unsupported symlink", f)
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
|
||||||
batch = batch[:0]
|
batch = batch[:0]
|
||||||
}
|
}
|
||||||
|
|
||||||
if ignores != nil && ignores.Match(f.Name) {
|
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
|
||||||
// File has been ignored. Set invalid bit.
|
// File has been ignored or an unsupported symlink. Set invalid bit.
|
||||||
l.Debugln("setting invalid bit on ignored", f)
|
l.Debugln("setting invalid bit on ignored", f)
|
||||||
nf := protocol.FileInfo{
|
nf := protocol.FileInfo{
|
||||||
Name: f.Name,
|
Name: f.Name,
|
||||||
|
@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
|
||||||
"size": f.Size(),
|
"size": f.Size(),
|
||||||
})
|
})
|
||||||
batch = append(batch, nf)
|
batch = append(batch, nf)
|
||||||
} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
|
} else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
|
||||||
// File has been deleted
|
// File has been deleted
|
||||||
nf := protocol.FileInfo{
|
nf := protocol.FileInfo{
|
||||||
Name: f.Name,
|
Name: f.Name,
|
||||||
|
@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func symlinkInvalid(isLink bool) bool {
|
||||||
|
if !symlinks.Supported && isLink {
|
||||||
|
SymlinkWarning.Do(func() {
|
||||||
|
l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.")
|
||||||
|
})
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -32,6 +33,7 @@ import (
|
||||||
"github.com/syncthing/syncthing/internal/osutil"
|
"github.com/syncthing/syncthing/internal/osutil"
|
||||||
"github.com/syncthing/syncthing/internal/protocol"
|
"github.com/syncthing/syncthing/internal/protocol"
|
||||||
"github.com/syncthing/syncthing/internal/scanner"
|
"github.com/syncthing/syncthing/internal/scanner"
|
||||||
|
"github.com/syncthing/syncthing/internal/symlinks"
|
||||||
"github.com/syncthing/syncthing/internal/versioner"
|
"github.com/syncthing/syncthing/internal/versioner"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -314,14 +316,15 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case file.IsDeleted():
|
case file.IsDeleted():
|
||||||
// A deleted file or directory
|
// A deleted file, directory or symlink
|
||||||
deletions = append(deletions, file)
|
deletions = append(deletions, file)
|
||||||
case file.IsDirectory():
|
case file.IsDirectory() && !file.IsSymlink():
|
||||||
// A new or changed directory
|
// A new or changed directory
|
||||||
p.handleDir(file)
|
p.handleDir(file)
|
||||||
default:
|
default:
|
||||||
// A new or changed file. This is the only case where we do stuff
|
// A new or changed file or symlink. This is the only case where we
|
||||||
// in the background; the other three are done synchronously.
|
// do stuff in the background; the other three are done
|
||||||
|
// synchronously.
|
||||||
p.handleFile(file, copyChan, finisherChan)
|
p.handleFile(file, copyChan, finisherChan)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
|
||||||
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
|
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
|
||||||
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
|
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
|
||||||
|
|
||||||
if len(curFile.Blocks) == len(file.Blocks) {
|
if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
|
||||||
for i := range file.Blocks {
|
|
||||||
if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
|
|
||||||
goto FilesAreDifferent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// We are supposed to copy the entire file, and then fetch nothing. We
|
// We are supposed to copy the entire file, and then fetch nothing. We
|
||||||
// are only updating metadata, so we don't actually *need* to make the
|
// are only updating metadata, so we don't actually *need* to make the
|
||||||
// copy.
|
// copy.
|
||||||
if debug {
|
if debug {
|
||||||
l.Debugln(p, "taking shortcut on", file.Name)
|
l.Debugln(p, "taking shortcut on", file.Name)
|
||||||
}
|
}
|
||||||
|
if file.IsSymlink() {
|
||||||
|
p.shortcutSymlink(curFile, file)
|
||||||
|
} else {
|
||||||
p.shortcutFile(file)
|
p.shortcutFile(file)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
FilesAreDifferent:
|
|
||||||
|
|
||||||
scanner.PopulateOffsets(file.Blocks)
|
scanner.PopulateOffsets(file.Blocks)
|
||||||
|
|
||||||
// Figure out the absolute filenames we need once and for all
|
// Figure out the absolute filenames we need once and for all
|
||||||
|
@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
|
||||||
p.model.updateLocal(p.folder, file)
|
p.model.updateLocal(p.folder, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shortcutSymlink changes the symlinks type if necessery.
|
||||||
|
func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
|
||||||
|
err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
|
||||||
|
if err != nil {
|
||||||
|
l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.model.updateLocal(p.folder, file)
|
||||||
|
}
|
||||||
|
|
||||||
// copierRoutine reads copierStates until the in channel closes and performs
|
// copierRoutine reads copierStates until the in channel closes and performs
|
||||||
// the relevant copies when possible, or passes it to the puller routine.
|
// the relevant copies when possible, or passes it to the puller routine.
|
||||||
func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
|
func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
|
||||||
|
@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If it's a symlink, the target of the symlink is inside the file.
|
||||||
|
if state.file.IsSymlink() {
|
||||||
|
content, err := ioutil.ReadFile(state.realName)
|
||||||
|
if err != nil {
|
||||||
|
l.Warnln("puller: final: reading symlink:", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove the file, and replace it with a symlink.
|
||||||
|
err = osutil.InWritableDir(func(path string) error {
|
||||||
|
os.Remove(path)
|
||||||
|
return symlinks.Create(path, string(content), state.file.Flags)
|
||||||
|
}, state.realName)
|
||||||
|
if err != nil {
|
||||||
|
l.Warnln("puller: final: creating symlink:", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Record the updated file in the index
|
// Record the updated file in the index
|
||||||
p.model.updateLocal(p.folder, state.file)
|
p.model.updateLocal(p.folder, state.file)
|
||||||
}
|
}
|
||||||
|
|
|
@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
|
||||||
|
|
||||||
func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
|
func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
|
||||||
for f := range inbox {
|
for f := range inbox {
|
||||||
if f.IsDirectory() || f.IsDeleted() {
|
if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
|
||||||
outbox <- f
|
outbox <- f
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BlockEqual returns whether two slices of blocks are exactly the same hash
|
||||||
|
// and index pair wise.
|
||||||
|
func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
|
||||||
|
if len(tgt) != len(src) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, sblk := range src {
|
||||||
|
if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
|
@ -27,6 +27,7 @@ import (
|
||||||
"github.com/syncthing/syncthing/internal/ignore"
|
"github.com/syncthing/syncthing/internal/ignore"
|
||||||
"github.com/syncthing/syncthing/internal/lamport"
|
"github.com/syncthing/syncthing/internal/lamport"
|
||||||
"github.com/syncthing/syncthing/internal/protocol"
|
"github.com/syncthing/syncthing/internal/protocol"
|
||||||
|
"github.com/syncthing/syncthing/internal/symlinks"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Walker struct {
|
type Walker struct {
|
||||||
|
@ -131,6 +132,70 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// We must perform this check, as symlinks on Windows are always
|
||||||
|
// .IsRegular or .IsDir unlike on Unix.
|
||||||
|
// Index wise symlinks are always files, regardless of what the target
|
||||||
|
// is, because symlinks carry their target path as their content.
|
||||||
|
isSymlink, _ := symlinks.IsSymlink(p)
|
||||||
|
if isSymlink {
|
||||||
|
var rval error
|
||||||
|
// If the target is a directory, do NOT descend down there.
|
||||||
|
// This will cause files to get tracked, and removing the symlink
|
||||||
|
// will as a result remove files in their real location.
|
||||||
|
// But do not SkipDir if the target is not a directory, as it will
|
||||||
|
// stop scanning the current directory.
|
||||||
|
if info.IsDir() {
|
||||||
|
rval = filepath.SkipDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// We always rehash symlinks as they have no modtime or
|
||||||
|
// permissions.
|
||||||
|
// We check if they point to the old target by checking that
|
||||||
|
// their existing blocks match with the blocks in the index.
|
||||||
|
// If we don't have a filer or don't support symlinks, skip.
|
||||||
|
if w.CurrentFiler == nil || !symlinks.Supported {
|
||||||
|
return rval
|
||||||
|
}
|
||||||
|
|
||||||
|
target, flags, err := symlinks.Read(p)
|
||||||
|
flags = flags & protocol.SymlinkTypeMask
|
||||||
|
if err != nil {
|
||||||
|
if debug {
|
||||||
|
l.Debugln("readlink error:", p, err)
|
||||||
|
}
|
||||||
|
return rval
|
||||||
|
}
|
||||||
|
|
||||||
|
blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
|
||||||
|
if err != nil {
|
||||||
|
if debug {
|
||||||
|
l.Debugln("hash link error:", p, err)
|
||||||
|
}
|
||||||
|
return rval
|
||||||
|
}
|
||||||
|
|
||||||
|
cf := w.CurrentFiler.CurrentFile(rn)
|
||||||
|
if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
|
||||||
|
return rval
|
||||||
|
}
|
||||||
|
|
||||||
|
f := protocol.FileInfo{
|
||||||
|
Name: rn,
|
||||||
|
Version: lamport.Default.Tick(0),
|
||||||
|
Flags: protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
|
||||||
|
Modified: 0,
|
||||||
|
Blocks: blocks,
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
l.Debugln("symlink to hash:", p, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
fchan <- f
|
||||||
|
|
||||||
|
return rval
|
||||||
|
}
|
||||||
|
|
||||||
if info.Mode().IsDir() {
|
if info.Mode().IsDir() {
|
||||||
if w.CurrentFiler != nil {
|
if w.CurrentFiler != nil {
|
||||||
cf := w.CurrentFiler.CurrentFile(rn)
|
cf := w.CurrentFiler.CurrentFile(rn)
|
||||||
|
@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool {
|
||||||
return a&0777 == b&0777
|
return a&0777 == b&0777
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the target is missing, Unix never knows what type of symlink it is
|
||||||
|
// and Windows always knows even if there is no target.
|
||||||
|
// Which means that without this special check a Unix node would be fighting
|
||||||
|
// with a Windows node about whether or not the target is known.
|
||||||
|
// Basically, if you don't know and someone else knows, just accept it.
|
||||||
|
// The fact that you don't know means you are on Unix, and on Unix you don't
|
||||||
|
// really care what the target type is. The moment you do know, and if something
|
||||||
|
// doesn't match, that will propogate throught the cluster.
|
||||||
|
func SymlinkTypeEqual(disk, index uint32) bool {
|
||||||
|
if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
|
||||||
|
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue