Merge pull request #1590 from calmh/long-filenames

Handle long filenames on Windows (fixes #1295)
This commit is contained in:
Audrius Butkevicius 2015-04-08 13:12:25 +01:00
commit a892f80e86
8 changed files with 120 additions and 50 deletions

View File

@ -670,7 +670,7 @@ func defaultConfig(myName string) config.Configuration {
newCfg.Folders = []config.FolderConfiguration{ newCfg.Folders = []config.FolderConfiguration{
{ {
ID: "default", ID: "default",
Path: locations[locDefFolder], RawPath: locations[locDefFolder],
RescanIntervalS: 60, RescanIntervalS: 60,
Devices: []config.FolderDeviceConfiguration{{DeviceID: myID}}, Devices: []config.FolderDeviceConfiguration{{DeviceID: myID}},
}, },

View File

@ -21,8 +21,8 @@ import (
func TestFolderErrors(t *testing.T) { func TestFolderErrors(t *testing.T) {
fcfg := config.FolderConfiguration{ fcfg := config.FolderConfiguration{
ID: "folder", ID: "folder",
Path: "testdata/testfolder", RawPath: "testdata/testfolder",
} }
cfg := config.Wrap("/tmp/test", config.Configuration{ cfg := config.Wrap("/tmp/test", config.Configuration{
Folders: []config.FolderConfiguration{fcfg}, Folders: []config.FolderConfiguration{fcfg},
@ -62,7 +62,7 @@ func TestFolderErrors(t *testing.T) {
// Case 2 - new folder, marker created // Case 2 - new folder, marker created
fcfg.Path = "testdata/" fcfg.RawPath = "testdata/"
cfg = config.Wrap("/tmp/test", config.Configuration{ cfg = config.Wrap("/tmp/test", config.Configuration{
Folders: []config.FolderConfiguration{fcfg}, Folders: []config.FolderConfiguration{fcfg},
}) })
@ -110,7 +110,7 @@ func TestFolderErrors(t *testing.T) {
os.Remove("testdata/testfolder/.stfolder") os.Remove("testdata/testfolder/.stfolder")
os.Remove("testdata/testfolder/") os.Remove("testdata/testfolder/")
fcfg.Path = "testdata/testfolder" fcfg.RawPath = "testdata/testfolder"
cfg = config.Wrap("testdata/subfolder", config.Configuration{ cfg = config.Wrap("testdata/subfolder", config.Configuration{
Folders: []config.FolderConfiguration{fcfg}, Folders: []config.FolderConfiguration{fcfg},
}) })

View File

@ -15,6 +15,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"runtime"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@ -70,7 +71,7 @@ func (orig Configuration) Copy() Configuration {
type FolderConfiguration struct { type FolderConfiguration struct {
ID string `xml:"id,attr" json:"id"` ID string `xml:"id,attr" json:"id"`
Path string `xml:"path,attr" json:"path"` RawPath string `xml:"path,attr" json:"path"`
Devices []FolderDeviceConfiguration `xml:"device" json:"devices"` Devices []FolderDeviceConfiguration `xml:"device" json:"devices"`
ReadOnly bool `xml:"ro,attr" json:"readOnly"` ReadOnly bool `xml:"ro,attr" json:"readOnly"`
RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"` RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
@ -94,9 +95,37 @@ func (orig FolderConfiguration) Copy() FolderConfiguration {
return c return c
} }
func (f FolderConfiguration) Path() string {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Path() should be valid.
// Attempt tilde expansion; leave unchanged in case of error
if path, err := osutil.ExpandTilde(f.RawPath); err == nil {
f.RawPath = path
}
// Attempt absolutification; leave unchanged in case of error
if !filepath.IsAbs(f.RawPath) {
// Abs() looks like a fairly expensive syscall on Windows, while
// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
// somewhat faster in the general case, hence the outer if...
if path, err := filepath.Abs(f.RawPath); err == nil {
f.RawPath = path
}
}
// Attempt to enable long filename support on Windows. We may still not
// have an absolute path here if the previous steps failed.
if runtime.GOOS == "windows" && filepath.IsAbs(f.RawPath) && !strings.HasPrefix(f.RawPath, `\\`) {
return `\\?\` + f.RawPath
}
return f.RawPath
}
func (f *FolderConfiguration) CreateMarker() error { func (f *FolderConfiguration) CreateMarker() error {
if !f.HasMarker() { if !f.HasMarker() {
marker := filepath.Join(f.Path, ".stfolder") marker := filepath.Join(f.Path(), ".stfolder")
fd, err := os.Create(marker) fd, err := os.Create(marker)
if err != nil { if err != nil {
return err return err
@ -109,7 +138,7 @@ func (f *FolderConfiguration) CreateMarker() error {
} }
func (f *FolderConfiguration) HasMarker() bool { func (f *FolderConfiguration) HasMarker() bool {
_, err := os.Stat(filepath.Join(f.Path, ".stfolder")) _, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
if err != nil { if err != nil {
return false return false
} }
@ -285,7 +314,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
for i := range cfg.Folders { for i := range cfg.Folders {
folder := &cfg.Folders[i] folder := &cfg.Folders[i]
if len(folder.Path) == 0 { if len(folder.RawPath) == 0 {
folder.Invalid = "no directory configured" folder.Invalid = "no directory configured"
continue continue
} }
@ -296,7 +325,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
// C:\somedir\ -> C:\somedir\\ -> C:\somedir // C:\somedir\ -> C:\somedir\\ -> C:\somedir
// This way in the tests, we get away without OS specific separators // This way in the tests, we get away without OS specific separators
// in the test configs. // in the test configs.
folder.Path = filepath.Dir(folder.Path + string(filepath.Separator)) folder.RawPath = filepath.Dir(folder.RawPath + string(filepath.Separator))
if folder.ID == "" { if folder.ID == "" {
folder.ID = "default" folder.ID = "default"

View File

@ -11,8 +11,10 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"path/filepath"
"reflect" "reflect"
"runtime" "runtime"
"strings"
"testing" "testing"
"github.com/syncthing/protocol" "github.com/syncthing/protocol"
@ -78,7 +80,7 @@ func TestDeviceConfig(t *testing.T) {
expectedFolders := []FolderConfiguration{ expectedFolders := []FolderConfiguration{
{ {
ID: "test", ID: "test",
Path: "testdata", RawPath: "testdata",
Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}}, Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
ReadOnly: true, ReadOnly: true,
RescanIntervalS: 600, RescanIntervalS: 600,
@ -297,10 +299,10 @@ func TestVersioningConfig(t *testing.T) {
func TestIssue1262(t *testing.T) { func TestIssue1262(t *testing.T) {
cfg, err := Load("testdata/issue-1262.xml", device4) cfg, err := Load("testdata/issue-1262.xml", device4)
if err != nil { if err != nil {
t.Error(err) t.Fatal(err)
} }
actual := cfg.Folders()["test"].Path actual := cfg.Folders()["test"].RawPath
expected := "e:" expected := "e:"
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
expected = `e:\` expected = `e:\`
@ -311,6 +313,51 @@ func TestIssue1262(t *testing.T) {
} }
} }
func TestWindowsPaths(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skip("Not useful on non-Windows")
return
}
folder := FolderConfiguration{
RawPath: `e:\`,
}
expected := `\\?\e:\`
actual := folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `\\192.0.2.22\network\share`
expected = folder.RawPath
actual = folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `relative\path`
expected = folder.RawPath
actual = folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
}
func TestFolderPath(t *testing.T) {
folder := FolderConfiguration{
RawPath: "~/tmp",
}
realPath := folder.Path()
if !filepath.IsAbs(realPath) {
t.Error(realPath, "should be absolute")
}
if strings.Contains(realPath, "~") {
t.Error(realPath, "should not contain ~")
}
}
func TestNewSaveLoad(t *testing.T) { func TestNewSaveLoad(t *testing.T) {
path := "testdata/temp.xml" path := "testdata/temp.xml"
os.Remove(path) os.Remove(path)
@ -391,8 +438,8 @@ func TestRequiresRestart(t *testing.T) {
newCfg = cfg newCfg = cfg
newCfg.Folders = append(newCfg.Folders, FolderConfiguration{ newCfg.Folders = append(newCfg.Folders, FolderConfiguration{
ID: "t1", ID: "t1",
Path: "t1", RawPath: "t1",
}) })
if !ChangeRequiresRestart(cfg, newCfg) { if !ChangeRequiresRestart(cfg, newCfg) {
t.Error("Adding a folder requires restart") t.Error("Adding a folder requires restart")
@ -411,7 +458,7 @@ func TestRequiresRestart(t *testing.T) {
if ChangeRequiresRestart(cfg, newCfg) { if ChangeRequiresRestart(cfg, newCfg) {
t.Error("No changes done yet") t.Error("No changes done yet")
} }
newCfg.Folders[0].Path = "different" newCfg.Folders[0].RawPath = "different"
if !ChangeRequiresRestart(cfg, newCfg) { if !ChangeRequiresRestart(cfg, newCfg) {
t.Error("Changing a folder requires restart") t.Error("Changing a folder requires restart")
} }

View File

@ -159,12 +159,6 @@ func (w *Wrapper) Folders() map[string]FolderConfiguration {
if w.folderMap == nil { if w.folderMap == nil {
w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders)) w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders))
for _, fld := range w.cfg.Folders { for _, fld := range w.cfg.Folders {
path, err := osutil.ExpandTilde(fld.Path)
if err != nil {
l.Warnln("home:", err)
continue
}
fld.Path = path
w.folderMap[fld.ID] = fld w.folderMap[fld.ID] = fld
} }
} }

View File

@ -153,7 +153,7 @@ func (m *Model) StartFolderRW(folder string) {
if !ok { if !ok {
l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type) l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
} }
p.versioner = factory(folder, cfg.Path, cfg.Versioning.Params) p.versioner = factory(folder, cfg.Path(), cfg.Versioning.Params)
} }
if cfg.LenientMtimes { if cfg.LenientMtimes {
@ -730,7 +730,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size) l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
} }
m.fmut.RLock() m.fmut.RLock()
fn := filepath.Join(m.folderCfgs[folder].Path, name) fn := filepath.Join(m.folderCfgs[folder].Path(), name)
m.fmut.RUnlock() m.fmut.RUnlock()
var reader io.ReaderAt var reader io.ReaderAt
@ -813,7 +813,7 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
return lines, nil, fmt.Errorf("Folder %s does not exist", folder) return lines, nil, fmt.Errorf("Folder %s does not exist", folder)
} }
fd, err := os.Open(filepath.Join(cfg.Path, ".stignore")) fd, err := os.Open(filepath.Join(cfg.Path(), ".stignore"))
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return lines, nil, nil return lines, nil, nil
@ -844,7 +844,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
return fmt.Errorf("Folder %s does not exist", folder) return fmt.Errorf("Folder %s does not exist", folder)
} }
fd, err := ioutil.TempFile(cfg.Path, ".syncthing.stignore-"+folder) fd, err := ioutil.TempFile(cfg.Path(), ".syncthing.stignore-"+folder)
if err != nil { if err != nil {
l.Warnln("Saving .stignore:", err) l.Warnln("Saving .stignore:", err)
return err return err
@ -865,7 +865,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
return err return err
} }
file := filepath.Join(cfg.Path, ".stignore") file := filepath.Join(cfg.Path(), ".stignore")
err = osutil.Rename(fd.Name(), file) err = osutil.Rename(fd.Name(), file)
if err != nil { if err != nil {
l.Warnln("Saving .stignore:", err) l.Warnln("Saving .stignore:", err)
@ -1076,7 +1076,7 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
} }
ignores := ignore.New(m.cfg.Options().CacheIgnoredFiles) ignores := ignore.New(m.cfg.Options().CacheIgnoredFiles)
_ = ignores.Load(filepath.Join(cfg.Path, ".stignore")) // Ignore error, there might not be an .stignore _ = ignores.Load(filepath.Join(cfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore
m.folderIgnores[cfg.ID] = ignores m.folderIgnores[cfg.ID] = ignores
m.addedFolder = true m.addedFolder = true
@ -1144,7 +1144,7 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
return errors.New("no such folder") return errors.New("no such folder")
} }
_ = ignores.Load(filepath.Join(folderCfg.Path, ".stignore")) // Ignore error, there might not be an .stignore _ = ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore
// Required to make sure that we start indexing at a directory we're already // Required to make sure that we start indexing at a directory we're already
// aware off. // aware off.
@ -1170,7 +1170,7 @@ nextSub:
subs = unifySubs subs = unifySubs
w := &scanner.Walker{ w := &scanner.Walker{
Dir: folderCfg.Path, Dir: folderCfg.Path(),
Subs: subs, Subs: subs,
Matcher: ignores, Matcher: ignores,
BlockSize: protocol.BlockSize, BlockSize: protocol.BlockSize,
@ -1268,7 +1268,7 @@ nextSub:
"size": f.Size(), "size": f.Size(),
}) })
batch = append(batch, nf) batch = append(batch, nf)
} else if _, err := os.Lstat(filepath.Join(folderCfg.Path, f.Name)); err != nil { } else if _, err := os.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
// File has been deleted. // File has been deleted.
// We don't specifically verify that the error is // We don't specifically verify that the error is
@ -1533,7 +1533,7 @@ func (m *Model) CheckFolderHealth(id string) error {
return errors.New("Folder does not exist") return errors.New("Folder does not exist")
} }
fi, err := os.Stat(folder.Path) fi, err := os.Stat(folder.Path())
if m.CurrentLocalVersion(id) > 0 { if m.CurrentLocalVersion(id) > 0 {
// Safety check. If the cached index contains files but the // Safety check. If the cached index contains files but the
// folder doesn't exist, we have a problem. We would assume // folder doesn't exist, we have a problem. We would assume
@ -1547,7 +1547,7 @@ func (m *Model) CheckFolderHealth(id string) error {
} else if os.IsNotExist(err) { } else if os.IsNotExist(err) {
// If we don't have any files in the index, and the directory // If we don't have any files in the index, and the directory
// doesn't exist, try creating it. // doesn't exist, try creating it.
err = os.MkdirAll(folder.Path, 0700) err = os.MkdirAll(folder.Path(), 0700)
if err == nil { if err == nil {
err = folder.CreateMarker() err = folder.CreateMarker()
} }

View File

@ -35,8 +35,8 @@ func init() {
device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
defaultFolderConfig = config.FolderConfiguration{ defaultFolderConfig = config.FolderConfiguration{
ID: "default", ID: "default",
Path: "testdata", RawPath: "testdata",
Devices: []config.FolderDeviceConfiguration{ Devices: []config.FolderDeviceConfiguration{
{ {
DeviceID: device1, DeviceID: device1,
@ -540,7 +540,7 @@ func TestIgnores(t *testing.T) {
t.Error("No error") t.Error("No error")
} }
m.AddFolder(config.FolderConfiguration{ID: "fresh", Path: "XXX"}) m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
ignores, _, err = m.GetIgnores("fresh") ignores, _, err = m.GetIgnores("fresh")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
@ -596,7 +596,7 @@ func TestROScanRecovery(t *testing.T) {
fcfg := config.FolderConfiguration{ fcfg := config.FolderConfiguration{
ID: "default", ID: "default",
Path: "testdata/rotestfolder", RawPath: "testdata/rotestfolder",
RescanIntervalS: 1, RescanIntervalS: 1,
} }
cfg := config.Wrap("/tmp/test", config.Configuration{ cfg := config.Wrap("/tmp/test", config.Configuration{
@ -608,7 +608,7 @@ func TestROScanRecovery(t *testing.T) {
}, },
}) })
os.RemoveAll(fcfg.Path) os.RemoveAll(fcfg.RawPath)
m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb) m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
@ -633,14 +633,14 @@ func TestROScanRecovery(t *testing.T) {
return return
} }
os.Mkdir(fcfg.Path, 0700) os.Mkdir(fcfg.RawPath, 0700)
if err := waitFor("Folder marker missing"); err != nil { if err := waitFor("Folder marker missing"); err != nil {
t.Error(err) t.Error(err)
return return
} }
fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -652,14 +652,14 @@ func TestROScanRecovery(t *testing.T) {
return return
} }
os.Remove(filepath.Join(fcfg.Path, ".stfolder")) os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
if err := waitFor("Folder marker missing"); err != nil { if err := waitFor("Folder marker missing"); err != nil {
t.Error(err) t.Error(err)
return return
} }
os.Remove(fcfg.Path) os.Remove(fcfg.RawPath)
if err := waitFor("Folder path missing"); err != nil { if err := waitFor("Folder path missing"); err != nil {
t.Error(err) t.Error(err)
@ -676,7 +676,7 @@ func TestRWScanRecovery(t *testing.T) {
fcfg := config.FolderConfiguration{ fcfg := config.FolderConfiguration{
ID: "default", ID: "default",
Path: "testdata/rwtestfolder", RawPath: "testdata/rwtestfolder",
RescanIntervalS: 1, RescanIntervalS: 1,
} }
cfg := config.Wrap("/tmp/test", config.Configuration{ cfg := config.Wrap("/tmp/test", config.Configuration{
@ -688,7 +688,7 @@ func TestRWScanRecovery(t *testing.T) {
}, },
}) })
os.RemoveAll(fcfg.Path) os.RemoveAll(fcfg.RawPath)
m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb) m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
@ -713,14 +713,14 @@ func TestRWScanRecovery(t *testing.T) {
return return
} }
os.Mkdir(fcfg.Path, 0700) os.Mkdir(fcfg.RawPath, 0700)
if err := waitFor("Folder marker missing"); err != nil { if err := waitFor("Folder marker missing"); err != nil {
t.Error(err) t.Error(err)
return return
} }
fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -732,14 +732,14 @@ func TestRWScanRecovery(t *testing.T) {
return return
} }
os.Remove(filepath.Join(fcfg.Path, ".stfolder")) os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
if err := waitFor("Folder marker missing"); err != nil { if err := waitFor("Folder marker missing"); err != nil {
t.Error(err) t.Error(err)
return return
} }
os.Remove(fcfg.Path) os.Remove(fcfg.RawPath)
if err := waitFor("Folder path missing"); err != nil { if err := waitFor("Folder path missing"); err != nil {
t.Error(err) t.Error(err)

View File

@ -82,7 +82,7 @@ func newRWFolder(m *Model, cfg config.FolderConfiguration) *rwFolder {
progressEmitter: m.progressEmitter, progressEmitter: m.progressEmitter,
folder: cfg.ID, folder: cfg.ID,
dir: cfg.Path, dir: cfg.Path(),
scanIntv: time.Duration(cfg.RescanIntervalS) * time.Second, scanIntv: time.Duration(cfg.RescanIntervalS) * time.Second,
ignorePerms: cfg.IgnorePerms, ignorePerms: cfg.IgnorePerms,
lenientMtimes: cfg.LenientMtimes, lenientMtimes: cfg.LenientMtimes,
@ -852,7 +852,7 @@ func (p *rwFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pull
folderRoots := make(map[string]string) folderRoots := make(map[string]string)
p.model.fmut.RLock() p.model.fmut.RLock()
for folder, cfg := range p.model.folderCfgs { for folder, cfg := range p.model.folderCfgs {
folderRoots[folder] = cfg.Path folderRoots[folder] = cfg.Path()
} }
p.model.fmut.RUnlock() p.model.fmut.RUnlock()