syncthing/lib/fs/casefs_test.go
Simon Frei 932d8c69de lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)
With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
2020-07-28 11:15:11 +02:00

280 lines
7.0 KiB
Go

// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestRealCase(t *testing.T) {
// Verify realCase lookups on various underlying filesystems.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCase(t, fsys)
})
}
func testRealCase(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
comps := []string{"Foo", "bar", "BAZ", "bAs"}
path := filepath.Join(comps...)
testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
fd, err := testFs.Create(path)
if err != nil {
t.Fatal(err)
}
fd.Close()
for i, tc := range []struct {
in string
len int
}{
{path, 4},
{strings.ToLower(path), 4},
{strings.ToUpper(path), 4},
{"foo", 1},
{"FOO", 1},
{"foO", 1},
{filepath.Join("Foo", "bar"), 2},
{filepath.Join("Foo", "bAr"), 2},
{filepath.Join("FoO", "bar"), 2},
{filepath.Join("foo", "bar", "BAZ"), 3},
{filepath.Join("Foo", "bar", "bAz"), 3},
{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
} {
out, err := testFs.realCase(tc.in)
if err != nil {
t.Error(err)
} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
}
}
}
func TestRealCaseSensitive(t *testing.T) {
// Verify that realCase returns the best on-disk case for case sensitive
// systems. Test is skipped if the underlying fs is insensitive.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCaseSensitive(t, fsys)
})
}
func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
names := make([]string, 2)
names[0] = "foo"
names[1] = strings.ToUpper(names[0])
for _, n := range names {
if err := testFs.MkdirAll(n, 0777); err != nil {
if IsErrCaseConflict(err) {
t.Skip("Filesystem is case-insensitive")
}
t.Fatal(err)
}
}
for _, n := range names {
if rn, err := testFs.realCase(n); err != nil {
t.Error(err)
} else if rn != n {
t.Errorf("Got %v, expected %v", rn, n)
}
}
}
func TestCaseFSStat(t *testing.T) {
// Verify that a Stat() lookup behaves in a case sensitive manner
// regardless of the underlying fs.
t.Run("fake-sensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testCaseFSStat(t, fsys)
})
}
func testCaseFSStat(t *testing.T, fsys Filesystem) {
fd, err := fsys.Create("foo")
if err != nil {
t.Fatal(err)
}
fd.Close()
// Check if the underlying fs is sensitive or not
sensitive := true
if _, err = fsys.Stat("FOO"); err == nil {
sensitive = false
}
testFs := NewCaseFilesystem(fsys)
_, err = testFs.Stat("FOO")
if sensitive {
if IsNotExist(err) {
t.Log("pass: case sensitive underlying fs")
} else {
t.Error("expected NotExist, not", err, "for sensitive fs")
}
} else if IsErrCaseConflict(err) {
t.Log("pass: case insensitive underlying fs")
} else {
t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
}
}
func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
b.Fatal(err)
}
slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
if err != nil {
b.Fatal(err)
}
b.Run("raw-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, fsys, paths)
b.ReportAllocs()
})
b.Run("case-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
b.ReportAllocs()
})
b.Run("raw-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, slowsys, paths)
b.ReportAllocs()
})
b.Run("case-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
b.ReportAllocs()
})
}
func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
// Simulate a scanner pass over the filesystem. First walk it to
// discover all names, then stat each name individually to check if it's
// been deleted or not (pretending that they all existed in the
// database).
var ms0 runtime.MemStats
runtime.ReadMemStats(&ms0)
t0 := time.Now()
for i := 0; i < b.N; i++ {
if err := doubleWalkFS(fsys, paths); err != nil {
b.Fatal(err)
}
}
t1 := time.Now()
var ms1 runtime.MemStats
runtime.ReadMemStats(&ms1)
// We add metrics per path entry
b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
}
func TestStressCaseFS(t *testing.T) {
// Exercise a bunch of paralell operations for stressing out race
// conditions in the realnamer cache etc.
const limit = 10 * time.Second
if testing.Short() {
t.Skip("long test")
}
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
t.Fatal(err)
}
for i := 0; i < runtime.NumCPU()/2+1; i++ {
t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
// Walk the filesystem and stat everything
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
if err := doubleWalkFS(fsys, paths); err != nil {
t.Fatal(err)
}
}
})
t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
// Touch all the things
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
for _, p := range paths {
now := time.Now()
if err := fsys.Chtimes(p, now, now); err != nil {
t.Fatal(err)
}
}
}
})
}
}
func doubleWalkFS(fsys Filesystem, paths []string) error {
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
return err
}); err != nil {
return err
}
for _, p := range paths {
if _, err := fsys.Lstat(p); err != nil {
return err
}
}
return nil
}
func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
var paths []string
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
paths = append(paths, path)
return err
}); err != nil {
return nil, nil, err
}
if len(paths) < nfiles {
return nil, nil, errors.New("didn't find enough stuff")
}
return fsys, paths, nil
}