From ee5d0dd43fe13788ef6ac86faf12f42029f455ea Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 17 Nov 2017 12:10:16 +0000 Subject: [PATCH] lib/fs: Add case insensitivity to MtimeFS This is step one of a hundred fifty on the path to case insensitivity. It brings in the basic case folding mechanism and adds it to the mtimefs, as this is something outside the fileset that touches stuff in the database based on name. No effort to convert or handle existing entries when the insensitivity is changed, I don't think we need it... Useless by itself but includes tests and will reduce the review load along the way. GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4521 --- lib/fs/folding.go | 19 +++++++++++++++++ lib/fs/folding_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++ lib/fs/mtimefs.go | 33 ++++++++++++++++++++++++----- lib/fs/mtimefs_test.go | 46 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 lib/fs/folding.go create mode 100644 lib/fs/folding_test.go diff --git a/lib/fs/folding.go b/lib/fs/folding.go new file mode 100644 index 000000000..e220855af --- /dev/null +++ b/lib/fs/folding.go @@ -0,0 +1,19 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "unicode" +) + +func UnicodeLowercase(s string) string { + rs := []rune(s) + for i, r := range rs { + rs[i] = unicode.ToLower(unicode.ToUpper(r)) + } + return string(rs) +} diff --git a/lib/fs/folding_test.go b/lib/fs/folding_test.go new file mode 100644 index 000000000..11e43962f --- /dev/null +++ b/lib/fs/folding_test.go @@ -0,0 +1,48 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import "testing" + +func TestUnicodeLowercase(t *testing.T) { + cases := [][2]string{ + {"", ""}, + {"hej", "hej"}, + {"HeJ!@#", "hej!@#"}, + // Western Europe diacritical stuff is trivial + {"ÜBERRÄKSMÖRGÅS", "überräksmörgås"}, + // Cyrillic seems regular as well + {"Привет", "привет"}, + // Greek has multiple lower case characters for things depending on + // context; we should always choose the right one. + {"Ὀδυσσεύς", "ὀδυσσεύσ"}, + {"ὈΔΥΣΣΕΎΣ", "ὀδυσσεύσ"}, + // German ß doesn't really have an upper case variant, and we + // shouldn't mess things up when lower casing it either. We don't + // attempt to make ß equivalant to "ss". + {"Reichwaldstraße", "reichwaldstraße"}, + // The Turks do their thing with the Is.... Like the Greek example + // we pick just the one canonicalized "i" although you can argue + // with this... From what I understand most operating systems don't + // get this right anyway. + {"İI", "ii"}, + // Arabic doesn't do case folding. + {"العَرَبِيَّة", "العَرَبِيَّة"}, + // Neither does Hebrew. + {"עברית", "עברית"}, + // Nor Chinese, in any variant. + {"汉语/漢語 or 中文", "汉语/漢語 or 中文"}, + // Niether katakana as far as I can tell. + {"チャーハン", "チャーハン"}, + } + for _, tc := range cases { + res := UnicodeLowercase(tc[0]) + if res != tc[1] { + t.Errorf("UnicodeLowercase(%q) => %q, expected %q", tc[0], res, tc[1]) + } + } +} diff --git a/lib/fs/mtimefs.go b/lib/fs/mtimefs.go index 1719b8cec..3f375640e 100644 --- a/lib/fs/mtimefs.go +++ b/lib/fs/mtimefs.go @@ -6,7 +6,9 @@ package fs -import "time" +import ( + "time" +) // The database is where we store the virtual mtimes type database interface { @@ -20,16 +22,29 @@ type database interface { // just does the underlying operations with no additions. type MtimeFS struct { Filesystem - chtimes func(string, time.Time, time.Time) error - db database + chtimes func(string, time.Time, time.Time) error + db database + caseInsensitive bool } -func NewMtimeFS(underlying Filesystem, db database) *MtimeFS { - return &MtimeFS{ +type MtimeFSOption func(*MtimeFS) + +func WithCaseInsensitivity(v bool) MtimeFSOption { + return func(f *MtimeFS) { + f.caseInsensitive = v + } +} + +func NewMtimeFS(underlying Filesystem, db database, options ...MtimeFSOption) *MtimeFS { + f := &MtimeFS{ Filesystem: underlying, chtimes: underlying.Chtimes, // for mocking it out in the tests db: db, } + for _, opt := range options { + opt(f) + } + return f } func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error { @@ -72,6 +87,10 @@ func (f *MtimeFS) Lstat(name string) (FileInfo, error) { // "virtual" is what want the timestamp to be func (f *MtimeFS) save(name string, real, virtual time.Time) { + if f.caseInsensitive { + name = UnicodeLowercase(name) + } + if real.Equal(virtual) { // If the virtual time and the real on disk time are equal we don't // need to store anything. @@ -88,6 +107,10 @@ func (f *MtimeFS) save(name string, real, virtual time.Time) { } func (f *MtimeFS) load(name string) (real, virtual time.Time) { + if f.caseInsensitive { + name = UnicodeLowercase(name) + } + data, exists := f.db.Bytes(name) if !exists { return diff --git a/lib/fs/mtimefs_test.go b/lib/fs/mtimefs_test.go index 80514e1cb..e8befb591 100644 --- a/lib/fs/mtimefs_test.go +++ b/lib/fs/mtimefs_test.go @@ -10,6 +10,7 @@ import ( "errors" "io/ioutil" "os" + "runtime" "testing" "time" ) @@ -80,6 +81,51 @@ func TestMtimeFS(t *testing.T) { } } +func TestMtimeFSInsensitive(t *testing.T) { + switch runtime.GOOS { + case "darwin", "windows": + // blatantly assume file systems here are case insensitive. Might be + // a spurious failure on oddly configured systems. + default: + t.Skip("need case insensitive FS") + } + + theTest := func(t *testing.T, fs *MtimeFS, shouldSucceed bool) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.Mkdir("testdata", 0755) + ioutil.WriteFile("testdata/FiLe", []byte("hello"), 0644) + + // a random time with nanosecond precision + testTime := time.Unix(1234567890, 123456789) + + // Do one call that gets struck by an exceptionally evil Chtimes, with a + // different case from what is on disk. + fs.chtimes = evilChtimes + if err := fs.Chtimes("testdata/fIlE", testTime, testTime); err != nil { + t.Error("Should not have failed:", err) + } + + // Check that we get back the mtime we set, if we were supposed to succed. + info, err := fs.Lstat("testdata/FILE") + if err != nil { + t.Error("Lstat shouldn't fail:", err) + } else if info.ModTime().Equal(testTime) != shouldSucceed { + t.Errorf("Time mismatch; got %v, comparison %v, expected equal=%v", info.ModTime(), testTime, shouldSucceed) + } + } + + // The test should fail with a case sensitive mtimefs + t.Run("with case sensitive mtimefs", func(t *testing.T) { + theTest(t, NewMtimeFS(newBasicFilesystem("."), make(mapStore)), false) + }) + + // And succeed with a case insensitive one. + t.Run("with case insensitive mtimefs", func(t *testing.T) { + theTest(t, NewMtimeFS(newBasicFilesystem("."), make(mapStore), WithCaseInsensitivity(true)), true) + }) +} + // The mapStore is a simple database type mapStore map[string][]byte