diff --git a/lib/db/keyer.go b/lib/db/keyer.go new file mode 100644 index 000000000..ff3ce603f --- /dev/null +++ b/lib/db/keyer.go @@ -0,0 +1,202 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "encoding/binary" +) + +const ( + keyPrefixLen = 1 + keyFolderLen = 4 // indexed + keyDeviceLen = 4 // indexed + keySequenceLen = 8 + keyHashLen = 32 + + maxInt64 int64 = 1<<63 - 1 +) + +const ( + KeyTypeDevice = 0 + KeyTypeGlobal = 1 + KeyTypeBlock = 2 + KeyTypeDeviceStatistic = 3 + KeyTypeFolderStatistic = 4 + KeyTypeVirtualMtime = 5 + KeyTypeFolderIdx = 6 + KeyTypeDeviceIdx = 7 + KeyTypeIndexID = 8 + KeyTypeFolderMeta = 9 + KeyTypeMiscData = 10 + KeyTypeSequence = 11 + KeyTypeNeed = 12 +) + +type keyer interface { + // device file key stuff + GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey + NameFromDeviceFileKey(key []byte) []byte + DeviceFromDeviceFileKey(key []byte) ([]byte, bool) + FolderFromDeviceFileKey(key []byte) ([]byte, bool) + + // global version key stuff + GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey + NameFromGlobalVersionKey(key []byte) []byte + FolderFromGlobalVersionKey(key []byte) ([]byte, bool) + + // file need index + GenerateNeedFileKey(key, folder, name []byte) needFileKey + + // file sequence index + GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey + SequenceFromSequenceKey(key []byte) int64 + + // index IDs + GenerateIndexIDKey(key, device, folder []byte) indexIDKey + DeviceFromIndexIDKey(key []byte) ([]byte, bool) + + // Mtimes + GenerateMtimesKey(key, folder []byte) mtimesKey + + // Folder metadata + GenerateFolderMetaKey(key, folder []byte) folderMetaKey +} + +// defaultKeyer implements our key scheme. It needs folder and device +// indexes. +type defaultKeyer struct { + folderIdx *smallIndex + deviceIdx *smallIndex +} + +func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer { + return defaultKeyer{ + folderIdx: folderIdx, + deviceIdx: deviceIdx, + } +} + +type deviceFileKey []byte + +func (k deviceFileKey) WithoutName() []byte { + return k[:keyPrefixLen+keyFolderLen+keyDeviceLen] +} + +func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey { + key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name)) + key[0] = KeyTypeDevice + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], k.deviceIdx.ID(device)) + copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name) + return key +} + +func (k defaultKeyer) NameFromDeviceFileKey(key []byte) []byte { + return key[keyPrefixLen+keyFolderLen+keyDeviceLen:] +} + +func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) { + return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:])) +} + +func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) { + return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) +} + +type globalVersionKey []byte + +func (k globalVersionKey) WithoutName() []byte { + return k[:keyPrefixLen+keyFolderLen] +} + +func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey { + key = resize(key, keyPrefixLen+keyFolderLen+len(name)) + key[0] = KeyTypeGlobal + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + copy(key[keyPrefixLen+keyFolderLen:], name) + return key +} + +func (k defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte { + return key[keyPrefixLen+keyFolderLen:] +} + +func (k defaultKeyer) FolderFromGlobalVersionKey(key []byte) ([]byte, bool) { + return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) +} + +type needFileKey []byte + +func (k needFileKey) WithoutName() []byte { + return k[:keyPrefixLen+keyFolderLen] +} + +func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) needFileKey { + key = resize(key, keyPrefixLen+keyFolderLen+len(name)) + key[0] = KeyTypeNeed + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + copy(key[keyPrefixLen+keyFolderLen:], name) + return key +} + +type sequenceKey []byte + +func (k sequenceKey) WithoutSequence() []byte { + return k[:keyPrefixLen+keyFolderLen] +} + +func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey { + key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen) + key[0] = KeyTypeSequence + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq)) + return key +} + +func (k defaultKeyer) SequenceFromSequenceKey(key []byte) int64 { + return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:])) +} + +type indexIDKey []byte + +func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) indexIDKey { + key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen) + key[0] = KeyTypeIndexID + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.deviceIdx.ID(device)) + binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], k.folderIdx.ID(folder)) + return key +} + +func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) { + return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) +} + +type mtimesKey []byte + +func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) mtimesKey { + key = resize(key, keyPrefixLen+keyFolderLen) + key[0] = KeyTypeVirtualMtime + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + return key +} + +type folderMetaKey []byte + +func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) folderMetaKey { + key = resize(key, keyPrefixLen+keyFolderLen) + key[0] = KeyTypeFolderMeta + binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder)) + return key +} + +// resize returns a byte slice of the specified size, reusing bs if possible +func resize(bs []byte, size int) []byte { + if cap(bs) < size { + return make([]byte, size) + } + return bs[:size] +} diff --git a/lib/db/keyer_test.go b/lib/db/keyer_test.go new file mode 100644 index 000000000..291a4dd59 --- /dev/null +++ b/lib/db/keyer_test.go @@ -0,0 +1,80 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "bytes" + "testing" +) + +func TestDeviceKey(t *testing.T) { + fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") + dev := []byte("device67890123456789012345678901") + name := []byte("name") + + db := OpenMemory() + + key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name) + + fld2, ok := db.keyer.FolderFromDeviceFileKey(key) + if !ok { + t.Fatal("unexpectedly not found") + } + if !bytes.Equal(fld2, fld) { + t.Errorf("wrong folder %q != %q", fld2, fld) + } + dev2, ok := db.keyer.DeviceFromDeviceFileKey(key) + if !ok { + t.Fatal("unexpectedly not found") + } + if !bytes.Equal(dev2, dev) { + t.Errorf("wrong device %q != %q", dev2, dev) + } + name2 := db.keyer.NameFromDeviceFileKey(key) + if !bytes.Equal(name2, name) { + t.Errorf("wrong name %q != %q", name2, name) + } +} + +func TestGlobalKey(t *testing.T) { + fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") + name := []byte("name") + + db := OpenMemory() + + key := db.keyer.GenerateGlobalVersionKey(nil, fld, name) + + fld2, ok := db.keyer.FolderFromGlobalVersionKey(key) + if !ok { + t.Error("should have been found") + } + if !bytes.Equal(fld2, fld) { + t.Errorf("wrong folder %q != %q", fld2, fld) + } + name2 := db.keyer.NameFromGlobalVersionKey(key) + if !bytes.Equal(name2, name) { + t.Errorf("wrong name %q != %q", name2, name) + } + + _, ok = db.keyer.FolderFromGlobalVersionKey([]byte{1, 2, 3, 4, 5}) + if ok { + t.Error("should not have been found") + } +} + +func TestSequenceKey(t *testing.T) { + fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") + + db := OpenMemory() + + const seq = 1234567890 + key := db.keyer.GenerateSequenceKey(nil, fld, seq) + outSeq := db.keyer.SequenceFromSequenceKey(key) + if outSeq != seq { + t.Errorf("sequence number mangled, %d != %d", outSeq, seq) + } +} diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index 358de8683..7ebb0e1c3 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -13,22 +13,6 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -const ( - KeyTypeDevice = iota - KeyTypeGlobal - KeyTypeBlock - KeyTypeDeviceStatistic - KeyTypeFolderStatistic - KeyTypeVirtualMtime - KeyTypeFolderIdx - KeyTypeDeviceIdx - KeyTypeIndexID - KeyTypeFolderMeta - KeyTypeMiscData - KeyTypeSequence - KeyTypeNeed -) - func (vl VersionList) String() string { var b bytes.Buffer var id protocol.DeviceID @@ -86,7 +70,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db * // to determine the winner.) // // A surprise missing file entry here is counted as a win for us. - if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) { + if of, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) { vl = vl.insertAt(i, nv) return vl, removedFV, removedAt, i } diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index b231e4520..ead076f7e 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -16,7 +16,6 @@ import ( "sync/atomic" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/sync" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -33,18 +32,9 @@ type Instance struct { location string folderIdx *smallIndex deviceIdx *smallIndex + keyer keyer } -const ( - keyPrefixLen = 1 - keyFolderLen = 4 // indexed - keyDeviceLen = 4 // indexed - keySequenceLen = 8 - keyHashLen = 32 - - maxInt64 int64 = 1<<63 - 1 -) - func Open(file string) (*Instance, error) { opts := &opt.Options{ OpenFilesCacheCapacity: 100, @@ -80,11 +70,12 @@ func OpenMemory() *Instance { func newDBInstance(db *leveldb.DB, location string) (*Instance, error) { i := &Instance{ - DB: db, - location: location, + DB: db, + location: location, + folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}), + deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}), } - i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx}) - i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx}) + i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx) err := i.updateSchema() return i, err } @@ -107,7 +98,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m var gk []byte for _, f := range fs { name := []byte(f.Name) - fk = db.deviceKeyInto(fk, folder, device, name) + fk = db.keyer.GenerateDeviceFileKey(fk, folder, device, name) // Get and unmarshal the file entry. If it doesn't exist or can't be // unmarshalled we'll add it as a new entry. @@ -131,7 +122,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m t.insertFile(fk, folder, device, f) - gk = db.globalKeyInto(gk, folder, name) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name) t.updateGlobal(gk, folder, device, f, meta) // Write out and reuse the batch every few records, to avoid the batch @@ -147,8 +138,8 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) { var sk []byte var dk []byte for _, f := range fs { - sk = db.sequenceKeyInto(sk, folder, f.Sequence) - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name)) + sk = db.keyer.GenerateSequenceKey(sk, folder, f.Sequence) + dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name)) t.Put(sk, dk) l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name) t.checkFlush() @@ -161,7 +152,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) { var sk []byte for _, f := range fs { - t.Delete(db.sequenceKeyInto(sk, folder, f.Sequence)) + t.Delete(db.keyer.GenerateSequenceKey(sk, folder, f.Sequence)) l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name) t.checkFlush() } @@ -176,7 +167,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It prefix = append(prefix, '/') } - if f, ok := db.getFileTrunc(db.deviceKey(folder, device, unslashedPrefix), true); ok && !fn(f) { + if f, ok := db.getFileTrunc(db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix), true); ok && !fn(f) { return } } @@ -184,11 +175,11 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It t := db.newReadOnlyTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, prefix)[:keyPrefixLen+keyFolderLen+keyDeviceLen+len(prefix)]), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)), nil) defer dbi.Release() for dbi.Next() { - name := db.deviceKeyName(dbi.Key()) + name := db.keyer.NameFromDeviceFileKey(dbi.Key()) if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { return } @@ -212,13 +203,13 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) t := db.newReadOnlyTransaction() defer t.close() - dbi := t.NewIterator(&util.Range{Start: db.sequenceKey(folder, startSeq), Limit: db.sequenceKey(folder, maxInt64)}, nil) + dbi := t.NewIterator(&util.Range{Start: db.keyer.GenerateSequenceKey(nil, folder, startSeq), Limit: db.keyer.GenerateSequenceKey(nil, folder, maxInt64)}, nil) defer dbi.Release() for dbi.Next() { f, ok := db.getFile(dbi.Value()) if !ok { - l.Debugln("missing file for sequence number", db.sequenceKeySequence(dbi.Key())) + l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key())) continue } @@ -239,13 +230,19 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, t := db.newReadWriteTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen]), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName()), nil) defer dbi.Release() var gk []byte for dbi.Next() { - device := db.deviceKeyDevice(dbi.Key()) + device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key()) + if !ok { + // Not having the device in the index is bad. Clear it. + t.Delete(dbi.Key()) + t.checkFlush() + continue + } var f FileInfoTruncated // The iterator function may keep a reference to the unmarshalled // struct, which in turn references the buffer it was unmarshalled @@ -261,7 +258,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, case "", ".", "..", "/": // A few obviously invalid filenames l.Infof("Dropping invalid filename %q from database", f.Name) name := []byte(f.Name) - gk = db.globalKeyInto(gk, folder, name) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name) t.removeFromGlobal(gk, folder, device, name, nil) t.Delete(dbi.Key()) t.checkFlush() @@ -308,7 +305,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo } func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) { - gk = db.globalKeyInto(gk, folder, file) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file) bs, err := t.Get(gk, nil) if err != nil { @@ -320,7 +317,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file [] return gk, dk, nil, false } - dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file) + dk = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, file) if fi, ok := db.getFileTrunc(dk, truncate); ok { return gk, dk, fi, true } @@ -345,12 +342,12 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator t := db.newReadOnlyTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, prefix)), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)), nil) defer dbi.Release() var fk []byte for dbi.Next() { - name := db.globalKeyName(dbi.Key()) + name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { return } @@ -360,7 +357,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator continue } - fk = db.deviceKeyInto(fk, folder, vl.Versions[0].Device, name) + fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[0].Device, name) f, ok := db.getFileTrunc(fk, truncate) if !ok { @@ -374,7 +371,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator } func (db *Instance) availability(folder, file []byte) []protocol.DeviceID { - k := db.globalKey(folder, file) + k := db.keyer.GenerateGlobalVersionKey(nil, folder, file) bs, err := db.Get(k, nil) if err == leveldb.ErrNotFound { return nil @@ -413,7 +410,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) t := db.newReadOnlyTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil) defer dbi.Release() var fk []byte @@ -431,7 +428,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) continue } - name := db.globalKeyName(dbi.Key()) + name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) needVersion := vl.Versions[0].Version needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device) @@ -446,7 +443,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) continue } - fk = db.deviceKeyInto(fk, folder, vl.Versions[i].Device, name) + fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[i].Device, name) bs, err := t.Get(fk, nil) if err != nil { l.Debugln("surprise error:", err) @@ -480,7 +477,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) { t := db.newReadOnlyTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName()), nil) defer dbi.Release() var dk []byte @@ -488,7 +485,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) { var f FileIntf var ok bool for dbi.Next() { - gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate) + gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate) if !ok { continue } @@ -507,7 +504,7 @@ func (db *Instance) ListFolders() []string { folderExists := make(map[string]bool) for dbi.Next() { - folder, ok := db.globalKeyFolder(dbi.Key()) + folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key()) if ok && !folderExists[string(folder)] { folderExists[string(folder)] = true } @@ -528,13 +525,13 @@ func (db *Instance) dropFolder(folder []byte) { for _, key := range [][]byte{ // Remove all items related to the given folder from the device->file bucket - db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen], + db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName(), // Remove all sequences related to the folder - db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen], + db.keyer.GenerateSequenceKey(nil, []byte(folder), 0).WithoutSequence(), // Remove all items related to the given folder from the global bucket - db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen], + db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName(), // Remove all needs related to the folder - db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen], + db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName(), } { t.deleteKeyPrefix(key) } @@ -544,15 +541,15 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke t := db.newReadWriteTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, nil)), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, nil)), nil) defer dbi.Release() var gk []byte for dbi.Next() { key := dbi.Key() - name := db.deviceKeyName(key) - gk = db.globalKeyInto(gk, folder, name) + name := db.keyer.NameFromDeviceFileKey(key) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name) t.removeFromGlobal(gk, folder, device, name, meta) t.Delete(key) t.checkFlush() @@ -563,7 +560,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { t := db.newReadWriteTransaction() defer t.close() - dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil) + dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil) defer dbi.Release() var fk []byte @@ -578,10 +575,10 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { // there are global entries pointing to no longer existing files. Here // we find those and clear them out. - name := db.globalKeyName(dbi.Key()) + name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) var newVL VersionList for i, version := range vl.Versions { - fk = db.deviceKeyInto(fk, folder, version.Device, name) + fk = db.keyer.GenerateDeviceFileKey(fk, folder, version.Device, name) _, err := t.Get(fk, nil) if err == leveldb.ErrNotFound { continue @@ -607,110 +604,8 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { l.Debugf("db check completed for %q", folder) } -// deviceKey returns a byte slice encoding the following information: -// keyTypeDevice (1 byte) -// folder (4 bytes) -// device (4 bytes) -// name (variable size) -func (db *Instance) deviceKey(folder, device, file []byte) []byte { - return db.deviceKeyInto(nil, folder, device, file) -} - -func (db *Instance) deviceKeyInto(k, folder, device, file []byte) []byte { - reqLen := keyPrefixLen + keyFolderLen + keyDeviceLen + len(file) - k = resize(k, reqLen) - k[0] = KeyTypeDevice - binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) - binary.BigEndian.PutUint32(k[keyPrefixLen+keyFolderLen:], db.deviceIdx.ID(device)) - copy(k[keyPrefixLen+keyFolderLen+keyDeviceLen:], file) - return k -} - -// deviceKeyName returns the device ID from the key -func (db *Instance) deviceKeyName(key []byte) []byte { - return key[keyPrefixLen+keyFolderLen+keyDeviceLen:] -} - -// deviceKeyFolder returns the folder name from the key -func (db *Instance) deviceKeyFolder(key []byte) []byte { - folder, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) - if !ok { - panic("bug: lookup of nonexistent folder ID") - } - return folder -} - -// deviceKeyDevice returns the device ID from the key -func (db *Instance) deviceKeyDevice(key []byte) []byte { - device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:])) - if !ok { - panic("bug: lookup of nonexistent device ID") - } - return device -} - -// globalKey returns a byte slice encoding the following information: -// keyTypeGlobal (1 byte) -// folder (4 bytes) -// name (variable size) -func (db *Instance) globalKey(folder, file []byte) []byte { - return db.globalKeyInto(nil, folder, file) -} - -func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte { - reqLen := keyPrefixLen + keyFolderLen + len(file) - gk = resize(gk, reqLen) - gk[0] = KeyTypeGlobal - binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder)) - copy(gk[keyPrefixLen+keyFolderLen:], file) - return gk -} - -// globalKeyName returns the filename from the key -func (db *Instance) globalKeyName(key []byte) []byte { - return key[keyPrefixLen+keyFolderLen:] -} - -// globalKeyFolder returns the folder name from the key -func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) { - return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) -} - -// needKey is a globalKey with a different prefix -func (db *Instance) needKey(folder, file []byte) []byte { - return db.needKeyInto(nil, folder, file) -} - -func (db *Instance) needKeyInto(k, folder, file []byte) []byte { - k = db.globalKeyInto(k, folder, file) - k[0] = KeyTypeNeed - return k -} - -// sequenceKey returns a byte slice encoding the following information: -// KeyTypeSequence (1 byte) -// folder (4 bytes) -// sequence number (8 bytes) -func (db *Instance) sequenceKey(folder []byte, seq int64) []byte { - return db.sequenceKeyInto(nil, folder, seq) -} - -func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte { - reqLen := keyPrefixLen + keyFolderLen + keySequenceLen - k = resize(k, reqLen) - k[0] = KeyTypeSequence - binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) - binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq)) - return k -} - -// sequenceKeySequence returns the sequence number from the key -func (db *Instance) sequenceKeySequence(key []byte) int64 { - return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:])) -} - func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID { - key := db.indexIDKey(device, folder) + key := db.keyer.GenerateIndexIDKey(nil, device, folder) cur, err := db.Get(key, nil) if err != nil { return 0 @@ -725,44 +620,13 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID { } func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) { - key := db.indexIDKey(device, folder) + key := db.keyer.GenerateIndexIDKey(nil, device, folder) bs, _ := id.Marshal() // marshalling can't fail if err := db.Put(key, bs, nil); err != nil { panic("storing index ID: " + err.Error()) } } -func (db *Instance) indexIDKey(device, folder []byte) []byte { - k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen) - k[0] = KeyTypeIndexID - binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device)) - binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder)) - return k -} - -func (db *Instance) indexIDDevice(key []byte) []byte { - device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) - if !ok { - // uuh ... - return nil - } - return device -} - -func (db *Instance) mtimesKey(folder []byte) []byte { - prefix := make([]byte, 5) // key type + 4 bytes folder idx number - prefix[0] = KeyTypeVirtualMtime - binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder)) - return prefix -} - -func (db *Instance) folderMetaKey(folder []byte) []byte { - prefix := make([]byte, 5) // key type + 4 bytes folder idx number - prefix[0] = KeyTypeFolderMeta - binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder)) - return prefix -} - // DropLocalDeltaIndexIDs removes all index IDs for the local device ID from // the database. This will cause a full index transmission on the next // connection. @@ -785,7 +649,7 @@ func (db *Instance) dropDeltaIndexIDs(local bool) { defer dbi.Release() for dbi.Next() { - device := db.indexIDDevice(dbi.Key()) + device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key()) if bytes.Equal(device, protocol.LocalDeviceID[:]) == local { t.Delete(dbi.Key()) } @@ -793,11 +657,11 @@ func (db *Instance) dropDeltaIndexIDs(local bool) { } func (db *Instance) dropMtimes(folder []byte) { - db.dropPrefix(db.mtimesKey(folder)) + db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder)) } func (db *Instance) dropFolderMeta(folder []byte) { - db.dropPrefix(db.folderMetaKey(folder)) + db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder)) } func (db *Instance) dropPrefix(prefix []byte) { @@ -853,99 +717,6 @@ func leveldbIsCorrupted(err error) bool { return false } -// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives -// fast lookups in both directions and persists to the database. Don't use for -// storing more items than fit comfortably in RAM. -type smallIndex struct { - db *Instance - prefix []byte - id2val map[uint32]string - val2id map[string]uint32 - nextID uint32 - mut sync.Mutex -} - -func newSmallIndex(db *Instance, prefix []byte) *smallIndex { - idx := &smallIndex{ - db: db, - prefix: prefix, - id2val: make(map[uint32]string), - val2id: make(map[string]uint32), - mut: sync.NewMutex(), - } - idx.load() - return idx -} - -// load iterates over the prefix space in the database and populates the in -// memory maps. -func (i *smallIndex) load() { - tr := i.db.newReadOnlyTransaction() - it := tr.NewIterator(util.BytesPrefix(i.prefix), nil) - for it.Next() { - val := string(it.Value()) - id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):]) - i.id2val[id] = val - i.val2id[val] = id - if id >= i.nextID { - i.nextID = id + 1 - } - } - it.Release() - tr.close() -} - -// ID returns the index number for the given byte slice, allocating a new one -// and persisting this to the database if necessary. -func (i *smallIndex) ID(val []byte) uint32 { - i.mut.Lock() - // intentionally avoiding defer here as we want this call to be as fast as - // possible in the general case (folder ID already exists). The map lookup - // with the conversion of []byte to string is compiler optimized to not - // copy the []byte, which is why we don't assign it to a temp variable - // here. - if id, ok := i.val2id[string(val)]; ok { - i.mut.Unlock() - return id - } - - id := i.nextID - i.nextID++ - - valStr := string(val) - i.val2id[valStr] = id - i.id2val[id] = valStr - - key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id - copy(key, i.prefix) - binary.BigEndian.PutUint32(key[len(i.prefix):], id) - i.db.Put(key, val, nil) - - i.mut.Unlock() - return id -} - -// Val returns the value for the given index number, or (nil, false) if there -// is no such index number. -func (i *smallIndex) Val(id uint32) ([]byte, bool) { - i.mut.Lock() - val, ok := i.id2val[id] - i.mut.Unlock() - if !ok { - return nil, false - } - - return []byte(val), true -} - -// resize returns a byte array of length reqLen, reusing k if possible -func resize(k []byte, reqLen int) []byte { - if cap(k) < reqLen { - return make([]byte, reqLen) - } - return k[:reqLen] -} - type errorSuggestion struct { inner error suggestion string diff --git a/lib/db/leveldb_dbinstance_updateschema.go b/lib/db/leveldb_dbinstance_updateschema.go index c685809a7..36d174bf4 100644 --- a/lib/db/leveldb_dbinstance_updateschema.go +++ b/lib/db/leveldb_dbinstance_updateschema.go @@ -91,16 +91,28 @@ func (db *Instance) updateSchema0to1() { var gk []byte for dbi.Next() { - folder := db.deviceKeyFolder(dbi.Key()) - device := db.deviceKeyDevice(dbi.Key()) - name := db.deviceKeyName(dbi.Key()) + folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key()) + if !ok { + // not having the folder in the index is bad; delete and continue + t.Delete(dbi.Key()) + t.checkFlush() + continue + } + device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key()) + if !ok { + // not having the device in the index is bad; delete and continue + t.Delete(dbi.Key()) + t.checkFlush() + continue + } + name := db.keyer.NameFromDeviceFileKey(dbi.Key()) // Remove files with absolute path (see #4799) if strings.HasPrefix(string(name), "/") { if _, ok := changedFolders[string(folder)]; !ok { changedFolders[string(folder)] = struct{}{} } - gk = db.globalKeyInto(gk, folder, name) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name) t.removeFromGlobal(gk, folder, device, nil, nil) t.Delete(dbi.Key()) t.checkFlush() @@ -130,7 +142,7 @@ func (db *Instance) updateSchema0to1() { // Add invalid files to global list if f.IsInvalid() { - gk = db.globalKeyInto(gk, folder, name) + gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name) if t.updateGlobal(gk, folder, device, f, meta) { if _, ok := changedFolders[string(folder)]; !ok { changedFolders[string(folder)] = struct{}{} @@ -156,8 +168,8 @@ func (db *Instance) updateSchema1to2() { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { - sk = db.sequenceKeyInto(sk, folder, f.SequenceNo()) - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName())) + sk = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo()) + dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName())) t.Put(sk, dk) t.checkFlush() return true @@ -176,7 +188,7 @@ func (db *Instance) updateSchema2to3() { folder := []byte(folderStr) db.withGlobal(folder, nil, true, func(f FileIntf) bool { name := []byte(f.FileName()) - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name) + dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name) var v protocol.Vector haveFile, ok := db.getFileTrunc(dk, true) if ok { @@ -185,7 +197,7 @@ func (db *Instance) updateSchema2to3() { if !need(f, ok, v) { return true } - nk = t.db.needKeyInto(nk, folder, []byte(f.FileName())) + nk = t.db.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName())) t.Put(nk, nil) t.checkFlush() return true @@ -201,7 +213,7 @@ func (db *Instance) updateSchemaTo5() { t := db.newReadWriteTransaction() var nk []byte for _, folderStr := range db.ListFolders() { - nk = db.needKeyInto(nk, []byte(folderStr), nil) + nk = db.keyer.GenerateNeedFileKey(nk, []byte(folderStr), nil) t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen]) } t.close() @@ -230,7 +242,7 @@ func (db *Instance) updateSchema5to6() { fi.LocalFlags = protocol.FlagLocalIgnored bs, _ := fi.Marshal() - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name)) + dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name)) t.Put(dk, bs) t.checkFlush() diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index dcdb5f41a..1699eccf4 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -7,7 +7,6 @@ package db import ( - "bytes" "os" "testing" @@ -15,58 +14,6 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -func TestDeviceKey(t *testing.T) { - fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") - dev := []byte("device67890123456789012345678901") - name := []byte("name") - - db := OpenMemory() - db.folderIdx.ID(fld) - db.deviceIdx.ID(dev) - - key := db.deviceKey(fld, dev, name) - - fld2 := db.deviceKeyFolder(key) - if !bytes.Equal(fld2, fld) { - t.Errorf("wrong folder %q != %q", fld2, fld) - } - dev2 := db.deviceKeyDevice(key) - if !bytes.Equal(dev2, dev) { - t.Errorf("wrong device %q != %q", dev2, dev) - } - name2 := db.deviceKeyName(key) - if !bytes.Equal(name2, name) { - t.Errorf("wrong name %q != %q", name2, name) - } -} - -func TestGlobalKey(t *testing.T) { - fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") - name := []byte("name") - - db := OpenMemory() - db.folderIdx.ID(fld) - - key := db.globalKey(fld, name) - - fld2, ok := db.globalKeyFolder(key) - if !ok { - t.Error("should have been found") - } - if !bytes.Equal(fld2, fld) { - t.Errorf("wrong folder %q != %q", fld2, fld) - } - name2 := db.globalKeyName(key) - if !bytes.Equal(name2, name) { - t.Errorf("wrong name %q != %q", name2, name) - } - - _, ok = db.globalKeyFolder([]byte{1, 2, 3, 4, 5}) - if ok { - t.Error("should not have been found") - } -} - func TestDropIndexIDs(t *testing.T) { db := OpenMemory() @@ -287,11 +234,11 @@ func TestUpdate0to3(t *testing.T) { db.updateSchema0to1() - if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok { + if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok { t.Error("File prefixed by '/' was not removed during transition to schema 1") } - if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil { + if _, err := db.Get(db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)), nil); err != nil { t.Error("Invalid file wasn't added to global list") } diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index 64496380f..7b04192b5 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -37,7 +37,7 @@ func (t readOnlyTransaction) close() { } func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool) { - return t.db.getFile(t.db.deviceKey(folder, device, file)) + return t.db.getFile(t.db.keyer.GenerateDeviceFileKey(nil, folder, device, file)) } // A readWriteTransaction is a readOnlyTransaction plus a batch for writes. @@ -111,7 +111,7 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto } // Fixup the list of files we need. - nk := t.db.needKey(folder, name) + nk := t.db.keyer.GenerateNeedFileKey(nil, folder, name) hasNeeded, _ := t.db.Has(nk, nil) if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) { if !hasNeeded { diff --git a/lib/db/meta.go b/lib/db/meta.go index 508b5250a..097df0a88 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -56,7 +56,7 @@ func (m *metadataTracker) Marshal() ([]byte, error) { // toDB saves the marshalled metadataTracker to the given db, under the key // corresponding to the given folder func (m *metadataTracker) toDB(db *Instance, folder []byte) error { - key := db.folderMetaKey(folder) + key := db.keyer.GenerateFolderMetaKey(nil, folder) bs, err := m.Marshal() if err != nil { return err @@ -67,7 +67,7 @@ func (m *metadataTracker) toDB(db *Instance, folder []byte) error { // fromDB initializes the metadataTracker from the marshalled data found in // the database under the key corresponding to the given folder func (m *metadataTracker) fromDB(db *Instance, folder []byte) error { - key := db.folderMetaKey(folder) + key := db.keyer.GenerateFolderMetaKey(nil, folder) bs, err := db.Get(key, nil) if err != nil { return err diff --git a/lib/db/set.go b/lib/db/set.go index e9a52e019..e979ae0f3 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -160,7 +160,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) { var dk []byte folder := []byte(s.folder) for _, nf := range oldFs { - dk = s.db.deviceKeyInto(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name))) + dk = s.db.keyer.GenerateDeviceFileKey(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name))) ef, ok := s.db.getFile(dk) if ok && ef.Version.Equal(nf.Version) && ef.IsInvalid() == nf.IsInvalid() { continue @@ -242,7 +242,7 @@ func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) { } func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) { - f, ok := s.db.getFile(s.db.deviceKey([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))) + f, ok := s.db.getFile(s.db.keyer.GenerateDeviceFileKey(nil, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))) f.Name = osutil.NativeFilename(f.Name) return f, ok } @@ -309,7 +309,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) { } func (s *FileSet) MtimeFS() *fs.MtimeFS { - prefix := s.db.mtimesKey([]byte(s.folder)) + prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder)) kv := NewNamespacedKV(s.db, string(prefix)) return fs.NewMtimeFS(s.fs, kv) } diff --git a/lib/db/smallindex.go b/lib/db/smallindex.go new file mode 100644 index 000000000..d171e0813 --- /dev/null +++ b/lib/db/smallindex.go @@ -0,0 +1,98 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "encoding/binary" + + "github.com/syncthing/syncthing/lib/sync" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" +) + +// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives +// fast lookups in both directions and persists to the database. Don't use for +// storing more items than fit comfortably in RAM. +type smallIndex struct { + db *leveldb.DB + prefix []byte + id2val map[uint32]string + val2id map[string]uint32 + nextID uint32 + mut sync.Mutex +} + +func newSmallIndex(db *leveldb.DB, prefix []byte) *smallIndex { + idx := &smallIndex{ + db: db, + prefix: prefix, + id2val: make(map[uint32]string), + val2id: make(map[string]uint32), + mut: sync.NewMutex(), + } + idx.load() + return idx +} + +// load iterates over the prefix space in the database and populates the in +// memory maps. +func (i *smallIndex) load() { + it := i.db.NewIterator(util.BytesPrefix(i.prefix), nil) + defer it.Release() + for it.Next() { + val := string(it.Value()) + id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):]) + i.id2val[id] = val + i.val2id[val] = id + if id >= i.nextID { + i.nextID = id + 1 + } + } +} + +// ID returns the index number for the given byte slice, allocating a new one +// and persisting this to the database if necessary. +func (i *smallIndex) ID(val []byte) uint32 { + i.mut.Lock() + // intentionally avoiding defer here as we want this call to be as fast as + // possible in the general case (folder ID already exists). The map lookup + // with the conversion of []byte to string is compiler optimized to not + // copy the []byte, which is why we don't assign it to a temp variable + // here. + if id, ok := i.val2id[string(val)]; ok { + i.mut.Unlock() + return id + } + + id := i.nextID + i.nextID++ + + valStr := string(val) + i.val2id[valStr] = id + i.id2val[id] = valStr + + key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id + copy(key, i.prefix) + binary.BigEndian.PutUint32(key[len(i.prefix):], id) + i.db.Put(key, val, nil) + + i.mut.Unlock() + return id +} + +// Val returns the value for the given index number, or (nil, false) if there +// is no such index number. +func (i *smallIndex) Val(id uint32) ([]byte, bool) { + i.mut.Lock() + val, ok := i.id2val[id] + i.mut.Unlock() + if !ok { + return nil, false + } + + return []byte(val), true +}