This breaks out the key generation stuff into a separate type. It's cleaner on its own, and it prepares for future stuff.
This commit is contained in:
parent
797a999585
commit
6a87aac84f
|
@ -0,0 +1,202 @@
|
||||||
|
// Copyright (C) 2018 The Syncthing Authors.
|
||||||
|
//
|
||||||
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||||
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
keyPrefixLen = 1
|
||||||
|
keyFolderLen = 4 // indexed
|
||||||
|
keyDeviceLen = 4 // indexed
|
||||||
|
keySequenceLen = 8
|
||||||
|
keyHashLen = 32
|
||||||
|
|
||||||
|
maxInt64 int64 = 1<<63 - 1
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
KeyTypeDevice = 0
|
||||||
|
KeyTypeGlobal = 1
|
||||||
|
KeyTypeBlock = 2
|
||||||
|
KeyTypeDeviceStatistic = 3
|
||||||
|
KeyTypeFolderStatistic = 4
|
||||||
|
KeyTypeVirtualMtime = 5
|
||||||
|
KeyTypeFolderIdx = 6
|
||||||
|
KeyTypeDeviceIdx = 7
|
||||||
|
KeyTypeIndexID = 8
|
||||||
|
KeyTypeFolderMeta = 9
|
||||||
|
KeyTypeMiscData = 10
|
||||||
|
KeyTypeSequence = 11
|
||||||
|
KeyTypeNeed = 12
|
||||||
|
)
|
||||||
|
|
||||||
|
type keyer interface {
|
||||||
|
// device file key stuff
|
||||||
|
GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey
|
||||||
|
NameFromDeviceFileKey(key []byte) []byte
|
||||||
|
DeviceFromDeviceFileKey(key []byte) ([]byte, bool)
|
||||||
|
FolderFromDeviceFileKey(key []byte) ([]byte, bool)
|
||||||
|
|
||||||
|
// global version key stuff
|
||||||
|
GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey
|
||||||
|
NameFromGlobalVersionKey(key []byte) []byte
|
||||||
|
FolderFromGlobalVersionKey(key []byte) ([]byte, bool)
|
||||||
|
|
||||||
|
// file need index
|
||||||
|
GenerateNeedFileKey(key, folder, name []byte) needFileKey
|
||||||
|
|
||||||
|
// file sequence index
|
||||||
|
GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey
|
||||||
|
SequenceFromSequenceKey(key []byte) int64
|
||||||
|
|
||||||
|
// index IDs
|
||||||
|
GenerateIndexIDKey(key, device, folder []byte) indexIDKey
|
||||||
|
DeviceFromIndexIDKey(key []byte) ([]byte, bool)
|
||||||
|
|
||||||
|
// Mtimes
|
||||||
|
GenerateMtimesKey(key, folder []byte) mtimesKey
|
||||||
|
|
||||||
|
// Folder metadata
|
||||||
|
GenerateFolderMetaKey(key, folder []byte) folderMetaKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultKeyer implements our key scheme. It needs folder and device
|
||||||
|
// indexes.
|
||||||
|
type defaultKeyer struct {
|
||||||
|
folderIdx *smallIndex
|
||||||
|
deviceIdx *smallIndex
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer {
|
||||||
|
return defaultKeyer{
|
||||||
|
folderIdx: folderIdx,
|
||||||
|
deviceIdx: deviceIdx,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type deviceFileKey []byte
|
||||||
|
|
||||||
|
func (k deviceFileKey) WithoutName() []byte {
|
||||||
|
return k[:keyPrefixLen+keyFolderLen+keyDeviceLen]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) deviceFileKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name))
|
||||||
|
key[0] = KeyTypeDevice
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], k.deviceIdx.ID(device))
|
||||||
|
copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) NameFromDeviceFileKey(key []byte) []byte {
|
||||||
|
return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) {
|
||||||
|
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) {
|
||||||
|
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
type globalVersionKey []byte
|
||||||
|
|
||||||
|
func (k globalVersionKey) WithoutName() []byte {
|
||||||
|
return k[:keyPrefixLen+keyFolderLen]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) globalVersionKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
|
||||||
|
key[0] = KeyTypeGlobal
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
copy(key[keyPrefixLen+keyFolderLen:], name)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte {
|
||||||
|
return key[keyPrefixLen+keyFolderLen:]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) FolderFromGlobalVersionKey(key []byte) ([]byte, bool) {
|
||||||
|
return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
type needFileKey []byte
|
||||||
|
|
||||||
|
func (k needFileKey) WithoutName() []byte {
|
||||||
|
return k[:keyPrefixLen+keyFolderLen]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) needFileKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen+len(name))
|
||||||
|
key[0] = KeyTypeNeed
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
copy(key[keyPrefixLen+keyFolderLen:], name)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
type sequenceKey []byte
|
||||||
|
|
||||||
|
func (k sequenceKey) WithoutSequence() []byte {
|
||||||
|
return k[:keyPrefixLen+keyFolderLen]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) sequenceKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen)
|
||||||
|
key[0] = KeyTypeSequence
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq))
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) SequenceFromSequenceKey(key []byte) int64 {
|
||||||
|
return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
type indexIDKey []byte
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) indexIDKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen)
|
||||||
|
key[0] = KeyTypeIndexID
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.deviceIdx.ID(device))
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], k.folderIdx.ID(folder))
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) {
|
||||||
|
return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
type mtimesKey []byte
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) mtimesKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen)
|
||||||
|
key[0] = KeyTypeVirtualMtime
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
type folderMetaKey []byte
|
||||||
|
|
||||||
|
func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) folderMetaKey {
|
||||||
|
key = resize(key, keyPrefixLen+keyFolderLen)
|
||||||
|
key[0] = KeyTypeFolderMeta
|
||||||
|
binary.BigEndian.PutUint32(key[keyPrefixLen:], k.folderIdx.ID(folder))
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
// resize returns a byte slice of the specified size, reusing bs if possible
|
||||||
|
func resize(bs []byte, size int) []byte {
|
||||||
|
if cap(bs) < size {
|
||||||
|
return make([]byte, size)
|
||||||
|
}
|
||||||
|
return bs[:size]
|
||||||
|
}
|
|
@ -0,0 +1,80 @@
|
||||||
|
// Copyright (C) 2018 The Syncthing Authors.
|
||||||
|
//
|
||||||
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||||
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDeviceKey(t *testing.T) {
|
||||||
|
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
|
||||||
|
dev := []byte("device67890123456789012345678901")
|
||||||
|
name := []byte("name")
|
||||||
|
|
||||||
|
db := OpenMemory()
|
||||||
|
|
||||||
|
key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
|
||||||
|
|
||||||
|
fld2, ok := db.keyer.FolderFromDeviceFileKey(key)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("unexpectedly not found")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(fld2, fld) {
|
||||||
|
t.Errorf("wrong folder %q != %q", fld2, fld)
|
||||||
|
}
|
||||||
|
dev2, ok := db.keyer.DeviceFromDeviceFileKey(key)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("unexpectedly not found")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(dev2, dev) {
|
||||||
|
t.Errorf("wrong device %q != %q", dev2, dev)
|
||||||
|
}
|
||||||
|
name2 := db.keyer.NameFromDeviceFileKey(key)
|
||||||
|
if !bytes.Equal(name2, name) {
|
||||||
|
t.Errorf("wrong name %q != %q", name2, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGlobalKey(t *testing.T) {
|
||||||
|
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
|
||||||
|
name := []byte("name")
|
||||||
|
|
||||||
|
db := OpenMemory()
|
||||||
|
|
||||||
|
key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
|
||||||
|
|
||||||
|
fld2, ok := db.keyer.FolderFromGlobalVersionKey(key)
|
||||||
|
if !ok {
|
||||||
|
t.Error("should have been found")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(fld2, fld) {
|
||||||
|
t.Errorf("wrong folder %q != %q", fld2, fld)
|
||||||
|
}
|
||||||
|
name2 := db.keyer.NameFromGlobalVersionKey(key)
|
||||||
|
if !bytes.Equal(name2, name) {
|
||||||
|
t.Errorf("wrong name %q != %q", name2, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, ok = db.keyer.FolderFromGlobalVersionKey([]byte{1, 2, 3, 4, 5})
|
||||||
|
if ok {
|
||||||
|
t.Error("should not have been found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequenceKey(t *testing.T) {
|
||||||
|
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
|
||||||
|
|
||||||
|
db := OpenMemory()
|
||||||
|
|
||||||
|
const seq = 1234567890
|
||||||
|
key := db.keyer.GenerateSequenceKey(nil, fld, seq)
|
||||||
|
outSeq := db.keyer.SequenceFromSequenceKey(key)
|
||||||
|
if outSeq != seq {
|
||||||
|
t.Errorf("sequence number mangled, %d != %d", outSeq, seq)
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,22 +13,6 @@ import (
|
||||||
"github.com/syncthing/syncthing/lib/protocol"
|
"github.com/syncthing/syncthing/lib/protocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
KeyTypeDevice = iota
|
|
||||||
KeyTypeGlobal
|
|
||||||
KeyTypeBlock
|
|
||||||
KeyTypeDeviceStatistic
|
|
||||||
KeyTypeFolderStatistic
|
|
||||||
KeyTypeVirtualMtime
|
|
||||||
KeyTypeFolderIdx
|
|
||||||
KeyTypeDeviceIdx
|
|
||||||
KeyTypeIndexID
|
|
||||||
KeyTypeFolderMeta
|
|
||||||
KeyTypeMiscData
|
|
||||||
KeyTypeSequence
|
|
||||||
KeyTypeNeed
|
|
||||||
)
|
|
||||||
|
|
||||||
func (vl VersionList) String() string {
|
func (vl VersionList) String() string {
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
var id protocol.DeviceID
|
var id protocol.DeviceID
|
||||||
|
@ -86,7 +70,7 @@ func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *
|
||||||
// to determine the winner.)
|
// to determine the winner.)
|
||||||
//
|
//
|
||||||
// A surprise missing file entry here is counted as a win for us.
|
// A surprise missing file entry here is counted as a win for us.
|
||||||
if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
|
if of, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
|
||||||
vl = vl.insertAt(i, nv)
|
vl = vl.insertAt(i, nv)
|
||||||
return vl, removedFV, removedAt, i
|
return vl, removedFV, removedAt, i
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,7 +16,6 @@ import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/syncthing/syncthing/lib/protocol"
|
"github.com/syncthing/syncthing/lib/protocol"
|
||||||
"github.com/syncthing/syncthing/lib/sync"
|
|
||||||
"github.com/syndtr/goleveldb/leveldb"
|
"github.com/syndtr/goleveldb/leveldb"
|
||||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||||
|
@ -33,18 +32,9 @@ type Instance struct {
|
||||||
location string
|
location string
|
||||||
folderIdx *smallIndex
|
folderIdx *smallIndex
|
||||||
deviceIdx *smallIndex
|
deviceIdx *smallIndex
|
||||||
|
keyer keyer
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
|
||||||
keyPrefixLen = 1
|
|
||||||
keyFolderLen = 4 // indexed
|
|
||||||
keyDeviceLen = 4 // indexed
|
|
||||||
keySequenceLen = 8
|
|
||||||
keyHashLen = 32
|
|
||||||
|
|
||||||
maxInt64 int64 = 1<<63 - 1
|
|
||||||
)
|
|
||||||
|
|
||||||
func Open(file string) (*Instance, error) {
|
func Open(file string) (*Instance, error) {
|
||||||
opts := &opt.Options{
|
opts := &opt.Options{
|
||||||
OpenFilesCacheCapacity: 100,
|
OpenFilesCacheCapacity: 100,
|
||||||
|
@ -82,9 +72,10 @@ func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
|
||||||
i := &Instance{
|
i := &Instance{
|
||||||
DB: db,
|
DB: db,
|
||||||
location: location,
|
location: location,
|
||||||
|
folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
|
||||||
|
deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
|
||||||
}
|
}
|
||||||
i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
|
i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
|
||||||
i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
|
|
||||||
err := i.updateSchema()
|
err := i.updateSchema()
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
@ -107,7 +98,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
|
||||||
var gk []byte
|
var gk []byte
|
||||||
for _, f := range fs {
|
for _, f := range fs {
|
||||||
name := []byte(f.Name)
|
name := []byte(f.Name)
|
||||||
fk = db.deviceKeyInto(fk, folder, device, name)
|
fk = db.keyer.GenerateDeviceFileKey(fk, folder, device, name)
|
||||||
|
|
||||||
// Get and unmarshal the file entry. If it doesn't exist or can't be
|
// Get and unmarshal the file entry. If it doesn't exist or can't be
|
||||||
// unmarshalled we'll add it as a new entry.
|
// unmarshalled we'll add it as a new entry.
|
||||||
|
@ -131,7 +122,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
|
||||||
|
|
||||||
t.insertFile(fk, folder, device, f)
|
t.insertFile(fk, folder, device, f)
|
||||||
|
|
||||||
gk = db.globalKeyInto(gk, folder, name)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
||||||
t.updateGlobal(gk, folder, device, f, meta)
|
t.updateGlobal(gk, folder, device, f, meta)
|
||||||
|
|
||||||
// Write out and reuse the batch every few records, to avoid the batch
|
// Write out and reuse the batch every few records, to avoid the batch
|
||||||
|
@ -147,8 +138,8 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
|
||||||
var sk []byte
|
var sk []byte
|
||||||
var dk []byte
|
var dk []byte
|
||||||
for _, f := range fs {
|
for _, f := range fs {
|
||||||
sk = db.sequenceKeyInto(sk, folder, f.Sequence)
|
sk = db.keyer.GenerateSequenceKey(sk, folder, f.Sequence)
|
||||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
|
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
|
||||||
t.Put(sk, dk)
|
t.Put(sk, dk)
|
||||||
l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
|
l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
|
@ -161,7 +152,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
|
||||||
|
|
||||||
var sk []byte
|
var sk []byte
|
||||||
for _, f := range fs {
|
for _, f := range fs {
|
||||||
t.Delete(db.sequenceKeyInto(sk, folder, f.Sequence))
|
t.Delete(db.keyer.GenerateSequenceKey(sk, folder, f.Sequence))
|
||||||
l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
|
l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
}
|
}
|
||||||
|
@ -176,7 +167,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
|
||||||
prefix = append(prefix, '/')
|
prefix = append(prefix, '/')
|
||||||
}
|
}
|
||||||
|
|
||||||
if f, ok := db.getFileTrunc(db.deviceKey(folder, device, unslashedPrefix), true); ok && !fn(f) {
|
if f, ok := db.getFileTrunc(db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix), true); ok && !fn(f) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -184,11 +175,11 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
|
||||||
t := db.newReadOnlyTransaction()
|
t := db.newReadOnlyTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, prefix)[:keyPrefixLen+keyFolderLen+keyDeviceLen+len(prefix)]), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
name := db.deviceKeyName(dbi.Key())
|
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
|
||||||
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -212,13 +203,13 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
|
||||||
t := db.newReadOnlyTransaction()
|
t := db.newReadOnlyTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(&util.Range{Start: db.sequenceKey(folder, startSeq), Limit: db.sequenceKey(folder, maxInt64)}, nil)
|
dbi := t.NewIterator(&util.Range{Start: db.keyer.GenerateSequenceKey(nil, folder, startSeq), Limit: db.keyer.GenerateSequenceKey(nil, folder, maxInt64)}, nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
f, ok := db.getFile(dbi.Value())
|
f, ok := db.getFile(dbi.Value())
|
||||||
if !ok {
|
if !ok {
|
||||||
l.Debugln("missing file for sequence number", db.sequenceKeySequence(dbi.Key()))
|
l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -239,13 +230,19 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
|
||||||
t := db.newReadWriteTransaction()
|
t := db.newReadWriteTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen]), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName()), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var gk []byte
|
var gk []byte
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
device := db.deviceKeyDevice(dbi.Key())
|
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
|
||||||
|
if !ok {
|
||||||
|
// Not having the device in the index is bad. Clear it.
|
||||||
|
t.Delete(dbi.Key())
|
||||||
|
t.checkFlush()
|
||||||
|
continue
|
||||||
|
}
|
||||||
var f FileInfoTruncated
|
var f FileInfoTruncated
|
||||||
// The iterator function may keep a reference to the unmarshalled
|
// The iterator function may keep a reference to the unmarshalled
|
||||||
// struct, which in turn references the buffer it was unmarshalled
|
// struct, which in turn references the buffer it was unmarshalled
|
||||||
|
@ -261,7 +258,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
|
||||||
case "", ".", "..", "/": // A few obviously invalid filenames
|
case "", ".", "..", "/": // A few obviously invalid filenames
|
||||||
l.Infof("Dropping invalid filename %q from database", f.Name)
|
l.Infof("Dropping invalid filename %q from database", f.Name)
|
||||||
name := []byte(f.Name)
|
name := []byte(f.Name)
|
||||||
gk = db.globalKeyInto(gk, folder, name)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
||||||
t.removeFromGlobal(gk, folder, device, name, nil)
|
t.removeFromGlobal(gk, folder, device, name, nil)
|
||||||
t.Delete(dbi.Key())
|
t.Delete(dbi.Key())
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
|
@ -308,7 +305,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
|
func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
|
||||||
gk = db.globalKeyInto(gk, folder, file)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
|
||||||
|
|
||||||
bs, err := t.Get(gk, nil)
|
bs, err := t.Get(gk, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -320,7 +317,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
|
||||||
return gk, dk, nil, false
|
return gk, dk, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
|
dk = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, file)
|
||||||
if fi, ok := db.getFileTrunc(dk, truncate); ok {
|
if fi, ok := db.getFileTrunc(dk, truncate); ok {
|
||||||
return gk, dk, fi, true
|
return gk, dk, fi, true
|
||||||
}
|
}
|
||||||
|
@ -345,12 +342,12 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
|
||||||
t := db.newReadOnlyTransaction()
|
t := db.newReadOnlyTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, prefix)), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var fk []byte
|
var fk []byte
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
name := db.globalKeyName(dbi.Key())
|
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
|
||||||
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -360,7 +357,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fk = db.deviceKeyInto(fk, folder, vl.Versions[0].Device, name)
|
fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[0].Device, name)
|
||||||
|
|
||||||
f, ok := db.getFileTrunc(fk, truncate)
|
f, ok := db.getFileTrunc(fk, truncate)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -374,7 +371,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
|
func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
|
||||||
k := db.globalKey(folder, file)
|
k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
|
||||||
bs, err := db.Get(k, nil)
|
bs, err := db.Get(k, nil)
|
||||||
if err == leveldb.ErrNotFound {
|
if err == leveldb.ErrNotFound {
|
||||||
return nil
|
return nil
|
||||||
|
@ -413,7 +410,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
|
||||||
t := db.newReadOnlyTransaction()
|
t := db.newReadOnlyTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var fk []byte
|
var fk []byte
|
||||||
|
@ -431,7 +428,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
name := db.globalKeyName(dbi.Key())
|
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
|
||||||
needVersion := vl.Versions[0].Version
|
needVersion := vl.Versions[0].Version
|
||||||
needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
|
needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
|
||||||
|
|
||||||
|
@ -446,7 +443,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fk = db.deviceKeyInto(fk, folder, vl.Versions[i].Device, name)
|
fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[i].Device, name)
|
||||||
bs, err := t.Get(fk, nil)
|
bs, err := t.Get(fk, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Debugln("surprise error:", err)
|
l.Debugln("surprise error:", err)
|
||||||
|
@ -480,7 +477,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
|
||||||
t := db.newReadOnlyTransaction()
|
t := db.newReadOnlyTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName()), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var dk []byte
|
var dk []byte
|
||||||
|
@ -488,7 +485,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
|
||||||
var f FileIntf
|
var f FileIntf
|
||||||
var ok bool
|
var ok bool
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
|
gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -507,7 +504,7 @@ func (db *Instance) ListFolders() []string {
|
||||||
|
|
||||||
folderExists := make(map[string]bool)
|
folderExists := make(map[string]bool)
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
folder, ok := db.globalKeyFolder(dbi.Key())
|
folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
|
||||||
if ok && !folderExists[string(folder)] {
|
if ok && !folderExists[string(folder)] {
|
||||||
folderExists[string(folder)] = true
|
folderExists[string(folder)] = true
|
||||||
}
|
}
|
||||||
|
@ -528,13 +525,13 @@ func (db *Instance) dropFolder(folder []byte) {
|
||||||
|
|
||||||
for _, key := range [][]byte{
|
for _, key := range [][]byte{
|
||||||
// Remove all items related to the given folder from the device->file bucket
|
// Remove all items related to the given folder from the device->file bucket
|
||||||
db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
|
db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName(),
|
||||||
// Remove all sequences related to the folder
|
// Remove all sequences related to the folder
|
||||||
db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
|
db.keyer.GenerateSequenceKey(nil, []byte(folder), 0).WithoutSequence(),
|
||||||
// Remove all items related to the given folder from the global bucket
|
// Remove all items related to the given folder from the global bucket
|
||||||
db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
|
db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName(),
|
||||||
// Remove all needs related to the folder
|
// Remove all needs related to the folder
|
||||||
db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
|
db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName(),
|
||||||
} {
|
} {
|
||||||
t.deleteKeyPrefix(key)
|
t.deleteKeyPrefix(key)
|
||||||
}
|
}
|
||||||
|
@ -544,15 +541,15 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
|
||||||
t := db.newReadWriteTransaction()
|
t := db.newReadWriteTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, nil)), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, nil)), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var gk []byte
|
var gk []byte
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
key := dbi.Key()
|
key := dbi.Key()
|
||||||
name := db.deviceKeyName(key)
|
name := db.keyer.NameFromDeviceFileKey(key)
|
||||||
gk = db.globalKeyInto(gk, folder, name)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
||||||
t.removeFromGlobal(gk, folder, device, name, meta)
|
t.removeFromGlobal(gk, folder, device, name, meta)
|
||||||
t.Delete(key)
|
t.Delete(key)
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
|
@ -563,7 +560,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
|
||||||
t := db.newReadWriteTransaction()
|
t := db.newReadWriteTransaction()
|
||||||
defer t.close()
|
defer t.close()
|
||||||
|
|
||||||
dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
|
dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
var fk []byte
|
var fk []byte
|
||||||
|
@ -578,10 +575,10 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
|
||||||
// there are global entries pointing to no longer existing files. Here
|
// there are global entries pointing to no longer existing files. Here
|
||||||
// we find those and clear them out.
|
// we find those and clear them out.
|
||||||
|
|
||||||
name := db.globalKeyName(dbi.Key())
|
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
|
||||||
var newVL VersionList
|
var newVL VersionList
|
||||||
for i, version := range vl.Versions {
|
for i, version := range vl.Versions {
|
||||||
fk = db.deviceKeyInto(fk, folder, version.Device, name)
|
fk = db.keyer.GenerateDeviceFileKey(fk, folder, version.Device, name)
|
||||||
_, err := t.Get(fk, nil)
|
_, err := t.Get(fk, nil)
|
||||||
if err == leveldb.ErrNotFound {
|
if err == leveldb.ErrNotFound {
|
||||||
continue
|
continue
|
||||||
|
@ -607,110 +604,8 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
|
||||||
l.Debugf("db check completed for %q", folder)
|
l.Debugf("db check completed for %q", folder)
|
||||||
}
|
}
|
||||||
|
|
||||||
// deviceKey returns a byte slice encoding the following information:
|
|
||||||
// keyTypeDevice (1 byte)
|
|
||||||
// folder (4 bytes)
|
|
||||||
// device (4 bytes)
|
|
||||||
// name (variable size)
|
|
||||||
func (db *Instance) deviceKey(folder, device, file []byte) []byte {
|
|
||||||
return db.deviceKeyInto(nil, folder, device, file)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) deviceKeyInto(k, folder, device, file []byte) []byte {
|
|
||||||
reqLen := keyPrefixLen + keyFolderLen + keyDeviceLen + len(file)
|
|
||||||
k = resize(k, reqLen)
|
|
||||||
k[0] = KeyTypeDevice
|
|
||||||
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
|
|
||||||
binary.BigEndian.PutUint32(k[keyPrefixLen+keyFolderLen:], db.deviceIdx.ID(device))
|
|
||||||
copy(k[keyPrefixLen+keyFolderLen+keyDeviceLen:], file)
|
|
||||||
return k
|
|
||||||
}
|
|
||||||
|
|
||||||
// deviceKeyName returns the device ID from the key
|
|
||||||
func (db *Instance) deviceKeyName(key []byte) []byte {
|
|
||||||
return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// deviceKeyFolder returns the folder name from the key
|
|
||||||
func (db *Instance) deviceKeyFolder(key []byte) []byte {
|
|
||||||
folder, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
|
||||||
if !ok {
|
|
||||||
panic("bug: lookup of nonexistent folder ID")
|
|
||||||
}
|
|
||||||
return folder
|
|
||||||
}
|
|
||||||
|
|
||||||
// deviceKeyDevice returns the device ID from the key
|
|
||||||
func (db *Instance) deviceKeyDevice(key []byte) []byte {
|
|
||||||
device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
|
|
||||||
if !ok {
|
|
||||||
panic("bug: lookup of nonexistent device ID")
|
|
||||||
}
|
|
||||||
return device
|
|
||||||
}
|
|
||||||
|
|
||||||
// globalKey returns a byte slice encoding the following information:
|
|
||||||
// keyTypeGlobal (1 byte)
|
|
||||||
// folder (4 bytes)
|
|
||||||
// name (variable size)
|
|
||||||
func (db *Instance) globalKey(folder, file []byte) []byte {
|
|
||||||
return db.globalKeyInto(nil, folder, file)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
|
|
||||||
reqLen := keyPrefixLen + keyFolderLen + len(file)
|
|
||||||
gk = resize(gk, reqLen)
|
|
||||||
gk[0] = KeyTypeGlobal
|
|
||||||
binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
|
|
||||||
copy(gk[keyPrefixLen+keyFolderLen:], file)
|
|
||||||
return gk
|
|
||||||
}
|
|
||||||
|
|
||||||
// globalKeyName returns the filename from the key
|
|
||||||
func (db *Instance) globalKeyName(key []byte) []byte {
|
|
||||||
return key[keyPrefixLen+keyFolderLen:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// globalKeyFolder returns the folder name from the key
|
|
||||||
func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
|
|
||||||
return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
// needKey is a globalKey with a different prefix
|
|
||||||
func (db *Instance) needKey(folder, file []byte) []byte {
|
|
||||||
return db.needKeyInto(nil, folder, file)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
|
|
||||||
k = db.globalKeyInto(k, folder, file)
|
|
||||||
k[0] = KeyTypeNeed
|
|
||||||
return k
|
|
||||||
}
|
|
||||||
|
|
||||||
// sequenceKey returns a byte slice encoding the following information:
|
|
||||||
// KeyTypeSequence (1 byte)
|
|
||||||
// folder (4 bytes)
|
|
||||||
// sequence number (8 bytes)
|
|
||||||
func (db *Instance) sequenceKey(folder []byte, seq int64) []byte {
|
|
||||||
return db.sequenceKeyInto(nil, folder, seq)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
|
|
||||||
reqLen := keyPrefixLen + keyFolderLen + keySequenceLen
|
|
||||||
k = resize(k, reqLen)
|
|
||||||
k[0] = KeyTypeSequence
|
|
||||||
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
|
|
||||||
binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
|
|
||||||
return k
|
|
||||||
}
|
|
||||||
|
|
||||||
// sequenceKeySequence returns the sequence number from the key
|
|
||||||
func (db *Instance) sequenceKeySequence(key []byte) int64 {
|
|
||||||
return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
|
func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
|
||||||
key := db.indexIDKey(device, folder)
|
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
|
||||||
cur, err := db.Get(key, nil)
|
cur, err := db.Get(key, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0
|
return 0
|
||||||
|
@ -725,44 +620,13 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
|
func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
|
||||||
key := db.indexIDKey(device, folder)
|
key := db.keyer.GenerateIndexIDKey(nil, device, folder)
|
||||||
bs, _ := id.Marshal() // marshalling can't fail
|
bs, _ := id.Marshal() // marshalling can't fail
|
||||||
if err := db.Put(key, bs, nil); err != nil {
|
if err := db.Put(key, bs, nil); err != nil {
|
||||||
panic("storing index ID: " + err.Error())
|
panic("storing index ID: " + err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) indexIDKey(device, folder []byte) []byte {
|
|
||||||
k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen)
|
|
||||||
k[0] = KeyTypeIndexID
|
|
||||||
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device))
|
|
||||||
binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder))
|
|
||||||
return k
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) indexIDDevice(key []byte) []byte {
|
|
||||||
device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
|
|
||||||
if !ok {
|
|
||||||
// uuh ...
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return device
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) mtimesKey(folder []byte) []byte {
|
|
||||||
prefix := make([]byte, 5) // key type + 4 bytes folder idx number
|
|
||||||
prefix[0] = KeyTypeVirtualMtime
|
|
||||||
binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
|
|
||||||
return prefix
|
|
||||||
}
|
|
||||||
|
|
||||||
func (db *Instance) folderMetaKey(folder []byte) []byte {
|
|
||||||
prefix := make([]byte, 5) // key type + 4 bytes folder idx number
|
|
||||||
prefix[0] = KeyTypeFolderMeta
|
|
||||||
binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
|
|
||||||
return prefix
|
|
||||||
}
|
|
||||||
|
|
||||||
// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
|
// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
|
||||||
// the database. This will cause a full index transmission on the next
|
// the database. This will cause a full index transmission on the next
|
||||||
// connection.
|
// connection.
|
||||||
|
@ -785,7 +649,7 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
|
||||||
defer dbi.Release()
|
defer dbi.Release()
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
device := db.indexIDDevice(dbi.Key())
|
device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
|
||||||
if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
|
if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
|
||||||
t.Delete(dbi.Key())
|
t.Delete(dbi.Key())
|
||||||
}
|
}
|
||||||
|
@ -793,11 +657,11 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) dropMtimes(folder []byte) {
|
func (db *Instance) dropMtimes(folder []byte) {
|
||||||
db.dropPrefix(db.mtimesKey(folder))
|
db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) dropFolderMeta(folder []byte) {
|
func (db *Instance) dropFolderMeta(folder []byte) {
|
||||||
db.dropPrefix(db.folderMetaKey(folder))
|
db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Instance) dropPrefix(prefix []byte) {
|
func (db *Instance) dropPrefix(prefix []byte) {
|
||||||
|
@ -853,99 +717,6 @@ func leveldbIsCorrupted(err error) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
|
|
||||||
// fast lookups in both directions and persists to the database. Don't use for
|
|
||||||
// storing more items than fit comfortably in RAM.
|
|
||||||
type smallIndex struct {
|
|
||||||
db *Instance
|
|
||||||
prefix []byte
|
|
||||||
id2val map[uint32]string
|
|
||||||
val2id map[string]uint32
|
|
||||||
nextID uint32
|
|
||||||
mut sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func newSmallIndex(db *Instance, prefix []byte) *smallIndex {
|
|
||||||
idx := &smallIndex{
|
|
||||||
db: db,
|
|
||||||
prefix: prefix,
|
|
||||||
id2val: make(map[uint32]string),
|
|
||||||
val2id: make(map[string]uint32),
|
|
||||||
mut: sync.NewMutex(),
|
|
||||||
}
|
|
||||||
idx.load()
|
|
||||||
return idx
|
|
||||||
}
|
|
||||||
|
|
||||||
// load iterates over the prefix space in the database and populates the in
|
|
||||||
// memory maps.
|
|
||||||
func (i *smallIndex) load() {
|
|
||||||
tr := i.db.newReadOnlyTransaction()
|
|
||||||
it := tr.NewIterator(util.BytesPrefix(i.prefix), nil)
|
|
||||||
for it.Next() {
|
|
||||||
val := string(it.Value())
|
|
||||||
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
|
|
||||||
i.id2val[id] = val
|
|
||||||
i.val2id[val] = id
|
|
||||||
if id >= i.nextID {
|
|
||||||
i.nextID = id + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
it.Release()
|
|
||||||
tr.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ID returns the index number for the given byte slice, allocating a new one
|
|
||||||
// and persisting this to the database if necessary.
|
|
||||||
func (i *smallIndex) ID(val []byte) uint32 {
|
|
||||||
i.mut.Lock()
|
|
||||||
// intentionally avoiding defer here as we want this call to be as fast as
|
|
||||||
// possible in the general case (folder ID already exists). The map lookup
|
|
||||||
// with the conversion of []byte to string is compiler optimized to not
|
|
||||||
// copy the []byte, which is why we don't assign it to a temp variable
|
|
||||||
// here.
|
|
||||||
if id, ok := i.val2id[string(val)]; ok {
|
|
||||||
i.mut.Unlock()
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
id := i.nextID
|
|
||||||
i.nextID++
|
|
||||||
|
|
||||||
valStr := string(val)
|
|
||||||
i.val2id[valStr] = id
|
|
||||||
i.id2val[id] = valStr
|
|
||||||
|
|
||||||
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
|
|
||||||
copy(key, i.prefix)
|
|
||||||
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
|
|
||||||
i.db.Put(key, val, nil)
|
|
||||||
|
|
||||||
i.mut.Unlock()
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
// Val returns the value for the given index number, or (nil, false) if there
|
|
||||||
// is no such index number.
|
|
||||||
func (i *smallIndex) Val(id uint32) ([]byte, bool) {
|
|
||||||
i.mut.Lock()
|
|
||||||
val, ok := i.id2val[id]
|
|
||||||
i.mut.Unlock()
|
|
||||||
if !ok {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return []byte(val), true
|
|
||||||
}
|
|
||||||
|
|
||||||
// resize returns a byte array of length reqLen, reusing k if possible
|
|
||||||
func resize(k []byte, reqLen int) []byte {
|
|
||||||
if cap(k) < reqLen {
|
|
||||||
return make([]byte, reqLen)
|
|
||||||
}
|
|
||||||
return k[:reqLen]
|
|
||||||
}
|
|
||||||
|
|
||||||
type errorSuggestion struct {
|
type errorSuggestion struct {
|
||||||
inner error
|
inner error
|
||||||
suggestion string
|
suggestion string
|
||||||
|
|
|
@ -91,16 +91,28 @@ func (db *Instance) updateSchema0to1() {
|
||||||
var gk []byte
|
var gk []byte
|
||||||
|
|
||||||
for dbi.Next() {
|
for dbi.Next() {
|
||||||
folder := db.deviceKeyFolder(dbi.Key())
|
folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
|
||||||
device := db.deviceKeyDevice(dbi.Key())
|
if !ok {
|
||||||
name := db.deviceKeyName(dbi.Key())
|
// not having the folder in the index is bad; delete and continue
|
||||||
|
t.Delete(dbi.Key())
|
||||||
|
t.checkFlush()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
|
||||||
|
if !ok {
|
||||||
|
// not having the device in the index is bad; delete and continue
|
||||||
|
t.Delete(dbi.Key())
|
||||||
|
t.checkFlush()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
|
||||||
|
|
||||||
// Remove files with absolute path (see #4799)
|
// Remove files with absolute path (see #4799)
|
||||||
if strings.HasPrefix(string(name), "/") {
|
if strings.HasPrefix(string(name), "/") {
|
||||||
if _, ok := changedFolders[string(folder)]; !ok {
|
if _, ok := changedFolders[string(folder)]; !ok {
|
||||||
changedFolders[string(folder)] = struct{}{}
|
changedFolders[string(folder)] = struct{}{}
|
||||||
}
|
}
|
||||||
gk = db.globalKeyInto(gk, folder, name)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
||||||
t.removeFromGlobal(gk, folder, device, nil, nil)
|
t.removeFromGlobal(gk, folder, device, nil, nil)
|
||||||
t.Delete(dbi.Key())
|
t.Delete(dbi.Key())
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
|
@ -130,7 +142,7 @@ func (db *Instance) updateSchema0to1() {
|
||||||
|
|
||||||
// Add invalid files to global list
|
// Add invalid files to global list
|
||||||
if f.IsInvalid() {
|
if f.IsInvalid() {
|
||||||
gk = db.globalKeyInto(gk, folder, name)
|
gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
||||||
if t.updateGlobal(gk, folder, device, f, meta) {
|
if t.updateGlobal(gk, folder, device, f, meta) {
|
||||||
if _, ok := changedFolders[string(folder)]; !ok {
|
if _, ok := changedFolders[string(folder)]; !ok {
|
||||||
changedFolders[string(folder)] = struct{}{}
|
changedFolders[string(folder)] = struct{}{}
|
||||||
|
@ -156,8 +168,8 @@ func (db *Instance) updateSchema1to2() {
|
||||||
for _, folderStr := range db.ListFolders() {
|
for _, folderStr := range db.ListFolders() {
|
||||||
folder := []byte(folderStr)
|
folder := []byte(folderStr)
|
||||||
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
|
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
|
||||||
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
|
sk = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
|
||||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
|
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
|
||||||
t.Put(sk, dk)
|
t.Put(sk, dk)
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
return true
|
return true
|
||||||
|
@ -176,7 +188,7 @@ func (db *Instance) updateSchema2to3() {
|
||||||
folder := []byte(folderStr)
|
folder := []byte(folderStr)
|
||||||
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
|
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
|
||||||
name := []byte(f.FileName())
|
name := []byte(f.FileName())
|
||||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
|
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
|
||||||
var v protocol.Vector
|
var v protocol.Vector
|
||||||
haveFile, ok := db.getFileTrunc(dk, true)
|
haveFile, ok := db.getFileTrunc(dk, true)
|
||||||
if ok {
|
if ok {
|
||||||
|
@ -185,7 +197,7 @@ func (db *Instance) updateSchema2to3() {
|
||||||
if !need(f, ok, v) {
|
if !need(f, ok, v) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
|
nk = t.db.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
|
||||||
t.Put(nk, nil)
|
t.Put(nk, nil)
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
return true
|
return true
|
||||||
|
@ -201,7 +213,7 @@ func (db *Instance) updateSchemaTo5() {
|
||||||
t := db.newReadWriteTransaction()
|
t := db.newReadWriteTransaction()
|
||||||
var nk []byte
|
var nk []byte
|
||||||
for _, folderStr := range db.ListFolders() {
|
for _, folderStr := range db.ListFolders() {
|
||||||
nk = db.needKeyInto(nk, []byte(folderStr), nil)
|
nk = db.keyer.GenerateNeedFileKey(nk, []byte(folderStr), nil)
|
||||||
t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen])
|
t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen])
|
||||||
}
|
}
|
||||||
t.close()
|
t.close()
|
||||||
|
@ -230,7 +242,7 @@ func (db *Instance) updateSchema5to6() {
|
||||||
fi.LocalFlags = protocol.FlagLocalIgnored
|
fi.LocalFlags = protocol.FlagLocalIgnored
|
||||||
bs, _ := fi.Marshal()
|
bs, _ := fi.Marshal()
|
||||||
|
|
||||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
|
dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
|
||||||
t.Put(dk, bs)
|
t.Put(dk, bs)
|
||||||
|
|
||||||
t.checkFlush()
|
t.checkFlush()
|
||||||
|
|
|
@ -7,7 +7,6 @@
|
||||||
package db
|
package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -15,58 +14,6 @@ import (
|
||||||
"github.com/syncthing/syncthing/lib/protocol"
|
"github.com/syncthing/syncthing/lib/protocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDeviceKey(t *testing.T) {
|
|
||||||
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
|
|
||||||
dev := []byte("device67890123456789012345678901")
|
|
||||||
name := []byte("name")
|
|
||||||
|
|
||||||
db := OpenMemory()
|
|
||||||
db.folderIdx.ID(fld)
|
|
||||||
db.deviceIdx.ID(dev)
|
|
||||||
|
|
||||||
key := db.deviceKey(fld, dev, name)
|
|
||||||
|
|
||||||
fld2 := db.deviceKeyFolder(key)
|
|
||||||
if !bytes.Equal(fld2, fld) {
|
|
||||||
t.Errorf("wrong folder %q != %q", fld2, fld)
|
|
||||||
}
|
|
||||||
dev2 := db.deviceKeyDevice(key)
|
|
||||||
if !bytes.Equal(dev2, dev) {
|
|
||||||
t.Errorf("wrong device %q != %q", dev2, dev)
|
|
||||||
}
|
|
||||||
name2 := db.deviceKeyName(key)
|
|
||||||
if !bytes.Equal(name2, name) {
|
|
||||||
t.Errorf("wrong name %q != %q", name2, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGlobalKey(t *testing.T) {
|
|
||||||
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
|
|
||||||
name := []byte("name")
|
|
||||||
|
|
||||||
db := OpenMemory()
|
|
||||||
db.folderIdx.ID(fld)
|
|
||||||
|
|
||||||
key := db.globalKey(fld, name)
|
|
||||||
|
|
||||||
fld2, ok := db.globalKeyFolder(key)
|
|
||||||
if !ok {
|
|
||||||
t.Error("should have been found")
|
|
||||||
}
|
|
||||||
if !bytes.Equal(fld2, fld) {
|
|
||||||
t.Errorf("wrong folder %q != %q", fld2, fld)
|
|
||||||
}
|
|
||||||
name2 := db.globalKeyName(key)
|
|
||||||
if !bytes.Equal(name2, name) {
|
|
||||||
t.Errorf("wrong name %q != %q", name2, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, ok = db.globalKeyFolder([]byte{1, 2, 3, 4, 5})
|
|
||||||
if ok {
|
|
||||||
t.Error("should not have been found")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDropIndexIDs(t *testing.T) {
|
func TestDropIndexIDs(t *testing.T) {
|
||||||
db := OpenMemory()
|
db := OpenMemory()
|
||||||
|
|
||||||
|
@ -287,11 +234,11 @@ func TestUpdate0to3(t *testing.T) {
|
||||||
|
|
||||||
db.updateSchema0to1()
|
db.updateSchema0to1()
|
||||||
|
|
||||||
if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
|
if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
|
||||||
t.Error("File prefixed by '/' was not removed during transition to schema 1")
|
t.Error("File prefixed by '/' was not removed during transition to schema 1")
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil {
|
if _, err := db.Get(db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)), nil); err != nil {
|
||||||
t.Error("Invalid file wasn't added to global list")
|
t.Error("Invalid file wasn't added to global list")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ func (t readOnlyTransaction) close() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
|
func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
|
||||||
return t.db.getFile(t.db.deviceKey(folder, device, file))
|
return t.db.getFile(t.db.keyer.GenerateDeviceFileKey(nil, folder, device, file))
|
||||||
}
|
}
|
||||||
|
|
||||||
// A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
|
// A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
|
||||||
|
@ -111,7 +111,7 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fixup the list of files we need.
|
// Fixup the list of files we need.
|
||||||
nk := t.db.needKey(folder, name)
|
nk := t.db.keyer.GenerateNeedFileKey(nil, folder, name)
|
||||||
hasNeeded, _ := t.db.Has(nk, nil)
|
hasNeeded, _ := t.db.Has(nk, nil)
|
||||||
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
|
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
|
||||||
if !hasNeeded {
|
if !hasNeeded {
|
||||||
|
|
|
@ -56,7 +56,7 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
|
||||||
// toDB saves the marshalled metadataTracker to the given db, under the key
|
// toDB saves the marshalled metadataTracker to the given db, under the key
|
||||||
// corresponding to the given folder
|
// corresponding to the given folder
|
||||||
func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
|
func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
|
||||||
key := db.folderMetaKey(folder)
|
key := db.keyer.GenerateFolderMetaKey(nil, folder)
|
||||||
bs, err := m.Marshal()
|
bs, err := m.Marshal()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -67,7 +67,7 @@ func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
|
||||||
// fromDB initializes the metadataTracker from the marshalled data found in
|
// fromDB initializes the metadataTracker from the marshalled data found in
|
||||||
// the database under the key corresponding to the given folder
|
// the database under the key corresponding to the given folder
|
||||||
func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
|
func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
|
||||||
key := db.folderMetaKey(folder)
|
key := db.keyer.GenerateFolderMetaKey(nil, folder)
|
||||||
bs, err := db.Get(key, nil)
|
bs, err := db.Get(key, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -160,7 +160,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||||
var dk []byte
|
var dk []byte
|
||||||
folder := []byte(s.folder)
|
folder := []byte(s.folder)
|
||||||
for _, nf := range oldFs {
|
for _, nf := range oldFs {
|
||||||
dk = s.db.deviceKeyInto(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
|
dk = s.db.keyer.GenerateDeviceFileKey(dk, folder, device[:], []byte(osutil.NormalizedFilename(nf.Name)))
|
||||||
ef, ok := s.db.getFile(dk)
|
ef, ok := s.db.getFile(dk)
|
||||||
if ok && ef.Version.Equal(nf.Version) && ef.IsInvalid() == nf.IsInvalid() {
|
if ok && ef.Version.Equal(nf.Version) && ef.IsInvalid() == nf.IsInvalid() {
|
||||||
continue
|
continue
|
||||||
|
@ -242,7 +242,7 @@ func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
|
func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
|
||||||
f, ok := s.db.getFile(s.db.deviceKey([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
|
f, ok := s.db.getFile(s.db.keyer.GenerateDeviceFileKey(nil, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))))
|
||||||
f.Name = osutil.NativeFilename(f.Name)
|
f.Name = osutil.NativeFilename(f.Name)
|
||||||
return f, ok
|
return f, ok
|
||||||
}
|
}
|
||||||
|
@ -309,7 +309,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *FileSet) MtimeFS() *fs.MtimeFS {
|
func (s *FileSet) MtimeFS() *fs.MtimeFS {
|
||||||
prefix := s.db.mtimesKey([]byte(s.folder))
|
prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
|
||||||
kv := NewNamespacedKV(s.db, string(prefix))
|
kv := NewNamespacedKV(s.db, string(prefix))
|
||||||
return fs.NewMtimeFS(s.fs, kv)
|
return fs.NewMtimeFS(s.fs, kv)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,98 @@
|
||||||
|
// Copyright (C) 2018 The Syncthing Authors.
|
||||||
|
//
|
||||||
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||||
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
|
||||||
|
"github.com/syncthing/syncthing/lib/sync"
|
||||||
|
"github.com/syndtr/goleveldb/leveldb"
|
||||||
|
"github.com/syndtr/goleveldb/leveldb/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
|
||||||
|
// fast lookups in both directions and persists to the database. Don't use for
|
||||||
|
// storing more items than fit comfortably in RAM.
|
||||||
|
type smallIndex struct {
|
||||||
|
db *leveldb.DB
|
||||||
|
prefix []byte
|
||||||
|
id2val map[uint32]string
|
||||||
|
val2id map[string]uint32
|
||||||
|
nextID uint32
|
||||||
|
mut sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSmallIndex(db *leveldb.DB, prefix []byte) *smallIndex {
|
||||||
|
idx := &smallIndex{
|
||||||
|
db: db,
|
||||||
|
prefix: prefix,
|
||||||
|
id2val: make(map[uint32]string),
|
||||||
|
val2id: make(map[string]uint32),
|
||||||
|
mut: sync.NewMutex(),
|
||||||
|
}
|
||||||
|
idx.load()
|
||||||
|
return idx
|
||||||
|
}
|
||||||
|
|
||||||
|
// load iterates over the prefix space in the database and populates the in
|
||||||
|
// memory maps.
|
||||||
|
func (i *smallIndex) load() {
|
||||||
|
it := i.db.NewIterator(util.BytesPrefix(i.prefix), nil)
|
||||||
|
defer it.Release()
|
||||||
|
for it.Next() {
|
||||||
|
val := string(it.Value())
|
||||||
|
id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
|
||||||
|
i.id2val[id] = val
|
||||||
|
i.val2id[val] = id
|
||||||
|
if id >= i.nextID {
|
||||||
|
i.nextID = id + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ID returns the index number for the given byte slice, allocating a new one
|
||||||
|
// and persisting this to the database if necessary.
|
||||||
|
func (i *smallIndex) ID(val []byte) uint32 {
|
||||||
|
i.mut.Lock()
|
||||||
|
// intentionally avoiding defer here as we want this call to be as fast as
|
||||||
|
// possible in the general case (folder ID already exists). The map lookup
|
||||||
|
// with the conversion of []byte to string is compiler optimized to not
|
||||||
|
// copy the []byte, which is why we don't assign it to a temp variable
|
||||||
|
// here.
|
||||||
|
if id, ok := i.val2id[string(val)]; ok {
|
||||||
|
i.mut.Unlock()
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
id := i.nextID
|
||||||
|
i.nextID++
|
||||||
|
|
||||||
|
valStr := string(val)
|
||||||
|
i.val2id[valStr] = id
|
||||||
|
i.id2val[id] = valStr
|
||||||
|
|
||||||
|
key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
|
||||||
|
copy(key, i.prefix)
|
||||||
|
binary.BigEndian.PutUint32(key[len(i.prefix):], id)
|
||||||
|
i.db.Put(key, val, nil)
|
||||||
|
|
||||||
|
i.mut.Unlock()
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// Val returns the value for the given index number, or (nil, false) if there
|
||||||
|
// is no such index number.
|
||||||
|
func (i *smallIndex) Val(id uint32) ([]byte, bool) {
|
||||||
|
i.mut.Lock()
|
||||||
|
val, ok := i.id2val[id]
|
||||||
|
i.mut.Unlock()
|
||||||
|
if !ok {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return []byte(val), true
|
||||||
|
}
|
Loading…
Reference in New Issue