Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors. Co-authored-by: Jakob Borg <jakob@kastelo.net>
524 lines
15 KiB
Go
524 lines
15 KiB
Go
// Copyright (C) 2014 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
//go:generate go run ../../script/protofmt.go structs.proto
|
|
//go:generate protoc -I ../../ -I . --gogofast_out=Mlib/protocol/bep.proto=github.com/syncthing/syncthing/lib/protocol:. structs.proto
|
|
|
|
package db
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
|
)
|
|
|
|
func (f FileInfoTruncated) String() string {
|
|
switch f.Type {
|
|
case protocol.FileInfoTypeDirectory:
|
|
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v}",
|
|
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions)
|
|
case protocol.FileInfoTypeFile:
|
|
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d}",
|
|
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.RawBlockSize)
|
|
case protocol.FileInfoTypeSymlink, protocol.FileInfoTypeDeprecatedSymlinkDirectory, protocol.FileInfoTypeDeprecatedSymlinkFile:
|
|
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q}",
|
|
f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.SymlinkTarget)
|
|
default:
|
|
panic("mystery file type detected")
|
|
}
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsDeleted() bool {
|
|
return f.Deleted
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsInvalid() bool {
|
|
return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsUnsupported() bool {
|
|
return f.LocalFlags&protocol.FlagLocalUnsupported != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsIgnored() bool {
|
|
return f.LocalFlags&protocol.FlagLocalIgnored != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) MustRescan() bool {
|
|
return f.LocalFlags&protocol.FlagLocalMustRescan != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsReceiveOnlyChanged() bool {
|
|
return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsDirectory() bool {
|
|
return f.Type == protocol.FileInfoTypeDirectory
|
|
}
|
|
|
|
func (f FileInfoTruncated) IsSymlink() bool {
|
|
switch f.Type {
|
|
case protocol.FileInfoTypeSymlink, protocol.FileInfoTypeDeprecatedSymlinkDirectory, protocol.FileInfoTypeDeprecatedSymlinkFile:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (f FileInfoTruncated) ShouldConflict() bool {
|
|
return f.LocalFlags&protocol.LocalConflictFlags != 0
|
|
}
|
|
|
|
func (f FileInfoTruncated) HasPermissionBits() bool {
|
|
return !f.NoPermissions
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileSize() int64 {
|
|
if f.Deleted {
|
|
return 0
|
|
}
|
|
if f.IsDirectory() || f.IsSymlink() {
|
|
return protocol.SyntheticDirectorySize
|
|
}
|
|
return f.Size
|
|
}
|
|
|
|
func (f FileInfoTruncated) BlockSize() int {
|
|
if f.RawBlockSize == 0 {
|
|
return protocol.MinBlockSize
|
|
}
|
|
return int(f.RawBlockSize)
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileName() string {
|
|
return f.Name
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileLocalFlags() uint32 {
|
|
return f.LocalFlags
|
|
}
|
|
|
|
func (f FileInfoTruncated) ModTime() time.Time {
|
|
return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
|
|
}
|
|
|
|
func (f FileInfoTruncated) SequenceNo() int64 {
|
|
return f.Sequence
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileVersion() protocol.Vector {
|
|
return f.Version
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileType() protocol.FileInfoType {
|
|
return f.Type
|
|
}
|
|
|
|
func (f FileInfoTruncated) FilePermissions() uint32 {
|
|
return f.Permissions
|
|
}
|
|
|
|
func (f FileInfoTruncated) FileModifiedBy() protocol.ShortID {
|
|
return f.ModifiedBy
|
|
}
|
|
|
|
func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protocol.FileInfo {
|
|
file := f.copyToFileInfo()
|
|
file.SetIgnored(by)
|
|
return file
|
|
}
|
|
|
|
func (f FileInfoTruncated) ConvertToDeletedFileInfo(by protocol.ShortID) protocol.FileInfo {
|
|
file := f.copyToFileInfo()
|
|
file.SetDeleted(by)
|
|
return file
|
|
}
|
|
|
|
// ConvertDeletedToFileInfo converts a deleted truncated file info to a regular file info
|
|
func (f FileInfoTruncated) ConvertDeletedToFileInfo() protocol.FileInfo {
|
|
if !f.Deleted {
|
|
panic("ConvertDeletedToFileInfo must only be called on deleted items")
|
|
}
|
|
return f.copyToFileInfo()
|
|
}
|
|
|
|
// copyToFileInfo just copies all members of FileInfoTruncated to protocol.FileInfo
|
|
func (f FileInfoTruncated) copyToFileInfo() protocol.FileInfo {
|
|
return protocol.FileInfo{
|
|
Name: f.Name,
|
|
Size: f.Size,
|
|
ModifiedS: f.ModifiedS,
|
|
ModifiedBy: f.ModifiedBy,
|
|
Version: f.Version,
|
|
Sequence: f.Sequence,
|
|
SymlinkTarget: f.SymlinkTarget,
|
|
BlocksHash: f.BlocksHash,
|
|
Type: f.Type,
|
|
Permissions: f.Permissions,
|
|
ModifiedNs: f.ModifiedNs,
|
|
RawBlockSize: f.RawBlockSize,
|
|
LocalFlags: f.LocalFlags,
|
|
Deleted: f.Deleted,
|
|
RawInvalid: f.RawInvalid,
|
|
NoPermissions: f.NoPermissions,
|
|
}
|
|
}
|
|
|
|
func (c Counts) Add(other Counts) Counts {
|
|
return Counts{
|
|
Files: c.Files + other.Files,
|
|
Directories: c.Directories + other.Directories,
|
|
Symlinks: c.Symlinks + other.Symlinks,
|
|
Deleted: c.Deleted + other.Deleted,
|
|
Bytes: c.Bytes + other.Bytes,
|
|
Sequence: c.Sequence + other.Sequence,
|
|
DeviceID: protocol.EmptyDeviceID[:],
|
|
LocalFlags: c.LocalFlags | other.LocalFlags,
|
|
}
|
|
}
|
|
|
|
func (c Counts) TotalItems() int32 {
|
|
return c.Files + c.Directories + c.Symlinks + c.Deleted
|
|
}
|
|
|
|
// Equal compares the numbers only, not sequence/dev/flags.
|
|
func (c Counts) Equal(o Counts) bool {
|
|
return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
|
|
}
|
|
|
|
func (vl VersionList) String() string {
|
|
var b bytes.Buffer
|
|
var id protocol.DeviceID
|
|
b.WriteString("{")
|
|
for i, v := range vl.RawVersions {
|
|
if i > 0 {
|
|
b.WriteString(", ")
|
|
}
|
|
fmt.Fprintf(&b, "{%v, {", v.Version)
|
|
for j, dev := range v.Devices {
|
|
if j > 0 {
|
|
b.WriteString(", ")
|
|
}
|
|
copy(id[:], dev)
|
|
fmt.Fprint(&b, id.Short())
|
|
}
|
|
b.WriteString("}, {")
|
|
for j, dev := range v.InvalidDevices {
|
|
if j > 0 {
|
|
b.WriteString(", ")
|
|
}
|
|
copy(id[:], dev)
|
|
fmt.Fprint(&b, id.Short())
|
|
}
|
|
fmt.Fprint(&b, "}}")
|
|
}
|
|
b.WriteString("}")
|
|
return b.String()
|
|
}
|
|
|
|
// update brings the VersionList up to date with file. It returns the updated
|
|
// VersionList, a device that has the global/newest version, a device that previously
|
|
// had the global/newest version, a boolean indicating if the global version has
|
|
// changed and if any error occurred (only possible in db interaction).
|
|
func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) {
|
|
if len(vl.RawVersions) == 0 {
|
|
nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
|
|
vl.RawVersions = append(vl.RawVersions, nv)
|
|
return nv, FileVersion{}, FileVersion{}, false, false, true, nil
|
|
}
|
|
|
|
// Get the current global (before updating)
|
|
oldFV, haveOldGlobal := vl.GetGlobal()
|
|
|
|
// Remove ourselves first
|
|
removedFV, haveRemoved, _, err := vl.pop(folder, device, []byte(file.FileName()), t)
|
|
if err == nil {
|
|
// Find position and insert the file
|
|
err = vl.insert(folder, device, file, t)
|
|
}
|
|
if err != nil {
|
|
return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err
|
|
}
|
|
|
|
newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty
|
|
|
|
if !haveOldGlobal {
|
|
return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil
|
|
}
|
|
|
|
globalChanged := true
|
|
if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) {
|
|
globalChanged = false
|
|
}
|
|
|
|
return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
|
|
}
|
|
|
|
func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error {
|
|
var added bool
|
|
var err error
|
|
i := 0
|
|
for ; i < len(vl.RawVersions); i++ {
|
|
// Insert our new version
|
|
added, err = vl.checkInsertAt(i, folder, device, file, t)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if added {
|
|
break
|
|
}
|
|
}
|
|
if i == len(vl.RawVersions) {
|
|
// Append to the end
|
|
vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (vl *VersionList) insertAt(i int, v FileVersion) {
|
|
vl.RawVersions = append(vl.RawVersions, FileVersion{})
|
|
copy(vl.RawVersions[i+1:], vl.RawVersions[i:])
|
|
vl.RawVersions[i] = v
|
|
}
|
|
|
|
// pop returns the VersionList without the entry for the given device, as well
|
|
// as the removed FileVersion, whether it was found/removed at all and whether
|
|
// the global changed in the process.
|
|
func (vl *VersionList) pop(folder, device, name []byte, t readOnlyTransaction) (FileVersion, bool, bool, error) {
|
|
invDevice, i, j, ok := vl.findDevice(device)
|
|
if !ok {
|
|
return FileVersion{}, false, false, nil
|
|
}
|
|
globalPos := vl.findGlobal()
|
|
|
|
if vl.RawVersions[i].deviceCount() == 1 {
|
|
fv := vl.RawVersions[i]
|
|
vl.popVersionAt(i)
|
|
return fv, true, globalPos == i, nil
|
|
}
|
|
|
|
if invDevice {
|
|
vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j)
|
|
} else {
|
|
vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j)
|
|
}
|
|
// If the last valid device of the previous global was removed above,
|
|
// the next entry is now the global entry (unless all entries are invalid).
|
|
if len(vl.RawVersions[i].Devices) == 0 && globalPos == i {
|
|
return vl.RawVersions[i], true, globalPos == vl.findGlobal(), nil
|
|
}
|
|
return vl.RawVersions[i], true, false, nil
|
|
}
|
|
|
|
// Get returns a FileVersion that contains the given device and whether it has
|
|
// been found at all.
|
|
func (vl *VersionList) Get(device []byte) (FileVersion, bool) {
|
|
_, i, _, ok := vl.findDevice(device)
|
|
if !ok {
|
|
return FileVersion{}, false
|
|
}
|
|
return vl.RawVersions[i], true
|
|
}
|
|
|
|
// GetGlobal returns the current global FileVersion. The returned FileVersion
|
|
// may be invalid, if all FileVersions are invalid. Returns false only if
|
|
// VersionList is empty.
|
|
func (vl *VersionList) GetGlobal() (FileVersion, bool) {
|
|
i := vl.findGlobal()
|
|
if i == -1 {
|
|
return FileVersion{}, false
|
|
}
|
|
return vl.RawVersions[i], true
|
|
}
|
|
|
|
func (vl *VersionList) Empty() bool {
|
|
return len(vl.RawVersions) == 0
|
|
}
|
|
|
|
// findGlobal returns the first version that isn't invalid, or if all versions are
|
|
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
|
|
func (vl *VersionList) findGlobal() int {
|
|
for i, fv := range vl.RawVersions {
|
|
if !fv.IsInvalid() {
|
|
return i
|
|
}
|
|
}
|
|
if len(vl.RawVersions) == 0 {
|
|
return -1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// findDevices returns whether the device is in InvalidVersions or Versions and
|
|
// in InvalidDevices or Devices (true for invalid), the positions in the version
|
|
// and device slices and whether it has been found at all.
|
|
func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
|
|
for i, v := range vl.RawVersions {
|
|
if j := deviceIndex(v.Devices, device); j != -1 {
|
|
return false, i, j, true
|
|
}
|
|
if j := deviceIndex(v.InvalidDevices, device); j != -1 {
|
|
return true, i, j, true
|
|
}
|
|
}
|
|
return false, -1, -1, false
|
|
}
|
|
|
|
func (vl *VersionList) popVersion(version protocol.Vector) (FileVersion, bool) {
|
|
i := vl.versionIndex(version)
|
|
if i == -1 {
|
|
return FileVersion{}, false
|
|
}
|
|
fv := vl.RawVersions[i]
|
|
vl.popVersionAt(i)
|
|
return fv, true
|
|
}
|
|
|
|
func (vl *VersionList) versionIndex(version protocol.Vector) int {
|
|
for i, v := range vl.RawVersions {
|
|
if version.Equal(v.Version) {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (vl *VersionList) popVersionAt(i int) {
|
|
vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...)
|
|
}
|
|
|
|
// checkInsertAt determines if the given device and associated file should be
|
|
// inserted into the FileVersion at position i or into a new FileVersion at
|
|
// position i.
|
|
func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
|
|
ordering := vl.RawVersions[i].Version.Compare(file.FileVersion())
|
|
if ordering == protocol.Equal {
|
|
if !file.IsInvalid() {
|
|
vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device)
|
|
} else {
|
|
vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device)
|
|
}
|
|
return true, nil
|
|
}
|
|
existingDevice, _ := vl.RawVersions[i].FirstDevice()
|
|
insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if insert {
|
|
vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
|
|
return true, nil
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// shouldInsertBefore determines whether the file comes before an existing
|
|
// entry, given the version ordering (existing compared to new one), existing
|
|
// device and if the existing version is invalid.
|
|
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
|
|
switch ordering {
|
|
case protocol.Lesser:
|
|
// The version at this point in the list is lesser
|
|
// ("older") than us. We insert ourselves in front of it.
|
|
return true, nil
|
|
|
|
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
|
|
// The version in conflict with us.
|
|
// Check if we can shortcut due to one being invalid.
|
|
if existingInvalid != file.IsInvalid() {
|
|
return existingInvalid, nil
|
|
}
|
|
// We must pull the actual file metadata to determine who wins.
|
|
// If we win, we insert ourselves in front of the loser here.
|
|
// (The "Lesser" and "Greater" in the condition above is just
|
|
// based on the device IDs in the version vector, which is not
|
|
// the only thing we use to determine the winner.)
|
|
of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
// A surprise missing file entry here is counted as a win for us.
|
|
if !ok {
|
|
return true, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if protocol.WinsConflict(file, of) {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func deviceIndex(devices [][]byte, device []byte) int {
|
|
for i, dev := range devices {
|
|
if bytes.Equal(device, dev) {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func popDeviceAt(devices [][]byte, i int) [][]byte {
|
|
return append(devices[:i], devices[i+1:]...)
|
|
}
|
|
|
|
func popDevice(devices [][]byte, device []byte) ([][]byte, bool) {
|
|
i := deviceIndex(devices, device)
|
|
if i == -1 {
|
|
return devices, false
|
|
}
|
|
return popDeviceAt(devices, i), true
|
|
}
|
|
|
|
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion {
|
|
fv := FileVersion{
|
|
Version: version,
|
|
Deleted: deleted,
|
|
}
|
|
if invalid {
|
|
fv.InvalidDevices = [][]byte{device}
|
|
} else {
|
|
fv.Devices = [][]byte{device}
|
|
}
|
|
return fv
|
|
}
|
|
|
|
func (fv FileVersion) FirstDevice() ([]byte, bool) {
|
|
if len(fv.Devices) != 0 {
|
|
return fv.Devices[0], true
|
|
}
|
|
if len(fv.InvalidDevices) != 0 {
|
|
return fv.InvalidDevices[0], true
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
func (fv FileVersion) IsInvalid() bool {
|
|
return len(fv.Devices) == 0
|
|
}
|
|
|
|
func (fv FileVersion) deviceCount() int {
|
|
return len(fv.Devices) + len(fv.InvalidDevices)
|
|
}
|
|
|
|
type fileList []protocol.FileInfo
|
|
|
|
func (fl fileList) Len() int {
|
|
return len(fl)
|
|
}
|
|
|
|
func (fl fileList) Swap(a, b int) {
|
|
fl[a], fl[b] = fl[b], fl[a]
|
|
}
|
|
|
|
func (fl fileList) Less(a, b int) bool {
|
|
return fl[a].Name < fl[b].Name
|
|
}
|