2015-10-29 08:07:51 +01:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
|
|
|
//
|
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 07:52:18 +01:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2015-10-29 08:07:51 +01:00
|
|
|
|
|
|
|
package db
|
|
|
|
|
|
|
|
import (
|
2020-01-21 18:23:08 +01:00
|
|
|
"bytes"
|
2020-03-19 14:32:22 +01:00
|
|
|
"errors"
|
2020-01-21 18:23:08 +01:00
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
"github.com/syncthing/syncthing/lib/db/backend"
|
2015-10-29 08:07:51 +01:00
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
|
|
|
)
|
|
|
|
|
2020-03-19 14:32:22 +01:00
|
|
|
var errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry")
|
2020-03-16 08:45:45 +01:00
|
|
|
|
2015-10-29 08:07:51 +01:00
|
|
|
// A readOnlyTransaction represents a database snapshot.
|
|
|
|
type readOnlyTransaction struct {
|
2019-11-29 09:11:52 +01:00
|
|
|
backend.ReadTransaction
|
2019-02-15 00:15:13 +01:00
|
|
|
keyer keyer
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2019-12-02 08:18:04 +01:00
|
|
|
func (db *Lowlevel) newReadOnlyTransaction() (readOnlyTransaction, error) {
|
2019-11-29 09:11:52 +01:00
|
|
|
tran, err := db.NewReadTransaction()
|
|
|
|
if err != nil {
|
|
|
|
return readOnlyTransaction{}, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return readOnlyTransaction{
|
|
|
|
ReadTransaction: tran,
|
|
|
|
keyer: db.keyer,
|
|
|
|
}, nil
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
func (t readOnlyTransaction) close() {
|
|
|
|
t.Release()
|
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readOnlyTransaction) getFile(folder, device, file []byte) (protocol.FileInfo, bool, error) {
|
|
|
|
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, file)
|
|
|
|
if err != nil {
|
|
|
|
return protocol.FileInfo{}, false, err
|
|
|
|
}
|
|
|
|
return t.getFileByKey(key)
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool, error) {
|
|
|
|
f, ok, err := t.getFileTrunc(key, false)
|
|
|
|
if err != nil || !ok {
|
|
|
|
return protocol.FileInfo{}, false, err
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return f.(protocol.FileInfo), true, nil
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, bool, error) {
|
|
|
|
bs, err := t.Get(key)
|
|
|
|
if backend.IsNotFound(err) {
|
|
|
|
return nil, false, nil
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
|
|
|
if err != nil {
|
2019-11-29 09:11:52 +01:00
|
|
|
return nil, false, err
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
f, err := t.unmarshalTrunc(bs, trunc)
|
2020-02-11 15:37:22 +01:00
|
|
|
if backend.IsNotFound(err) {
|
|
|
|
return nil, false, nil
|
|
|
|
}
|
2019-01-18 11:34:18 +01:00
|
|
|
if err != nil {
|
2019-11-29 09:11:52 +01:00
|
|
|
return nil, false, err
|
2019-01-18 11:34:18 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return f, true, nil
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) {
|
|
|
|
if trunc {
|
|
|
|
var tf FileInfoTruncated
|
|
|
|
err := tf.Unmarshal(bs)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return tf, nil
|
|
|
|
}
|
|
|
|
|
2020-02-27 11:19:21 +01:00
|
|
|
var fi protocol.FileInfo
|
|
|
|
if err := fi.Unmarshal(bs); err != nil {
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
return nil, err
|
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
if err := t.fillFileInfo(&fi); err != nil {
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
return nil, err
|
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
return fi, nil
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
}
|
|
|
|
|
2020-02-27 11:19:21 +01:00
|
|
|
// fillFileInfo follows the (possible) indirection of blocks and fills it out.
|
|
|
|
func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
|
|
|
|
var key []byte
|
|
|
|
|
|
|
|
if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
|
|
|
|
// The blocks list is indirected and we need to load it.
|
|
|
|
key = t.keyer.GenerateBlockListKey(key, fi.BlocksHash)
|
|
|
|
bs, err := t.Get(key)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
var bl BlockList
|
|
|
|
if err := bl.Unmarshal(bs); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fi.Blocks = bl.Blocks
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) {
|
|
|
|
var err error
|
|
|
|
keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, false, err
|
|
|
|
}
|
2019-01-18 13:01:39 +01:00
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
bs, err := t.Get(keyBuf)
|
|
|
|
if backend.IsNotFound(err) {
|
|
|
|
return keyBuf, nil, false, nil
|
|
|
|
}
|
2019-01-18 13:01:39 +01:00
|
|
|
if err != nil {
|
2019-11-29 09:11:52 +01:00
|
|
|
return nil, nil, false, err
|
2019-01-18 13:01:39 +01:00
|
|
|
}
|
|
|
|
|
2020-03-16 10:09:27 +01:00
|
|
|
var vl VersionList
|
|
|
|
if err := vl.Unmarshal(bs); err != nil {
|
|
|
|
return nil, nil, false, err
|
2019-01-18 13:01:39 +01:00
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, false, err
|
2019-01-18 13:01:39 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
fi, ok, err := t.getFileTrunc(keyBuf, truncate)
|
|
|
|
if err != nil || !ok {
|
|
|
|
return keyBuf, nil, false, err
|
|
|
|
}
|
|
|
|
return keyBuf, fi, true, nil
|
2019-01-18 13:01:39 +01:00
|
|
|
}
|
|
|
|
|
2020-01-21 18:23:08 +01:00
|
|
|
func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
|
|
|
|
if len(prefix) > 0 {
|
|
|
|
unslashedPrefix := prefix
|
|
|
|
if bytes.HasSuffix(prefix, []byte{'/'}) {
|
|
|
|
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
|
|
|
|
} else {
|
|
|
|
prefix = append(prefix, '/')
|
|
|
|
}
|
|
|
|
|
|
|
|
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if f, ok, err := t.getFileTrunc(key, true); err != nil {
|
|
|
|
return err
|
|
|
|
} else if ok && !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewPrefixIterator(key)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
for dbi.Next() {
|
|
|
|
name := t.keyer.NameFromDeviceFileKey(dbi.Key())
|
|
|
|
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
f, err := t.unmarshalTrunc(dbi.Value(), truncate)
|
2020-01-21 18:23:08 +01:00
|
|
|
if err != nil {
|
|
|
|
l.Debugln("unmarshal error:", err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dbi.Error()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
|
|
|
|
first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewRangeIterator(first, last)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
for dbi.Next() {
|
|
|
|
f, ok, err := t.getFileByKey(dbi.Value())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
l.Debugln("missing file for sequence number", t.keyer.SequenceFromSequenceKey(dbi.Key()))
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if shouldDebug() {
|
|
|
|
if seq := t.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
|
|
|
|
l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
|
|
|
|
panic("sequence index corruption")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dbi.Error()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error {
|
|
|
|
if len(prefix) > 0 {
|
|
|
|
unslashedPrefix := prefix
|
|
|
|
if bytes.HasSuffix(prefix, []byte{'/'}) {
|
|
|
|
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
|
|
|
|
} else {
|
|
|
|
prefix = append(prefix, '/')
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil {
|
|
|
|
return err
|
|
|
|
} else if ok && !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, prefix)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewPrefixIterator(key)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
var dk []byte
|
|
|
|
for dbi.Next() {
|
|
|
|
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
|
|
|
|
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-03-16 10:09:27 +01:00
|
|
|
var vl VersionList
|
|
|
|
if err := vl.Unmarshal(dbi.Value()); err != nil {
|
|
|
|
return err
|
2020-01-21 18:23:08 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
f, ok, err := t.getFileTrunc(dk, truncate)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return dbi.Error()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.DeviceID, error) {
|
|
|
|
k, err := t.keyer.GenerateGlobalVersionKey(nil, folder, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
bs, err := t.Get(k)
|
|
|
|
if backend.IsNotFound(err) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2020-03-16 10:09:27 +01:00
|
|
|
var vl VersionList
|
|
|
|
if err := vl.Unmarshal(bs); err != nil {
|
|
|
|
return nil, err
|
2020-01-21 18:23:08 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
var devices []protocol.DeviceID
|
|
|
|
for _, v := range vl.Versions {
|
|
|
|
if !v.Version.Equal(vl.Versions[0].Version) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if v.Invalid {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
n := protocol.DeviceIDFromBytes(v.Device)
|
|
|
|
devices = append(devices, n)
|
|
|
|
}
|
|
|
|
|
|
|
|
return devices, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn Iterator) error {
|
|
|
|
if bytes.Equal(device, protocol.LocalDeviceID[:]) {
|
|
|
|
return t.withNeedLocal(folder, truncate, fn)
|
|
|
|
}
|
|
|
|
|
|
|
|
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewPrefixIterator(key.WithoutName())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
var dk []byte
|
|
|
|
devID := protocol.DeviceIDFromBytes(device)
|
|
|
|
for dbi.Next() {
|
2020-03-16 10:09:27 +01:00
|
|
|
var vl VersionList
|
|
|
|
if err := vl.Unmarshal(dbi.Value()); err != nil {
|
|
|
|
return err
|
2020-01-21 18:23:08 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
haveFV, have := vl.Get(device)
|
|
|
|
|
|
|
|
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
|
2020-03-16 08:45:45 +01:00
|
|
|
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
gf, ok, err := t.getFileTrunc(dk, truncate)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !ok {
|
2020-03-19 14:32:22 +01:00
|
|
|
return errEntryFromGlobalMissing
|
2020-03-16 08:45:45 +01:00
|
|
|
}
|
|
|
|
if !need(gf, have, haveFV.Version) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, vl.Versions[0].Version, vl.Versions[0].Device)
|
|
|
|
if !fn(gf) {
|
|
|
|
return dbi.Error()
|
2020-01-21 18:23:08 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return dbi.Error()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Iterator) error {
|
|
|
|
key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewPrefixIterator(key.WithoutName())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
var keyBuf []byte
|
|
|
|
var f FileIntf
|
|
|
|
var ok bool
|
|
|
|
for dbi.Next() {
|
|
|
|
keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !fn(f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dbi.Error()
|
|
|
|
}
|
|
|
|
|
2015-10-29 08:07:51 +01:00
|
|
|
// A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
|
|
|
|
// The batch will be committed on close() or by checkFlush() if it exceeds the
|
|
|
|
// batch size.
|
|
|
|
type readWriteTransaction struct {
|
2019-11-29 09:11:52 +01:00
|
|
|
backend.WriteTransaction
|
2015-10-29 08:07:51 +01:00
|
|
|
readOnlyTransaction
|
|
|
|
}
|
|
|
|
|
2019-12-02 08:18:04 +01:00
|
|
|
func (db *Lowlevel) newReadWriteTransaction() (readWriteTransaction, error) {
|
2019-11-29 09:11:52 +01:00
|
|
|
tran, err := db.NewWriteTransaction()
|
|
|
|
if err != nil {
|
|
|
|
return readWriteTransaction{}, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return readWriteTransaction{
|
|
|
|
WriteTransaction: tran,
|
|
|
|
readOnlyTransaction: readOnlyTransaction{
|
|
|
|
ReadTransaction: tran,
|
|
|
|
keyer: db.keyer,
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2020-02-12 11:59:55 +01:00
|
|
|
func (t readWriteTransaction) Commit() error {
|
2019-11-29 09:11:52 +01:00
|
|
|
t.readOnlyTransaction.close()
|
|
|
|
return t.WriteTransaction.Commit()
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
func (t readWriteTransaction) close() {
|
|
|
|
t.readOnlyTransaction.close()
|
2019-11-29 09:11:52 +01:00
|
|
|
t.WriteTransaction.Release()
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2020-03-20 12:07:14 +01:00
|
|
|
// putFile stores a file in the database, taking care of indirected fields.
|
|
|
|
// Set the truncated flag when putting a file that deliberatly can have an
|
|
|
|
// empty block list but a non-empty block list hash. This should normally be
|
|
|
|
// false.
|
|
|
|
func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo, truncated bool) error {
|
2020-02-27 11:19:21 +01:00
|
|
|
var bkey []byte
|
|
|
|
|
2020-03-20 12:07:14 +01:00
|
|
|
// Always set the blocks hash when there are blocks. Leave the blocks
|
|
|
|
// hash alone when there are no blocks and we might be putting a
|
|
|
|
// "truncated" FileInfo (no blocks, but the hash reference is live).
|
2020-02-27 11:19:21 +01:00
|
|
|
if len(fi.Blocks) > 0 {
|
|
|
|
fi.BlocksHash = protocol.BlocksHash(fi.Blocks)
|
2020-03-20 12:07:14 +01:00
|
|
|
} else if !truncated {
|
2020-02-27 11:19:21 +01:00
|
|
|
fi.BlocksHash = nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Indirect the blocks if the block list is large enough.
|
|
|
|
if len(fi.Blocks) > blocksIndirectionCutoff {
|
|
|
|
bkey = t.keyer.GenerateBlockListKey(bkey, fi.BlocksHash)
|
|
|
|
if _, err := t.Get(bkey); backend.IsNotFound(err) {
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
// Marshal the block list and save it
|
|
|
|
blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks})
|
2020-02-27 11:19:21 +01:00
|
|
|
if err := t.Put(bkey, blocksBs); err != nil {
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
fi.Blocks = nil
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
fiBs := mustMarshal(&fi)
|
2020-02-27 11:19:21 +01:00
|
|
|
return t.Put(fkey, fiBs)
|
lib/db: Deduplicate block lists in database (fixes #5898) (#6283)
* lib/db: Deduplicate block lists in database (fixes #5898)
This moves the block list in the database out from being just a field on
the FileInfo to being an object of its own. When putting a FileInfo we
marshal the block list separately and store it keyed by the sha256 of
the marshalled block list. When getting, if we are not doing a
"truncated" get, we do an extra read and unmarshal for the block list.
Old block lists are cleared out by a periodic GC sweep. The alternative
would be to use refcounting, but:
- There is a larger risk of getting that wrong and either dropping a
block list in error or keeping them around forever.
- It's tricky with our current database, as we don't have dirty reads.
This means that if we update two FileInfos with identical block lists in
the same transaction we can't just do read/modify/write for the ref
counters as we wouldn't see our own first update. See above about
tracking this and risks about getting it wrong.
GC uses a bloom filter for keys to avoid heavy RAM usage. GC can't run
concurrently with FileInfo updates so there is a new lock around those
operation at the lowlevel.
The end result is a much more compact database, especially for setups
with many peers where files get duplicated many times.
This is per-key-class stats for a large database I'm currently working
with, under the current schema:
```
0x00: 9138161 items, 870876 KB keys + 7397482 KB data, 95 B + 809 B avg, 1637651 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x08: 1349 items, 12 KB keys + 10 KB data, 9 B + 8 B avg, 17 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 7 B avg, 30 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
Total 10426475 items, 968490 KB keys + 9202925 KB data.
```
Note 7.4 GB of data in class 00, total size 9.2 GB. After running the
migration we get this instead:
```
0x00: 9138161 items, 870876 KB keys + 2611392 KB data, 95 B + 285 B avg, 4788 B max
0x01: 185656 items, 10388 KB keys + 1790909 KB data, 55 B + 9646 B avg, 924525 B max
0x02: 916890 items, 84795 KB keys + 3667 KB data, 92 B + 4 B avg, 192 B max
0x03: 384 items, 27 KB keys + 5 KB data, 72 B + 15 B avg, 87 B max
0x04: 1109 items, 17 KB keys + 17 KB data, 15 B + 15 B avg, 69 B max
0x06: 383 items, 3 KB keys + 0 KB data, 9 B + 2 B avg, 18 B max
0x07: 510 items, 4 KB keys + 12 KB data, 9 B + 24 B avg, 41 B max
0x09: 194 items, 0 KB keys + 123 KB data, 5 B + 634 B avg, 11484 B max
0x0a: 3 items, 0 KB keys + 0 KB data, 14 B + 17 B avg, 51 B max
0x0b: 181836 items, 2363 KB keys + 10694 KB data, 13 B + 58 B avg, 173 B max
0x0d: 44282 items, 1461 KB keys + 61081 KB data, 33 B + 1379 B avg, 1637399 B max
Total 10469408 items, 969939 KB keys + 4477905 KB data.
```
Class 00 is now down to 2.6 GB, with just 61 MB added in class 0d.
There will be some additional reads in some cases which theoretically
hurts performance, but this will be more than compensated for by smaller
writes and better compaction.
On my own home setup which just has three devices and a handful of
folders the difference is smaller in absolute numbers of course, but
still less than half the old size:
```
0x00: 297122 items, 20894 KB keys + 306860 KB data, 70 B + 1032 B avg, 103237 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
Total 1947412 items, 151268 KB keys + 337485 KB data.
```
to:
```
0x00: 297122 items, 20894 KB keys + 37038 KB data, 70 B + 124 B avg, 520 B max
0x01: 115299 items, 7738 KB keys + 17542 KB data, 67 B + 152 B avg, 419 B max
0x02: 1430537 items, 121223 KB keys + 5722 KB data, 84 B + 4 B avg, 253 B max
...
0x0d: 18041 items, 595 KB keys + 71964 KB data, 33 B + 3988 B avg, 101109 B max
Total 1965447 items, 151863 KB keys + 139628 KB data.
```
* wip
* wip
* wip
* wip
2020-01-24 08:35:44 +01:00
|
|
|
}
|
|
|
|
|
2015-10-29 08:07:51 +01:00
|
|
|
// updateGlobal adds this device+version to the version list for the given
|
|
|
|
// file. If the device is already present in the list, the version is updated.
|
|
|
|
// If the file does not have an entry in the global list, it is created.
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) ([]byte, bool, error) {
|
2018-06-24 09:50:18 +02:00
|
|
|
l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.IsInvalid())
|
2015-10-29 08:07:51 +01:00
|
|
|
|
2016-05-31 21:29:26 +02:00
|
|
|
var fl VersionList
|
2019-11-29 09:11:52 +01:00
|
|
|
svl, err := t.Get(gk)
|
|
|
|
if err == nil {
|
|
|
|
_ = fl.Unmarshal(svl) // Ignore error, continue with empty fl
|
|
|
|
} else if !backend.IsNotFound(err) {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
fl, removedFV, removedAt, insertedAt, err := fl.update(folder, device, file, t.readOnlyTransaction)
|
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2018-06-18 08:23:40 +02:00
|
|
|
name := []byte(file.Name)
|
|
|
|
|
2020-04-07 13:13:18 +02:00
|
|
|
var global FileIntf
|
2018-06-02 15:08:32 +02:00
|
|
|
if insertedAt == 0 {
|
|
|
|
// Inserted a new newest version
|
2018-10-30 05:40:51 +01:00
|
|
|
global = file
|
2018-06-02 15:08:32 +02:00
|
|
|
} else {
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name)
|
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
2019-01-20 08:47:20 +01:00
|
|
|
}
|
2020-04-07 13:13:18 +02:00
|
|
|
new, ok, err := t.getFileTrunc(keyBuf, true)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err != nil || !ok {
|
|
|
|
return keyBuf, false, err
|
|
|
|
}
|
|
|
|
global = new
|
2018-06-02 15:08:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Fixup the list of files we need.
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.updateLocalNeed(keyBuf, folder, name, fl, global)
|
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
2015-10-29 08:07:51 +01:00
|
|
|
|
2018-07-10 17:32:34 +02:00
|
|
|
if removedAt != 0 && insertedAt != 0 {
|
|
|
|
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
|
|
|
return keyBuf, true, nil
|
2018-07-10 17:32:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Remove the old global from the global size counter
|
|
|
|
var oldGlobalFV FileVersion
|
|
|
|
if removedAt == 0 {
|
|
|
|
oldGlobalFV = removedFV
|
|
|
|
} else if len(fl.Versions) > 1 {
|
|
|
|
// The previous newest version is now at index 1
|
|
|
|
oldGlobalFV = fl.Versions[1]
|
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name)
|
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
2020-04-07 13:13:18 +02:00
|
|
|
oldFile, ok, err := t.getFileTrunc(keyBuf, true)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
|
|
|
if ok {
|
2018-07-10 17:32:34 +02:00
|
|
|
// A failure to get the file here is surprising and our
|
|
|
|
// global size data will be incorrect until a restart...
|
2018-07-12 10:15:57 +02:00
|
|
|
meta.removeFile(protocol.GlobalDeviceID, oldFile)
|
2018-07-10 17:32:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add the new global to the global size counter
|
2018-10-30 05:40:51 +01:00
|
|
|
meta.addFile(protocol.GlobalDeviceID, global)
|
2018-07-10 17:32:34 +02:00
|
|
|
|
2018-06-02 15:08:32 +02:00
|
|
|
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
2015-10-29 08:07:51 +01:00
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
return keyBuf, true, nil
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2019-01-20 08:47:20 +01:00
|
|
|
// updateLocalNeed checks whether the given file is still needed on the local
|
2018-10-30 05:40:51 +01:00
|
|
|
// device according to the version list and global FileInfo given and updates
|
|
|
|
// the db accordingly.
|
2020-04-07 13:13:18 +02:00
|
|
|
func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, fl VersionList, global FileIntf) ([]byte, error) {
|
2019-11-29 09:11:52 +01:00
|
|
|
var err error
|
|
|
|
keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, name)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
_, err = t.Get(keyBuf)
|
|
|
|
if err != nil && !backend.IsNotFound(err) {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
hasNeeded := err == nil
|
2018-10-30 05:40:51 +01:00
|
|
|
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(global, haveLocalFV, localFV.Version) {
|
|
|
|
if !hasNeeded {
|
|
|
|
l.Debugf("local need insert; folder=%q, name=%q", folder, name)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Put(keyBuf, nil); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-10-30 05:40:51 +01:00
|
|
|
}
|
|
|
|
} else if hasNeeded {
|
|
|
|
l.Debugf("local need delete; folder=%q, name=%q", folder, name)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Delete(keyBuf); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-10-30 05:40:51 +01:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return keyBuf, nil
|
2018-10-30 05:40:51 +01:00
|
|
|
}
|
|
|
|
|
2018-06-02 15:08:32 +02:00
|
|
|
func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool {
|
|
|
|
// We never need an invalid file.
|
|
|
|
if global.IsInvalid() {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// We don't need a deleted file if we don't have it.
|
|
|
|
if global.IsDeleted() && !haveLocal {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// We don't need the global file if we already have the same version.
|
2018-10-10 12:43:07 +02:00
|
|
|
if haveLocal && localVersion.GreaterEqual(global.FileVersion()) {
|
2018-06-02 15:08:32 +02:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2015-10-29 08:07:51 +01:00
|
|
|
// removeFromGlobal removes the device from the global version list for the
|
|
|
|
// given file. If the version list is empty after this, the file entry is
|
|
|
|
// removed entirely.
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) {
|
2015-10-29 08:07:51 +01:00
|
|
|
l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
svl, err := t.Get(gk)
|
|
|
|
if backend.IsNotFound(err) {
|
2015-10-29 08:07:51 +01:00
|
|
|
// We might be called to "remove" a global version that doesn't exist
|
|
|
|
// if the first update for the file is already marked invalid.
|
2019-11-29 09:11:52 +01:00
|
|
|
return keyBuf, nil
|
|
|
|
} else if err != nil {
|
|
|
|
return nil, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2016-05-31 21:29:26 +02:00
|
|
|
var fl VersionList
|
2016-07-04 12:40:29 +02:00
|
|
|
err = fl.Unmarshal(svl)
|
2015-10-29 08:07:51 +01:00
|
|
|
if err != nil {
|
2019-11-29 09:11:52 +01:00
|
|
|
return nil, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2018-10-30 05:40:51 +01:00
|
|
|
fl, _, removedAt := fl.pop(device)
|
|
|
|
if removedAt == -1 {
|
|
|
|
// There is no version for the given device
|
2019-11-29 09:11:52 +01:00
|
|
|
return keyBuf, nil
|
2018-10-30 05:40:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if removedAt == 0 {
|
|
|
|
// A failure to get the file here is surprising and our
|
|
|
|
// global size data will be incorrect until a restart...
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, device, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-04-07 13:13:18 +02:00
|
|
|
if f, ok, err := t.getFileTrunc(keyBuf, true); err != nil {
|
2020-03-19 14:32:22 +01:00
|
|
|
return nil, err
|
2019-11-29 09:11:52 +01:00
|
|
|
} else if ok {
|
2018-10-30 05:40:51 +01:00
|
|
|
meta.removeFile(protocol.GlobalDeviceID, f)
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-04 12:40:29 +02:00
|
|
|
if len(fl.Versions) == 0 {
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateNeedFileKey(keyBuf, folder, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err := t.Delete(keyBuf); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err := t.Delete(gk); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return keyBuf, nil
|
2017-11-11 20:18:17 +01:00
|
|
|
}
|
2018-10-30 05:40:51 +01:00
|
|
|
|
|
|
|
if removedAt == 0 {
|
2019-11-29 09:11:52 +01:00
|
|
|
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-04-07 13:13:18 +02:00
|
|
|
global, ok, err := t.getFileTrunc(keyBuf, true)
|
2020-03-19 14:32:22 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if !ok {
|
|
|
|
return nil, errEntryFromGlobalMissing
|
2019-11-29 09:11:52 +01:00
|
|
|
}
|
|
|
|
keyBuf, err = t.updateLocalNeed(keyBuf, folder, file, fl, global)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
2018-10-30 05:40:51 +01:00
|
|
|
meta.addFile(protocol.GlobalDeviceID, global)
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
2018-10-30 05:40:51 +01:00
|
|
|
|
|
|
|
l.Debugf("new global after remove: %v", fl)
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2019-01-20 08:47:20 +01:00
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
return keyBuf, nil
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
|
|
|
|
2019-11-29 09:11:52 +01:00
|
|
|
func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) error {
|
|
|
|
dbi, err := t.NewPrefixIterator(prefix)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
2018-06-02 15:08:32 +02:00
|
|
|
for dbi.Next() {
|
2019-11-29 09:11:52 +01:00
|
|
|
if err := t.Delete(dbi.Key()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-06-02 15:08:32 +02:00
|
|
|
}
|
2019-11-29 09:11:52 +01:00
|
|
|
return dbi.Error()
|
2015-10-29 08:07:51 +01:00
|
|
|
}
|
2016-07-04 12:40:29 +02:00
|
|
|
|
2020-01-21 18:23:08 +01:00
|
|
|
func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
|
|
|
|
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dbi.Release()
|
|
|
|
|
|
|
|
var gk, keyBuf []byte
|
|
|
|
for dbi.Next() {
|
|
|
|
device, ok := t.keyer.DeviceFromDeviceFileKey(dbi.Key())
|
|
|
|
if !ok {
|
|
|
|
// Not having the device in the index is bad. Clear it.
|
|
|
|
if err := t.Delete(dbi.Key()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
|
|
|
|
intf, err := t.unmarshalTrunc(dbi.Value(), true)
|
2020-01-21 18:23:08 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-02-27 11:19:21 +01:00
|
|
|
f := intf.(FileInfoTruncated)
|
2020-01-21 18:23:08 +01:00
|
|
|
|
|
|
|
switch f.Name {
|
|
|
|
case "", ".", "..", "/": // A few obviously invalid filenames
|
|
|
|
l.Infof("Dropping invalid filename %q from database", f.Name)
|
|
|
|
name := []byte(f.Name)
|
|
|
|
gk, err = t.keyer.GenerateGlobalVersionKey(gk, folder, name)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := t.Delete(dbi.Key()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if !fn(device, f) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2020-02-14 08:11:24 +01:00
|
|
|
return dbi.Error()
|
2020-01-21 18:23:08 +01:00
|
|
|
}
|
|
|
|
|
2016-07-04 12:40:29 +02:00
|
|
|
type marshaller interface {
|
|
|
|
Marshal() ([]byte, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
func mustMarshal(f marshaller) []byte {
|
|
|
|
bs, err := f.Marshal()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
return bs
|
|
|
|
}
|