Flush the batch when exceeding a certain size, instead of when reaching a number of batched operations. Move batch to lowlevel to be able to use it in NamespacedKV. Increase the leveldb memory buffer from 4 to 16 MiB.
157 lines
4.2 KiB
Go
157 lines
4.2 KiB
Go
// Copyright (C) 2018 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
package db
|
|
|
|
import (
|
|
"os"
|
|
"strings"
|
|
"sync/atomic"
|
|
|
|
"github.com/syndtr/goleveldb/leveldb"
|
|
"github.com/syndtr/goleveldb/leveldb/errors"
|
|
"github.com/syndtr/goleveldb/leveldb/opt"
|
|
"github.com/syndtr/goleveldb/leveldb/storage"
|
|
)
|
|
|
|
const (
|
|
dbMaxOpenFiles = 100
|
|
dbWriteBuffer = 16 << 20
|
|
dbFlushBatch = dbWriteBuffer / 4 // Some leeway for any leveldb in-memory optimizations
|
|
)
|
|
|
|
// Lowlevel is the lowest level database interface. It has a very simple
|
|
// purpose: hold the actual *leveldb.DB database, and the in-memory state
|
|
// that belong to that database. In the same way that a single on disk
|
|
// database can only be opened once, there should be only one Lowlevel for
|
|
// any given *leveldb.DB.
|
|
type Lowlevel struct {
|
|
committed int64 // atomic, must come first
|
|
*leveldb.DB
|
|
location string
|
|
folderIdx *smallIndex
|
|
deviceIdx *smallIndex
|
|
}
|
|
|
|
// Open attempts to open the database at the given location, and runs
|
|
// recovery on it if opening fails. Worst case, if recovery is not possible,
|
|
// the database is erased and created from scratch.
|
|
func Open(location string) (*Lowlevel, error) {
|
|
opts := &opt.Options{
|
|
OpenFilesCacheCapacity: dbMaxOpenFiles,
|
|
WriteBuffer: dbWriteBuffer,
|
|
}
|
|
return open(location, opts)
|
|
}
|
|
|
|
// OpenRO attempts to open the database at the given location, read only.
|
|
func OpenRO(location string) (*Lowlevel, error) {
|
|
opts := &opt.Options{
|
|
OpenFilesCacheCapacity: dbMaxOpenFiles,
|
|
ReadOnly: true,
|
|
}
|
|
return open(location, opts)
|
|
}
|
|
|
|
func open(location string, opts *opt.Options) (*Lowlevel, error) {
|
|
db, err := leveldb.OpenFile(location, opts)
|
|
if leveldbIsCorrupted(err) {
|
|
db, err = leveldb.RecoverFile(location, opts)
|
|
}
|
|
if leveldbIsCorrupted(err) {
|
|
// The database is corrupted, and we've tried to recover it but it
|
|
// didn't work. At this point there isn't much to do beyond dropping
|
|
// the database and reindexing...
|
|
l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
|
|
if err := os.RemoveAll(location); err != nil {
|
|
return nil, errorSuggestion{err, "failed to delete corrupted database"}
|
|
}
|
|
db, err = leveldb.OpenFile(location, opts)
|
|
}
|
|
if err != nil {
|
|
return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
|
|
}
|
|
return NewLowlevel(db, location), nil
|
|
}
|
|
|
|
// OpenMemory returns a new Lowlevel referencing an in-memory database.
|
|
func OpenMemory() *Lowlevel {
|
|
db, _ := leveldb.Open(storage.NewMemStorage(), nil)
|
|
return NewLowlevel(db, "<memory>")
|
|
}
|
|
|
|
// ListFolders returns the list of folders currently in the database
|
|
func (db *Lowlevel) ListFolders() []string {
|
|
return db.folderIdx.Values()
|
|
}
|
|
|
|
// Committed returns the number of items committed to the database since startup
|
|
func (db *Lowlevel) Committed() int64 {
|
|
return atomic.LoadInt64(&db.committed)
|
|
}
|
|
|
|
func (db *Lowlevel) Put(key, val []byte, wo *opt.WriteOptions) error {
|
|
atomic.AddInt64(&db.committed, 1)
|
|
return db.DB.Put(key, val, wo)
|
|
}
|
|
|
|
func (db *Lowlevel) Delete(key []byte, wo *opt.WriteOptions) error {
|
|
atomic.AddInt64(&db.committed, 1)
|
|
return db.DB.Delete(key, wo)
|
|
}
|
|
|
|
// NewLowlevel wraps the given *leveldb.DB into a *lowlevel
|
|
func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
|
|
return &Lowlevel{
|
|
DB: db,
|
|
location: location,
|
|
folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
|
|
deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
|
|
}
|
|
}
|
|
|
|
// A "better" version of leveldb's errors.IsCorrupted.
|
|
func leveldbIsCorrupted(err error) bool {
|
|
switch {
|
|
case err == nil:
|
|
return false
|
|
|
|
case errors.IsCorrupted(err):
|
|
return true
|
|
|
|
case strings.Contains(err.Error(), "corrupted"):
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
type batch struct {
|
|
*leveldb.Batch
|
|
db *Lowlevel
|
|
}
|
|
|
|
func (db *Lowlevel) newBatch() *batch {
|
|
return &batch{
|
|
Batch: new(leveldb.Batch),
|
|
db: db,
|
|
}
|
|
}
|
|
|
|
// checkFlush flushes and resets the batch if its size exceeds dbFlushBatch.
|
|
func (b *batch) checkFlush() {
|
|
if len(b.Dump()) > dbFlushBatch {
|
|
b.flush()
|
|
b.Reset()
|
|
}
|
|
}
|
|
|
|
func (b *batch) flush() {
|
|
if err := b.db.Write(b.Batch, nil); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|