From 94beed5c109b60f1bbcbc8cee357cb70b1c81a2d Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 29 May 2020 12:43:02 +0100 Subject: [PATCH] lib/db: Add Badger backend (fixes #5910) (#6250) --- go.mod | 2 +- go.sum | 40 +++ lib/db/backend/backend.go | 86 ++++++ lib/db/backend/badger_backend.go | 443 +++++++++++++++++++++++++++++++ lib/db/backend/badger_test.go | 38 +++ lib/db/backend/leveldb_open.go | 10 +- lib/db/lowlevel.go | 5 +- lib/db/schemaupdater.go | 4 + lib/db/transactions.go | 4 +- lib/locations/locations.go | 21 +- 10 files changed, 644 insertions(+), 9 deletions(-) create mode 100644 lib/db/backend/badger_backend.go create mode 100644 lib/db/backend/badger_test.go diff --git a/go.mod b/go.mod index ea68889ba..f9360ae8f 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 // indirect github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 github.com/d4l3k/messagediff v1.2.1 + github.com/dgraph-io/badger/v2 v2.0.3 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/getsentry/raven-go v0.2.0 github.com/go-ldap/ldap/v3 v3.1.10 @@ -45,7 +46,6 @@ require ( golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index aa1aceb3e..67f9f2e4b 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,9 @@ github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg= github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -25,6 +28,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -42,6 +46,8 @@ github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZ github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE= github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 h1:8k9FLYBLKT+9v2HQJ/a95ZemmTx+/ltJcAiRhVushG8= github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= @@ -49,7 +55,12 @@ github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wX github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 h1:Wg96Dh0MLTanEaPO0OkGtUIaa2jOnShAIOVUIzRHUxo= github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= @@ -57,6 +68,14 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= +github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= +github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= @@ -132,8 +151,10 @@ github.com/greatroar/blobloom v0.2.1 h1:Ie7+kTQFhcvfFHhzOYSJA2XE5sp8c9iB5iapu2tI github.com/greatroar/blobloom v0.2.1/go.mod h1:we9vO6GNYMmsNvCWINtZnQbcGEHUT6hGBAznNHd6RlE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackpal/gateway v1.0.6 h1:/MJORKvJEwNVldtGVJC2p2cwCnsSoLn3hl3zxmZT7tk= github.com/jackpal/gateway v1.0.6/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -163,6 +184,7 @@ github.com/lucas-clemente/quic-go v0.15.6/go.mod h1:Myi1OyS0FOjL3not4BxT7KN29bRk github.com/lucas-clemente/quic-go v0.15.7 h1:Pu7To5/G9JoP1mwlrcIvfV8ByPBlCzif3MCl8+1W83I= github.com/lucas-clemente/quic-go v0.15.7/go.mod h1:Myi1OyS0FOjL3not4BxT7KN29bRkcMUV5JVVFLKtDp8= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ= @@ -183,6 +205,8 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -204,6 +228,7 @@ github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9Pvr github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls= github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -276,6 +301,14 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -293,6 +326,7 @@ github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thejerf/suture v3.0.2+incompatible h1:GtMydYcnK4zBJ0KL6Lx9vLzl6Oozb65wh252FTBxrvM= github.com/thejerf/suture v3.0.2+incompatible/go.mod h1:ibKwrVj+Uzf3XZdAiNWUouPaAbSoemxOHLmJmwheEMc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= @@ -301,11 +335,13 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI= github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -330,6 +366,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -349,6 +386,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= @@ -356,6 +394,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= @@ -411,6 +450,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/lib/db/backend/backend.go b/lib/db/backend/backend.go index 3320da03c..04499aad0 100644 --- a/lib/db/backend/backend.go +++ b/lib/db/backend/backend.go @@ -7,7 +7,12 @@ package backend import ( + "os" + "strings" "sync" + "time" + + "github.com/syncthing/syncthing/lib/locations" ) // The Reader interface specifies the read-only operations available on the @@ -118,10 +123,24 @@ const ( ) func Open(path string, tuning Tuning) (Backend, error) { + if os.Getenv("USE_BADGER") != "" { + l.Warnln("Using experimental badger db") + if err := maybeCopyDatabase(path, strings.Replace(path, locations.BadgerDir, locations.LevelDBDir, 1), OpenBadger, OpenLevelDBRO); err != nil { + return nil, err + } + return OpenBadger(path) + } + + if err := maybeCopyDatabase(path, strings.Replace(path, locations.LevelDBDir, locations.BadgerDir, 1), OpenLevelDBAuto, OpenBadger); err != nil { + return nil, err + } return OpenLevelDB(path, tuning) } func OpenMemory() Backend { + if os.Getenv("USE_BADGER") != "" { + return OpenBadgerMemory() + } return OpenLevelDBMemory() } @@ -202,3 +221,70 @@ func (cg *closeWaitGroup) CloseWait() { cg.closeMut.Unlock() cg.WaitGroup.Wait() } + +type opener func(path string) (Backend, error) + +// maybeCopyDatabase copies the database if the destination doesn't exist +// but the source does. +func maybeCopyDatabase(toPath, fromPath string, toOpen, fromOpen opener) error { + if _, err := os.Lstat(toPath); !os.IsNotExist(err) { + // Destination database exists (or is otherwise unavailable), do not + // attempt to overwrite it. + return nil + } + + if _, err := os.Lstat(fromPath); err != nil { + // Source database is not available, so nothing to copy + return nil + } + + fromDB, err := fromOpen(fromPath) + if err != nil { + return err + } + defer fromDB.Close() + + toDB, err := toOpen(toPath) + if err != nil { + // That's odd, but it will be handled & reported in the usual path + // so we can ignore it here. + return err + } + defer toDB.Close() + + l.Infoln("Copying database for format conversion...") + if err := copyBackend(toDB, fromDB); err != nil { + return err + } + + // Move the old database out of the way to mark it as migrated. + fromDB.Close() + _ = os.Rename(fromPath, fromPath+".migrated."+time.Now().Format("20060102150405")) + return nil +} + +func copyBackend(to, from Backend) error { + srcIt, err := from.NewPrefixIterator(nil) + if err != nil { + return err + } + defer srcIt.Release() + + dstTx, err := to.NewWriteTransaction() + if err != nil { + return err + } + defer dstTx.Release() + + for srcIt.Next() { + if err := dstTx.Put(srcIt.Key(), srcIt.Value()); err != nil { + return err + } + } + if srcIt.Error() != nil { + return err + } + srcIt.Release() + + return dstTx.Commit() +} diff --git a/lib/db/backend/badger_backend.go b/lib/db/backend/badger_backend.go new file mode 100644 index 000000000..1b5333731 --- /dev/null +++ b/lib/db/backend/badger_backend.go @@ -0,0 +1,443 @@ +// Copyright (C) 2019 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package backend + +import ( + "bytes" + "errors" + "time" + + badger "github.com/dgraph-io/badger/v2" +) + +const ( + checkpointFlushMinSize = 128 << KiB + maxCacheSize = 64 << MiB +) + +func OpenBadger(path string) (Backend, error) { + opts := badger.DefaultOptions(path) + opts = opts.WithMaxCacheSize(maxCacheSize).WithCompactL0OnClose(false) + opts.Logger = nil + return openBadger(opts) +} + +func OpenBadgerMemory() Backend { + opts := badger.DefaultOptions("").WithInMemory(true) + opts.Logger = nil + backend, err := openBadger(opts) + if err != nil { + // Opening in-memory should never be able to fail, and is anyway + // used just by tests. + panic(err) + } + return backend +} + +func openBadger(opts badger.Options) (Backend, error) { + // XXX: We should find good values for memory utilization in the "small" + // and "large" cases we support for LevelDB. Some notes here: + // https://github.com/dgraph-io/badger/tree/v2.0.3#memory-usage + bdb, err := badger.Open(opts) + if err != nil { + return nil, wrapBadgerErr(err) + } + return &badgerBackend{ + bdb: bdb, + closeWG: &closeWaitGroup{}, + }, nil +} + +// badgerBackend implements Backend on top of a badger +type badgerBackend struct { + bdb *badger.DB + closeWG *closeWaitGroup +} + +func (b *badgerBackend) NewReadTransaction() (ReadTransaction, error) { + rel, err := newReleaser(b.closeWG) + if err != nil { + return nil, err + } + return badgerSnapshot{ + txn: b.bdb.NewTransaction(false), + rel: rel, + }, nil +} + +func (b *badgerBackend) NewWriteTransaction() (WriteTransaction, error) { + rel1, err := newReleaser(b.closeWG) + if err != nil { + return nil, err + } + rel2, err := newReleaser(b.closeWG) + if err != nil { + rel1.Release() + return nil, err + } + + // We use two transactions here to preserve the property that our + // leveldb wrapper has, that writes in a transaction are completely + // invisible until it's committed, even inside that same transaction. + rtxn := b.bdb.NewTransaction(false) + wtxn := b.bdb.NewTransaction(true) + return &badgerTransaction{ + badgerSnapshot: badgerSnapshot{ + txn: rtxn, + rel: rel1, + }, + txn: wtxn, + bdb: b.bdb, + rel: rel2, + }, nil +} + +func (b *badgerBackend) Close() error { + b.closeWG.CloseWait() + return wrapBadgerErr(b.bdb.Close()) +} + +func (b *badgerBackend) Get(key []byte) ([]byte, error) { + if err := b.closeWG.Add(1); err != nil { + return nil, err + } + defer b.closeWG.Done() + + txn := b.bdb.NewTransaction(false) + defer txn.Discard() + item, err := txn.Get(key) + if err != nil { + return nil, wrapBadgerErr(err) + } + val, err := item.ValueCopy(nil) + if err != nil { + return nil, wrapBadgerErr(err) + } + return val, nil +} + +func (b *badgerBackend) NewPrefixIterator(prefix []byte) (Iterator, error) { + if err := b.closeWG.Add(1); err != nil { + return nil, err + } + + txn := b.bdb.NewTransaction(false) + it := badgerPrefixIterator(txn, prefix) + it.releaseFn = func() { + defer b.closeWG.Done() + txn.Discard() + } + return it, nil +} + +func (b *badgerBackend) NewRangeIterator(first, last []byte) (Iterator, error) { + if err := b.closeWG.Add(1); err != nil { + return nil, err + } + + txn := b.bdb.NewTransaction(false) + it := badgerRangeIterator(txn, first, last) + it.releaseFn = func() { + defer b.closeWG.Done() + txn.Discard() + } + return it, nil +} + +func (b *badgerBackend) Put(key, val []byte) error { + if err := b.closeWG.Add(1); err != nil { + return err + } + defer b.closeWG.Done() + + txn := b.bdb.NewTransaction(true) + if err := txn.Set(key, val); err != nil { + txn.Discard() + return wrapBadgerErr(err) + } + return wrapBadgerErr(txn.Commit()) +} + +func (b *badgerBackend) Delete(key []byte) error { + if err := b.closeWG.Add(1); err != nil { + return err + } + defer b.closeWG.Done() + + txn := b.bdb.NewTransaction(true) + if err := txn.Delete(key); err != nil { + txn.Discard() + return wrapBadgerErr(err) + } + return wrapBadgerErr(txn.Commit()) +} + +func (b *badgerBackend) Compact() error { + if err := b.closeWG.Add(1); err != nil { + return err + } + defer b.closeWG.Done() + + // This weird looking loop is as recommended in the README + // (https://github.com/dgraph-io/badger/tree/v2.0.3#garbage-collection). + // Basically, the RunValueLogGC will pick some promising thing to + // garbage collect at random and return nil if it improved the + // situation, then return ErrNoRewrite when there is nothing more to GC. + // The 0.5 is the discard ratio, for which the method docs say they + // "recommend setting discardRatio to 0.5, thus indicating that a file + // be rewritten if half the space can be discarded". + var err error + t0 := time.Now() + for err == nil { + if time.Since(t0) > time.Hour { + l.Warnln("Database compaction is taking a long time, performance may be impacted. Consider investigating and/or opening an issue if this warning repeats.") + t0 = time.Now() + } + err = b.bdb.RunValueLogGC(0.5) + } + + if errors.Is(err, badger.ErrNoRewrite) { + // GC did nothing, because nothing needed to be done + return nil + } + if errors.Is(err, badger.ErrRejected) { + // GC was already running (could possibly happen), or the database + // is closed (can't happen). + return nil + } + if errors.Is(err, badger.ErrGCInMemoryMode) { + // GC in in-memory mode, which is fine. + return nil + } + return err +} + +// badgerSnapshot implements backend.ReadTransaction +type badgerSnapshot struct { + txn *badger.Txn + rel *releaser +} + +func (l badgerSnapshot) Get(key []byte) ([]byte, error) { + item, err := l.txn.Get(key) + if err != nil { + return nil, wrapBadgerErr(err) + } + val, err := item.ValueCopy(nil) + if err != nil { + return nil, wrapBadgerErr(err) + } + return val, nil +} + +func (l badgerSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) { + return badgerPrefixIterator(l.txn, prefix), nil +} + +func (l badgerSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) { + return badgerRangeIterator(l.txn, first, last), nil +} + +func (l badgerSnapshot) Release() { + defer l.rel.Release() + l.txn.Discard() +} + +type badgerTransaction struct { + badgerSnapshot + txn *badger.Txn + bdb *badger.DB + rel *releaser + size int +} + +func (t *badgerTransaction) Delete(key []byte) error { + t.size += len(key) + kc := make([]byte, len(key)) + copy(kc, key) + return t.transactionRetried(func(txn *badger.Txn) error { + return txn.Delete(kc) + }) +} + +func (t *badgerTransaction) Put(key, val []byte) error { + t.size += len(key) + len(val) + kc := make([]byte, len(key)) + copy(kc, key) + vc := make([]byte, len(val)) + copy(vc, val) + return t.transactionRetried(func(txn *badger.Txn) error { + return txn.Set(kc, vc) + }) +} + +// transactionRetried performs the given operation in the current +// transaction, with commit and retry if Badger says the transaction has +// grown too large. +func (t *badgerTransaction) transactionRetried(fn func(*badger.Txn) error) error { + if err := fn(t.txn); err == badger.ErrTxnTooBig { + if err := t.txn.Commit(); err != nil { + return wrapBadgerErr(err) + } + t.size = 0 + t.txn = t.bdb.NewTransaction(true) + return wrapBadgerErr(fn(t.txn)) + } else if err != nil { + return wrapBadgerErr(err) + } + return nil +} + +func (t *badgerTransaction) Commit() error { + defer t.rel.Release() + defer t.badgerSnapshot.Release() + return wrapBadgerErr(t.txn.Commit()) +} + +func (t *badgerTransaction) Checkpoint(preFlush ...func() error) error { + if t.size < checkpointFlushMinSize { + return nil + } + for _, hook := range preFlush { + if err := hook(); err != nil { + return err + } + } + err := t.txn.Commit() + if err == nil { + t.size = 0 + t.txn = t.bdb.NewTransaction(true) + } + return wrapBadgerErr(err) +} + +func (t *badgerTransaction) Release() { + defer t.rel.Release() + defer t.badgerSnapshot.Release() + t.txn.Discard() +} + +type badgerIterator struct { + it *badger.Iterator + prefix []byte + first []byte + last []byte + releaseFn func() + didSeek bool + err error +} + +func (i *badgerIterator) Next() bool { + if i.err != nil { + return false + } + for { + if !i.didSeek { + if i.first != nil { + // Range iterator + i.it.Seek(i.first) + } else { + // Prefix iterator + i.it.Seek(i.prefix) + } + i.didSeek = true + } else { + i.it.Next() + } + + if !i.it.ValidForPrefix(i.prefix) { + // Done + return false + } + if i.first == nil && i.last == nil { + // No range checks required + return true + } + + key := i.it.Item().Key() + if bytes.Compare(key, i.last) > 0 { + // Key is after range last + return false + } + return true + } +} + +func (i *badgerIterator) Key() []byte { + if i.err != nil { + return nil + } + return i.it.Item().Key() +} + +func (i *badgerIterator) Value() []byte { + if i.err != nil { + return nil + } + val, err := i.it.Item().ValueCopy(nil) + if err != nil { + i.err = err + } + return val +} + +func (i *badgerIterator) Error() error { + return wrapBadgerErr(i.err) +} + +func (i *badgerIterator) Release() { + i.it.Close() + if i.releaseFn != nil { + i.releaseFn() + } +} + +// wrapBadgerErr wraps errors so that the backend package can recognize them +func wrapBadgerErr(err error) error { + if err == nil { + return nil + } + if err == badger.ErrDiscardedTxn { + return errClosed{} + } + if err == badger.ErrKeyNotFound { + return errNotFound{} + } + return err +} + +func badgerPrefixIterator(txn *badger.Txn, prefix []byte) *badgerIterator { + it := iteratorForPrefix(txn, prefix) + return &badgerIterator{it: it, prefix: prefix} +} + +func badgerRangeIterator(txn *badger.Txn, first, last []byte) *badgerIterator { + prefix := commonPrefix(first, last) + it := iteratorForPrefix(txn, prefix) + return &badgerIterator{it: it, prefix: prefix, first: first, last: last} +} + +func iteratorForPrefix(txn *badger.Txn, prefix []byte) *badger.Iterator { + opts := badger.DefaultIteratorOptions + opts.Prefix = prefix + return txn.NewIterator(opts) +} + +func commonPrefix(a, b []byte) []byte { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + prefix := make([]byte, 0, minLen) + for i := 0; i < minLen; i++ { + if a[i] != b[i] { + break + } + prefix = append(prefix, a[i]) + } + return prefix +} diff --git a/lib/db/backend/badger_test.go b/lib/db/backend/badger_test.go new file mode 100644 index 000000000..8a4ef9daa --- /dev/null +++ b/lib/db/backend/badger_test.go @@ -0,0 +1,38 @@ +// Copyright (C) 2019 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package backend + +import "testing" + +func TestCommonPrefix(t *testing.T) { + cases := []struct { + a string + b string + common string + }{ + {"", "", ""}, + {"a", "b", ""}, + {"aa", "ab", "a"}, + {"aa", "a", "a"}, + {"a", "aa", "a"}, + {"aabab", "ab", "a"}, + {"ab", "aabab", "a"}, + {"abac", "ababab", "aba"}, + {"ababab", "abac", "aba"}, + } + + for _, tc := range cases { + pref := string(commonPrefix([]byte(tc.a), []byte(tc.b))) + if pref != tc.common { + t.Errorf("commonPrefix(%q, %q) => %q, expected %q", tc.a, tc.b, pref, tc.common) + } + } +} + +func TestBadgerBackendBehavior(t *testing.T) { + testBackendBehavior(t, OpenBadgerMemory) +} diff --git a/lib/db/backend/leveldb_open.go b/lib/db/backend/leveldb_open.go index 8e981b37b..cedd909f2 100644 --- a/lib/db/backend/leveldb_open.go +++ b/lib/db/backend/leveldb_open.go @@ -33,7 +33,7 @@ const ( MiB = 20 ) -// Open attempts to open the database at the given location, and runs +// OpenLevelDB attempts to open the database at the given location, and runs // recovery on it if opening fails. Worst case, if recovery is not possible, // the database is erased and created from scratch. func OpenLevelDB(location string, tuning Tuning) (Backend, error) { @@ -45,7 +45,13 @@ func OpenLevelDB(location string, tuning Tuning) (Backend, error) { return newLeveldbBackend(ldb), nil } -// OpenRO attempts to open the database at the given location, read only. +// OpenLevelDBAuto is OpenLevelDB with TuningAuto tuning. +func OpenLevelDBAuto(location string) (Backend, error) { + return OpenLevelDB(location, TuningAuto) +} + +// OpenLevelDBRO attempts to open the database at the given location, read +// only. func OpenLevelDBRO(location string) (Backend, error) { opts := &opt.Options{ OpenFilesCacheCapacity: dbMaxOpenFiles, diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index ff4062fe9..4a35e0779 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -382,6 +382,8 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke if err != nil { return err } + defer dbi.Release() + var gk, keyBuf []byte for dbi.Next() { name := db.keyer.NameFromDeviceFileKey(dbi.Key()) @@ -400,10 +402,10 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke return err } } + dbi.Release() if err := dbi.Error(); err != nil { return err } - dbi.Release() if bytes.Equal(device, protocol.LocalDeviceID[:]) { key, err := db.keyer.GenerateBlockMapKey(nil, folder, nil, nil) @@ -491,6 +493,7 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error { } } } + dbi.Release() if err := dbi.Error(); err != nil { return err } diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 98f180f05..16ac28dcc 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -212,6 +212,10 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { return err } } + dbi.Release() + if err != dbi.Error() { + return err + } for folder := range changedFolders { if err := db.dropFolderMeta([]byte(folder)); err != nil { diff --git a/lib/db/transactions.go b/lib/db/transactions.go index bd7c9fc78..4660247a3 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -509,7 +509,9 @@ func (db *Lowlevel) newReadWriteTransaction() (readWriteTransaction, error) { } func (t readWriteTransaction) Commit() error { - t.readOnlyTransaction.close() + // The readOnlyTransaction must close after commit, because they may be + // backed by the same actual lower level transaction. + defer t.readOnlyTransaction.close() return t.WriteTransaction.Commit() } diff --git a/lib/locations/locations.go b/lib/locations/locations.go index 31b1ddfca..702683f26 100644 --- a/lib/locations/locations.go +++ b/lib/locations/locations.go @@ -44,12 +44,20 @@ const ( DataBaseDir BaseDirEnum = "data" // User's home directory, *not* -home flag UserHomeBaseDir BaseDirEnum = "userHome" + + LevelDBDir = "index-v0.14.0.db" + BadgerDir = "indexdb.badger" ) // Platform dependent directories var baseDirs = make(map[BaseDirEnum]string, 3) func init() { + if os.Getenv("USE_BADGER") != "" { + // XXX: Replace the leveldb name with the badger name. + locationTemplates[Database] = strings.Replace(locationTemplates[Database], LevelDBDir, BadgerDir, 1) + } + userHome := userHomeDir() config := defaultConfigDir(userHome) baseDirs[UserHomeBaseDir] = userHome @@ -80,8 +88,6 @@ func GetBaseDir(baseDir BaseDirEnum) string { return baseDirs[baseDir] } -var databaseDirname = "index-v0.14.0.db" - // Use the variables from baseDirs here var locationTemplates = map[LocationEnum]string{ ConfigFile: "${config}/config.xml", @@ -89,7 +95,7 @@ var locationTemplates = map[LocationEnum]string{ KeyFile: "${config}/key.pem", HTTPSCertFile: "${config}/https-cert.pem", HTTPSKeyFile: "${config}/https-key.pem", - Database: "${data}/" + databaseDirname, + Database: "${data}/" + LevelDBDir, LogFile: "${data}/syncthing.log", // -logfile on Windows CsrfTokens: "${data}/csrftokens.txt", PanicLog: "${data}/panic-${timestamp}.log", @@ -150,7 +156,14 @@ func defaultDataDir(userHome, config string) string { default: // If a database exists at the "normal" location, use that anyway. - if _, err := os.Lstat(filepath.Join(config, databaseDirname)); err == nil { + // We look for both LevelDB and Badger variants here regardless of + // what we're currently configured to use, because we might be + // starting up in Badger mode with only a LevelDB database present + // (will be converted), or vice versa. + if _, err := os.Lstat(filepath.Join(config, LevelDBDir)); err == nil { + return config + } + if _, err := os.Lstat(filepath.Join(config, BadgerDir)); err == nil { return config } // Always use this env var, as it's explicitly set by the user