This is a new revision of the discovery server. Relevant changes and non-changes: - Protocol towards clients is unchanged. - Recommended large scale design is still to be deployed nehind nginx (I tested, and it's still a lot faster at terminating TLS). - Database backend is leveldb again, only. It scales enough, is easy to setup, and we don't need any backend to take care of. - Server supports replication. This is a simple TCP channel - protect it with a firewall when deploying over the internet. (We deploy this within the same datacenter, and with firewall.) Any incoming client announces are sent over the replication channel(s) to other peer discosrvs. Incoming replication changes are applied to the database as if they came from clients, but without the TLS/certificate overhead. - Metrics are exposed using the prometheus library, when enabled. - The database values and replication protocol is protobuf, because JSON was quite CPU intensive when I tried that and benchmarked it. - The "Retry-After" value for failed lookups gets slowly increased from a default of 120 seconds, by 5 seconds for each failed lookup, independently by each discosrv. This lowers the query load over time for clients that are never seen. The Retry-After maxes out at 3600 after a couple of weeks of this increase. The number of failed lookups is stored in the database, now and then (avoiding making each lookup a database put). All in all this means clients can be pointed towards a cluster using just multiple A / AAAA records to gain both load sharing and redundancy (if one is down, clients will talk to the remaining ones). GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4648
243 lines
6.3 KiB
Go
243 lines
6.3 KiB
Go
package toml
|
|
|
|
// Struct field handling is adapted from code in encoding/json:
|
|
//
|
|
// Copyright 2010 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the Go distribution.
|
|
|
|
import (
|
|
"reflect"
|
|
"sort"
|
|
"sync"
|
|
)
|
|
|
|
// A field represents a single field found in a struct.
|
|
type field struct {
|
|
name string // the name of the field (`toml` tag included)
|
|
tag bool // whether field has a `toml` tag
|
|
index []int // represents the depth of an anonymous field
|
|
typ reflect.Type // the type of the field
|
|
}
|
|
|
|
// byName sorts field by name, breaking ties with depth,
|
|
// then breaking ties with "name came from toml tag", then
|
|
// breaking ties with index sequence.
|
|
type byName []field
|
|
|
|
func (x byName) Len() int { return len(x) }
|
|
|
|
func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
|
|
|
|
func (x byName) Less(i, j int) bool {
|
|
if x[i].name != x[j].name {
|
|
return x[i].name < x[j].name
|
|
}
|
|
if len(x[i].index) != len(x[j].index) {
|
|
return len(x[i].index) < len(x[j].index)
|
|
}
|
|
if x[i].tag != x[j].tag {
|
|
return x[i].tag
|
|
}
|
|
return byIndex(x).Less(i, j)
|
|
}
|
|
|
|
// byIndex sorts field by index sequence.
|
|
type byIndex []field
|
|
|
|
func (x byIndex) Len() int { return len(x) }
|
|
|
|
func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
|
|
|
|
func (x byIndex) Less(i, j int) bool {
|
|
for k, xik := range x[i].index {
|
|
if k >= len(x[j].index) {
|
|
return false
|
|
}
|
|
if xik != x[j].index[k] {
|
|
return xik < x[j].index[k]
|
|
}
|
|
}
|
|
return len(x[i].index) < len(x[j].index)
|
|
}
|
|
|
|
// typeFields returns a list of fields that TOML should recognize for the given
|
|
// type. The algorithm is breadth-first search over the set of structs to
|
|
// include - the top struct and then any reachable anonymous structs.
|
|
func typeFields(t reflect.Type) []field {
|
|
// Anonymous fields to explore at the current level and the next.
|
|
current := []field{}
|
|
next := []field{{typ: t}}
|
|
|
|
// Count of queued names for current level and the next.
|
|
count := map[reflect.Type]int{}
|
|
nextCount := map[reflect.Type]int{}
|
|
|
|
// Types already visited at an earlier level.
|
|
visited := map[reflect.Type]bool{}
|
|
|
|
// Fields found.
|
|
var fields []field
|
|
|
|
for len(next) > 0 {
|
|
current, next = next, current[:0]
|
|
count, nextCount = nextCount, map[reflect.Type]int{}
|
|
|
|
for _, f := range current {
|
|
if visited[f.typ] {
|
|
continue
|
|
}
|
|
visited[f.typ] = true
|
|
|
|
// Scan f.typ for fields to include.
|
|
for i := 0; i < f.typ.NumField(); i++ {
|
|
sf := f.typ.Field(i)
|
|
if sf.PkgPath != "" && !sf.Anonymous { // unexported
|
|
continue
|
|
}
|
|
opts := getOptions(sf.Tag)
|
|
if opts.skip {
|
|
continue
|
|
}
|
|
index := make([]int, len(f.index)+1)
|
|
copy(index, f.index)
|
|
index[len(f.index)] = i
|
|
|
|
ft := sf.Type
|
|
if ft.Name() == "" && ft.Kind() == reflect.Ptr {
|
|
// Follow pointer.
|
|
ft = ft.Elem()
|
|
}
|
|
|
|
// Record found field and index sequence.
|
|
if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
|
|
tagged := opts.name != ""
|
|
name := opts.name
|
|
if name == "" {
|
|
name = sf.Name
|
|
}
|
|
fields = append(fields, field{name, tagged, index, ft})
|
|
if count[f.typ] > 1 {
|
|
// If there were multiple instances, add a second,
|
|
// so that the annihilation code will see a duplicate.
|
|
// It only cares about the distinction between 1 or 2,
|
|
// so don't bother generating any more copies.
|
|
fields = append(fields, fields[len(fields)-1])
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Record new anonymous struct to explore in next round.
|
|
nextCount[ft]++
|
|
if nextCount[ft] == 1 {
|
|
f := field{name: ft.Name(), index: index, typ: ft}
|
|
next = append(next, f)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
sort.Sort(byName(fields))
|
|
|
|
// Delete all fields that are hidden by the Go rules for embedded fields,
|
|
// except that fields with TOML tags are promoted.
|
|
|
|
// The fields are sorted in primary order of name, secondary order
|
|
// of field index length. Loop over names; for each name, delete
|
|
// hidden fields by choosing the one dominant field that survives.
|
|
out := fields[:0]
|
|
for advance, i := 0, 0; i < len(fields); i += advance {
|
|
// One iteration per name.
|
|
// Find the sequence of fields with the name of this first field.
|
|
fi := fields[i]
|
|
name := fi.name
|
|
for advance = 1; i+advance < len(fields); advance++ {
|
|
fj := fields[i+advance]
|
|
if fj.name != name {
|
|
break
|
|
}
|
|
}
|
|
if advance == 1 { // Only one field with this name
|
|
out = append(out, fi)
|
|
continue
|
|
}
|
|
dominant, ok := dominantField(fields[i : i+advance])
|
|
if ok {
|
|
out = append(out, dominant)
|
|
}
|
|
}
|
|
|
|
fields = out
|
|
sort.Sort(byIndex(fields))
|
|
|
|
return fields
|
|
}
|
|
|
|
// dominantField looks through the fields, all of which are known to
|
|
// have the same name, to find the single field that dominates the
|
|
// others using Go's embedding rules, modified by the presence of
|
|
// TOML tags. If there are multiple top-level fields, the boolean
|
|
// will be false: This condition is an error in Go and we skip all
|
|
// the fields.
|
|
func dominantField(fields []field) (field, bool) {
|
|
// The fields are sorted in increasing index-length order. The winner
|
|
// must therefore be one with the shortest index length. Drop all
|
|
// longer entries, which is easy: just truncate the slice.
|
|
length := len(fields[0].index)
|
|
tagged := -1 // Index of first tagged field.
|
|
for i, f := range fields {
|
|
if len(f.index) > length {
|
|
fields = fields[:i]
|
|
break
|
|
}
|
|
if f.tag {
|
|
if tagged >= 0 {
|
|
// Multiple tagged fields at the same level: conflict.
|
|
// Return no field.
|
|
return field{}, false
|
|
}
|
|
tagged = i
|
|
}
|
|
}
|
|
if tagged >= 0 {
|
|
return fields[tagged], true
|
|
}
|
|
// All remaining fields have the same length. If there's more than one,
|
|
// we have a conflict (two fields named "X" at the same level) and we
|
|
// return no field.
|
|
if len(fields) > 1 {
|
|
return field{}, false
|
|
}
|
|
return fields[0], true
|
|
}
|
|
|
|
var fieldCache struct {
|
|
sync.RWMutex
|
|
m map[reflect.Type][]field
|
|
}
|
|
|
|
// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
|
|
func cachedTypeFields(t reflect.Type) []field {
|
|
fieldCache.RLock()
|
|
f := fieldCache.m[t]
|
|
fieldCache.RUnlock()
|
|
if f != nil {
|
|
return f
|
|
}
|
|
|
|
// Compute fields without lock.
|
|
// Might duplicate effort but won't hold other computations back.
|
|
f = typeFields(t)
|
|
if f == nil {
|
|
f = []field{}
|
|
}
|
|
|
|
fieldCache.Lock()
|
|
if fieldCache.m == nil {
|
|
fieldCache.m = map[reflect.Type][]field{}
|
|
}
|
|
fieldCache.m[t] = f
|
|
fieldCache.Unlock()
|
|
return f
|
|
}
|