2014-12-08 14:02:27 +01:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
|
|
|
//
|
2015-03-07 21:36:35 +01:00
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 07:52:18 +01:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2014-12-08 14:02:27 +01:00
|
|
|
|
2014-11-17 00:18:59 +01:00
|
|
|
package model
|
|
|
|
|
|
|
|
import (
|
2015-06-03 09:47:39 +02:00
|
|
|
"fmt"
|
2014-11-17 00:18:59 +01:00
|
|
|
"time"
|
|
|
|
|
2019-07-09 11:40:30 +02:00
|
|
|
"github.com/thejerf/suture"
|
|
|
|
|
2015-08-06 11:29:25 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/config"
|
|
|
|
"github.com/syncthing/syncthing/lib/events"
|
2016-04-15 12:59:41 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
2015-08-06 11:29:25 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/sync"
|
2019-07-09 11:40:30 +02:00
|
|
|
"github.com/syncthing/syncthing/lib/util"
|
2014-11-17 00:18:59 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
type ProgressEmitter struct {
|
2019-07-09 11:40:30 +02:00
|
|
|
suture.Service
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
registry map[string]map[string]*sharedPullerState // folder: name: puller
|
2016-04-15 12:59:41 +02:00
|
|
|
interval time.Duration
|
|
|
|
minBlocks int
|
|
|
|
sentDownloadStates map[protocol.DeviceID]*sentDownloadState // States representing what we've sent to the other peer via DownloadProgress messages.
|
2019-04-13 14:20:51 +02:00
|
|
|
connections map[protocol.DeviceID]protocol.Connection
|
|
|
|
foldersByConns map[protocol.DeviceID][]string
|
|
|
|
disabled bool
|
2019-08-15 16:29:37 +02:00
|
|
|
evLogger events.Logger
|
2016-04-15 12:59:41 +02:00
|
|
|
mut sync.Mutex
|
2014-11-17 00:18:59 +01:00
|
|
|
|
|
|
|
timer *time.Timer
|
|
|
|
}
|
|
|
|
|
2015-04-28 22:32:10 +02:00
|
|
|
// NewProgressEmitter creates a new progress emitter which emits
|
|
|
|
// DownloadProgress events every interval.
|
2019-08-15 16:29:37 +02:00
|
|
|
func NewProgressEmitter(cfg config.Wrapper, evLogger events.Logger) *ProgressEmitter {
|
2014-11-17 00:18:59 +01:00
|
|
|
t := &ProgressEmitter{
|
2019-04-13 14:20:51 +02:00
|
|
|
registry: make(map[string]map[string]*sharedPullerState),
|
2016-04-15 12:59:41 +02:00
|
|
|
timer: time.NewTimer(time.Millisecond),
|
|
|
|
sentDownloadStates: make(map[protocol.DeviceID]*sentDownloadState),
|
2019-04-13 14:20:51 +02:00
|
|
|
connections: make(map[protocol.DeviceID]protocol.Connection),
|
|
|
|
foldersByConns: make(map[protocol.DeviceID][]string),
|
2019-08-15 16:29:37 +02:00
|
|
|
evLogger: evLogger,
|
2016-04-15 12:59:41 +02:00
|
|
|
mut: sync.NewMutex(),
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
2019-07-09 11:40:30 +02:00
|
|
|
t.Service = util.AsService(t.serve)
|
2015-06-03 09:47:39 +02:00
|
|
|
|
2016-11-12 10:34:18 +01:00
|
|
|
t.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
|
2014-11-17 00:18:59 +01:00
|
|
|
cfg.Subscribe(t)
|
2015-06-03 09:47:39 +02:00
|
|
|
|
2014-11-17 00:18:59 +01:00
|
|
|
return t
|
|
|
|
}
|
|
|
|
|
2019-07-09 11:40:30 +02:00
|
|
|
// serve starts the progress emitter which starts emitting DownloadProgress
|
2015-04-28 22:32:10 +02:00
|
|
|
// events as the progress happens.
|
2019-07-09 11:40:30 +02:00
|
|
|
func (t *ProgressEmitter) serve(stop chan struct{}) {
|
2016-04-15 12:59:41 +02:00
|
|
|
var lastUpdate time.Time
|
|
|
|
var lastCount, newCount int
|
2014-11-17 00:18:59 +01:00
|
|
|
for {
|
|
|
|
select {
|
2019-07-09 11:40:30 +02:00
|
|
|
case <-stop:
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("progress emitter: stopping")
|
2014-11-17 00:18:59 +01:00
|
|
|
return
|
|
|
|
case <-t.timer.C:
|
2014-11-29 22:51:13 +01:00
|
|
|
t.mut.Lock()
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("progress emitter: timer - looking after", len(t.registry))
|
2016-04-15 12:59:41 +02:00
|
|
|
|
|
|
|
newLastUpdated := lastUpdate
|
2019-04-13 14:20:51 +02:00
|
|
|
newCount = t.lenRegistryLocked()
|
|
|
|
for _, pullers := range t.registry {
|
|
|
|
for _, puller := range pullers {
|
|
|
|
if updated := puller.Updated(); updated.After(newLastUpdated) {
|
|
|
|
newLastUpdated = updated
|
|
|
|
}
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
|
|
|
if !newLastUpdated.Equal(lastUpdate) || newCount != lastCount {
|
|
|
|
lastUpdate = newLastUpdated
|
|
|
|
lastCount = newCount
|
2019-04-13 14:20:51 +02:00
|
|
|
t.sendDownloadProgressEventLocked()
|
2016-04-15 12:59:41 +02:00
|
|
|
if len(t.connections) > 0 {
|
2019-04-13 14:20:51 +02:00
|
|
|
t.sendDownloadProgressMessagesLocked()
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
2015-10-03 17:25:21 +02:00
|
|
|
} else {
|
2014-11-17 00:18:59 +01:00
|
|
|
l.Debugln("progress emitter: nothing new")
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
|
|
|
if newCount != 0 {
|
2014-11-17 00:18:59 +01:00
|
|
|
t.timer.Reset(t.interval)
|
|
|
|
}
|
|
|
|
t.mut.Unlock()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) sendDownloadProgressEventLocked() {
|
2016-04-15 12:59:41 +02:00
|
|
|
output := make(map[string]map[string]*pullerProgress)
|
2019-04-13 14:20:51 +02:00
|
|
|
for folder, pullers := range t.registry {
|
|
|
|
if len(pullers) == 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
output[folder] = make(map[string]*pullerProgress)
|
|
|
|
for name, puller := range pullers {
|
|
|
|
output[folder][name] = puller.Progress()
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
}
|
2019-08-15 16:29:37 +02:00
|
|
|
t.evLogger.Log(events.DownloadProgress, output)
|
2016-04-15 12:59:41 +02:00
|
|
|
l.Debugf("progress emitter: emitting %#v", output)
|
|
|
|
}
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) sendDownloadProgressMessagesLocked() {
|
|
|
|
for id, conn := range t.connections {
|
|
|
|
for _, folder := range t.foldersByConns[id] {
|
|
|
|
pullers, ok := t.registry[folder]
|
|
|
|
if !ok {
|
|
|
|
// There's never been any puller registered for this folder yet
|
|
|
|
continue
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
|
|
|
state, ok := t.sentDownloadStates[id]
|
|
|
|
if !ok {
|
|
|
|
state = &sentDownloadState{
|
|
|
|
folderStates: make(map[string]*sentFolderDownloadState),
|
|
|
|
}
|
|
|
|
t.sentDownloadStates[id] = state
|
|
|
|
}
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
activePullers := make([]*sharedPullerState, 0, len(pullers))
|
|
|
|
for _, puller := range pullers {
|
2016-04-15 12:59:41 +02:00
|
|
|
if puller.folder != folder || puller.file.IsSymlink() || puller.file.IsDirectory() || len(puller.file.Blocks) <= t.minBlocks {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
activePullers = append(activePullers, puller)
|
|
|
|
}
|
|
|
|
|
|
|
|
// For every new puller that hasn't yet been seen, it will send all the blocks the puller has available
|
|
|
|
// For every existing puller, it will check for new blocks, and send update for the new blocks only
|
|
|
|
// For every puller that we've seen before but is no longer there, we will send a forget message
|
|
|
|
updates := state.update(folder, activePullers)
|
|
|
|
|
|
|
|
if len(updates) > 0 {
|
2016-07-04 12:40:29 +02:00
|
|
|
conn.DownloadProgress(folder, updates)
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Clean up sentDownloadStates for devices which we are no longer connected to.
|
|
|
|
for id := range t.sentDownloadStates {
|
2019-04-13 14:20:51 +02:00
|
|
|
_, ok := t.connections[id]
|
2016-04-15 12:59:41 +02:00
|
|
|
if !ok {
|
|
|
|
// Null out outstanding entries for device
|
|
|
|
delete(t.sentDownloadStates, id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If a folder was unshared from some device, tell it that all temp files
|
|
|
|
// are now gone.
|
2019-04-13 14:20:51 +02:00
|
|
|
for id, state := range t.sentDownloadStates {
|
2016-04-15 12:59:41 +02:00
|
|
|
// For each of the folders that the state is aware of,
|
|
|
|
// try to match it with a shared folder we've discovered above,
|
2019-04-13 14:20:51 +02:00
|
|
|
nextFolder:
|
2016-04-15 12:59:41 +02:00
|
|
|
for _, folder := range state.folders() {
|
2019-04-13 14:20:51 +02:00
|
|
|
for _, existingFolder := range t.foldersByConns[id] {
|
2016-04-15 12:59:41 +02:00
|
|
|
if existingFolder == folder {
|
|
|
|
continue nextFolder
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If we fail to find that folder, we tell the state to forget about it
|
|
|
|
// and return us a list of updates which would clean up the state
|
|
|
|
// on the remote end.
|
2019-02-02 12:09:07 +01:00
|
|
|
state.cleanup(folder)
|
|
|
|
// updates := state.cleanup(folder)
|
|
|
|
// if len(updates) > 0 {
|
|
|
|
// XXX: Don't send this now, as the only way we've unshared a folder
|
|
|
|
// is by breaking the connection and reconnecting, hence sending
|
|
|
|
// forget messages for some random folder currently makes no sense.
|
|
|
|
// deviceConns[id].DownloadProgress(folder, updates, 0, nil)
|
|
|
|
// }
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-03 09:47:39 +02:00
|
|
|
// VerifyConfiguration implements the config.Committer interface
|
|
|
|
func (t *ProgressEmitter) VerifyConfiguration(from, to config.Configuration) error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// CommitConfiguration implements the config.Committer interface
|
|
|
|
func (t *ProgressEmitter) CommitConfiguration(from, to config.Configuration) bool {
|
2014-11-17 00:18:59 +01:00
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
switch {
|
|
|
|
case t.disabled && to.Options.ProgressUpdateIntervalS >= 0:
|
|
|
|
t.disabled = false
|
|
|
|
l.Debugln("progress emitter: enabled")
|
|
|
|
fallthrough
|
|
|
|
case !t.disabled && from.Options.ProgressUpdateIntervalS != to.Options.ProgressUpdateIntervalS:
|
|
|
|
t.interval = time.Duration(to.Options.ProgressUpdateIntervalS) * time.Second
|
|
|
|
if t.interval < time.Second {
|
|
|
|
t.interval = time.Second
|
|
|
|
}
|
|
|
|
l.Debugln("progress emitter: updated interval", t.interval)
|
|
|
|
case !t.disabled && to.Options.ProgressUpdateIntervalS < 0:
|
|
|
|
t.clearLocked()
|
|
|
|
t.disabled = true
|
|
|
|
l.Debugln("progress emitter: disabled")
|
2016-08-16 20:22:01 +02:00
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
t.minBlocks = to.Options.TempIndexMinBlocks
|
2015-06-03 09:47:39 +02:00
|
|
|
|
|
|
|
return true
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Register a puller with the emitter which will start broadcasting pullers
|
|
|
|
// progress.
|
|
|
|
func (t *ProgressEmitter) Register(s *sharedPullerState) {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
2019-04-13 14:20:51 +02:00
|
|
|
if t.disabled {
|
|
|
|
l.Debugln("progress emitter: disabled, skip registering")
|
|
|
|
return
|
|
|
|
}
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugln("progress emitter: registering", s.folder, s.file.Name)
|
2019-04-13 14:20:51 +02:00
|
|
|
if t.emptyLocked() {
|
2014-11-17 00:18:59 +01:00
|
|
|
t.timer.Reset(t.interval)
|
|
|
|
}
|
2019-04-13 14:20:51 +02:00
|
|
|
if _, ok := t.registry[s.folder]; !ok {
|
|
|
|
t.registry[s.folder] = make(map[string]*sharedPullerState)
|
|
|
|
}
|
|
|
|
t.registry[s.folder][s.file.Name] = s
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
|
|
|
|
2015-04-28 17:34:55 +02:00
|
|
|
// Deregister a puller which will stop broadcasting pullers state.
|
2014-11-17 00:18:59 +01:00
|
|
|
func (t *ProgressEmitter) Deregister(s *sharedPullerState) {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
2016-04-15 12:59:41 +02:00
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
if t.disabled {
|
|
|
|
l.Debugln("progress emitter: disabled, skip deregistering")
|
|
|
|
return
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
l.Debugln("progress emitter: deregistering", s.folder, s.file.Name)
|
|
|
|
delete(t.registry[s.folder], s.file.Name)
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
|
|
|
|
2015-04-28 22:32:10 +02:00
|
|
|
// BytesCompleted returns the number of bytes completed in the given folder.
|
2014-11-17 00:18:59 +01:00
|
|
|
func (t *ProgressEmitter) BytesCompleted(folder string) (bytes int64) {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
for _, s := range t.registry[folder] {
|
|
|
|
bytes += s.Progress().BytesDone
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
2015-10-03 17:25:21 +02:00
|
|
|
l.Debugf("progress emitter: bytes completed for %s: %d", folder, bytes)
|
2014-11-17 00:18:59 +01:00
|
|
|
return
|
|
|
|
}
|
2015-06-03 09:47:39 +02:00
|
|
|
|
|
|
|
func (t *ProgressEmitter) String() string {
|
|
|
|
return fmt.Sprintf("ProgressEmitter@%p", t)
|
|
|
|
}
|
2015-10-20 08:51:14 +02:00
|
|
|
|
|
|
|
func (t *ProgressEmitter) lenRegistry() int {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
2019-04-13 14:20:51 +02:00
|
|
|
return t.lenRegistryLocked()
|
2015-10-20 08:51:14 +02:00
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) lenRegistryLocked() (out int) {
|
|
|
|
for _, pullers := range t.registry {
|
|
|
|
out += len(pullers)
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
2019-04-13 14:20:51 +02:00
|
|
|
return out
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) emptyLocked() bool {
|
|
|
|
for _, pullers := range t.registry {
|
|
|
|
if len(pullers) != 0 {
|
|
|
|
return false
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
}
|
2019-04-13 14:20:51 +02:00
|
|
|
return true
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) temporaryIndexSubscribe(conn protocol.Connection, folders []string) {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
|
|
|
t.connections[conn.ID()] = conn
|
|
|
|
t.foldersByConns[conn.ID()] = folders
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *ProgressEmitter) temporaryIndexUnsubscribe(conn protocol.Connection) {
|
|
|
|
t.mut.Lock()
|
|
|
|
defer t.mut.Unlock()
|
|
|
|
delete(t.connections, conn.ID())
|
|
|
|
delete(t.foldersByConns, conn.ID())
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
|
2019-04-13 14:20:51 +02:00
|
|
|
func (t *ProgressEmitter) clearLocked() {
|
|
|
|
for id, state := range t.sentDownloadStates {
|
|
|
|
conn, ok := t.connections[id]
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, folder := range state.folders() {
|
|
|
|
if updates := state.cleanup(folder); len(updates) > 0 {
|
|
|
|
conn.DownloadProgress(folder, updates)
|
|
|
|
}
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|
|
|
|
}
|
2019-04-13 14:20:51 +02:00
|
|
|
t.registry = make(map[string]map[string]*sharedPullerState)
|
|
|
|
t.sentDownloadStates = make(map[protocol.DeviceID]*sentDownloadState)
|
|
|
|
t.connections = make(map[protocol.DeviceID]protocol.Connection)
|
|
|
|
t.foldersByConns = make(map[protocol.DeviceID][]string)
|
2016-04-15 12:59:41 +02:00
|
|
|
}
|