2014-11-16 21:13:20 +01:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
2014-09-29 21:43:32 +02:00
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify it
|
|
|
|
// under the terms of the GNU General Public License as published by the Free
|
|
|
|
// Software Foundation, either version 3 of the License, or (at your option)
|
|
|
|
// any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful, but WITHOUT
|
|
|
|
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
// more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU General Public License along
|
|
|
|
// with this program. If not, see <http://www.gnu.org/licenses/>.
|
2014-06-01 22:50:14 +02:00
|
|
|
|
2014-05-15 05:26:55 +02:00
|
|
|
package model
|
2014-03-28 14:36:57 +01:00
|
|
|
|
|
|
|
import (
|
2014-10-09 00:41:23 +02:00
|
|
|
"bytes"
|
2014-10-25 00:20:08 +02:00
|
|
|
"crypto/sha256"
|
2014-03-28 14:36:57 +01:00
|
|
|
"errors"
|
2014-08-25 17:45:13 +02:00
|
|
|
"fmt"
|
2014-11-09 05:26:52 +01:00
|
|
|
"io/ioutil"
|
2014-03-28 14:36:57 +01:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2014-09-27 14:44:15 +02:00
|
|
|
"sync"
|
2014-03-28 14:36:57 +01:00
|
|
|
"time"
|
2014-06-20 00:27:54 +02:00
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
"github.com/AudriusButkevicius/lfu-go"
|
|
|
|
|
2014-09-22 21:42:11 +02:00
|
|
|
"github.com/syncthing/syncthing/internal/config"
|
|
|
|
"github.com/syncthing/syncthing/internal/events"
|
|
|
|
"github.com/syncthing/syncthing/internal/osutil"
|
|
|
|
"github.com/syncthing/syncthing/internal/protocol"
|
|
|
|
"github.com/syncthing/syncthing/internal/scanner"
|
2014-11-09 05:26:52 +01:00
|
|
|
"github.com/syncthing/syncthing/internal/symlinks"
|
2014-09-22 21:42:11 +02:00
|
|
|
"github.com/syncthing/syncthing/internal/versioner"
|
2014-03-28 14:36:57 +01:00
|
|
|
)
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// TODO: Stop on errors
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
const (
|
2014-11-23 01:02:09 +01:00
|
|
|
pauseIntv = 60 * time.Second
|
|
|
|
nextPullIntv = 10 * time.Second
|
|
|
|
checkPullIntv = 1 * time.Second
|
2014-09-27 14:44:15 +02:00
|
|
|
)
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// A pullBlockState is passed to the puller routine for each block that needs
|
|
|
|
// to be fetched.
|
|
|
|
type pullBlockState struct {
|
|
|
|
*sharedPullerState
|
|
|
|
block protocol.BlockInfo
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// A copyBlocksState is passed to copy routine if the file has blocks to be
|
2014-10-09 00:41:23 +02:00
|
|
|
// copied.
|
2014-09-27 14:44:15 +02:00
|
|
|
type copyBlocksState struct {
|
|
|
|
*sharedPullerState
|
|
|
|
blocks []protocol.BlockInfo
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
var (
|
2014-09-28 13:05:25 +02:00
|
|
|
activity = newDeviceActivity()
|
2014-09-28 13:00:38 +02:00
|
|
|
errNoDevice = errors.New("no available source device")
|
2014-09-27 14:44:15 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
type Puller struct {
|
2014-11-17 00:18:59 +01:00
|
|
|
folder string
|
|
|
|
dir string
|
|
|
|
scanIntv time.Duration
|
|
|
|
model *Model
|
|
|
|
stop chan struct{}
|
|
|
|
versioner versioner.Versioner
|
|
|
|
ignorePerms bool
|
|
|
|
lenientMtimes bool
|
|
|
|
progressEmitter *ProgressEmitter
|
|
|
|
copiers int
|
|
|
|
pullers int
|
|
|
|
finishers int
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Serve will run scans and pulls. It will return when Stop()ed or on a
|
|
|
|
// critical error.
|
|
|
|
func (p *Puller) Serve() {
|
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "starting")
|
|
|
|
defer l.Debugln(p, "exiting")
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
p.stop = make(chan struct{})
|
2014-05-25 20:49:08 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
pullTimer := time.NewTimer(checkPullIntv)
|
2014-09-30 17:34:31 +02:00
|
|
|
scanTimer := time.NewTimer(time.Millisecond) // The first scan should be done immediately.
|
2014-10-15 11:30:04 +02:00
|
|
|
cleanTimer := time.NewTicker(time.Hour)
|
2014-09-27 14:44:15 +02:00
|
|
|
|
|
|
|
defer func() {
|
|
|
|
pullTimer.Stop()
|
|
|
|
scanTimer.Stop()
|
2014-10-15 11:30:04 +02:00
|
|
|
cleanTimer.Stop()
|
2014-09-28 13:00:38 +02:00
|
|
|
// TODO: Should there be an actual FolderStopped state?
|
|
|
|
p.model.setState(p.folder, FolderIdle)
|
2014-09-27 14:44:15 +02:00
|
|
|
}()
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-06-20 00:27:54 +02:00
|
|
|
var prevVer uint64
|
2014-07-24 09:38:16 +02:00
|
|
|
|
2014-10-15 11:30:04 +02:00
|
|
|
// Clean out old temporaries before we start pulling
|
|
|
|
p.clean()
|
|
|
|
|
2014-09-30 17:34:31 +02:00
|
|
|
// We don't start pulling files until a scan has been completed.
|
|
|
|
initialScanCompleted := false
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
loop:
|
2014-03-28 14:36:57 +01:00
|
|
|
for {
|
2014-09-27 14:44:15 +02:00
|
|
|
select {
|
|
|
|
case <-p.stop:
|
|
|
|
return
|
2014-07-24 09:38:16 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// TODO: We could easily add a channel here for notifications from
|
|
|
|
// Index(), so that we immediately start a pull when new index
|
|
|
|
// information is available. Before that though, I'd like to build a
|
|
|
|
// repeatable benchmark of how long it takes to sync a change from
|
2014-09-28 13:00:38 +02:00
|
|
|
// device A to device B, so we have something to work against.
|
2014-09-27 14:44:15 +02:00
|
|
|
case <-pullTimer.C:
|
2014-09-30 17:34:31 +02:00
|
|
|
if !initialScanCompleted {
|
2014-10-03 17:55:03 +02:00
|
|
|
// How did we even get here?
|
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "skip (initial)")
|
|
|
|
}
|
2014-10-03 16:10:35 +02:00
|
|
|
pullTimer.Reset(nextPullIntv)
|
2014-09-30 17:34:31 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// RemoteLocalVersion() is a fast call, doesn't touch the database.
|
2014-09-28 13:00:38 +02:00
|
|
|
curVer := p.model.RemoteLocalVersion(p.folder)
|
2014-09-27 14:44:15 +02:00
|
|
|
if curVer == prevVer {
|
2014-10-03 17:55:03 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "skip (curVer == prevVer)", prevVer)
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
pullTimer.Reset(checkPullIntv)
|
|
|
|
continue
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-08-04 22:02:44 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "pulling", prevVer, curVer)
|
2014-08-04 22:02:44 +02:00
|
|
|
}
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.setState(p.folder, FolderSyncing)
|
2014-09-27 14:44:15 +02:00
|
|
|
tries := 0
|
2014-10-25 00:20:08 +02:00
|
|
|
checksum := false
|
2014-09-27 14:44:15 +02:00
|
|
|
for {
|
|
|
|
tries++
|
2014-10-25 00:20:08 +02:00
|
|
|
// Last resort mode, to get around corrupt/invalid block maps.
|
|
|
|
if tries == 10 {
|
|
|
|
l.Infoln("Desperation mode ON")
|
|
|
|
checksum = true
|
|
|
|
}
|
|
|
|
|
2014-11-23 01:02:09 +01:00
|
|
|
changed := p.pullerIteration(checksum)
|
2014-09-27 14:44:15 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "changed", changed)
|
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
if changed == 0 {
|
|
|
|
// No files were changed by the puller, so we are in
|
|
|
|
// sync. Remember the local version number and
|
|
|
|
// schedule a resync a little bit into the future.
|
2014-09-28 07:56:05 +02:00
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
if lv := p.model.RemoteLocalVersion(p.folder); lv < curVer {
|
|
|
|
// There's a corner case where the device we needed
|
2014-09-28 07:56:05 +02:00
|
|
|
// files from disconnected during the puller
|
|
|
|
// iteration. The files will have been removed from
|
|
|
|
// the index, so we've concluded that we don't need
|
|
|
|
// them, but at the same time we have the local
|
|
|
|
// version that includes those files in curVer. So we
|
|
|
|
// catch the case that localVersion might have
|
|
|
|
// decresed here.
|
2014-09-28 13:05:25 +02:00
|
|
|
l.Debugln(p, "adjusting curVer", lv)
|
2014-09-28 07:56:05 +02:00
|
|
|
curVer = lv
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
prevVer = curVer
|
2014-10-16 09:32:23 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "next pull in", nextPullIntv)
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
pullTimer.Reset(nextPullIntv)
|
|
|
|
break
|
|
|
|
}
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
if tries > 10 {
|
|
|
|
// We've tried a bunch of times to get in sync, but
|
|
|
|
// we're not making it. Probably there are write
|
|
|
|
// errors preventing us. Flag this with a warning and
|
|
|
|
// wait a bit longer before retrying.
|
2014-09-28 13:00:38 +02:00
|
|
|
l.Warnf("Folder %q isn't making progress - check logs for possible root cause. Pausing puller for %v.", p.folder, pauseIntv)
|
2014-10-16 09:32:23 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "next pull in", pauseIntv)
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
pullTimer.Reset(pauseIntv)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.setState(p.folder, FolderIdle)
|
2014-04-14 09:58:17 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// The reason for running the scanner from within the puller is that
|
|
|
|
// this is the easiest way to make sure we are not doing both at the
|
|
|
|
// same time.
|
|
|
|
case <-scanTimer.C:
|
2014-05-15 05:26:55 +02:00
|
|
|
if debug {
|
2014-09-27 14:44:15 +02:00
|
|
|
l.Debugln(p, "rescan")
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.setState(p.folder, FolderScanning)
|
|
|
|
if err := p.model.ScanFolder(p.folder); err != nil {
|
2014-10-06 09:25:45 +02:00
|
|
|
p.model.cfg.InvalidateFolder(p.folder, err.Error())
|
2014-09-27 14:44:15 +02:00
|
|
|
break loop
|
2014-05-04 18:20:25 +02:00
|
|
|
}
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.setState(p.folder, FolderIdle)
|
2014-10-15 10:51:09 +02:00
|
|
|
if p.scanIntv > 0 {
|
2014-10-16 09:32:23 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "next rescan in", p.scanIntv)
|
|
|
|
}
|
2014-10-15 10:51:09 +02:00
|
|
|
scanTimer.Reset(p.scanIntv)
|
|
|
|
}
|
2014-09-30 17:34:31 +02:00
|
|
|
if !initialScanCompleted {
|
|
|
|
l.Infoln("Completed initial scan (rw) of folder", p.folder)
|
|
|
|
initialScanCompleted = true
|
|
|
|
}
|
2014-10-16 09:32:23 +02:00
|
|
|
|
2014-10-15 11:30:04 +02:00
|
|
|
// Clean out old temporaries
|
|
|
|
case <-cleanTimer.C:
|
|
|
|
p.clean()
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
func (p *Puller) Stop() {
|
|
|
|
close(p.stop)
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
func (p *Puller) String() string {
|
2014-09-28 13:00:38 +02:00
|
|
|
return fmt.Sprintf("puller/%s@%p", p.folder, p)
|
2014-09-27 14:44:15 +02:00
|
|
|
}
|
2014-09-07 21:29:06 +02:00
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
// pullerIteration runs a single puller iteration for the given folder and
|
2014-09-27 14:44:15 +02:00
|
|
|
// returns the number items that should have been synced (even those that
|
|
|
|
// might have failed). One puller iteration handles all files currently
|
2014-11-23 01:02:09 +01:00
|
|
|
// flagged as needed in the folder.
|
|
|
|
func (p *Puller) pullerIteration(checksum bool) int {
|
2014-09-27 14:44:15 +02:00
|
|
|
pullChan := make(chan pullBlockState)
|
|
|
|
copyChan := make(chan copyBlocksState)
|
|
|
|
finisherChan := make(chan *sharedPullerState)
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
var copyWg sync.WaitGroup
|
|
|
|
var pullWg sync.WaitGroup
|
2014-09-27 14:44:15 +02:00
|
|
|
var doneWg sync.WaitGroup
|
|
|
|
|
2014-11-23 19:43:49 +01:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "c", p.copiers, "p", p.pullers, "f", p.finishers)
|
|
|
|
}
|
|
|
|
|
2014-11-23 01:02:09 +01:00
|
|
|
for i := 0; i < p.copiers; i++ {
|
2014-10-09 00:41:23 +02:00
|
|
|
copyWg.Add(1)
|
2014-09-27 14:44:15 +02:00
|
|
|
go func() {
|
|
|
|
// copierRoutine finishes when copyChan is closed
|
2014-10-25 00:20:08 +02:00
|
|
|
p.copierRoutine(copyChan, pullChan, finisherChan, checksum)
|
2014-10-09 00:41:23 +02:00
|
|
|
copyWg.Done()
|
2014-09-27 14:44:15 +02:00
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2014-11-23 01:02:09 +01:00
|
|
|
for i := 0; i < p.pullers; i++ {
|
2014-10-09 00:41:23 +02:00
|
|
|
pullWg.Add(1)
|
2014-09-27 14:44:15 +02:00
|
|
|
go func() {
|
|
|
|
// pullerRoutine finishes when pullChan is closed
|
|
|
|
p.pullerRoutine(pullChan, finisherChan)
|
2014-10-09 00:41:23 +02:00
|
|
|
pullWg.Done()
|
2014-09-27 14:44:15 +02:00
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2014-11-23 01:02:09 +01:00
|
|
|
for i := 0; i < p.finishers; i++ {
|
2014-09-27 14:44:15 +02:00
|
|
|
doneWg.Add(1)
|
|
|
|
// finisherRoutine finishes when finisherChan is closed
|
|
|
|
go func() {
|
|
|
|
p.finisherRoutine(finisherChan)
|
|
|
|
doneWg.Done()
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2014-09-28 13:39:39 +02:00
|
|
|
p.model.fmut.RLock()
|
2014-09-28 13:00:38 +02:00
|
|
|
files := p.model.folderFiles[p.folder]
|
2014-09-28 13:39:39 +02:00
|
|
|
p.model.fmut.RUnlock()
|
2014-09-27 14:44:15 +02:00
|
|
|
|
|
|
|
// !!!
|
|
|
|
// WithNeed takes a database snapshot (by necessity). By the time we've
|
|
|
|
// handled a bunch of files it might have become out of date and we might
|
|
|
|
// be attempting to sync with an old version of a file...
|
|
|
|
// !!!
|
|
|
|
|
|
|
|
changed := 0
|
2014-10-12 23:01:57 +02:00
|
|
|
|
|
|
|
var deletions []protocol.FileInfo
|
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
|
2014-09-28 07:56:05 +02:00
|
|
|
|
|
|
|
// Needed items are delivered sorted lexicographically. This isn't
|
|
|
|
// really optimal from a performance point of view - it would be
|
|
|
|
// better if files were handled in random order, to spread the load
|
|
|
|
// over the cluster. But it means that we can be sure that we fully
|
|
|
|
// handle directories before the files that go inside them, which is
|
|
|
|
// nice.
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
file := intf.(protocol.FileInfo)
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
events.Default.Log(events.ItemStarted, map[string]string{
|
2014-09-28 13:00:38 +02:00
|
|
|
"folder": p.folder,
|
2014-09-28 13:05:25 +02:00
|
|
|
"item": file.Name,
|
2014-09-27 14:44:15 +02:00
|
|
|
})
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "handling", file.Name)
|
2014-05-25 20:49:08 +02:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
switch {
|
2014-11-05 00:22:15 +01:00
|
|
|
case file.IsDeleted():
|
2014-11-09 05:26:52 +01:00
|
|
|
// A deleted file, directory or symlink
|
2014-10-12 23:01:57 +02:00
|
|
|
deletions = append(deletions, file)
|
2014-11-09 05:26:52 +01:00
|
|
|
case file.IsDirectory() && !file.IsSymlink():
|
2014-09-27 14:44:15 +02:00
|
|
|
// A new or changed directory
|
|
|
|
p.handleDir(file)
|
|
|
|
default:
|
2014-11-09 05:26:52 +01:00
|
|
|
// A new or changed file or symlink. This is the only case where we
|
|
|
|
// do stuff in the background; the other three are done
|
|
|
|
// synchronously.
|
2014-10-09 00:41:23 +02:00
|
|
|
p.handleFile(file, copyChan, finisherChan)
|
2014-04-01 23:18:32 +02:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
changed++
|
|
|
|
return true
|
|
|
|
})
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Signal copy and puller routines that we are done with the in data for
|
2014-10-09 00:41:23 +02:00
|
|
|
// this iteration. Wait for them to finish.
|
2014-09-27 14:44:15 +02:00
|
|
|
close(copyChan)
|
2014-10-09 00:41:23 +02:00
|
|
|
copyWg.Wait()
|
2014-09-27 14:44:15 +02:00
|
|
|
close(pullChan)
|
2014-10-09 00:41:23 +02:00
|
|
|
pullWg.Wait()
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
// Signal the finisher chan that there will be no more input.
|
2014-09-27 14:44:15 +02:00
|
|
|
close(finisherChan)
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Wait for the finisherChan to finish.
|
|
|
|
doneWg.Wait()
|
2014-05-19 22:31:28 +02:00
|
|
|
|
2014-10-12 23:01:57 +02:00
|
|
|
for i := range deletions {
|
|
|
|
deletion := deletions[len(deletions)-i-1]
|
|
|
|
if deletion.IsDirectory() {
|
|
|
|
p.deleteDir(deletion)
|
|
|
|
} else {
|
|
|
|
p.deleteFile(deletion)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
return changed
|
|
|
|
}
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// handleDir creates or updates the given directory
|
|
|
|
func (p *Puller) handleDir(file protocol.FileInfo) {
|
|
|
|
realName := filepath.Join(p.dir, file.Name)
|
|
|
|
mode := os.FileMode(file.Flags & 0777)
|
2014-10-10 00:34:32 +02:00
|
|
|
if p.ignorePerms {
|
|
|
|
mode = 0755
|
|
|
|
}
|
2014-05-19 22:31:28 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
if debug {
|
2014-09-28 13:00:38 +02:00
|
|
|
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
|
2014-09-27 14:44:15 +02:00
|
|
|
l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
|
2014-04-01 23:18:32 +02:00
|
|
|
}
|
|
|
|
|
2014-11-17 12:25:32 +01:00
|
|
|
info, err := os.Lstat(realName)
|
|
|
|
isLink, _ := symlinks.IsSymlink(realName)
|
2014-11-13 23:59:40 +01:00
|
|
|
switch {
|
2014-11-17 12:25:32 +01:00
|
|
|
// There is already something under that name, but it's a file/link.
|
|
|
|
// Most likely a file/link is getting replaced with a directory.
|
|
|
|
// Remove the file/link and fall through to directory creation.
|
|
|
|
case isLink || (err == nil && !info.IsDir()):
|
2014-11-13 23:59:40 +01:00
|
|
|
err = osutil.InWritableDir(os.Remove, realName)
|
|
|
|
if err != nil {
|
|
|
|
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
|
2014-09-28 01:54:25 +02:00
|
|
|
return
|
|
|
|
}
|
2014-11-13 23:59:40 +01:00
|
|
|
fallthrough
|
|
|
|
// The directory doesn't exist, so we create it with the right
|
|
|
|
// mode bits from the start.
|
|
|
|
case err != nil && os.IsNotExist(err):
|
|
|
|
// We declare a function that acts on only the path name, so
|
|
|
|
// we can pass it to InWritableDir. We use a regular Mkdir and
|
|
|
|
// not MkdirAll because the parent should already exist.
|
|
|
|
mkdir := func(path string) error {
|
|
|
|
return os.Mkdir(path, mode)
|
|
|
|
}
|
2014-09-28 01:54:25 +02:00
|
|
|
|
2014-11-13 23:59:40 +01:00
|
|
|
if err = osutil.InWritableDir(mkdir, realName); err == nil {
|
|
|
|
p.model.updateLocal(p.folder, file)
|
|
|
|
} else {
|
|
|
|
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
|
|
|
|
}
|
2014-09-28 01:54:25 +02:00
|
|
|
return
|
2014-11-13 23:59:40 +01:00
|
|
|
// Weird error when stat()'ing the dir. Probably won't work to do
|
|
|
|
// anything else with it if we can't even stat() it.
|
|
|
|
case err != nil:
|
|
|
|
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
|
2014-03-28 14:36:57 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2014-09-28 01:54:25 +02:00
|
|
|
// The directory already exists, so we just correct the mode bits. (We
|
|
|
|
// don't handle modification times on directories, because that sucks...)
|
|
|
|
// It's OK to change mode bits on stuff within non-writable directories.
|
|
|
|
|
2014-10-10 00:34:32 +02:00
|
|
|
if p.ignorePerms {
|
|
|
|
p.model.updateLocal(p.folder, file)
|
|
|
|
} else if err := os.Chmod(realName, mode); err == nil {
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.updateLocal(p.folder, file)
|
2014-09-28 01:54:25 +02:00
|
|
|
} else {
|
2014-10-10 21:20:46 +02:00
|
|
|
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
|
2014-07-15 13:04:37 +02:00
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// deleteDir attempts to delete the given directory
|
|
|
|
func (p *Puller) deleteDir(file protocol.FileInfo) {
|
|
|
|
realName := filepath.Join(p.dir, file.Name)
|
2014-10-29 12:19:48 +01:00
|
|
|
// Delete any temporary files lying around in the directory
|
|
|
|
dir, _ := os.Open(realName)
|
|
|
|
if dir != nil {
|
|
|
|
files, _ := dir.Readdirnames(-1)
|
|
|
|
for _, file := range files {
|
|
|
|
if defTempNamer.IsTemporary(file) {
|
|
|
|
osutil.InWritableDir(os.Remove, filepath.Join(realName, file))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-09-28 01:54:25 +02:00
|
|
|
err := osutil.InWritableDir(os.Remove, realName)
|
2014-09-27 14:44:15 +02:00
|
|
|
if err == nil || os.IsNotExist(err) {
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.updateLocal(p.folder, file)
|
2014-10-10 21:20:46 +02:00
|
|
|
} else {
|
|
|
|
l.Infof("Puller (folder %q, dir %q): delete: %v", p.folder, file.Name, err)
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// deleteFile attempts to delete the given file
|
|
|
|
func (p *Puller) deleteFile(file protocol.FileInfo) {
|
|
|
|
realName := filepath.Join(p.dir, file.Name)
|
2014-04-01 23:18:32 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
var err error
|
|
|
|
if p.versioner != nil {
|
2014-09-28 01:54:25 +02:00
|
|
|
err = osutil.InWritableDir(p.versioner.Archive, realName)
|
2014-09-27 14:44:15 +02:00
|
|
|
} else {
|
2014-09-28 01:54:25 +02:00
|
|
|
err = osutil.InWritableDir(os.Remove, realName)
|
2014-09-27 14:44:15 +02:00
|
|
|
}
|
2014-07-13 21:07:24 +02:00
|
|
|
|
2014-10-06 21:47:42 +02:00
|
|
|
if err != nil && !os.IsNotExist(err) {
|
2014-09-28 13:00:38 +02:00
|
|
|
l.Infof("Puller (folder %q, file %q): delete: %v", p.folder, file.Name, err)
|
2014-09-27 14:44:15 +02:00
|
|
|
} else {
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.updateLocal(p.folder, file)
|
2014-05-28 11:45:45 +02:00
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
}
|
2014-05-28 11:45:45 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// handleFile queues the copies and pulls as necessary for a single new or
|
|
|
|
// changed file.
|
2014-10-09 00:41:23 +02:00
|
|
|
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
|
2014-09-28 13:00:38 +02:00
|
|
|
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-11-09 05:26:52 +01:00
|
|
|
if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
|
2014-09-27 14:44:15 +02:00
|
|
|
// We are supposed to copy the entire file, and then fetch nothing. We
|
|
|
|
// are only updating metadata, so we don't actually *need* to make the
|
|
|
|
// copy.
|
2014-05-15 05:26:55 +02:00
|
|
|
if debug {
|
2014-09-27 14:44:15 +02:00
|
|
|
l.Debugln(p, "taking shortcut on", file.Name)
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-11-09 05:26:52 +01:00
|
|
|
if file.IsSymlink() {
|
|
|
|
p.shortcutSymlink(curFile, file)
|
|
|
|
} else {
|
|
|
|
p.shortcutFile(file)
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
return
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-10-18 00:16:29 +02:00
|
|
|
scanner.PopulateOffsets(file.Blocks)
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Figure out the absolute filenames we need once and for all
|
|
|
|
tempName := filepath.Join(p.dir, defTempNamer.TempName(file.Name))
|
|
|
|
realName := filepath.Join(p.dir, file.Name)
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-10-12 22:38:22 +02:00
|
|
|
reused := 0
|
2014-10-09 00:41:23 +02:00
|
|
|
var blocks []protocol.BlockInfo
|
2014-10-04 00:15:54 +02:00
|
|
|
|
|
|
|
// Check for an old temporary file which might have some blocks we could
|
|
|
|
// reuse.
|
|
|
|
tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize)
|
|
|
|
if err == nil {
|
|
|
|
// Check for any reusable blocks in the temp file
|
|
|
|
tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
|
|
|
|
|
|
|
|
// block.String() returns a string unique to the block
|
|
|
|
existingBlocks := make(map[string]bool, len(tempCopyBlocks))
|
|
|
|
for _, block := range tempCopyBlocks {
|
|
|
|
existingBlocks[block.String()] = true
|
|
|
|
}
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
// Since the blocks are already there, we don't need to get them.
|
|
|
|
for _, block := range file.Blocks {
|
2014-10-04 00:15:54 +02:00
|
|
|
_, ok := existingBlocks[block.String()]
|
|
|
|
if !ok {
|
2014-10-09 00:41:23 +02:00
|
|
|
blocks = append(blocks, block)
|
2014-10-04 00:15:54 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-12 22:38:22 +02:00
|
|
|
// The sharedpullerstate will know which flags to use when opening the
|
|
|
|
// temp file depending if we are reusing any blocks or not.
|
|
|
|
reused = len(file.Blocks) - len(blocks)
|
|
|
|
if reused == 0 {
|
2014-10-04 00:15:54 +02:00
|
|
|
// Otherwise, discard the file ourselves in order for the
|
|
|
|
// sharedpuller not to panic when it fails to exlusively create a
|
|
|
|
// file which already exists
|
|
|
|
os.Remove(tempName)
|
|
|
|
}
|
2014-10-09 00:41:23 +02:00
|
|
|
} else {
|
|
|
|
blocks = file.Blocks
|
2014-10-04 00:15:54 +02:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
s := sharedPullerState{
|
|
|
|
file: file,
|
2014-09-28 13:05:25 +02:00
|
|
|
folder: p.folder,
|
2014-09-27 14:44:15 +02:00
|
|
|
tempName: tempName,
|
|
|
|
realName: realName,
|
2014-11-17 00:18:59 +01:00
|
|
|
copyTotal: uint32(len(blocks)),
|
|
|
|
copyNeeded: uint32(len(blocks)),
|
|
|
|
reused: uint32(reused),
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-05-15 05:26:55 +02:00
|
|
|
if debug {
|
2014-10-12 22:38:22 +02:00
|
|
|
l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused)
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
cs := copyBlocksState{
|
|
|
|
sharedPullerState: &s,
|
|
|
|
blocks: blocks,
|
2014-10-06 10:14:36 +02:00
|
|
|
}
|
2014-10-09 00:41:23 +02:00
|
|
|
copyChan <- cs
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// shortcutFile sets file mode and modification time, when that's the only
|
|
|
|
// thing that has changed.
|
|
|
|
func (p *Puller) shortcutFile(file protocol.FileInfo) {
|
|
|
|
realName := filepath.Join(p.dir, file.Name)
|
2014-10-10 00:34:32 +02:00
|
|
|
if !p.ignorePerms {
|
|
|
|
err := os.Chmod(realName, os.FileMode(file.Flags&0777))
|
|
|
|
if err != nil {
|
|
|
|
l.Infof("Puller (folder %q, file %q): shortcut: %v", p.folder, file.Name, err)
|
|
|
|
return
|
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
t := time.Unix(file.Modified, 0)
|
2014-10-10 00:34:32 +02:00
|
|
|
err := os.Chtimes(realName, t, t)
|
2014-09-27 14:44:15 +02:00
|
|
|
if err != nil {
|
2014-10-14 08:48:35 +02:00
|
|
|
if p.lenientMtimes {
|
|
|
|
// We accept the failure with a warning here and allow the sync to
|
|
|
|
// continue. We'll sync the new mtime back to the other devices later.
|
|
|
|
// If they have the same problem & setting, we might never get in
|
|
|
|
// sync.
|
|
|
|
l.Infof("Puller (folder %q, file %q): shortcut: %v (continuing anyway as requested)", p.folder, file.Name, err)
|
|
|
|
} else {
|
|
|
|
l.Infof("Puller (folder %q, file %q): shortcut: %v", p.folder, file.Name, err)
|
|
|
|
return
|
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
p.model.updateLocal(p.folder, file)
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-11-09 05:26:52 +01:00
|
|
|
// shortcutSymlink changes the symlinks type if necessery.
|
|
|
|
func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
|
|
|
|
err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
|
|
|
|
if err != nil {
|
|
|
|
l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
p.model.updateLocal(p.folder, file)
|
|
|
|
}
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
// copierRoutine reads copierStates until the in channel closes and performs
|
|
|
|
// the relevant copies when possible, or passes it to the puller routine.
|
2014-10-25 00:20:08 +02:00
|
|
|
func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
|
2014-09-30 00:01:17 +02:00
|
|
|
buf := make([]byte, protocol.BlockSize)
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
nextFile:
|
|
|
|
for state := range in {
|
|
|
|
dstFd, err := state.tempFile()
|
|
|
|
if err != nil {
|
|
|
|
// Nothing more to do for this failed file (the error was logged
|
|
|
|
// when it happened)
|
|
|
|
continue nextFile
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-11-17 00:18:59 +01:00
|
|
|
if p.progressEmitter != nil {
|
|
|
|
p.progressEmitter.Register(state.sharedPullerState)
|
|
|
|
}
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
evictionChan := make(chan lfu.Eviction)
|
|
|
|
|
|
|
|
fdCache := lfu.New()
|
|
|
|
fdCache.UpperBound = 50
|
|
|
|
fdCache.LowerBound = 20
|
|
|
|
fdCache.EvictionChannel = evictionChan
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
for item := range evictionChan {
|
|
|
|
item.Value.(*os.File).Close()
|
|
|
|
}
|
|
|
|
}()
|
2014-08-07 21:38:04 +02:00
|
|
|
|
2014-11-09 20:03:56 +01:00
|
|
|
folderRoots := make(map[string]string)
|
|
|
|
p.model.fmut.RLock()
|
|
|
|
for folder, cfg := range p.model.folderCfgs {
|
|
|
|
folderRoots[folder] = cfg.Path
|
|
|
|
}
|
|
|
|
p.model.fmut.RUnlock()
|
|
|
|
|
2014-10-25 00:20:08 +02:00
|
|
|
hasher := sha256.New()
|
2014-09-27 14:44:15 +02:00
|
|
|
for _, block := range state.blocks {
|
|
|
|
buf = buf[:int(block.Size)]
|
2014-10-25 00:20:08 +02:00
|
|
|
found := p.model.finder.Iterate(block.Hash, func(folder, file string, index uint32) bool {
|
2014-11-09 20:03:56 +01:00
|
|
|
path := filepath.Join(folderRoots[folder], file)
|
2014-10-09 00:41:23 +02:00
|
|
|
|
|
|
|
var fd *os.File
|
|
|
|
|
|
|
|
fdi := fdCache.Get(path)
|
|
|
|
if fdi != nil {
|
|
|
|
fd = fdi.(*os.File)
|
|
|
|
} else {
|
|
|
|
fd, err = os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
fdCache.Set(path, fd)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2014-10-25 00:20:08 +02:00
|
|
|
// Only done on second to last puller attempt
|
|
|
|
if checksum {
|
|
|
|
hasher.Write(buf)
|
|
|
|
hash := hasher.Sum(nil)
|
|
|
|
hasher.Reset()
|
|
|
|
if !bytes.Equal(hash, block.Hash) {
|
|
|
|
if debug {
|
|
|
|
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
|
|
|
|
}
|
|
|
|
err = p.model.finder.Fix(folder, file, index, block.Hash, hash)
|
|
|
|
if err != nil {
|
|
|
|
l.Warnln("finder fix:", err)
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-09 00:41:23 +02:00
|
|
|
_, err = dstFd.WriteAt(buf, block.Offset)
|
|
|
|
if err != nil {
|
|
|
|
state.earlyClose("dst write", err)
|
|
|
|
}
|
2014-10-12 22:38:22 +02:00
|
|
|
if file == state.file.Name {
|
|
|
|
state.copiedFromOrigin()
|
|
|
|
}
|
2014-10-09 00:41:23 +02:00
|
|
|
return true
|
|
|
|
})
|
|
|
|
|
|
|
|
if state.failed() != nil {
|
|
|
|
break
|
2014-08-27 07:00:15 +02:00
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
|
2014-10-25 00:20:08 +02:00
|
|
|
if !found {
|
2014-10-09 00:41:23 +02:00
|
|
|
state.pullStarted()
|
|
|
|
ps := pullBlockState{
|
|
|
|
sharedPullerState: state.sharedPullerState,
|
|
|
|
block: block,
|
|
|
|
}
|
|
|
|
pullChan <- ps
|
2014-10-12 22:38:22 +02:00
|
|
|
} else {
|
|
|
|
state.copyDone()
|
2014-05-25 20:49:08 +02:00
|
|
|
}
|
2014-05-19 23:42:08 +02:00
|
|
|
}
|
2014-10-09 00:41:23 +02:00
|
|
|
fdCache.Evict(fdCache.Len())
|
|
|
|
close(evictionChan)
|
2014-09-27 14:44:15 +02:00
|
|
|
out <- state.sharedPullerState
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
func (p *Puller) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
|
|
|
|
nextBlock:
|
|
|
|
for state := range in {
|
|
|
|
if state.failed() != nil {
|
|
|
|
continue nextBlock
|
|
|
|
}
|
2014-07-24 09:38:16 +02:00
|
|
|
|
2014-10-21 13:12:57 +02:00
|
|
|
// Select the least busy device to pull the block from. If we found no
|
2014-09-28 13:00:38 +02:00
|
|
|
// feasible device at all, fail the block (and in the long run, the
|
2014-09-27 14:44:15 +02:00
|
|
|
// file).
|
2014-09-28 13:00:38 +02:00
|
|
|
potentialDevices := p.model.availability(p.folder, state.file.Name)
|
|
|
|
selected := activity.leastBusy(potentialDevices)
|
|
|
|
if selected == (protocol.DeviceID{}) {
|
|
|
|
state.earlyClose("pull", errNoDevice)
|
2014-09-27 14:44:15 +02:00
|
|
|
continue nextBlock
|
|
|
|
}
|
2014-07-24 09:38:16 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Get an fd to the temporary file. Tehcnically we don't need it until
|
|
|
|
// after fetching the block, but if we run into an error here there is
|
|
|
|
// no point in issuing the request to the network.
|
|
|
|
fd, err := state.tempFile()
|
|
|
|
if err != nil {
|
|
|
|
continue nextBlock
|
2014-07-24 09:38:16 +02:00
|
|
|
}
|
2014-08-05 09:46:11 +02:00
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
// Fetch the block, while marking the selected device as in use so that
|
|
|
|
// leastBusy can select another device when someone else asks.
|
2014-09-27 14:44:15 +02:00
|
|
|
activity.using(selected)
|
2014-09-28 13:00:38 +02:00
|
|
|
buf, err := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash)
|
2014-09-27 14:44:15 +02:00
|
|
|
activity.done(selected)
|
|
|
|
if err != nil {
|
|
|
|
state.earlyClose("pull", err)
|
|
|
|
continue nextBlock
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-08-05 09:46:11 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// Save the block data we got from the cluster
|
|
|
|
_, err = fd.WriteAt(buf, state.block.Offset)
|
|
|
|
if err != nil {
|
|
|
|
state.earlyClose("save", err)
|
|
|
|
continue nextBlock
|
|
|
|
}
|
2014-07-24 09:38:16 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
state.pullDone()
|
|
|
|
out <- state.sharedPullerState
|
2014-07-24 09:38:16 +02:00
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
|
2014-11-17 00:18:59 +01:00
|
|
|
func (p *Puller) performFinish(state *sharedPullerState) {
|
2014-11-29 23:18:56 +01:00
|
|
|
// Verify the file against expected hashes
|
|
|
|
fd, err := os.Open(state.tempName)
|
|
|
|
if err != nil {
|
|
|
|
l.Warnln("puller: final:", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
err = scanner.Verify(fd, protocol.BlockSize, state.file.Blocks)
|
|
|
|
fd.Close()
|
|
|
|
if err != nil {
|
|
|
|
l.Infoln("puller:", state.file.Name, err, "(file changed during pull?)")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the correct permission bits on the new file
|
|
|
|
if !p.ignorePerms {
|
|
|
|
err = os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777))
|
2014-11-17 00:18:59 +01:00
|
|
|
if err != nil {
|
|
|
|
l.Warnln("puller: final:", err)
|
|
|
|
return
|
|
|
|
}
|
2014-11-29 23:18:56 +01:00
|
|
|
}
|
2014-11-17 00:18:59 +01:00
|
|
|
|
2014-11-29 23:18:56 +01:00
|
|
|
// Set the correct timestamp on the new file
|
|
|
|
t := time.Unix(state.file.Modified, 0)
|
|
|
|
err = os.Chtimes(state.tempName, t, t)
|
|
|
|
if err != nil {
|
|
|
|
if p.lenientMtimes {
|
|
|
|
// We accept the failure with a warning here and allow the sync to
|
|
|
|
// continue. We'll sync the new mtime back to the other devices later.
|
|
|
|
// If they have the same problem & setting, we might never get in
|
|
|
|
// sync.
|
|
|
|
l.Infof("Puller (folder %q, file %q): final: %v (continuing anyway as requested)", p.folder, state.file.Name, err)
|
|
|
|
} else {
|
2014-11-17 00:18:59 +01:00
|
|
|
l.Warnln("puller: final:", err)
|
|
|
|
return
|
|
|
|
}
|
2014-11-29 23:18:56 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// If we should use versioning, let the versioner archive the old
|
|
|
|
// file before we replace it. Archiving a non-existent file is not
|
|
|
|
// an error.
|
|
|
|
if p.versioner != nil {
|
|
|
|
err = p.versioner.Archive(state.realName)
|
2014-11-17 00:18:59 +01:00
|
|
|
if err != nil {
|
2014-11-29 23:18:56 +01:00
|
|
|
l.Warnln("puller: final:", err)
|
2014-11-17 00:18:59 +01:00
|
|
|
return
|
|
|
|
}
|
2014-11-29 23:18:56 +01:00
|
|
|
}
|
2014-11-17 00:18:59 +01:00
|
|
|
|
2014-11-29 23:18:56 +01:00
|
|
|
// If the target path is a symlink or a directory, we cannot copy
|
|
|
|
// over it, hence remove it before proceeding.
|
|
|
|
stat, err := os.Lstat(state.realName)
|
|
|
|
isLink, _ := symlinks.IsSymlink(state.realName)
|
|
|
|
if isLink || (err == nil && stat.IsDir()) {
|
|
|
|
osutil.InWritableDir(os.Remove, state.realName)
|
|
|
|
}
|
|
|
|
// Replace the original content with the new one
|
|
|
|
err = osutil.Rename(state.tempName, state.realName)
|
|
|
|
if err != nil {
|
|
|
|
l.Warnln("puller: final:", err)
|
|
|
|
return
|
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
|
2014-11-29 23:18:56 +01:00
|
|
|
// If it's a symlink, the target of the symlink is inside the file.
|
|
|
|
if state.file.IsSymlink() {
|
|
|
|
content, err := ioutil.ReadFile(state.realName)
|
2014-11-17 00:18:59 +01:00
|
|
|
if err != nil {
|
2014-11-29 23:18:56 +01:00
|
|
|
l.Warnln("puller: final: reading symlink:", err)
|
|
|
|
return
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
|
2014-11-29 23:18:56 +01:00
|
|
|
// Remove the file, and replace it with a symlink.
|
|
|
|
err = osutil.InWritableDir(func(path string) error {
|
|
|
|
os.Remove(path)
|
|
|
|
return symlinks.Create(path, string(content), state.file.Flags)
|
|
|
|
}, state.realName)
|
2014-11-17 00:18:59 +01:00
|
|
|
if err != nil {
|
2014-11-29 23:18:56 +01:00
|
|
|
l.Warnln("puller: final: creating symlink:", err)
|
2014-11-17 00:18:59 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2014-11-29 23:18:56 +01:00
|
|
|
|
|
|
|
// Record the updated file in the index
|
|
|
|
p.model.updateLocal(p.folder, state.file)
|
2014-11-17 00:18:59 +01:00
|
|
|
}
|
2014-11-09 05:26:52 +01:00
|
|
|
|
2014-11-17 00:18:59 +01:00
|
|
|
func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
|
|
|
|
for state := range in {
|
2014-11-29 23:18:56 +01:00
|
|
|
if closed, err := state.finalClose(); closed {
|
|
|
|
if debug {
|
|
|
|
l.Debugln(p, "closing", state.file.Name)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
l.Warnln("puller: final:", err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
p.performFinish(state)
|
|
|
|
if p.progressEmitter != nil {
|
|
|
|
p.progressEmitter.Deregister(state)
|
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
}
|
|
|
|
}
|
2014-09-27 14:44:15 +02:00
|
|
|
}
|
2014-04-27 12:14:53 +02:00
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
// clean deletes orphaned temporary files
|
|
|
|
func (p *Puller) clean() {
|
2014-10-06 09:25:45 +02:00
|
|
|
keep := time.Duration(p.model.cfg.Options().KeepTemporariesH) * time.Hour
|
2014-10-04 00:15:54 +02:00
|
|
|
now := time.Now()
|
2014-09-27 14:44:15 +02:00
|
|
|
filepath.Walk(p.dir, func(path string, info os.FileInfo, err error) error {
|
2014-08-04 22:02:44 +02:00
|
|
|
if err != nil {
|
2014-09-27 14:44:15 +02:00
|
|
|
return err
|
2014-05-23 14:31:16 +02:00
|
|
|
}
|
2014-05-25 20:49:08 +02:00
|
|
|
|
2014-10-04 00:15:54 +02:00
|
|
|
if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) && info.ModTime().Add(keep).Before(now) {
|
2014-09-27 14:44:15 +02:00
|
|
|
os.Remove(path)
|
2014-05-25 20:49:08 +02:00
|
|
|
}
|
|
|
|
|
2014-09-27 14:44:15 +02:00
|
|
|
return nil
|
|
|
|
})
|
2014-04-27 12:14:53 +02:00
|
|
|
}
|
2014-05-15 02:18:09 +02:00
|
|
|
|
2014-09-28 13:00:38 +02:00
|
|
|
func invalidateFolder(cfg *config.Configuration, folderID string, err error) {
|
|
|
|
for i := range cfg.Folders {
|
|
|
|
folder := &cfg.Folders[i]
|
|
|
|
if folder.ID == folderID {
|
|
|
|
folder.Invalid = err.Error()
|
2014-05-15 02:18:09 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|