265 lines
5.8 KiB
Go
265 lines
5.8 KiB
Go
// Copyright (C) 2014 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"runtime"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/syncthing/syncthing/lib/osutil"
|
|
"github.com/syncthing/syncthing/lib/sync"
|
|
)
|
|
|
|
var (
|
|
stdoutFirstLines []string // The first 10 lines of stdout
|
|
stdoutLastLines []string // The last 50 lines of stdout
|
|
stdoutMut = sync.NewMutex()
|
|
)
|
|
|
|
const (
|
|
countRestarts = 4
|
|
loopThreshold = 60 * time.Second
|
|
)
|
|
|
|
func monitorMain() {
|
|
os.Setenv("STNORESTART", "yes")
|
|
os.Setenv("STMONITORED", "yes")
|
|
l.SetPrefix("[monitor] ")
|
|
|
|
var err error
|
|
var dst io.Writer = os.Stdout
|
|
|
|
if logFile != "-" {
|
|
var fileDst io.Writer
|
|
|
|
fileDst, err = os.Create(logFile)
|
|
if err != nil {
|
|
l.Fatalln("log file:", err)
|
|
}
|
|
|
|
if runtime.GOOS == "windows" {
|
|
// Translate line breaks to Windows standard
|
|
fileDst = osutil.ReplacingWriter{
|
|
Writer: fileDst,
|
|
From: '\n',
|
|
To: []byte{'\r', '\n'},
|
|
}
|
|
}
|
|
|
|
// Log to both stdout and file.
|
|
dst = io.MultiWriter(dst, fileDst)
|
|
|
|
l.Infof(`Log output saved to file "%s"`, logFile)
|
|
}
|
|
|
|
args := os.Args
|
|
var restarts [countRestarts]time.Time
|
|
|
|
sign := make(chan os.Signal, 1)
|
|
sigTerm := syscall.Signal(0xf)
|
|
signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)
|
|
|
|
for {
|
|
if t := time.Since(restarts[0]); t < loopThreshold {
|
|
l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
|
|
os.Exit(exitError)
|
|
}
|
|
|
|
copy(restarts[0:], restarts[1:])
|
|
restarts[len(restarts)-1] = time.Now()
|
|
|
|
cmd := exec.Command(args[0], args[1:]...)
|
|
|
|
stderr, err := cmd.StderrPipe()
|
|
if err != nil {
|
|
l.Fatalln("stderr:", err)
|
|
}
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
l.Fatalln("stdout:", err)
|
|
}
|
|
|
|
l.Infoln("Starting syncthing")
|
|
err = cmd.Start()
|
|
if err != nil {
|
|
l.Fatalln(err)
|
|
}
|
|
|
|
// Let the next child process know that this is not the first time
|
|
// it's starting up.
|
|
os.Setenv("STRESTART", "yes")
|
|
|
|
stdoutMut.Lock()
|
|
stdoutFirstLines = make([]string, 0, 10)
|
|
stdoutLastLines = make([]string, 0, 50)
|
|
stdoutMut.Unlock()
|
|
|
|
wg := sync.NewWaitGroup()
|
|
|
|
wg.Add(1)
|
|
go func() {
|
|
copyStderr(stderr, dst)
|
|
wg.Done()
|
|
}()
|
|
|
|
wg.Add(1)
|
|
go func() {
|
|
copyStdout(stdout, dst)
|
|
wg.Done()
|
|
}()
|
|
|
|
exit := make(chan error)
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
exit <- cmd.Wait()
|
|
}()
|
|
|
|
select {
|
|
case s := <-sign:
|
|
l.Infof("Signal %d received; exiting", s)
|
|
cmd.Process.Kill()
|
|
<-exit
|
|
return
|
|
|
|
case err = <-exit:
|
|
if err == nil {
|
|
// Successful exit indicates an intentional shutdown
|
|
return
|
|
} else if exiterr, ok := err.(*exec.ExitError); ok {
|
|
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
|
switch status.ExitStatus() {
|
|
case exitUpgrading:
|
|
// Restart the monitor process to release the .old
|
|
// binary as part of the upgrade process.
|
|
l.Infoln("Restarting monitor...")
|
|
os.Setenv("STNORESTART", "")
|
|
if err = restartMonitor(args); err != nil {
|
|
l.Warnln("Restart:", err)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
l.Infoln("Syncthing exited:", err)
|
|
time.Sleep(1 * time.Second)
|
|
}
|
|
}
|
|
|
|
func copyStderr(stderr io.Reader, dst io.Writer) {
|
|
br := bufio.NewReader(stderr)
|
|
|
|
var panicFd *os.File
|
|
for {
|
|
line, err := br.ReadString('\n')
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if panicFd == nil {
|
|
dst.Write([]byte(line))
|
|
|
|
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
|
|
panicFd, err = os.Create(timestampedLoc(locPanicLog))
|
|
if err != nil {
|
|
l.Warnln("Create panic log:", err)
|
|
continue
|
|
}
|
|
|
|
l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
|
|
l.Warnln("Please check for existing issues with similar panic message at https://github.com/syncthing/syncthing/issues/")
|
|
l.Warnln("If no issue with similar panic message exists, please create a new issue with the panic log attached")
|
|
|
|
stdoutMut.Lock()
|
|
for _, line := range stdoutFirstLines {
|
|
panicFd.WriteString(line)
|
|
}
|
|
panicFd.WriteString("...\n")
|
|
for _, line := range stdoutLastLines {
|
|
panicFd.WriteString(line)
|
|
}
|
|
stdoutMut.Unlock()
|
|
}
|
|
|
|
panicFd.WriteString("Panic at " + time.Now().Format(time.RFC3339) + "\n")
|
|
}
|
|
|
|
if panicFd != nil {
|
|
panicFd.WriteString(line)
|
|
}
|
|
}
|
|
}
|
|
|
|
func copyStdout(stdout io.Reader, dst io.Writer) {
|
|
br := bufio.NewReader(stdout)
|
|
for {
|
|
line, err := br.ReadString('\n')
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
stdoutMut.Lock()
|
|
if len(stdoutFirstLines) < cap(stdoutFirstLines) {
|
|
stdoutFirstLines = append(stdoutFirstLines, line)
|
|
} else {
|
|
if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
|
|
stdoutLastLines = stdoutLastLines[:l-1]
|
|
}
|
|
stdoutLastLines = append(stdoutLastLines, line)
|
|
}
|
|
stdoutMut.Unlock()
|
|
|
|
dst.Write([]byte(line))
|
|
}
|
|
}
|
|
|
|
func restartMonitor(args []string) error {
|
|
if runtime.GOOS != "windows" {
|
|
// syscall.Exec is the cleanest way to restart on Unixes as it
|
|
// replaces the current process with the new one, keeping the pid and
|
|
// controlling terminal and so on
|
|
return restartMonitorUnix(args)
|
|
}
|
|
|
|
// but it isn't supported on Windows, so there we start a normal
|
|
// exec.Command and return.
|
|
return restartMonitorWindows(args)
|
|
}
|
|
|
|
func restartMonitorUnix(args []string) error {
|
|
if !strings.ContainsRune(args[0], os.PathSeparator) {
|
|
// The path to the binary doesn't contain a slash, so it should be
|
|
// found in $PATH.
|
|
binary, err := exec.LookPath(args[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
args[0] = binary
|
|
}
|
|
|
|
return syscall.Exec(args[0], args, os.Environ())
|
|
}
|
|
|
|
func restartMonitorWindows(args []string) error {
|
|
cmd := exec.Command(args[0], args[1:]...)
|
|
// Retain the standard streams
|
|
cmd.Stderr = os.Stderr
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stdin = os.Stdin
|
|
return cmd.Start()
|
|
}
|