557 lines
15 KiB
Go
557 lines
15 KiB
Go
package suture
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
notRunning = iota
|
|
normal
|
|
paused
|
|
)
|
|
|
|
type supervisorID uint32
|
|
type serviceID uint32
|
|
|
|
var currentSupervisorIDL sync.Mutex
|
|
var currentSupervisorID uint32
|
|
|
|
// ErrWrongSupervisor is returned by the (*Supervisor).Remove method
|
|
// if you pass a ServiceToken from the wrong Supervisor.
|
|
var ErrWrongSupervisor = errors.New("wrong supervisor for this service token, no service removed")
|
|
|
|
// ServiceToken is an opaque identifier that can be used to terminate a service that
|
|
// has been Add()ed to a Supervisor.
|
|
type ServiceToken struct {
|
|
id uint64
|
|
}
|
|
|
|
type serviceWithName struct {
|
|
Service Service
|
|
name string
|
|
}
|
|
|
|
/*
|
|
Supervisor is the core type of the module that represents a Supervisor.
|
|
|
|
Supervisors should be constructed either by New or NewSimple.
|
|
|
|
Once constructed, a Supervisor should be started in one of three ways:
|
|
|
|
1. Calling .Serve().
|
|
2. Calling .ServeBackground().
|
|
3. Adding it to an existing Supervisor.
|
|
|
|
Calling Serve will cause the supervisor to run until it is shut down by
|
|
an external user calling Stop() on it. If that never happens, it simply
|
|
runs forever. I suggest creating your services in Supervisors, then making
|
|
a Serve() call on your top-level Supervisor be the last line of your main
|
|
func.
|
|
|
|
Calling ServeBackground will CORRECTLY start the supervisor running in a
|
|
new goroutine. You do not want to just:
|
|
|
|
go supervisor.Serve()
|
|
|
|
because that will briefly create a race condition as it starts up, if you
|
|
try to .Add() services immediately afterward.
|
|
|
|
*/
|
|
type Supervisor struct {
|
|
Name string
|
|
id supervisorID
|
|
|
|
failureDecay float64
|
|
failureThreshold float64
|
|
failureBackoff time.Duration
|
|
timeout time.Duration
|
|
log func(string)
|
|
services map[serviceID]serviceWithName
|
|
servicesShuttingDown map[serviceID]serviceWithName
|
|
lastFail time.Time
|
|
failures float64
|
|
restartQueue []serviceID
|
|
serviceCounter serviceID
|
|
control chan supervisorMessage
|
|
liveness chan struct{}
|
|
resumeTimer <-chan time.Time
|
|
|
|
// The testing uses the ability to grab these individual logging functions
|
|
// and get inside of suture's handling at a deep level.
|
|
// If you ever come up with some need to get into these, submit a pull
|
|
// request to make them public and some smidge of justification, and
|
|
// I'll happily do it.
|
|
// But since I've now changed the signature on these once, I'm glad I
|
|
// didn't start with them public... :)
|
|
logBadStop func(*Supervisor, Service, string)
|
|
logFailure func(supervisor *Supervisor, service Service, serviceName string, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
|
|
logBackoff func(*Supervisor, bool)
|
|
|
|
// avoid a dependency on github.com/thejerf/abtime by just implementing
|
|
// a minimal chunk.
|
|
getNow func() time.Time
|
|
getAfterChan func(time.Duration) <-chan time.Time
|
|
|
|
sync.Mutex
|
|
state uint8
|
|
}
|
|
|
|
// Spec is used to pass arguments to the New function to create a
|
|
// supervisor. See the New function for full documentation.
|
|
type Spec struct {
|
|
Log func(string)
|
|
FailureDecay float64
|
|
FailureThreshold float64
|
|
FailureBackoff time.Duration
|
|
Timeout time.Duration
|
|
}
|
|
|
|
/*
|
|
|
|
New is the full constructor function for a supervisor.
|
|
|
|
The name is a friendly human name for the supervisor, used in logging. Suture
|
|
does not care if this is unique, but it is good for your sanity if it is.
|
|
|
|
If not set, the following values are used:
|
|
|
|
* Log: A function is created that uses log.Print.
|
|
* FailureDecay: 30 seconds
|
|
* FailureThreshold: 5 failures
|
|
* FailureBackoff: 15 seconds
|
|
* Timeout: 10 seconds
|
|
|
|
The Log function will be called when errors occur. Suture will log the
|
|
following:
|
|
|
|
* When a service has failed, with a descriptive message about the
|
|
current backoff status, and whether it was immediately restarted
|
|
* When the supervisor has gone into its backoff mode, and when it
|
|
exits it
|
|
* When a service fails to stop
|
|
|
|
The failureRate, failureThreshold, and failureBackoff controls how failures
|
|
are handled, in order to avoid the supervisor failure case where the
|
|
program does nothing but restarting failed services. If you do not
|
|
care how failures behave, the default values should be fine for the
|
|
vast majority of services, but if you want the details:
|
|
|
|
The supervisor tracks the number of failures that have occurred, with an
|
|
exponential decay on the count. Every FailureDecay seconds, the number of
|
|
failures that have occurred is cut in half. (This is done smoothly with an
|
|
exponential function.) When a failure occurs, the number of failures
|
|
is incremented by one. When the number of failures passes the
|
|
FailureThreshold, the entire service waits for FailureBackoff seconds
|
|
before attempting any further restarts, at which point it resets its
|
|
failure count to zero.
|
|
|
|
Timeout is how long Suture will wait for a service to properly terminate.
|
|
|
|
*/
|
|
func New(name string, spec Spec) (s *Supervisor) {
|
|
s = new(Supervisor)
|
|
|
|
s.Name = name
|
|
currentSupervisorIDL.Lock()
|
|
currentSupervisorID++
|
|
s.id = supervisorID(currentSupervisorID)
|
|
currentSupervisorIDL.Unlock()
|
|
|
|
if spec.Log == nil {
|
|
s.log = func(msg string) {
|
|
log.Print(fmt.Sprintf("Supervisor %s: %s", s.Name, msg))
|
|
}
|
|
} else {
|
|
s.log = spec.Log
|
|
}
|
|
|
|
if spec.FailureDecay == 0 {
|
|
s.failureDecay = 30
|
|
} else {
|
|
s.failureDecay = spec.FailureDecay
|
|
}
|
|
if spec.FailureThreshold == 0 {
|
|
s.failureThreshold = 5
|
|
} else {
|
|
s.failureThreshold = spec.FailureThreshold
|
|
}
|
|
if spec.FailureBackoff == 0 {
|
|
s.failureBackoff = time.Second * 15
|
|
} else {
|
|
s.failureBackoff = spec.FailureBackoff
|
|
}
|
|
if spec.Timeout == 0 {
|
|
s.timeout = time.Second * 10
|
|
} else {
|
|
s.timeout = spec.Timeout
|
|
}
|
|
|
|
// overriding these allows for testing the threshold behavior
|
|
s.getNow = time.Now
|
|
s.getAfterChan = time.After
|
|
|
|
s.control = make(chan supervisorMessage)
|
|
s.liveness = make(chan struct{})
|
|
s.services = make(map[serviceID]serviceWithName)
|
|
s.servicesShuttingDown = make(map[serviceID]serviceWithName)
|
|
s.restartQueue = make([]serviceID, 0, 1)
|
|
s.resumeTimer = make(chan time.Time)
|
|
|
|
// set up the default logging handlers
|
|
s.logBadStop = func(supervisor *Supervisor, service Service, name string) {
|
|
s.log(fmt.Sprintf("%s: Service %s failed to terminate in a timely manner", supervisor.Name, name))
|
|
}
|
|
s.logFailure = func(supervisor *Supervisor, service Service, serviceName string, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
|
|
var errString string
|
|
|
|
e, canError := err.(error)
|
|
if canError {
|
|
errString = e.Error()
|
|
} else {
|
|
errString = fmt.Sprintf("%#v", err)
|
|
}
|
|
|
|
s.log(fmt.Sprintf("%s: Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", supervisor.Name, serviceName, failures, threshold, restarting, errString, string(st)))
|
|
}
|
|
s.logBackoff = func(s *Supervisor, entering bool) {
|
|
if entering {
|
|
s.log("Entering the backoff state.")
|
|
} else {
|
|
s.log("Exiting backoff state.")
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func serviceName(service Service) (serviceName string) {
|
|
stringer, canStringer := service.(fmt.Stringer)
|
|
if canStringer {
|
|
serviceName = stringer.String()
|
|
} else {
|
|
serviceName = fmt.Sprintf("%#v", service)
|
|
}
|
|
return
|
|
}
|
|
|
|
// NewSimple is a convenience function to create a service with just a name
|
|
// and the sensible defaults.
|
|
func NewSimple(name string) *Supervisor {
|
|
return New(name, Spec{})
|
|
}
|
|
|
|
/*
|
|
Add adds a service to this supervisor.
|
|
|
|
If the supervisor is currently running, the service will be started
|
|
immediately. If the supervisor is not currently running, the service
|
|
will be started when the supervisor is.
|
|
|
|
The returned ServiceID may be passed to the Remove method of the Supervisor
|
|
to terminate the service.
|
|
|
|
As a special behavior, if the service added is itself a supervisor, the
|
|
supervisor being added will copy the Log function from the Supervisor it
|
|
is being added to. This allows factoring out providing a Supervisor
|
|
from its logging.
|
|
|
|
*/
|
|
func (s *Supervisor) Add(service Service) ServiceToken {
|
|
if s == nil {
|
|
panic("can't add service to nil *suture.Supervisor")
|
|
}
|
|
|
|
if supervisor, isSupervisor := service.(*Supervisor); isSupervisor {
|
|
supervisor.logBadStop = s.logBadStop
|
|
supervisor.logFailure = s.logFailure
|
|
supervisor.logBackoff = s.logBackoff
|
|
}
|
|
|
|
s.Lock()
|
|
if s.state == notRunning {
|
|
id := s.serviceCounter
|
|
s.serviceCounter++
|
|
|
|
s.services[id] = serviceWithName{service, serviceName(service)}
|
|
s.restartQueue = append(s.restartQueue, id)
|
|
|
|
s.Unlock()
|
|
return ServiceToken{uint64(s.id)<<32 | uint64(id)}
|
|
}
|
|
s.Unlock()
|
|
|
|
response := make(chan serviceID)
|
|
s.control <- addService{service, serviceName(service), response}
|
|
return ServiceToken{uint64(s.id)<<32 | uint64(<-response)}
|
|
}
|
|
|
|
// ServeBackground starts running a supervisor in its own goroutine. This
|
|
// method does not return until it is safe to use .Add() on the Supervisor.
|
|
func (s *Supervisor) ServeBackground() {
|
|
go s.Serve()
|
|
s.sync()
|
|
}
|
|
|
|
/*
|
|
Serve starts the supervisor. You should call this on the top-level supervisor,
|
|
but nothing else.
|
|
*/
|
|
func (s *Supervisor) Serve() {
|
|
if s == nil {
|
|
panic("Can't serve with a nil *suture.Supervisor")
|
|
}
|
|
if s.id == 0 {
|
|
panic("Can't call Serve on an incorrectly-constructed *suture.Supervisor")
|
|
}
|
|
|
|
defer func() {
|
|
s.Lock()
|
|
s.state = notRunning
|
|
s.Unlock()
|
|
}()
|
|
|
|
s.Lock()
|
|
if s.state != notRunning {
|
|
s.Unlock()
|
|
panic("Running a supervisor while it is already running?")
|
|
}
|
|
|
|
s.state = normal
|
|
s.Unlock()
|
|
|
|
// for all the services I currently know about, start them
|
|
for _, id := range s.restartQueue {
|
|
namedService, present := s.services[id]
|
|
if present {
|
|
s.runService(namedService.Service, id)
|
|
}
|
|
}
|
|
s.restartQueue = make([]serviceID, 0, 1)
|
|
|
|
for {
|
|
select {
|
|
case m := <-s.control:
|
|
switch msg := m.(type) {
|
|
case serviceFailed:
|
|
s.handleFailedService(msg.id, msg.err, msg.stacktrace)
|
|
case serviceEnded:
|
|
service, monitored := s.services[msg.id]
|
|
if monitored {
|
|
s.handleFailedService(msg.id, fmt.Sprintf("%s returned unexpectedly", service), []byte("[unknown stack trace]"))
|
|
}
|
|
case addService:
|
|
id := s.serviceCounter
|
|
s.serviceCounter++
|
|
|
|
s.services[id] = serviceWithName{msg.service, msg.name}
|
|
s.runService(msg.service, id)
|
|
|
|
msg.response <- id
|
|
case removeService:
|
|
s.removeService(msg.id, s.control)
|
|
case serviceTerminated:
|
|
delete(s.servicesShuttingDown, msg.id)
|
|
case stopSupervisor:
|
|
s.stopSupervisor()
|
|
msg.done <- struct{}{}
|
|
return
|
|
case listServices:
|
|
services := []Service{}
|
|
for _, service := range s.services {
|
|
services = append(services, service.Service)
|
|
}
|
|
msg.c <- services
|
|
case syncSupervisor:
|
|
// this does nothing on purpose; its sole purpose is to
|
|
// introduce a sync point via the channel receive
|
|
case panicSupervisor:
|
|
// used only by tests
|
|
panic("Panicking as requested!")
|
|
}
|
|
case _ = <-s.resumeTimer:
|
|
// We're resuming normal operation after a pause due to
|
|
// excessive thrashing
|
|
// FIXME: Ought to permit some spacing of these functions, rather
|
|
// than simply hammering through them
|
|
s.Lock()
|
|
s.state = normal
|
|
s.Unlock()
|
|
s.failures = 0
|
|
s.logBackoff(s, false)
|
|
for _, id := range s.restartQueue {
|
|
namedService, present := s.services[id]
|
|
if present {
|
|
s.runService(namedService.Service, id)
|
|
}
|
|
}
|
|
s.restartQueue = make([]serviceID, 0, 1)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktrace []byte) {
|
|
now := s.getNow()
|
|
|
|
if s.lastFail.IsZero() {
|
|
s.lastFail = now
|
|
s.failures = 1.0
|
|
} else {
|
|
sinceLastFail := now.Sub(s.lastFail).Seconds()
|
|
intervals := sinceLastFail / s.failureDecay
|
|
s.failures = s.failures*math.Pow(.5, intervals) + 1
|
|
}
|
|
|
|
if s.failures > s.failureThreshold {
|
|
s.Lock()
|
|
s.state = paused
|
|
s.Unlock()
|
|
s.logBackoff(s, true)
|
|
s.resumeTimer = s.getAfterChan(s.failureBackoff)
|
|
}
|
|
|
|
s.lastFail = now
|
|
|
|
failedService, monitored := s.services[id]
|
|
|
|
// It is possible for a service to be no longer monitored
|
|
// by the time we get here. In that case, just ignore it.
|
|
if monitored {
|
|
// this may look dangerous because the state could change, but this
|
|
// code is only ever run in the one goroutine that is permitted to
|
|
// change the state, so nothing else will.
|
|
s.Lock()
|
|
curState := s.state
|
|
s.Unlock()
|
|
if curState == normal {
|
|
s.runService(failedService.Service, id)
|
|
s.logFailure(s, failedService.Service, failedService.name, s.failures, s.failureThreshold, true, err, stacktrace)
|
|
} else {
|
|
// FIXME: When restarting, check that the service still
|
|
// exists (it may have been stopped in the meantime)
|
|
s.restartQueue = append(s.restartQueue, id)
|
|
s.logFailure(s, failedService.Service, failedService.name, s.failures, s.failureThreshold, false, err, stacktrace)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Supervisor) runService(service Service, id serviceID) {
|
|
go func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
buf := make([]byte, 65535, 65535)
|
|
written := runtime.Stack(buf, false)
|
|
buf = buf[:written]
|
|
s.fail(id, r, buf)
|
|
}
|
|
}()
|
|
|
|
service.Serve()
|
|
|
|
s.serviceEnded(id)
|
|
}()
|
|
}
|
|
|
|
func (s *Supervisor) removeService(id serviceID, removedChan chan supervisorMessage) {
|
|
namedService, present := s.services[id]
|
|
if present {
|
|
delete(s.services, id)
|
|
s.servicesShuttingDown[id] = namedService
|
|
go func() {
|
|
successChan := make(chan bool)
|
|
go func() {
|
|
namedService.Service.Stop()
|
|
successChan <- true
|
|
}()
|
|
|
|
select {
|
|
case <-successChan:
|
|
// Life is good!
|
|
case <-s.getAfterChan(s.timeout):
|
|
s.logBadStop(s, namedService.Service, namedService.name)
|
|
}
|
|
removedChan <- serviceTerminated{id}
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (s *Supervisor) stopSupervisor() {
|
|
notifyDone := make(chan serviceID)
|
|
|
|
for id := range s.services {
|
|
namedService, present := s.services[id]
|
|
if present {
|
|
delete(s.services, id)
|
|
s.servicesShuttingDown[id] = namedService
|
|
go func(sID serviceID) {
|
|
namedService.Service.Stop()
|
|
notifyDone <- sID
|
|
}(id)
|
|
}
|
|
}
|
|
|
|
timeout := s.getAfterChan(s.timeout)
|
|
for len(s.servicesShuttingDown) > 0 {
|
|
select {
|
|
case id := <-notifyDone:
|
|
delete(s.servicesShuttingDown, id)
|
|
case <-timeout:
|
|
for _, namedService := range s.servicesShuttingDown {
|
|
s.logBadStop(s, namedService.Service, namedService.name)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
close(s.liveness)
|
|
return
|
|
}
|
|
|
|
// String implements the fmt.Stringer interface.
|
|
func (s *Supervisor) String() string {
|
|
return s.Name
|
|
}
|
|
|
|
func (s *Supervisor) sendControl(sm supervisorMessage) bool {
|
|
select {
|
|
case s.control <- sm:
|
|
return true
|
|
case _, _ = (<-s.liveness):
|
|
return false
|
|
}
|
|
}
|
|
|
|
/*
|
|
Remove will remove the given service from the Supervisor, and attempt to Stop() it.
|
|
The ServiceID token comes from the Add() call.
|
|
*/
|
|
func (s *Supervisor) Remove(id ServiceToken) error {
|
|
sID := supervisorID(id.id >> 32)
|
|
if sID != s.id {
|
|
return ErrWrongSupervisor
|
|
}
|
|
s.sendControl(removeService{serviceID(id.id & 0xffffffff)})
|
|
return nil
|
|
}
|
|
|
|
/*
|
|
|
|
Services returns a []Service containing a snapshot of the services this
|
|
Supervisor is managing.
|
|
|
|
*/
|
|
func (s *Supervisor) Services() []Service {
|
|
ls := listServices{make(chan []Service)}
|
|
|
|
if s.sendControl(ls) {
|
|
return <-ls.c
|
|
} else {
|
|
return nil
|
|
}
|
|
}
|