syncthing/main.go
2014-01-13 10:29:23 -07:00

455 lines
12 KiB
Go

package main
import (
"compress/gzip"
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
_ "net/http/pprof"
"os"
"path"
"runtime"
"runtime/debug"
"strconv"
"strings"
"time"
"github.com/calmh/ini"
"github.com/calmh/syncthing/discover"
flags "github.com/calmh/syncthing/github.com/jessevdk/go-flags"
"github.com/calmh/syncthing/model"
"github.com/calmh/syncthing/protocol"
)
type Options struct {
ConfDir string `short:"c" long:"cfg" description:"Configuration directory" default:"~/.syncthing" value-name:"DIR"`
Listen string `short:"l" long:"listen" description:"Listen address" default:":22000" value-name:"ADDR"`
ReadOnly bool `short:"r" long:"ro" description:"Repository is read only"`
Rehash bool `long:"rehash" description:"Ignore cache and rehash all files in repository"`
NoDelete bool `long:"no-delete" description:"Never delete files"`
NoSymlinks bool `long:"no-symlinks" description:"Don't follow first level symlinks in the repo"`
NoStats bool `long:"no-stats" description:"Don't print model and connection statistics"`
NoGUI bool `long:"no-gui" description:"Don't start GUI"`
GUIAddr string `long:"gui-addr" description:"GUI listen address" default:"127.0.0.1:8080" value-name:"ADDR"`
ShowVersion bool `short:"v" long:"version" description:"Show version"`
Discovery DiscoveryOptions `group:"Discovery Options"`
Advanced AdvancedOptions `group:"Advanced Options"`
Debug DebugOptions `group:"Debugging Options"`
}
type DebugOptions struct {
LogSource bool `long:"log-source"`
TraceModel []string `long:"trace-model" value-name:"TRACE" description:"idx, net, file, need, pull"`
TraceConnect bool `long:"trace-connect"`
Profiler string `long:"profiler" value-name:"ADDR"`
}
type DiscoveryOptions struct {
ExternalServer string `long:"ext-server" description:"External discovery server" value-name:"NAME" default:"syncthing.nym.se"`
ExternalPort int `short:"e" long:"ext-port" description:"External listen port" value-name:"PORT" default:"22000"`
NoExternalDiscovery bool `short:"n" long:"no-ext-announce" description:"Do not announce presence externally"`
NoLocalDiscovery bool `short:"N" long:"no-local-announce" description:"Do not announce presence locally"`
}
type AdvancedOptions struct {
RequestsInFlight int `long:"reqs-in-flight" description:"Parallell in flight requests per node" default:"8" value-name:"REQS"`
LimitRate int `long:"send-rate" description:"Rate limit for outgoing data" default:"0" value-name:"KBPS"`
ScanInterval time.Duration `long:"scan-intv" description:"Repository scan interval" default:"60s" value-name:"INTV"`
ConnInterval time.Duration `long:"conn-intv" description:"Node reconnect interval" default:"60s" value-name:"INTV"`
}
var opts Options
var Version string = "unknown-dev"
const (
confFileName = "syncthing.ini"
)
var (
myID string
config ini.Config
nodeAddrs = make(map[string][]string)
)
func main() {
_, err := flags.Parse(&opts)
if err != nil {
if err, ok := err.(*flags.Error); ok {
if err.Type == flags.ErrHelp {
os.Exit(0)
}
}
fatalln(err)
}
if opts.ShowVersion {
fmt.Println(Version)
os.Exit(0)
}
if len(os.Getenv("GOGC")) == 0 {
debug.SetGCPercent(25)
}
if len(os.Getenv("GOMAXPROCS")) == 0 {
runtime.GOMAXPROCS(runtime.NumCPU())
}
log.SetOutput(os.Stderr)
logger = log.New(os.Stderr, "", log.Flags())
if len(opts.Debug.TraceModel) > 0 || opts.Debug.LogSource {
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
logger.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
}
opts.ConfDir = expandTilde(opts.ConfDir)
infoln("Version", Version)
// Ensure that our home directory exists and that we have a certificate and key.
ensureDir(opts.ConfDir, 0700)
cert, err := loadCert(opts.ConfDir)
if err != nil {
newCertificate(opts.ConfDir)
cert, err = loadCert(opts.ConfDir)
fatalErr(err)
}
myID = string(certId(cert.Certificate[0]))
infoln("My ID:", myID)
if opts.Debug.Profiler != "" {
go func() {
err := http.ListenAndServe(opts.Debug.Profiler, nil)
if err != nil {
warnln(err)
}
}()
}
// The TLS configuration is used for both the listening socket and outgoing
// connections.
cfg := &tls.Config{
Certificates: []tls.Certificate{cert},
NextProtos: []string{"bep/1.0"},
ServerName: myID,
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS12,
}
// Load the configuration file, if it exists.
cf, err := os.Open(path.Join(opts.ConfDir, confFileName))
if err != nil {
fatalln("No config file")
config = ini.Config{}
}
config = ini.Parse(cf)
cf.Close()
var dir = expandTilde(config.Get("repository", "dir"))
// Create a map of desired node connections based on the configuration file
// directives.
for nodeID, addrs := range config.OptionMap("nodes") {
addrs := strings.Fields(addrs)
nodeAddrs[nodeID] = addrs
}
ensureDir(dir, -1)
m := model.NewModel(dir)
for _, t := range opts.Debug.TraceModel {
m.Trace(t)
}
if opts.Advanced.LimitRate > 0 {
m.LimitRate(opts.Advanced.LimitRate)
}
// GUI
if !opts.NoGUI && opts.GUIAddr != "" {
host, port, err := net.SplitHostPort(opts.GUIAddr)
if err != nil {
warnf("Cannot start GUI on %q: %v", opts.GUIAddr, err)
} else {
if len(host) > 0 {
infof("Starting web GUI on http://%s", opts.GUIAddr)
} else {
infof("Starting web GUI on port %s", port)
}
startGUI(opts.GUIAddr, m)
}
}
// Walk the repository and update the local model before establishing any
// connections to other nodes.
if !opts.Rehash {
infoln("Loading index cache")
loadIndex(m)
}
infoln("Populating repository index")
updateLocalModel(m)
// Routine to listen for incoming connections
infoln("Listening for incoming connections")
go listen(myID, opts.Listen, m, cfg)
// Routine to connect out to configured nodes
infoln("Attempting to connect to other nodes")
go connect(myID, opts.Listen, nodeAddrs, m, cfg)
// Routine to pull blocks from other nodes to synchronize the local
// repository. Does not run when we are in read only (publish only) mode.
if !opts.ReadOnly {
if opts.NoDelete {
infoln("Deletes from peer nodes will be ignored")
} else {
infoln("Deletes from peer nodes are allowed")
}
okln("Ready to synchronize (read-write)")
m.StartRW(!opts.NoDelete, opts.Advanced.RequestsInFlight)
} else {
okln("Ready to synchronize (read only; no external updates accepted)")
}
// Periodically scan the repository and update the local model.
// XXX: Should use some fsnotify mechanism.
go func() {
for {
time.Sleep(opts.Advanced.ScanInterval)
updateLocalModel(m)
}
}()
if !opts.NoStats {
// Periodically print statistics
go printStatsLoop(m)
}
select {}
}
func printStatsLoop(m *model.Model) {
var lastUpdated int64
var lastStats = make(map[string]model.ConnectionInfo)
for {
time.Sleep(60 * time.Second)
for node, stats := range m.ConnectionStats() {
secs := time.Since(lastStats[node].At).Seconds()
inbps := 8 * int(float64(stats.InBytesTotal-lastStats[node].InBytesTotal)/secs)
outbps := 8 * int(float64(stats.OutBytesTotal-lastStats[node].OutBytesTotal)/secs)
if inbps+outbps > 0 {
infof("%s: %sb/s in, %sb/s out", node, MetricPrefix(inbps), MetricPrefix(outbps))
}
lastStats[node] = stats
}
if lu := m.Generation(); lu > lastUpdated {
lastUpdated = lu
files, _, bytes := m.GlobalSize()
infof("%6d files, %9sB in cluster", files, BinaryPrefix(bytes))
files, _, bytes = m.LocalSize()
infof("%6d files, %9sB in local repo", files, BinaryPrefix(bytes))
needFiles, bytes := m.NeedFiles()
infof("%6d files, %9sB to synchronize", len(needFiles), BinaryPrefix(bytes))
}
}
}
func listen(myID string, addr string, m *model.Model, cfg *tls.Config) {
l, err := tls.Listen("tcp", addr, cfg)
fatalErr(err)
listen:
for {
conn, err := l.Accept()
if err != nil {
warnln(err)
continue
}
if opts.Debug.TraceConnect {
debugln("NET: Connect from", conn.RemoteAddr())
}
tc := conn.(*tls.Conn)
err = tc.Handshake()
if err != nil {
warnln(err)
tc.Close()
continue
}
remoteID := certId(tc.ConnectionState().PeerCertificates[0].Raw)
if remoteID == myID {
warnf("Connect from myself (%s) - should not happen", remoteID)
conn.Close()
continue
}
if m.ConnectedTo(remoteID) {
warnf("Connect from connected node (%s)", remoteID)
}
for nodeID := range nodeAddrs {
if nodeID == remoteID {
protoConn := protocol.NewConnection(remoteID, conn, conn, m)
m.AddConnection(conn, protoConn)
continue listen
}
}
conn.Close()
}
}
func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.Model, cfg *tls.Config) {
_, portstr, err := net.SplitHostPort(addr)
fatalErr(err)
port, _ := strconv.Atoi(portstr)
if opts.Discovery.NoLocalDiscovery {
port = -1
} else {
infoln("Sending local discovery announcements")
}
if opts.Discovery.NoExternalDiscovery {
opts.Discovery.ExternalPort = -1
} else {
infoln("Sending external discovery announcements")
}
disc, err := discover.NewDiscoverer(myID, port, opts.Discovery.ExternalPort, opts.Discovery.ExternalServer)
if err != nil {
warnf("No discovery possible (%v)", err)
}
for {
nextNode:
for nodeID, addrs := range nodeAddrs {
if nodeID == myID {
continue
}
if m.ConnectedTo(nodeID) {
continue
}
for _, addr := range addrs {
if addr == "dynamic" {
var ok bool
if disc != nil {
addr, ok = disc.Lookup(nodeID)
}
if !ok {
continue
}
}
if opts.Debug.TraceConnect {
debugln("NET: Dial", nodeID, addr)
}
conn, err := tls.Dial("tcp", addr, cfg)
if err != nil {
if opts.Debug.TraceConnect {
debugln("NET:", err)
}
continue
}
remoteID := certId(conn.ConnectionState().PeerCertificates[0].Raw)
if remoteID != nodeID {
warnln("Unexpected nodeID", remoteID, "!=", nodeID)
conn.Close()
continue
}
protoConn := protocol.NewConnection(remoteID, conn, conn, m)
m.AddConnection(conn, protoConn)
continue nextNode
}
}
time.Sleep(opts.Advanced.ConnInterval)
}
}
func updateLocalModel(m *model.Model) {
files, _ := m.Walk(!opts.NoSymlinks)
m.ReplaceLocal(files)
saveIndex(m)
}
func saveIndex(m *model.Model) {
name := m.RepoID() + ".idx.gz"
fullName := path.Join(opts.ConfDir, name)
idxf, err := os.Create(fullName + ".tmp")
if err != nil {
return
}
gzw := gzip.NewWriter(idxf)
protocol.WriteIndex(gzw, m.ProtocolIndex())
gzw.Close()
idxf.Close()
os.Rename(fullName+".tmp", fullName)
}
func loadIndex(m *model.Model) {
name := m.RepoID() + ".idx.gz"
idxf, err := os.Open(path.Join(opts.ConfDir, name))
if err != nil {
return
}
defer idxf.Close()
gzr, err := gzip.NewReader(idxf)
if err != nil {
return
}
defer gzr.Close()
idx, err := protocol.ReadIndex(gzr)
if err != nil {
return
}
m.SeedLocal(idx)
}
func ensureDir(dir string, mode int) {
fi, err := os.Stat(dir)
if os.IsNotExist(err) {
err := os.MkdirAll(dir, 0700)
fatalErr(err)
} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
err := os.Chmod(dir, os.FileMode(mode))
fatalErr(err)
}
}
func expandTilde(p string) string {
if strings.HasPrefix(p, "~/") {
return strings.Replace(p, "~", getHomeDir(), 1)
}
return p
}
func getHomeDir() string {
home := os.Getenv("HOME")
if home == "" {
fatalln("No home directory?")
}
return home
}