Store reports in PostgreSQL
This commit is contained in:
parent
13f82a06ee
commit
ed240145c7
334
main.go
334
main.go
|
@ -2,8 +2,8 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"html/template"
|
"html/template"
|
||||||
"io"
|
"io"
|
||||||
|
@ -11,20 +11,20 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/lib/pq"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
keyFile = flag.String("key", "", "Key file")
|
keyFile = getEnvDefault("UR_KEY_FILE", "key.pem")
|
||||||
certFile = flag.String("cert", "", "Certificate file")
|
certFile = getEnvDefault("UR_CRT_FILE", "crt.pem")
|
||||||
dbDir = flag.String("db", "", "Database directory")
|
dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
|
||||||
port = flag.Int("port", 8443, "Listen port")
|
listenAddr = getEnvDefault("UR_LISTEN", "0.0.0.0:8443")
|
||||||
tpl *template.Template
|
tpl *template.Template
|
||||||
)
|
)
|
||||||
|
|
||||||
var funcs = map[string]interface{}{
|
var funcs = map[string]interface{}{
|
||||||
|
@ -32,30 +32,128 @@ var funcs = map[string]interface{}{
|
||||||
"number": number,
|
"number": number,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getEnvDefault(key, def string) string {
|
||||||
|
if val := os.Getenv(key); val != "" {
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
type report struct {
|
||||||
|
Received time.Time // Only from DB
|
||||||
|
|
||||||
|
UniqueID string
|
||||||
|
Version string
|
||||||
|
LongVersion string
|
||||||
|
Platform string
|
||||||
|
NumFolders int
|
||||||
|
NumDevices int
|
||||||
|
TotFiles int
|
||||||
|
FolderMaxFiles int
|
||||||
|
TotMiB int
|
||||||
|
FolderMaxMiB int
|
||||||
|
MemoryUsageMiB int
|
||||||
|
SHA256Perf float64
|
||||||
|
MemorySize int
|
||||||
|
|
||||||
|
Date string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *report) Validate() error {
|
||||||
|
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
|
||||||
|
return fmt.Errorf("missing required field")
|
||||||
|
}
|
||||||
|
if len(r.Date) != 8 {
|
||||||
|
return fmt.Errorf("date not initialized")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupDB(db *sql.DB) error {
|
||||||
|
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
|
||||||
|
Received TIMESTAMP NOT NULL,
|
||||||
|
UniqueID VARCHAR(32) NOT NULL,
|
||||||
|
Version VARCHAR(32) NOT NULL,
|
||||||
|
LongVersion VARCHAR(256) NOT NULL,
|
||||||
|
Platform VARCHAR(32) NOT NULL,
|
||||||
|
NumFolders INTEGER NOT NULL,
|
||||||
|
NumDevices INTEGER NOT NULL,
|
||||||
|
TotFiles INTEGER NOT NULL,
|
||||||
|
FolderMaxFiles INTEGER NOT NULL,
|
||||||
|
TotMiB INTEGER NOT NULL,
|
||||||
|
FolderMaxMiB INTEGER NOT NULL,
|
||||||
|
MemoryUsageMiB INTEGER NOT NULL,
|
||||||
|
SHA256Perf DOUBLE PRECISION NOT NULL,
|
||||||
|
MemorySize INTEGER NOT NULL,
|
||||||
|
Date VARCHAR(8) NOT NULL
|
||||||
|
)`)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`)
|
||||||
|
if err := row.Scan(nil); err != nil {
|
||||||
|
_, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`)
|
||||||
|
if err := row.Scan(nil); err != nil {
|
||||||
|
_, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func insertReport(db *sql.DB, r report) error {
|
||||||
|
_, err := db.Exec(`INSERT INTO Reports VALUES (now(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
|
||||||
|
r.UniqueID, r.Version, r.LongVersion, r.Platform, r.NumFolders,
|
||||||
|
r.NumDevices, r.TotFiles, r.FolderMaxFiles, r.TotMiB, r.FolderMaxMiB,
|
||||||
|
r.MemoryUsageMiB, r.SHA256Perf, r.MemorySize, r.Date)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request)
|
||||||
|
|
||||||
|
func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
f(db, w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
log.SetFlags(log.Lshortfile)
|
log.SetFlags(log.Ltime | log.Ldate)
|
||||||
log.SetOutput(os.Stdout)
|
log.SetOutput(os.Stdout)
|
||||||
flag.Parse()
|
|
||||||
|
// Template
|
||||||
|
|
||||||
fd, err := os.Open("static/index.html")
|
fd, err := os.Open("static/index.html")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalln("template:", err)
|
||||||
}
|
}
|
||||||
bs, err := ioutil.ReadAll(fd)
|
bs, err := ioutil.ReadAll(fd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalln("template:", err)
|
||||||
}
|
}
|
||||||
fd.Close()
|
fd.Close()
|
||||||
tpl = template.Must(template.New("index.html").Funcs(funcs).Parse(string(bs)))
|
tpl = template.Must(template.New("index.html").Funcs(funcs).Parse(string(bs)))
|
||||||
|
|
||||||
http.HandleFunc("/", rootHandler)
|
// DB
|
||||||
http.HandleFunc("/newdata", newDataHandler)
|
|
||||||
http.HandleFunc("/report", reportHandler)
|
|
||||||
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
|
|
||||||
|
|
||||||
cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
|
db, err := sql.Open("postgres", dbConn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalln("database:", err)
|
||||||
|
}
|
||||||
|
err = setupDB(db)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln("database:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TLS
|
||||||
|
|
||||||
|
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln("tls:", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := &tls.Config{
|
cfg := &tls.Config{
|
||||||
|
@ -63,28 +161,32 @@ func main() {
|
||||||
SessionTicketsDisabled: true,
|
SessionTicketsDisabled: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
listener, err := tls.Listen("tcp", fmt.Sprintf(":%d", *port), cfg)
|
// HTTPS
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("Listening on", listener.Addr())
|
listener, err := tls.Listen("tcp", listenAddr, cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln("https:", err)
|
||||||
|
}
|
||||||
|
|
||||||
srv := http.Server{
|
srv := http.Server{
|
||||||
ReadTimeout: 5 * time.Second,
|
ReadTimeout: 5 * time.Second,
|
||||||
WriteTimeout: 5 * time.Second,
|
WriteTimeout: 5 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
http.HandleFunc("/", withDB(db, rootHandler))
|
||||||
|
http.HandleFunc("/newdata", withDB(db, newDataHandler))
|
||||||
|
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
|
||||||
|
|
||||||
err = srv.Serve(listener)
|
err = srv.Serve(listener)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatalln("https:", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func rootHandler(w http.ResponseWriter, r *http.Request) {
|
func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path == "/" || r.URL.Path == "/index.html" {
|
if r.URL.Path == "/" || r.URL.Path == "/index.html" {
|
||||||
k := timestamp()
|
k := timestamp()
|
||||||
rep := getReport(k)
|
rep := getReport(db, k)
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
err := tpl.Execute(w, rep)
|
err := tpl.Execute(w, rep)
|
||||||
|
@ -96,126 +198,32 @@ func rootHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func reportHandler(w http.ResponseWriter, r *http.Request) {
|
func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
defer r.Body.Close()
|
||||||
k := timestamp()
|
|
||||||
rep := getReport(k)
|
|
||||||
json.NewEncoder(w).Encode(rep)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDataHandler(w http.ResponseWriter, r *http.Request) {
|
var rep report
|
||||||
today := time.Now().Format("20060102")
|
rep.Date = time.Now().UTC().Format("20060102")
|
||||||
dir := filepath.Join(*dbDir, today)
|
|
||||||
ensureDir(dir, 0700)
|
|
||||||
|
|
||||||
var m map[string]interface{}
|
|
||||||
lr := &io.LimitedReader{R: r.Body, N: 10240}
|
lr := &io.LimitedReader{R: r.Body, N: 10240}
|
||||||
err := json.NewDecoder(lr).Decode(&m)
|
if err := json.NewDecoder(lr).Decode(&rep); err != nil {
|
||||||
|
log.Println("json decode:", err)
|
||||||
if err != nil {
|
http.Error(w, "JSON Decode Error", http.StatusInternalServerError)
|
||||||
log.Println(err)
|
|
||||||
http.Error(w, err.Error(), 500)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
id, ok := m["uniqueID"]
|
if err := rep.Validate(); err != nil {
|
||||||
if ok {
|
log.Println("validate:", err)
|
||||||
idStr, ok := id.(string)
|
log.Printf("%#v", rep)
|
||||||
if !ok {
|
http.Error(w, "Validation Error", http.StatusInternalServerError)
|
||||||
if err != nil {
|
|
||||||
log.Printf("No ID (type was %T)", id)
|
|
||||||
http.Error(w, "No ID", 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if idStr == "" {
|
|
||||||
log.Println("No ID (empty)")
|
|
||||||
http.Error(w, "No ID", 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The ID is base64 encoded, so can contain slashes. Replace those with dots instead.
|
|
||||||
idStr = strings.Replace(idStr, "/", ".", -1)
|
|
||||||
|
|
||||||
f, err := os.Create(path.Join(dir, idStr+".json"))
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
http.Error(w, err.Error(), 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
err = json.NewEncoder(f).Encode(m)
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
http.Error(w, err.Error(), 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
err = f.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
http.Error(w, err.Error(), 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Report from %q", id)
|
|
||||||
} else {
|
|
||||||
log.Println("No ID (missing)")
|
|
||||||
http.Error(w, "No ID", 500)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
type report struct {
|
if err := insertReport(db, rep); err != nil {
|
||||||
UniqueID string
|
log.Println("insert:", err)
|
||||||
Version string
|
log.Printf("%#v", rep)
|
||||||
Platform string
|
http.Error(w, "Database Error", http.StatusInternalServerError)
|
||||||
NumRepos int
|
return
|
||||||
NumNodes int
|
|
||||||
TotFiles int
|
|
||||||
RepoMaxFiles int
|
|
||||||
TotMiB int
|
|
||||||
RepoMaxMiB int
|
|
||||||
MemoryUsageMiB int
|
|
||||||
SHA256Perf float64
|
|
||||||
MemorySize int
|
|
||||||
}
|
|
||||||
|
|
||||||
func fileList() ([]string, error) {
|
|
||||||
files := make(map[string]string)
|
|
||||||
t0 := time.Now().Add(-24 * time.Hour).Format("20060102")
|
|
||||||
t1 := time.Now().Format("20060102")
|
|
||||||
|
|
||||||
dir := filepath.Join(*dbDir, t0)
|
|
||||||
gr, err := filepath.Glob(filepath.Join(dir, "*.json"))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
for _, f := range gr {
|
|
||||||
bn := filepath.Base(f)
|
|
||||||
files[bn] = f
|
|
||||||
}
|
|
||||||
|
|
||||||
dir = filepath.Join(*dbDir, t1)
|
|
||||||
gr, err = filepath.Glob(filepath.Join(dir, "*.json"))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, f := range gr {
|
|
||||||
bn := filepath.Base(f)
|
|
||||||
files[bn] = f
|
|
||||||
}
|
|
||||||
|
|
||||||
l := make([]string, 0, len(files))
|
|
||||||
for _, f := range files {
|
|
||||||
si, err := os.Stat(f)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if time.Since(si.ModTime()) < 24*time.Hour {
|
|
||||||
l = append(l, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return l, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type category struct {
|
type category struct {
|
||||||
|
@ -229,7 +237,7 @@ type category struct {
|
||||||
var reportCache map[string]interface{}
|
var reportCache map[string]interface{}
|
||||||
var reportMutex sync.Mutex
|
var reportMutex sync.Mutex
|
||||||
|
|
||||||
func getReport(key string) map[string]interface{} {
|
func getReport(db *sql.DB, key string) map[string]interface{} {
|
||||||
reportMutex.Lock()
|
reportMutex.Lock()
|
||||||
defer reportMutex.Unlock()
|
defer reportMutex.Unlock()
|
||||||
|
|
||||||
|
@ -237,17 +245,12 @@ func getReport(key string) map[string]interface{} {
|
||||||
return reportCache
|
return reportCache
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := fileList()
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
nodes := 0
|
nodes := 0
|
||||||
var versions []string
|
var versions []string
|
||||||
var platforms []string
|
var platforms []string
|
||||||
var oses []string
|
var oses []string
|
||||||
var numRepos []int
|
var numFolders []int
|
||||||
var numNodes []int
|
var numDevices []int
|
||||||
var totFiles []int
|
var totFiles []int
|
||||||
var maxFiles []int
|
var maxFiles []int
|
||||||
var totMiB []int
|
var totMiB []int
|
||||||
|
@ -256,41 +259,48 @@ func getReport(key string) map[string]interface{} {
|
||||||
var sha256Perf []float64
|
var sha256Perf []float64
|
||||||
var memorySize []int
|
var memorySize []int
|
||||||
|
|
||||||
for _, fn := range files {
|
rows, err := db.Query(`SELECT * FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
|
||||||
f, err := os.Open(fn)
|
if err != nil {
|
||||||
if err != nil {
|
log.Println("sql:", err)
|
||||||
continue
|
return nil
|
||||||
}
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
|
||||||
var rep report
|
var rep report
|
||||||
err = json.NewDecoder(f).Decode(&rep)
|
err := rows.Scan(&rep.Received, &rep.UniqueID, &rep.Version,
|
||||||
|
&rep.LongVersion, &rep.Platform, &rep.NumFolders, &rep.NumDevices,
|
||||||
|
&rep.TotFiles, &rep.FolderMaxFiles, &rep.TotMiB, &rep.FolderMaxMiB,
|
||||||
|
&rep.MemoryUsageMiB, &rep.SHA256Perf, &rep.MemorySize, &rep.Date)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
log.Println("sql:", err)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
f.Close()
|
|
||||||
|
|
||||||
nodes++
|
nodes++
|
||||||
versions = append(versions, transformVersion(rep.Version))
|
versions = append(versions, transformVersion(rep.Version))
|
||||||
platforms = append(platforms, rep.Platform)
|
platforms = append(platforms, rep.Platform)
|
||||||
ps := strings.Split(rep.Platform, "-")
|
ps := strings.Split(rep.Platform, "-")
|
||||||
oses = append(oses, ps[0])
|
oses = append(oses, ps[0])
|
||||||
if rep.NumRepos > 0 {
|
if rep.NumFolders > 0 {
|
||||||
numRepos = append(numRepos, rep.NumRepos)
|
numFolders = append(numFolders, rep.NumFolders)
|
||||||
}
|
}
|
||||||
if rep.NumNodes > 0 {
|
if rep.NumDevices > 0 {
|
||||||
numNodes = append(numNodes, rep.NumNodes)
|
numDevices = append(numDevices, rep.NumDevices)
|
||||||
}
|
}
|
||||||
if rep.TotFiles > 0 {
|
if rep.TotFiles > 0 {
|
||||||
totFiles = append(totFiles, rep.TotFiles)
|
totFiles = append(totFiles, rep.TotFiles)
|
||||||
}
|
}
|
||||||
if rep.RepoMaxFiles > 0 {
|
if rep.FolderMaxFiles > 0 {
|
||||||
maxFiles = append(maxFiles, rep.RepoMaxFiles)
|
maxFiles = append(maxFiles, rep.FolderMaxFiles)
|
||||||
}
|
}
|
||||||
if rep.TotMiB > 0 {
|
if rep.TotMiB > 0 {
|
||||||
totMiB = append(totMiB, rep.TotMiB*(1<<20))
|
totMiB = append(totMiB, rep.TotMiB*(1<<20))
|
||||||
}
|
}
|
||||||
if rep.RepoMaxMiB > 0 {
|
if rep.FolderMaxMiB > 0 {
|
||||||
maxMiB = append(maxMiB, rep.RepoMaxMiB*(1<<20))
|
maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
|
||||||
}
|
}
|
||||||
if rep.MemoryUsageMiB > 0 {
|
if rep.MemoryUsageMiB > 0 {
|
||||||
memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
|
memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
|
||||||
|
@ -329,12 +339,12 @@ func getReport(key string) map[string]interface{} {
|
||||||
})
|
})
|
||||||
|
|
||||||
categories = append(categories, category{
|
categories = append(categories, category{
|
||||||
Values: statsForInts(numNodes),
|
Values: statsForInts(numDevices),
|
||||||
Descr: "Number of Devices in Cluster",
|
Descr: "Number of Devices in Cluster",
|
||||||
})
|
})
|
||||||
|
|
||||||
categories = append(categories, category{
|
categories = append(categories, category{
|
||||||
Values: statsForInts(numRepos),
|
Values: statsForInts(numFolders),
|
||||||
Descr: "Number of Folders Configured",
|
Descr: "Number of Folders Configured",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue