Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Report historical performance

This commit is contained in:
Jakob Borg 2016-09-06 20:15:18 +02:00
parent dd175c5431
commit 2b5a735091
4 changed files with 254 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
cmd/aggregate/main.go
View File

@ -53,6 +53,14 @@ func runAggregation(db *sql.DB) {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
log.Println("Aggregating Performance data")
since = maxIndexedDay(db, "Performance")
rows, err = aggregatePerformance(db, since)
if err != nil {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
}
func sleepUntilNext(intv, margin time.Duration) {
@ -82,21 +90,40 @@ func setupDB(db *sql.DB) error {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
Day TIMESTAMP NOT NULL,
TotFiles INTEGER NOT NULL,
TotMiB INTEGER NOT NULL,
SHA256Perf DOUBLE PRECISION NOT NULL,
MemorySize INTEGER NOT NULL,
MemoryUsageMiB INTEGER NOT NULL
)`)
if err != nil {
return err
}
var t string
row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
if err := row.Scan(nil); err != nil {
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
}
row = db.QueryRow(`SELECT 'DayIndex'::regclass`)
if err := row.Scan(nil); err != nil {
_, err = db.Exec(`CREATE INDEX DayIndex ON VerionSummary (Day)`)
row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
}
row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
if err := row.Scan(nil); err != nil {
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
}
row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
}
return err
}
@ -209,3 +236,27 @@ func aggregateUserMovement(db *sql.DB) (int64, error) {
return int64(len(sumRows)), tx.Commit()
}
func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO Performance (
SELECT
DATE_TRUNC('day', Received) AS Day,
AVG(TotFiles) As TotFiles,
AVG(TotMiB) As TotMiB,
AVG(SHA256Perf) As SHA256Perf,
AVG(MemorySize) As MemorySize,
AVG(MemoryUsageMiB) As MemoryUsageMiB
FROM Reports
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v0.%'
GROUP BY Day
);
`, since)
if err != nil {
return 0, err
}
return res.RowsAffected()
}

47
cmd/ursrv/main.go
View File

@ -350,6 +350,7 @@ func main() {
http.HandleFunc("/newdata", withDB(db, newDataHandler))
http.HandleFunc("/summary.json", withDB(db, summaryHandler))
http.HandleFunc("/movement.json", withDB(db, movementHandler))
http.HandleFunc("/performance.json", withDB(db, performanceHandler))
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
err = srv.Serve(listener)
@ -458,6 +459,25 @@ func movementHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
w.Write(bs)
}
func performanceHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
s, err := getPerformance(db)
if err != nil {
log.Println("performanceHandler:", err)
http.Error(w, "Database Error", http.StatusInternalServerError)
return
}
bs, err := json.Marshal(s)
if err != nil {
log.Println("performanceHandler:", err)
http.Error(w, "JSON Encode Error", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(bs)
}
type category struct {
Values [4]float64
Key string
@ -919,6 +939,33 @@ func getMovement(db *sql.DB) ([][]interface{}, error) {
return res, nil
}
func getPerformance(db *sql.DB) ([][]interface{}, error) {
rows, err := db.Query(`SELECT Day, TotFiles, TotMiB, SHA256Perf, MemorySize, MemoryUsageMiB FROM Performance WHERE Day > '2014-06-20'::TIMESTAMP ORDER BY Day`)
if err != nil {
return nil, err
}
defer rows.Close()
res := [][]interface{}{
{"Day", "TotFiles", "TotMiB", "SHA256Perf", "MemorySize", "MemoryUsageMiB"},
}
for rows.Next() {
var day time.Time
var sha256Perf float64
var totFiles, totMiB, memorySize, memoryUsage int
err := rows.Scan(&day, &totFiles, &totMiB, &sha256Perf, &memorySize, &memoryUsage)
if err != nil {
return nil, err
}
row := []interface{}{day.Format("2006-01-02"), totFiles, totMiB, float64(int(sha256Perf*10)) / 10, memorySize, memoryUsage}
res = append(res, row)
}
return res, nil
}
type sortableFeatureList []feature
func (l sortableFeatureList) Len() int {

40
static/index.html
View File

@ -41,6 +41,7 @@ found in the LICENSE file.
<script type="text/javascript">
google.setOnLoadCallback(drawVersionChart);
google.setOnLoadCallback(drawMovementChart);
google.setOnLoadCallback(drawPerformanceChart);
function drawVersionChart() {
var jsonData = $.ajax({url: "summary.json", dataType:"json", async: false}).responseText;
@ -97,6 +98,41 @@ found in the LICENSE file.
var chart = new google.visualization.AreaChart(document.getElementById('movementChart'));
chart.draw(data, options);
}
function drawPerformanceChart() {
var jsonData = $.ajax({url: "performance.json", dataType:"json", async: false}).responseText;
var rows = JSON.parse(jsonData);
var data = new google.visualization.DataTable();
data.addColumn('date', 'Day');
data.addColumn('number', 'Hash Performance (MiB/s)');
data.addColumn('number', 'Memory Usage (MiB)');
for (var i = 1; i < rows.length; i++){
rows[i][0] = new Date(rows[i][0]);
if (rows[i][1] > 500) {
rows[i][1] = null;
}
data.addRow(rows[i]);
};
var options = {
legend: { position: 'bottom', alignment: 'center' },
colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
chartArea: {left: 80, top: 20, width: '940', height: '300'},
series: {
0: {targetAxisIndex: 0},
1: {targetAxisIndex: 1},
},
vAxes: {
0: {title: 'MiB/s'},
1: {title: 'MiB'},
}
};
var chart = new google.visualization.LineChart(document.getElementById('performanceChart'));
chart.draw(data, options);
}
</script>
</head>
@ -119,9 +155,11 @@ found in the LICENSE file.
<div class="img-thumbnail" id="movementChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<p class="text-muted">
Reappearance of users cause the "left" data to shrink retroactively.
Spikes in December 2014 were due to the unique ID format changing and have been partly removed to avoid skewing the graph scale.
</p>
<div class="img-thumbnail" id="performanceChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<h4 id="metrics">Usage Metrics</h4>
<p>
This is the aggregated usage report data for the last 24 hours. Data based on <b>{{.nodes}}</b> devices that have reported in.

112
static/performance.html Normal file
View File

@ -0,0 +1,112 @@
<!DOCTYPE html>
<!--
Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
Use of this source code is governed by an MIT-style license that can be
found in the LICENSE file.
-->
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="">
<meta name="author" content="">
<link rel="shortcut icon" href="assets/img/favicon.png">
<title>Historical Performance Data</title>
<link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script type="text/javascript" src="bootstrap/js/bootstrap.min.js"></script>
<style type="text/css">
body {
margin: 40px;
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
}
tr.main td {
font-weight: bold;
}
tr.child td.first {
padding-left: 2em;
}
</style>
<script type="text/javascript"
src="https://www.google.com/jsapi?autoload={
'modules':[{
'name':'visualization',
'version':'1',
'packages':['corechart']
}]
}"></script>
<script type="text/javascript">
google.setOnLoadCallback(drawPerformanceCharts);
function drawPerformanceCharts() {
var jsonData = $.ajax({url: "/performance.json", dataType:"json", async: false}).responseText;
var rows = JSON.parse(jsonData);
for (var i = 1; i < rows.length; i++){
rows[i][0] = new Date(rows[i][0]);
}
drawChart(rows, 1, 'Total Number of Files', 'totFilesChart', 1e6, 1);
drawChart(rows, 2, 'Total Folder Size (GiB)', 'totMiBChart', 1e6, 1024);
drawChart(rows, 3, 'Hash Performance (MiB/s)', 'hashPerfChart', 1000, 1);
drawChart(rows, 4, 'System RAM Size (GiB)', 'memSizeChart', 1e6, 1024);
drawChart(rows, 5, 'Memory Usage (MiB)', 'memUsageChart', 250, 1);
}
// {"Day", "TotFiles", "TotMiB", "SHA256Perf", "MemorySize", "MemoryUsageMiB"},
function drawChart(rows, index, title, id, cutoff, divisor) {
var data = new google.visualization.DataTable();
data.addColumn('date', 'Day');
data.addColumn('number', title);
var row;
for (var i = 1; i < rows.length; i++){
row = [rows[i][0], rows[i][index] / divisor];
if (row[1] > cutoff) {
row[1] = null;
}
data.addRow(row);
}
var options = {
legend: { position: 'bottom', alignment: 'center' },
colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
chartArea: {left: 80, top: 20, width: '1020', height: '300'},
vAxes: {0: {minValue: 0}},
};
var chart = new google.visualization.LineChart(document.getElementById(id));
chart.draw(data, options);
}
</script>
</head>
<body>
<div class="container">
<div class="row">
<div class="col-md-12">
<h1>Historical Performance Data</h1>
<p>These charts are all the average of the corresponding metric, for the entire population of a given day.</p>
<h4 id="active-users">Hash Performance (MiB/s)</h4>
<div class="img-thumbnail" id="hashPerfChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<h4 id="active-users">Memory Usage (MiB)</h4>
<div class="img-thumbnail" id="memUsageChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<h4 id="active-users">Total Number of Files</h4>
<div class="img-thumbnail" id="totFilesChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<h4 id="active-users">Total Folder Size (GiB)</h4>
<div class="img-thumbnail" id="totMiBChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
<h4 id="active-users">System RAM Size (GiB)</h4>
<div class="img-thumbnail" id="memSizeChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
</div>
</div>
</body>
</html>