From 1be1a06281dccada078a2a51e8b483811af8f596 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 29 Mar 2024 15:02:10 +0100 Subject: all: refactor stats Add ability for each package to create and export own stats. Each stat is self-contained, describes how it should be presented, and there is not need to copy them from one package to another. Stats also keep historical data and allow building graphs over time. --- pkg/stats/set.go | 499 ++++++++++++++++++++++++++++++++++++++++++++++++++ pkg/stats/set_test.go | 219 ++++++++++++++++++++++ 2 files changed, 718 insertions(+) create mode 100644 pkg/stats/set.go create mode 100644 pkg/stats/set_test.go (limited to 'pkg/stats') diff --git a/pkg/stats/set.go b/pkg/stats/set.go new file mode 100644 index 000000000..54e44e457 --- /dev/null +++ b/pkg/stats/set.go @@ -0,0 +1,499 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package stats + +import ( + "bytes" + "fmt" + "reflect" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/bsm/histogram/v3" + "github.com/google/syzkaller/pkg/html/pages" + "github.com/prometheus/client_golang/prometheus" +) + +// This file provides prometheus/streamz style metrics (Val type) for instrumenting code for monitoring. +// It also provides a registry for such metrics (set type) and a global default registry. +// +// Simple uses of metrics: +// +// statFoo := stats.Create("metric name", "metric description") +// statFoo.Add(1) +// +// stats.Create("metric name", "metric description", LenOf(mySlice, rwMutex)) +// +// Metric visualization code uses Collect/RenderHTML functions to obtain values of all registered metrics. + +type UI struct { + Name string + Desc string + Link string + Level Level + Value string + V int +} + +func Create(name, desc string, opts ...any) *Val { + return global.Create(name, desc, opts...) +} + +func Collect(level Level) []UI { + return global.Collect(level) +} + +func RenderHTML() ([]byte, error) { + return global.RenderHTML() +} + +func Import(named map[string]uint64) { + global.Import(named) +} + +var global = newSet(256, true) + +type set struct { + mu sync.Mutex + vals map[string]*Val + graphs map[string]*graph + totalTicks int + historySize int + historyTicks int + historyPos int + historyScale int +} + +type graph struct { + level Level + stacked bool + lines map[string]*line +} + +type line struct { + desc string + rate bool + data []float64 + hist []*histogram.Histogram +} + +const ( + tickPeriod = time.Second + histogramBuckets = 255 +) + +func newSet(histSize int, tick bool) *set { + s := &set{ + vals: make(map[string]*Val), + historySize: histSize, + historyScale: 1, + graphs: make(map[string]*graph), + } + if tick { + go func() { + for range time.NewTicker(tickPeriod).C { + s.tick() + } + }() + } + return s +} + +func (s *set) Collect(level Level) []UI { + s.mu.Lock() + defer s.mu.Unlock() + period := time.Duration(s.totalTicks) * tickPeriod + if period == 0 { + period = tickPeriod + } + var res []UI + for _, v := range s.vals { + if v.level < level { + continue + } + val := v.Val() + res = append(res, UI{ + Name: v.name, + Desc: v.desc, + Link: v.link, + Level: v.level, + Value: v.fmt(val, period), + V: val, + }) + } + sort.Slice(res, func(i, j int) bool { + if res[i].Level != res[j].Level { + return res[i].Level > res[j].Level + } + return res[i].Name < res[j].Name + }) + return res +} + +func (s *set) Import(named map[string]uint64) { + s.mu.Lock() + defer s.mu.Unlock() + for name, val := range named { + v := s.vals[name] + if v == nil { + panic(fmt.Sprintf("imported stat %v is missing", name)) + } + v.Add(int(val)) + } +} + +// Additional options for Val metrics. + +// Level controls if the metric should be printed to console in periodic heartbeat logs, +// or showed on the simple web interface, or showed in the expert interface only. +type Level int + +const ( + All Level = iota + Simple + Console +) + +// Link adds a hyperlink to metric name. +type Link string + +// Prometheus exports the metric to Prometheus under the given name. +type Prometheus string + +// Rate says to collect/visualize metric rate per unit of time rather then total value. +type Rate struct{} + +// Distribution says to collect/visualize histogram of individual sample distributions. +type Distribution struct{} + +// Graph allows to combine multiple related metrics on a single graph. +type Graph string + +// StackedGraph is like Graph, but shows metrics on a stacked graph. +type StackedGraph string + +// NoGraph says to not visualize the metric as a graph. +const NoGraph Graph = "" + +// LenOf reads the metric value from the given slice/map/chan. +func LenOf(containerPtr any, mu *sync.RWMutex) func() int { + v := reflect.ValueOf(containerPtr) + _ = v.Elem().Len() // panics if container is not slice/map/chan + return func() int { + mu.RLock() + defer mu.RUnlock() + return v.Elem().Len() + } +} + +// Addittionally a custom 'func() int' can be passed to read the metric value from the function. +// and 'func(int, time.Duration) string' can be passed for custom formatting of the metric value. + +func (s *set) Create(name, desc string, opts ...any) *Val { + v := &Val{ + name: name, + desc: desc, + graph: name, + fmt: func(v int, period time.Duration) string { return strconv.Itoa(v) }, + } + stacked := false + for _, o := range opts { + switch opt := o.(type) { + case Level: + v.level = opt + case Link: + v.link = string(opt) + case Graph: + v.graph = string(opt) + case StackedGraph: + v.graph = string(opt) + stacked = true + case Rate: + v.rate = true + v.fmt = formatRate + case Distribution: + v.hist = true + case func() int: + v.ext = opt + case func(int, time.Duration) string: + v.fmt = opt + case Prometheus: + // Prometheus Instrumentation https://prometheus.io/docs/guides/go-application. + prometheus.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: string(opt), + Help: desc, + }, + func() float64 { return float64(v.Val()) }, + )) + default: + panic(fmt.Sprintf("unknown stats option %#v", o)) + } + } + s.mu.Lock() + defer s.mu.Unlock() + s.vals[name] = v + if v.graph != "" { + if s.graphs[v.graph] == nil { + s.graphs[v.graph] = &graph{ + lines: make(map[string]*line), + } + } + if s.graphs[v.graph].level < v.level { + s.graphs[v.graph].level = v.level + } + s.graphs[v.graph].stacked = stacked + } + return v +} + +type Val struct { + name string + desc string + link string + graph string + level Level + val atomic.Uint64 + ext func() int + fmt func(int, time.Duration) string + rate bool + hist bool + prev int + histMu sync.Mutex + histVal *histogram.Histogram +} + +func (v *Val) Add(val int) { + if v.ext != nil { + panic(fmt.Sprintf("stat %v is in external mode", v.name)) + } + if v.hist { + v.histMu.Lock() + if v.histVal == nil { + v.histVal = histogram.New(histogramBuckets) + } + v.histVal.Add(float64(val)) + v.histMu.Unlock() + return + } + v.val.Add(uint64(val)) +} + +func (v *Val) Val() int { + if v.ext != nil { + return v.ext() + } + if v.hist { + v.histMu.Lock() + defer v.histMu.Unlock() + if v.histVal == nil { + return 0 + } + return int(v.histVal.Mean()) + } + return int(v.val.Load()) +} + +func formatRate(v int, period time.Duration) string { + secs := int(period.Seconds()) + if x := v / secs; x >= 10 { + return fmt.Sprintf("%v (%v/sec)", v, x) + } + if x := v * 60 / secs; x >= 10 { + return fmt.Sprintf("%v (%v/min)", v, x) + } + x := v * 60 * 60 / secs + return fmt.Sprintf("%v (%v/hour)", v, x) +} + +func (s *set) tick() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.historyPos == s.historySize { + s.compress() + } + + s.totalTicks++ + s.historyTicks++ + for _, v := range s.vals { + if v.graph == "" { + continue + } + graph := s.graphs[v.graph] + ln := graph.lines[v.name] + if ln == nil { + ln = &line{ + desc: v.desc, + rate: v.rate, + } + if v.hist { + ln.hist = make([]*histogram.Histogram, s.historySize) + } else { + ln.data = make([]float64, s.historySize) + } + graph.lines[v.name] = ln + } + if v.hist { + if s.historyTicks == s.historyScale { + v.histMu.Lock() + ln.hist[s.historyPos] = v.histVal + v.histVal = nil + v.histMu.Unlock() + } + } else { + val := v.Val() + pv := &ln.data[s.historyPos] + if v.rate { + *pv += float64(val-v.prev) / float64(s.historyScale) + v.prev = val + } else { + if *pv < float64(val) { + *pv = float64(val) + } + } + } + } + if s.historyTicks != s.historyScale { + return + } + s.historyTicks = 0 + s.historyPos++ +} + +func (s *set) compress() { + half := s.historySize / 2 + s.historyPos = half + s.historyScale *= 2 + for _, graph := range s.graphs { + for _, line := range graph.lines { + for i := 0; i < half; i++ { + if line.hist != nil { + h1, h2 := line.hist[2*i], line.hist[2*i+1] + line.hist[2*i], line.hist[2*i+1] = nil, nil + line.hist[i] = h1 + if h1 == nil { + line.hist[i] = h2 + } + } else { + v1, v2 := line.data[2*i], line.data[2*i+1] + line.data[2*i], line.data[2*i+1] = 0, 0 + if line.rate { + line.data[i] = (v1 + v2) / 2 + } else { + line.data[i] = v1 + if v2 > v1 { + line.data[i] = v2 + } + } + } + } + } + } +} + +func (s *set) RenderHTML() ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + type Point struct { + X int + Y []float64 + } + type Graph struct { + ID int + Title string + Stacked bool + Level Level + Lines []string + Points []Point + } + var graphs []Graph + tick := s.historyScale * int(tickPeriod.Seconds()) + for title, graph := range s.graphs { + if len(graph.lines) == 0 { + continue + } + g := Graph{ + ID: len(graphs), + Title: title, + Stacked: graph.stacked, + Level: graph.level, + Points: make([]Point, s.historyPos), + } + for i := 0; i < s.historyPos; i++ { + g.Points[i].X = i * tick + } + for name, ln := range graph.lines { + if ln.hist == nil { + g.Lines = append(g.Lines, name+": "+ln.desc) + for i := 0; i < s.historyPos; i++ { + g.Points[i].Y = append(g.Points[i].Y, ln.data[i]) + } + } else { + for _, percent := range []int{10, 50, 90} { + g.Lines = append(g.Lines, fmt.Sprintf("%v%%", percent)) + for i := 0; i < s.historyPos; i++ { + v := 0.0 + if ln.hist[i] != nil { + v = ln.hist[i].Quantile(float64(percent) / 100) + } + g.Points[i].Y = append(g.Points[i].Y, v) + } + } + } + } + graphs = append(graphs, g) + } + sort.Slice(graphs, func(i, j int) bool { + if graphs[i].Level != graphs[j].Level { + return graphs[i].Level > graphs[j].Level + } + return graphs[i].Title < graphs[j].Title + }) + buf := new(bytes.Buffer) + err := htmlTemplate.Execute(buf, graphs) + return buf.Bytes(), err +} + +var htmlTemplate = pages.Create(` + + + + syzkaller stats + + {{HEAD}} + + +{{range $g := .}} +
+ +{{end}} + + +`) diff --git a/pkg/stats/set_test.go b/pkg/stats/set_test.go new file mode 100644 index 000000000..9a19011bb --- /dev/null +++ b/pkg/stats/set_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package stats + +import ( + "fmt" + "math/rand" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestSet(t *testing.T) { + a := assert.New(t) + set := newSet(4, false) + a.Empty(set.Collect(All)) + _, err := set.RenderHTML() + a.NoError(err) + + v0 := set.Create("v0", "desc0") + a.Equal(v0.Val(), 0) + v0.Add(1) + a.Equal(v0.Val(), 1) + v0.Add(1) + a.Equal(v0.Val(), 2) + + vv1 := 0 + v1 := set.Create("v1", "desc1", Simple, func() int { return vv1 }) + a.Equal(v1.Val(), 0) + vv1 = 11 + a.Equal(v1.Val(), 11) + a.Panics(func() { v1.Add(1) }) + + v2 := set.Create("v2", "desc2", Console, func(v int, period time.Duration) string { + return fmt.Sprintf("v2 %v %v", v, period) + }) + v2.Add(100) + + v3 := set.Create("v3", "desc3", Link("/v3"), NoGraph, Distribution{}) + a.Equal(v3.Val(), 0) + v3.Add(10) + a.Equal(v3.Val(), 10) + v3.Add(20) + a.Equal(v3.Val(), 15) + v3.Add(20) + a.Equal(v3.Val(), 16) + v3.Add(30) + a.Equal(v3.Val(), 20) + v3.Add(30) + a.Equal(v3.Val(), 22) + v3.Add(30) + v3.Add(30) + a.Equal(v3.Val(), 24) + + v4 := set.Create("v4", "desc4", Rate{}, Graph("graph")) + v4.Add(10) + a.Equal(v4.Val(), 10) + v4.Add(10) + a.Equal(v4.Val(), 20) + + a.Panics(func() { set.Create("v0", "desc0", float64(1)) }) + + ui := set.Collect(All) + a.Equal(len(ui), 5) + a.Equal(ui[0], UI{"v2", "desc2", "", Console, "v2 100 1s", 100}) + a.Equal(ui[1], UI{"v1", "desc1", "", Simple, "11", 11}) + a.Equal(ui[2], UI{"v0", "desc0", "", All, "2", 2}) + a.Equal(ui[3], UI{"v3", "desc3", "/v3", All, "24", 24}) + a.Equal(ui[4], UI{"v4", "desc4", "", All, "20 (20/sec)", 20}) + + ui1 := set.Collect(Simple) + a.Equal(len(ui1), 2) + a.Equal(ui1[0].Name, "v2") + a.Equal(ui1[1].Name, "v1") + + ui2 := set.Collect(Console) + a.Equal(len(ui2), 1) + a.Equal(ui2[0].Name, "v2") + + _, err = set.RenderHTML() + a.NoError(err) +} + +func TestSetRateFormat(t *testing.T) { + a := assert.New(t) + set := newSet(4, false) + v := set.Create("v", "desc", Rate{}) + a.Equal(set.Collect(All)[0].Value, "0 (0/hour)") + v.Add(1) + a.Equal(set.Collect(All)[0].Value, "1 (60/min)") + v.Add(99) + a.Equal(set.Collect(All)[0].Value, "100 (100/sec)") +} + +func TestSetHistoryCounter(t *testing.T) { + a := assert.New(t) + set := newSet(4, false) + v := set.Create("v0", "desc0") + set.tick() + hist := func() []float64 { return set.graphs["v0"].lines["v0"].data[:set.historyPos] } + step := func(n int) []float64 { + v.Add(n) + set.tick() + return hist() + } + a.Equal(hist(), []float64{0}) + v.Add(1) + v.Add(1) + a.Equal(hist(), []float64{0}) + set.tick() + a.Equal(hist(), []float64{0, 2}) + v.Add(3) + a.Equal(hist(), []float64{0, 2}) + set.tick() + a.Equal(hist(), []float64{0, 2, 5}) + a.Equal(step(-1), []float64{0, 2, 5, 4}) + // Compacted, each new history value will require 2 steps. + a.Equal(step(7), []float64{2, 5}) + a.Equal(step(-10), []float64{2, 5, 11}) + a.Equal(step(2), []float64{2, 5, 11}) + a.Equal(step(1), []float64{2, 5, 11, 4}) + // 4 steps for each new value. + a.Equal(step(1), []float64{5, 11}) + a.Equal(step(1), []float64{5, 11}) + a.Equal(step(1), []float64{5, 11}) + a.Equal(step(1), []float64{5, 11, 8}) +} + +func TestSetHistoryRate(t *testing.T) { + a := assert.New(t) + set := newSet(4, false) + v := set.Create("v0", "desc0", Rate{}) + step := func(n int) []float64 { + v.Add(n) + set.tick() + return set.graphs["v0"].lines["v0"].data[:set.historyPos] + } + a.Equal(step(3), []float64{3}) + a.Equal(step(1), []float64{3, 1}) + a.Equal(step(2), []float64{3, 1, 2}) + a.Equal(step(5), []float64{3, 1, 2, 5}) + a.Equal(step(1), []float64{2, 3.5}) + a.Equal(step(2), []float64{2, 3.5, 1.5}) + a.Equal(step(2), []float64{2, 3.5, 1.5}) + a.Equal(step(4), []float64{2, 3.5, 1.5, 3}) + a.Equal(step(1), []float64{2.75, 2.25}) + a.Equal(step(2), []float64{2.75, 2.25}) + a.Equal(step(3), []float64{2.75, 2.25}) + a.Equal(step(4), []float64{2.75, 2.25, 2.5}) +} + +func TestSetHistoryDistribution(t *testing.T) { + a := assert.New(t) + set := newSet(4, false) + v := set.Create("v0", "desc0", Distribution{}) + step := func(n int) [3][]float64 { + v.Add(n) + set.tick() + var history [3][]float64 + for p, percent := range []int{10, 50, 90} { + history[p] = make([]float64, set.historyPos) + for i := 0; i < set.historyPos; i++ { + hist := set.graphs["v0"].lines["v0"].hist[i] + if hist != nil { + history[p][i] = hist.Quantile(float64(percent) / 100) + } + } + } + return history + } + a.Equal(step(3), [3][]float64{{3}, {3}, {3}}) + a.Equal(step(6), [3][]float64{{3, 6}, {3, 6}, {3, 6}}) + a.Equal(step(1), [3][]float64{{3, 6, 1}, {3, 6, 1}, {3, 6, 1}}) + a.Equal(step(2), [3][]float64{{3, 6, 1, 2}, {3, 6, 1, 2}, {3, 6, 1, 2}}) + a.Equal(step(1), [3][]float64{{3, 1}, {3, 1}, {3, 1}}) + a.Equal(step(10), [3][]float64{{3, 1, 1}, {3, 1, 10}, {3, 1, 10}}) +} + +func TestSetStress(t *testing.T) { + set := newSet(4, false) + var stop atomic.Bool + var seq atomic.Uint64 + start := func(f func()) { + go func() { + for !stop.Load() { + f() + } + }() + } + for p := 0; p < 2; p++ { + for _, opt := range []any{Link(""), NoGraph, Rate{}, Distribution{}} { + opt := opt + go func() { + v := set.Create(fmt.Sprintf("v%v", seq.Add(1)), "desc", opt) + for p1 := 0; p1 < 2; p1++ { + start(func() { v.Val() }) + start(func() { v.Add(rand.Intn(10000)) }) + } + }() + } + go func() { + var vv atomic.Uint64 + v := set.Create(fmt.Sprintf("v%v", seq.Add(1)), "desc", + func() int { return int(vv.Load()) }) + for p1 := 0; p1 < 2; p1++ { + start(func() { v.Val() }) + start(func() { vv.Store(uint64(rand.Intn(10000))) }) + } + }() + start(func() { set.Collect(All) }) + start(func() { set.RenderHTML() }) + start(func() { set.tick() }) + } + time.Sleep(time.Second) + stop.Store(true) +} -- cgit mrf-deployment