diff options
| author | Dmitry Vyukov <dvyukov@google.com> | 2017-02-24 14:11:40 +0300 |
|---|---|---|
| committer | Dmitry Vyukov <dvyukov@google.com> | 2017-02-24 22:01:03 +0300 |
| commit | a460a8a08280201c1f728c7939e0751cbb6b87cc (patch) | |
| tree | 7a5b30e67a20904cc3f98fba14d98665382898d1 | |
| parent | 19d8bc6235424c4b1734ded2f3cf723639bc2608 (diff) | |
syz-dash: assorted improvments
| -rw-r--r-- | syz-dash/api.go | 86 | ||||
| -rw-r--r-- | syz-dash/app.yaml | 6 | ||||
| -rw-r--r-- | syz-dash/bug.html | 89 | ||||
| -rw-r--r-- | syz-dash/common.html | 8 | ||||
| -rw-r--r-- | syz-dash/dash.html | 14 | ||||
| -rw-r--r-- | syz-dash/error.html | 12 | ||||
| -rw-r--r-- | syz-dash/handler.go | 250 | ||||
| -rw-r--r-- | syz-dash/index.yaml | 21 | ||||
| -rw-r--r-- | syz-dash/memcache.go | 106 | ||||
| -rw-r--r-- | syz-dash/patch.go | 2 | ||||
| -rw-r--r-- | syz-dash/patch_test.go | 18 | ||||
| -rw-r--r-- | syz-dash/static/style.css | 86 | ||||
| -rw-r--r-- | tools/syz-dashtool/dashtool.go | 62 |
13 files changed, 672 insertions, 88 deletions
diff --git a/syz-dash/api.go b/syz-dash/api.go index a1a049979..789ebf123 100644 --- a/syz-dash/api.go +++ b/syz-dash/api.go @@ -68,7 +68,7 @@ const ( BugStatusReported BugStatusFixed BugStatusUnclear - BugStatusClosed + BugStatusClosed = 1000 + iota BugStatusDeleted ) @@ -187,6 +187,8 @@ const ( maxLinkLen = 1000 maxOptsLen = 1000 maxCommentLen = 4000 + + maxCrashes = 20 ) func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { @@ -194,14 +196,16 @@ func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal crash: %v", err) } + addedBug := false + var group *Group if err := ds.RunInTransaction(c, func(c appengine.Context) error { now := time.Now() + addedBug = false manager := r.FormValue("client") crash := &Crash{ Manager: limitLength(manager, maxTextLen), Tag: limitLength(req.Tag, maxTextLen), - //Title: limitLength(req.Desc, maxTitleLen), - Time: now, + Time: now, } var err error if crash.Log, err = putText(c, "CrashLog", req.Log); err != nil { @@ -211,7 +215,7 @@ func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { return err } - group := &Group{Title: limitLength(req.Desc, maxTitleLen), Seq: 0} + group = &Group{Title: limitLength(req.Desc, maxTitleLen), Seq: 0} for { if err := ds.Get(c, group.Key(c), group); err != nil { if err != ds.ErrNoSuchEntity { @@ -220,7 +224,6 @@ func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { bug := &Bug{ Title: group.DisplayTitle(), Status: BugStatusNew, - //Updated: now, Groups: []string{group.hash()}, } bugKey, err := ds.Put(c, ds.NewIncompleteKey(c, "Bug", nil), bug) @@ -235,6 +238,7 @@ func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { if _, err := ds.Put(c, group.Key(c), group); err != nil { return err } + addedBug = true break } bug := new(Bug) @@ -269,9 +273,61 @@ func handleAddCrash(c appengine.Context, r *http.Request) (interface{}, error) { }, &ds.TransactionOptions{XG: true}); err != nil { return nil, err } + if addedBug { + dropCached(c) + } + purgeOldCrashes(c, group) return nil, nil } +func purgeOldCrashes(c appengine.Context, group *Group) int { + if group.NumCrashes <= maxCrashes { + return 0 + } + var keys []*ds.Key + var crashes []*Crash + keys, err := ds.NewQuery("Crash").Ancestor(group.Key(c)).Order("Time").Limit(2000).GetAll(c, &crashes) + if err != nil { + c.Errorf("Error: failed to fetch purge group crashes: %v", err) + return -1 + } + if len(keys) <= maxCrashes { + return 0 + } + keys = keys[:len(keys)-maxCrashes] + crashes = crashes[:len(crashes)-maxCrashes] + nn := len(keys) + for len(keys) != 0 { + n := len(keys) + if n > 200 { + n = 200 + } + var textKeys []*ds.Key + for _, crash := range crashes[:n] { + if crash.Log != 0 { + textKeys = append(textKeys, ds.NewKey(c, "Text", "", crash.Log, nil)) + } + if crash.Report != 0 { + textKeys = append(textKeys, ds.NewKey(c, "Text", "", crash.Report, nil)) + } + } + if len(textKeys) != 0 { + if err := ds.DeleteMulti(c, textKeys); err != nil { + c.Errorf("Error: failed to delete old crash texts: %v", err) + return -1 + } + } + if err := ds.DeleteMulti(c, keys[:n]); err != nil { + c.Errorf("Error: failed to delete old crashes: %v", err) + return -1 + } + keys = keys[n:] + crashes = crashes[n:] + } + c.Infof("deleted %v crashes '%v'", nn, group.Title) + return nn +} + func handleAddRepro(c appengine.Context, r *http.Request) (interface{}, error) { req := new(dashboard.Repro) if err := json.NewDecoder(r.Body).Decode(req); err != nil { @@ -394,10 +450,24 @@ func putText(c appengine.Context, tag string, data []byte) (int64, error) { if len(data) == 0 { return 0, nil } + const ( + maxTextLen = 2 << 20 + maxCompressedLen = 1000 << 10 // datastore entity limit is 1MB + ) + if len(data) > maxTextLen { + data = data[:maxTextLen] + } b := new(bytes.Buffer) - z, _ := gzip.NewWriterLevel(b, gzip.BestCompression) - z.Write(data) - z.Close() + for { + z, _ := gzip.NewWriterLevel(b, gzip.BestCompression) + z.Write(data) + z.Close() + if len(b.Bytes()) < maxCompressedLen { + break + } + data = data[:len(data)/10*9] + b.Reset() + } text := &Text{ Tag: tag, Text: b.Bytes(), diff --git a/syz-dash/app.yaml b/syz-dash/app.yaml index 72aa23688..a5d59f2fb 100644 --- a/syz-dash/app.yaml +++ b/syz-dash/app.yaml @@ -7,7 +7,11 @@ handlers: - url: /static static_dir: static secure: always -- url: /(|bug|text|client) +- url: /(|bug|text) + script: _go_app + login: required + secure: always +- url: /(client) script: _go_app login: admin secure: always diff --git a/syz-dash/bug.html b/syz-dash/bug.html index ffaf29272..a42d886b3 100644 --- a/syz-dash/bug.html +++ b/syz-dash/bug.html @@ -5,20 +5,26 @@ <link rel="stylesheet" href="/static/style.css"/> </head> <body> + {{template "header" .Header}} + <form action="/bug?id={{.ID}}" method="post"> <table> <tr> <td>Title:</td> - <td><input name="title" type="text" size="200" maxlength="200" value="{{.Title}}" required/></td> + <td><input name="title" type="text" size="200" maxlength="200" value="{{.Title}}" required {{if .Closed}}disabled{{end}}/></td> </tr> <tr> <td>Status:</td> - <td><select name="status"> - <option value="new" {{if eq .Status "new"}}selected{{end}}>New</option> - <option value="reported" {{if eq .Status "reported"}}selected{{end}}>Reported</option> - <option value="fixed" {{if eq .Status "fixed"}}selected{{end}}>Fixed</option> - <option value="unclear" {{if eq .Status "unclear"}}selected{{end}}>Unclear</option> - </select></td> + {{if .Closed}} + <td>{{.Status}}</td> + {{else}} + <td><select name="status"> + <option value="new" {{if eq .Status "new"}}selected{{end}}>New</option> + <option value="reported" {{if eq .Status "reported"}}selected{{end}}>Reported</option> + <option value="fixed" {{if eq .Status "fixed"}}selected{{end}}>Fixed</option> + <option value="unclear" {{if eq .Status "unclear"}}selected{{end}}>Unclear</option> + </select></td> + {{end}} </tr> <tr> <td>Crashes:</td> @@ -34,24 +40,28 @@ </tr> <tr> <td>Report link:</td> - <td><input name="report_link" type="text" size="200" maxlength="1000" value="{{.ReportLink}}"/></td> + <td><input name="report_link" type="text" size="200" maxlength="1000" value="{{.ReportLink}}" {{if .Closed}}disabled{{end}}/></td> </tr> <tr> <td>CVE:</td> - <td><input name="cve" type="text" size="200" maxlength="100" value="{{.CVE}}"/></td> + <td><input name="cve" type="text" size="200" maxlength="100" value="{{.CVE}}" {{if .Closed}}disabled{{end}}/></td> </tr> <tr> <td>Comment:</td> - <td><input name="comment" type="text" size="200" maxlength="4000" value="{{.Comment}}"/></td> + <td><input name="comment" type="text" size="200" maxlength="4000" value="{{.Comment}}" {{if .Closed}}disabled{{end}}/></td> </tr> </table> <input name="ver" type="hidden" value="{{.Version}}"/> - <input type="submit" name="action" value="Update"/> - <input type="submit" name="action" value="Close" title="fixed and is not happening anymore + {{if .Closed}} + <input type="submit" name="action" value="Reopen" class="button"/> + {{else}} + <input type="submit" name="action" value="Update" class="button"/> + <input type="submit" name="action" value="Close" class="button" title="fixed and is not happening anymore new crashes won't be associated with this bug and instead produce a new bug"/> - <input type="submit" name="action" value="Delete" title="trash/not interesting + <input type="submit" name="action" value="Delete" class="button" title="trash/not interesting new crashes will produce a new bug"/> + {{end}} <b>{{.Message}}</b> </form> <br> @@ -63,7 +73,8 @@ new crashes will produce a new bug"/> <option value="{{$b.ID}}">{{$b.Title}}</option> {{end}} </select> - <input type="submit" name="action" value="Merge"/> + <input type="submit" name="action" value="Merge" class="button"/> + <input name="ver" type="hidden" value="{{.Version}}"/> </form> <br> @@ -76,7 +87,7 @@ new crashes will produce a new bug"/> <td> <form action="/bug?id={{$.ID}}" method="post"> <input name="hash" type="hidden" value="{{$g.Hash}}"/> - <input type="submit" name="action" value="Unmerge"/> + <input type="submit" name="action" value="Unmerge" class="button"/> </form> </td> </tr> @@ -94,7 +105,7 @@ new crashes will produce a new bug"/> <td> <form action="/bug?id={{$.ID}}" method="post"> <input name="title" type="hidden" value="{{$p.Title}}"/> - <input type="submit" name="action" value="Delete patch"/> + <input type="submit" name="action" value="Delete patch" class="button"/> </form> </td> </tr> @@ -105,11 +116,40 @@ new crashes will produce a new bug"/> <form action="/bug?id={{$.ID}}" method="post"> <p><textarea name="patch" cols="88" rows="10" required></textarea></p> - <input type="submit" name="action" value="Add patch"/> + <input type="submit" name="action" value="Add patch" class="button"/> </form> <br> - <table> + {{if $.Repros}} + <table class="list_table"> + <caption>Reproducers:</caption> + <tr> + <th>Title</th> + <th>Manager</th> + <th>Time</th> + <th>Tag</th> + <th>Opts</th> + <th>Report</th> + <th>Prog</th> + <th>C prog</th> + </tr> + {{range $c := $.Repros}} + <tr> + <td class="title">{{$c.Title}}</td> + <td class="manager">{{$c.Manager}}</td> + <td class="time">{{formatTime $c.Time}}</td> + <td class="tag">{{$c.Tag}}</td> + <td class="opts" title="{{$c.Opts}}">{{$c.Opts}}</td> + <td class="report">{{if $c.Report}}<a href="/text?id={{$c.Report}}">report</a>{{end}}</td> + <td class="prog">{{if $c.Prog}}<a href="/text?id={{$c.Prog}}">prog</a>{{end}}</td> + <td class="cprog">{{if $c.CProg}}<a href="/text?id={{$c.CProg}}">C prog</a>{{end}}</td> + </tr> + {{end}} + </table> + <br> + {{end}} + + <table class="list_table"> <caption>Crashes:</caption> <tr> <th>Title</th> @@ -121,14 +161,15 @@ new crashes will produce a new bug"/> </tr> {{range $c := $.Crashes}} <tr> - <td>{{$c.Title}}</td> - <td>{{$c.Manager}}</td> - <td>{{formatTime $c.Time}}</td> - <td>{{$c.Tag}}</td> - <td>{{if $c.Log}}<a href="/text?id={{$c.Log}}">log</a>{{end}}</td> - <td>{{if $c.Report}}<a href="/text?id={{$c.Report}}">report</a>{{end}}</td> + <td class="title">{{$c.Title}}</td> + <td class="manager">{{$c.Manager}}</td> + <td class="time">{{formatTime $c.Time}}</td> + <td class="tag">{{$c.Tag}}</td> + <td class="log">{{if $c.Log}}<a href="/text?id={{$c.Log}}">log</a>{{end}}</td> + <td class="report">{{if $c.Report}}<a href="/text?id={{$c.Report}}">report</a>{{end}}</td> </tr> {{end}} </table> + <br> </body> </html> diff --git a/syz-dash/common.html b/syz-dash/common.html new file mode 100644 index 000000000..ab9e8c66d --- /dev/null +++ b/syz-dash/common.html @@ -0,0 +1,8 @@ +{{define "header"}} + <header id="topbar"> + <h1><a href="/">syzkaller</a></h1> + {{.Found}} bugs found, {{.Fixed}} bugs fixed, {{.Crashed}} kernels crashed + </header> +{{end}} + + diff --git a/syz-dash/dash.html b/syz-dash/dash.html index a7ca4159b..4b14a418c 100644 --- a/syz-dash/dash.html +++ b/syz-dash/dash.html @@ -1,18 +1,20 @@ {{define "bug_table"}} - <table> + <table class="list_table"> <caption>{{.Name}}:</caption> <tr> <th>Title</th> <th>Count</th> + <th>Repro</th> <th>Time</th> <th>Happens on</th> </tr> {{range $b := $.Bugs}} <tr> - <td><a href="/bug?id={{$b.ID}}">{{$b.Title}}</a></td> - <td>{{$b.NumCrashes}}</td> - <td>{{formatTime $b.LastTime}} - {{formatTime $b.FirstTime}}</td> - <td>{{$b.Managers}}</td> + <td class="title"><a href="/bug?id={{$b.ID}}">{{$b.Title}}</a></td> + <td class="count">{{$b.NumCrashes}}</td> + <td class="repro">{{$b.Repro}}</td> + <td class="time">{{formatTime $b.LastTime}}</td> + <td class="managers" title="{{$b.Managers}}">{{$b.Managers}}</td> </tr> {{end}} </table> @@ -25,6 +27,8 @@ <link rel="stylesheet" href="/static/style.css"/> </head> <body> + {{template "header" .Header}} + {{range $g := $.BugGroups}} {{if $g.Bugs}} {{template "bug_table" $g}} <br> diff --git a/syz-dash/error.html b/syz-dash/error.html new file mode 100644 index 000000000..aec5f52e1 --- /dev/null +++ b/syz-dash/error.html @@ -0,0 +1,12 @@ +<!doctype html> +<html> +<head> + <title>Syzkaller Dashboard</title> + <link rel="stylesheet" href="/static/style.css"/> +</head> +<body> + {{.}} + <br> + <a href="javascript:history.back()">back</a> +</body> +</html> diff --git a/syz-dash/handler.go b/syz-dash/handler.go index e08a882f0..7861fee0d 100644 --- a/syz-dash/handler.go +++ b/syz-dash/handler.go @@ -17,25 +17,40 @@ import ( "appengine" ds "appengine/datastore" + "appengine/user" ) func init() { - http.Handle("/", handlerWrapper(handleDash)) - http.Handle("/bug", handlerWrapper(handleBug)) - http.Handle("/text", handlerWrapper(handleText)) - http.Handle("/client", handlerWrapper(handleClient)) + http.Handle("/", handlerWrapper(handleAuth(handleDash))) + http.Handle("/bug", handlerWrapper(handleAuth(handleBug))) + http.Handle("/text", handlerWrapper(handleAuth(handleText))) + http.Handle("/client", handlerWrapper(handleAuth(handleClient))) } -func handlerWrapper(fn func(c appengine.Context, w http.ResponseWriter, r *http.Request) error) http.Handler { +type aeHandler func(c appengine.Context, w http.ResponseWriter, r *http.Request) error + +func handlerWrapper(fn aeHandler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) if err := fn(c, w, r); err != nil { c.Errorf("Error: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) + if err1 := templates.ExecuteTemplate(w, "error.html", err.Error()); err1 != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } }) } +func handleAuth(fn aeHandler) aeHandler { + return func(c appengine.Context, w http.ResponseWriter, r *http.Request) error { + u := user.Current(c) + if !u.Admin && (u.AuthDomain != "gmail.com" || !strings.HasSuffix(u.Email, "@google.com")) { + return fmt.Errorf("You are not authorized to view this. This incident will be reported.") + } + return fn(c, w, r) + } +} + func handleClient(c appengine.Context, w http.ResponseWriter, r *http.Request) error { name := r.FormValue("name") if name == "" { @@ -44,7 +59,7 @@ func handleClient(c appengine.Context, w http.ResponseWriter, r *http.Request) e return fmt.Errorf("failed to fetch clients: %v", err) } for _, client := range clients { - fmt.Fprintf(w, "%v: %v<br>\n", client.Name, client.Key) + fmt.Fprintf(w, "%v: %v\n", client.Name, client.Key) } return nil } @@ -79,10 +94,21 @@ func handleDash(c appengine.Context, w http.ResponseWriter, r *http.Request) err } data.BugGroups = append(data.BugGroups, bugGroups[BugStatusNew], bugGroups[BugStatusReported], bugGroups[BugStatusUnclear], bugGroups[BugStatusFixed]) + all := r.FormValue("all") != "" + if all { + bugGroups[BugStatusClosed] = &uiBugGroup{Name: "Closed bugs"} + bugGroups[BugStatusDeleted] = &uiBugGroup{Name: "Deleted bugs"} + data.BugGroups = append(data.BugGroups, bugGroups[BugStatusClosed], bugGroups[BugStatusDeleted]) + } + var bugs []*Bug var keys []*ds.Key var err error - if keys, err = ds.NewQuery("Bug").Filter("Status <", BugStatusClosed).GetAll(c, &bugs); err != nil { + query := ds.NewQuery("Bug").Project("Title", "Status") + if !all { + query = query.Filter("Status <", BugStatusClosed) + } + if keys, err = query.GetAll(c, &bugs); err != nil { return fmt.Errorf("failed to fetch bugs: %v", err) } bugMap := make(map[int64]*uiBug) @@ -90,10 +116,9 @@ func handleDash(c appengine.Context, w http.ResponseWriter, r *http.Request) err for i, bug := range bugs { id := keys[i].IntID() ui := &uiBug{ - ID: id, - Title: bug.Title, - Status: statusToString(bug.Status), - Comment: bug.Comment, + ID: id, + Title: bug.Title, + Status: statusToString(bug.Status), } bugMap[id] = ui managers[id] = make(map[string]bool) @@ -107,9 +132,17 @@ func handleDash(c appengine.Context, w http.ResponseWriter, r *http.Request) err for _, group := range groups { ui := bugMap[group.Bug] if ui == nil { + if !all { + continue + } return fmt.Errorf("failed to find bug for crash %v (%v)", group.Title, group.Seq) } ui.NumCrashes += group.NumCrashes + if group.HasCRepro { + ui.Repro = "C repro" + } else if group.HasRepro && ui.Repro != "C repro" { + ui.Repro = "repro" + } if ui.FirstTime.IsZero() || ui.FirstTime.After(group.FirstTime) { ui.FirstTime = group.FirstTime } @@ -134,7 +167,14 @@ func handleDash(c appengine.Context, w http.ResponseWriter, r *http.Request) err for _, group := range data.BugGroups { sort.Sort(uiBugArray(group.Bugs)) } - return templateDash.Execute(w, data) + + cached, err := getCached(c) + if err != nil { + return err + } + data.Header = headerFromCached(cached) + + return templates.ExecuteTemplate(w, "dash.html", data) } func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) error { @@ -142,9 +182,14 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro if err != nil { return fmt.Errorf("failed to parse bug id: %v", err) } + action := r.FormValue("action") + if action != "" && !user.IsAdmin(c) { + return fmt.Errorf("can't touch this") + } + msg := "" bug := new(Bug) - switch r.FormValue("action") { + switch action { case "Update": ver, err := strconv.ParseInt(r.FormValue("ver"), 10, 64) if err != nil { @@ -162,14 +207,17 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro return fmt.Errorf("title can't be empty") } switch status { - case BugStatusReported: - case BugStatusFixed: + case BugStatusReported, BugStatusFixed: + if reportLink == "" { + return fmt.Errorf("enter report link") + } case BugStatusUnclear: if comment == "" { return fmt.Errorf("enter comment as to why it's unclear") } } + flushCached := false if err := ds.RunInTransaction(c, func(c appengine.Context) error { if err := ds.Get(c, ds.NewKey(c, "Bug", "", id, nil), bug); err != nil { return err @@ -177,6 +225,10 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro if bug.Version != ver { return fmt.Errorf("bug has changed by somebody else") } + if status == BugStatusFixed && len(bug.Patches) == 0 { + return fmt.Errorf("add a patch for fixed bugs") + } + flushCached = bug.Status != status bug.Title = title bug.Status = status bug.ReportLink = reportLink @@ -190,7 +242,47 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro }, nil); err != nil { return err } + if flushCached { + dropCached(c) + } + msg = "bug is updated" + case "Close", "Delete", "Reopen": + ver, err := strconv.ParseInt(r.FormValue("ver"), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse bug version: %v", err) + } + if err := ds.RunInTransaction(c, func(c appengine.Context) error { + if err := ds.Get(c, ds.NewKey(c, "Bug", "", id, nil), bug); err != nil { + return err + } + if bug.Version != ver { + return fmt.Errorf("bug has changed by somebody else") + } + switch action { + case "Close": + bug.Status = BugStatusClosed + msg = "bug is closed" + case "Delete": + bug.Status = BugStatusDeleted + msg = "bug is deleted" + case "Reopen": + bug.Status = BugStatusNew + msg = "bug is reopened" + } + bug.Version++ + if _, err := ds.Put(c, ds.NewKey(c, "Bug", "", id, nil), bug); err != nil { + return err + } + return nil + }, nil); err != nil { + return err + } + dropCached(c) case "Merge": + ver, err := strconv.ParseInt(r.FormValue("ver"), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse bug version: %v", err) + } otherID, err := strconv.ParseInt(r.FormValue("bug_id"), 10, 64) if err != nil { return fmt.Errorf("failed to parse bug id: %v", err) @@ -200,10 +292,29 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro if err := ds.Get(c, ds.NewKey(c, "Bug", "", id, nil), srcBug); err != nil { return err } + if srcBug.Version != ver { + return fmt.Errorf("bug has changed by somebody else") + } dstBug := new(Bug) if err := ds.Get(c, ds.NewKey(c, "Bug", "", otherID, nil), dstBug); err != nil { return err } + if dstBug.Status >= BugStatusClosed { + return fmt.Errorf("target bug is already closed") + } + mergeStrings := func(s1, s2 string) string { + if s1 == "" { + return s2 + } else if s2 == "" { + return s1 + } else { + return s1 + ", " + s2 + } + } + dstBug.Version++ + dstBug.ReportLink = mergeStrings(dstBug.ReportLink, srcBug.ReportLink) + dstBug.Comment = mergeStrings(dstBug.Comment, srcBug.Comment) + dstBug.CVE = mergeStrings(dstBug.ReportLink, srcBug.CVE) var groupKeys []*ds.Key var groups []*Group for _, hash := range srcBug.Groups { @@ -220,6 +331,15 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro } } dstBug.Groups = append(dstBug.Groups, srcBug.Groups...) + nextPatch: + for _, patch := range srcBug.Patches { + for _, patch1 := range dstBug.Patches { + if patch1.Title == patch.Title { + continue nextPatch + } + } + dstBug.Patches = append(dstBug.Patches, patch) + } if _, err := ds.Put(c, ds.NewKey(c, "Bug", "", otherID, nil), dstBug); err != nil { return err } @@ -232,6 +352,7 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro }, &ds.TransactionOptions{XG: true}); err != nil { return err } + dropCached(c) http.Redirect(w, r, fmt.Sprintf("bug?id=%v", otherID), http.StatusMovedPermanently) return nil case "Unmerge": @@ -263,7 +384,6 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro newBug := &Bug{ Title: group.DisplayTitle(), Status: BugStatusNew, - //Updated: now, Groups: []string{group.hash()}, } bugKey, err := ds.Put(c, ds.NewIncompleteKey(c, "Bug", nil), newBug) @@ -274,10 +394,12 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro if _, err := ds.Put(c, group.Key(c), group); err != nil { return err } + msg = fmt.Sprintf("group '%v' is unmerged into separate bug", group.Title) return nil }, &ds.TransactionOptions{XG: true}); err != nil { return err } + dropCached(c) case "Add patch": title, diff, err := parsePatch(r.FormValue("patch")) if err != nil { @@ -308,6 +430,7 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro }, &ds.TransactionOptions{XG: true}); err != nil { return err } + msg = fmt.Sprintf("patch '%v' added", title) case "Delete patch": title := r.FormValue("title") if err := ds.RunInTransaction(c, func(c appengine.Context) error { @@ -333,6 +456,7 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro }, &ds.TransactionOptions{XG: true}); err != nil { return err } + msg = fmt.Sprintf("patch '%v' deleted", title) case "": if err := ds.Get(c, ds.NewKey(c, "Bug", "", id, nil), bug); err != nil { return err @@ -349,22 +473,22 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro data.CVE = bug.CVE data.Comment = bug.Comment data.Status = statusToString(bug.Status) + data.Closed = bug.Status >= BugStatusClosed data.Patches = bug.Patches - //data.Updated = bug.Updated + data.Message = msg - var bugs []*Bug - var keys []*ds.Key - if keys, err = ds.NewQuery("Bug").Filter("Status <", BugStatusClosed).GetAll(c, &bugs); err != nil { - return fmt.Errorf("failed to fetch bugs: %v", err) + cached, err := getCached(c) + if err != nil { + return err } - for i, bug1 := range bugs { - id1 := keys[i].IntID() - if id1 == id { + data.Header = headerFromCached(cached) + for _, bug1 := range cached.Bugs { + if bug1.ID == id { continue } data.AllBugs = append(data.AllBugs, &uiBug{ - ID: id1, - Title: fmt.Sprintf("%v (%v)", bug1.Title, statusToString(bug1.Status)), + ID: bug1.ID, + Title: bug1.Title, }) } sort.Sort(uiBugTitleSorter(data.AllBugs)) @@ -402,9 +526,27 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro Report: crash.Report, }) } + + var repros []*Repro + if _, err := ds.NewQuery("Repro").Ancestor(group.Key(c)).GetAll(c, &repros); err != nil { + return fmt.Errorf("failed to fetch repros: %v", err) + } + for _, repro := range repros { + data.Repros = append(data.Repros, &uiRepro{ + Title: group.DisplayTitle(), + Manager: repro.Manager, + Tag: repro.Tag, + Time: repro.Time, + Report: repro.Report, + Opts: repro.Opts, + Prog: repro.Prog, + CProg: repro.CProg, + }) + } } sort.Sort(uiCrashArray(data.Crashes)) + sort.Sort(uiReproArray(data.Repros)) if len(data.Groups) == 1 { data.Groups = nil @@ -417,7 +559,7 @@ func handleBug(c appengine.Context, w http.ResponseWriter, r *http.Request) erro sort.Strings(arr) data.Managers = strings.Join(arr, ", ") - return templateBug.Execute(w, data) + return templates.ExecuteTemplate(w, "bug.html", data) } func handleText(c appengine.Context, w http.ResponseWriter, r *http.Request) error { @@ -434,13 +576,30 @@ func handleText(c appengine.Context, w http.ResponseWriter, r *http.Request) err return nil } +type dataHeader struct { + Found int64 + Fixed int64 + Crashed int64 +} + +func headerFromCached(cached *Cached) *dataHeader { + return &dataHeader{ + Found: cached.Found, + Fixed: cached.Fixed, + Crashed: cached.Crashed, + } +} + type dataDash struct { + Header *dataHeader BugGroups []*uiBugGroup } type dataBug struct { + Header *dataHeader uiBug Crashes []*uiCrash + Repros []*uiRepro Message string AllBugs []*uiBug } @@ -460,10 +619,11 @@ type uiBug struct { Version int64 Title string Status string + Closed bool NumCrashes int64 + Repro string FirstTime time.Time LastTime time.Time - Updated time.Time Managers string ReportLink string Comment string @@ -481,6 +641,17 @@ type uiCrash struct { Report int64 } +type uiRepro struct { + Title string + Manager string + Tag string + Time time.Time + Report int64 + Opts string + Prog int64 + CProg int64 +} + type uiBugArray []*uiBug func (a uiBugArray) Len() int { @@ -516,13 +687,27 @@ func (a uiCrashArray) Len() int { } func (a uiCrashArray) Less(i, j int) bool { - return a[i].Time.Before(a[j].Time) + return a[i].Time.After(a[j].Time) } func (a uiCrashArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +type uiReproArray []*uiRepro + +func (a uiReproArray) Len() int { + return len(a) +} + +func (a uiReproArray) Less(i, j int) bool { + return a[i].Time.After(a[j].Time) +} + +func (a uiReproArray) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + type dataPatches struct { Message string Patches []*Patch @@ -536,7 +721,4 @@ func formatTime(t time.Time) string { return t.Format("Jan 02 15:04") } -var ( - templateDash = template.Must(template.New("dash.html").Funcs(tmplFuncs).ParseFiles("dash.html")) - templateBug = template.Must(template.New("bug.html").Funcs(tmplFuncs).ParseFiles("bug.html")) -) +var templates = template.Must(template.New("").Funcs(tmplFuncs).ParseGlob("*.html")) diff --git a/syz-dash/index.yaml b/syz-dash/index.yaml new file mode 100644 index 000000000..d330080f1 --- /dev/null +++ b/syz-dash/index.yaml @@ -0,0 +1,21 @@ +indexes: + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. + +- kind: Bug + properties: + - name: Status + - name: Title + +- kind: Crash + ancestor: yes + properties: + - name: Time diff --git a/syz-dash/memcache.go b/syz-dash/memcache.go new file mode 100644 index 000000000..9653eef90 --- /dev/null +++ b/syz-dash/memcache.go @@ -0,0 +1,106 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +// +build appengine + +package dash + +import ( + "fmt" + "time" + + "appengine" + ds "appengine/datastore" + "appengine/memcache" +) + +const cachedKey = "cached" + +type Cached struct { + Bugs []CachedBug + Found int64 + Fixed int64 + Crashed int64 +} + +type CachedBug struct { + ID int64 + Title string +} + +func getCached(c appengine.Context) (*Cached, error) { + cached := new(Cached) + if _, err := memcache.Gob.Get(c, cachedKey, cached); err == nil { + return cached, nil + } else if err != memcache.ErrCacheMiss { + c.Errorf("failed to get cached object: %v", err) + } + cached, err := buildCached(c) + if err != nil { + return nil, fmt.Errorf("failed to build cached object: %v", err) + } + item := &memcache.Item{ + Key: cachedKey, + Object: cached, + Expiration: time.Hour, + } + if err := memcache.Gob.Set(c, item); err != nil { + c.Errorf("failed to set cached object: %v", err) + } + return cached, nil +} + +func dropCached(c appengine.Context) { + if err := memcache.Delete(c, cachedKey); err != nil && err != memcache.ErrCacheMiss { + c.Errorf("failed to drop memcache: %v", err) + } +} + +func buildCached(c appengine.Context) (*Cached, error) { + cached := &Cached{} + var bugs []*Bug + var keys []*ds.Key + var err error + if keys, err = ds.NewQuery("Bug").Project("Title", "Status").GetAll(c, &bugs); err != nil { + return nil, fmt.Errorf("failed to fetch bugs: %v", err) + } + bugStatus := make(map[int64]int) + for i, bug := range bugs { + id := keys[i].IntID() + bugStatus[id] = bug.Status + if bug.Status < BugStatusClosed { + cached.Bugs = append(cached.Bugs, CachedBug{ + ID: id, + Title: fmt.Sprintf("%v (%v)", bug.Title, statusToString(bug.Status)), + }) + } + switch bug.Status { + case BugStatusNew, BugStatusReported, BugStatusUnclear: + cached.Found++ + case BugStatusFixed, BugStatusClosed: + cached.Found++ + cached.Fixed++ + case BugStatusDeleted: + default: + return nil, fmt.Errorf("unknown status %v", bug.Status) + } + } + var groups []*Group + if _, err := ds.NewQuery("Group").GetAll(c, &groups); err != nil { + return nil, fmt.Errorf("failed to fetch crash groups: %v", err) + } + for _, group := range groups { + status, ok := bugStatus[group.Bug] + if !ok { + return nil, fmt.Errorf("failed to find bug for crash %v (%v)", group.Title, group.Seq) + } + switch status { + case BugStatusNew, BugStatusReported, BugStatusFixed, BugStatusUnclear, BugStatusClosed: + cached.Crashed += group.NumCrashes + case BugStatusDeleted: + default: + return nil, fmt.Errorf("unknown status %v", status) + } + } + return cached, nil +} diff --git a/syz-dash/patch.go b/syz-dash/patch.go index d3e6b4773..b7f76c902 100644 --- a/syz-dash/patch.go +++ b/syz-dash/patch.go @@ -16,7 +16,7 @@ func parsePatch(text string) (title string, diff string, err error) { lastLine := "" for s.Scan() { ln := s.Text() - if strings.HasPrefix(ln, "--- a/") { + if strings.HasPrefix(ln, "--- a/") || strings.HasPrefix(ln, "--- /dev/null") { parsingDiff = true if title == "" { title = lastLine diff --git a/syz-dash/patch_test.go b/syz-dash/patch_test.go index 5d45a3187..e36d068d5 100644 --- a/syz-dash/patch_test.go +++ b/syz-dash/patch_test.go @@ -285,4 +285,22 @@ Subject: Re: [PATCH v3] net/irda: fix lockdep annotation }; `, }, + + { + text: `syz-dash: first version of dashboard app +diff --git a/syz-dash/api.go b/syz-dash/api.go +new file mode 100644 +index 0000000..a1a0499 +--- /dev/null ++++ b/syz-dash/api.go +@@ -0,0 +1,444 @@ ++package dash +`, + title: "syz-dash: first version of dashboard app", + diff: `--- /dev/null ++++ b/syz-dash/api.go +@@ -0,0 +1,444 @@ ++package dash +`, + }, } diff --git a/syz-dash/static/style.css b/syz-dash/static/style.css index edd3be3a8..e2558bacf 100644 --- a/syz-dash/static/style.css +++ b/syz-dash/static/style.css @@ -1,15 +1,85 @@ +#topbar { + padding: 5px 10px; + background: #E0EBF5; +} + +#topbar a { + color: #375EAB; + text-decoration: none; +} + +h1, h2, h3, h4 { + margin: 0; + padding: 0; + color: #375EAB; + font-weight: bold; +} + table { - border-collapse:collapse; - border:1px solid; + border: 1px solid #ccc; + margin: 20px 5px; + border-collapse: collapse; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; } + table caption { font-weight: bold; } -table td { - border:1px solid; - padding: 3px; + +table td, table th { + vertical-align: top; + padding: 2px 8px; + text-overflow: ellipsis; + overflow: hidden; +} + +.list_table td, .list_table th { + border-left: 1px solid #ccc; +} + +.list_table th { + background: #F4F4F4; +} + +.list_table tr:nth-child(2n+1) { + background: #F4F4F4; +} + +.list_table tr:hover { + background: #ffff99; +} + +.list_table .title { + width: 400pt; + max-width: 400pt; +} + +.list_table .count { + text-align: right; +} + +.list_table .tag { + font-family: monospace; + font-size: 9pt; +} + +.list_table .opts { + width: 40pt; + max-width: 40pt; +} + +.list_table .managers { + width: 100pt; + max-width: 100pt; } -table th { - border:1px solid; - padding: 3px; + +.button { + color: #222; + border: 1px solid #375EAB; + background: #E0EBF5; + border-radius: 3px; + cursor: pointer; + margin-left: 10px; } diff --git a/tools/syz-dashtool/dashtool.go b/tools/syz-dashtool/dashtool.go index d3d37c7b6..947b028d8 100644 --- a/tools/syz-dashtool/dashtool.go +++ b/tools/syz-dashtool/dashtool.go @@ -6,6 +6,7 @@ package main import ( + "bytes" "flag" "fmt" "io/ioutil" @@ -36,16 +37,22 @@ func main() { Key: *flagKey, } if len(flag.Args()) == 0 { - fmt.Fprintf(os.Stderr, "specify command: report, report-all\n") + fmt.Fprintf(os.Stderr, "specify command: report-crash/report-repro/report-all\n") os.Exit(1) } switch flag.Args()[0] { - case "report": + case "report-crash": if len(flag.Args()) != 2 { - fmt.Fprintf(os.Stderr, "usage: report logN\n") + fmt.Fprintf(os.Stderr, "usage: report-crash logN\n") os.Exit(1) } - report(dash, flag.Args()[1]) + reportCrash(dash, flag.Args()[1]) + case "report-repro": + if len(flag.Args()) != 2 { + fmt.Fprintf(os.Stderr, "usage: report-repro crashdir\n") + os.Exit(1) + } + reportRepro(dash, flag.Args()[1]) case "report-all": if len(flag.Args()) != 2 { fmt.Fprintf(os.Stderr, "usage: report-all workdir/crashes\n") @@ -58,7 +65,7 @@ func main() { } } -func report(dash *dashboard.Dashboard, logfile string) { +func reportCrash(dash *dashboard.Dashboard, logfile string) { n := -1 for i := range logfile { x, err := strconv.Atoi(logfile[i:]) @@ -80,7 +87,7 @@ func report(dash *dashboard.Dashboard, logfile string) { } desc, err := ioutil.ReadFile(filepath.Join(dir, "description")) if err != nil { - fmt.Fprintf(os.Stderr, "failed to description file: %v\n", err) + fmt.Fprintf(os.Stderr, "failed to read description file: %v\n", err) os.Exit(1) } tag, _ := ioutil.ReadFile(filepath.Join(dir, fmt.Sprintf("tag%v", n))) @@ -99,6 +106,47 @@ func report(dash *dashboard.Dashboard, logfile string) { } } +func reportRepro(dash *dashboard.Dashboard, crashdir string) { + desc, err := ioutil.ReadFile(filepath.Join(crashdir, "description")) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read description file: %v\n", err) + os.Exit(1) + } + prog, err := ioutil.ReadFile(filepath.Join(crashdir, "repro.prog")) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to repro.prog file: %v\n", err) + os.Exit(1) + } + report, err := ioutil.ReadFile(filepath.Join(crashdir, "repro.report")) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to repro.report file: %v\n", err) + os.Exit(1) + } + tag, _ := ioutil.ReadFile(filepath.Join(crashdir, "repro.tag")) + cprog, _ := ioutil.ReadFile(filepath.Join(crashdir, "repro.cprog")) + opts := "" + if nl := bytes.IndexByte(prog, '\n'); nl > 1 && prog[0] == '#' { + opts = string(prog[:nl-1]) + prog = prog[nl+1:] + } + + repro := &dashboard.Repro{ + Crash: dashboard.Crash{ + Tag: string(tag), + Desc: string(desc), + Report: report, + }, + Reproduced: true, + Opts: opts, + Prog: prog, + CProg: cprog, + } + if err := dash.ReportRepro(repro); err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(1) + } +} + func reportAll(dash *dashboard.Dashboard, crashes string) { dirs, err := ioutil.ReadDir(crashes) if err != nil { @@ -115,7 +163,7 @@ func reportAll(dash *dashboard.Dashboard, crashes string) { if !strings.HasPrefix(file.Name(), "log") { continue } - report(dash, filepath.Join(crashes, dir.Name(), file.Name())) + reportCrash(dash, filepath.Join(crashes, dir.Name(), file.Name())) } } } |
