aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDmitry Vyukov <dvyukov@google.com>2017-08-17 19:09:07 +0200
committerDmitry Vyukov <dvyukov@google.com>2017-08-17 19:42:11 +0200
commit172189e9551b768d48c6d5c038fbf3d5cd88aa8e (patch)
treeb03dcab8c156938bd4ac02def77a15a326a04550
parent2dfba870d0da6e4638fd58c47099bdea9495ac25 (diff)
dashboard/app: heavylifting of email reporting
- save Message-ID and use In-Reply-To in subsequent messages - remember additional CC entries added manually - don't mail to maintainers if maintainers list is empty - improve mail formatting and add a footer - implement upstream/fix/dup/invalid commands over email - add tests
-rw-r--r--dashboard/app/api.go5
-rw-r--r--dashboard/app/app_test.go4
-rw-r--r--dashboard/app/config.go2
-rw-r--r--dashboard/app/email_test.go419
-rw-r--r--dashboard/app/entities.go28
-rw-r--r--dashboard/app/mail_bug.txt24
-rw-r--r--dashboard/app/main.go2
-rw-r--r--dashboard/app/reporting.go118
-rw-r--r--dashboard/app/reporting_email.go60
-rw-r--r--dashboard/app/reporting_test.go2
-rw-r--r--dashboard/app/util_test.go13
-rw-r--r--dashboard/dashapi/dashapi.go7
-rw-r--r--pkg/email/parser.go90
-rw-r--r--pkg/email/parser_test.go126
-rw-r--r--pkg/email/reply_test.go12
15 files changed, 763 insertions, 149 deletions
diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 9fd7d36d4..c59e42983 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -16,6 +16,7 @@ import (
"unicode/utf8"
"github.com/google/syzkaller/dashboard/dashapi"
+ "github.com/google/syzkaller/pkg/email"
"github.com/google/syzkaller/pkg/hash"
"golang.org/x/net/context"
"google.golang.org/appengine"
@@ -319,9 +320,7 @@ func apiReportCrash(c context.Context, ns string, r *http.Request) (interface{},
return nil, fmt.Errorf("failed to unmarshal request: %v", err)
}
req.Title = limitLength(req.Title, maxTextLen)
- if len(req.Maintainers) > maxMaintainers {
- req.Maintainers = req.Maintainers[:maxMaintainers]
- }
+ req.Maintainers = email.MergeEmailLists(req.Maintainers)
build, err := loadBuild(c, ns, req.BuildID)
if err != nil {
diff --git a/dashboard/app/app_test.go b/dashboard/app/app_test.go
index cc5f471b7..03dfde6c2 100644
--- a/dashboard/app/app_test.go
+++ b/dashboard/app/app_test.go
@@ -83,6 +83,10 @@ func (cfg *TestConfig) Type() string {
return "test"
}
+func (cfg *TestConfig) NeedMaintainers() bool {
+ return false
+}
+
func (cfg *TestConfig) Validate() error {
return nil
}
diff --git a/dashboard/app/config.go b/dashboard/app/config.go
index a94540d0e..a40526b97 100644
--- a/dashboard/app/config.go
+++ b/dashboard/app/config.go
@@ -57,6 +57,8 @@ type Reporting struct {
type ReportingType interface {
// Type returns a unique string that identifies this reporting type (e.g. "email").
Type() string
+ // NeedMaintainers says if this reporting requires non-empty maintainers list.
+ NeedMaintainers() bool
// Validate validates the current object, this is called only during init.
Validate() error
}
diff --git a/dashboard/app/email_test.go b/dashboard/app/email_test.go
index 389bbbd1f..61f5da351 100644
--- a/dashboard/app/email_test.go
+++ b/dashboard/app/email_test.go
@@ -6,8 +6,11 @@
package dash
import (
+ "fmt"
+ "strings"
"testing"
+ "github.com/google/syzkaller/dashboard/dashapi"
"github.com/google/syzkaller/pkg/email"
)
@@ -22,35 +25,417 @@ func TestEmailReport(t *testing.T) {
crash.Maintainers = []string{`"Foo Bar" <foo@bar.com>`, `bar@foo.com`}
c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
- c.expectOK(c.GET("/email_poll"))
- c.expectEQ(len(c.emailSink), 1)
- msg := <-c.emailSink
- sender, _, err := email.RemoveAddrContext(msg.Sender)
- if err != nil {
- t.Fatalf("failed to remove sender context: %v", err)
+ // Report the crash over email and check all fields.
+ sender0 := ""
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ sender0 = msg.Sender
+ sender, _, err := email.RemoveAddrContext(msg.Sender)
+ if err != nil {
+ t.Fatalf("failed to remove sender context: %v", err)
+ }
+ c.expectEQ(sender, fromAddr(c.ctx))
+ to := config.Namespaces["test2"].Reporting[0].Config.(*EmailConfig).Email
+ c.expectEQ(msg.To, []string{to})
+ c.expectEQ(msg.Subject, crash.Title)
+ c.expectEQ(len(msg.Attachments), 2)
+ c.expectEQ(msg.Attachments[0].Name, "config.txt")
+ c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+ c.expectEQ(msg.Attachments[1].Name, "raw.log")
+ c.expectEQ(msg.Attachments[1].Data, crash.Log)
+ body := `Hello,
+
+syzkaller hit the following crash on kernel_commit1
+repo1/branch1
+compiler: compiler1
+.config is attached
+Raw console output is attached.
+
+
+CC: [bar@foo.com foo@bar.com]
+
+report1
+
+---
+This bug is generated by a dumb bot. It may contain errors.
+See https://goo.gl/tpsmEJ for details.
+Direct all questions to syzkaller@googlegroups.com.
+
+syzbot will keep track of this bug report.
+Once a fix for this bug is committed, please reply to this email with:
+#syz fix: exact-commit-title
+To mark this as a duplicate of another syzbot report, please reply with:
+#syz dup: exact-subject-of-another-report
+If it's a one-off invalid bug report, please reply with:
+#syz invalid
+Note: if the crash happens again, it will cause creation of a new bug report.
+To upstream this report, please reply with:
+#syz upstream`
+ c.expectEQ(msg.Body, body)
}
- c.expectEQ(sender, fromAddr(c.ctx))
- to := config.Namespaces["test2"].Reporting[0].Config.(*EmailConfig).Email
- c.expectEQ(msg.To, []string{to})
- c.expectEQ(msg.Subject, crash.Title)
- c.expectEQ(len(msg.Attachments), 1)
- c.expectEQ(msg.Attachments[0].Name, "config.txt")
- c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
- body := `
-Hello,syzkaller hit the following crash on kernel_commit1
+
+ // Emulate receive of the report from a mailing list.
+ // This should update the bug with the link/Message-ID.
+ incoming1 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <1234>
+Subject: crash1
+From: %v
+To: foo@bar.com
+Content-Type: text/plain
+
+Hello
+
+syzbot will keep track of this bug report.
+Once a fix for this bug is committed, please reply to this email with:
+#syz fix: exact-commit-title
+To mark this as a duplicate of another syzbot report, please reply with:
+#syz dup: exact-subject-of-another-report
+If it's a one-off invalid bug report, please reply with:
+#syz invalid
+
+--
+You received this message because you are subscribed to the Google Groups "syzkaller" group.
+To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller+unsubscribe@googlegroups.com.
+To post to this group, send email to syzkaller@googlegroups.com.
+To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller/1234@google.com.
+For more options, visit https://groups.google.com/d/optout.
+`, sender0)
+
+ c.expectOK(c.POST("/_ah/mail/", incoming1))
+
+ // Now report syz reproducer and check updated email.
+ crash.ReproOpts = []byte("repro opts")
+ crash.ReproSyz = []byte("getpid()")
+ syzRepro := []byte(fmt.Sprintf("#%s\n%s", crash.ReproOpts, crash.ReproSyz))
+ c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ c.expectEQ(msg.Sender, sender0)
+ sender, _, err := email.RemoveAddrContext(msg.Sender)
+ if err != nil {
+ t.Fatalf("failed to remove sender context: %v", err)
+ }
+ c.expectEQ(sender, fromAddr(c.ctx))
+ var to []string
+ to = append(to, "foo@bar.com")
+ to = append(to, config.Namespaces["test2"].Reporting[0].Config.(*EmailConfig).Email)
+ c.expectEQ(msg.To, to)
+ c.expectEQ(msg.Subject, crash.Title)
+ c.expectEQ(len(msg.Attachments), 3)
+ c.expectEQ(msg.Attachments[0].Name, "config.txt")
+ c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+ c.expectEQ(msg.Attachments[1].Name, "raw.log")
+ c.expectEQ(msg.Attachments[1].Data, crash.Log)
+ c.expectEQ(msg.Attachments[2].Name, "repro.txt")
+ c.expectEQ(msg.Attachments[2].Data, syzRepro)
+ c.expectEQ(msg.Headers["In-Reply-To"], []string{"<1234>"})
+ body := `syzkaller has found reproducer for the following crash on kernel_commit1
repo1/branch1
compiler: compiler1
.config is attached
+Raw console output is attached.
+syzkaller reproducer is attached. See https://goo.gl/kgGztJ
+for information about syzkaller reproducers
+CC: [bar@foo.com foo@bar.com]
+
+report1
+`
+ c.expectEQ(msg.Body, body)
+ }
+
+ // Now upstream the bug and check that it reaches the next reporting.
+ incoming2 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <1234>
+Subject: crash1
+From: foo@bar.com
+To: foo@bar.com
+Cc: %v
+Content-Type: text/plain
+
+#syz upstream
+`, sender0)
+
+ c.expectOK(c.POST("/_ah/mail/", incoming2))
+
+ sender1 := ""
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ sender1 = msg.Sender
+ if sender1 == sender0 {
+ t.Fatalf("same ID in different reporting")
+ }
+ sender, _, err := email.RemoveAddrContext(msg.Sender)
+ if err != nil {
+ t.Fatalf("failed to remove sender context: %v", err)
+ }
+ c.expectEQ(sender, fromAddr(c.ctx))
+ c.expectEQ(msg.To, []string{"bar@foo.com", "bugs@syzkaller.com", "foo@bar.com"})
+ c.expectEQ(msg.Subject, crash.Title)
+ c.expectEQ(len(msg.Attachments), 3)
+ c.expectEQ(msg.Attachments[0].Name, "config.txt")
+ c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+ c.expectEQ(msg.Attachments[1].Name, "raw.log")
+ c.expectEQ(msg.Attachments[1].Data, crash.Log)
+ c.expectEQ(msg.Attachments[2].Name, "repro.txt")
+ c.expectEQ(msg.Attachments[2].Data, syzRepro)
+ body := `Hello,
+
+syzkaller hit the following crash on kernel_commit1
+repo1/branch1
+compiler: compiler1
+.config is attached
+Raw console output is attached.
+
+syzkaller reproducer is attached. See https://goo.gl/kgGztJ
+for information about syzkaller reproducers
-CC: ["Foo Bar" <foo@bar.com> bar@foo.com]
report1
---
This bug is generated by a dumb bot. It may contain errors.
+See https://goo.gl/tpsmEJ for details.
Direct all questions to syzkaller@googlegroups.com.
+syzbot will keep track of this bug report.
+Once a fix for this bug is committed, please reply to this email with:
+#syz fix: exact-commit-title
+To mark this as a duplicate of another syzbot report, please reply with:
+#syz dup: exact-subject-of-another-report
+If it's a one-off invalid bug report, please reply with:
+#syz invalid
+Note: if the crash happens again, it will cause creation of a new bug report.
`
- c.expectEQ(msg.Body, body)
+ c.expectEQ(msg.Body, body)
+ }
+
+ // Model that somebody adds more emails to CC list.
+ incoming3 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <1234>
+Subject: crash1
+From: foo@bar.com
+To: %v
+CC: new@new.com, "another" <another@another.com>, bar@foo.com, bugs@syzkaller.com, foo@bar.com
+Content-Type: text/plain
+
++more people
+`, sender1)
+
+ c.expectOK(c.POST("/_ah/mail/", incoming3))
+
+ // Now upload a C reproducer.
+ crash.ReproC = []byte("int main() {}")
+ crash.Maintainers = []string{"\"qux\" <qux@qux.com>"}
+ c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ c.expectEQ(msg.Sender, sender1)
+ sender, _, err := email.RemoveAddrContext(msg.Sender)
+ if err != nil {
+ t.Fatalf("failed to remove sender context: %v", err)
+ }
+ c.expectEQ(sender, fromAddr(c.ctx))
+ c.expectEQ(msg.To, []string{"another@another.com", "bar@foo.com", "bugs@syzkaller.com", "foo@bar.com", "new@new.com", "qux@qux.com"})
+ c.expectEQ(msg.Subject, crash.Title)
+ c.expectEQ(len(msg.Attachments), 4)
+ c.expectEQ(msg.Attachments[0].Name, "config.txt")
+ c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+ c.expectEQ(msg.Attachments[1].Name, "raw.log")
+ c.expectEQ(msg.Attachments[1].Data, crash.Log)
+ c.expectEQ(msg.Attachments[2].Name, "repro.txt")
+ c.expectEQ(msg.Attachments[2].Data, syzRepro)
+ c.expectEQ(msg.Attachments[3].Name, "repro.c")
+ c.expectEQ(msg.Attachments[3].Data, crash.ReproC)
+ body := `syzkaller has found reproducer for the following crash on kernel_commit1
+repo1/branch1
+compiler: compiler1
+.config is attached
+Raw console output is attached.
+C reproducer is attached
+syzkaller reproducer is attached. See https://goo.gl/kgGztJ
+for information about syzkaller reproducers
+
+
+report1
+`
+ c.expectEQ(msg.Body, body)
+ }
+
+ // Send an invalid command.
+ incoming4 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <abcdef>
+Subject: title1
+From: foo@bar.com
+To: %v
+Content-Type: text/plain
+
+#syz bad-command
+`, sender1)
+
+ c.expectOK(c.POST("/_ah/mail/", incoming4))
+
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ c.expectEQ(msg.To, []string{"<foo@bar.com>"})
+ c.expectEQ(msg.Subject, crash.Title)
+ c.expectEQ(msg.Headers["In-Reply-To"], []string{"<abcdef>"})
+ if !strings.Contains(msg.Body, `> #syz bad-command
+
+unknown command "bad-command"
+`) {
+ t.Fatal("no unknown command reply for bad command")
+ }
+ }
+
+ // Now mark the bug as invalid.
+ incoming5 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <abcdef>
+Subject: title1
+From: foo@bar.com
+To: %v
+Content-Type: text/plain
+
+#syz fix: some: commit title
+`, sender1)
+ c.expectOK(c.POST("/_ah/mail/", incoming5))
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 0)
+
+ // Check that the commit is now passed to builders.
+ builderPollReq := &dashapi.BuilderPollReq{build.Manager}
+ builderPollResp := new(dashapi.BuilderPollResp)
+ c.expectOK(c.API(client2, key2, "builder_poll", builderPollReq, builderPollResp))
+ c.expectEQ(len(builderPollResp.PendingCommits), 1)
+ c.expectEQ(builderPollResp.PendingCommits[0], "some: commit title")
+
+ build2 := testBuild(2)
+ build2.Manager = build.Manager
+ build2.Commits = []string{"some: commit title"}
+ c.expectOK(c.API(client2, key2, "upload_build", build2, nil))
+
+ // New crash must produce new bug in the first reporting.
+ c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ c.expectEQ(msg.Subject, crash.Title+" (2)")
+ if msg.Sender == sender0 {
+ t.Fatalf("same reporting ID for new bug")
+ }
+ }
+}
+
+// Bug must not be mailed to maintainers if maintainers list is empty.
+func TestEmailNoMaintainers(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ build := testBuild(1)
+ c.expectOK(c.API(client2, key2, "upload_build", build, nil))
+
+ crash := testCrash(build, 1)
+ c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ sender := (<-c.emailSink).Sender
+
+ incoming1 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <1234>
+Subject: crash1
+From: %v
+To: foo@bar.com
+Content-Type: text/plain
+
+#syz upstream
+`, sender)
+ c.expectOK(c.POST("/_ah/mail/", incoming1))
+
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 0)
+}
+
+// Basic dup scenario: mark one bug as dup of another.
+func TestEmailDup(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ build := testBuild(1)
+ c.expectOK(c.API(client2, key2, "upload_build", build, nil))
+
+ crash1 := testCrash(build, 1)
+ crash1.Title = "BUG: slightly more elaborate title"
+ c.expectOK(c.API(client2, key2, "report_crash", crash1, nil))
+
+ crash2 := testCrash(build, 2)
+ crash1.Title = "KASAN: another title"
+ c.expectOK(c.API(client2, key2, "report_crash", crash2, nil))
+
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 2)
+ msg1 := <-c.emailSink
+ msg2 := <-c.emailSink
+
+ // Dup crash2 to crash1.
+ incoming1 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <12345>
+Subject: title1
+From: foo@bar.com
+To: %v
+Content-Type: text/plain
+
+#syz dup: BUG: slightly more elaborate title
+`, msg2.Sender)
+ c.expectOK(c.POST("/_ah/mail/", incoming1))
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 0)
+
+ // Second crash happens again
+ crash2.ReproC = []byte("int main() {}")
+ c.expectOK(c.API(client2, key2, "report_crash", crash2, nil))
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 0)
+
+ // Now close the original bug, and check that new bugs for dup are now created.
+ incoming2 := fmt.Sprintf(`Sender: syzkaller@googlegroups.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <12345>
+Subject: title1
+From: foo@bar.com
+To: %v
+Content-Type: text/plain
+
+#syz invalid
+`, msg1.Sender)
+ c.expectOK(c.POST("/_ah/mail/", incoming2))
+
+ // New crash must produce new bug in the first reporting.
+ c.expectOK(c.API(client2, key2, "report_crash", crash2, nil))
+ {
+ c.expectOK(c.GET("/email_poll"))
+ c.expectEQ(len(c.emailSink), 1)
+ msg := <-c.emailSink
+ c.expectEQ(msg.Subject, crash2.Title+" (2)")
+ }
}
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index b2a35c16f..948002076 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -5,6 +5,8 @@ package dash
import (
"fmt"
+ "regexp"
+ "strconv"
"time"
"github.com/google/syzkaller/dashboard/dashapi"
@@ -16,9 +18,8 @@ import (
// This file contains definitions of entities stored in datastore.
const (
- maxMaintainers = 50
- maxTextLen = 200
- MaxStringLen = 1024
+ maxTextLen = 200
+ MaxStringLen = 1024
maxCrashes = 20
)
@@ -56,7 +57,9 @@ type Bug struct {
type BugReporting struct {
Name string // refers to Reporting.Name
ID string // unique ID per BUG/BugReporting used in commucation with external systems
+ ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport
Link string
+ CC string // additional emails added to CC list (|-delimited list)
ReproLevel dashapi.ReproLevel
Reported time.Time
Closed time.Time
@@ -136,6 +139,25 @@ func (bug *Bug) displayTitle() string {
return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
}
+var displayTitleRe = regexp.MustCompile("^(.*) \\(([0-9]+)\\)$")
+
+func splitDisplayTitle(display string) (string, int64, error) {
+ match := displayTitleRe.FindStringSubmatchIndex(display)
+ if match == nil {
+ return display, 0, nil
+ }
+ title := display[match[2]:match[3]]
+ seqStr := display[match[4]:match[5]]
+ seq, err := strconv.ParseInt(seqStr, 10, 64)
+ if err != nil {
+ return "", 0, fmt.Errorf("failed to parse bug title: %v", err)
+ }
+ if seq <= 0 || seq > 1e6 {
+ return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
+ }
+ return title, seq - 1, nil
+}
+
func canonicalBug(c context.Context, bug *Bug) (*Bug, error) {
for {
if bug.Status != BugStatusDup {
diff --git a/dashboard/app/mail_bug.txt b/dashboard/app/mail_bug.txt
index 51169837c..2c3eca7f1 100644
--- a/dashboard/app/mail_bug.txt
+++ b/dashboard/app/mail_bug.txt
@@ -1,18 +1,34 @@
-{{if .First}}
+{{if .First -}}
Hello,
-{{- end -}}
+{{end -}}
syzkaller {{if .First}}hit{{else}}has found reproducer for{{end}} the following crash on {{.KernelCommit}}
{{.KernelRepo}}/{{.KernelBranch}}
compiler: {{.CompilerID}}
.config is attached
+{{if .HasLog}}Raw console output is attached.{{end}}
{{if .ReproC}}C reproducer is attached{{end}}
-{{if .ReproSyz}}syzkaller reproducer is attached. See https://github.com/google/syzkaller/blob/master/docs/executing_syzkaller_programs.md for information about syzkaller reproducers{{end}}
+{{if .ReproSyz}}syzkaller reproducer is attached. See https://goo.gl/kgGztJ
+for information about syzkaller reproducers{{end}}
{{if .Moderation}}CC: {{.Maintainers}}{{end}}
{{printf "%s" .Report}}
{{if .First}}
---
This bug is generated by a dumb bot. It may contain errors.
+See https://goo.gl/tpsmEJ for details.
Direct all questions to syzkaller@googlegroups.com.
-{{end}}
+
+syzbot will keep track of this bug report.
+Once a fix for this bug is committed, please reply to this email with:
+#syz fix: exact-commit-title
+To mark this as a duplicate of another syzbot report, please reply with:
+#syz dup: exact-subject-of-another-report
+If it's a one-off invalid bug report, please reply with:
+#syz invalid
+Note: if the crash happens again, it will cause creation of a new bug report.
+{{if .Moderation -}}
+To upstream this report, please reply with:
+#syz upstream
+{{- end -}}
+{{- end -}}
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 766f686c0..2261b3177 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -167,7 +167,7 @@ func fetchBugs(c context.Context) ([]*uiBugGroup, error) {
}
func createUIBug(c context.Context, bug *Bug, state *ReportingState) *uiBug {
- _, _, reportingIdx, status, link, err := needReport(c, "", state, bug)
+ _, _, _, reportingIdx, status, link, err := needReport(c, "", state, bug)
if err != nil {
status = err.Error()
}
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 13957263e..589183f3d 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -8,9 +8,11 @@ import (
"errors"
"fmt"
"sort"
+ "strings"
"time"
"github.com/google/syzkaller/dashboard/dashapi"
+ "github.com/google/syzkaller/pkg/email"
"golang.org/x/net/context"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
@@ -60,11 +62,11 @@ func reportingPoll(c context.Context, typ string) []*dashapi.BugReport {
}
func handleReportBug(c context.Context, typ string, state *ReportingState, bug *Bug) (*dashapi.BugReport, error) {
- reporting, bugReporting, _, _, _, err := needReport(c, typ, state, bug)
+ reporting, bugReporting, crash, _, _, _, err := needReport(c, typ, state, bug)
if err != nil || reporting == nil {
return nil, err
}
- rep, err := createBugReport(c, bug, bugReporting, reporting.Config)
+ rep, err := createBugReport(c, bug, crash, bugReporting, reporting.Config)
if err != nil {
return nil, err
}
@@ -72,15 +74,14 @@ func handleReportBug(c context.Context, typ string, state *ReportingState, bug *
return rep, nil
}
-func needReport(c context.Context, typ string, state *ReportingState, bug *Bug) (reporting *Reporting, bugReporting *BugReporting, reportingIdx int, status, link string, err error) {
+func needReport(c context.Context, typ string, state *ReportingState, bug *Bug) (reporting *Reporting, bugReporting *BugReporting, crash *Crash, reportingIdx int, status, link string, err error) {
reporting, bugReporting, reportingIdx, status, err = currentReporting(c, bug)
if err != nil || reporting == nil {
return
}
if typ != "" && typ != reporting.Config.Type() {
status = "on a different reporting"
- reporting = nil
- bugReporting = nil
+ reporting, bugReporting = nil, nil
return
}
link = bugReporting.Link
@@ -88,40 +89,49 @@ func needReport(c context.Context, typ string, state *ReportingState, bug *Bug)
status = fmt.Sprintf("%v: reported%v on %v",
reporting.Name, reproStr(bugReporting.ReproLevel),
formatTime(bugReporting.Reported))
- reporting = nil
- bugReporting = nil
+ reporting, bugReporting = nil, nil
return
}
ent := state.getEntry(timeNow(c), bug.Namespace, reporting.Name)
+ cfg := config.Namespaces[bug.Namespace]
+ if bug.ReproLevel < ReproLevelC && timeSince(c, bug.FirstTime) < cfg.WaitForRepro {
+ status = fmt.Sprintf("%v: waiting for C repro", reporting.Name)
+ reporting, bugReporting = nil, nil
+ return
+ }
+ if !cfg.MailWithoutReport && !bug.HasReport {
+ status = fmt.Sprintf("%v: no report", reporting.Name)
+ reporting, bugReporting = nil, nil
+ return
+ }
+
+ crash, err = findCrashForBug(c, bug)
+ if err != nil {
+ status = fmt.Sprintf("%v: no crashes!", reporting.Name)
+ reporting, bugReporting = nil, nil
+ return
+ }
+ if reporting.Config.NeedMaintainers() && len(crash.Maintainers) == 0 {
+ status = fmt.Sprintf("%v: no maintainers", reporting.Name)
+ reporting, bugReporting = nil, nil
+ return
+ }
+
// Limit number of reports sent per day,
// but don't limit sending repros to already reported bugs.
if bugReporting.Reported.IsZero() && reporting.DailyLimit != 0 &&
ent.Sent >= reporting.DailyLimit {
status = fmt.Sprintf("%v: out of quota for today", reporting.Name)
- reporting = nil
- bugReporting = nil
+ reporting, bugReporting = nil, nil
return
}
+
+ // Ready to be reported.
if bugReporting.Reported.IsZero() {
// This update won't be committed, but it will prevent us from
// reporting too many bugs in a single poll.
ent.Sent++
}
- cfg := config.Namespaces[bug.Namespace]
- if bug.ReproLevel < ReproLevelC && timeSince(c, bug.FirstTime) < cfg.WaitForRepro {
- status = fmt.Sprintf("%v: waiting for C repro", reporting.Name)
- reporting = nil
- bugReporting = nil
- return
- }
- if !cfg.MailWithoutReport && !bug.HasReport {
- status = fmt.Sprintf("%v: no report", reporting.Name)
- reporting = nil
- bugReporting = nil
- return
- }
-
- // Ready to be reported.
status = fmt.Sprintf("%v: ready to report", reporting.Name)
if !bugReporting.Reported.IsZero() {
status += fmt.Sprintf(" (reported%v on %v)",
@@ -162,16 +172,11 @@ func reproStr(level dashapi.ReproLevel) string {
}
}
-func createBugReport(c context.Context, bug *Bug, bugReporting *BugReporting, config interface{}) (*dashapi.BugReport, error) {
+func createBugReport(c context.Context, bug *Bug, crash *Crash, bugReporting *BugReporting, config interface{}) (*dashapi.BugReport, error) {
reportingConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
- bugKey := datastore.NewKey(c, "Bug", bugKeyHash(bug.Namespace, bug.Title, bug.Seq), 0, nil)
- crash, err := findCrashForBug(c, bug, bugKey)
- if err != nil {
- return nil, err
- }
crashLog, err := getText(c, "CrashLog", crash.Log)
if err != nil {
return nil, err
@@ -212,6 +217,7 @@ func createBugReport(c context.Context, bug *Bug, bugReporting *BugReporting, co
rep := &dashapi.BugReport{
Config: reportingConfig,
ID: bugReporting.ID,
+ ExtID: bugReporting.ExtID,
First: bugReporting.Reported.IsZero(),
Title: bug.displayTitle(),
Log: crashLog,
@@ -225,12 +231,15 @@ func createBugReport(c context.Context, bug *Bug, bugReporting *BugReporting, co
ReproC: reproC,
ReproSyz: reproSyz,
}
+ if bugReporting.CC != "" {
+ rep.CC = strings.Split(bugReporting.CC, "|")
+ }
return rep, nil
}
// incomingCommand is entry point to bug status updates.
func incomingCommand(c context.Context, cmd *dashapi.BugUpdate) (string, bool) {
- log.Infof(c, "got command: %+v", cmd)
+ log.Infof(c, "got command: %+q", cmd)
reply, err := incomingCommandImpl(c, cmd)
if err != nil {
log.Errorf(c, "%v", err)
@@ -247,9 +256,25 @@ func incomingCommandImpl(c context.Context, cmd *dashapi.BugUpdate) (string, err
now := timeNow(c)
dupHash := ""
if cmd.Status == dashapi.BugStatusDup {
+ bugReporting, _ := bugReportingByID(bug, cmd.ID, now)
dup, dupKey, err := findBugByReportingID(c, cmd.DupOf)
if err != nil {
- return "can't find the dup bug", err
+ // Email reporting passes bug title in cmd.DupOf, try to find bug by title.
+ dup, dupKey, err = findDupByTitle(c, bug.Namespace, cmd.DupOf)
+ if err != nil {
+ return "can't find the dup bug", err
+ }
+ cmd.DupOf = ""
+ for i := range dup.Reporting {
+ if dup.Reporting[i].Name == bugReporting.Name {
+ cmd.DupOf = dup.Reporting[i].ID
+ break
+ }
+ }
+ if cmd.DupOf == "" {
+ return "can't find the dup bug",
+ fmt.Errorf("dup does not have reporting %q", bugReporting.Name)
+ }
}
if bugKey.StringID() == dupKey.StringID() {
return "can't dup bug to itself", fmt.Errorf("can't dup bug to itself")
@@ -258,7 +283,6 @@ func incomingCommandImpl(c context.Context, cmd *dashapi.BugUpdate) (string, err
return "can't find the dup bug",
fmt.Errorf("inter-namespace dup: %v->%v", bug.Namespace, dup.Namespace)
}
- bugReporting, _ := bugReportingByID(bug, cmd.ID, now)
dupReporting, _ := bugReportingByID(dup, cmd.DupOf, now)
if bugReporting == nil || dupReporting == nil {
return internalError, fmt.Errorf("can't find bug reporting")
@@ -348,6 +372,8 @@ func incomingCommandTx(c context.Context, now time.Time, cmd *dashapi.BugUpdate,
bug.Status = BugStatusDup
bug.Closed = now
bug.DupOf = dupHash
+ case dashapi.BugStatusUpdate:
+ // Just update Link, Commits, etc below.
default:
return "unknown bug status", fmt.Errorf("unknown bug status %v", cmd.Status)
}
@@ -380,9 +406,16 @@ func incomingCommandTx(c context.Context, now time.Time, cmd *dashapi.BugUpdate,
bug.PatchedOn = nil
}
}
- if cmd.Link != "" {
+ if bugReporting.ExtID == "" {
+ bugReporting.ExtID = cmd.ExtID
+ }
+ if bugReporting.Link == "" {
bugReporting.Link = cmd.Link
}
+ if len(cmd.CC) != 0 {
+ merged := email.MergeEmailLists(strings.Split(bugReporting.CC, "|"), cmd.CC)
+ bugReporting.CC = strings.Join(merged, "|")
+ }
if bugReporting.ReproLevel < cmd.ReproLevel {
bugReporting.ReproLevel = cmd.ReproLevel
}
@@ -416,6 +449,20 @@ func findBugByReportingID(c context.Context, id string) (*Bug, *datastore.Key, e
return bugs[0], keys[0], nil
}
+func findDupByTitle(c context.Context, ns, title string) (*Bug, *datastore.Key, error) {
+ title, seq, err := splitDisplayTitle(title)
+ if err != nil {
+ return nil, nil, err
+ }
+ bugHash := bugKeyHash(ns, title, seq)
+ bugKey := datastore.NewKey(c, "Bug", bugHash, 0, nil)
+ bug := new(Bug)
+ if err := datastore.Get(c, bugKey, bug); err != nil {
+ return nil, nil, fmt.Errorf("failed to get dup: %v", err)
+ }
+ return bug, bugKey, nil
+}
+
func bugReportingByID(bug *Bug, id string, now time.Time) (*BugReporting, bool) {
for i := range bug.Reporting {
if bug.Reporting[i].ID == id {
@@ -442,7 +489,8 @@ func queryCrashesForBug(c context.Context, bugKey *datastore.Key, limit int) ([]
return crashes, nil
}
-func findCrashForBug(c context.Context, bug *Bug, bugKey *datastore.Key) (*Crash, error) {
+func findCrashForBug(c context.Context, bug *Bug) (*Crash, error) {
+ bugKey := datastore.NewKey(c, "Bug", bugKeyHash(bug.Namespace, bug.Title, bug.Seq), 0, nil)
crashes, err := queryCrashesForBug(c, bugKey, 1)
if err != nil {
return nil, fmt.Errorf("failed to fetch crashes: %v", err)
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 0407f2766..c9caa7123 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -38,6 +38,10 @@ func (cfg *EmailConfig) Type() string {
return emailType
}
+func (cfg *EmailConfig) NeedMaintainers() bool {
+ return cfg.MailMaintainers
+}
+
func (cfg *EmailConfig) Validate() error {
if _, err := mail.ParseAddress(cfg.Email); err != nil {
return fmt.Errorf("bad email address %q: %v", cfg.Email, err)
@@ -81,12 +85,19 @@ func emailReport(c context.Context, rep *dashapi.BugReport) error {
}
to = append(to, rep.Maintainers...)
}
+ to = email.MergeEmailLists(to, rep.CC)
attachments := []aemail.Attachment{
{
Name: "config.txt",
Data: rep.KernelConfig,
},
}
+ if len(rep.Log) != 0 {
+ attachments = append(attachments, aemail.Attachment{
+ Name: "raw.log",
+ Data: rep.Log,
+ })
+ }
repro := dashapi.ReproLevelNone
if len(rep.ReproSyz) != 0 {
repro = dashapi.ReproLevelSyz
@@ -116,6 +127,7 @@ func emailReport(c context.Context, rep *dashapi.BugReport) error {
KernelBranch string
KernelCommit string
Report []byte
+ HasLog bool
ReproSyz bool
ReproC bool
}
@@ -128,10 +140,12 @@ func emailReport(c context.Context, rep *dashapi.BugReport) error {
KernelBranch: rep.KernelBranch,
KernelCommit: rep.KernelCommit,
Report: rep.Report,
+ HasLog: len(rep.Log) != 0,
ReproSyz: len(rep.ReproSyz) != 0,
ReproC: len(rep.ReproC) != 0,
}
- if err := sendMailTemplate(c, rep.Title, from, to, attachments, "mail_bug.txt", data); err != nil {
+ err = sendMailTemplate(c, rep.Title, from, to, rep.ExtID, attachments, "mail_bug.txt", data)
+ if err != nil {
return err
}
cmd := &dashapi.BugUpdate{
@@ -157,30 +171,49 @@ func incomingMail(c context.Context, r *http.Request) error {
if err != nil {
return err
}
- log.Infof(c, "received email: subject '%v', from '%v', cc '%v', msg '%v', bug '%v', cmd '%v'",
- msg.Subject, msg.From, msg.Cc, msg.MessageID, msg.BugID, msg.Command)
- var status dashapi.BugStatus
+ log.Infof(c, "received email: subject %q, from %q, cc %q, msg %q, bug %q, cmd %q, link %q",
+ msg.Subject, msg.From, msg.Cc, msg.MessageID, msg.BugID, msg.Command, msg.Link)
+ // Don't send replies yet.
+ // It is not tested and it is unclear how verbose we want to be.
+ sendReply := false
+ cmd := &dashapi.BugUpdate{
+ ID: msg.BugID,
+ ExtID: msg.MessageID,
+ Link: msg.Link,
+ CC: msg.Cc,
+ }
switch msg.Command {
case "":
- return nil
+ cmd.Status = dashapi.BugStatusUpdate
case "upstream":
- status = dashapi.BugStatusUpstream
+ cmd.Status = dashapi.BugStatusUpstream
case "invalid":
- status = dashapi.BugStatusInvalid
+ cmd.Status = dashapi.BugStatusInvalid
+ case "fix:":
+ if msg.CommandArgs == "" {
+ return replyTo(c, msg, fmt.Sprintf("no commit title"), nil)
+ }
+ cmd.Status = dashapi.BugStatusOpen
+ cmd.FixCommits = []string{msg.CommandArgs}
+ case "dup:":
+ if msg.CommandArgs == "" {
+ return replyTo(c, msg, fmt.Sprintf("no dup title"), nil)
+ }
+ cmd.Status = dashapi.BugStatusDup
+ cmd.DupOf = msg.CommandArgs
default:
return replyTo(c, msg, fmt.Sprintf("unknown command %q", msg.Command), nil)
}
- cmd := &dashapi.BugUpdate{
- ID: msg.BugID,
- Status: status,
- }
reply, _ := incomingCommand(c, cmd)
+ if !sendReply {
+ return nil
+ }
return replyTo(c, msg, reply, nil)
}
var mailTemplates = template.Must(template.New("").ParseGlob("mail_*.txt"))
-func sendMailTemplate(c context.Context, subject, from string, to []string,
+func sendMailTemplate(c context.Context, subject, from string, to []string, replyTo string,
attachments []aemail.Attachment, template string, data interface{}) error {
body := new(bytes.Buffer)
if err := mailTemplates.ExecuteTemplate(body, template, data); err != nil {
@@ -193,6 +226,9 @@ func sendMailTemplate(c context.Context, subject, from string, to []string,
Body: body.String(),
Attachments: attachments,
}
+ if replyTo != "" {
+ msg.Headers = mail.Header{"In-Reply-To": []string{replyTo}}
+ }
return sendEmail(c, msg)
}
diff --git a/dashboard/app/reporting_test.go b/dashboard/app/reporting_test.go
index 9208ed576..9d6368b97 100644
--- a/dashboard/app/reporting_test.go
+++ b/dashboard/app/reporting_test.go
@@ -50,7 +50,7 @@ func TestReportBug(t *testing.T) {
ID: rep.ID,
First: true,
Title: "title1",
- Maintainers: []string{`"Foo Bar" <foo@bar.com>`, `bar@foo.com`},
+ Maintainers: []string{"bar@foo.com", "foo@bar.com"},
CompilerID: "compiler1",
KernelRepo: "repo1",
KernelBranch: "branch1",
diff --git a/dashboard/app/util_test.go b/dashboard/app/util_test.go
index 1b12c857d..068b05cbb 100644
--- a/dashboard/app/util_test.go
+++ b/dashboard/app/util_test.go
@@ -136,8 +136,17 @@ func (c *Ctx) API(client, key, method string, req, reply interface{}) error {
// GET sends authorized HTTP GET request to the app.
func (c *Ctx) GET(url string) error {
- c.t.Logf("GET: %v", url)
- r, err := c.inst.NewRequest("GET", url, nil)
+ return c.httpRequest("GET", url, "")
+}
+
+// POST sends authorized HTTP POST request to the app.
+func (c *Ctx) POST(url, body string) error {
+ return c.httpRequest("POST", url, body)
+}
+
+func (c *Ctx) httpRequest(method, url, body string) error {
+ c.t.Logf("%v: %v", method, url)
+ r, err := c.inst.NewRequest(method, url, strings.NewReader(body))
if err != nil {
c.t.Fatal(err)
}
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index 4871f97ad..ddc20ebe4 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -136,9 +136,11 @@ func (dash *Dashboard) LogError(name, msg string, args ...interface{}) {
type BugReport struct {
Config []byte
ID string
- First bool // Set for first report for this bug.
+ ExtID string // arbitrary reporting ID forwarded from BugUpdate.ExtID
+ First bool // Set for first report for this bug.
Title string
Maintainers []string
+ CC []string // additional CC emails
CompilerID string
KernelRepo string
KernelBranch string
@@ -152,11 +154,13 @@ type BugReport struct {
type BugUpdate struct {
ID string
+ ExtID string
Link string
Status BugStatus
ReproLevel ReproLevel
DupOf string
FixCommits []string // Titles of commits that fix this bug.
+ CC []string // Additional emails to add to CC list in future emails.
}
type BugUpdateReply struct {
@@ -182,6 +186,7 @@ const (
BugStatusUpstream
BugStatusInvalid
BugStatusDup
+ BugStatusUpdate // aux info update (i.e. ExtID/Link/CC)
)
const (
diff --git a/pkg/email/parser.go b/pkg/email/parser.go
index ed90338a0..56579157f 100644
--- a/pkg/email/parser.go
+++ b/pkg/email/parser.go
@@ -12,22 +12,27 @@ import (
"mime"
"mime/multipart"
"net/mail"
+ "regexp"
+ "sort"
"strings"
)
type Email struct {
BugID string
MessageID string
+ Link string
Subject string
From string
Cc []string
- Body string // text/plain part
- Patch string // attached patch, if any
- Command string // command to bot (#syzbot is stripped)
- CommandArgs []string // arguments for the command
+ Body string // text/plain part
+ Patch string // attached patch, if any
+ Command string // command to bot (#syz is stripped)
+ CommandArgs string // arguments for the command
}
-const commandPrefix = "#syzbot "
+const commandPrefix = "#syz "
+
+var groupsLinkRe = regexp.MustCompile("\nTo view this discussion on the web visit (https://groups\\.google\\.com/.*?)\\.(?:\r)?\n")
func Parse(r io.Reader, ownEmail string) (*Email, error) {
msg, err := mail.ReadMessage(r)
@@ -52,7 +57,14 @@ func Parse(r io.Reader, ownEmail string) (*Email, error) {
if addr, err := mail.ParseAddress(ownEmail); err == nil {
ownEmail = addr.Address
}
- for _, addr := range append(cc, to...) {
+ fromMe := false
+ for _, addr := range from {
+ cleaned, _, _ := RemoveAddrContext(addr.Address)
+ if addr, err := mail.ParseAddress(cleaned); err == nil && addr.Address == ownEmail {
+ fromMe = true
+ }
+ }
+ for _, addr := range append(append(cc, to...), from...) {
cleaned, context, _ := RemoveAddrContext(addr.Address)
if addr, err := mail.ParseAddress(cleaned); err == nil {
cleaned = addr.Address
@@ -62,27 +74,36 @@ func Parse(r io.Reader, ownEmail string) (*Email, error) {
bugID = context
}
} else {
- ccList = append(ccList, addr.String())
+ ccList = append(ccList, cleaned)
}
}
+ ccList = MergeEmailLists(ccList)
body, attachments, err := parseBody(msg.Body, msg.Header)
if err != nil {
return nil, err
}
- patch := ""
- for _, a := range attachments {
- _, patch, _ = ParsePatch(string(a))
- if patch != "" {
- break
+ bodyStr := string(body)
+ patch, cmd, cmdArgs := "", "", ""
+ if !fromMe {
+ for _, a := range attachments {
+ _, patch, _ = ParsePatch(string(a))
+ if patch != "" {
+ break
+ }
}
+ if patch == "" {
+ _, patch, _ = ParsePatch(bodyStr)
+ }
+ cmd, cmdArgs = extractCommand(body)
}
- if patch == "" {
- _, patch, _ = ParsePatch(string(body))
+ link := ""
+ if match := groupsLinkRe.FindStringSubmatchIndex(bodyStr); match != nil {
+ link = bodyStr[match[2]:match[3]]
}
- cmd, cmdArgs := extractCommand(body)
email := &Email{
BugID: bugID,
MessageID: msg.Header.Get("Message-ID"),
+ Link: link,
Subject: msg.Header.Get("Subject"),
From: from[0].String(),
Cc: ccList,
@@ -131,13 +152,13 @@ func RemoveAddrContext(email string) (string, string, error) {
// extractCommand extracts command to syzbot from email body.
// Commands are of the following form:
-// ^#syzbot cmd args...
-func extractCommand(body []byte) (cmd string, args []string) {
+// ^#syz cmd args...
+func extractCommand(body []byte) (cmd, args string) {
cmdPos := bytes.Index(append([]byte{'\n'}, body...), []byte("\n"+commandPrefix))
if cmdPos == -1 {
return
}
- cmdPos += 8
+ cmdPos += len(commandPrefix)
cmdEnd := bytes.IndexByte(body[cmdPos:], '\n')
if cmdEnd == -1 {
cmdEnd = len(body) - cmdPos
@@ -148,10 +169,8 @@ func extractCommand(body []byte) (cmd string, args []string) {
}
split := strings.Split(cmdLine, " ")
cmd = split[0]
- for _, arg := range split[1:] {
- if trimmed := strings.TrimSpace(arg); trimmed != "" {
- args = append(args, trimmed)
- }
+ if len(split) > 1 {
+ args = strings.TrimSpace(strings.Join(split[1:], " "))
}
return
}
@@ -202,3 +221,30 @@ func parseBody(r io.Reader, headers mail.Header) (body []byte, attachments [][]b
attachments = append(attachments, attachments1...)
}
}
+
+// MergeEmailLists merges several email lists removing duplicates and invalid entries.
+func MergeEmailLists(lists ...[]string) []string {
+ const (
+ maxEmailLen = 1000
+ maxEmails = 50
+ )
+ merged := make(map[string]bool)
+ for _, list := range lists {
+ for _, email := range list {
+ addr, err := mail.ParseAddress(email)
+ if err != nil || len(addr.Address) > maxEmailLen {
+ continue
+ }
+ merged[addr.Address] = true
+ }
+ }
+ var result []string
+ for e := range merged {
+ result = append(result, e)
+ }
+ sort.Strings(result)
+ if len(result) > maxEmails {
+ result = result[:maxEmails]
+ }
+ return result
+}
diff --git a/pkg/email/parser_test.go b/pkg/email/parser_test.go
index dd758462d..5156690d0 100644
--- a/pkg/email/parser_test.go
+++ b/pkg/email/parser_test.go
@@ -10,6 +10,7 @@ import (
"testing"
)
+//!!! add tests with \r\n
func TestExtractCommand(t *testing.T) {
for i, test := range extractCommandTests {
t.Run(fmt.Sprint(i), func(t *testing.T) {
@@ -77,58 +78,65 @@ func TestAddRemoveAddrContext(t *testing.T) {
func TestParse(t *testing.T) {
for i, test := range parseTests {
- t.Run(fmt.Sprint(i), func(t *testing.T) {
+ body := func(t *testing.T, test ParseTest) {
email, err := Parse(strings.NewReader(test.email), "bot <foo@bar.com>")
if err != nil {
t.Fatal(err)
}
- if !reflect.DeepEqual(email, test.res) {
- t.Logf("expect:\n%#v", test.res)
+ if !reflect.DeepEqual(email, &test.res) {
+ t.Logf("expect:\n%#v", &test.res)
t.Logf("got:\n%#v", email)
t.Fail()
}
- })
+ }
+ t.Run(fmt.Sprint(i), func(t *testing.T) { body(t, test) })
+
+ test.email = strings.Replace(test.email, "\n", "\r\n", -1)
+ test.res.Body = strings.Replace(test.res.Body, "\n", "\r\n", -1)
+ t.Run(fmt.Sprint(i)+"rn", func(t *testing.T) { body(t, test) })
}
}
var extractCommandTests = []struct {
body string
cmd string
- args []string
+ args string
}{
{
body: `Hello,
line1
-#syzbot foo bar baz`,
+#syz foo bar baz `,
cmd: "foo",
- args: []string{"bar", "baz"},
+ args: "bar baz",
},
{
body: `Hello,
line1
-#syzbot foo bar baz
+#syz foo bar baz
line 2
`,
- cmd: "foo",
- args: []string{"bar", "baz"},
+ cmd: "foo",
+ args: "bar baz",
},
{
body: `
line1
-> #syzbot foo bar baz
+> #syz foo bar baz
line 2
`,
cmd: "",
- args: nil,
+ args: "",
},
}
-var parseTests = []struct {
+type ParseTest struct {
email string
- res *Email
-}{
+ res Email
+}
+
+var parseTests = []ParseTest{
{`Date: Sun, 7 May 2017 19:54:00 -0700
Message-ID: <123>
Subject: test subject
@@ -138,20 +146,54 @@ Content-Type: text/plain; charset="UTF-8"
text body
second line
-#syzbot command arg1 arg2 arg3
-last line`,
- &Email{
+#syz command arg1 arg2 arg3
+last line
+--
+You received this message because you are subscribed to the Google Groups "syzkaller" group.
+To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller+unsubscribe@googlegroups.com.
+To post to this group, send email to syzkaller@googlegroups.com.
+To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller/abcdef@google.com.
+For more options, visit https://groups.google.com/d/optout.`,
+ Email{
BugID: "4564456",
MessageID: "<123>",
+ Link: "https://groups.google.com/d/msgid/syzkaller/abcdef@google.com",
Subject: "test subject",
From: "\"Bob\" <bob@example.com>",
+ Cc: []string{"bob@example.com"},
Body: `text body
second line
-#syzbot command arg1 arg2 arg3
-last line`,
+#syz command arg1 arg2 arg3
+last line
+--
+You received this message because you are subscribed to the Google Groups "syzkaller" group.
+To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller+unsubscribe@googlegroups.com.
+To post to this group, send email to syzkaller@googlegroups.com.
+To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller/abcdef@google.com.
+For more options, visit https://groups.google.com/d/optout.`,
Patch: "",
Command: "command",
- CommandArgs: []string{"arg1", "arg2", "arg3"},
+ CommandArgs: "arg1 arg2 arg3",
+ }},
+
+ {`Date: Sun, 7 May 2017 19:54:00 -0700
+Message-ID: <123>
+Subject: test subject
+From: syzbot <foo+4564456@bar.com>
+To: Bob <bob@example.com>
+Content-Type: text/plain; charset="UTF-8"
+
+text body
+last line`,
+ Email{
+ BugID: "4564456",
+ MessageID: "<123>",
+ Subject: "test subject",
+ From: "\"syzbot\" <foo+4564456@bar.com>",
+ Cc: []string{"bob@example.com"},
+ Body: `text body
+last line`,
+ Patch: "",
}},
{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -161,22 +203,22 @@ From: Bob <bob@example.com>
To: syzbot <bot@example.com>, Alice <alice@example.com>
Content-Type: text/plain
-#syzbot command
+#syz command
text body
second line
last line`,
- &Email{
+ Email{
MessageID: "<123>",
Subject: "test subject",
From: "\"Bob\" <bob@example.com>",
- Cc: []string{"\"syzbot\" <bot@example.com>", "\"Alice\" <alice@example.com>"},
- Body: `#syzbot command
+ Cc: []string{"alice@example.com", "bob@example.com", "bot@example.com"},
+ Body: `#syz command
text body
second line
last line`,
Patch: "",
Command: "command",
- CommandArgs: nil,
+ CommandArgs: "",
}},
{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -189,19 +231,19 @@ Content-Type: text/plain
text body
second line
last line
-#syzbot command`,
- &Email{
+#syz command`,
+ Email{
MessageID: "<123>",
Subject: "test subject",
From: "\"Bob\" <bob@example.com>",
- Cc: []string{"\"syzbot\" <bot@example.com>", "\"Alice\" <alice@example.com>"},
+ Cc: []string{"alice@example.com", "bob@example.com", "bot@example.com"},
Body: `text body
second line
last line
-#syzbot command`,
+#syz command`,
Patch: "",
Command: "command",
- CommandArgs: nil,
+ CommandArgs: "",
}},
{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -215,7 +257,7 @@ Content-Type: multipart/mixed; boundary="001a114ce0b01684a6054f0d8b81"
Content-Type: text/plain; charset="UTF-8"
body text
->#syzbot test
+>#syz test
--001a114ce0b01684a6054f0d8b81
Content-Type: text/x-patch; charset="US-ASCII"; name="patch.patch"
@@ -230,13 +272,13 @@ YXNrX3N0cnVjdCAqdCkKIAlrY292ID0gdC0+a2NvdjsKIAlpZiAoa2NvdiA9PSBOVUxMKQogCQly
ZXR1cm47Ci0Jc3Bpbl9sb2NrKCZrY292LT5sb2NrKTsKIAlpZiAoV0FSTl9PTihrY292LT50ICE9
IHQpKSB7CiAJCXNwaW5fdW5sb2NrKCZrY292LT5sb2NrKTsKIAkJcmV0dXJuOwo=
--001a114ce0b01684a6054f0d8b81--`,
- &Email{
+ Email{
MessageID: "<123>",
Subject: "test subject",
From: "\"Bob\" <bob@example.com>",
- Cc: []string{"\"syzbot\" <bot@example.com>"},
+ Cc: []string{"bob@example.com", "bot@example.com"},
Body: `body text
->#syzbot test
+>#syz test
`,
Patch: `--- a/kernel/kcov.c
+++ b/kernel/kcov.c
@@ -250,7 +292,7 @@ IHQpKSB7CiAJCXNwaW5fdW5sb2NrKCZrY292LT5sb2NrKTsKIAkJcmV0dXJuOwo=
return;
`,
Command: "",
- CommandArgs: nil,
+ CommandArgs: "",
}},
{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -266,7 +308,7 @@ Content-Type: text/plain; charset="UTF-8"
On Mon, May 8, 2017 at 6:47 PM, Bob wrote:
> body text
-#syzbot test
+#syz test
commit 59372bbf3abd5b24a7f6f676a3968685c280f955
Date: Thu Apr 27 13:54:11 2017 +0200
@@ -295,7 +337,7 @@ Content-Transfer-Encoding: quoted-printable
<div dir=3D"ltr">On Mon, May 8, 2017 at 6:47 PM, Dmitry Vyukov &lt;<a href=
=3D"mailto:bob@example.com">bob@example.com</a>&gt; wrote:<br>&gt; bo=
-dy text<br><br>#syzbot test<br><br><div><div>commit 59372bbf3abd5b24a7f6f67=
+dy text<br><br>#syz test<br><br><div><div>commit 59372bbf3abd5b24a7f6f67=
6a3968685c280f955</div><div>Date: =C2=A0 Thu Apr 27 13:54:11 2017 +0200</di=
v><div><br></div><div>=C2=A0 =C2=A0 statx: correct error handling of NULL p=
athname</div><div>=C2=A0 =C2=A0=C2=A0</div><div>=C2=A0 =C2=A0 test patch.</=
@@ -316,15 +358,15 @@ ce:pre">=09=09</span>return -EINVAL;</div><div>=C2=A0</div><div>=C2=A0<span=
or)</div></div></div>
--f403043eee70018593054f0d9f1f--`,
- &Email{
+ Email{
MessageID: "<123>",
Subject: "test subject",
From: "\"Bob\" <bob@example.com>",
- Cc: []string{"\"syzbot\" <bot@example.com>"},
+ Cc: []string{"bob@example.com", "bot@example.com"},
Body: `On Mon, May 8, 2017 at 6:47 PM, Bob wrote:
> body text
-#syzbot test
+#syz test
commit 59372bbf3abd5b24a7f6f676a3968685c280f955
Date: Thu Apr 27 13:54:11 2017 +0200
@@ -360,6 +402,6 @@ index 3d85747bd86e..a257b872a53d 100644
if (error)
`,
Command: "test",
- CommandArgs: nil,
+ CommandArgs: "",
}},
}
diff --git a/pkg/email/reply_test.go b/pkg/email/reply_test.go
index 2dd9d894d..75d2eca09 100644
--- a/pkg/email/reply_test.go
+++ b/pkg/email/reply_test.go
@@ -29,13 +29,13 @@ var formReplyTests = []struct {
{
email: `line1
line2
-#syzbot foo
+#syz foo
line3
`,
reply: "this is reply",
result: `> line1
> line2
-> #syzbot foo
+> #syz foo
this is reply
@@ -45,13 +45,13 @@ this is reply
{
email: `> line1
> line2
-#syzbot foo
+#syz foo
line3
`,
reply: "this is reply\n",
result: `>> line1
>> line2
-> #syzbot foo
+> #syz foo
this is reply
@@ -61,11 +61,11 @@ this is reply
{
email: `line1
line2
-#syzbot foo`,
+#syz foo`,
reply: "this is reply 1\nthis is reply 2",
result: `> line1
> line2
-> #syzbot foo
+> #syz foo
this is reply 1
this is reply 2