dashboard/app: heavylifting of email reporting

- save Message-ID and use In-Reply-To in subsequent messages
- remember additional CC entries added manually
- don't mail to maintainers if maintainers list is empty
- improve mail formatting and add a footer
- implement upstream/fix/dup/invalid commands over email
- add tests

3 files changed, 158 insertions, 70 deletions

diff --git a/pkg/email/parser.go b/pkg/email/parser.go
index ed90338a0..56579157f 100644
--- a/pkg/email/parser.go
+++ b/pkg/email/parser.go
@@ -12,22 +12,27 @@ import (
 	"mime"
 	"mime/multipart"
 	"net/mail"
+	"regexp"
+	"sort"
 	"strings"
 )
 
 type Email struct {
 	BugID       string
 	MessageID   string
+	Link        string
 	Subject     string
 	From        string
 	Cc          []string
-	Body        string   // text/plain part
-	Patch       string   // attached patch, if any
-	Command     string   // command to bot (#syzbot is stripped)
-	CommandArgs []string // arguments for the command
+	Body        string // text/plain part
+	Patch       string // attached patch, if any
+	Command     string // command to bot (#syz is stripped)
+	CommandArgs string // arguments for the command
 }
 
-const commandPrefix = "#syzbot "
+const commandPrefix = "#syz "
+
+var groupsLinkRe = regexp.MustCompile("\nTo view this discussion on the web visit (https://groups\\.google\\.com/.*?)\\.(?:\r)?\n")
 
 func Parse(r io.Reader, ownEmail string) (*Email, error) {
 	msg, err := mail.ReadMessage(r)
@@ -52,7 +57,14 @@ func Parse(r io.Reader, ownEmail string) (*Email, error) {
 	if addr, err := mail.ParseAddress(ownEmail); err == nil {
 		ownEmail = addr.Address
 	}
-	for _, addr := range append(cc, to...) {
+	fromMe := false
+	for _, addr := range from {
+		cleaned, _, _ := RemoveAddrContext(addr.Address)
+		if addr, err := mail.ParseAddress(cleaned); err == nil && addr.Address == ownEmail {
+			fromMe = true
+		}
+	}
+	for _, addr := range append(append(cc, to...), from...) {
 		cleaned, context, _ := RemoveAddrContext(addr.Address)
 		if addr, err := mail.ParseAddress(cleaned); err == nil {
 			cleaned = addr.Address
@@ -62,27 +74,36 @@ func Parse(r io.Reader, ownEmail string) (*Email, error) {
 				bugID = context
 			}
 		} else {
-			ccList = append(ccList, addr.String())
+			ccList = append(ccList, cleaned)
 		}
 	}
+	ccList = MergeEmailLists(ccList)
 	body, attachments, err := parseBody(msg.Body, msg.Header)
 	if err != nil {
 		return nil, err
 	}
-	patch := ""
-	for _, a := range attachments {
-		_, patch, _ = ParsePatch(string(a))
-		if patch != "" {
-			break
+	bodyStr := string(body)
+	patch, cmd, cmdArgs := "", "", ""
+	if !fromMe {
+		for _, a := range attachments {
+			_, patch, _ = ParsePatch(string(a))
+			if patch != "" {
+				break
+			}
 		}
+		if patch == "" {
+			_, patch, _ = ParsePatch(bodyStr)
+		}
+		cmd, cmdArgs = extractCommand(body)
 	}
-	if patch == "" {
-		_, patch, _ = ParsePatch(string(body))
+	link := ""
+	if match := groupsLinkRe.FindStringSubmatchIndex(bodyStr); match != nil {
+		link = bodyStr[match[2]:match[3]]
 	}
-	cmd, cmdArgs := extractCommand(body)
 	email := &Email{
 		BugID:       bugID,
 		MessageID:   msg.Header.Get("Message-ID"),
+		Link:        link,
 		Subject:     msg.Header.Get("Subject"),
 		From:        from[0].String(),
 		Cc:          ccList,
@@ -131,13 +152,13 @@ func RemoveAddrContext(email string) (string, string, error) {
 
 // extractCommand extracts command to syzbot from email body.
 // Commands are of the following form:
-// ^#syzbot cmd args...
-func extractCommand(body []byte) (cmd string, args []string) {
+// ^#syz cmd args...
+func extractCommand(body []byte) (cmd, args string) {
 	cmdPos := bytes.Index(append([]byte{'\n'}, body...), []byte("\n"+commandPrefix))
 	if cmdPos == -1 {
 		return
 	}
-	cmdPos += 8
+	cmdPos += len(commandPrefix)
 	cmdEnd := bytes.IndexByte(body[cmdPos:], '\n')
 	if cmdEnd == -1 {
 		cmdEnd = len(body) - cmdPos
@@ -148,10 +169,8 @@ func extractCommand(body []byte) (cmd string, args []string) {
 	}
 	split := strings.Split(cmdLine, " ")
 	cmd = split[0]
-	for _, arg := range split[1:] {
-		if trimmed := strings.TrimSpace(arg); trimmed != "" {
-			args = append(args, trimmed)
-		}
+	if len(split) > 1 {
+		args = strings.TrimSpace(strings.Join(split[1:], " "))
 	}
 	return
 }
@@ -202,3 +221,30 @@ func parseBody(r io.Reader, headers mail.Header) (body []byte, attachments [][]b
 		attachments = append(attachments, attachments1...)
 	}
 }
+
+// MergeEmailLists merges several email lists removing duplicates and invalid entries.
+func MergeEmailLists(lists ...[]string) []string {
+	const (
+		maxEmailLen = 1000
+		maxEmails   = 50
+	)
+	merged := make(map[string]bool)
+	for _, list := range lists {
+		for _, email := range list {
+			addr, err := mail.ParseAddress(email)
+			if err != nil || len(addr.Address) > maxEmailLen {
+				continue
+			}
+			merged[addr.Address] = true
+		}
+	}
+	var result []string
+	for e := range merged {
+		result = append(result, e)
+	}
+	sort.Strings(result)
+	if len(result) > maxEmails {
+		result = result[:maxEmails]
+	}
+	return result
+}
diff --git a/pkg/email/parser_test.go b/pkg/email/parser_test.go
index dd758462d..5156690d0 100644
--- a/pkg/email/parser_test.go
+++ b/pkg/email/parser_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 )
 
+//!!! add tests with \r\n
 func TestExtractCommand(t *testing.T) {
 	for i, test := range extractCommandTests {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
@@ -77,58 +78,65 @@ func TestAddRemoveAddrContext(t *testing.T) {
 
 func TestParse(t *testing.T) {
 	for i, test := range parseTests {
-		t.Run(fmt.Sprint(i), func(t *testing.T) {
+		body := func(t *testing.T, test ParseTest) {
 			email, err := Parse(strings.NewReader(test.email), "bot <foo@bar.com>")
 			if err != nil {
 				t.Fatal(err)
 			}
-			if !reflect.DeepEqual(email, test.res) {
-				t.Logf("expect:\n%#v", test.res)
+			if !reflect.DeepEqual(email, &test.res) {
+				t.Logf("expect:\n%#v", &test.res)
 				t.Logf("got:\n%#v", email)
 				t.Fail()
 			}
-		})
+		}
+		t.Run(fmt.Sprint(i), func(t *testing.T) { body(t, test) })
+
+		test.email = strings.Replace(test.email, "\n", "\r\n", -1)
+		test.res.Body = strings.Replace(test.res.Body, "\n", "\r\n", -1)
+		t.Run(fmt.Sprint(i)+"rn", func(t *testing.T) { body(t, test) })
 	}
 }
 
 var extractCommandTests = []struct {
 	body string
 	cmd  string
-	args []string
+	args string
 }{
 	{
 		body: `Hello,
 
 line1
-#syzbot foo bar baz`,
+#syz  foo  bar baz 	`,
 		cmd:  "foo",
-		args: []string{"bar", "baz"},
+		args: "bar baz",
 	},
 	{
 		body: `Hello,
 
 line1
-#syzbot foo bar  	 baz
+#syz foo bar  	 baz
 line 2
 `,
-		cmd:  "foo",
-		args: []string{"bar", "baz"},
+		cmd: "foo",
+		args: "bar  	 baz",
 	},
 	{
 		body: `
 line1
-> #syzbot foo bar   baz
+> #syz foo bar   baz
 line 2
 `,
 		cmd:  "",
-		args: nil,
+		args: "",
 	},
 }
 
-var parseTests = []struct {
+type ParseTest struct {
 	email string
-	res   *Email
-}{
+	res   Email
+}
+
+var parseTests = []ParseTest{
 	{`Date: Sun, 7 May 2017 19:54:00 -0700
 Message-ID: <123>
 Subject: test subject
@@ -138,20 +146,54 @@ Content-Type: text/plain; charset="UTF-8"
 
 text body
 second line
-#syzbot command arg1 arg2 arg3
-last line`,
-		&Email{
+#syz command 	 arg1 arg2 arg3 	
+last line
+-- 
+You received this message because you are subscribed to the Google Groups "syzkaller" group.
+To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller+unsubscribe@googlegroups.com.
+To post to this group, send email to syzkaller@googlegroups.com.
+To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller/abcdef@google.com.
+For more options, visit https://groups.google.com/d/optout.`,
+		Email{
 			BugID:     "4564456",
 			MessageID: "<123>",
+			Link:      "https://groups.google.com/d/msgid/syzkaller/abcdef@google.com",
 			Subject:   "test subject",
 			From:      "\"Bob\" <bob@example.com>",
+			Cc:        []string{"bob@example.com"},
 			Body: `text body
 second line
-#syzbot command arg1 arg2 arg3
-last line`,
+#syz command 	 arg1 arg2 arg3 	
+last line
+-- 
+You received this message because you are subscribed to the Google Groups "syzkaller" group.
+To unsubscribe from this group and stop receiving emails from it, send an email to syzkaller+unsubscribe@googlegroups.com.
+To post to this group, send email to syzkaller@googlegroups.com.
+To view this discussion on the web visit https://groups.google.com/d/msgid/syzkaller/abcdef@google.com.
+For more options, visit https://groups.google.com/d/optout.`,
 			Patch:       "",
 			Command:     "command",
-			CommandArgs: []string{"arg1", "arg2", "arg3"},
+			CommandArgs: "arg1 arg2 arg3",
+		}},
+
+	{`Date: Sun, 7 May 2017 19:54:00 -0700
+Message-ID: <123>
+Subject: test subject
+From: syzbot <foo+4564456@bar.com>
+To: Bob <bob@example.com>
+Content-Type: text/plain; charset="UTF-8"
+
+text body
+last line`,
+		Email{
+			BugID:     "4564456",
+			MessageID: "<123>",
+			Subject:   "test subject",
+			From:      "\"syzbot\" <foo+4564456@bar.com>",
+			Cc:        []string{"bob@example.com"},
+			Body: `text body
+last line`,
+			Patch: "",
 		}},
 
 	{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -161,22 +203,22 @@ From: Bob <bob@example.com>
 To: syzbot <bot@example.com>, Alice <alice@example.com>
 Content-Type: text/plain
 
-#syzbot command
+#syz  command   	 
 text body
 second line
 last line`,
-		&Email{
+		Email{
 			MessageID: "<123>",
 			Subject:   "test subject",
 			From:      "\"Bob\" <bob@example.com>",
-			Cc:        []string{"\"syzbot\" <bot@example.com>", "\"Alice\" <alice@example.com>"},
-			Body: `#syzbot command
+			Cc:        []string{"alice@example.com", "bob@example.com", "bot@example.com"},
+			Body: `#syz  command   	 
 text body
 second line
 last line`,
 			Patch:       "",
 			Command:     "command",
-			CommandArgs: nil,
+			CommandArgs: "",
 		}},
 
 	{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -189,19 +231,19 @@ Content-Type: text/plain
 text body
 second line
 last line
-#syzbot command`,
-		&Email{
+#syz command`,
+		Email{
 			MessageID: "<123>",
 			Subject:   "test subject",
 			From:      "\"Bob\" <bob@example.com>",
-			Cc:        []string{"\"syzbot\" <bot@example.com>", "\"Alice\" <alice@example.com>"},
+			Cc:        []string{"alice@example.com", "bob@example.com", "bot@example.com"},
 			Body: `text body
 second line
 last line
-#syzbot command`,
+#syz command`,
 			Patch:       "",
 			Command:     "command",
-			CommandArgs: nil,
+			CommandArgs: "",
 		}},
 
 	{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -215,7 +257,7 @@ Content-Type: multipart/mixed; boundary="001a114ce0b01684a6054f0d8b81"
 Content-Type: text/plain; charset="UTF-8"
 
 body text
->#syzbot test
+>#syz test
 
 --001a114ce0b01684a6054f0d8b81
 Content-Type: text/x-patch; charset="US-ASCII"; name="patch.patch"
@@ -230,13 +272,13 @@ YXNrX3N0cnVjdCAqdCkKIAlrY292ID0gdC0+a2NvdjsKIAlpZiAoa2NvdiA9PSBOVUxMKQogCQly
 ZXR1cm47Ci0Jc3Bpbl9sb2NrKCZrY292LT5sb2NrKTsKIAlpZiAoV0FSTl9PTihrY292LT50ICE9
 IHQpKSB7CiAJCXNwaW5fdW5sb2NrKCZrY292LT5sb2NrKTsKIAkJcmV0dXJuOwo=
 --001a114ce0b01684a6054f0d8b81--`,
-		&Email{
+		Email{
 			MessageID: "<123>",
 			Subject:   "test subject",
 			From:      "\"Bob\" <bob@example.com>",
-			Cc:        []string{"\"syzbot\" <bot@example.com>"},
+			Cc:        []string{"bob@example.com", "bot@example.com"},
 			Body: `body text
->#syzbot test
+>#syz test
 `,
 			Patch: `--- a/kernel/kcov.c
 +++ b/kernel/kcov.c
@@ -250,7 +292,7 @@ IHQpKSB7CiAJCXNwaW5fdW5sb2NrKCZrY292LT5sb2NrKTsKIAkJcmV0dXJuOwo=
  		return;
 `,
 			Command:     "",
-			CommandArgs: nil,
+			CommandArgs: "",
 		}},
 
 	{`Date: Sun, 7 May 2017 19:54:00 -0700
@@ -266,7 +308,7 @@ Content-Type: text/plain; charset="UTF-8"
 On Mon, May 8, 2017 at 6:47 PM, Bob wrote:
 > body text
 
-#syzbot test
+#syz test
 
 commit 59372bbf3abd5b24a7f6f676a3968685c280f955
 Date:   Thu Apr 27 13:54:11 2017 +0200
@@ -295,7 +337,7 @@ Content-Transfer-Encoding: quoted-printable
 
 <div dir=3D"ltr">On Mon, May 8, 2017 at 6:47 PM, Dmitry Vyukov &lt;<a href=
 =3D"mailto:bob@example.com">bob@example.com</a>&gt; wrote:<br>&gt; bo=
-dy text<br><br>#syzbot test<br><br><div><div>commit 59372bbf3abd5b24a7f6f67=
+dy text<br><br>#syz test<br><br><div><div>commit 59372bbf3abd5b24a7f6f67=
 6a3968685c280f955</div><div>Date: =C2=A0 Thu Apr 27 13:54:11 2017 +0200</di=
 v><div><br></div><div>=C2=A0 =C2=A0 statx: correct error handling of NULL p=
 athname</div><div>=C2=A0 =C2=A0=C2=A0</div><div>=C2=A0 =C2=A0 test patch.</=
@@ -316,15 +358,15 @@ ce:pre">=09=09</span>return -EINVAL;</div><div>=C2=A0</div><div>=C2=A0<span=
 or)</div></div></div>
 
 --f403043eee70018593054f0d9f1f--`,
-		&Email{
+		Email{
 			MessageID: "<123>",
 			Subject:   "test subject",
 			From:      "\"Bob\" <bob@example.com>",
-			Cc:        []string{"\"syzbot\" <bot@example.com>"},
+			Cc:        []string{"bob@example.com", "bot@example.com"},
 			Body: `On Mon, May 8, 2017 at 6:47 PM, Bob wrote:
 > body text
 
-#syzbot test
+#syz test
 
 commit 59372bbf3abd5b24a7f6f676a3968685c280f955
 Date:   Thu Apr 27 13:54:11 2017 +0200
@@ -360,6 +402,6 @@ index 3d85747bd86e..a257b872a53d 100644
   if (error)
 `,
 			Command:     "test",
-			CommandArgs: nil,
+			CommandArgs: "",
 		}},
 }
diff --git a/pkg/email/reply_test.go b/pkg/email/reply_test.go
index 2dd9d894d..75d2eca09 100644
--- a/pkg/email/reply_test.go
+++ b/pkg/email/reply_test.go
@@ -29,13 +29,13 @@ var formReplyTests = []struct {
 	{
 		email: `line1
 line2
-#syzbot foo
+#syz foo
 line3
 `,
 		reply: "this is reply",
 		result: `> line1
 > line2
-> #syzbot foo
+> #syz foo
 
 this is reply
 
@@ -45,13 +45,13 @@ this is reply
 	{
 		email: `> line1
 > line2
-#syzbot foo
+#syz foo
 line3
 `,
 		reply: "this is reply\n",
 		result: `>> line1
 >> line2
-> #syzbot foo
+> #syz foo
 
 this is reply
 
@@ -61,11 +61,11 @@ this is reply
 	{
 		email: `line1
 line2
-#syzbot foo`,
+#syz foo`,
 		reply: "this is reply 1\nthis is reply 2",
 		result: `> line1
 > line2
-> #syzbot foo
+> #syz foo
 
 this is reply 1
 this is reply 2