syz-cluster: recognize bug reports by addr context

For the emails sent via GAE, we don't immediately know the MessageID.
Rely on addr context for their identification.

4 files changed, 55 insertions, 11 deletions

diff --git a/syz-cluster/email-reporter/handler_test.go b/syz-cluster/email-reporter/handler_test.go
index b01cae14b..af3e452a7 100644
--- a/syz-cluster/email-reporter/handler_test.go
+++ b/syz-cluster/email-reporter/handler_test.go
@@ -165,4 +165,8 @@ var testEmailConfig = &app.EmailConfig{
 	DocsLink:       "docs",
 	ModerationList: "moderation@list.com",
 	ArchiveList:    "archive@list.com",
+	Sender:         app.SenderSMTP,
+	SMTP: &app.SMTPConfig{
+		From: "a@b.com",
+	},
 }
diff --git a/syz-cluster/email-reporter/main.go b/syz-cluster/email-reporter/main.go
index da1ebb755..277482687 100644
--- a/syz-cluster/email-reporter/main.go
+++ b/syz-cluster/email-reporter/main.go
@@ -51,8 +51,7 @@ func main() {
 	msgCh := make(chan *email.Email, 16)
 	eg, loopCtx := errgroup.WithContext(ctx)
 	if cfg.EmailReporting.LoreArchiveURL != "" {
-		fetcher := NewLKMLEmailStream("/lore-repo",
-			cfg.EmailReporting.LoreArchiveURL, reporterClient, msgCh)
+		fetcher := NewLKMLEmailStream("/lore-repo", reporterClient, cfg.EmailReporting, msgCh)
 		eg.Go(func() error { return fetcher.Loop(loopCtx, fetcherPollPeriod) })
 	}
 	eg.Go(func() error {
diff --git a/syz-cluster/email-reporter/stream.go b/syz-cluster/email-reporter/stream.go
index 5138df0a6..c76826e27 100644
--- a/syz-cluster/email-reporter/stream.go
+++ b/syz-cluster/email-reporter/stream.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/google/syzkaller/pkg/email"
@@ -17,8 +18,9 @@ import (
 )
 
 type LKMLEmailStream struct {
+	cfg            *app.EmailConfig
+	ownEmails      []string
 	reporterName   string
-	repoURL        string
 	repoFolder     string
 	client         *api.ReporterClient
 	newMessages    chan *email.Email
@@ -26,11 +28,16 @@ type LKMLEmailStream struct {
 	lastCommit     string
 }
 
-func NewLKMLEmailStream(repoFolder, repoURL string, client *api.ReporterClient,
-	writeTo chan *email.Email) *LKMLEmailStream {
+func NewLKMLEmailStream(repoFolder string, client *api.ReporterClient,
+	cfg *app.EmailConfig, writeTo chan *email.Email) *LKMLEmailStream {
+	var ownEmails []string
+	if cfg.Dashapi != nil {
+		ownEmails = append(ownEmails, cfg.Dashapi.From)
+	}
 	return &LKMLEmailStream{
+		cfg:          cfg,
+		ownEmails:    ownEmails,
 		reporterName: api.LKMLReporter,
-		repoURL:      repoURL,
 		repoFolder:   repoFolder,
 		client:       client,
 		newMessages:  writeTo,
@@ -69,7 +76,7 @@ func (s *LKMLEmailStream) Loop(ctx context.Context, pollPeriod time.Duration) er
 
 func (s *LKMLEmailStream) fetchMessages(ctx context.Context) error {
 	gitRepo := vcs.NewLKMLRepo(s.repoFolder)
-	_, err := gitRepo.Poll(s.repoURL, "master")
+	_, err := gitRepo.Poll(s.cfg.LoreArchiveURL, "master")
 	if err != nil {
 		return err
 	}
@@ -86,7 +93,7 @@ func (s *LKMLEmailStream) fetchMessages(ctx context.Context) error {
 	// From oldest to newest.
 	for i := len(messages) - 1; i >= 0; i-- {
 		msg := messages[i]
-		parsed, err := msg.Parse(nil, nil)
+		parsed, err := msg.Parse(s.ownEmails, nil)
 		if err != nil || parsed == nil {
 			log.Printf("failed to parse the email from hash %q: %v", msg.Hash, err)
 			continue
@@ -104,6 +111,7 @@ func (s *LKMLEmailStream) fetchMessages(ctx context.Context) error {
 		}
 		resp, err := s.client.RecordReply(ctx, &api.RecordReplyReq{
 			MessageID: parsed.MessageID,
+			ReportID:  s.extractMessageID(parsed),
 			InReplyTo: parsed.InReplyTo,
 			Reporter:  s.reporterName,
 			Time:      messageDate,
@@ -125,3 +133,17 @@ func (s *LKMLEmailStream) fetchMessages(ctx context.Context) error {
 	}
 	return nil
 }
+
+// If the message was sent via the dashapi sender, the report ID wil be a part of the email address.
+func (s *LKMLEmailStream) extractMessageID(msg *email.Email) string {
+	if s.cfg.Dashapi == nil {
+		// The mode is not configured.
+		return ""
+	}
+	for _, id := range msg.BugIDs {
+		if strings.HasPrefix(id, s.cfg.Dashapi.ContextPrefix) {
+			return strings.TrimPrefix(id, s.cfg.Dashapi.ContextPrefix)
+		}
+	}
+	return ""
+}
diff --git a/syz-cluster/email-reporter/stream_test.go b/syz-cluster/email-reporter/stream_test.go
index 0d79b158f..e98fd26d4 100644
--- a/syz-cluster/email-reporter/stream_test.go
+++ b/syz-cluster/email-reporter/stream_test.go
@@ -39,8 +39,14 @@ func TestEmailStream(t *testing.T) {
 	// Emulate the lore archive and set up the loop.
 	loreArchive := newLoreArchive(t)
 	writeTo := make(chan *email.Email, 16)
-	stream := NewLKMLEmailStream(t.TempDir(), loreArchive.remoteRef(), reporterClient, writeTo)
-
+	emailCfg := &app.EmailConfig{
+		LoreArchiveURL: loreArchive.remoteRef(),
+		Dashapi: &app.DashapiConfig{
+			From:          "bot@syzbot.org",
+			ContextPrefix: "ci_",
+		},
+	}
+	stream := NewLKMLEmailStream(t.TempDir(), reporterClient, emailCfg, writeTo)
 	cancel := startStreamLoop(t, ctx, stream)
 
 	t.Logf("sending a direct reply")
@@ -79,11 +85,24 @@ Content-Type: text/plain
 	msg = <-writeTo
 	assert.Len(t, msg.BugIDs, 0)
 
+	t.Logf("identify by email context")
+	loreArchive.saveMessage(t, `Date: Sun, 7 May 2017 19:55:00 -0700
+Subject: New thread
+Message-ID: <new-thread>
+In-Reply-To: <whatever>
+From: Someone Else <b@syzbot.org>
+Cc: <bot+ci_`+report.ID+`@syzbot.org>
+Content-Type: text/plain
+
+`)
+	msg = <-writeTo
+	assert.Equal(t, []string{report.ID}, msg.BugIDs)
+
 	t.Logf("stopping the loop")
 	cancel()
 
 	// Emulate service restart.
-	stream = NewLKMLEmailStream(t.TempDir(), loreArchive.remoteRef(), reporterClient, writeTo)
+	stream = NewLKMLEmailStream(t.TempDir(), reporterClient, emailCfg, writeTo)
 	cancel = startStreamLoop(t, ctx, stream)
 	defer cancel()
 	// Only the unrelated message is expected to pop up.