pkg/kcidb: assorted improvements for the schema

Based on @spbnick comments:
https://groups.google.com/g/syzkaller/c/Gd-hTPH4Two/m/6viE0QLgCQAJ

Normalize git.kernel.org repo addresses.
Send dashboard link in misc rather than in output files.
Don't omit waived.

Also check sent messages with kcidb-validate if it's in PATH.

2 files changed, 44 insertions, 5 deletions

diff --git a/pkg/kcidb/client.go b/pkg/kcidb/client.go
index 16fced9c5..e04ba5dec 100644
--- a/pkg/kcidb/client.go
+++ b/pkg/kcidb/client.go
@@ -4,9 +4,12 @@
 package kcidb
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"os/exec"
 	"strings"
 	"time"
 
@@ -56,10 +59,34 @@ func (c *Client) Publish(bug *dashapi.BugReport) error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal kcidb json: %v", err)
 	}
+	if err := kcidbValidate(data); err != nil {
+		return err
+	}
 	_, err = c.topic.Publish(c.ctx, &pubsub.Message{Data: data}).Get(c.ctx)
 	return err
 }
 
+var Validate bool
+
+func kcidbValidate(data []byte) error {
+	if !Validate {
+		return nil
+	}
+	const bin = "kcidb-validate"
+	if _, err := exec.LookPath(bin); err != nil {
+		fmt.Fprintf(os.Stderr, "%v is not found\n", bin)
+		return nil
+	}
+	cmd := exec.Command(bin)
+	cmd.Stdin = bytes.NewReader(data)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%v failed (%v) on:\n%s\n\nerror: %s",
+			bin, err, data, output)
+	}
+	return nil
+}
+
 func (c *Client) convert(target *targets.Target, bug *dashapi.BugReport) *Kcidb {
 	res := &Kcidb{
 		Version: &Version{
@@ -70,7 +97,7 @@ func (c *Client) convert(target *targets.Target, bug *dashapi.BugReport) *Kcidb
 			{
 				Origin:              c.origin,
 				ID:                  bug.KernelCommit,
-				GitRepositoryURL:    bug.KernelRepo,
+				GitRepositoryURL:    normalizeRepo(bug.KernelRepo),
 				GitCommitHash:       bug.KernelCommit,
 				GitRepositoryBranch: bug.KernelBranch,
 				Description:         bug.KernelCommitTitle,
@@ -97,12 +124,11 @@ func (c *Client) convert(target *targets.Target, bug *dashapi.BugReport) *Kcidb
 		build.Valid = false
 		build.LogURL = bug.LogLink
 		build.Misc = &BuildMisc{
+			OriginURL:  bug.Link,
 			ReportedBy: bug.CreditEmail,
 		}
 	} else {
-		outputFiles := []*Resource{
-			{Name: "dashboard", URL: bug.Link},
-		}
+		var outputFiles []*Resource
 		if bug.ReportLink != "" {
 			outputFiles = append(outputFiles, &Resource{Name: "report.txt", URL: bug.ReportLink})
 		}
@@ -134,6 +160,7 @@ func (c *Client) convert(target *targets.Target, bug *dashapi.BugReport) *Kcidb
 				Status:      "FAIL",
 				Waived:      false,
 				Misc: &TestMisc{
+					OriginURL:       bug.Link,
 					ReportedBy:      bug.CreditEmail,
 					UserSpaceArch:   bug.UserSpaceArch,
 					CauseRevisionID: causeRevisionID,
@@ -144,6 +171,16 @@ func (c *Client) convert(target *targets.Target, bug *dashapi.BugReport) *Kcidb
 	return res
 }
 
+func normalizeRepo(repo string) string {
+	// Kcidb needs normalized repo addresses to match reports from different
+	// origins and with subscriptions. "https:" is always preferred over "git:"
+	// where available. Unfortunately we don't know where it's available
+	// and where it isn't. We know that "https:" is supported on kernel.org,
+	// and that's the main case we need to fix up. "https:" is always used
+	// for github.com and googlesource.com.
+	return strings.Replace(repo, "git://git.kernel.org", "https://git.kernel.org", -1)
+}
+
 func (c *Client) extID(id string) string {
 	return c.origin + ":" + id
 }
diff --git a/pkg/kcidb/schema.go b/pkg/kcidb/schema.go
index 9e2f34370..1d4549f2f 100644
--- a/pkg/kcidb/schema.go
+++ b/pkg/kcidb/schema.go
@@ -86,6 +86,7 @@ type Build struct {
 
 // Miscellaneous extra data about the build.
 type BuildMisc struct {
+	OriginURL  string `json:"origin_url,omitempty"`
 	ReportedBy string `json:"reported_by,omitempty"`
 }
 
@@ -234,11 +235,12 @@ type Test struct {
 	// Could be used for reporting test results without affecting the overall test status and alerting
 	// the contacts concerned with the tested code revision. For example, for collecting test reliability
 	// statistics when the test is first introduced, or is being fixed.
-	Waived bool `json:"waived,omitempty"`
+	Waived bool `json:"waived"`
 }
 
 // Miscellaneous extra data about the test.
 type TestMisc struct {
+	OriginURL       string `json:"origin_url,omitempty"`
 	ReportedBy      string `json:"reported_by,omitempty"`
 	UserSpaceArch   string `json:"user_space_arch,omitempty"`
 	CauseRevisionID string `json:"cause_revision_id,omitempty"`