dashboard: implement asset storage

Asset is a file/attachment relevant for debugging a kernel crash or the
syzbot itself.

Dashboard keeps track of the uploaded assets and manages their lifetime.
Some of the assets are attached to the bug reports sent by syzkaller,
some (like html coverage report) are displayed on the web portal.

Two new API requests:
* add_build_assets -- let dashboard remember one more build-related
  asset.
* needed_assets -- query the list of assets that are still needed by the
  dashboard.

13 files changed, 788 insertions, 12 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index e76a9d746..411c46440 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"reflect"
 	"regexp"
 	"sort"
@@ -18,6 +19,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/asset"
 	"github.com/google/syzkaller/pkg/auth"
 	"github.com/google/syzkaller/pkg/email"
 	"github.com/google/syzkaller/pkg/hash"
@@ -40,6 +42,7 @@ var apiHandlers = map[string]APIHandler{
 	"reporting_poll_notifs": apiReportingPollNotifications,
 	"reporting_poll_closed": apiReportingPollClosed,
 	"reporting_update":      apiReportingUpdate,
+	"needed_assets":         apiNeededAssetsList,
 }
 
 var apiNamespaceHandlers = map[string]APINamespaceHandler{
@@ -54,6 +57,7 @@ var apiNamespaceHandlers = map[string]APINamespaceHandler{
 	"upload_commits":      apiUploadCommits,
 	"bug_list":            apiBugList,
 	"load_bug":            apiLoadBug,
+	"add_build_assets":    apiAddBuildAssets,
 }
 
 type JSONHandler func(c context.Context, r *http.Request) (interface{}, error)
@@ -417,10 +421,17 @@ func apiUploadBuild(c context.Context, ns string, r *http.Request, payload []byt
 
 func uploadBuild(c context.Context, now time.Time, ns string, req *dashapi.Build, typ BuildType) (
 	*Build, bool, error) {
+	newAssets := []Asset{}
+	for i, toAdd := range req.Assets {
+		newAsset, err := parseIncomingAsset(c, toAdd)
+		if err != nil {
+			return nil, false, fmt.Errorf("failed to parse asset #%d: %w", i, err)
+		}
+		newAssets = append(newAssets, newAsset)
+	}
 	if build, err := loadBuild(c, ns, req.ID); err == nil {
 		return build, false, nil
 	}
-
 	checkStrLen := func(str, name string, maxLen int) error {
 		if str == "" {
 			return fmt.Errorf("%v is empty", name)
@@ -473,6 +484,7 @@ func uploadBuild(c context.Context, now time.Time, ns string, req *dashapi.Build
 		KernelCommitTitle:   req.KernelCommitTitle,
 		KernelCommitDate:    req.KernelCommitDate,
 		KernelConfig:        configID,
+		Assets:              newAssets,
 	}
 	if _, err := db.Put(c, buildKey(c, ns, req.ID), build); err != nil {
 		return nil, false, err
@@ -1058,6 +1070,46 @@ func loadBugReport(c context.Context, bug *Bug) (*dashapi.BugReport, error) {
 	return createBugReport(c, bug, crash, crashKey, bugReporting, reporting)
 }
 
+func apiAddBuildAssets(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
+	req := new(dashapi.AddBuildAssetsReq)
+	if err := json.Unmarshal(payload, req); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
+	}
+	assets := []Asset{}
+	for i, toAdd := range req.Assets {
+		asset, err := parseIncomingAsset(c, toAdd)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse asset #%d: %w", i, err)
+		}
+		assets = append(assets, asset)
+	}
+	_, err := appendBuildAssets(c, ns, req.BuildID, assets)
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func parseIncomingAsset(c context.Context, newAsset dashapi.NewAsset) (Asset, error) {
+	typeInfo := asset.GetTypeDescription(newAsset.Type)
+	if typeInfo == nil {
+		return Asset{}, fmt.Errorf("unknown asset type")
+	}
+	_, err := url.ParseRequestURI(newAsset.DownloadURL)
+	if err != nil {
+		return Asset{}, fmt.Errorf("invalid URL: %w", err)
+	}
+	return Asset{
+		Type:        newAsset.Type,
+		DownloadURL: newAsset.DownloadURL,
+		CreateDate:  timeNow(c),
+	}, nil
+}
+
+func apiNeededAssetsList(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	return queryNeededAssets(c)
+}
+
 func findExistingBugForCrash(c context.Context, ns string, titles []string) (*Bug, error) {
 	// First, try to find an existing bug that we already used to report this crash title.
 	var bugs []*Bug
diff --git a/dashboard/app/app.yaml b/dashboard/app/app.yaml
index 67305a710..a4b947a96 100644
--- a/dashboard/app/app.yaml
+++ b/dashboard/app/app.yaml
@@ -25,7 +25,7 @@ handlers:
 - url: /static
   static_dir: static
   secure: always
-- url: /(admin|email_poll|cache_update|kcidb_poll)
+- url: /(admin|email_poll|cache_update|kcidb_poll|deprecate_assets)
   script: auto
   login: admin
   secure: always
diff --git a/dashboard/app/asset_storage.go b/dashboard/app/asset_storage.go
new file mode 100644
index 000000000..e618727cd
--- /dev/null
+++ b/dashboard/app/asset_storage.go
@@ -0,0 +1,338 @@
+// Copyright 2022 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/asset"
+	"golang.org/x/net/context"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/appengine/v2"
+	db "google.golang.org/appengine/v2/datastore"
+	"google.golang.org/appengine/v2/log"
+)
+
+// TODO: decide if we want to save job-related assets.
+
+func appendBuildAssets(c context.Context, ns, buildID string, assets []Asset) (*Build, error) {
+	var retBuild *Build
+	tx := func(c context.Context) error {
+		build, err := loadBuild(c, ns, buildID)
+		if err != nil {
+			return err
+		}
+		retBuild = build
+		appendedOk := false
+		var appendErr error
+		for _, newAsset := range assets {
+			appendErr = build.AppendAsset(newAsset)
+			if appendErr == nil {
+				appendedOk = true
+			}
+		}
+		// It took quite a number of resources to upload the files, so we return success
+		// even if we managed to save at least one of the new assets.
+		if !appendedOk {
+			return fmt.Errorf("failed to append all assets, last error %w", appendErr)
+		}
+		if _, err := db.Put(c, buildKey(c, ns, buildID), build); err != nil {
+			return fmt.Errorf("failed to put build: %w", err)
+		}
+		log.Infof(c, "updated build: %#v", build)
+		return nil
+	}
+	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
+		return nil, err
+	}
+	return retBuild, nil
+}
+
+var ErrAssetDuplicated = errors.New("an asset of this type is already present")
+
+func (build *Build) AppendAsset(addAsset Asset) error {
+	typeInfo := asset.GetTypeDescription(addAsset.Type)
+	if typeInfo == nil {
+		return fmt.Errorf("unknown asset type")
+	}
+	if !typeInfo.AllowMultiple {
+		for _, obj := range build.Assets {
+			if obj.Type == addAsset.Type {
+				return ErrAssetDuplicated
+			}
+		}
+	}
+	build.Assets = append(build.Assets, addAsset)
+	return nil
+}
+
+func queryNeededAssets(c context.Context) (*dashapi.NeededAssetsResp, error) {
+	// So far only build assets.
+	var builds []*Build
+	_, err := db.NewQuery("Build").
+		Filter("Assets.DownloadURL>", "").
+		Project("Assets.DownloadURL").
+		GetAll(c, &builds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query builds: %w", err)
+	}
+	log.Infof(c, "queried %v builds with assets", len(builds))
+	resp := &dashapi.NeededAssetsResp{}
+	for _, build := range builds {
+		for _, asset := range build.Assets {
+			resp.DownloadURLs = append(resp.DownloadURLs, asset.DownloadURL)
+		}
+	}
+	return resp, nil
+}
+
+func handleDeprecateAssets(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	for ns := range config.Namespaces {
+		err := deprecateNamespaceAssets(c, ns)
+		if err != nil {
+			log.Errorf(c, "deprecateNamespaceAssets failed for ns=%v: %v", ns, err)
+		}
+	}
+}
+
+func deprecateNamespaceAssets(c context.Context, ns string) error {
+	ad := assetDeprecator{
+		ns: ns,
+		c:  c,
+	}
+	const buildBatchSize = 16
+	err := ad.batchProcessBuilds(buildBatchSize)
+	if err != nil {
+		return fmt.Errorf("build batch processing failed: %w", err)
+	}
+	return nil
+}
+
+type assetDeprecator struct {
+	ns           string
+	c            context.Context
+	bugsQueried  bool
+	relevantBugs map[string]bool
+}
+
+const keepAssetsForClosedBugs = time.Hour * 24 * 30
+
+func (ad *assetDeprecator) queryBugs() error {
+	if ad.bugsQueried {
+		return nil
+	}
+	var openBugKeys []*db.Key
+	var closedBugKeys []*db.Key
+	g, _ := errgroup.WithContext(context.Background())
+	g.Go(func() error {
+		// Query open bugs.
+		var err error
+		openBugKeys, err = db.NewQuery("Bug").
+			Filter("Namespace=", ad.ns).
+			Filter("Status=", BugStatusOpen).
+			KeysOnly().
+			GetAll(ad.c, nil)
+		if err != nil {
+			return fmt.Errorf("failed to fetch open builds: %w", err)
+		}
+		return nil
+	})
+	g.Go(func() error {
+		// Query recently closed bugs.
+		var err error
+		closedBugKeys, err = db.NewQuery("Bug").
+			Filter("Namespace=", ad.ns).
+			Filter("Closed>", timeNow(ad.c).Add(-keepAssetsForClosedBugs)).
+			KeysOnly().
+			GetAll(ad.c, nil)
+		if err != nil {
+			return fmt.Errorf("failed to fetch closed builds: %w", err)
+		}
+		return nil
+	})
+	err := g.Wait()
+	if err != nil {
+		return fmt.Errorf("failed to query bugs: %w", err)
+	}
+	ad.relevantBugs = map[string]bool{}
+	for _, key := range append(append([]*db.Key{}, openBugKeys...), closedBugKeys...) {
+		ad.relevantBugs[key.String()] = true
+	}
+	return nil
+}
+
+func (ad *assetDeprecator) buildArchivePolicy(build *Build, asset *Asset) (bool, error) {
+	// If the asset is reasonably new, we always keep it.
+	const alwaysKeepPeriod = time.Hour * 24 * 14
+	if asset.CreateDate.After(timeNow(ad.c).Add(-alwaysKeepPeriod)) {
+		return true, nil
+	}
+	// Query builds to see whether there's a newer same-type asset on the same week.
+	var builds []*Build
+	_, err := db.NewQuery("Build").
+		Filter("Namespace=", ad.ns).
+		Filter("Manager=", build.Manager).
+		Filter("Assets.Type=", asset.Type).
+		Filter("Assets.CreateDate>", asset.CreateDate).
+		Limit(1).
+		Order("Assets.CreateDate").
+		GetAll(ad.c, &builds)
+	if err != nil {
+		return false, fmt.Errorf("failed to query newer assets: %w", err)
+	}
+	log.Infof(ad.c, "running archive policy for %s, date %s; queried %d builds",
+		asset.DownloadURL, asset.CreateDate, len(builds))
+	sameWeek := false
+	if len(builds) > 0 {
+		origY, origW := asset.CreateDate.ISOWeek()
+		for _, nextAsset := range builds[0].Assets {
+			if nextAsset.Type != asset.Type {
+				continue
+			}
+			if nextAsset.CreateDate.Before(asset.CreateDate) ||
+				nextAsset.CreateDate.Equal(asset.CreateDate) {
+				continue
+			}
+			nextY, nextW := nextAsset.CreateDate.ISOWeek()
+			if origY == nextY && origW == nextW {
+				log.Infof(ad.c, "found a newer asset: %s, date %s",
+					nextAsset.DownloadURL, nextAsset.CreateDate)
+				sameWeek = true
+				break
+			}
+		}
+	}
+	return !sameWeek, nil
+}
+
+func (ad *assetDeprecator) buildBugStatusPolicy(build *Build) (bool, error) {
+	if err := ad.queryBugs(); err != nil {
+		return false, fmt.Errorf("failed to query bugs: %w", err)
+	}
+	keys, err := db.NewQuery("Crash").
+		Filter("BuildID=", build.ID).
+		KeysOnly().
+		GetAll(ad.c, nil)
+	if err != nil {
+		return false, fmt.Errorf("failed to query crashes: %w", err)
+	}
+	for _, key := range keys {
+		bugKey := key.Parent()
+		if _, ok := ad.relevantBugs[bugKey.String()]; ok {
+			// At least one crash is related to an opened/recently closed bug.
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (ad *assetDeprecator) needThisBuildAsset(build *Build, buildAsset *Asset) (bool, error) {
+	if buildAsset.Type == dashapi.HTMLCoverageReport {
+		// We want to keep coverage reports forever, not just
+		// while there are any open bugs. But we don't want to
+		// keep all coverage reports, just a share of them.
+		return ad.buildArchivePolicy(build, buildAsset)
+	}
+	if build.Type == BuildNormal {
+		// A build-related asset, keep it only while there are open bugs with crashes
+		// related to this build.
+		return ad.buildBugStatusPolicy(build)
+	}
+	// TODO: fix this once this is no longer the case.
+	return false, fmt.Errorf("job-related assets are not supported yet")
+}
+
+func (ad *assetDeprecator) updateBuild(buildID string, urlsToDelete []string) error {
+	toDelete := map[string]bool{}
+	for _, url := range urlsToDelete {
+		toDelete[url] = true
+	}
+	tx := func(c context.Context) error {
+		build, err := loadBuild(ad.c, ad.ns, buildID)
+		if build == nil || err != nil {
+			// Assume the DB has been updated in the meanwhile.
+			return nil
+		}
+		newAssets := []Asset{}
+		for _, asset := range build.Assets {
+			if _, ok := toDelete[asset.DownloadURL]; !ok {
+				newAssets = append(newAssets, asset)
+			}
+		}
+		build.Assets = newAssets
+		build.AssetsLastCheck = timeNow(ad.c)
+		if _, err := db.Put(ad.c, buildKey(ad.c, ad.ns, buildID), build); err != nil {
+			return fmt.Errorf("failed to save build: %w", err)
+		}
+		return nil
+	}
+	if err := db.RunInTransaction(ad.c, tx, nil); err != nil {
+		return fmt.Errorf("failed to update build: %w", err)
+	}
+	return nil
+}
+
+func (ad *assetDeprecator) batchProcessBuilds(count int) error {
+	// We cannot query only the Build with non-empty Assets array and yet sort
+	// by AssetsLastCheck. The datastore returns "The first sort property must
+	// be the same as the property to which the inequality filter is applied.
+	// In your query the first sort property is AssetsLastCheck but the inequality
+	// filter is on Assets.DownloadURL.
+	// So we have to omit Filter("Assets.DownloadURL>", ""). here.
+	var builds []*Build
+	_, err := db.NewQuery("Build").
+		Filter("Namespace=", ad.ns).
+		Order("AssetsLastCheck").
+		Limit(count).
+		GetAll(ad.c, &builds)
+	if err != nil {
+		return fmt.Errorf("failed to fetch builds: %w", err)
+	}
+	for _, build := range builds {
+		toDelete := []string{}
+		for _, asset := range build.Assets {
+			needed, err := ad.needThisBuildAsset(build, &asset)
+			if err != nil {
+				return fmt.Errorf("failed to test asset: %w", err)
+			} else if !needed {
+				toDelete = append(toDelete, asset.DownloadURL)
+			}
+		}
+		err := ad.updateBuild(build.ID, toDelete)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func queryLatestManagerAssets(c context.Context, ns string, assetType dashapi.AssetType,
+	period time.Duration) (map[string]Asset, error) {
+	var builds []*Build
+	startTime := timeNow(c).Add(-period)
+	_, err := db.NewQuery("Build").
+		Filter("Namespace=", ns).
+		Filter("Assets.Type=", assetType).
+		Filter("Assets.CreateDate>", startTime).
+		Order("Assets.CreateDate").
+		GetAll(c, &builds)
+	if err != nil {
+		return nil, err
+	}
+	ret := map[string]Asset{}
+	for _, build := range builds {
+		for _, asset := range build.Assets {
+			if asset.Type != assetType {
+				continue
+			}
+			ret[build.Manager] = asset
+		}
+	}
+	return ret, nil
+}
diff --git a/dashboard/app/asset_storage_test.go b/dashboard/app/asset_storage_test.go
new file mode 100644
index 000000000..0fe6b5c32
--- /dev/null
+++ b/dashboard/app/asset_storage_test.go
@@ -0,0 +1,259 @@
+// Copyright 2022 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/email"
+)
+
+func TestBuildAssetLifetime(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	build := testBuild(1)
+	// Embed one of the assets right away.
+	build.Assets = []dashapi.NewAsset{
+		{
+			Type:        dashapi.KernelObject,
+			DownloadURL: "http://google.com/vmlinux",
+		},
+	}
+	c.client2.UploadBuild(build)
+
+	// "Upload" several more assets.
+	c.expectOK(c.client2.AddBuildAssets(&dashapi.AddBuildAssetsReq{
+		BuildID: build.ID,
+		Assets: []dashapi.NewAsset{
+			{
+				Type:        dashapi.BootableDisk,
+				DownloadURL: "http://google.com/bootable_disk",
+			},
+		},
+	}))
+	c.expectOK(c.client2.AddBuildAssets(&dashapi.AddBuildAssetsReq{
+		BuildID: build.ID,
+		Assets: []dashapi.NewAsset{
+			{
+				Type:        dashapi.HTMLCoverageReport,
+				DownloadURL: "http://google.com/coverage.html",
+			},
+		},
+	}))
+
+	crash := testCrash(build, 1)
+	crash.Maintainers = []string{`"Foo Bar" <foo@bar.com>`, `bar@foo.com`, `idont@want.EMAILS`}
+	c.client2.ReportCrash(crash)
+
+	// Test that the reporting email is correct.
+	msg := c.pollEmailBug()
+	sender, extBugID, err := email.RemoveAddrContext(msg.Sender)
+	c.expectOK(err)
+	_, dbCrash, dbBuild := c.loadBug(extBugID)
+	crashLogLink := externalLink(c.ctx, textCrashLog, dbCrash.Log)
+	kernelConfigLink := externalLink(c.ctx, textKernelConfig, dbBuild.KernelConfig)
+	c.expectEQ(sender, fromAddr(c.ctx))
+	to := config.Namespaces["test2"].Reporting[0].Config.(*EmailConfig).Email
+	c.expectEQ(msg.To, []string{to})
+	c.expectEQ(msg.Subject, crash.Title)
+	c.expectEQ(len(msg.Attachments), 0)
+	c.expectEQ(msg.Body, fmt.Sprintf(`Hello,
+
+syzbot found the following issue on:
+
+HEAD commit:    111111111111 kernel_commit_title1
+git tree:       repo1 branch1
+console output: %[2]v
+kernel config:  %[3]v
+dashboard link: https://testapp.appspot.com/bug?extid=%[1]v
+compiler:       compiler1
+CC:             [bar@foo.com foo@bar.com idont@want.EMAILS]
+
+Unfortunately, I don't have any reproducer for this issue yet.
+
+Downloadable assets:
+disk image: http://google.com/bootable_disk
+vmlinux: http://google.com/vmlinux
+
+IMPORTANT: if you fix the issue, please add the following tag to the commit:
+Reported-by: syzbot+%[1]v@testapp.appspotmail.com
+
+report1
+
+---
+This report is generated by a bot. It may contain errors.
+See https://goo.gl/tpsmEJ for more information about syzbot.
+syzbot engineers can be reached at syzkaller@googlegroups.com.
+
+syzbot will keep track of this issue. See:
+https://goo.gl/tpsmEJ#status for how to communicate with syzbot.`,
+		extBugID, crashLogLink, kernelConfigLink))
+	c.checkURLContents(crashLogLink, crash.Log)
+	c.checkURLContents(kernelConfigLink, build.KernelConfig)
+
+	// We query the needed assets. We need all 3.
+	needed, err := c.client2.NeededAssetsList()
+	c.expectOK(err)
+	sort.Strings(needed.DownloadURLs)
+	allDownloadURLs := []string{
+		"http://google.com/bootable_disk",
+		"http://google.com/coverage.html",
+		"http://google.com/vmlinux",
+	}
+	c.expectEQ(needed.DownloadURLs, allDownloadURLs)
+
+	// Invalidate the bug.
+	c.client.updateBug(extBugID, dashapi.BugStatusInvalid, "")
+	_, err = c.GET("/deprecate_assets")
+	c.expectOK(err)
+
+	// Query the needed assets once more, so far there should be no change.
+	needed, err = c.client2.NeededAssetsList()
+	c.expectOK(err)
+	sort.Strings(needed.DownloadURLs)
+	c.expectEQ(needed.DownloadURLs, allDownloadURLs)
+
+	// Skip one month and deprecate assets.
+	c.advanceTime(time.Hour * 24 * 31)
+	_, err = c.GET("/deprecate_assets")
+	c.expectOK(err)
+
+	// Only the html asset should have persisted.
+	needed, err = c.client2.NeededAssetsList()
+	c.expectOK(err)
+	c.expectEQ(needed.DownloadURLs, []string{"http://google.com/coverage.html"})
+}
+
+func TestCoverReportDisplay(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	build := testBuild(1)
+	c.client.UploadBuild(build)
+
+	// Upload the second build to just make sure coverage reports are assigned per-manager.
+	c.client.UploadBuild(testBuild(2))
+
+	// We expect no coverage reports to be present.
+	uiManagers, err := loadManagers(c.ctx, AccessAdmin, "test1", "")
+	c.expectOK(err)
+	c.expectEQ(len(uiManagers), 2)
+	c.expectEQ(uiManagers[0].CoverLink, "")
+	c.expectEQ(uiManagers[1].CoverLink, "")
+
+	// Upload an asset.
+	origHTMLAsset := "http://google.com/coverage0.html"
+	c.expectOK(c.client.AddBuildAssets(&dashapi.AddBuildAssetsReq{
+		BuildID: build.ID,
+		Assets: []dashapi.NewAsset{
+			{
+				Type:        dashapi.HTMLCoverageReport,
+				DownloadURL: origHTMLAsset,
+			},
+		},
+	}))
+	uiManagers, err = loadManagers(c.ctx, AccessAdmin, "test1", "")
+	c.expectOK(err)
+	c.expectEQ(len(uiManagers), 2)
+	c.expectEQ(uiManagers[0].CoverLink, origHTMLAsset)
+	c.expectEQ(uiManagers[1].CoverLink, "")
+
+	// Upload a newer coverage.
+	newHTMLAsset := "http://google.com/coverage1.html"
+	c.expectOK(c.client.AddBuildAssets(&dashapi.AddBuildAssetsReq{
+		BuildID: build.ID,
+		Assets: []dashapi.NewAsset{
+			{
+				Type:        dashapi.HTMLCoverageReport,
+				DownloadURL: newHTMLAsset,
+			},
+		},
+	}))
+	uiManagers, err = loadManagers(c.ctx, AccessAdmin, "test1", "")
+	c.expectOK(err)
+	c.expectEQ(len(uiManagers), 2)
+	c.expectEQ(uiManagers[0].CoverLink, newHTMLAsset)
+	c.expectEQ(uiManagers[1].CoverLink, "")
+}
+
+func TestCoverReportDeprecation(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	ensureNeeded := func(needed []string) {
+		_, err := c.GET("/deprecate_assets")
+		c.expectOK(err)
+		neededResp, err := c.client.NeededAssetsList()
+		c.expectOK(err)
+		sort.Strings(neededResp.DownloadURLs)
+		sort.Strings(needed)
+		c.expectEQ(neededResp.DownloadURLs, needed)
+	}
+
+	build := testBuild(1)
+	c.client.UploadBuild(build)
+
+	uploadReport := func(url string) {
+		c.expectOK(c.client.AddBuildAssets(&dashapi.AddBuildAssetsReq{
+			BuildID: build.ID,
+			Assets: []dashapi.NewAsset{
+				{
+					Type:        dashapi.HTMLCoverageReport,
+					DownloadURL: url,
+				},
+			},
+		}))
+	}
+
+	// Week 1. Saturday Jan 1st, 2000.
+	weekOneFirst := "http://google.com/coverage1_1.html"
+	uploadReport(weekOneFirst)
+
+	// Week 1. Sunday Jan 2nd, 2000.
+	weekOneSecond := "http://google.com/coverage1_2.html"
+	c.advanceTime(time.Hour * 24)
+	uploadReport(weekOneSecond)
+	ensureNeeded([]string{weekOneFirst, weekOneSecond})
+
+	// Week 2. Tuesday Jan 4nd, 2000.
+	weekTwoFirst := "http://google.com/coverage2_1.html"
+	c.advanceTime(time.Hour * 24 * 2)
+	uploadReport(weekTwoFirst)
+	ensureNeeded([]string{weekOneFirst, weekOneSecond, weekTwoFirst})
+
+	// Week 2. Thu Jan 6nd, 2000.
+	weekTwoSecond := "http://google.com/coverage2_2.html"
+	c.advanceTime(time.Hour * 24 * 2)
+	uploadReport(weekTwoSecond)
+	ensureNeeded([]string{weekOneFirst, weekOneSecond, weekTwoFirst, weekTwoSecond})
+
+	// Week 3. Monday Jan 10th, 2000.
+	weekThreeFirst := "http://google.com/coverage3_1.html"
+	c.advanceTime(time.Hour * 24 * 4)
+	uploadReport(weekThreeFirst)
+	ensureNeeded([]string{weekOneFirst, weekOneSecond, weekTwoFirst, weekTwoSecond, weekThreeFirst})
+
+	// Week 4. Monday Jan 17th, 2000.
+	weekFourFirst := "http://google.com/coverage4_1.html"
+	c.advanceTime(time.Hour * 24 * 7)
+	uploadReport(weekFourFirst)
+
+	t.Logf("embargo is over, time is %s", timeNow(c.ctx))
+	// Note that now that the two week deletion embargo has passed, the first asset
+	// begins to falls out.
+	ensureNeeded([]string{weekOneSecond, weekTwoFirst, weekTwoSecond, weekThreeFirst, weekFourFirst})
+
+	// Week 5. Monday Jan 24th, 2000.
+	c.advanceTime(time.Hour * 24 * 7)
+	ensureNeeded([]string{weekOneSecond, weekTwoSecond, weekThreeFirst, weekFourFirst})
+
+	// A year later.
+	c.advanceTime(time.Hour * 24 * 365)
+	ensureNeeded([]string{weekOneSecond, weekTwoSecond, weekThreeFirst, weekFourFirst})
+}
diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml
index 1e8ee2d63..f46621a05 100644
--- a/dashboard/app/cron.yaml
+++ b/dashboard/app/cron.yaml
@@ -6,6 +6,8 @@ cron:
   schedule: every 1 minutes
 - url: /cache_update
   schedule: every 1 hours
+- url: /deprecate_assets
+  schedule: every 1 hours
 - url: /kcidb_poll
   schedule: every 5 minutes
 - url: /_ah/datastore_admin/backup.create?name=backup&filesystem=gs&gs_bucket_name=syzkaller-backups&kind=Bug&kind=Build&kind=Crash&kind=CrashLog&kind=CrashReport&kind=Error&kind=Job&kind=KernelConfig&kind=Manager&kind=ManagerStats&kind=Patch&kind=ReportingState&kind=ReproC&kind=ReproSyz
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 71dd57548..e19d3a9e2 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -50,6 +50,12 @@ type ManagerStats struct {
 	TotalExecs        int64
 }
 
+type Asset struct {
+	Type        dashapi.AssetType
+	DownloadURL string
+	CreateDate  time.Time
+}
+
 type Build struct {
 	Namespace           string
 	Manager             string
@@ -68,6 +74,8 @@ type Build struct {
 	KernelCommitTitle   string    `datastore:",noindex"`
 	KernelCommitDate    time.Time `datastore:",noindex"`
 	KernelConfig        int64     // reference to KernelConfig text entity
+	Assets              []Asset   // build-related assets
+	AssetsLastCheck     time.Time // the last time we checked the assets for deprecation
 }
 
 type Bug struct {
@@ -177,6 +185,7 @@ type ReportingStateEntry struct {
 //   - test of a committed fix
 //   - reproduce crash
 //   - test that crash still happens on HEAD
+//
 // Job has Bug as parent entity.
 type Job struct {
 	Type      JobType
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
index be7438bbc..54ae729db 100644
--- a/dashboard/app/index.yaml
+++ b/dashboard/app/index.yaml
@@ -102,6 +102,24 @@ indexes:
   - name: Time
     direction: desc
 
+- kind: Build
+  properties:
+  - name: Namespace
+  - name: Manager
+  - name: Assets.Type
+  - name: Assets.CreateDate
+
+- kind: Build
+  properties:
+  - name: Namespace
+  - name: Assets.Type
+  - name: Assets.CreateDate
+
+- kind: Build
+  properties:
+  - name: Namespace
+  - name: AssetsLastCheck
+
 - kind: Crash
   ancestor: yes
   properties:
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
index 1e1ff85bd..e4c2436b7 100644
--- a/dashboard/app/jobs.go
+++ b/dashboard/app/jobs.go
@@ -364,7 +364,10 @@ func createBisectJobForBug(c context.Context, bug0 *Bug, crash *Crash, bugKey, c
 		}
 		return markCrashReported(c, job.CrashID, bugKey, now)
 	}
-	if err := db.RunInTransaction(c, tx, nil); err != nil {
+	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{
+		// We're accessing two different kinds in markCrashReported.
+		XG: true,
+	}); err != nil {
 		return nil, nil, fmt.Errorf("create bisect job tx failed: %v", err)
 	}
 	return job, jobKey, nil
diff --git a/dashboard/app/mail_bug.txt b/dashboard/app/mail_bug.txt
index c2b2e652d..8c4e46b7e 100644
--- a/dashboard/app/mail_bug.txt
+++ b/dashboard/app/mail_bug.txt
@@ -16,7 +16,10 @@ git tree:       {{.KernelRepoAlias}}
 {{end}}{{if and .Moderation .Maintainers}}CC:             {{.Maintainers}}
 {{end}}{{if and (not .NoRepro) (not .ReproCLink) (not .ReproSyzLink)}}
 Unfortunately, I don't have any reproducer for this issue yet.
-{{end}}
+{{end}}{{if .Assets}}
+Downloadable assets:
+{{range $i, $asset := .Assets}}{{$asset.Title}}: {{$asset.DownloadURL}}
+{{end}}{{end}}
 {{if .BisectCause}}{{if .BisectCause.Commit}}The issue was bisected to:
 
 commit {{.BisectCause.Commit.Hash}}
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 1232b52fc..73568da16 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -23,6 +23,7 @@ import (
 	"github.com/google/syzkaller/pkg/html"
 	"github.com/google/syzkaller/pkg/vcs"
 	"golang.org/x/net/context"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/api/iterator"
 	"google.golang.org/appengine/v2"
 	db "google.golang.org/appengine/v2/datastore"
@@ -58,6 +59,7 @@ func initHTTPHandlers() {
 		http.Handle("/"+ns+"/graph/crashes", handlerWrapper(handleGraphCrashes))
 	}
 	http.HandleFunc("/cache_update", cacheUpdate)
+	http.HandleFunc("/deprecate_assets", handleDeprecateAssets)
 }
 
 type uiMainPage struct {
@@ -1095,17 +1097,30 @@ func loadManagers(c context.Context, accessLevel AccessLevel, ns, manager string
 		}
 	}
 	builds := make([]*Build, len(buildKeys))
-	if err := db.GetMulti(c, buildKeys, builds); err != nil {
-		return nil, err
+	stats := make([]*ManagerStats, len(statsKeys))
+	coverAssets := map[string]Asset{}
+	g, _ := errgroup.WithContext(context.Background())
+	g.Go(func() error {
+		return db.GetMulti(c, buildKeys, builds)
+	})
+	g.Go(func() error {
+		return db.GetMulti(c, statsKeys, stats)
+	})
+	g.Go(func() error {
+		// Get the last coverage report asset for the last week.
+		const maxDuration = time.Hour * 24 * 7
+		var err error
+		coverAssets, err = queryLatestManagerAssets(c, ns, dashapi.HTMLCoverageReport, maxDuration)
+		return err
+	})
+	err = g.Wait()
+	if err != nil {
+		return nil, fmt.Errorf("failed to query manager-related info: %w", err)
 	}
 	uiBuilds := make(map[string]*uiBuild)
 	for _, build := range builds {
 		uiBuilds[build.Namespace+"|"+build.ID] = makeUIBuild(build)
 	}
-	stats := make([]*ManagerStats, len(statsKeys))
-	if err := db.GetMulti(c, statsKeys, stats); err != nil {
-		return nil, fmt.Errorf("fetching manager stats: %v", err)
-	}
 	var fullStats []*ManagerStats
 	for _, mgr := range managers {
 		if timeDate(mgr.LastAlive) != date {
@@ -1126,12 +1141,20 @@ func loadManagers(c context.Context, accessLevel AccessLevel, ns, manager string
 		if now.Sub(mgr.LastAlive) > 6*time.Hour {
 			uptime = 0
 		}
+		// TODO: also display how fresh the coverage report is (to display it on
+		// the main page -- this will reduce confusion).
+		coverURL := ""
+		if config.CoverPath != "" {
+			coverURL = config.CoverPath + mgr.Name + ".html"
+		} else if asset, ok := coverAssets[mgr.Name]; ok {
+			coverURL = asset.DownloadURL
+		}
 		ui := &uiManager{
 			Now:                   timeNow(c),
 			Namespace:             mgr.Namespace,
 			Name:                  mgr.Name,
 			Link:                  link,
-			CoverLink:             config.CoverPath + mgr.Name + ".html",
+			CoverLink:             coverURL,
 			CurrentBuild:          uiBuilds[mgr.Namespace+"|"+mgr.CurrentBuild],
 			FailedBuildBugLink:    bugLink(mgr.FailedBuildBug),
 			FailedSyzBuildBugLink: bugLink(mgr.FailedSyzBuildBug),
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index c9556c34c..1c48d28a2 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/asset"
 	"github.com/google/syzkaller/pkg/email"
 	"github.com/google/syzkaller/pkg/html"
 	"github.com/google/syzkaller/sys/targets"
@@ -366,6 +367,7 @@ func reproStr(level dashapi.ReproLevel) string {
 	}
 }
 
+// nolint: gocyclo
 func createBugReport(c context.Context, bug *Bug, crash *Crash, crashKey *db.Key,
 	bugReporting *BugReporting, reporting *Reporting) (*dashapi.BugReport, error) {
 	reportingConfig, err := json.Marshal(reporting.Config)
@@ -422,7 +424,22 @@ func createBugReport(c context.Context, bug *Bug, crash *Crash, crashKey *db.Key
 	if !bugReporting.Reported.IsZero() {
 		typ = dashapi.ReportRepro
 	}
-
+	assetList := []dashapi.Asset{}
+	for _, buildAsset := range build.Assets {
+		typeDescr := asset.GetTypeDescription(buildAsset.Type)
+		if typeDescr == nil || typeDescr.NoReporting {
+			continue
+		}
+		assetList = append(assetList, dashapi.Asset{
+			Title:       typeDescr.GetTitle(targets.Get(build.OS, build.Arch)),
+			DownloadURL: buildAsset.DownloadURL,
+			Type:        buildAsset.Type,
+		})
+	}
+	sort.SliceStable(assetList, func(i, j int) bool {
+		return asset.GetTypeDescription(assetList[i].Type).ReportingPrio <
+			asset.GetTypeDescription(assetList[j].Type).ReportingPrio
+	})
 	kernelRepo := kernelRepoInfo(build)
 	rep := &dashapi.BugReport{
 		Type:            typ,
@@ -448,6 +465,7 @@ func createBugReport(c context.Context, bug *Bug, crash *Crash, crashKey *db.Key
 		CrashTime:       crash.Time,
 		NumCrashes:      bug.NumCrashes,
 		HappenedOn:      managersToRepos(c, bug.Namespace, bug.HappenedOn),
+		Assets:          assetList,
 	}
 	if bugReporting.CC != "" {
 		rep.CC = append(rep.CC, strings.Split(bugReporting.CC, "|")...)
diff --git a/dashboard/app/reporting_test.go b/dashboard/app/reporting_test.go
index 5b61099cd..2c4f862f5 100644
--- a/dashboard/app/reporting_test.go
+++ b/dashboard/app/reporting_test.go
@@ -82,6 +82,7 @@ func TestReportBug(t *testing.T) {
 		CrashTime:         timeNow(c.ctx),
 		NumCrashes:        1,
 		HappenedOn:        []string{"repo1 branch1"},
+		Assets:            []dashapi.Asset{},
 	}
 	c.expectEQ(want, rep)
 
@@ -244,6 +245,7 @@ func TestInvalidBug(t *testing.T) {
 		CrashTime:         timeNow(c.ctx),
 		NumCrashes:        1,
 		HappenedOn:        []string{"repo1 branch1"},
+		Assets:            []dashapi.Asset{},
 	}
 	c.expectEQ(want, rep)
 	c.client.ReportFailedRepro(testCrashID(crash1))
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index 028448b33..cdefb8080 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -102,6 +102,7 @@ type Build struct {
 	KernelConfig        []byte
 	Commits             []string // see BuilderPoll
 	FixCommits          []Commit
+	Assets              []NewAsset
 }
 
 type Commit struct {
@@ -396,8 +397,27 @@ type BugReport struct {
 	PatchLink      string
 	BisectCause    *BisectResult
 	BisectFix      *BisectResult
+	Assets         []Asset
 }
 
+type Asset struct {
+	Title       string
+	DownloadURL string
+	Type        AssetType
+}
+
+type AssetType string
+
+// Asset types used throughout the system.
+// DO NOT change them, this will break compatibility with DB content.
+const (
+	BootableDisk       AssetType = "bootable_disk"
+	NonBootableDisk    AssetType = "non_bootable_disk"
+	KernelObject       AssetType = "kernel_object"
+	KernelImage        AssetType = "kernel_image"
+	HTMLCoverageReport AssetType = "html_coverage_report"
+)
+
 type BisectResult struct {
 	Commit          *Commit   // for conclusive bisection
 	Commits         []*Commit // for inconclusive bisection
@@ -537,6 +557,35 @@ func (dash *Dashboard) UploadManagerStats(req *ManagerStatsReq) error {
 	return dash.Query("manager_stats", req, nil)
 }
 
+// Asset lifetime:
+// 1. syz-ci uploads it to GCS and reports to the dashboard via add_build_asset.
+// 2. dashboard periodically checks if the asset is still needed.
+// 3. syz-ci queries needed_assets to figure out which assets are still needed.
+// 4. Once an asset is not needed, syz-ci removes the corresponding file.
+type NewAsset struct {
+	DownloadURL string
+	Type        AssetType
+}
+
+type AddBuildAssetsReq struct {
+	BuildID string
+	Assets  []NewAsset
+}
+
+func (dash *Dashboard) AddBuildAssets(req *AddBuildAssetsReq) error {
+	return dash.Query("add_build_assets", req, nil)
+}
+
+type NeededAssetsResp struct {
+	DownloadURLs []string
+}
+
+func (dash *Dashboard) NeededAssetsList() (*NeededAssetsResp, error) {
+	resp := new(NeededAssetsResp)
+	err := dash.Query("needed_assets", nil, resp)
+	return resp, err
+}
+
 type BugListResp struct {
 	List []string
 }