From da7439a87ada53c4aa2ce9da7c20f827d842ffe1 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 17 Jun 2024 11:51:04 +0200 Subject: dashboard/app/entities.go: rename because datastore is not the only --- dashboard/app/entities.go | 1111 ------------------------------ dashboard/app/entities_datastore.go | 1111 ++++++++++++++++++++++++++++++ dashboard/app/entities_datastore_test.go | 62 ++ dashboard/app/entities_test.go | 62 -- 4 files changed, 1173 insertions(+), 1173 deletions(-) delete mode 100644 dashboard/app/entities.go create mode 100644 dashboard/app/entities_datastore.go create mode 100644 dashboard/app/entities_datastore_test.go delete mode 100644 dashboard/app/entities_test.go diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go deleted file mode 100644 index 3d81722fc..000000000 --- a/dashboard/app/entities.go +++ /dev/null @@ -1,1111 +0,0 @@ -// Copyright 2017 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "context" - "fmt" - "regexp" - "strconv" - "strings" - "time" - - "github.com/google/syzkaller/dashboard/dashapi" - "github.com/google/syzkaller/pkg/hash" - "github.com/google/syzkaller/pkg/subsystem" - db "google.golang.org/appengine/v2/datastore" -) - -// This file contains definitions of entities stored in datastore. - -const ( - maxTextLen = 200 - MaxStringLen = 1024 - - maxBugHistoryDays = 365 * 5 -) - -type Manager struct { - Namespace string - Name string - Link string - CurrentBuild string - FailedBuildBug string - FailedSyzBuildBug string - LastAlive time.Time - CurrentUpTime time.Duration - LastGeneratedJob time.Time -} - -// ManagerStats holds per-day manager runtime stats. -// Has Manager as parent entity. Keyed by Date. -type ManagerStats struct { - Date int // YYYYMMDD - MaxCorpus int64 - MaxPCs int64 // coverage - MaxCover int64 // what we call feedback signal everywhere else - TotalFuzzingTime time.Duration - TotalCrashes int64 - CrashTypes int64 // unique crash types - SuppressedCrashes int64 - TotalExecs int64 - // These are only recorded once right after corpus is triaged. - TriagedCoverage int64 - TriagedPCs int64 -} - -type Asset struct { - Type dashapi.AssetType - DownloadURL string - CreateDate time.Time -} - -type Build struct { - Namespace string - Manager string - ID string // unique ID generated by syz-ci - Type BuildType - Time time.Time - OS string - Arch string - VMArch string - SyzkallerCommit string - SyzkallerCommitDate time.Time - CompilerID string - KernelRepo string - KernelBranch string - KernelCommit string - KernelCommitTitle string `datastore:",noindex"` - KernelCommitDate time.Time `datastore:",noindex"` - KernelConfig int64 // reference to KernelConfig text entity - Assets []Asset // build-related assets - AssetsLastCheck time.Time // the last time we checked the assets for deprecation -} - -type Bug struct { - Namespace string - Seq int64 // sequences of the bug with the same title - Title string - MergedTitles []string // crash titles that we already merged into this bug - AltTitles []string // alternative crash titles that we may merge into this bug - Status int - StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why - DupOf string - NumCrashes int64 - NumRepro int64 - // ReproLevel is the best ever found repro level for this bug. - // HeadReproLevel is best known repro level that still works on the HEAD commit. - ReproLevel dashapi.ReproLevel - HeadReproLevel dashapi.ReproLevel `datastore:"HeadReproLevel"` - BisectCause BisectStatus - BisectFix BisectStatus - HasReport bool - NeedCommitInfo bool - FirstTime time.Time - LastTime time.Time - LastSavedCrash time.Time - LastReproTime time.Time - LastCauseBisect time.Time - FixTime time.Time // when we become aware of the fixing commit - LastActivity time.Time // last time we observed any activity related to the bug - Closed time.Time - SubsystemsTime time.Time // when we have updated subsystems last time - SubsystemsRev int - Reporting []BugReporting - Commits []string // titles of fixing commmits - CommitInfo []Commit // additional info for commits (for historical reasons parallel array to Commits) - HappenedOn []string // list of managers - PatchedOn []string `datastore:",noindex"` // list of managers - UNCC []string // don't CC these emails on this bug - // Kcidb publishing status bitmask: - // bit 0 - the bug is published - // bit 1 - don't want to publish it (syzkaller build/test errors) - KcidbStatus int64 - DailyStats []BugDailyStats - Labels []BugLabel - DiscussionInfo []BugDiscussionInfo - TreeTests BugTreeTestInfo - // FixCandidateJob holds the key of the latest successful cross-tree fix bisection job. - FixCandidateJob string - ReproAttempts []BugReproAttempt -} - -type BugTreeTestInfo struct { - // NeedPoll is set to true if this bug needs to be considered ASAP. - NeedPoll bool - // NextPoll can be used to delay the next inspection of the bug. - NextPoll time.Time - // List contains latest data about cross-tree patch tests. - List []BugTreeTest -} - -type BugTreeTest struct { - CrashID int64 - Repo string - Branch string // May be also equal to a commit. - // If the values below are set, the testing was done on a merge base. - MergeBaseRepo string - MergeBaseBranch string - // Below are job keys. - First string // The first job that finished successfully. - FirstOK string - FirstCrash string - Last string - Error string // If some job succeeds afterwards, it should be cleared. - Pending string -} - -type BugLabelType string - -type BugLabel struct { - Label BugLabelType - // Either empty (for flags) or contains the value. - Value string - // The email of the user who manually set this subsystem tag. - // If empty, the label was set automatically. - SetBy string - // Link to the message. - Link string -} - -func (label BugLabel) String() string { - if label.Value == "" { - return string(label.Label) - } - return string(label.Label) + ":" + label.Value -} - -// BugReproAttempt describes a single attempt to generate a repro for a bug. -type BugReproAttempt struct { - Time time.Time - Manager string - Log int64 -} - -func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem, now time.Time, rev int) { - bug.SubsystemsRev = rev - bug.SubsystemsTime = now - var objects []BugLabel - for _, item := range list { - objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name}) - } - bug.SetLabels(makeLabelSet(c, bug.Namespace), objects) -} - -func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error { - tx := func(c context.Context) error { - bug := new(Bug) - if err := db.Get(c, bugKey, bug); err != nil { - return fmt.Errorf("failed to get bug: %w", err) - } - err := transform(bug) - if err != nil { - return err - } - if _, err := db.Put(c, bugKey, bug); err != nil { - return fmt.Errorf("failed to put bug: %w", err) - } - return nil - } - return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) -} - -func (bug *Bug) hasUserSubsystems() bool { - return bug.HasUserLabel(SubsystemLabel) -} - -// Initially, subsystem labels were stored as Tags.Subsystems, but over time -// it turned out that we'd better store all labels together. -// Let's keep this conversion code until "Tags" are removed from all bugs. -// Then it can be removed. - -type Bug202304 struct { - Tags BugTags202304 -} - -type BugTags202304 struct { - Subsystems []BugTag202304 -} - -type BugTag202304 struct { - Name string - SetBy string -} - -func (bug *Bug) Load(origProps []db.Property) error { - // First filer out Tag properties. - var tags, ps []db.Property - for _, p := range origProps { - if strings.HasPrefix(p.Name, "Tags.") { - tags = append(tags, p) - } else { - ps = append(ps, p) - } - } - if err := db.LoadStruct(bug, ps); err != nil { - return err - } - if len(tags) > 0 { - old := Bug202304{} - if err := db.LoadStruct(&old, tags); err != nil { - return err - } - for _, entry := range old.Tags.Subsystems { - bug.Labels = append(bug.Labels, BugLabel{ - Label: SubsystemLabel, - SetBy: entry.SetBy, - Value: entry.Name, - }) - } - } - headReproFound := false - for _, p := range ps { - if p.Name == "HeadReproLevel" { - headReproFound = true - break - } - } - if !headReproFound { - // The field is new, so it won't be set in all entities. - // Assume it to be equal to the best found repro for the bug. - bug.HeadReproLevel = bug.ReproLevel - } - return nil -} - -func (bug *Bug) Save() ([]db.Property, error) { - return db.SaveStruct(bug) -} - -type BugDailyStats struct { - Date int // YYYYMMDD - CrashCount int -} - -type Commit struct { - Hash string - Title string - Author string - AuthorName string - CC string `datastore:",noindex"` // (|-delimited list) - Date time.Time -} - -func (com Commit) toDashapi() *dashapi.Commit { - return &dashapi.Commit{ - Hash: com.Hash, - Title: com.Title, - Author: com.Author, - AuthorName: com.AuthorName, - Date: com.Date, - } -} - -type BugDiscussionInfo struct { - Source string - Summary DiscussionSummary -} - -type DiscussionSummary struct { - AllMessages int - ExternalMessages int - LastMessage time.Time - LastPatchMessage time.Time -} - -type BugReporting struct { - Name string // refers to Reporting.Name - ID string // unique ID per BUG/BugReporting used in commucation with external systems - ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport - Link string - CC string // additional emails added to CC list (|-delimited list) - CrashID int64 // crash that we've last reported in this reporting - Auto bool // was it auto-upstreamed/obsoleted? - // If Dummy is true, the corresponding Reporting stage was introduced later and the object was just - // inserted to preserve consistency across the system. Even though it's indicated as Closed and Reported, - // it never actually was. - Dummy bool - ReproLevel dashapi.ReproLevel // may be less then bug.ReproLevel if repro arrived but we didn't report it yet - Labels string // a comma-separated string of already reported labels - OnHold time.Time // if set, the bug must not be upstreamed - Reported time.Time - Closed time.Time -} - -func (r *BugReporting) GetLabels() []string { - return strings.Split(r.Labels, ",") -} - -func (r *BugReporting) AddLabel(label string) { - newList := unique(append(r.GetLabels(), label)) - r.Labels = strings.Join(newList, ",") -} - -type Crash struct { - // May be different from bug.Title due to AltTitles. - // May be empty for old bugs, in such case bug.Title is the right title. - Title string - Manager string - BuildID string - Time time.Time - Reported time.Time // set if this crash was ever reported - References []CrashReference - Maintainers []string `datastore:",noindex"` - Log int64 // reference to CrashLog text entity - Flags int64 // properties of the Crash - Report int64 // reference to CrashReport text entity - ReportElements CrashReportElements // parsed parts of the crash report - ReproOpts []byte `datastore:",noindex"` - ReproSyz int64 // reference to ReproSyz text entity - ReproC int64 // reference to ReproC text entity - ReproIsRevoked bool // the repro no longer triggers the bug on HEAD - ReproLog int64 // reference to ReproLog text entity - LastReproRetest time.Time // the last time when the repro was re-checked - MachineInfo int64 // Reference to MachineInfo text entity. - // Custom crash priority for reporting (greater values are higher priority). - // For example, a crash in mainline kernel has higher priority than a crash in a side branch. - // For historical reasons this is called ReportLen. - ReportLen int64 - Assets []Asset // crash-related assets - AssetsLastCheck time.Time // the last time we checked the assets for deprecation -} - -type CrashReportElements struct { - GuiltyFiles []string // guilty files as determined during the crash report parsing -} - -type CrashReferenceType string - -const ( - CrashReferenceReporting = "reporting" - CrashReferenceJob = "job" - // This one is needed for backward compatibility. - crashReferenceUnknown = "unknown" -) - -type CrashReference struct { - Type CrashReferenceType - // For CrashReferenceReporting, it refers to Reporting.Name - // For CrashReferenceJob, it refers to extJobID(jobKey) - Key string - Time time.Time -} - -func (crash *Crash) AddReference(newRef CrashReference) { - crash.Reported = newRef.Time - for i, ref := range crash.References { - if ref.Type != newRef.Type || ref.Key != newRef.Key { - continue - } - crash.References[i].Time = newRef.Time - return - } - crash.References = append(crash.References, newRef) -} - -func (crash *Crash) ClearReference(t CrashReferenceType, key string) { - newRefs := []CrashReference{} - crash.Reported = time.Time{} - for _, ref := range crash.References { - if ref.Type == t && ref.Key == key { - continue - } - if ref.Time.After(crash.Reported) { - crash.Reported = ref.Time - } - newRefs = append(newRefs, ref) - } - crash.References = newRefs -} - -func (crash *Crash) Load(ps []db.Property) error { - if err := db.LoadStruct(crash, ps); err != nil { - return err - } - // Earlier we only relied on Reported, which does not let us reliably unreport a crash. - // We need some means of ref counting, so let's create a dummy reference to keep the - // crash from being purged. - if !crash.Reported.IsZero() && len(crash.References) == 0 { - crash.References = append(crash.References, CrashReference{ - Type: crashReferenceUnknown, - Time: crash.Reported, - }) - } - return nil -} - -func (crash *Crash) Save() ([]db.Property, error) { - return db.SaveStruct(crash) -} - -type Discussion struct { - ID string // the base message ID - Source string - Type string - Subject string - BugKeys []string - // Message contains last N messages. - // N is supposed to be big enough, so that in almost all cases - // AllMessages == len(Messages) holds true. - Messages []DiscussionMessage - // Since Messages could be trimmed, we have to keep aggregate stats. - Summary DiscussionSummary -} - -func discussionKey(c context.Context, source, id string) *db.Key { - return db.NewKey(c, "Discussion", fmt.Sprintf("%v-%v", source, id), 0, nil) -} - -func (d *Discussion) key(c context.Context) *db.Key { - return discussionKey(c, d.Source, d.ID) -} - -type DiscussionMessage struct { - ID string - // External is true if the message is not from the bot itself. - // Let's use a shorter name to save space. - External bool `datastore:"e"` - Time time.Time `datastore:",noindex"` -} - -// ReportingState holds dynamic info associated with reporting. -type ReportingState struct { - Entries []ReportingStateEntry -} - -type ReportingStateEntry struct { - Namespace string - Name string - // Current reporting quota consumption. - Sent int - Date int // YYYYMMDD -} - -// Subsystem holds the history of grouped per-subsystem open bug reminders. -type Subsystem struct { - Namespace string - Name string - // ListsQueried is the last time bug lists were queried for the subsystem. - ListsQueried time.Time - // LastBugList is the last time we have actually managed to generate a bug list. - LastBugList time.Time -} - -// SubsystemReport holds a single report about open bugs in a subsystem. -// There'll be one record for moderation (if it's needed) and one for actual reporting. -type SubsystemReport struct { - Created time.Time - BugKeys []string `datastore:",noindex"` - TotalStats SubsystemReportStats - PeriodStats SubsystemReportStats - Stages []SubsystemReportStage -} - -func (r *SubsystemReport) getBugKeys() ([]*db.Key, error) { - ret := []*db.Key{} - for _, encoded := range r.BugKeys { - key, err := db.DecodeKey(encoded) - if err != nil { - return nil, fmt.Errorf("failed to parse %#v: %w", encoded, err) - } - ret = append(ret, key) - } - return ret, nil -} - -func (r *SubsystemReport) findStage(id string) *SubsystemReportStage { - for j := range r.Stages { - stage := &r.Stages[j] - if stage.ID == id { - return stage - } - } - return nil -} - -type SubsystemReportStats struct { - Reported int - LowPrio int - Fixed int -} - -func (s *SubsystemReportStats) toDashapi() dashapi.BugListReportStats { - return dashapi.BugListReportStats{ - Reported: s.Reported, - LowPrio: s.LowPrio, - Fixed: s.Fixed, - } -} - -// There can be at most two stages. -// One has Moderation=true, the other one has Moderation=false. -type SubsystemReportStage struct { - ID string - ExtID string - Link string - Reported time.Time - Closed time.Time - Moderation bool -} - -// Job represent a single patch testing or bisection job for syz-ci. -// Later we may want to extend this to other types of jobs: -// - test of a committed fix -// - reproduce crash -// - test that crash still happens on HEAD -// -// Job has Bug as parent entity. -type Job struct { - Type JobType - Created time.Time - User string - CC []string - Reporting string - ExtID string // email Message-ID - Link string // web link for the job (e.g. email in the group) - Namespace string - Manager string - BugTitle string - CrashID int64 - - // Provided by user: - KernelRepo string - KernelBranch string - Patch int64 // reference to Patch text entity - KernelConfig int64 // reference to the kernel config entity - - Attempts int // number of times we tried to execute this job - IsRunning bool // the job might have been started, but never finished - LastStarted time.Time `datastore:"Started"` - Finished time.Time // if set, job is finished - TreeOrigin bool // whether the job is related to tree origin detection - - // If patch test should be done on the merge base between two branches. - MergeBaseRepo string - MergeBaseBranch string - - // By default, bisection starts from the revision of the associated crash. - // The BisectFrom field can override this. - BisectFrom string - - // Result of execution: - CrashTitle string // if empty, we did not hit crash during testing - CrashLog int64 // reference to CrashLog text entity - CrashReport int64 // reference to CrashReport text entity - Commits []Commit - BuildID string - Log int64 // reference to Log text entity - Error int64 // reference to Error text entity, if set job failed - Flags dashapi.JobDoneFlags - - Reported bool // have we reported result back to user? - InvalidatedBy string // user who marked this bug as invalid, empty by default - BackportedCommit Commit -} - -func (job *Job) IsBisection() bool { - return job.Type == JobBisectCause || job.Type == JobBisectFix -} - -func (job *Job) IsFinished() bool { - return !job.Finished.IsZero() -} - -type JobType int - -const ( - JobTestPatch JobType = iota - JobBisectCause - JobBisectFix -) - -func (typ JobType) toDashapiReportType() dashapi.ReportType { - switch typ { - case JobTestPatch: - return dashapi.ReportTestPatch - case JobBisectCause: - return dashapi.ReportBisectCause - case JobBisectFix: - return dashapi.ReportBisectFix - default: - panic(fmt.Sprintf("unknown job type %v", typ)) - } -} - -func (job *Job) isUnreliableBisect() bool { - if job.Type != JobBisectCause && job.Type != JobBisectFix { - panic(fmt.Sprintf("bad job type %v", job.Type)) - } - // If a bisection points to a merge or a commit that does not affect the kernel binary, - // it is considered an unreliable/wrong result and should not be reported in emails. - return job.Flags&dashapi.BisectResultMerge != 0 || - job.Flags&dashapi.BisectResultNoop != 0 || - job.Flags&dashapi.BisectResultRelease != 0 || - job.Flags&dashapi.BisectResultIgnore != 0 -} - -func (job *Job) IsCrossTree() bool { - return job.MergeBaseRepo != "" && job.IsBisection() -} - -// Text holds text blobs (crash logs, reports, reproducers, etc). -type Text struct { - Namespace string - Text []byte `datastore:",noindex"` // gzip-compressed text -} - -const ( - textCrashLog = "CrashLog" - textCrashReport = "CrashReport" - textReproSyz = "ReproSyz" - textReproC = "ReproC" - textMachineInfo = "MachineInfo" - textKernelConfig = "KernelConfig" - textPatch = "Patch" - textLog = "Log" - textError = "Error" - textReproLog = "ReproLog" -) - -const ( - BugStatusOpen = iota -) - -const ( - BugStatusFixed = 1000 + iota - BugStatusInvalid - BugStatusDup -) - -const ( - ReproLevelNone = dashapi.ReproLevelNone - ReproLevelSyz = dashapi.ReproLevelSyz - ReproLevelC = dashapi.ReproLevelC -) - -type BuildType int - -const ( - BuildNormal BuildType = iota - BuildFailed - BuildJob -) - -type BisectStatus int - -const ( - BisectNot BisectStatus = iota - BisectPending - BisectError - BisectYes // have 1 commit - BisectUnreliable // have 1 commit, but suspect it's wrong - BisectInconclusive // multiple commits due to skips - BisectHorizont // happens on the oldest commit we can test (or HEAD for fix bisection) - bisectStatusLast // this value can be changed (not stored in datastore) -) - -func (status BisectStatus) String() string { - switch status { - case BisectError: - return "error" - case BisectYes: - return "done" - case BisectUnreliable: - return "unreliable" - case BisectInconclusive: - return "inconclusive" - case BisectHorizont: - return "inconclusive" - default: - return "" - } -} - -func mgrKey(c context.Context, ns, name string) *db.Key { - return db.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil) -} - -func (mgr *Manager) key(c context.Context) *db.Key { - return mgrKey(c, mgr.Namespace, mgr.Name) -} - -func loadManager(c context.Context, ns, name string) (*Manager, error) { - mgr := new(Manager) - if err := db.Get(c, mgrKey(c, ns, name), mgr); err != nil { - if err != db.ErrNoSuchEntity { - return nil, fmt.Errorf("failed to get manager %v/%v: %w", ns, name, err) - } - mgr = &Manager{ - Namespace: ns, - Name: name, - } - } - return mgr, nil -} - -// updateManager does transactional compare-and-swap on the manager and its current stats. -func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats) error) error { - date := timeDate(timeNow(c)) - tx := func(c context.Context) error { - mgr, err := loadManager(c, ns, name) - if err != nil { - return err - } - mgrKey := mgr.key(c) - stats := new(ManagerStats) - statsKey := db.NewKey(c, "ManagerStats", "", int64(date), mgrKey) - if err := db.Get(c, statsKey, stats); err != nil { - if err != db.ErrNoSuchEntity { - return fmt.Errorf("failed to get stats %v/%v/%v: %w", ns, name, date, err) - } - stats = &ManagerStats{ - Date: date, - } - } - - if err := fn(mgr, stats); err != nil { - return err - } - - if _, err := db.Put(c, mgrKey, mgr); err != nil { - return fmt.Errorf("failed to put manager: %w", err) - } - if _, err := db.Put(c, statsKey, stats); err != nil { - return fmt.Errorf("failed to put manager stats: %w", err) - } - return nil - } - return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) -} - -func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) { - var managers []*Manager - query := db.NewQuery("Manager") - if ns != "" { - query = query.Filter("Namespace=", ns) - } - keys, err := query.GetAll(c, &managers) - if err != nil { - return nil, nil, fmt.Errorf("failed to query managers: %w", err) - } - var result []*Manager - var resultKeys []*db.Key - for i, mgr := range managers { - if getNsConfig(c, mgr.Namespace).Managers[mgr.Name].Decommissioned { - continue - } - result = append(result, mgr) - resultKeys = append(resultKeys, keys[i]) - } - return result, resultKeys, nil -} - -func buildKey(c context.Context, ns, id string) *db.Key { - if ns == "" { - panic("requesting build key outside of namespace") - } - h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id))) - return db.NewKey(c, "Build", h, 0, nil) -} - -func loadBuild(c context.Context, ns, id string) (*Build, error) { - build := new(Build) - if err := db.Get(c, buildKey(c, ns, id), build); err != nil { - if err == db.ErrNoSuchEntity { - return nil, fmt.Errorf("unknown build %v/%v", ns, id) - } - return nil, fmt.Errorf("failed to get build %v/%v: %w", ns, id, err) - } - return build, nil -} - -func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) { - mgr, err := loadManager(c, ns, manager) - if err != nil { - return nil, err - } - if mgr.CurrentBuild == "" { - return nil, fmt.Errorf("failed to fetch manager build: no builds") - } - return loadBuild(c, ns, mgr.CurrentBuild) -} - -func loadBuilds(c context.Context, ns, manager string, typ BuildType) ([]*Build, error) { - const limit = 500 - var builds []*Build - _, err := db.NewQuery("Build"). - Filter("Namespace=", ns). - Filter("Manager=", manager). - Filter("Type=", typ). - Order("-Time"). - Limit(limit). - GetAll(c, &builds) - if err != nil { - return nil, err - } - return builds, nil -} - -func (bug *Bug) displayTitle() string { - if bug.Seq == 0 { - return bug.Title - } - return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1) -} - -var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`) - -func splitDisplayTitle(display string) (string, int64, error) { - match := displayTitleRe.FindStringSubmatchIndex(display) - if match == nil { - return display, 0, nil - } - title := display[match[2]:match[3]] - seqStr := display[match[4]:match[5]] - seq, err := strconv.ParseInt(seqStr, 10, 64) - if err != nil { - return "", 0, fmt.Errorf("failed to parse bug title: %w", err) - } - if seq <= 0 || seq > 1e6 { - return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq) - } - return title, seq - 1, nil -} - -func canonicalBug(c context.Context, bug *Bug) (*Bug, error) { - for { - if bug.Status != BugStatusDup { - return bug, nil - } - canon := new(Bug) - bugKey := db.NewKey(c, "Bug", bug.DupOf, 0, nil) - if err := db.Get(c, bugKey, canon); err != nil { - return nil, fmt.Errorf("failed to get dup bug %q for %q: %w", - bug.DupOf, bug.keyHash(c), err) - } - bug = canon - } -} - -func (bug *Bug) key(c context.Context) *db.Key { - return db.NewKey(c, "Bug", bug.keyHash(c), 0, nil) -} - -func (bug *Bug) keyHash(c context.Context) string { - return bugKeyHash(c, bug.Namespace, bug.Title, bug.Seq) -} - -func bugKeyHash(c context.Context, ns, title string, seq int64) string { - return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq))) -} - -func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) { - domain := getNsConfig(c, bug.Namespace).SimilarityDomain - dedup := make(map[string]bool) - dedup[bug.keyHash(c)] = true - - ret := []*Bug{} - for _, title := range bug.AltTitles { - var similar []*Bug - _, err := db.NewQuery("Bug"). - Filter("AltTitles=", title). - GetAll(c, &similar) - if err != nil { - return nil, err - } - for _, bug := range similar { - if getNsConfig(c, bug.Namespace).SimilarityDomain != domain || - dedup[bug.keyHash(c)] { - continue - } - dedup[bug.keyHash(c)] = true - ret = append(ret, bug) - } - } - return ret, nil -} - -// Since these IDs appear in Reported-by tags in commit, we slightly limit their size. -const reportingHashLen = 20 - -func bugReportingHash(bugHash, reporting string) string { - return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:reportingHashLen] -} - -func looksLikeReportingHash(id string) bool { - // This is only used as best-effort check. - // Now we produce 20-chars ids, but we used to use full sha1 hash. - return len(id) == reportingHashLen || len(id) == 2*len(hash.Sig{}) -} - -func (bug *Bug) updateCommits(commits []string, now time.Time) { - bug.Commits = commits - bug.CommitInfo = nil - bug.NeedCommitInfo = true - bug.FixTime = now - bug.PatchedOn = nil -} - -func (bug *Bug) getCommitInfo(i int) Commit { - if i < len(bug.CommitInfo) { - return bug.CommitInfo[i] - } - return Commit{} -} - -func (bug *Bug) increaseCrashStats(now time.Time) { - bug.NumCrashes++ - date := timeDate(now) - if len(bug.DailyStats) == 0 || bug.DailyStats[len(bug.DailyStats)-1].Date < date { - bug.DailyStats = append(bug.DailyStats, BugDailyStats{date, 1}) - } else { - // It is theoretically possible that this method might get into a situation, when - // the latest saved date is later than now. But we assume that this can only happen - // in a small window around the start of the day and it is better to attribute a - // crash to the next day than to get a mismatch between NumCrashes and the sum of - // CrashCount. - bug.DailyStats[len(bug.DailyStats)-1].CrashCount++ - } - - if len(bug.DailyStats) > maxBugHistoryDays { - bug.DailyStats = bug.DailyStats[len(bug.DailyStats)-maxBugHistoryDays:] - } -} - -func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats { - startDate := timeDate(from) - startPos := len(bug.DailyStats) - for ; startPos > 0; startPos-- { - if bug.DailyStats[startPos-1].Date < startDate { - break - } - } - return bug.DailyStats[startPos:] -} - -func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) { - var status dashapi.BugStatus - switch bug.Status { - case BugStatusOpen: - status = dashapi.BugStatusOpen - case BugStatusFixed: - status = dashapi.BugStatusFixed - case BugStatusInvalid: - status = dashapi.BugStatusInvalid - case BugStatusDup: - status = dashapi.BugStatusDup - default: - return status, fmt.Errorf("unknown bugs status %v", bug.Status) - } - return status, nil -} - -// If an entity of type EmergencyStop exists, syzbot's operation is paused until -// a support engineer deletes it from the DB. -type EmergencyStop struct { - Time time.Time - User string -} - -func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error { - crash := new(Crash) - crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) - if err := db.Get(c, crashKey, crash); err != nil { - return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) - } - crash.AddReference(ref) - if _, err := db.Put(c, crashKey, crash); err != nil { - return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) - } - return nil -} - -func removeCrashReference(c context.Context, crashID int64, bugKey *db.Key, - t CrashReferenceType, key string) error { - crash := new(Crash) - crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) - if err := db.Get(c, crashKey, crash); err != nil { - return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) - } - crash.ClearReference(t, key) - if _, err := db.Put(c, crashKey, crash); err != nil { - return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) - } - return nil -} - -func kernelRepoInfo(c context.Context, build *Build) KernelRepo { - return kernelRepoInfoRaw(c, build.Namespace, build.KernelRepo, build.KernelBranch) -} - -func kernelRepoInfoRaw(c context.Context, ns, url, branch string) KernelRepo { - var info KernelRepo - for _, repo := range getNsConfig(c, ns).Repos { - if repo.URL == url && repo.Branch == branch { - info = repo - break - } - } - if info.Alias == "" { - info.Alias = url - if branch != "" { - info.Alias += " " + branch - } - } - return info -} - -func textLink(tag string, id int64) string { - if id == 0 { - return "" - } - return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16)) -} - -// timeDate returns t's date as a single int YYYYMMDD. -func timeDate(t time.Time) int { - year, month, day := t.Date() - return year*10000 + int(month)*100 + day -} - -func stringInList(list []string, str string) bool { - for _, s := range list { - if s == str { - return true - } - } - return false -} - -func stringListsIntersect(a, b []string) bool { - m := map[string]bool{} - for _, strA := range a { - m[strA] = true - } - for _, strB := range b { - if m[strB] { - return true - } - } - return false -} - -func mergeString(list []string, str string) []string { - if !stringInList(list, str) { - list = append(list, str) - } - return list -} - -func mergeStringList(list, add []string) []string { - for _, str := range add { - list = mergeString(list, str) - } - return list -} - -// dateTime converts date in YYYYMMDD format back to Time. -func dateTime(date int) time.Time { - return time.Date(date/10000, time.Month(date/100%100), date%100, 0, 0, 0, 0, time.UTC) -} diff --git a/dashboard/app/entities_datastore.go b/dashboard/app/entities_datastore.go new file mode 100644 index 000000000..3d81722fc --- /dev/null +++ b/dashboard/app/entities_datastore.go @@ -0,0 +1,1111 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/google/syzkaller/dashboard/dashapi" + "github.com/google/syzkaller/pkg/hash" + "github.com/google/syzkaller/pkg/subsystem" + db "google.golang.org/appengine/v2/datastore" +) + +// This file contains definitions of entities stored in datastore. + +const ( + maxTextLen = 200 + MaxStringLen = 1024 + + maxBugHistoryDays = 365 * 5 +) + +type Manager struct { + Namespace string + Name string + Link string + CurrentBuild string + FailedBuildBug string + FailedSyzBuildBug string + LastAlive time.Time + CurrentUpTime time.Duration + LastGeneratedJob time.Time +} + +// ManagerStats holds per-day manager runtime stats. +// Has Manager as parent entity. Keyed by Date. +type ManagerStats struct { + Date int // YYYYMMDD + MaxCorpus int64 + MaxPCs int64 // coverage + MaxCover int64 // what we call feedback signal everywhere else + TotalFuzzingTime time.Duration + TotalCrashes int64 + CrashTypes int64 // unique crash types + SuppressedCrashes int64 + TotalExecs int64 + // These are only recorded once right after corpus is triaged. + TriagedCoverage int64 + TriagedPCs int64 +} + +type Asset struct { + Type dashapi.AssetType + DownloadURL string + CreateDate time.Time +} + +type Build struct { + Namespace string + Manager string + ID string // unique ID generated by syz-ci + Type BuildType + Time time.Time + OS string + Arch string + VMArch string + SyzkallerCommit string + SyzkallerCommitDate time.Time + CompilerID string + KernelRepo string + KernelBranch string + KernelCommit string + KernelCommitTitle string `datastore:",noindex"` + KernelCommitDate time.Time `datastore:",noindex"` + KernelConfig int64 // reference to KernelConfig text entity + Assets []Asset // build-related assets + AssetsLastCheck time.Time // the last time we checked the assets for deprecation +} + +type Bug struct { + Namespace string + Seq int64 // sequences of the bug with the same title + Title string + MergedTitles []string // crash titles that we already merged into this bug + AltTitles []string // alternative crash titles that we may merge into this bug + Status int + StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why + DupOf string + NumCrashes int64 + NumRepro int64 + // ReproLevel is the best ever found repro level for this bug. + // HeadReproLevel is best known repro level that still works on the HEAD commit. + ReproLevel dashapi.ReproLevel + HeadReproLevel dashapi.ReproLevel `datastore:"HeadReproLevel"` + BisectCause BisectStatus + BisectFix BisectStatus + HasReport bool + NeedCommitInfo bool + FirstTime time.Time + LastTime time.Time + LastSavedCrash time.Time + LastReproTime time.Time + LastCauseBisect time.Time + FixTime time.Time // when we become aware of the fixing commit + LastActivity time.Time // last time we observed any activity related to the bug + Closed time.Time + SubsystemsTime time.Time // when we have updated subsystems last time + SubsystemsRev int + Reporting []BugReporting + Commits []string // titles of fixing commmits + CommitInfo []Commit // additional info for commits (for historical reasons parallel array to Commits) + HappenedOn []string // list of managers + PatchedOn []string `datastore:",noindex"` // list of managers + UNCC []string // don't CC these emails on this bug + // Kcidb publishing status bitmask: + // bit 0 - the bug is published + // bit 1 - don't want to publish it (syzkaller build/test errors) + KcidbStatus int64 + DailyStats []BugDailyStats + Labels []BugLabel + DiscussionInfo []BugDiscussionInfo + TreeTests BugTreeTestInfo + // FixCandidateJob holds the key of the latest successful cross-tree fix bisection job. + FixCandidateJob string + ReproAttempts []BugReproAttempt +} + +type BugTreeTestInfo struct { + // NeedPoll is set to true if this bug needs to be considered ASAP. + NeedPoll bool + // NextPoll can be used to delay the next inspection of the bug. + NextPoll time.Time + // List contains latest data about cross-tree patch tests. + List []BugTreeTest +} + +type BugTreeTest struct { + CrashID int64 + Repo string + Branch string // May be also equal to a commit. + // If the values below are set, the testing was done on a merge base. + MergeBaseRepo string + MergeBaseBranch string + // Below are job keys. + First string // The first job that finished successfully. + FirstOK string + FirstCrash string + Last string + Error string // If some job succeeds afterwards, it should be cleared. + Pending string +} + +type BugLabelType string + +type BugLabel struct { + Label BugLabelType + // Either empty (for flags) or contains the value. + Value string + // The email of the user who manually set this subsystem tag. + // If empty, the label was set automatically. + SetBy string + // Link to the message. + Link string +} + +func (label BugLabel) String() string { + if label.Value == "" { + return string(label.Label) + } + return string(label.Label) + ":" + label.Value +} + +// BugReproAttempt describes a single attempt to generate a repro for a bug. +type BugReproAttempt struct { + Time time.Time + Manager string + Log int64 +} + +func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem, now time.Time, rev int) { + bug.SubsystemsRev = rev + bug.SubsystemsTime = now + var objects []BugLabel + for _, item := range list { + objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name}) + } + bug.SetLabels(makeLabelSet(c, bug.Namespace), objects) +} + +func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error { + tx := func(c context.Context) error { + bug := new(Bug) + if err := db.Get(c, bugKey, bug); err != nil { + return fmt.Errorf("failed to get bug: %w", err) + } + err := transform(bug) + if err != nil { + return err + } + if _, err := db.Put(c, bugKey, bug); err != nil { + return fmt.Errorf("failed to put bug: %w", err) + } + return nil + } + return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) +} + +func (bug *Bug) hasUserSubsystems() bool { + return bug.HasUserLabel(SubsystemLabel) +} + +// Initially, subsystem labels were stored as Tags.Subsystems, but over time +// it turned out that we'd better store all labels together. +// Let's keep this conversion code until "Tags" are removed from all bugs. +// Then it can be removed. + +type Bug202304 struct { + Tags BugTags202304 +} + +type BugTags202304 struct { + Subsystems []BugTag202304 +} + +type BugTag202304 struct { + Name string + SetBy string +} + +func (bug *Bug) Load(origProps []db.Property) error { + // First filer out Tag properties. + var tags, ps []db.Property + for _, p := range origProps { + if strings.HasPrefix(p.Name, "Tags.") { + tags = append(tags, p) + } else { + ps = append(ps, p) + } + } + if err := db.LoadStruct(bug, ps); err != nil { + return err + } + if len(tags) > 0 { + old := Bug202304{} + if err := db.LoadStruct(&old, tags); err != nil { + return err + } + for _, entry := range old.Tags.Subsystems { + bug.Labels = append(bug.Labels, BugLabel{ + Label: SubsystemLabel, + SetBy: entry.SetBy, + Value: entry.Name, + }) + } + } + headReproFound := false + for _, p := range ps { + if p.Name == "HeadReproLevel" { + headReproFound = true + break + } + } + if !headReproFound { + // The field is new, so it won't be set in all entities. + // Assume it to be equal to the best found repro for the bug. + bug.HeadReproLevel = bug.ReproLevel + } + return nil +} + +func (bug *Bug) Save() ([]db.Property, error) { + return db.SaveStruct(bug) +} + +type BugDailyStats struct { + Date int // YYYYMMDD + CrashCount int +} + +type Commit struct { + Hash string + Title string + Author string + AuthorName string + CC string `datastore:",noindex"` // (|-delimited list) + Date time.Time +} + +func (com Commit) toDashapi() *dashapi.Commit { + return &dashapi.Commit{ + Hash: com.Hash, + Title: com.Title, + Author: com.Author, + AuthorName: com.AuthorName, + Date: com.Date, + } +} + +type BugDiscussionInfo struct { + Source string + Summary DiscussionSummary +} + +type DiscussionSummary struct { + AllMessages int + ExternalMessages int + LastMessage time.Time + LastPatchMessage time.Time +} + +type BugReporting struct { + Name string // refers to Reporting.Name + ID string // unique ID per BUG/BugReporting used in commucation with external systems + ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport + Link string + CC string // additional emails added to CC list (|-delimited list) + CrashID int64 // crash that we've last reported in this reporting + Auto bool // was it auto-upstreamed/obsoleted? + // If Dummy is true, the corresponding Reporting stage was introduced later and the object was just + // inserted to preserve consistency across the system. Even though it's indicated as Closed and Reported, + // it never actually was. + Dummy bool + ReproLevel dashapi.ReproLevel // may be less then bug.ReproLevel if repro arrived but we didn't report it yet + Labels string // a comma-separated string of already reported labels + OnHold time.Time // if set, the bug must not be upstreamed + Reported time.Time + Closed time.Time +} + +func (r *BugReporting) GetLabels() []string { + return strings.Split(r.Labels, ",") +} + +func (r *BugReporting) AddLabel(label string) { + newList := unique(append(r.GetLabels(), label)) + r.Labels = strings.Join(newList, ",") +} + +type Crash struct { + // May be different from bug.Title due to AltTitles. + // May be empty for old bugs, in such case bug.Title is the right title. + Title string + Manager string + BuildID string + Time time.Time + Reported time.Time // set if this crash was ever reported + References []CrashReference + Maintainers []string `datastore:",noindex"` + Log int64 // reference to CrashLog text entity + Flags int64 // properties of the Crash + Report int64 // reference to CrashReport text entity + ReportElements CrashReportElements // parsed parts of the crash report + ReproOpts []byte `datastore:",noindex"` + ReproSyz int64 // reference to ReproSyz text entity + ReproC int64 // reference to ReproC text entity + ReproIsRevoked bool // the repro no longer triggers the bug on HEAD + ReproLog int64 // reference to ReproLog text entity + LastReproRetest time.Time // the last time when the repro was re-checked + MachineInfo int64 // Reference to MachineInfo text entity. + // Custom crash priority for reporting (greater values are higher priority). + // For example, a crash in mainline kernel has higher priority than a crash in a side branch. + // For historical reasons this is called ReportLen. + ReportLen int64 + Assets []Asset // crash-related assets + AssetsLastCheck time.Time // the last time we checked the assets for deprecation +} + +type CrashReportElements struct { + GuiltyFiles []string // guilty files as determined during the crash report parsing +} + +type CrashReferenceType string + +const ( + CrashReferenceReporting = "reporting" + CrashReferenceJob = "job" + // This one is needed for backward compatibility. + crashReferenceUnknown = "unknown" +) + +type CrashReference struct { + Type CrashReferenceType + // For CrashReferenceReporting, it refers to Reporting.Name + // For CrashReferenceJob, it refers to extJobID(jobKey) + Key string + Time time.Time +} + +func (crash *Crash) AddReference(newRef CrashReference) { + crash.Reported = newRef.Time + for i, ref := range crash.References { + if ref.Type != newRef.Type || ref.Key != newRef.Key { + continue + } + crash.References[i].Time = newRef.Time + return + } + crash.References = append(crash.References, newRef) +} + +func (crash *Crash) ClearReference(t CrashReferenceType, key string) { + newRefs := []CrashReference{} + crash.Reported = time.Time{} + for _, ref := range crash.References { + if ref.Type == t && ref.Key == key { + continue + } + if ref.Time.After(crash.Reported) { + crash.Reported = ref.Time + } + newRefs = append(newRefs, ref) + } + crash.References = newRefs +} + +func (crash *Crash) Load(ps []db.Property) error { + if err := db.LoadStruct(crash, ps); err != nil { + return err + } + // Earlier we only relied on Reported, which does not let us reliably unreport a crash. + // We need some means of ref counting, so let's create a dummy reference to keep the + // crash from being purged. + if !crash.Reported.IsZero() && len(crash.References) == 0 { + crash.References = append(crash.References, CrashReference{ + Type: crashReferenceUnknown, + Time: crash.Reported, + }) + } + return nil +} + +func (crash *Crash) Save() ([]db.Property, error) { + return db.SaveStruct(crash) +} + +type Discussion struct { + ID string // the base message ID + Source string + Type string + Subject string + BugKeys []string + // Message contains last N messages. + // N is supposed to be big enough, so that in almost all cases + // AllMessages == len(Messages) holds true. + Messages []DiscussionMessage + // Since Messages could be trimmed, we have to keep aggregate stats. + Summary DiscussionSummary +} + +func discussionKey(c context.Context, source, id string) *db.Key { + return db.NewKey(c, "Discussion", fmt.Sprintf("%v-%v", source, id), 0, nil) +} + +func (d *Discussion) key(c context.Context) *db.Key { + return discussionKey(c, d.Source, d.ID) +} + +type DiscussionMessage struct { + ID string + // External is true if the message is not from the bot itself. + // Let's use a shorter name to save space. + External bool `datastore:"e"` + Time time.Time `datastore:",noindex"` +} + +// ReportingState holds dynamic info associated with reporting. +type ReportingState struct { + Entries []ReportingStateEntry +} + +type ReportingStateEntry struct { + Namespace string + Name string + // Current reporting quota consumption. + Sent int + Date int // YYYYMMDD +} + +// Subsystem holds the history of grouped per-subsystem open bug reminders. +type Subsystem struct { + Namespace string + Name string + // ListsQueried is the last time bug lists were queried for the subsystem. + ListsQueried time.Time + // LastBugList is the last time we have actually managed to generate a bug list. + LastBugList time.Time +} + +// SubsystemReport holds a single report about open bugs in a subsystem. +// There'll be one record for moderation (if it's needed) and one for actual reporting. +type SubsystemReport struct { + Created time.Time + BugKeys []string `datastore:",noindex"` + TotalStats SubsystemReportStats + PeriodStats SubsystemReportStats + Stages []SubsystemReportStage +} + +func (r *SubsystemReport) getBugKeys() ([]*db.Key, error) { + ret := []*db.Key{} + for _, encoded := range r.BugKeys { + key, err := db.DecodeKey(encoded) + if err != nil { + return nil, fmt.Errorf("failed to parse %#v: %w", encoded, err) + } + ret = append(ret, key) + } + return ret, nil +} + +func (r *SubsystemReport) findStage(id string) *SubsystemReportStage { + for j := range r.Stages { + stage := &r.Stages[j] + if stage.ID == id { + return stage + } + } + return nil +} + +type SubsystemReportStats struct { + Reported int + LowPrio int + Fixed int +} + +func (s *SubsystemReportStats) toDashapi() dashapi.BugListReportStats { + return dashapi.BugListReportStats{ + Reported: s.Reported, + LowPrio: s.LowPrio, + Fixed: s.Fixed, + } +} + +// There can be at most two stages. +// One has Moderation=true, the other one has Moderation=false. +type SubsystemReportStage struct { + ID string + ExtID string + Link string + Reported time.Time + Closed time.Time + Moderation bool +} + +// Job represent a single patch testing or bisection job for syz-ci. +// Later we may want to extend this to other types of jobs: +// - test of a committed fix +// - reproduce crash +// - test that crash still happens on HEAD +// +// Job has Bug as parent entity. +type Job struct { + Type JobType + Created time.Time + User string + CC []string + Reporting string + ExtID string // email Message-ID + Link string // web link for the job (e.g. email in the group) + Namespace string + Manager string + BugTitle string + CrashID int64 + + // Provided by user: + KernelRepo string + KernelBranch string + Patch int64 // reference to Patch text entity + KernelConfig int64 // reference to the kernel config entity + + Attempts int // number of times we tried to execute this job + IsRunning bool // the job might have been started, but never finished + LastStarted time.Time `datastore:"Started"` + Finished time.Time // if set, job is finished + TreeOrigin bool // whether the job is related to tree origin detection + + // If patch test should be done on the merge base between two branches. + MergeBaseRepo string + MergeBaseBranch string + + // By default, bisection starts from the revision of the associated crash. + // The BisectFrom field can override this. + BisectFrom string + + // Result of execution: + CrashTitle string // if empty, we did not hit crash during testing + CrashLog int64 // reference to CrashLog text entity + CrashReport int64 // reference to CrashReport text entity + Commits []Commit + BuildID string + Log int64 // reference to Log text entity + Error int64 // reference to Error text entity, if set job failed + Flags dashapi.JobDoneFlags + + Reported bool // have we reported result back to user? + InvalidatedBy string // user who marked this bug as invalid, empty by default + BackportedCommit Commit +} + +func (job *Job) IsBisection() bool { + return job.Type == JobBisectCause || job.Type == JobBisectFix +} + +func (job *Job) IsFinished() bool { + return !job.Finished.IsZero() +} + +type JobType int + +const ( + JobTestPatch JobType = iota + JobBisectCause + JobBisectFix +) + +func (typ JobType) toDashapiReportType() dashapi.ReportType { + switch typ { + case JobTestPatch: + return dashapi.ReportTestPatch + case JobBisectCause: + return dashapi.ReportBisectCause + case JobBisectFix: + return dashapi.ReportBisectFix + default: + panic(fmt.Sprintf("unknown job type %v", typ)) + } +} + +func (job *Job) isUnreliableBisect() bool { + if job.Type != JobBisectCause && job.Type != JobBisectFix { + panic(fmt.Sprintf("bad job type %v", job.Type)) + } + // If a bisection points to a merge or a commit that does not affect the kernel binary, + // it is considered an unreliable/wrong result and should not be reported in emails. + return job.Flags&dashapi.BisectResultMerge != 0 || + job.Flags&dashapi.BisectResultNoop != 0 || + job.Flags&dashapi.BisectResultRelease != 0 || + job.Flags&dashapi.BisectResultIgnore != 0 +} + +func (job *Job) IsCrossTree() bool { + return job.MergeBaseRepo != "" && job.IsBisection() +} + +// Text holds text blobs (crash logs, reports, reproducers, etc). +type Text struct { + Namespace string + Text []byte `datastore:",noindex"` // gzip-compressed text +} + +const ( + textCrashLog = "CrashLog" + textCrashReport = "CrashReport" + textReproSyz = "ReproSyz" + textReproC = "ReproC" + textMachineInfo = "MachineInfo" + textKernelConfig = "KernelConfig" + textPatch = "Patch" + textLog = "Log" + textError = "Error" + textReproLog = "ReproLog" +) + +const ( + BugStatusOpen = iota +) + +const ( + BugStatusFixed = 1000 + iota + BugStatusInvalid + BugStatusDup +) + +const ( + ReproLevelNone = dashapi.ReproLevelNone + ReproLevelSyz = dashapi.ReproLevelSyz + ReproLevelC = dashapi.ReproLevelC +) + +type BuildType int + +const ( + BuildNormal BuildType = iota + BuildFailed + BuildJob +) + +type BisectStatus int + +const ( + BisectNot BisectStatus = iota + BisectPending + BisectError + BisectYes // have 1 commit + BisectUnreliable // have 1 commit, but suspect it's wrong + BisectInconclusive // multiple commits due to skips + BisectHorizont // happens on the oldest commit we can test (or HEAD for fix bisection) + bisectStatusLast // this value can be changed (not stored in datastore) +) + +func (status BisectStatus) String() string { + switch status { + case BisectError: + return "error" + case BisectYes: + return "done" + case BisectUnreliable: + return "unreliable" + case BisectInconclusive: + return "inconclusive" + case BisectHorizont: + return "inconclusive" + default: + return "" + } +} + +func mgrKey(c context.Context, ns, name string) *db.Key { + return db.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil) +} + +func (mgr *Manager) key(c context.Context) *db.Key { + return mgrKey(c, mgr.Namespace, mgr.Name) +} + +func loadManager(c context.Context, ns, name string) (*Manager, error) { + mgr := new(Manager) + if err := db.Get(c, mgrKey(c, ns, name), mgr); err != nil { + if err != db.ErrNoSuchEntity { + return nil, fmt.Errorf("failed to get manager %v/%v: %w", ns, name, err) + } + mgr = &Manager{ + Namespace: ns, + Name: name, + } + } + return mgr, nil +} + +// updateManager does transactional compare-and-swap on the manager and its current stats. +func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats) error) error { + date := timeDate(timeNow(c)) + tx := func(c context.Context) error { + mgr, err := loadManager(c, ns, name) + if err != nil { + return err + } + mgrKey := mgr.key(c) + stats := new(ManagerStats) + statsKey := db.NewKey(c, "ManagerStats", "", int64(date), mgrKey) + if err := db.Get(c, statsKey, stats); err != nil { + if err != db.ErrNoSuchEntity { + return fmt.Errorf("failed to get stats %v/%v/%v: %w", ns, name, date, err) + } + stats = &ManagerStats{ + Date: date, + } + } + + if err := fn(mgr, stats); err != nil { + return err + } + + if _, err := db.Put(c, mgrKey, mgr); err != nil { + return fmt.Errorf("failed to put manager: %w", err) + } + if _, err := db.Put(c, statsKey, stats); err != nil { + return fmt.Errorf("failed to put manager stats: %w", err) + } + return nil + } + return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) +} + +func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) { + var managers []*Manager + query := db.NewQuery("Manager") + if ns != "" { + query = query.Filter("Namespace=", ns) + } + keys, err := query.GetAll(c, &managers) + if err != nil { + return nil, nil, fmt.Errorf("failed to query managers: %w", err) + } + var result []*Manager + var resultKeys []*db.Key + for i, mgr := range managers { + if getNsConfig(c, mgr.Namespace).Managers[mgr.Name].Decommissioned { + continue + } + result = append(result, mgr) + resultKeys = append(resultKeys, keys[i]) + } + return result, resultKeys, nil +} + +func buildKey(c context.Context, ns, id string) *db.Key { + if ns == "" { + panic("requesting build key outside of namespace") + } + h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id))) + return db.NewKey(c, "Build", h, 0, nil) +} + +func loadBuild(c context.Context, ns, id string) (*Build, error) { + build := new(Build) + if err := db.Get(c, buildKey(c, ns, id), build); err != nil { + if err == db.ErrNoSuchEntity { + return nil, fmt.Errorf("unknown build %v/%v", ns, id) + } + return nil, fmt.Errorf("failed to get build %v/%v: %w", ns, id, err) + } + return build, nil +} + +func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) { + mgr, err := loadManager(c, ns, manager) + if err != nil { + return nil, err + } + if mgr.CurrentBuild == "" { + return nil, fmt.Errorf("failed to fetch manager build: no builds") + } + return loadBuild(c, ns, mgr.CurrentBuild) +} + +func loadBuilds(c context.Context, ns, manager string, typ BuildType) ([]*Build, error) { + const limit = 500 + var builds []*Build + _, err := db.NewQuery("Build"). + Filter("Namespace=", ns). + Filter("Manager=", manager). + Filter("Type=", typ). + Order("-Time"). + Limit(limit). + GetAll(c, &builds) + if err != nil { + return nil, err + } + return builds, nil +} + +func (bug *Bug) displayTitle() string { + if bug.Seq == 0 { + return bug.Title + } + return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1) +} + +var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`) + +func splitDisplayTitle(display string) (string, int64, error) { + match := displayTitleRe.FindStringSubmatchIndex(display) + if match == nil { + return display, 0, nil + } + title := display[match[2]:match[3]] + seqStr := display[match[4]:match[5]] + seq, err := strconv.ParseInt(seqStr, 10, 64) + if err != nil { + return "", 0, fmt.Errorf("failed to parse bug title: %w", err) + } + if seq <= 0 || seq > 1e6 { + return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq) + } + return title, seq - 1, nil +} + +func canonicalBug(c context.Context, bug *Bug) (*Bug, error) { + for { + if bug.Status != BugStatusDup { + return bug, nil + } + canon := new(Bug) + bugKey := db.NewKey(c, "Bug", bug.DupOf, 0, nil) + if err := db.Get(c, bugKey, canon); err != nil { + return nil, fmt.Errorf("failed to get dup bug %q for %q: %w", + bug.DupOf, bug.keyHash(c), err) + } + bug = canon + } +} + +func (bug *Bug) key(c context.Context) *db.Key { + return db.NewKey(c, "Bug", bug.keyHash(c), 0, nil) +} + +func (bug *Bug) keyHash(c context.Context) string { + return bugKeyHash(c, bug.Namespace, bug.Title, bug.Seq) +} + +func bugKeyHash(c context.Context, ns, title string, seq int64) string { + return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq))) +} + +func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) { + domain := getNsConfig(c, bug.Namespace).SimilarityDomain + dedup := make(map[string]bool) + dedup[bug.keyHash(c)] = true + + ret := []*Bug{} + for _, title := range bug.AltTitles { + var similar []*Bug + _, err := db.NewQuery("Bug"). + Filter("AltTitles=", title). + GetAll(c, &similar) + if err != nil { + return nil, err + } + for _, bug := range similar { + if getNsConfig(c, bug.Namespace).SimilarityDomain != domain || + dedup[bug.keyHash(c)] { + continue + } + dedup[bug.keyHash(c)] = true + ret = append(ret, bug) + } + } + return ret, nil +} + +// Since these IDs appear in Reported-by tags in commit, we slightly limit their size. +const reportingHashLen = 20 + +func bugReportingHash(bugHash, reporting string) string { + return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:reportingHashLen] +} + +func looksLikeReportingHash(id string) bool { + // This is only used as best-effort check. + // Now we produce 20-chars ids, but we used to use full sha1 hash. + return len(id) == reportingHashLen || len(id) == 2*len(hash.Sig{}) +} + +func (bug *Bug) updateCommits(commits []string, now time.Time) { + bug.Commits = commits + bug.CommitInfo = nil + bug.NeedCommitInfo = true + bug.FixTime = now + bug.PatchedOn = nil +} + +func (bug *Bug) getCommitInfo(i int) Commit { + if i < len(bug.CommitInfo) { + return bug.CommitInfo[i] + } + return Commit{} +} + +func (bug *Bug) increaseCrashStats(now time.Time) { + bug.NumCrashes++ + date := timeDate(now) + if len(bug.DailyStats) == 0 || bug.DailyStats[len(bug.DailyStats)-1].Date < date { + bug.DailyStats = append(bug.DailyStats, BugDailyStats{date, 1}) + } else { + // It is theoretically possible that this method might get into a situation, when + // the latest saved date is later than now. But we assume that this can only happen + // in a small window around the start of the day and it is better to attribute a + // crash to the next day than to get a mismatch between NumCrashes and the sum of + // CrashCount. + bug.DailyStats[len(bug.DailyStats)-1].CrashCount++ + } + + if len(bug.DailyStats) > maxBugHistoryDays { + bug.DailyStats = bug.DailyStats[len(bug.DailyStats)-maxBugHistoryDays:] + } +} + +func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats { + startDate := timeDate(from) + startPos := len(bug.DailyStats) + for ; startPos > 0; startPos-- { + if bug.DailyStats[startPos-1].Date < startDate { + break + } + } + return bug.DailyStats[startPos:] +} + +func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) { + var status dashapi.BugStatus + switch bug.Status { + case BugStatusOpen: + status = dashapi.BugStatusOpen + case BugStatusFixed: + status = dashapi.BugStatusFixed + case BugStatusInvalid: + status = dashapi.BugStatusInvalid + case BugStatusDup: + status = dashapi.BugStatusDup + default: + return status, fmt.Errorf("unknown bugs status %v", bug.Status) + } + return status, nil +} + +// If an entity of type EmergencyStop exists, syzbot's operation is paused until +// a support engineer deletes it from the DB. +type EmergencyStop struct { + Time time.Time + User string +} + +func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error { + crash := new(Crash) + crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) + if err := db.Get(c, crashKey, crash); err != nil { + return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) + } + crash.AddReference(ref) + if _, err := db.Put(c, crashKey, crash); err != nil { + return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) + } + return nil +} + +func removeCrashReference(c context.Context, crashID int64, bugKey *db.Key, + t CrashReferenceType, key string) error { + crash := new(Crash) + crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) + if err := db.Get(c, crashKey, crash); err != nil { + return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) + } + crash.ClearReference(t, key) + if _, err := db.Put(c, crashKey, crash); err != nil { + return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) + } + return nil +} + +func kernelRepoInfo(c context.Context, build *Build) KernelRepo { + return kernelRepoInfoRaw(c, build.Namespace, build.KernelRepo, build.KernelBranch) +} + +func kernelRepoInfoRaw(c context.Context, ns, url, branch string) KernelRepo { + var info KernelRepo + for _, repo := range getNsConfig(c, ns).Repos { + if repo.URL == url && repo.Branch == branch { + info = repo + break + } + } + if info.Alias == "" { + info.Alias = url + if branch != "" { + info.Alias += " " + branch + } + } + return info +} + +func textLink(tag string, id int64) string { + if id == 0 { + return "" + } + return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16)) +} + +// timeDate returns t's date as a single int YYYYMMDD. +func timeDate(t time.Time) int { + year, month, day := t.Date() + return year*10000 + int(month)*100 + day +} + +func stringInList(list []string, str string) bool { + for _, s := range list { + if s == str { + return true + } + } + return false +} + +func stringListsIntersect(a, b []string) bool { + m := map[string]bool{} + for _, strA := range a { + m[strA] = true + } + for _, strB := range b { + if m[strB] { + return true + } + } + return false +} + +func mergeString(list []string, str string) []string { + if !stringInList(list, str) { + list = append(list, str) + } + return list +} + +func mergeStringList(list, add []string) []string { + for _, str := range add { + list = mergeString(list, str) + } + return list +} + +// dateTime converts date in YYYYMMDD format back to Time. +func dateTime(date int) time.Time { + return time.Date(date/10000, time.Month(date/100%100), date%100, 0, 0, 0, 0, time.UTC) +} diff --git a/dashboard/app/entities_datastore_test.go b/dashboard/app/entities_datastore_test.go new file mode 100644 index 000000000..7db3d5951 --- /dev/null +++ b/dashboard/app/entities_datastore_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + db "google.golang.org/appengine/v2/datastore" +) + +func TestOldBugTagsConversion(t *testing.T) { + oldBug := &struct { + Namespace string + Title string + Tags BugTags202304 + }{ + Namespace: "some-ns", + Title: "some title", + Tags: BugTags202304{ + Subsystems: []BugTag202304{ + { + Name: "first", + SetBy: "user", + }, + { + Name: "second", + }, + }, + }, + } + + fields, err := db.SaveStruct(oldBug) + if err != nil { + t.Fatal(err) + } + + newBug := &Bug{} + err = newBug.Load(fields) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(&Bug{ + Namespace: "some-ns", + Title: "some title", + Labels: []BugLabel{ + { + Value: "first", + SetBy: "user", + Label: SubsystemLabel, + }, + { + Value: "second", + Label: SubsystemLabel, + }, + }, + }, newBug); diff != "" { + t.Fatal(diff) + } +} diff --git a/dashboard/app/entities_test.go b/dashboard/app/entities_test.go deleted file mode 100644 index 7db3d5951..000000000 --- a/dashboard/app/entities_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2023 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - db "google.golang.org/appengine/v2/datastore" -) - -func TestOldBugTagsConversion(t *testing.T) { - oldBug := &struct { - Namespace string - Title string - Tags BugTags202304 - }{ - Namespace: "some-ns", - Title: "some title", - Tags: BugTags202304{ - Subsystems: []BugTag202304{ - { - Name: "first", - SetBy: "user", - }, - { - Name: "second", - }, - }, - }, - } - - fields, err := db.SaveStruct(oldBug) - if err != nil { - t.Fatal(err) - } - - newBug := &Bug{} - err = newBug.Load(fields) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(&Bug{ - Namespace: "some-ns", - Title: "some title", - Labels: []BugLabel{ - { - Value: "first", - SetBy: "user", - Label: SubsystemLabel, - }, - { - Value: "second", - Label: SubsystemLabel, - }, - }, - }, newBug); diff != "" { - t.Fatal(diff) - } -} -- cgit mrf-deployment