From 5e7b4bcaa61e8bb9b1d1fbca21684fe490f69133 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 7 Oct 2024 14:09:32 +0200 Subject: dashboard/app: introduce batch_reproexport This PR exports the latest reproducer for every bug. Reproducers are exported to the "bug_id/repro_id.c" files. This approach allows to add some metadata files or export more reproducers/bug later. All the files are then archived and uploaded to the preconfigured location. --- dashboard/app/batch_coverage.go | 153 ++++++++++++++++++++++++ dashboard/app/batch_main.go | 99 ++++++++++++++++ dashboard/app/batch_reproexport.go | 37 ++++++ dashboard/app/config.go | 4 +- dashboard/app/coverage_batch.go | 237 ------------------------------------- 5 files changed, 292 insertions(+), 238 deletions(-) create mode 100644 dashboard/app/batch_coverage.go create mode 100644 dashboard/app/batch_main.go create mode 100644 dashboard/app/batch_reproexport.go delete mode 100644 dashboard/app/coverage_batch.go (limited to 'dashboard/app') diff --git a/dashboard/app/batch_coverage.go b/dashboard/app/batch_coverage.go new file mode 100644 index 000000000..09f9e71ff --- /dev/null +++ b/dashboard/app/batch_coverage.go @@ -0,0 +1,153 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "cloud.google.com/go/batch/apiv1/batchpb" + "cloud.google.com/go/bigquery" + "cloud.google.com/go/civil" + "github.com/google/syzkaller/pkg/coveragedb" + "google.golang.org/api/iterator" + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/log" +) + +const batchCoverageTimeoutSeconds = 60 * 60 * 12 + +func handleBatchCoverage(w http.ResponseWriter, r *http.Request) { + ctx := appengine.NewContext(r) + doQuarters := r.FormValue("quarters") == "true" + doMonths := r.FormValue("months") == "true" + doDays := r.FormValue("days") == "true" + maxSteps, err := strconv.Atoi(r.FormValue("steps")) + if err != nil { + log.Errorf(ctx, "failed to convert &steps= into maxSteps: %s", err.Error()) + return + } + for ns, nsConfig := range getConfig(ctx).Namespaces { + if nsConfig.Coverage == nil { + continue + } + repo, branch := nsConfig.mainRepoBranch() + if repo == "" || branch == "" { + log.Errorf(ctx, "can't find default repo or branch for ns %s", ns) + continue + } + daysAvailable, rowsAvailable, err := nsDataAvailable(ctx, ns) + if err != nil { + log.Errorf(ctx, "failed nsDataAvailable(%s): %s", ns, err) + } + periodsMerged, rowsMerged, err := coveragedb.NsDataMerged(ctx, "syzkaller", ns) + if err != nil { + log.Errorf(ctx, "failed coveragedb.NsDataMerged(%s): %s", ns, err) + } + var periods []coveragedb.TimePeriod + if doDays { + periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, + &coveragedb.DayPeriodOps{})...) + } + if doMonths { + periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, + &coveragedb.MonthPeriodOps{})...) + } + if doQuarters { + periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, + &coveragedb.QuarterPeriodOps{})...) + } + if len(periods) == 0 { + log.Infof(ctx, "there is no new coverage for merging available in %s", ns) + continue + } + periods = coveragedb.AtMostNLatestPeriods(periods, maxSteps) + nsCovConfig := nsConfig.Coverage + serviceAccount := &batchpb.ServiceAccount{ + Email: nsCovConfig.BatchServiceAccount, + Scopes: nsCovConfig.BatchScopes, + } + if err := createScriptJob(ctx, nsCovConfig.BatchProject, "coverage-merge", + batchCoverageScript(ns, repo, branch, periods, + nsCovConfig.JobInitScript, + nsCovConfig.SyzEnvInitScript, + nsCovConfig.DashboardClientName), + batchCoverageTimeoutSeconds, + serviceAccount, + ); err != nil { + log.Errorf(ctx, "failed to batchCoverageScript: %s", err.Error()) + } + } +} + +func batchCoverageScript(ns, repo, branch string, periods []coveragedb.TimePeriod, + jobInitScript, syzEnvInitScript, clientName string) string { + if clientName == "" { + clientName = defaultDashboardClientName + } + script := jobInitScript + "\n" + script += "git clone --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + + "cd syzkaller\n" + + "export CI=1\n" + + "./tools/syz-env \"" + if syzEnvInitScript != "" { + script += syzEnvInitScript + "; " + } + for _, period := range periods { + script += "./tools/syz-bq.sh" + + " -w ../workdir-cover-aggregation/" + + " -n " + ns + + " -r " + repo + + " -b " + branch + + " -d " + strconv.Itoa(period.Days) + + " -t " + period.DateTo.String() + + " -c " + clientName + + " 2>&1; " // we don't want stderr output to be logged as errors + } + script += "\"" + return script +} + +func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, []int64, error) { + client, err := bigquery.NewClient(ctx, "syzkaller") + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize bigquery client: %w", err) + } + if err := client.EnableStorageReadClient(ctx); err != nil { + return nil, nil, fmt.Errorf("failed to client.EnableStorageReadClient: %w", err) + } + q := client.Query(fmt.Sprintf(` + SELECT + PARSE_DATE('%%Y%%m%%d', partition_id) as partitiondate, + total_rows as records + FROM + syzkaller.syzbot_coverage.INFORMATION_SCHEMA.PARTITIONS + WHERE table_name LIKE '%s' + `, ns)) + it, err := q.Read(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to Read() from bigquery: %w", err) + } + + var periods []coveragedb.TimePeriod + var recordsCount []int64 + for { + var values struct { + PartitionDate civil.Date + Records int64 + } + err = it.Next(&values) + if err == iterator.Done { + break + } + if err != nil { + return nil, nil, fmt.Errorf("failed to it.Next() bigquery records: %w", err) + } + periods = append(periods, coveragedb.TimePeriod{DateTo: values.PartitionDate, Days: 1}) + recordsCount = append(recordsCount, values.Records) + } + return periods, recordsCount, nil +} diff --git a/dashboard/app/batch_main.go b/dashboard/app/batch_main.go new file mode 100644 index 000000000..acf37ee8f --- /dev/null +++ b/dashboard/app/batch_main.go @@ -0,0 +1,99 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "fmt" + "net/http" + + "cloud.google.com/go/batch/apiv1" + "cloud.google.com/go/batch/apiv1/batchpb" + "github.com/google/uuid" + "google.golang.org/appengine/v2/log" + "google.golang.org/protobuf/types/known/durationpb" +) + +func initBatchProcessors() { + http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) + http.HandleFunc("/cron/batch_reproexport", handleBatchReproExport) +} + +// from https://cloud.google.com/batch/docs/samples/batch-create-script-job +func createScriptJob(ctx context.Context, projectID, jobNamePrefix, script string, + timeout int64, sa *batchpb.ServiceAccount) error { + region := "us-central1" + jobName := fmt.Sprintf("%s-%s", jobNamePrefix, uuid.New().String()) + + batchClient, err := batch.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed NewClient: %w", err) + } + defer batchClient.Close() + + taskGroups := []*batchpb.TaskGroup{ + { + TaskSpec: &batchpb.TaskSpec{ + Runnables: []*batchpb.Runnable{{ + Executable: &batchpb.Runnable_Script_{ + Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{ + Text: script, + }}, + }, + }}, + ComputeResource: &batchpb.ComputeResource{ + // CpuMilli is milliseconds per cpu-second. This means the task requires 2 whole CPUs. + CpuMilli: 4000, + MemoryMib: 12 * 1024, + }, + MaxRunDuration: &durationpb.Duration{ + Seconds: timeout, + }, + }, + }, + } + + // Policies are used to define on what kind of virtual machines the tasks will run on. + // In this case, we tell the system to use "e2-standard-4" machine type. + // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types + allocationPolicy := &batchpb.AllocationPolicy{ + Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{ + PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{ + Policy: &batchpb.AllocationPolicy_InstancePolicy{ + ProvisioningModel: batchpb.AllocationPolicy_SPOT, + MachineType: "c3-standard-4", + }, + }, + }}, + ServiceAccount: sa, + } + + logsPolicy := &batchpb.LogsPolicy{ + Destination: batchpb.LogsPolicy_CLOUD_LOGGING, + } + + // The job's parent is the region in which the job will run. + parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region) + + job := batchpb.Job{ + TaskGroups: taskGroups, + AllocationPolicy: allocationPolicy, + LogsPolicy: logsPolicy, + } + + req := &batchpb.CreateJobRequest{ + Parent: parent, + JobId: jobName, + Job: &job, + } + + createdJob, err := batchClient.CreateJob(ctx, req) + if err != nil { + return fmt.Errorf("unable to create job: %w", err) + } + + log.Infof(ctx, "job created: %v\n", createdJob) + + return nil +} diff --git a/dashboard/app/batch_reproexport.go b/dashboard/app/batch_reproexport.go new file mode 100644 index 000000000..845d901fe --- /dev/null +++ b/dashboard/app/batch_reproexport.go @@ -0,0 +1,37 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "net/http" + + "google.golang.org/appengine/v2" + "google.golang.org/appengine/v2/log" +) + +const exportTimeoutSeconds = 60 * 60 * 6 + +func handleBatchReproExport(w http.ResponseWriter, r *http.Request) { + ctx := appengine.NewContext(r) + for ns, nsConfig := range getConfig(ctx).Namespaces { + if nsConfig.ReproExportPath == "" { + continue + } + if err := createScriptJob(ctx, "syzkaller", "export-repro", + exportReproScript(ns, nsConfig.ReproExportPath), exportTimeoutSeconds, nil); err != nil { + log.Errorf(ctx, "createScriptJob: %s", err.Error()) + } + } +} + +func exportReproScript(srcNamespace, archivePath string) string { + return "\n" + + "git clone --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + + "cd syzkaller\n" + + "./tools/syz-env \"" + + "go run ./tools/syz-reprolist/... -namespace " + srcNamespace + " && " + + "tar -czvf reproducers.tar.gz ./repros/ && " + + "gsutil -m cp reproducers.tar.gz " + archivePath + + "\"" +} diff --git a/dashboard/app/config.go b/dashboard/app/config.go index 0e30c9320..b8d810440 100644 --- a/dashboard/app/config.go +++ b/dashboard/app/config.go @@ -124,6 +124,8 @@ type Config struct { CacheUIPages bool // Enables coverage aggregation. Coverage *CoverageConfig + // Reproducers export path. + ReproExportPath string } const defaultDashboardClientName = "coverage-merger" @@ -393,7 +395,7 @@ func installConfig(cfg *GlobalConfig) { initHTTPHandlers() initAPIHandlers() initKcidb() - initCoverageBatches() + initBatchProcessors() } var contextConfigKey = "Updated config (to be used during tests). Use only in tests!" diff --git a/dashboard/app/coverage_batch.go b/dashboard/app/coverage_batch.go deleted file mode 100644 index 8a720066c..000000000 --- a/dashboard/app/coverage_batch.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2017 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "context" - "fmt" - "net/http" - "strconv" - - "cloud.google.com/go/batch/apiv1" - "cloud.google.com/go/batch/apiv1/batchpb" - "cloud.google.com/go/bigquery" - "cloud.google.com/go/civil" - "github.com/google/syzkaller/pkg/coveragedb" - "github.com/google/uuid" - "google.golang.org/api/iterator" - "google.golang.org/appengine/v2" - "google.golang.org/appengine/v2/log" - "google.golang.org/protobuf/types/known/durationpb" -) - -func initCoverageBatches() { - http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) -} - -const batchTimeoutSeconds = 60 * 60 * 12 - -func handleBatchCoverage(w http.ResponseWriter, r *http.Request) { - ctx := appengine.NewContext(r) - doQuarters := r.FormValue("quarters") == "true" - doMonths := r.FormValue("months") == "true" - doDays := r.FormValue("days") == "true" - maxSteps, err := strconv.Atoi(r.FormValue("steps")) - if err != nil { - log.Errorf(ctx, "failed to convert &steps= into maxSteps: %s", err.Error()) - return - } - for ns, nsConfig := range getConfig(ctx).Namespaces { - if nsConfig.Coverage == nil { - continue - } - repo, branch := nsConfig.mainRepoBranch() - if repo == "" || branch == "" { - log.Errorf(ctx, "can't find default repo or branch for ns %s", ns) - continue - } - daysAvailable, rowsAvailable, err := nsDataAvailable(ctx, ns) - if err != nil { - log.Errorf(ctx, "failed nsDataAvailable(%s): %s", ns, err) - } - periodsMerged, rowsMerged, err := coveragedb.NsDataMerged(ctx, "syzkaller", ns) - if err != nil { - log.Errorf(ctx, "failed coveragedb.NsDataMerged(%s): %s", ns, err) - } - var periods []coveragedb.TimePeriod - if doDays { - periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, - &coveragedb.DayPeriodOps{})...) - } - if doMonths { - periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, - &coveragedb.MonthPeriodOps{})...) - } - if doQuarters { - periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged, - &coveragedb.QuarterPeriodOps{})...) - } - if len(periods) == 0 { - log.Infof(ctx, "there is no new coverage for merging available in %s", ns) - continue - } - periods = coveragedb.AtMostNLatestPeriods(periods, maxSteps) - nsCovConfig := nsConfig.Coverage - if err := createScriptJob( - ctx, - nsCovConfig.BatchProject, - nsCovConfig.BatchServiceAccount, - batchScript(ns, repo, branch, periods, - nsCovConfig.JobInitScript, - nsCovConfig.SyzEnvInitScript, - nsCovConfig.DashboardClientName), - nsCovConfig.BatchScopes); err != nil { - log.Errorf(ctx, "failed to batchScript: %s", err.Error()) - } - } -} - -func batchScript(ns, repo, branch string, periods []coveragedb.TimePeriod, - jobInitScript, syzEnvInitScript, clientName string) string { - if clientName == "" { - clientName = defaultDashboardClientName - } - script := jobInitScript + "\n" - script += "git clone --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + - "cd syzkaller\n" + - "export CI=1\n" + - "./tools/syz-env \"" - if syzEnvInitScript != "" { - script += syzEnvInitScript + "; " - } - for _, period := range periods { - script += "./tools/syz-bq.sh" + - " -w ../workdir-cover-aggregation/" + - " -n " + ns + - " -r " + repo + - " -b " + branch + - " -d " + strconv.Itoa(period.Days) + - " -t " + period.DateTo.String() + - " -c " + clientName + - " 2>&1; " // we don't want stderr output to be logged as errors - } - script += "\"" - return script -} - -// from https://cloud.google.com/batch/docs/samples/batch-create-script-job -func createScriptJob(ctx context.Context, projectID, serviceAccount, script string, scopes []string) error { - region := "us-central1" - jobName := fmt.Sprintf("coverage-merge-%s", uuid.New().String()) - - batchClient, err := batch.NewClient(ctx) - if err != nil { - return fmt.Errorf("failed NewClient: %w", err) - } - defer batchClient.Close() - - taskGroups := []*batchpb.TaskGroup{ - { - TaskSpec: &batchpb.TaskSpec{ - Runnables: []*batchpb.Runnable{{ - Executable: &batchpb.Runnable_Script_{ - Script: &batchpb.Runnable_Script{Command: &batchpb.Runnable_Script_Text{ - Text: script, - }}, - }, - }}, - ComputeResource: &batchpb.ComputeResource{ - // CpuMilli is milliseconds per cpu-second. This means the task requires 2 whole CPUs. - CpuMilli: 4000, - MemoryMib: 12 * 1024, - }, - MaxRunDuration: &durationpb.Duration{ - Seconds: batchTimeoutSeconds, - }, - }, - }, - } - - // Policies are used to define on what kind of virtual machines the tasks will run on. - // In this case, we tell the system to use "e2-standard-4" machine type. - // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types - allocationPolicy := &batchpb.AllocationPolicy{ - Instances: []*batchpb.AllocationPolicy_InstancePolicyOrTemplate{{ - PolicyTemplate: &batchpb.AllocationPolicy_InstancePolicyOrTemplate_Policy{ - Policy: &batchpb.AllocationPolicy_InstancePolicy{ - ProvisioningModel: batchpb.AllocationPolicy_SPOT, - MachineType: "c3-standard-4", - }, - }, - }}, - ServiceAccount: &batchpb.ServiceAccount{ - Email: serviceAccount, - Scopes: scopes, - }, - } - - logsPolicy := &batchpb.LogsPolicy{ - Destination: batchpb.LogsPolicy_CLOUD_LOGGING, - } - - // The job's parent is the region in which the job will run. - parent := fmt.Sprintf("projects/%s/locations/%s", projectID, region) - - job := batchpb.Job{ - TaskGroups: taskGroups, - AllocationPolicy: allocationPolicy, - LogsPolicy: logsPolicy, - } - - req := &batchpb.CreateJobRequest{ - Parent: parent, - JobId: jobName, - Job: &job, - } - - createdJob, err := batchClient.CreateJob(ctx, req) - if err != nil { - return fmt.Errorf("unable to create job: %w", err) - } - - log.Infof(ctx, "job created: %v\n", createdJob) - - return nil -} - -func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, []int64, error) { - client, err := bigquery.NewClient(ctx, "syzkaller") - if err != nil { - return nil, nil, fmt.Errorf("failed to initialize bigquery client: %w", err) - } - if err := client.EnableStorageReadClient(ctx); err != nil { - return nil, nil, fmt.Errorf("failed to client.EnableStorageReadClient: %w", err) - } - q := client.Query(fmt.Sprintf(` - SELECT - PARSE_DATE('%%Y%%m%%d', partition_id) as partitiondate, - total_rows as records - FROM - syzkaller.syzbot_coverage.INFORMATION_SCHEMA.PARTITIONS - WHERE table_name LIKE '%s' - `, ns)) - it, err := q.Read(ctx) - if err != nil { - return nil, nil, fmt.Errorf("failed to Read() from bigquery: %w", err) - } - - var periods []coveragedb.TimePeriod - var recordsCount []int64 - for { - var values struct { - PartitionDate civil.Date - Records int64 - } - err = it.Next(&values) - if err == iterator.Done { - break - } - if err != nil { - return nil, nil, fmt.Errorf("failed to it.Next() bigquery records: %w", err) - } - periods = append(periods, coveragedb.TimePeriod{DateTo: values.PartitionDate, Days: 1}) - recordsCount = append(recordsCount, values.Records) - } - return periods, recordsCount, nil -} -- cgit mrf-deployment