From a37f70eacae01881d3ed517cdf93f59b5a0156e1 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Fri, 14 Feb 2025 17:15:00 +0100 Subject: pkg/manager: export programs + coverage jsonl The export is quite big but is generated fast. Every line is a valid json object representing the single program coverage. --- pkg/cover/html.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++-- pkg/cover/report.go | 4 +-- pkg/manager/http.go | 15 +++++++++ 3 files changed, 109 insertions(+), 4 deletions(-) (limited to 'pkg') diff --git a/pkg/cover/html.go b/pkg/cover/html.go index 599db6141..074f10815 100644 --- a/pkg/cover/html.go +++ b/pkg/cover/html.go @@ -178,7 +178,7 @@ func fileLineContents(file *file, lines [][]byte) lineCoverExport { func (rg *ReportGenerator) DoRawCoverFiles(w io.Writer, params HandlerParams) error { progs := fixUpPCs(params.Progs, params.Filter) - if err := rg.symbolizePCs(uniquePCs(progs)); err != nil { + if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil { return err } @@ -223,7 +223,7 @@ func (rg *ReportGenerator) DoCoverJSONL(w io.Writer, params HandlerParams) error } } progs := fixUpPCs(params.Progs, params.Filter) - if err := rg.symbolizePCs(uniquePCs(progs)); err != nil { + if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil { return err } pcProgCount := make(map[uint64]int) @@ -256,6 +256,96 @@ func (rg *ReportGenerator) DoCoverJSONL(w io.Writer, params HandlerParams) error return nil } +type CoveredBlock struct { + FromLine int `json:"from_line"` + FromCol int `json:"from_column"` + ToLine int `json:"to_line"` + ToCol int `json:"to_column"` +} + +type FunctionCoverage struct { + FuncName string `json:"func_name"` + Instrumented int `json:"total_blocks,omitempty"` + Blocks []*CoveredBlock `json:"covered_blocks"` +} + +type FileCoverage struct { + Repo string `json:"repo,omitempty"` + Commit string `json:"commit,omitempty"` + FilePath string `json:"file_path"` + Functions []*FunctionCoverage `json:"functions"` +} + +type ProgramCoverage struct { + Program string `json:"program"` + CoveredFiles []*FileCoverage `json:"coverage"` +} + +// DoCoverPrograms returns the corpus programs with the associated coverage. +// The result is a jsonl stream. +// Each line is a single ProgramCoverage record. +func (rg *ReportGenerator) DoCoverPrograms(w io.Writer, params HandlerParams) error { + if rg.CallbackPoints != nil { + if err := rg.symbolizePCs(rg.CallbackPoints); err != nil { + return fmt.Errorf("failed to symbolize PCs(): %w", err) + } + } + pcToFrames := map[uint64][]*backend.Frame{} + for _, frame := range rg.Frames { + pcToFrames[frame.PC] = append(pcToFrames[frame.PC], frame) + } + encoder := json.NewEncoder(w) + for _, prog := range params.Progs { + fileFuncFrames := map[string]map[string][]*backend.Frame{} + for _, pc := range uniquePCs(prog) { + for _, frame := range pcToFrames[pc] { + if fileFuncFrames[frame.Name] == nil { + fileFuncFrames[frame.Name] = map[string][]*backend.Frame{} + } + frames := fileFuncFrames[frame.Name][frame.FuncName] + frames = append(frames, frame) + fileFuncFrames[frame.Name][frame.FuncName] = frames + } + } + + var progCoverage []*FileCoverage + for filePath, functions := range fileFuncFrames { + var expFuncs []*FunctionCoverage + for funcName, frames := range functions { + var expCoveredBlocks []*CoveredBlock + for _, frame := range frames { + endCol := frame.EndCol + if endCol == backend.LineEnd { + endCol = -1 + } + expCoveredBlocks = append(expCoveredBlocks, &CoveredBlock{ + FromCol: frame.StartCol, + FromLine: frame.StartLine, + ToCol: endCol, + ToLine: frame.EndLine, + }) + } + expFuncs = append(expFuncs, &FunctionCoverage{ + FuncName: funcName, + Blocks: expCoveredBlocks, + }) + } + progCoverage = append(progCoverage, &FileCoverage{ + FilePath: filePath, + Functions: expFuncs, + }) + } + + if err := encoder.Encode(&ProgramCoverage{ + Program: prog.Data, + CoveredFiles: progCoverage, + }); err != nil { + return fmt.Errorf("encoder.Encode: %w", err) + } + } + return nil +} + func (rg *ReportGenerator) DoRawCover(w io.Writer, params HandlerParams) error { progs := fixUpPCs(params.Progs, params.Filter) var pcs []uint64 diff --git a/pkg/cover/report.go b/pkg/cover/report.go index 3802f7b39..6d4eab0c1 100644 --- a/pkg/cover/report.go +++ b/pkg/cover/report.go @@ -81,7 +81,7 @@ type line struct { type fileMap map[string]*file func (rg *ReportGenerator) prepareFileMap(progs []Prog, force, debug bool) (fileMap, error) { - if err := rg.symbolizePCs(uniquePCs(progs)); err != nil { + if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil { return nil, err } files := make(fileMap) @@ -187,7 +187,7 @@ func coverageCallbackMismatch(debug bool, numPCs int, unmatchedPCs map[uint64]bo len(unmatchedPCs), numPCs, debugStr) } -func uniquePCs(progs []Prog) []uint64 { +func uniquePCs(progs ...Prog) []uint64 { PCs := make(map[uint64]bool) for _, p := range progs { for _, pc := range p.PCs { diff --git a/pkg/manager/http.go b/pkg/manager/http.go index 99af2c0ed..50f672207 100644 --- a/pkg/manager/http.go +++ b/pkg/manager/http.go @@ -86,6 +86,7 @@ func (serv *HTTPServer) Serve(ctx context.Context) error { handle("/corpus", serv.httpCorpus) handle("/corpus.db", serv.httpDownloadCorpus) handle("/cover", serv.httpCover) + handle("/coverprogs", serv.httpPrograms) handle("/debuginput", serv.httpDebugInput) handle("/file", serv.httpFile) handle("/filecover", serv.httpFileCover) @@ -444,6 +445,7 @@ const ( DoRawCover DoFilterPCs DoCoverJSONL + DoCoverPrograms ) func (serv *HTTPServer) httpCover(w http.ResponseWriter, r *http.Request) { @@ -458,6 +460,18 @@ func (serv *HTTPServer) httpCover(w http.ResponseWriter, r *http.Request) { serv.httpCoverCover(w, r, DoHTML) } +func (serv *HTTPServer) httpPrograms(w http.ResponseWriter, r *http.Request) { + if !serv.Cfg.Cover { + http.Error(w, "coverage is not enabled", http.StatusInternalServerError) + return + } + if r.FormValue("jsonl") != "1" { + http.Error(w, "only ?jsonl=1 param is supported", http.StatusBadRequest) + return + } + serv.httpCoverCover(w, r, DoCoverPrograms) +} + func (serv *HTTPServer) httpSubsystemCover(w http.ResponseWriter, r *http.Request) { if !serv.Cfg.Cover { serv.httpCoverFallback(w, r) @@ -577,6 +591,7 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f DoRawCover: {rg.DoRawCover, ctTextPlain}, DoFilterPCs: {rg.DoFilterPCs, ctTextPlain}, DoCoverJSONL: {rg.DoCoverJSONL, ctApplicationJSON}, + DoCoverPrograms: {rg.DoCoverPrograms, ctApplicationJSON}, } if ct := flagToFunc[funcFlag].contentType; ct != "" { -- cgit mrf-deployment