From c53ea9c902c6745d750c5700631e1556716f4cd4 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Wed, 2 Apr 2025 20:28:15 +0200 Subject: pkg/covermerger: read gcs files sequentially Quarter long aggregation means thousands of gzip files. Opening all the files in parallel we struggle from: 1. Memory overhead. 2. GCS API errors. It can't read Attrs for 1500+ files. --- pkg/cover/file.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'pkg/cover') diff --git a/pkg/cover/file.go b/pkg/cover/file.go index f96deb677..97b6be6b7 100644 --- a/pkg/cover/file.go +++ b/pkg/cover/file.go @@ -63,21 +63,11 @@ func GetMergeResult(c context.Context, ns, repo, forCommit, sourceCommit, filePa } fromDate, toDate := tp.DatesFromTo() - dbReader := covermerger.MakeBQCSVReader() - if err := dbReader.InitNsRecords(c, - ns, - filePath, - sourceCommit, - fromDate, - toDate, - ); err != nil { - return nil, fmt.Errorf("failed to dbReader.InitNsRecords: %w", err) - } - defer dbReader.Close() - csvReader, err := dbReader.Reader() + csvReader, err := covermerger.InitNsRecords(c, ns, filePath, sourceCommit, fromDate, toDate) if err != nil { - return nil, fmt.Errorf("failed to dbReader.Reader: %w", err) + return nil, fmt.Errorf("failed to covermerger.InitNsRecords: %w", err) } + defer csvReader.Close() ch := make(chan *covermerger.FileMergeResult, 1) if err := covermerger.MergeCSVData(c, config, csvReader, ch); err != nil { -- cgit mrf-deployment