From c53ea9c902c6745d750c5700631e1556716f4cd4 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Wed, 2 Apr 2025 20:28:15 +0200 Subject: pkg/covermerger: read gcs files sequentially Quarter long aggregation means thousands of gzip files. Opening all the files in parallel we struggle from: 1. Memory overhead. 2. GCS API errors. It can't read Attrs for 1500+ files. --- tools/syz-covermerger/syz_covermerger.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/syz-covermerger/syz_covermerger.go b/tools/syz-covermerger/syz_covermerger.go index 6cd714421..7b51c9997 100644 --- a/tools/syz-covermerger/syz_covermerger.go +++ b/tools/syz-covermerger/syz_covermerger.go @@ -71,21 +71,17 @@ func do() error { panic(fmt.Sprintf("failed to parse time_to: %s", err.Error())) } dateFrom = dateTo.AddDays(-int(*flagDuration)) - dbReader := covermerger.MakeBQCSVReader() - if err = dbReader.InitNsRecords(context.Background(), + csvReader, err := covermerger.InitNsRecords(context.Background(), *flagNamespace, *flagFilePathPrefix, "", dateFrom, dateTo, - ); err != nil { + ) + if err != nil { panic(fmt.Sprintf("failed to dbReader.InitNsRecords: %v", err.Error())) } - defer dbReader.Close() - csvReader, errReader := dbReader.Reader() - if errReader != nil { - panic(fmt.Sprintf("failed to dbReader.Reader: %v", errReader.Error())) - } + defer csvReader.Close() var wc io.WriteCloser url := *flagToGCS if *flagToDashAPI != "" { -- cgit mrf-deployment