diff options
| author | Taras Madan <tarasmadan@google.com> | 2025-04-02 20:28:15 +0200 |
|---|---|---|
| committer | Taras Madan <tarasmadan@google.com> | 2025-04-04 14:52:42 +0000 |
| commit | c53ea9c902c6745d750c5700631e1556716f4cd4 (patch) | |
| tree | 855f9483db0aeedee6fe7dbd34e746356db13085 /tools | |
| parent | 1c4febdbbe8f34403baef506b22cd689576927f6 (diff) | |
pkg/covermerger: read gcs files sequentially
Quarter long aggregation means thousands of gzip files.
Opening all the files in parallel we struggle from:
1. Memory overhead.
2. GCS API errors. It can't read Attrs for 1500+ files.
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/syz-covermerger/syz_covermerger.go | 12 |
1 files changed, 4 insertions, 8 deletions
diff --git a/tools/syz-covermerger/syz_covermerger.go b/tools/syz-covermerger/syz_covermerger.go index 6cd714421..7b51c9997 100644 --- a/tools/syz-covermerger/syz_covermerger.go +++ b/tools/syz-covermerger/syz_covermerger.go @@ -71,21 +71,17 @@ func do() error { panic(fmt.Sprintf("failed to parse time_to: %s", err.Error())) } dateFrom = dateTo.AddDays(-int(*flagDuration)) - dbReader := covermerger.MakeBQCSVReader() - if err = dbReader.InitNsRecords(context.Background(), + csvReader, err := covermerger.InitNsRecords(context.Background(), *flagNamespace, *flagFilePathPrefix, "", dateFrom, dateTo, - ); err != nil { + ) + if err != nil { panic(fmt.Sprintf("failed to dbReader.InitNsRecords: %v", err.Error())) } - defer dbReader.Close() - csvReader, errReader := dbReader.Reader() - if errReader != nil { - panic(fmt.Sprintf("failed to dbReader.Reader: %v", errReader.Error())) - } + defer csvReader.Close() var wc io.WriteCloser url := *flagToGCS if *flagToDashAPI != "" { |