From df6d5ba592799e2e819c8b3447b9a08e7e8d1986 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 9 Sep 2024 11:58:14 +0200 Subject: dashboard/app: use day long aggregations not week long Current coverage is a week long data merge for every day. The goal is to show a monthly data by default and make daily data available for &period=day requests. This commit makes the first two steps: 1. Make the day a day long, not a week long aggregation. 2. Enable the month long merges available for &period=month. --- pkg/cover/heatmap.go | 60 ++++++++++++++++++++------------------ pkg/coveragedb/time_period.go | 11 +++++-- pkg/coveragedb/time_period_test.go | 18 ++++++++---- 3 files changed, 51 insertions(+), 38 deletions(-) (limited to 'pkg') diff --git a/pkg/cover/heatmap.go b/pkg/cover/heatmap.go index 63ea96515..7faea553a 100644 --- a/pkg/cover/heatmap.go +++ b/pkg/cover/heatmap.go @@ -125,7 +125,7 @@ func filesCoverageToTemplateData(fCov []*fileCoverageWithDetails) *templateHeatm return &res } -func filesCoverageWithDetailsStmt(ns, subsystem string, fromDate, toDate civil.Date) spanner.Statement { +func filesCoverageWithDetailsStmt(ns, subsystem string, timePeriod coveragedb.TimePeriod) spanner.Statement { stmt := spanner.Statement{ SQL: ` select @@ -140,11 +140,11 @@ from merge_history join file_subsystems on merge_history.namespace = file_subsystems.namespace and files.filepath = file_subsystems.filepath where - merge_history.namespace=$1 and dateto>=$2 and dateto<=$3`, + merge_history.namespace=$1 and dateto=$2 and duration=$3`, Params: map[string]interface{}{ "p1": ns, - "p2": fromDate, - "p3": toDate, + "p2": timePeriod.DateTo, + "p3": timePeriod.Days, }, } if subsystem != "" { @@ -154,7 +154,7 @@ where return stmt } -func filesCoverageWithDetails(ctx context.Context, projectID, ns, subsystem string, fromDate, toDate civil.Date, +func filesCoverageWithDetails(ctx context.Context, projectID, ns, subsystem string, timePeriods []coveragedb.TimePeriod, ) ([]*fileCoverageWithDetails, error) { client, err := coveragedb.NewClient(ctx, projectID) if err != nil { @@ -162,23 +162,25 @@ func filesCoverageWithDetails(ctx context.Context, projectID, ns, subsystem stri } defer client.Close() - stmt := filesCoverageWithDetailsStmt(ns, subsystem, fromDate, toDate) - iter := client.Single().Query(ctx, stmt) - defer iter.Stop() res := []*fileCoverageWithDetails{} - for { - row, err := iter.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, fmt.Errorf("failed to iter.Next() spanner DB: %w", err) - } - var r fileCoverageWithDetails - if err = row.ToStruct(&r); err != nil { - return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err) + for _, timePeriod := range timePeriods { + stmt := filesCoverageWithDetailsStmt(ns, subsystem, timePeriod) + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("failed to iter.Next() spanner DB: %w", err) + } + var r fileCoverageWithDetails + if err = row.ToStruct(&r); err != nil { + return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err) + } + res = append(res, &r) } - res = append(res, &r) } return res, nil } @@ -190,8 +192,8 @@ type StyleBodyJS struct { } // nolint: dupl -func DoDirHeatMap(w io.Writer, projectID, ns string, dateFrom, dateTo civil.Date) error { - style, body, js, err := DoHeatMapStyleBodyJS(context.Background(), projectID, ns, "", dateFrom, dateTo) +func DoDirHeatMap(w io.Writer, projectID, ns string, periods []coveragedb.TimePeriod) error { + style, body, js, err := DoHeatMapStyleBodyJS(context.Background(), projectID, ns, "", periods) if err != nil { return fmt.Errorf("failed to DoHeatMapStyleBodyJS() %w", err) } @@ -203,8 +205,8 @@ func DoDirHeatMap(w io.Writer, projectID, ns string, dateFrom, dateTo civil.Date } // nolint: dupl -func DoSubsystemsHeatMap(w io.Writer, projectID, ns string, dateFrom, dateTo civil.Date) error { - style, body, js, err := DoSubsystemsHeatMapStyleBodyJS(context.Background(), projectID, ns, "", dateFrom, dateTo) +func DoSubsystemsHeatMap(w io.Writer, projectID, ns string, periods []coveragedb.TimePeriod) error { + style, body, js, err := DoSubsystemsHeatMapStyleBodyJS(context.Background(), projectID, ns, "", periods) if err != nil { return fmt.Errorf("failed to DoSubsystemsHeatMapStyleBodyJS() %w", err) } @@ -232,9 +234,9 @@ func stylesBodyJSTemplate(templData *templateHeatmap, template.HTML(js.Bytes()), nil } -func DoHeatMapStyleBodyJS(ctx context.Context, projectID, ns, subsystem string, dateFrom, dateTo civil.Date, +func DoHeatMapStyleBodyJS(ctx context.Context, projectID, ns, subsystem string, periods []coveragedb.TimePeriod, ) (template.CSS, template.HTML, template.HTML, error) { - covAndDates, err := filesCoverageWithDetails(ctx, projectID, ns, subsystem, dateFrom, dateTo) + covAndDates, err := filesCoverageWithDetails(ctx, projectID, ns, subsystem, periods) if err != nil { return "", "", "", fmt.Errorf("failed to filesCoverageWithDetails: %w", err) } @@ -242,9 +244,9 @@ func DoHeatMapStyleBodyJS(ctx context.Context, projectID, ns, subsystem string, return stylesBodyJSTemplate(templData) } -func DoSubsystemsHeatMapStyleBodyJS(ctx context.Context, projectID, ns, subsystem string, dateFrom, dateTo civil.Date, -) (template.CSS, template.HTML, template.HTML, error) { - covWithDetails, err := filesCoverageWithDetails(ctx, projectID, ns, subsystem, dateFrom, dateTo) +func DoSubsystemsHeatMapStyleBodyJS(ctx context.Context, projectID, ns, subsystem string, + periods []coveragedb.TimePeriod) (template.CSS, template.HTML, template.HTML, error) { + covWithDetails, err := filesCoverageWithDetails(ctx, projectID, ns, subsystem, periods) if err != nil { panic(err) } diff --git a/pkg/coveragedb/time_period.go b/pkg/coveragedb/time_period.go index b3a2aad73..bfa1b2789 100644 --- a/pkg/coveragedb/time_period.go +++ b/pkg/coveragedb/time_period.go @@ -17,6 +17,7 @@ type TimePeriod struct { } const ( + DayPeriod = "day" MonthPeriod = "month" QuarterPeriod = "quarter" ) @@ -25,6 +26,8 @@ var errUnknownTimePeriodType = errors.New("unknown time period type") func MinMaxDays(periodType string) (int, int, error) { switch periodType { + case DayPeriod: + return 1, 1, nil case MonthPeriod: return 28, 31, nil case QuarterPeriod: @@ -36,6 +39,8 @@ func MinMaxDays(periodType string) (int, int, error) { func PeriodOps(periodType string) (periodOps, error) { switch periodType { + case DayPeriod: + return &DayPeriodOps{}, nil case MonthPeriod: return &MonthPeriodOps{}, nil case QuarterPeriod: @@ -51,11 +56,11 @@ type periodOps interface { pointedPeriodDays(d civil.Date) int } -func GenNPeriodEndDatesTill(n int, d civil.Date, po periodOps) []civil.Date { - var res []civil.Date +func GenNPeriodsTill(n int, d civil.Date, po periodOps) []TimePeriod { + var res []TimePeriod for i := 0; i < n; i++ { d = po.lastPeriodDate(d) - res = append(res, d) + res = append(res, TimePeriod{DateTo: d, Days: po.pointedPeriodDays(d)}) d = d.AddDays(-po.pointedPeriodDays(d)) } slices.Reverse(res) diff --git a/pkg/coveragedb/time_period_test.go b/pkg/coveragedb/time_period_test.go index 86201e4db..4ccd18f12 100644 --- a/pkg/coveragedb/time_period_test.go +++ b/pkg/coveragedb/time_period_test.go @@ -25,8 +25,10 @@ func TestDayPeriodOps(t *testing.T) { assert.Equal(t, 1, ops.pointedPeriodDays(d)) assert.Equal(t, - []civil.Date{{Year: 2024, Month: time.February, Day: 19}, {Year: 2024, Month: time.February, Day: 20}}, - GenNPeriodEndDatesTill(2, d, ops)) + []TimePeriod{ + {DateTo: civil.Date{Year: 2024, Month: time.February, Day: 19}, Days: 1}, + {DateTo: civil.Date{Year: 2024, Month: time.February, Day: 20}, Days: 1}}, + GenNPeriodsTill(2, d, ops)) } func TestMonthPeriodOps(t *testing.T) { @@ -48,8 +50,10 @@ func TestMonthPeriodOps(t *testing.T) { assert.Equal(t, 29, ops.pointedPeriodDays(midMonthDate)) assert.Equal(t, - []civil.Date{{Year: 2024, Month: time.January, Day: 31}, {Year: 2024, Month: time.February, Day: 29}}, - GenNPeriodEndDatesTill(2, goodPeriod.DateTo, ops)) + []TimePeriod{ + {DateTo: civil.Date{Year: 2024, Month: time.January, Day: 31}, Days: 31}, + {DateTo: civil.Date{Year: 2024, Month: time.February, Day: 29}, Days: 29}}, + GenNPeriodsTill(2, goodPeriod.DateTo, ops)) } func TestQuarterPeriodOps(t *testing.T) { @@ -72,8 +76,10 @@ func TestQuarterPeriodOps(t *testing.T) { assert.Equal(t, 31+29+31, ops.pointedPeriodDays(midQuarterDate)) assert.Equal(t, - []civil.Date{{Year: 2023, Month: time.December, Day: 31}, {Year: 2024, Month: time.March, Day: 31}}, - GenNPeriodEndDatesTill(2, goodPeriod.DateTo, ops)) + []TimePeriod{ + {DateTo: civil.Date{Year: 2023, Month: time.December, Day: 31}, Days: 31 + 30 + 31}, + {DateTo: civil.Date{Year: 2024, Month: time.March, Day: 31}, Days: 31 + 29 + 31}}, + GenNPeriodsTill(2, goodPeriod.DateTo, ops)) } func TestPeriodsToMerge(t *testing.T) { -- cgit mrf-deployment