From a19f666144a849b339cdda7e4186046b4dec7dd8 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 13 Dec 2024 13:07:37 +0100 Subject: pkg/declextract: speed up sortAndDedupSlice Marshal to JSON only once and dedup based on hash before sorting. --- pkg/declextract/entity.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'pkg/declextract') diff --git a/pkg/declextract/entity.go b/pkg/declextract/entity.go index 5b7fe8813..eff8eda90 100644 --- a/pkg/declextract/entity.go +++ b/pkg/declextract/entity.go @@ -5,6 +5,7 @@ package declextract import ( "bytes" + "crypto/sha256" "encoding/json" "slices" ) @@ -186,15 +187,20 @@ func (out *Output) SetSourceFile(file string) { } } -func sortAndDedupSlice[Slice ~[]E, E any](s Slice) Slice { +func sortAndDedupSlice[Slice ~[]E, E comparable](s Slice) Slice { + dedup := make(map[[sha256.Size]byte]E) + text := make(map[E][]byte) + for _, e := range s { + t, _ := json.Marshal(e) + dedup[sha256.Sum256(t)] = e + text[e] = t + } + s = make([]E, 0, len(dedup)) + for _, e := range dedup { + s = append(s, e) + } slices.SortFunc(s, func(a, b E) int { - aa, _ := json.Marshal(a) - bb, _ := json.Marshal(b) - return bytes.Compare(aa, bb) - }) - return slices.CompactFunc(s, func(a, b E) bool { - aa, _ := json.Marshal(a) - bb, _ := json.Marshal(b) - return bytes.Equal(aa, bb) + return bytes.Compare(text[a], text[b]) }) + return s } -- cgit mrf-deployment