From 0fcd5fd3ddb45f5571658c4fce1589427a5bf090 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 25 May 2017 16:37:05 +0200 Subject: all: speed up tests Mark tests as parallel where makes sense. Speed up sys.TransitivelyEnabledCalls. Execution time is now: ok github.com/google/syzkaller/config 0.172s ok github.com/google/syzkaller/cover 0.060s ok github.com/google/syzkaller/csource 3.081s ok github.com/google/syzkaller/db 0.395s ok github.com/google/syzkaller/executor 0.060s ok github.com/google/syzkaller/fileutil 0.106s ok github.com/google/syzkaller/host 1.530s ok github.com/google/syzkaller/ifuzz 0.491s ok github.com/google/syzkaller/ipc 1.374s ok github.com/google/syzkaller/log 0.014s ok github.com/google/syzkaller/prog 2.604s ok github.com/google/syzkaller/report 0.045s ok github.com/google/syzkaller/symbolizer 0.062s ok github.com/google/syzkaller/sys 0.365s ok github.com/google/syzkaller/syz-dash 0.014s ok github.com/google/syzkaller/syz-hub/state 0.427s ok github.com/google/syzkaller/vm 0.052s However, main time is still taken by rebuilding sys package. Fixes #182 --- sys/decl.go | 16 ++++++++++++++-- sys/decl_test.go | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'sys') diff --git a/sys/decl.go b/sys/decl.go index 270ae4582..0956a94ee 100644 --- a/sys/decl.go +++ b/sys/decl.go @@ -521,14 +521,26 @@ func TransitivelyEnabledCalls(enabled map[*Call]bool) map[*Call]bool { for c := range enabled { supported[c] = true } + inputResources := make(map[*Call][]*ResourceType) + ctors := make(map[string][]*Call) + for c := range supported { + inputs := c.InputResources() + inputResources[c] = inputs + for _, res := range inputs { + if _, ok := ctors[res.Desc.Name]; ok { + continue + } + ctors[res.Desc.Name] = resourceCtors(res.Desc.Kind, true) + } + } for { n := len(supported) haveGettime := supported[CallMap["clock_gettime"]] for c := range supported { canCreate := true - for _, res := range c.InputResources() { + for _, res := range inputResources[c] { noctors := true - for _, ctor := range resourceCtors(res.Desc.Kind, true) { + for _, ctor := range ctors[res.Desc.Name] { if supported[ctor] { noctors = false break diff --git a/sys/decl_test.go b/sys/decl_test.go index c41a95e43..18385c321 100644 --- a/sys/decl_test.go +++ b/sys/decl_test.go @@ -8,6 +8,7 @@ import ( ) func TestTransitivelyEnabledCalls(t *testing.T) { + t.Parallel() calls := make(map[*Call]bool) for _, c := range Calls { calls[c] = true @@ -37,6 +38,7 @@ func TestTransitivelyEnabledCalls(t *testing.T) { } func TestClockGettime(t *testing.T) { + t.Parallel() calls := make(map[*Call]bool) for _, c := range Calls { calls[c] = true -- cgit mrf-deployment From 1f710b08fc40d6077b0065a25c5d7ab5d0750e7a Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 26 May 2017 15:32:51 +0200 Subject: sys: generate arrays instead of maps Compilation of large maps is super slow. Generate arrays instead and converet to maps at runtime. Reduces build time from ~40s to ~2s. Update #182 --- sys/align.go | 2 +- sys/decl.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++------ sysgen/sysgen.go | 51 +++++++++++++----------------- 3 files changed, 108 insertions(+), 39 deletions(-) (limited to 'sys') diff --git a/sys/align.go b/sys/align.go index e1ce27d56..00d464bcb 100644 --- a/sys/align.go +++ b/sys/align.go @@ -32,7 +32,7 @@ func initAlign() { } } - for _, s := range Structs { + for _, s := range keyedStructs { rec(s) } } diff --git a/sys/decl.go b/sys/decl.go index 0956a94ee..28c18d488 100644 --- a/sys/decl.go +++ b/sys/decl.go @@ -430,7 +430,86 @@ func (t *UnionType) Align() uintptr { return align } -var ctors = make(map[string][]*Call) +var ( + CallMap = make(map[string]*Call) + structs map[string]Type + keyedStructs map[structKey]Type + Resources map[string]*ResourceDesc + ctors = make(map[string][]*Call) +) + +type structKey struct { + name string + field string + dir Dir +} + +func getStruct(key structKey) Type { + if structs == nil { + structs = make(map[string]Type) + keyedStructs = make(map[structKey]Type) + for _, str := range structArray { + structs[str.Name()] = str + } + } + str := keyedStructs[key] + if str == nil { + proto := structs[key.name] + if proto == nil { + panic(fmt.Sprintf("missing struct prototype for %v", key.name)) + } + switch typed := proto.(type) { + case *StructType: + newStr := new(StructType) + *newStr = *typed + newStr.FldName = key.field + newStr.ArgDir = key.dir + str = newStr + case *UnionType: + newStr := new(UnionType) + *newStr = *typed + newStr.FldName = key.field + newStr.ArgDir = key.dir + str = newStr + default: + panic(fmt.Sprintf("unexpected type of struct prototype for %v: %+v", key.name, proto)) + } + keyedStructs[key] = str + } + return str +} + +func initStructFields() { + missed := 0 + for _, f := range structFields { + untyped := keyedStructs[f.key] + if untyped == nil { + missed++ + continue + } + switch str := untyped.(type) { + case *StructType: + str.Fields = f.fields + case *UnionType: + str.Options = f.fields + default: + panic(fmt.Sprintf("unexpected type of struct prototype for %v: %+v", f.key.name, untyped)) + } + } + fmt.Printf("missed %v/%v\n", missed, len(structFields)) +} + +func resource(name string) *ResourceDesc { + if Resources == nil { + // This is first called during init of sys package, so does not need to be thread-safe. + // resourceArray is in sys_GOARCH.go (generated by sysgen). + Resources = make(map[string]*ResourceDesc) + for _, res := range resourceArray { + Resources[res.Name] = res + } + } + return Resources[name] +} // ResourceConstructors returns a list of calls that can create a resource of the given kind. func ResourceConstructors(name string) []*Call { @@ -438,8 +517,9 @@ func ResourceConstructors(name string) []*Call { } func initResources() { - for name, res := range Resources { - ctors[name] = resourceCtors(res.Kind, false) + resource("") // init resources, if it's not done yet + for _, res := range resourceArray { + ctors[res.Name] = resourceCtors(res.Kind, false) } } @@ -611,16 +691,12 @@ func ForeachType(meta *Call, f func(Type)) { } } -var ( - Calls []*Call - CallMap = make(map[string]*Call) -) - func init() { - initCalls() initStructFields() initResources() initAlign() + keyedStructs = nil + structs = nil for i, c := range Calls { c.ID = i diff --git a/sysgen/sysgen.go b/sysgen/sysgen.go index 12bcf26f7..b9212cbba 100644 --- a/sysgen/sysgen.go +++ b/sysgen/sysgen.go @@ -147,7 +147,7 @@ func generate(arch string, desc *Description, consts map[string]uint64, out io.W generateResources(desc, consts, out) generateStructs(desc, consts, out) - fmt.Fprintf(out, "func initCalls() {\n") + fmt.Fprintf(out, "var Calls = []*Call{\n") for _, s := range desc.Syscalls { logf(4, " generate population code for %v", s.Name) skipCurrentSyscall = "" @@ -160,7 +160,7 @@ func generate(arch string, desc *Description, consts map[string]uint64, out io.W logf(0, "unsupported syscall: %v", s.CallName) } } - fmt.Fprintf(out, "func() { Calls = append(Calls, &Call{Name: \"%v\", CallName: \"%v\"", s.Name, s.CallName) + fmt.Fprintf(out, "&Call{Name: \"%v\", CallName: \"%v\"", s.Name, s.CallName) if len(s.Ret) != 0 { fmt.Fprintf(out, ", Ret: ") generateArg("", "ret", s.Ret[0], "out", s.Ret[1:], desc, consts, true, false, out) @@ -177,7 +177,7 @@ func generate(arch string, desc *Description, consts map[string]uint64, out io.W logf(0, "unsupported syscall: %v due to %v", s.Name, skipCurrentSyscall) syscallNR = -1 } - fmt.Fprintf(out, "}, NR: %v})}()\n", syscallNR) + fmt.Fprintf(out, "}, NR: %v},\n", syscallNR) } fmt.Fprintf(out, "}\n\n") @@ -201,7 +201,7 @@ func generateResources(desc *Description, consts map[string]uint64, out io.Write } sort.Sort(resArray) - fmt.Fprintf(out, "var Resources = map[string]*ResourceDesc{\n") + fmt.Fprintf(out, "var resourceArray = []*ResourceDesc{\n") for _, res := range resArray { underlying := "" name := res.Name @@ -230,7 +230,7 @@ func generateResources(desc *Description, consts map[string]uint64, out io.Write res = desc.Resources[res.Base] } } - fmt.Fprintf(out, "\"%v\": &ResourceDesc{Name: \"%v\", Type: ", name, name) + fmt.Fprintf(out, "&ResourceDesc{Name: \"%v\", Type: ", name) generateArg("", "resource-type", underlying, "inout", nil, desc, consts, true, true, out) fmt.Fprintf(out, ", Kind: []string{") for i, k := range kind { @@ -260,7 +260,7 @@ type structKey struct { dir string } -func generateStructEntry(str Struct, key structKey, out io.Writer) { +func generateStructEntry(str Struct, out io.Writer) { typ := "StructType" if str.IsUnion { typ = "UnionType" @@ -277,36 +277,29 @@ func generateStructEntry(str Struct, key structKey, out io.Writer) { if str.Align != 0 { align = fmt.Sprintf(", align: %v", str.Align) } - fmt.Fprintf(out, "\"%v\": &%v{TypeCommon: TypeCommon{TypeName: \"%v\", FldName: \"%v\", ArgDir: %v, IsOptional: %v} %v %v %v},\n", - key, typ, key.name, key.field, fmtDir(key.dir), false, packed, align, varlen) + fmt.Fprintf(out, "&%v{TypeCommon: TypeCommon{TypeName: \"%v\", IsOptional: %v} %v %v %v},\n", + typ, str.Name, false, packed, align, varlen) } func generateStructFields(str Struct, key structKey, desc *Description, consts map[string]uint64, out io.Writer) { - typ := "StructType" - fields := "Fields" - if str.IsUnion { - typ = "UnionType" - fields = "Options" - } - fmt.Fprintf(out, "func() { s := Structs[\"%v\"].(*%v)\n", key, typ) + fmt.Fprintf(out, "{structKey{\"%v\", \"%v\", %v}, []Type{\n", key.name, key.field, fmtDir(key.dir)) for _, a := range str.Flds { - fmt.Fprintf(out, "s.%v = append(s.%v, ", fields, fields) generateArg(str.Name, a[0], a[1], key.dir, a[2:], desc, consts, false, true, out) - fmt.Fprintf(out, ")\n") + fmt.Fprintf(out, ",\n") } - fmt.Fprintf(out, "}()\n") + fmt.Fprintf(out, "}},\n") + } func generateStructs(desc *Description, consts map[string]uint64, out io.Writer) { // Struct fields can refer to other structs. Go compiler won't like if - // we refer to Structs map during Structs map initialization. So we do - // it in 2 passes: on the first pass create types and assign them to - // the map, on the second pass fill in fields. + // we refer to Structs during Structs initialization. So we do + // it in 2 passes: on the first pass create struct types without fields, + // on the second pass we fill in fields. // Since structs of the same type can be fields with different names // of multiple other structs, we have an instance of those structs - // for each field indexed by the name of the parent struct and the - // field name. + // for each field indexed by the name of the parent struct, field name and dir. structMap := make(map[structKey]Struct) for _, str := range desc.Structs { @@ -322,13 +315,13 @@ func generateStructs(desc *Description, consts map[string]uint64, out io.Writer) } } - fmt.Fprintf(out, "var Structs = map[string]Type{\n") - for key, str := range structMap { - generateStructEntry(str, key, out) + fmt.Fprintf(out, "var structArray = []Type{\n") + for _, str := range desc.Structs { + generateStructEntry(str, out) } fmt.Fprintf(out, "}\n") - fmt.Fprintf(out, "func initStructFields() {\n") + fmt.Fprintf(out, "var structFields = []struct{key structKey; fields []Type}{") for key, str := range structMap { generateStructFields(str, key, desc, consts, out) } @@ -692,12 +685,12 @@ func generateArg( if len(a) != 0 { failf("struct '%v' has args", typ) } - fmt.Fprintf(out, "Structs[\"%v\"]", structKey{typ, origName, dir}) + fmt.Fprintf(out, "getStruct(structKey{\"%v\", \"%v\", %v})", typ, origName, fmtDir(dir)) } else if _, ok := desc.Resources[typ]; ok { if len(a) != 0 { failf("resource '%v' has args", typ) } - fmt.Fprintf(out, "&ResourceType{%v, Desc: Resources[\"%v\"]}", common(), typ) + fmt.Fprintf(out, "&ResourceType{%v, Desc: resource(\"%v\")}", common(), typ) return } else { failf("unknown arg type \"%v\" for %v", typ, name) -- cgit mrf-deployment