Makefile: generate descriptions on-the-fly

Checking in the generated descriptions files makes few things simpler,
but causes pain for pull requests: (1) PRs that touch descriptions
_always_ conflict, (2) PRs are large and harder to review,
(3) people sometimes forget to add auto-generated files.

The proposed way does not require us to hardcode lots of dependencies
in the Makefile (which is nice) and seem to work.
Let's see how it works.

The main contributor-visible consequence is that the auto-generated
files do not need to be checked-in now.

Credit for figuring the Makefile magic goes to @melver.

Fixes #1291

1 files changed, 36 insertions, 30 deletions

diff --git a/Makefile b/Makefile
index fa6e7583c..f7d026e86 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,7 @@ endif
 	manager runtest fuzzer executor \
 	ci hub \
 	execprog mutate prog2c trace2syz stress repro upgrade db \
-	bin/syz-sysgen bin/syz-extract bin/syz-fmt \
+	bin/syz-extract bin/syz-fmt \
 	extract generate generate_go generate_sys \
 	format format_go format_cpp format_sys \
 	tidy test test_race check_copyright check_links check_diff \
@@ -110,8 +110,7 @@ all: host target
 host: manager runtest repro mutate prog2c db upgrade
 target: fuzzer execprog stress executor
 
-# executor uses stacks of limited size, so no jumbo frames.
-executor:
+executor: descriptions
 ifneq ("$(BUILDOS)", "$(NATIVEBUILDOS)")
 	$(info ************************************************************************************)
 	$(info Executor will not be built)
@@ -132,49 +131,61 @@ else
 endif
 endif
 
-manager:
+# .descriptions is a stub file that serves as a substitute for all files generated by syz-sysgen:
+# sys/*/gen/*.go, executor/defs.h, executor/syscalls.h
+# syz-sysgen generates them all at once, so we can't make each of them an independent target.
+.PHONY: descriptions
+descriptions:
+	export GOBIN=$(PWD)/bin; go list -f '{{.Stale}}' ./sys/syz-sysgen | grep -q false || go install ./sys/syz-sysgen
+	$(MAKE) .descriptions
+
+.descriptions: sys/*/*.txt sys/*/*.const bin/syz-sysgen
+	bin/syz-sysgen
+	touch .descriptions
+
+manager: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-manager github.com/google/syzkaller/syz-manager
 
-runtest:
+runtest: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-runtest github.com/google/syzkaller/tools/syz-runtest
 
-fuzzer:
+fuzzer: descriptions
 	GOOS=$(TARGETGOOS) GOARCH=$(TARGETGOARCH) $(GO) build $(GOTARGETFLAGS) -o ./bin/$(TARGETOS)_$(TARGETVMARCH)/syz-fuzzer$(EXE) github.com/google/syzkaller/syz-fuzzer
 
-execprog:
+execprog: descriptions
 	GOOS=$(TARGETGOOS) GOARCH=$(TARGETGOARCH) $(GO) build $(GOTARGETFLAGS) -o ./bin/$(TARGETOS)_$(TARGETVMARCH)/syz-execprog$(EXE) github.com/google/syzkaller/tools/syz-execprog
 
-ci:
+ci: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-ci github.com/google/syzkaller/syz-ci
 
-hub:
+hub: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-hub github.com/google/syzkaller/syz-hub
 
-repro:
+repro: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-repro github.com/google/syzkaller/tools/syz-repro
 
-mutate:
+mutate: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-mutate github.com/google/syzkaller/tools/syz-mutate
 
-prog2c:
+prog2c: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-prog2c github.com/google/syzkaller/tools/syz-prog2c
 
-stress:
+stress: descriptions
 	GOOS=$(TARGETGOOS) GOARCH=$(TARGETGOARCH) $(GO) build $(GOTARGETFLAGS) -o ./bin/$(TARGETOS)_$(TARGETVMARCH)/syz-stress$(EXE) github.com/google/syzkaller/tools/syz-stress
 
-db:
+db: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-db github.com/google/syzkaller/tools/syz-db
 
-upgrade:
+upgrade: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-upgrade github.com/google/syzkaller/tools/syz-upgrade
 
-trace2syz:
+trace2syz: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-trace2syz github.com/google/syzkaller/tools/syz-trace2syz
 
 usbgen:
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-usbgen github.com/google/syzkaller/tools/syz-usbgen
 
-expand:
+expand: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-expand github.com/google/syzkaller/tools/syz-expand
 
 # `extract` extracts const files from various kernel sources, and may only
@@ -186,20 +197,18 @@ ifeq ($(TARGETOS),fuchsia)
 else
 endif
 	bin/syz-extract -build -os=$(TARGETOS) -sourcedir=$(SOURCEDIR) $(FILES)
+
 bin/syz-extract:
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o $@ ./sys/syz-extract
 
 # `generate` does *not* depend on any kernel sources, and generates everything
 # in one pass, for all arches. It can be run on a bare syzkaller checkout.
-generate: generate_go generate_sys
+generate: descriptions generate_go
 	$(MAKE) format
 
-generate_go: bin/syz-sysgen format_cpp
+generate_go: format_cpp
 	$(GO) generate ./pkg/csource ./executor ./pkg/ifuzz ./pkg/build ./pkg/html
 
-generate_sys: bin/syz-sysgen
-	bin/syz-sysgen
-
 generate_fidl:
 ifeq ($(TARGETOS),fuchsia)
 	$(HOSTGO) generate ./sys/fuchsia
@@ -211,9 +220,6 @@ generate_trace2syz:
 	(cd tools/syz-trace2syz/parser; ragel -Z -G2 -o lex.go straceLex.rl)
 	(cd tools/syz-trace2syz/parser; goyacc -o strace.go -p Strace -v="" strace.y)
 
-bin/syz-sysgen:
-	$(GO) build $(GOHOSTFLAGS) -o $@ ./sys/syz-sysgen
-
 format: format_go format_cpp format_sys
 
 format_go:
@@ -303,7 +309,7 @@ arch_test:
 	env TARGETOS=test TARGETARCH=32_shmem $(MAKE) executor
 	env TARGETOS=test TARGETARCH=32_fork_shmem $(MAKE) executor
 
-presubmit:
+presubmit: descriptions
 	$(MAKE) generate
 	$(MAKE) check_diff
 	$(MAKE) check_copyright 
@@ -314,8 +320,8 @@ presubmit:
 
 presubmit_parallel: test test_race arch
 
-test:
-ifeq ("$(TRAVIS)$(shell go version | grep 1.11)", "true")
+test: descriptions
+ifeq ("$(TRAVIS)$(shell go version | grep 1.13)", "true")
 	# Collect coverage report for codecov.io when testing Go 1.12 on travis (uploaded in .travis.yml).
 	env CGO_ENABLED=1 $(GO) test -short -coverprofile=coverage.txt ./...
 else
@@ -323,13 +329,13 @@ else
 	env CGO_ENABLED=1 $(GO) test -short ./...
 endif
 
-test_race:
+test_race: descriptions
 	env CGO_ENABLED=1 $(GO) test -race; if test $$? -ne 2; then \
 	env CGO_ENABLED=1 $(GO) test -race -short -bench=.* -benchtime=.2s ./... ;\
 	fi
 
 clean:
-	rm -rf ./bin/
+	rm -rf ./bin .descriptions sys/*/gen executor/defs.h executor/syscalls.h
 
 # For a tupical Ubuntu/Debian distribution.
 # We use "|| true" for apt-get install because packages are all different on different distros,