From c6f13480f3e562921f700b3e4f878ca5d54d958f Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 21 Dec 2021 13:57:26 +0100 Subject: .github/workflows: split work across more jobs CI become quite slow, especially the 'arch' job. Split it into 3 separate jobs. Also split main build/lint/test from all of the aux checks. Update #2886 --- .github/workflows/ci.yml | 25 +++++++++++++++++++++---- Makefile | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97b993a42..6da8258e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ name: ci on: [push, pull_request] jobs: - smoke: + aux: runs-on: ubuntu-latest steps: # Checks out syzkaller repo at the path. @@ -25,13 +25,27 @@ jobs: with: path: .cache key: cache - # Run make presubmit_smoke. + # Run make presubmit_aux. - name: run env: GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} GITHUB_PR_COMMITS: ${{ github.event.pull_request.commits }} - run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env make presubmit_smoke + run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env make presubmit_aux + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + with: + path: gopath/src/github.com/google/syzkaller + - name: cache + uses: actions/cache@v1 + with: + path: .cache + key: cache + - name: run + run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env make presubmit_build # Upload coverage report to codecov.io. For reference see: # https://github.com/codecov/codecov-action/blob/master/README.md - name: codecov @@ -60,6 +74,9 @@ jobs: flags: dashboard arch: runs-on: ubuntu-latest + strategy: + matrix: + target: [presubmit_arch_linux, presubmit_arch_freebsd, presubmit_arch_other, presubmit_arch_executor] steps: - name: checkout uses: actions/checkout@v2 @@ -71,7 +88,7 @@ jobs: path: .cache key: cache - name: run - run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-big-env make presubmit_arch + run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-big-env make ${{ matrix.target }} race: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 37cbbb17a..4f4dc0e1a 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,8 @@ endif format format_go format_cpp format_sys \ tidy test test_race \ check_copyright check_language check_whitespace check_links check_diff check_commits check_shebang \ - presubmit presubmit_smoke presubmit_build presubmit_arch presubmit_big presubmit_race presubmit_old + presubmit presubmit_aux presubmit_build presubmit_arch_linux presubmit_arch_freebsd presubmit_arch_other \ + presubmit_arch_executor presubmit_big presubmit_race presubmit_old all: host target host: manager runtest repro mutate prog2c db upgrade @@ -281,29 +282,28 @@ lint: bin/golangci-lint run ./... presubmit: - $(MAKE) presubmit_smoke - $(MAKE) presubmit_arch + $(MAKE) presubmit_aux + $(MAKE) presubmit_build + $(MAKE) presubmit_arch_linux + $(MAKE) presubmit_arch_freebsd + $(MAKE) presubmit_arch_other + $(MAKE) presubmit_arch_executor $(MAKE) presubmit_race -presubmit_smoke: +presubmit_aux: $(MAKE) generate - $(MAKE) -j100 check_commits check_diff check_copyright check_language check_whitespace check_links check_shebang presubmit_build tidy - $(MAKE) test + $(MAKE) -j100 check_commits check_diff check_copyright check_language check_whitespace check_links check_shebang tidy -presubmit_build: +presubmit_build: descriptions # Run go build before lint for better error messages if build is broken. # This does not check build of test files, but running go test takes too long (even for building). $(GO) build ./... $(MAKE) lint + $(MAKE) test -presubmit_arch: descriptions +presubmit_arch_linux: descriptions env HOSTOS=linux HOSTARCH=amd64 $(MAKE) host - env HOSTOS=freebsd HOSTARCH=amd64 $(MAKE) host - env HOSTOS=netbsd HOSTARCH=amd64 $(MAKE) host - env HOSTOS=openbsd HOSTARCH=amd64 $(MAKE) host - env HOSTOS=darwin HOSTARCH=amd64 $(MAKE) host env TARGETOS=linux TARGETARCH=amd64 $(MAKE) target - env TARGETOS=linux TARGETARCH=amd64 SYZ_CLANG=yes $(MAKE) executor env TARGETOS=linux TARGETARCH=386 $(MAKE) target env TARGETOS=linux TARGETARCH=arm64 $(MAKE) target env TARGETOS=linux TARGETARCH=arm $(MAKE) target @@ -311,11 +311,22 @@ presubmit_arch: descriptions env TARGETOS=linux TARGETARCH=ppc64le $(MAKE) target env TARGETOS=linux TARGETARCH=riscv64 $(MAKE) target env TARGETOS=linux TARGETARCH=s390x $(MAKE) target + +presubmit_arch_freebsd: descriptions + env HOSTOS=freebsd HOSTARCH=amd64 $(MAKE) host env TARGETOS=freebsd TARGETARCH=amd64 $(MAKE) target env TARGETOS=freebsd TARGETARCH=386 $(MAKE) target + +presubmit_arch_other: descriptions + env HOSTOS=netbsd HOSTARCH=amd64 $(MAKE) host + env HOSTOS=openbsd HOSTARCH=amd64 $(MAKE) host + env HOSTOS=darwin HOSTARCH=amd64 $(MAKE) host env TARGETOS=netbsd TARGETARCH=amd64 $(MAKE) target env TARGETOS=openbsd TARGETARCH=amd64 $(MAKE) target env TARGETOS=windows TARGETARCH=amd64 $(MAKE) target + +presubmit_arch_executor: descriptions + env TARGETOS=linux TARGETARCH=amd64 SYZ_CLANG=yes $(MAKE) executor env TARGETOS=akaros TARGETARCH=amd64 $(MAKE) executor env TARGETOS=fuchsia TARGETARCH=amd64 $(MAKE) executor env TARGETOS=fuchsia TARGETARCH=arm64 $(MAKE) executor @@ -323,6 +334,10 @@ presubmit_arch: descriptions env TARGETOS=test TARGETARCH=64_fork $(MAKE) executor env TARGETOS=test TARGETARCH=32_shmem $(MAKE) executor env TARGETOS=test TARGETARCH=32_fork_shmem $(MAKE) executor + env TARGETOS=test TARGETARCH=64 $(MAKE) executor + env TARGETOS=test TARGETARCH=64_fork $(MAKE) executor + env TARGETOS=test TARGETARCH=32_shmem $(MAKE) executor + env TARGETOS=test TARGETARCH=32_fork_shmem $(MAKE) executor presubmit_big: descriptions # This target runs on CI in syz-big-env, -- cgit mrf-deployment