| 1 | // Copyright 2021 The Go Authors. All rights reserved. |
|---|---|
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package main |
| 6 | |
| 7 | import ( |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "runtime" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | |
| 16 | "golang.org/x/tools/internal/testenv" |
| 17 | ) |
| 18 | |
| 19 | func canRace(t *testing.T) bool { |
| 20 | _, err := exec.Command("go", "run", "-race", "./testdata/himom.go").CombinedOutput() |
| 21 | return err == nil |
| 22 | } |
| 23 | |
| 24 | // buildRunner builds the fuzz-runner executable, returning its path. |
| 25 | func buildRunner(t *testing.T) string { |
| 26 | bindir := filepath.Join(t.TempDir(), "bin") |
| 27 | err := os.Mkdir(bindir, os.ModePerm) |
| 28 | if err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | binary := filepath.Join(bindir, "runner") |
| 32 | if runtime.GOOS == "windows" { |
| 33 | binary += ".exe" |
| 34 | } |
| 35 | cmd := exec.Command("go", "build", "-o", binary) |
| 36 | if err := cmd.Run(); err != nil { |
| 37 | t.Fatalf("Building fuzz-runner: %v", err) |
| 38 | } |
| 39 | return binary |
| 40 | } |
| 41 | |
| 42 | // TestRunner builds the binary, then kicks off a collection of sub-tests that invoke it. |
| 43 | func TestRunner(t *testing.T) { |
| 44 | testenv.NeedsTool(t, "go") |
| 45 | if runtime.GOOS == "android" { |
| 46 | t.Skipf("the dependencies are not available on android") |
| 47 | } |
| 48 | binaryPath := buildRunner(t) |
| 49 | |
| 50 | // Sub-tests using the binary built above. |
| 51 | t.Run("Basic", func(t *testing.T) { testBasic(t, binaryPath) }) |
| 52 | t.Run("Race", func(t *testing.T) { testRace(t, binaryPath) }) |
| 53 | t.Run("Minimization1", func(t *testing.T) { testMinimization1(t, binaryPath) }) |
| 54 | t.Run("Minimization2", func(t *testing.T) { testMinimization2(t, binaryPath) }) |
| 55 | } |
| 56 | |
| 57 | func testBasic(t *testing.T, binaryPath string) { |
| 58 | t.Parallel() |
| 59 | args := []string{"-numit=1", "-numfcns=1", "-numpkgs=1", "-seed=103", "-cleancache=0"} |
| 60 | c := exec.Command(binaryPath, args...) |
| 61 | b, err := c.CombinedOutput() |
| 62 | t.Logf("%s\n", b) |
| 63 | if err != nil { |
| 64 | t.Fatalf("error invoking fuzz-runner: %v", err) |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | func testRace(t *testing.T, binaryPath string) { |
| 69 | t.Parallel() |
| 70 | // For this test to work, the current test platform has to support the |
| 71 | // race detector. Check to see if that is the case by running a very |
| 72 | // simple Go program through it. |
| 73 | if !canRace(t) { |
| 74 | t.Skip("current platform does not appear to support the race detector") |
| 75 | } |
| 76 | |
| 77 | args := []string{"-v=1", "-numit=1", "-race", "-numfcns=3", "-numpkgs=3", "-seed=987", "-cleancache=0"} |
| 78 | c := exec.Command(binaryPath, args...) |
| 79 | b, err := c.CombinedOutput() |
| 80 | t.Logf("%s\n", b) |
| 81 | if err != nil { |
| 82 | t.Fatalf("error invoking fuzz-runner: %v", err) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func testMinimization1(t *testing.T, binaryPath string) { |
| 87 | if binaryPath == "" { |
| 88 | t.Skipf("No runner binary") |
| 89 | } |
| 90 | t.Parallel() |
| 91 | // Fire off the runner passing it -emitbad=1, so that the generated code |
| 92 | // contains illegal Go code (which will force the build to fail). Verify that |
| 93 | // it does fail, that the error reflects the nature of the failure, and that |
| 94 | // we can minimize the error down to a single package. |
| 95 | args := []string{"-emitbad=1", "-badfcnidx=2", "-badpkgidx=2", |
| 96 | "-forcetmpclean", "-cleancache=0", |
| 97 | "-numit=1", "-numfcns=3", "-numpkgs=3", "-seed=909"} |
| 98 | invocation := fmt.Sprintf("%s %v", binaryPath, args) |
| 99 | c := exec.Command(binaryPath, args...) |
| 100 | b, err := c.CombinedOutput() |
| 101 | t.Logf("%s\n", b) |
| 102 | if err == nil { |
| 103 | t.Fatalf("unexpected pass of fuzz-runner (invocation %q): %v", invocation, err) |
| 104 | } |
| 105 | result := string(b) |
| 106 | if !strings.Contains(result, "syntax error") { |
| 107 | t.Fatalf("-emitbad=1 did not trigger syntax error (invocation %q): output: %s", invocation, result) |
| 108 | } |
| 109 | if !strings.Contains(result, "package minimization succeeded: found bad pkg 2") { |
| 110 | t.Fatalf("failed to minimize package (invocation %q): output: %s", invocation, result) |
| 111 | } |
| 112 | if !strings.Contains(result, "function minimization succeeded: found bad fcn 2") { |
| 113 | t.Fatalf("failed to minimize package (invocation %q): output: %s", invocation, result) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | func testMinimization2(t *testing.T, binaryPath string) { |
| 118 | if binaryPath == "" { |
| 119 | t.Skipf("No runner binary") |
| 120 | } |
| 121 | t.Parallel() |
| 122 | // Fire off the runner passing it -emitbad=2, so that the |
| 123 | // generated code forces a runtime error. Verify that it does |
| 124 | // fail, and that the error is reflective. |
| 125 | args := []string{"-emitbad=2", "-badfcnidx=1", "-badpkgidx=1", |
| 126 | "-forcetmpclean", "-cleancache=0", |
| 127 | "-numit=1", "-numfcns=3", "-numpkgs=3", "-seed=55909"} |
| 128 | invocation := fmt.Sprintf("%s %v", binaryPath, args) |
| 129 | c := exec.Command(binaryPath, args...) |
| 130 | b, err := c.CombinedOutput() |
| 131 | t.Logf("%s\n", b) |
| 132 | if err == nil { |
| 133 | t.Fatalf("unexpected pass of fuzz-runner (invocation %q): %v", invocation, err) |
| 134 | } |
| 135 | result := string(b) |
| 136 | if !strings.Contains(result, "Error: fail") || !strings.Contains(result, "Checker1.Test1") { |
| 137 | t.Fatalf("-emitbad=2 did not trigger runtime error (invocation %q): output: %s", invocation, result) |
| 138 | } |
| 139 | if !strings.Contains(result, "package minimization succeeded: found bad pkg 1") { |
| 140 | t.Fatalf("failed to minimize package (invocation %q): output: %s", invocation, result) |
| 141 | } |
| 142 | if !strings.Contains(result, "function minimization succeeded: found bad fcn 1") { |
| 143 | t.Fatalf("failed to minimize package (invocation %q): output: %s", invocation, result) |
| 144 | } |
| 145 | } |
| 146 |
Members