| 1 | // Copyright 2021 The Go Authors. All rights reserved. |
|---|---|
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | // Program for performing test runs using "fuzz-driver". |
| 6 | // Main loop iteratively runs "fuzz-driver" to create a corpus, |
| 7 | // then builds and runs the code. If a failure in the run is |
| 8 | // detected, then a testcase minimization phase kicks in. |
| 9 | |
| 10 | package main |
| 11 | |
| 12 | import ( |
| 13 | "flag" |
| 14 | "fmt" |
| 15 | "io/ioutil" |
| 16 | "log" |
| 17 | "os" |
| 18 | "os/exec" |
| 19 | "path/filepath" |
| 20 | "runtime" |
| 21 | "strconv" |
| 22 | "strings" |
| 23 | "time" |
| 24 | |
| 25 | generator "golang.org/x/tools/cmd/signature-fuzzer/internal/fuzz-generator" |
| 26 | ) |
| 27 | |
| 28 | const pkName = "fzTest" |
| 29 | |
| 30 | // Basic options |
| 31 | var verbflag = flag.Int("v", 0, "Verbose trace output level") |
| 32 | var loopitflag = flag.Int("numit", 10, "Number of main loop iterations to run") |
| 33 | var seedflag = flag.Int64("seed", -1, "Random seed") |
| 34 | var execflag = flag.Bool("execdriver", false, "Exec fuzz-driver binary instead of invoking generator directly") |
| 35 | var numpkgsflag = flag.Int("numpkgs", 50, "Number of test packages") |
| 36 | var numfcnsflag = flag.Int("numfcns", 20, "Number of test functions per package.") |
| 37 | |
| 38 | // Debugging/testing options. These tell the generator to emit "bad" code so as to |
| 39 | // test the logic for detecting errors and/or minimization. |
| 40 | var emitbadflag = flag.Int("emitbad", -1, "[Testing only] force generator to emit 'bad' code.") |
| 41 | var selbadpkgflag = flag.Int("badpkgidx", 0, "[Testing only] select index of bad package (used with -emitbad)") |
| 42 | var selbadfcnflag = flag.Int("badfcnidx", 0, "[Testing only] select index of bad function (used with -emitbad)") |
| 43 | var forcetmpcleanflag = flag.Bool("forcetmpclean", false, "[Testing only] force cleanup of temp dir") |
| 44 | var cleancacheflag = flag.Bool("cleancache", true, "[Testing only] don't clean the go cache") |
| 45 | var raceflag = flag.Bool("race", false, "[Testing only] build generated code with -race") |
| 46 | |
| 47 | func verb(vlevel int, s string, a ...interface{}) { |
| 48 | if *verbflag >= vlevel { |
| 49 | fmt.Printf(s, a...) |
| 50 | fmt.Printf("\n") |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | func warn(s string, a ...interface{}) { |
| 55 | fmt.Fprintf(os.Stderr, s, a...) |
| 56 | fmt.Fprintf(os.Stderr, "\n") |
| 57 | } |
| 58 | |
| 59 | func fatal(s string, a ...interface{}) { |
| 60 | fmt.Fprintf(os.Stderr, s, a...) |
| 61 | fmt.Fprintf(os.Stderr, "\n") |
| 62 | os.Exit(1) |
| 63 | } |
| 64 | |
| 65 | type config struct { |
| 66 | generator.GenConfig |
| 67 | tmpdir string |
| 68 | gendir string |
| 69 | buildOutFile string |
| 70 | runOutFile string |
| 71 | gcflags string |
| 72 | nerrors int |
| 73 | } |
| 74 | |
| 75 | func usage(msg string) { |
| 76 | if len(msg) > 0 { |
| 77 | fmt.Fprintf(os.Stderr, "error: %s\n", msg) |
| 78 | } |
| 79 | fmt.Fprintf(os.Stderr, "usage: fuzz-runner [flags]\n\n") |
| 80 | flag.PrintDefaults() |
| 81 | fmt.Fprintf(os.Stderr, "Example:\n\n") |
| 82 | fmt.Fprintf(os.Stderr, " fuzz-runner -numit=500 -numpkgs=11 -numfcns=13 -seed=10101\n\n") |
| 83 | fmt.Fprintf(os.Stderr, " \tRuns 500 rounds of test case generation\n") |
| 84 | fmt.Fprintf(os.Stderr, " \tusing random see 10101, in each round emitting\n") |
| 85 | fmt.Fprintf(os.Stderr, " \t11 packages each with 13 function pairs.\n") |
| 86 | |
| 87 | os.Exit(2) |
| 88 | } |
| 89 | |
| 90 | // docmd executes the specified command in the dir given and pipes the |
| 91 | // output to stderr. return status is 0 if command passed, 1 |
| 92 | // otherwise. |
| 93 | func docmd(cmd []string, dir string) int { |
| 94 | verb(2, "docmd: %s", strings.Join(cmd, " ")) |
| 95 | c := exec.Command(cmd[0], cmd[1:]...) |
| 96 | if dir != "" { |
| 97 | c.Dir = dir |
| 98 | } |
| 99 | b, err := c.CombinedOutput() |
| 100 | st := 0 |
| 101 | if err != nil { |
| 102 | warn("error executing cmd %s: %v", |
| 103 | strings.Join(cmd, " "), err) |
| 104 | st = 1 |
| 105 | } |
| 106 | os.Stderr.Write(b) |
| 107 | return st |
| 108 | } |
| 109 | |
| 110 | // docodmout forks and execs command 'cmd' in dir 'dir', redirecting |
| 111 | // stderr and stdout from the execution to file 'outfile'. |
| 112 | func docmdout(cmd []string, dir string, outfile string) int { |
| 113 | of, err := os.OpenFile(outfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) |
| 114 | if err != nil { |
| 115 | fatal("opening outputfile %s: %v", outfile, err) |
| 116 | } |
| 117 | c := exec.Command(cmd[0], cmd[1:]...) |
| 118 | defer of.Close() |
| 119 | if dir != "" { |
| 120 | verb(2, "setting cmd.Dir to %s", dir) |
| 121 | c.Dir = dir |
| 122 | } |
| 123 | verb(2, "docmdout: %s > %s", strings.Join(cmd, " "), outfile) |
| 124 | c.Stdout = of |
| 125 | c.Stderr = of |
| 126 | err = c.Run() |
| 127 | st := 0 |
| 128 | if err != nil { |
| 129 | warn("error executing cmd %s: %v", |
| 130 | strings.Join(cmd, " "), err) |
| 131 | st = 1 |
| 132 | } |
| 133 | return st |
| 134 | } |
| 135 | |
| 136 | // gen is the main hook for kicking off code generation. For |
| 137 | // non-minimization runs, 'singlepk' and 'singlefn' will both be -1 |
| 138 | // (indicating that we want all functions and packages to be |
| 139 | // generated). If 'singlepk' is set to a non-negative value, then |
| 140 | // code generation will be restricted to the single package with that |
| 141 | // index (as a try at minimization), similarly with 'singlefn' |
| 142 | // restricting the codegen to a single specified function. |
| 143 | func (c *config) gen(singlepk int, singlefn int) { |
| 144 | |
| 145 | // clean the output dir |
| 146 | verb(2, "cleaning outdir %s", c.gendir) |
| 147 | if err := os.RemoveAll(c.gendir); err != nil { |
| 148 | fatal("error cleaning gen dir %s: %v", c.gendir, err) |
| 149 | } |
| 150 | |
| 151 | // emit code into the output dir. Here we either invoke the |
| 152 | // generator directly, or invoke fuzz-driver if -execflag is |
| 153 | // set. If the code generation process itself fails, this is |
| 154 | // typically a bug in the fuzzer itself, so it gets reported |
| 155 | // as a fatal error. |
| 156 | if *execflag { |
| 157 | args := []string{"fuzz-driver", |
| 158 | "-numpkgs", strconv.Itoa(c.NumTestPackages), |
| 159 | "-numfcns", strconv.Itoa(c.NumTestFunctions), |
| 160 | "-seed", strconv.Itoa(int(c.Seed)), |
| 161 | "-outdir", c.OutDir, |
| 162 | "-pkgpath", pkName, |
| 163 | "-maxfail", strconv.Itoa(c.MaxFail)} |
| 164 | if singlepk != -1 { |
| 165 | args = append(args, "-pkgmask", strconv.Itoa(singlepk)) |
| 166 | } |
| 167 | if singlefn != -1 { |
| 168 | args = append(args, "-fcnmask", strconv.Itoa(singlefn)) |
| 169 | } |
| 170 | if *emitbadflag != 0 { |
| 171 | args = append(args, "-emitbad", strconv.Itoa(*emitbadflag), |
| 172 | "-badpkgidx", strconv.Itoa(*selbadpkgflag), |
| 173 | "-badfcnidx", strconv.Itoa(*selbadfcnflag)) |
| 174 | } |
| 175 | verb(1, "invoking fuzz-driver with args: %v", args) |
| 176 | st := docmd(args, "") |
| 177 | if st != 0 { |
| 178 | fatal("fatal error: generation failed, cmd was: %v", args) |
| 179 | } |
| 180 | } else { |
| 181 | if singlepk != -1 { |
| 182 | c.PkgMask = map[int]int{singlepk: 1} |
| 183 | } |
| 184 | if singlefn != -1 { |
| 185 | c.FcnMask = map[int]int{singlefn: 1} |
| 186 | } |
| 187 | verb(1, "invoking generator.Generate with config: %v", c.GenConfig) |
| 188 | errs := generator.Generate(c.GenConfig) |
| 189 | if errs != 0 { |
| 190 | log.Fatal("errors during generation") |
| 191 | } |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // action performs a selected action/command in the generated code dir. |
| 196 | func (c *config) action(cmd []string, outfile string, emitout bool) int { |
| 197 | st := docmdout(cmd, c.gendir, outfile) |
| 198 | if emitout { |
| 199 | content, err := ioutil.ReadFile(outfile) |
| 200 | if err != nil { |
| 201 | log.Fatal(err) |
| 202 | } |
| 203 | fmt.Fprintf(os.Stderr, "%s", content) |
| 204 | } |
| 205 | return st |
| 206 | } |
| 207 | |
| 208 | func binaryName() string { |
| 209 | if runtime.GOOS == "windows" { |
| 210 | return pkName + ".exe" |
| 211 | } else { |
| 212 | return "./" + pkName |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | // build builds a generated corpus of Go code. If 'emitout' is set, then dump out the |
| 217 | // results of the build after it completes (during minimization emitout is set to false, |
| 218 | // since there is no need to see repeated errors). |
| 219 | func (c *config) build(emitout bool) int { |
| 220 | // Issue a build of the generated code. |
| 221 | c.buildOutFile = filepath.Join(c.tmpdir, "build.err.txt") |
| 222 | cmd := []string{"go", "build", "-o", binaryName()} |
| 223 | if c.gcflags != "" { |
| 224 | cmd = append(cmd, "-gcflags=all="+c.gcflags) |
| 225 | } |
| 226 | if *raceflag { |
| 227 | cmd = append(cmd, "-race") |
| 228 | } |
| 229 | cmd = append(cmd, ".") |
| 230 | verb(1, "build command is: %v", cmd) |
| 231 | return c.action(cmd, c.buildOutFile, emitout) |
| 232 | } |
| 233 | |
| 234 | // run invokes a binary built from a generated corpus of Go code. If |
| 235 | // 'emitout' is set, then dump out the results of the run after it |
| 236 | // completes. |
| 237 | func (c *config) run(emitout bool) int { |
| 238 | // Issue a run of the generated code. |
| 239 | c.runOutFile = filepath.Join(c.tmpdir, "run.err.txt") |
| 240 | cmd := []string{filepath.Join(c.gendir, binaryName())} |
| 241 | verb(1, "run command is: %v", cmd) |
| 242 | return c.action(cmd, c.runOutFile, emitout) |
| 243 | } |
| 244 | |
| 245 | type minimizeMode int |
| 246 | |
| 247 | const ( |
| 248 | minimizeBuildFailure = iota |
| 249 | minimizeRuntimeFailure |
| 250 | ) |
| 251 | |
| 252 | // minimize tries to minimize a failing scenario down to a single |
| 253 | // package and/or function if possible. This is done using an |
| 254 | // iterative search. Here 'minimizeMode' tells us whether we're |
| 255 | // looking for a compile-time error or a runtime error. |
| 256 | func (c *config) minimize(mode minimizeMode) int { |
| 257 | |
| 258 | verb(0, "... starting minimization for failed directory %s", c.gendir) |
| 259 | |
| 260 | foundPkg := -1 |
| 261 | foundFcn := -1 |
| 262 | |
| 263 | // Locate bad package. Uses brute-force linear search, could do better... |
| 264 | for pidx := 0; pidx < c.NumTestPackages; pidx++ { |
| 265 | verb(1, "minimization: trying package %d", pidx) |
| 266 | c.gen(pidx, -1) |
| 267 | st := c.build(false) |
| 268 | if mode == minimizeBuildFailure { |
| 269 | if st != 0 { |
| 270 | // Found. |
| 271 | foundPkg = pidx |
| 272 | c.nerrors++ |
| 273 | break |
| 274 | } |
| 275 | } else { |
| 276 | if st != 0 { |
| 277 | warn("run minimization: unexpected build failed while searching for bad pkg") |
| 278 | return 1 |
| 279 | } |
| 280 | st := c.run(false) |
| 281 | if st != 0 { |
| 282 | // Found. |
| 283 | c.nerrors++ |
| 284 | verb(1, "run minimization found bad package: %d", pidx) |
| 285 | foundPkg = pidx |
| 286 | break |
| 287 | } |
| 288 | } |
| 289 | } |
| 290 | if foundPkg == -1 { |
| 291 | verb(0, "** minimization failed, could not locate bad package") |
| 292 | return 1 |
| 293 | } |
| 294 | warn("package minimization succeeded: found bad pkg %d", foundPkg) |
| 295 | |
| 296 | // clean unused packages |
| 297 | for pidx := 0; pidx < c.NumTestPackages; pidx++ { |
| 298 | if pidx != foundPkg { |
| 299 | chp := filepath.Join(c.gendir, fmt.Sprintf("%s%s%d", c.Tag, generator.CheckerName, pidx)) |
| 300 | if err := os.RemoveAll(chp); err != nil { |
| 301 | fatal("failed to clean pkg subdir %s: %v", chp, err) |
| 302 | } |
| 303 | clp := filepath.Join(c.gendir, fmt.Sprintf("%s%s%d", c.Tag, generator.CallerName, pidx)) |
| 304 | if err := os.RemoveAll(clp); err != nil { |
| 305 | fatal("failed to clean pkg subdir %s: %v", clp, err) |
| 306 | } |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | // Locate bad function. Again, brute force. |
| 311 | for fidx := 0; fidx < c.NumTestFunctions; fidx++ { |
| 312 | c.gen(foundPkg, fidx) |
| 313 | st := c.build(false) |
| 314 | if mode == minimizeBuildFailure { |
| 315 | if st != 0 { |
| 316 | // Found. |
| 317 | verb(1, "build minimization found bad function: %d", fidx) |
| 318 | foundFcn = fidx |
| 319 | break |
| 320 | } |
| 321 | } else { |
| 322 | if st != 0 { |
| 323 | warn("run minimization: unexpected build failed while searching for bad fcn") |
| 324 | return 1 |
| 325 | } |
| 326 | st := c.run(false) |
| 327 | if st != 0 { |
| 328 | // Found. |
| 329 | verb(1, "run minimization found bad function: %d", fidx) |
| 330 | foundFcn = fidx |
| 331 | break |
| 332 | } |
| 333 | } |
| 334 | // not the function we want ... continue the hunt |
| 335 | } |
| 336 | if foundFcn == -1 { |
| 337 | verb(0, "** function minimization failed, could not locate bad function") |
| 338 | return 1 |
| 339 | } |
| 340 | warn("function minimization succeeded: found bad fcn %d", foundFcn) |
| 341 | |
| 342 | return 0 |
| 343 | } |
| 344 | |
| 345 | // cleanTemp removes the temp dir we've been working with. |
| 346 | func (c *config) cleanTemp() { |
| 347 | if !*forcetmpcleanflag { |
| 348 | if c.nerrors != 0 { |
| 349 | verb(1, "preserving temp dir %s", c.tmpdir) |
| 350 | return |
| 351 | } |
| 352 | } |
| 353 | verb(1, "cleaning temp dir %s", c.tmpdir) |
| 354 | os.RemoveAll(c.tmpdir) |
| 355 | } |
| 356 | |
| 357 | // perform is the top level driver routine for the program, containing the |
| 358 | // main loop. Each iteration of the loop performs a generate/build/run |
| 359 | // sequence, and then updates the seed afterwards if no failure is found. |
| 360 | // If a failure is detected, we try to minimize it and then return without |
| 361 | // attempting any additional tests. |
| 362 | func (c *config) perform() int { |
| 363 | defer c.cleanTemp() |
| 364 | |
| 365 | // Main loop |
| 366 | for iter := 0; iter < *loopitflag; iter++ { |
| 367 | if iter != 0 && iter%50 == 0 { |
| 368 | // Note: cleaning the Go cache periodically is |
| 369 | // pretty much a requirement if you want to do |
| 370 | // things like overnight runs of the fuzzer, |
| 371 | // but it is also a very unfriendly thing do |
| 372 | // to if we're executing as part of a unit |
| 373 | // test run (in which case there may be other |
| 374 | // tests running in parallel with this |
| 375 | // one). Check the "cleancache" flag before |
| 376 | // doing this. |
| 377 | if *cleancacheflag { |
| 378 | docmd([]string{"go", "clean", "-cache"}, "") |
| 379 | } |
| 380 | } |
| 381 | verb(0, "... begin iteration %d with current seed %d", iter, c.Seed) |
| 382 | c.gen(-1, -1) |
| 383 | st := c.build(true) |
| 384 | if st != 0 { |
| 385 | c.minimize(minimizeBuildFailure) |
| 386 | return 1 |
| 387 | } |
| 388 | st = c.run(true) |
| 389 | if st != 0 { |
| 390 | c.minimize(minimizeRuntimeFailure) |
| 391 | return 1 |
| 392 | } |
| 393 | // update seed so that we get different code on the next iter. |
| 394 | c.Seed += 101 |
| 395 | } |
| 396 | return 0 |
| 397 | } |
| 398 | |
| 399 | func main() { |
| 400 | log.SetFlags(0) |
| 401 | log.SetPrefix("fuzz-runner: ") |
| 402 | flag.Parse() |
| 403 | if flag.NArg() != 0 { |
| 404 | usage("unknown extra arguments") |
| 405 | } |
| 406 | verb(1, "in main, verblevel=%d", *verbflag) |
| 407 | |
| 408 | tmpdir, err := ioutil.TempDir("", "fuzzrun") |
| 409 | if err != nil { |
| 410 | fatal("creation of tempdir failed: %v", err) |
| 411 | } |
| 412 | gendir := filepath.Join(tmpdir, "fuzzTest") |
| 413 | |
| 414 | // select starting seed |
| 415 | if *seedflag == -1 { |
| 416 | now := time.Now() |
| 417 | *seedflag = now.UnixNano() % 123456789 |
| 418 | } |
| 419 | |
| 420 | // set up params for this run |
| 421 | c := &config{ |
| 422 | GenConfig: generator.GenConfig{ |
| 423 | NumTestPackages: *numpkgsflag, // 100 |
| 424 | NumTestFunctions: *numfcnsflag, // 20 |
| 425 | Seed: *seedflag, |
| 426 | OutDir: gendir, |
| 427 | Pragma: "-maxfail=9999", |
| 428 | PkgPath: pkName, |
| 429 | EmitBad: *emitbadflag, |
| 430 | BadPackageIdx: *selbadpkgflag, |
| 431 | BadFuncIdx: *selbadfcnflag, |
| 432 | }, |
| 433 | tmpdir: tmpdir, |
| 434 | gendir: gendir, |
| 435 | } |
| 436 | |
| 437 | // kick off the main loop. |
| 438 | st := c.perform() |
| 439 | |
| 440 | // done |
| 441 | verb(1, "leaving main, num errors=%d", c.nerrors) |
| 442 | os.Exit(st) |
| 443 | } |
| 444 |
Members