SATestBuild.py source code [clang_source_code/utils/analyzer/SATestBuild.py]

1	#!/usr/bin/env python
2
3	"""
4	Static Analyzer qualification infrastructure.
5
6	The goal is to test the analyzer against different projects,
7	check for failures, compare results, and measure performance.
8
9	Repository Directory will contain sources of the projects as well as the
10	information on how to build them and the expected output.
11	Repository Directory structure:
12	- ProjectMap file
13	- Historical Performance Data
14	- Project Dir1
15	- ReferenceOutput
16	- Project Dir2
17	- ReferenceOutput
18	..
19	Note that the build tree must be inside the project dir.
20
21	To test the build of the analyzer one would:
22	- Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
23	the build directory does not pollute the repository to min network
24	traffic).
25	- Build all projects, until error. Produce logs to report errors.
26	- Compare results.
27
28	The files which should be kept around for failure investigations:
29	RepositoryCopy/Project DirI/ScanBuildResults
30	RepositoryCopy/Project DirI/run_static_analyzer.log
31
32	Assumptions (TODO: shouldn't need to assume these.):
33	The script is being run from the Repository Directory.
34	The compiler for scan-build and scan-build are in the PATH.
35	export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
36
37	For more logging, set the env variables:
38	zaks:TI zaks$ export CCC_ANALYZER_LOG=1
39	zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
40
41	The list of checkers tested are hardcoded in the Checkers variable.
42	For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
43	variable. It should contain a comma separated list.
44	"""
45	import CmpRuns
46	import SATestUtils
47
48	from subprocess import CalledProcessError, check_call
49	import argparse
50	import csv
51	import glob
52	import logging
53	import math
54	import multiprocessing
55	import os
56	import plistlib
57	import shutil
58	import sys
59	import threading
60	import time
61	try:
62	import queue
63	except ImportError:
64	import Queue as queue
65
66	###############################################################################
67	# Helper functions.
68	###############################################################################
69
70	Local = threading.local()
71	Local.stdout = sys.stdout
72	Local.stderr = sys.stderr
73	logging.basicConfig(
74	level=logging.DEBUG,
75	format='%(asctime)s:%(levelname)s:%(name)s: %(message)s')
76
77	class StreamToLogger(object):
78	def __init__(self, logger, log_level=logging.INFO):
79	self.logger = logger
80	self.log_level = log_level
81
82	def write(self, buf):
83	# Rstrip in order not to write an extra newline.
84	self.logger.log(self.log_level, buf.rstrip())
85
86	def flush(self):
87	pass
88
89	def fileno(self):
90	return 0
91
92
93	def getProjectMapPath():
94	ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
95	ProjectMapFile)
96	if not os.path.exists(ProjectMapPath):
97	Local.stdout.write("Error: Cannot find the Project Map file " +
98	ProjectMapPath +
99	"\nRunning script for the wrong directory?\n")
100	sys.exit(1)
101	return ProjectMapPath
102
103
104	def getProjectDir(ID):
105	return os.path.join(os.path.abspath(os.curdir), ID)
106
107
108	def getSBOutputDirName(IsReferenceBuild):
109	if IsReferenceBuild:
110	return SBOutputDirReferencePrefix + SBOutputDirName
111	else:
112	return SBOutputDirName
113
114	###############################################################################
115	# Configuration setup.
116	###############################################################################
117
118
119	# Find Clang for static analysis.
120	if 'CC' in os.environ:
121	Clang = os.environ['CC']
122	else:
123	Clang = SATestUtils.which("clang", os.environ['PATH'])
124	if not Clang:
125	print("Error: cannot find 'clang' in PATH")
126	sys.exit(1)
127
128	# Number of jobs.
129	MaxJobs = int(math.ceil(multiprocessing.cpu_count() * 0.75))
130
131	# Project map stores info about all the "registered" projects.
132	ProjectMapFile = "projectMap.csv"
133
134	# Names of the project specific scripts.
135	# The script that downloads the project.
136	DownloadScript = "download_project.sh"
137	# The script that needs to be executed before the build can start.
138	CleanupScript = "cleanup_run_static_analyzer.sh"
139	# This is a file containing commands for scan-build.
140	BuildScript = "run_static_analyzer.cmd"
141
142	# A comment in a build script which disables wrapping.
143	NoPrefixCmd = "#NOPREFIX"
144
145	# The log file name.
146	LogFolderName = "Logs"
147	BuildLogName = "run_static_analyzer.log"
148	# Summary file - contains the summary of the failures. Ex: This info can be be
149	# displayed when buildbot detects a build failure.
150	NumOfFailuresInSummary = 10
151	FailuresSummaryFileName = "failures.txt"
152
153	# The scan-build result directory.
154	SBOutputDirName = "ScanBuildResults"
155	SBOutputDirReferencePrefix = "Ref"
156
157	# The name of the directory storing the cached project source. If this
158	# directory does not exist, the download script will be executed.
159	# That script should create the "CachedSource" directory and download the
160	# project source into it.
161	CachedSourceDirName = "CachedSource"
162
163	# The name of the directory containing the source code that will be analyzed.
164	# Each time a project is analyzed, a fresh copy of its CachedSource directory
165	# will be copied to the PatchedSource directory and then the local patches
166	# in PatchfileName will be applied (if PatchfileName exists).
167	PatchedSourceDirName = "PatchedSource"
168
169	# The name of the patchfile specifying any changes that should be applied
170	# to the CachedSource before analyzing.
171	PatchfileName = "changes_for_analyzer.patch"
172
173	# The list of checkers used during analyzes.
174	# Currently, consists of all the non-experimental checkers, plus a few alpha
175	# checkers we don't want to regress on.
176	Checkers = ",".join([
177	"alpha.unix.SimpleStream",
178	"alpha.security.taint",
179	"cplusplus.NewDeleteLeaks",
180	"core",
181	"cplusplus",
182	"deadcode",
183	"security",
184	"unix",
185	"osx",
186	"nullability"
187	])
188
189	Verbose = 0
190
191	###############################################################################
192	# Test harness logic.
193	###############################################################################
194
195
196	def runCleanupScript(Dir, PBuildLogFile):
197	"""
198	Run pre-processing script if any.
199	"""
200	Cwd = os.path.join(Dir, PatchedSourceDirName)
201	ScriptPath = os.path.join(Dir, CleanupScript)
202	SATestUtils.runScript(ScriptPath, PBuildLogFile, Cwd,
203	Stdout=Local.stdout, Stderr=Local.stderr)
204
205
206	def runDownloadScript(Dir, PBuildLogFile):
207	"""
208	Run the script to download the project, if it exists.
209	"""
210	ScriptPath = os.path.join(Dir, DownloadScript)
211	SATestUtils.runScript(ScriptPath, PBuildLogFile, Dir,
212	Stdout=Local.stdout, Stderr=Local.stderr)
213
214
215	def downloadAndPatch(Dir, PBuildLogFile):
216	"""
217	Download the project and apply the local patchfile if it exists.
218	"""
219	CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName)
220
221	# If the we don't already have the cached source, run the project's
222	# download script to download it.
223	if not os.path.exists(CachedSourceDirPath):
224	runDownloadScript(Dir, PBuildLogFile)
225	if not os.path.exists(CachedSourceDirPath):
226	Local.stderr.write("Error: '%s' not found after download.\n" % (
227	CachedSourceDirPath))
228	exit(1)
229
230	PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
231
232	# Remove potentially stale patched source.
233	if os.path.exists(PatchedSourceDirPath):
234	shutil.rmtree(PatchedSourceDirPath)
235
236	# Copy the cached source and apply any patches to the copy.
237	shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True)
238	applyPatch(Dir, PBuildLogFile)
239
240
241	def applyPatch(Dir, PBuildLogFile):
242	PatchfilePath = os.path.join(Dir, PatchfileName)
243	PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
244	if not os.path.exists(PatchfilePath):
245	Local.stdout.write(" No local patches.\n")
246	return
247
248	Local.stdout.write(" Applying patch.\n")
249	try:
250	check_call("patch -p1 < '%s'" % (PatchfilePath),
251	cwd=PatchedSourceDirPath,
252	stderr=PBuildLogFile,
253	stdout=PBuildLogFile,
254	shell=True)
255	except:
256	Local.stderr.write("Error: Patch failed. See %s for details.\n" % (
257	PBuildLogFile.name))
258	sys.exit(1)
259
260
261	def generateAnalyzerConfig(Args):
262	Out = "serialize-stats=true,stable-report-filename=true"
263	if Args.extra_analyzer_config:
264	Out += "," + Args.extra_analyzer_config
265	return Out
266
267
268	def runScanBuild(Args, Dir, SBOutputDir, PBuildLogFile):
269	"""
270	Build the project with scan-build by reading in the commands and
271	prefixing them with the scan-build options.
272	"""
273	BuildScriptPath = os.path.join(Dir, BuildScript)
274	if not os.path.exists(BuildScriptPath):
275	Local.stderr.write(
276	"Error: build script is not defined: %s\n" % BuildScriptPath)
277	sys.exit(1)
278
279	AllCheckers = Checkers
280	if 'SA_ADDITIONAL_CHECKERS' in os.environ:
281	AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS']
282
283	# Run scan-build from within the patched source directory.
284	SBCwd = os.path.join(Dir, PatchedSourceDirName)
285
286	SBOptions = "--use-analyzer '%s' " % Clang
287	SBOptions += "-plist-html -o '%s' " % SBOutputDir
288	SBOptions += "-enable-checker " + AllCheckers + " "
289	SBOptions += "--keep-empty "
290	SBOptions += "-analyzer-config '%s' " % generateAnalyzerConfig(Args)
291
292	# Always use ccc-analyze to ensure that we can locate the failures
293	# directory.
294	SBOptions += "--override-compiler "
295	ExtraEnv = {}
296	try:
297	SBCommandFile = open(BuildScriptPath, "r")
298	SBPrefix = "scan-build " + SBOptions + " "
299	for Command in SBCommandFile:
300	Command = Command.strip()
301	if len(Command) == 0:
302	continue
303
304	# Custom analyzer invocation specified by project.
305	# Communicate required information using environment variables
306	# instead.
307	if Command == NoPrefixCmd:
308	SBPrefix = ""
309	ExtraEnv['OUTPUT'] = SBOutputDir
310	ExtraEnv['CC'] = Clang
311	ExtraEnv['ANALYZER_CONFIG'] = generateAnalyzerConfig(Args)
312	continue
313
314	# If using 'make', auto imply a -jX argument
315	# to speed up analysis. xcodebuild will
316	# automatically use the maximum number of cores.
317	if (Command.startswith("make ") or Command == "make") and \
318	"-j" not in Command:
319	Command += " -j%d" % MaxJobs
320	SBCommand = SBPrefix + Command
321
322	if Verbose == 1:
323	Local.stdout.write(" Executing: %s\n" % (SBCommand,))
324	check_call(SBCommand, cwd=SBCwd,
325	stderr=PBuildLogFile,
326	stdout=PBuildLogFile,
327	env=dict(os.environ, **ExtraEnv),
328	shell=True)
329	except CalledProcessError:
330	Local.stderr.write("Error: scan-build failed. Its output was: \n")
331	PBuildLogFile.seek(0)
332	shutil.copyfileobj(PBuildLogFile, Local.stderr)
333	sys.exit(1)
334
335
336	def runAnalyzePreprocessed(Args, Dir, SBOutputDir, Mode):
337	"""
338	Run analysis on a set of preprocessed files.
339	"""
340	if os.path.exists(os.path.join(Dir, BuildScript)):
341	Local.stderr.write(
342	"Error: The preprocessed files project should not contain %s\n" % (
343	BuildScript))
344	raise Exception()
345
346	CmdPrefix = Clang + " -cc1 "
347
348	# For now, we assume the preprocessed files should be analyzed
349	# with the OS X SDK.
350	SDKPath = SATestUtils.getSDKPath("macosx")
351	if SDKPath is not None:
352	CmdPrefix += "-isysroot " + SDKPath + " "
353
354	CmdPrefix += "-analyze -analyzer-output=plist -w "
355	CmdPrefix += "-analyzer-checker=" + Checkers
356	CmdPrefix += " -fcxx-exceptions -fblocks "
357	CmdPrefix += " -analyzer-config %s " % generateAnalyzerConfig(Args)
358
359	if (Mode == 2):
360	CmdPrefix += "-std=c++11 "
361
362	PlistPath = os.path.join(Dir, SBOutputDir, "date")
363	FailPath = os.path.join(PlistPath, "failures")
364	os.makedirs(FailPath)
365
366	for FullFileName in glob.glob(Dir + "/*"):
367	FileName = os.path.basename(FullFileName)
368	Failed = False
369
370	# Only run the analyzes on supported files.
371	if SATestUtils.hasNoExtension(FileName):
372	continue
373	if not SATestUtils.isValidSingleInputFile(FileName):
374	Local.stderr.write(
375	"Error: Invalid single input file %s.\n" % (FullFileName,))
376	raise Exception()
377
378	# Build and call the analyzer command.
379	OutputOption = "-o '%s.plist' " % os.path.join(PlistPath, FileName)
380	Command = CmdPrefix + OutputOption + ("'%s'" % FileName)
381	LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
382	try:
383	if Verbose == 1:
384	Local.stdout.write(" Executing: %s\n" % (Command,))
385	check_call(Command, cwd=Dir, stderr=LogFile,
386	stdout=LogFile,
387	shell=True)
388	except CalledProcessError as e:
389	Local.stderr.write("Error: Analyzes of %s failed. "
390	"See %s for details."
391	"Error code %d.\n" % (
392	FullFileName, LogFile.name, e.returncode))
393	Failed = True
394	finally:
395	LogFile.close()
396
397	# If command did not fail, erase the log file.
398	if not Failed:
399	os.remove(LogFile.name)
400
401
402	def getBuildLogPath(SBOutputDir):
403	return os.path.join(SBOutputDir, LogFolderName, BuildLogName)
404
405
406	def removeLogFile(SBOutputDir):
407	BuildLogPath = getBuildLogPath(SBOutputDir)
408	# Clean up the log file.
409	if (os.path.exists(BuildLogPath)):
410	RmCommand = "rm '%s'" % BuildLogPath
411	if Verbose == 1:
412	Local.stdout.write(" Executing: %s\n" % (RmCommand,))
413	check_call(RmCommand, shell=True)
414
415
416	def buildProject(Args, Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
417	TBegin = time.time()
418
419	BuildLogPath = getBuildLogPath(SBOutputDir)
420	Local.stdout.write("Log file: %s\n" % (BuildLogPath,))
421	Local.stdout.write("Output directory: %s\n" % (SBOutputDir, ))
422
423	removeLogFile(SBOutputDir)
424
425	# Clean up scan build results.
426	if (os.path.exists(SBOutputDir)):
427	RmCommand = "rm -r '%s'" % SBOutputDir
428	if Verbose == 1:
429	Local.stdout.write(" Executing: %s\n" % (RmCommand,))
430	check_call(RmCommand, shell=True, stdout=Local.stdout,
431	stderr=Local.stderr)
432	assert(not os.path.exists(SBOutputDir))
433	os.makedirs(os.path.join(SBOutputDir, LogFolderName))
434
435	# Build and analyze the project.
436	with open(BuildLogPath, "wb+") as PBuildLogFile:
437	if (ProjectBuildMode == 1):
438	downloadAndPatch(Dir, PBuildLogFile)
439	runCleanupScript(Dir, PBuildLogFile)
440	runScanBuild(Args, Dir, SBOutputDir, PBuildLogFile)
441	else:
442	runAnalyzePreprocessed(Args, Dir, SBOutputDir, ProjectBuildMode)
443
444	if IsReferenceBuild:
445	runCleanupScript(Dir, PBuildLogFile)
446	normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode)
447
448	Local.stdout.write("Build complete (time: %.2f). "
449	"See the log for more details: %s\n" % (
450	(time.time() - TBegin), BuildLogPath))
451
452
453	def normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode):
454	"""
455	Make the absolute paths relative in the reference results.
456	"""
457	for (DirPath, Dirnames, Filenames) in os.walk(SBOutputDir):
458	for F in Filenames:
459	if (not F.endswith('plist')):
460	continue
461	Plist = os.path.join(DirPath, F)
462	Data = plistlib.readPlist(Plist)
463	PathPrefix = Dir
464	if (ProjectBuildMode == 1):
465	PathPrefix = os.path.join(Dir, PatchedSourceDirName)
466	Paths = [SourceFile[len(PathPrefix) + 1:]
467	if SourceFile.startswith(PathPrefix)
468	else SourceFile for SourceFile in Data['files']]
469	Data['files'] = Paths
470
471	# Remove transient fields which change from run to run.
472	for Diag in Data['diagnostics']:
473	if 'HTMLDiagnostics_files' in Diag:
474	Diag.pop('HTMLDiagnostics_files')
475	if 'clang_version' in Data:
476	Data.pop('clang_version')
477
478	plistlib.writePlist(Data, Plist)
479
480
481	def CleanUpEmptyPlists(SBOutputDir):
482	"""
483	A plist file is created for each call to the analyzer(each source file).
484	We are only interested on the once that have bug reports,
485	so delete the rest.
486	"""
487	for F in glob.glob(SBOutputDir + "//.plist"):
488	P = os.path.join(SBOutputDir, F)
489
490	Data = plistlib.readPlist(P)
491	# Delete empty reports.
492	if not Data['files']:
493	os.remove(P)
494	continue
495
496
497	def CleanUpEmptyFolders(SBOutputDir):
498	"""
499	Remove empty folders from results, as git would not store them.
500	"""
501	Subfolders = glob.glob(SBOutputDir + "/*")
502	for Folder in Subfolders:
503	if not os.listdir(Folder):
504	os.removedirs(Folder)
505
506
507	def checkBuild(SBOutputDir):
508	"""
509	Given the scan-build output directory, checks if the build failed
510	(by searching for the failures directories). If there are failures, it
511	creates a summary file in the output directory.
512
513	"""
514	# Check if there are failures.
515	Failures = glob.glob(SBOutputDir + "//failures/.stderr.txt")
516	TotalFailed = len(Failures)
517	if TotalFailed == 0:
518	CleanUpEmptyPlists(SBOutputDir)
519	CleanUpEmptyFolders(SBOutputDir)
520	Plists = glob.glob(SBOutputDir + "//.plist")
521	Local.stdout.write(
522	"Number of bug reports (non-empty plist files) produced: %d\n" %
523	len(Plists))
524	return
525
526	Local.stderr.write("Error: analysis failed.\n")
527	Local.stderr.write("Total of %d failures discovered.\n" % TotalFailed)
528	if TotalFailed > NumOfFailuresInSummary:
529	Local.stderr.write(
530	"See the first %d below.\n" % NumOfFailuresInSummary)
531	# TODO: Add a line "See the results folder for more."
532
533	Idx = 0
534	for FailLogPathI in Failures:
535	if Idx >= NumOfFailuresInSummary:
536	break
537	Idx += 1
538	Local.stderr.write("\n-- Error #%d -----------\n" % Idx)
539	with open(FailLogPathI, "r") as FailLogI:
540	shutil.copyfileobj(FailLogI, Local.stdout)
541
542	sys.exit(1)
543
544
545	def runCmpResults(Dir, Strictness=0):
546	"""
547	Compare the warnings produced by scan-build.
548	Strictness defines the success criteria for the test:
549	0 - success if there are no crashes or analyzer failure.
550	1 - success if there are no difference in the number of reported bugs.
551	2 - success if all the bug reports are identical.
552
553	:return success: Whether tests pass according to the Strictness
554	criteria.
555	"""
556	TestsPassed = True
557	TBegin = time.time()
558
559	RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
560	NewDir = os.path.join(Dir, SBOutputDirName)
561
562	# We have to go one level down the directory tree.
563	RefList = glob.glob(RefDir + "/*")
564	NewList = glob.glob(NewDir + "/*")
565
566	# Log folders are also located in the results dir, so ignore them.
567	RefLogDir = os.path.join(RefDir, LogFolderName)
568	if RefLogDir in RefList:
569	RefList.remove(RefLogDir)
570	NewList.remove(os.path.join(NewDir, LogFolderName))
571
572	if len(RefList) != len(NewList):
573	print("Mismatch in number of results folders: %s vs %s" % (
574	RefList, NewList))
575	sys.exit(1)
576
577	# There might be more then one folder underneath - one per each scan-build
578	# command (Ex: one for configure and one for make).
579	if (len(RefList) > 1):
580	# Assume that the corresponding folders have the same names.
581	RefList.sort()
582	NewList.sort()
583
584	# Iterate and find the differences.
585	NumDiffs = 0
586	for P in zip(RefList, NewList):
587	RefDir = P[0]
588	NewDir = P[1]
589
590	assert(RefDir != NewDir)
591	if Verbose == 1:
592	Local.stdout.write(" Comparing Results: %s %s\n" % (
593	RefDir, NewDir))
594
595	PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
596	Opts, Args = CmpRuns.generate_option_parser().parse_args(
597	["--rootA", "", "--rootB", PatchedSourceDirPath])
598	# Scan the results, delete empty plist files.
599	NumDiffs, ReportsInRef, ReportsInNew = \
600	CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
601	deleteEmpty=False,
602	Stdout=Local.stdout)
603	if (NumDiffs > 0):
604	Local.stdout.write("Warning: %s differences in diagnostics.\n"
605	% NumDiffs)
606	if Strictness >= 2 and NumDiffs > 0:
607	Local.stdout.write("Error: Diffs found in strict mode (2).\n")
608	TestsPassed = False
609	elif Strictness >= 1 and ReportsInRef != ReportsInNew:
610	Local.stdout.write("Error: The number of results are different " +
611	" strict mode (1).\n")
612	TestsPassed = False
613
614	Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" % (
615	time.time() - TBegin))
616	return TestsPassed
617
618
619	def cleanupReferenceResults(SBOutputDir):
620	"""
621	Delete html, css, and js files from reference results. These can
622	include multiple copies of the benchmark source and so get very large.
623	"""
624	Extensions = ["html", "css", "js"]
625	for E in Extensions:
626	for F in glob.glob("%s//.%s" % (SBOutputDir, E)):
627	P = os.path.join(SBOutputDir, F)
628	RmCommand = "rm '%s'" % P
629	check_call(RmCommand, shell=True)
630
631	# Remove the log file. It leaks absolute path names.
632	removeLogFile(SBOutputDir)
633
634
635	class TestProjectThread(threading.Thread):
636	def __init__(self, Args, TasksQueue, ResultsDiffer, FailureFlag):
637	"""
638	:param ResultsDiffer: Used to signify that results differ from
639	the canonical ones.
640	:param FailureFlag: Used to signify a failure during the run.
641	"""
642	self.Args = Args
643	self.TasksQueue = TasksQueue
644	self.ResultsDiffer = ResultsDiffer
645	self.FailureFlag = FailureFlag
646	super(TestProjectThread, self).__init__()
647
648	# Needed to gracefully handle interrupts with Ctrl-C
649	self.daemon = True
650
651	def run(self):
652	while not self.TasksQueue.empty():
653	try:
654	ProjArgs = self.TasksQueue.get()
655	Logger = logging.getLogger(ProjArgs[0])
656	Local.stdout = StreamToLogger(Logger, logging.INFO)
657	Local.stderr = StreamToLogger(Logger, logging.ERROR)
658	if not testProject(Args, *ProjArgs):
659	self.ResultsDiffer.set()
660	self.TasksQueue.task_done()
661	except:
662	self.FailureFlag.set()
663	raise
664
665
666	def testProject(Args, ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
667	"""
668	Test a given project.
669	:return TestsPassed: Whether tests have passed according
670	to the :param Strictness: criteria.
671	"""
672	Local.stdout.write(" \n\n--- Building project %s\n" % (ID,))
673
674	TBegin = time.time()
675
676	Dir = getProjectDir(ID)
677	if Verbose == 1:
678	Local.stdout.write(" Build directory: %s.\n" % (Dir,))
679
680	# Set the build results directory.
681	RelOutputDir = getSBOutputDirName(IsReferenceBuild)
682	SBOutputDir = os.path.join(Dir, RelOutputDir)
683
684	buildProject(Args, Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
685
686	checkBuild(SBOutputDir)
687
688	if IsReferenceBuild:
689	cleanupReferenceResults(SBOutputDir)
690	TestsPassed = True
691	else:
692	TestsPassed = runCmpResults(Dir, Strictness)
693
694	Local.stdout.write("Completed tests for project %s (time: %.2f).\n" % (
695	ID, (time.time() - TBegin)))
696	return TestsPassed
697
698
699	def projectFileHandler():
700	return open(getProjectMapPath(), "rb")
701
702
703	def iterateOverProjects(PMapFile):
704	"""
705	Iterate over all projects defined in the project file handler `PMapFile`
706	from the start.
707	"""
708	PMapFile.seek(0)
709	for I in csv.reader(PMapFile):
710	if (SATestUtils.isCommentCSVLine(I)):
711	continue
712	yield I
713
714
715	def validateProjectFile(PMapFile):
716	"""
717	Validate project file.
718	"""
719	for I in iterateOverProjects(PMapFile):
720	if len(I) != 2:
721	print("Error: Rows in the ProjectMapFile should have 2 entries.")
722	raise Exception()
723	if I[1] not in ('0', '1', '2'):
724	print("Error: Second entry in the ProjectMapFile should be 0" \
725	" (single file), 1 (project), or 2(single file c++11).")
726	raise Exception()
727
728	def singleThreadedTestAll(Args, ProjectsToTest):
729	"""
730	Run all projects.
731	:return: whether tests have passed.
732	"""
733	Success = True
734	for ProjArgs in ProjectsToTest:
735	Success &= testProject(Args, *ProjArgs)
736	return Success
737
738	def multiThreadedTestAll(Args, ProjectsToTest, Jobs):
739	"""
740	Run each project in a separate thread.
741
742	This is OK despite GIL, as testing is blocked
743	on launching external processes.
744
745	:return: whether tests have passed.
746	"""
747	TasksQueue = queue.Queue()
748
749	for ProjArgs in ProjectsToTest:
750	TasksQueue.put(ProjArgs)
751
752	ResultsDiffer = threading.Event()
753	FailureFlag = threading.Event()
754
755	for i in range(Jobs):
756	T = TestProjectThread(Args, TasksQueue, ResultsDiffer, FailureFlag)
757	T.start()
758
759	# Required to handle Ctrl-C gracefully.
760	while TasksQueue.unfinished_tasks:
761	time.sleep(0.1) # Seconds.
762	if FailureFlag.is_set():
763	Local.stderr.write("Test runner crashed\n")
764	sys.exit(1)
765	return not ResultsDiffer.is_set()
766
767
768	def testAll(Args):
769	ProjectsToTest = []
770
771	with projectFileHandler() as PMapFile:
772	validateProjectFile(PMapFile)
773
774	# Test the projects.
775	for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
776	ProjectsToTest.append((ProjName,
777	int(ProjBuildMode),
778	Args.regenerate,
779	Args.strictness))
780	if Args.jobs <= 1:
781	return singleThreadedTestAll(Args, ProjectsToTest)
782	else:
783	return multiThreadedTestAll(Args, ProjectsToTest, Args.jobs)
784
785
786	if __name__ == '__main__':
787	# Parse command line arguments.
788	Parser = argparse.ArgumentParser(
789	description='Test the Clang Static Analyzer.')
790	Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
791	help='0 to fail on runtime errors, 1 to fail when the \
792	number of found bugs are different from the \
793	reference, 2 to fail on any difference from the \
794	reference. Default is 0.')
795	Parser.add_argument('-r', dest='regenerate', action='store_true',
796	default=False, help='Regenerate reference output.')
797	Parser.add_argument('-j', '--jobs', dest='jobs', type=int,
798	default=0,
799	help='Number of projects to test concurrently')
800	Parser.add_argument('--extra-analyzer-config', dest='extra_analyzer_config',
801	type=str,
802	default="",
803	help="Arguments passed to to -analyzer-config")
804	Args = Parser.parse_args()
805
806	TestsPassed = testAll(Args)
807	if not TestsPassed:
808	print("ERROR: Tests failed.")
809	sys.exit(42)
810

Clang Project