author | Danek Duvall <danek.duvall@sun.com> |
Fri, 20 Nov 2009 10:57:30 -0800 | |
changeset 1513 | 3ece69a26155 |
parent 1505 | cc598d70bbbe |
child 1516 | 8c950a3b4171 |
permissions | -rw-r--r-- |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/python2.4 |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
2 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
3 |
# CDDL HEADER START |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
4 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
5 |
# The contents of this file are subject to the terms of the |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
6 |
# Common Development and Distribution License (the "License"). |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
7 |
# You may not use this file except in compliance with the License. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
8 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
10 |
# or http://www.opensolaris.org/os/licensing. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
11 |
# See the License for the specific language governing permissions |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
12 |
# and limitations under the License. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
13 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
19 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
20 |
# CDDL HEADER END |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
21 |
# |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
22 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
23 |
# Copyright 2008 Sun Microsystems, Inc. All rights reserved. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
24 |
# Use is subject to license terms. |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
25 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
26 |
import baseline |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
27 |
import copy |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
28 |
import string |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
29 |
import sys |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
30 |
import time |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
31 |
import unittest |
616
6d307b088d92
4056 make_uninstall_plan misreports trying to uninstall a known, but uninstalled package
Brock Pytlik <bpytlik@sun.com>
parents:
571
diff
changeset
|
32 |
import gettext |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
33 |
import os |
616
6d307b088d92
4056 make_uninstall_plan misreports trying to uninstall a known, but uninstalled package
Brock Pytlik <bpytlik@sun.com>
parents:
571
diff
changeset
|
34 |
|
6d307b088d92
4056 make_uninstall_plan misreports trying to uninstall a known, but uninstalled package
Brock Pytlik <bpytlik@sun.com>
parents:
571
diff
changeset
|
35 |
gettext.install("pkg", "/usr/lib/locale") |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
36 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
37 |
OUTPUT_DOTS=0 # Dots ... |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
38 |
OUTPUT_VERBOSE=1 # Verbose |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
39 |
OUTPUT_PARSEABLE=2 # Machine readable |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
40 |
|
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
41 |
class EarlyTearDownException(Exception): |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
42 |
"""An exception for inidicating early teardown of the testcase is |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
43 |
desired. This exception is only useful for testing depot startup and |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
44 |
whatnot from the test case setUp() functions, and shouldn't be used or |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
45 |
inherited from for other purposes. |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
46 |
""" |
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
47 |
pass |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
48 |
|
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
49 |
class Pkg5TestCase(unittest.TestCase): |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
50 |
|
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
51 |
# Needed for compatability |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
52 |
failureException = AssertionError |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
53 |
|
683
5e5d509f00e3
4422 t_pkg_authority pinging extant node accidentally
Brock Pytlik <bpytlik@sun.com>
parents:
616
diff
changeset
|
54 |
bogus_url = "test.invalid" |
5e5d509f00e3
4422 t_pkg_authority pinging extant node accidentally
Brock Pytlik <bpytlik@sun.com>
parents:
616
diff
changeset
|
55 |
|
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
56 |
def __init__(self, methodName='runTest'): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
57 |
super(Pkg5TestCase, self).__init__(methodName) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
58 |
self.__testMethodName = methodName |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
59 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
60 |
def __str__(self): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
61 |
return "%s.py %s.%s" % (self.__class__.__module__, |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
62 |
self.__class__.__name__, self.__testMethodName) |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
63 |
def getTeardownFunc(self): |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
64 |
return (self, self.tearDown) |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
65 |
|
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
66 |
def run(self, result=None): |
436 | 67 |
if result is None: |
68 |
result = self.defaultTestResult() |
|
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
69 |
result.startTest(self) |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
70 |
testMethod = getattr(self, self.__testMethodName) |
1513
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
71 |
if result.coverage: |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
72 |
self.coverage_cmd, self.coverage_env = result.coverage |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
73 |
else: |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
74 |
self.coverage_cmd, self.coverage_env = "", {} |
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
75 |
try: |
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
76 |
try: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
77 |
self.setUp() |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
78 |
except KeyboardInterrupt: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
79 |
raise |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
80 |
except EarlyTearDownException: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
81 |
self.tearDown() |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
82 |
result.addSuccess(self) |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
83 |
return |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
84 |
except: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
85 |
result.addError(self, sys.exc_info()) |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
86 |
return |
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
87 |
|
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
88 |
ok = False |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
89 |
try: |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
90 |
testMethod() |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
91 |
ok = True |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
92 |
except self.failureException: |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
93 |
result.addFailure(self, sys.exc_info()) |
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
94 |
except KeyboardInterrupt: |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
95 |
raise |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
96 |
except: |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
97 |
result.addError(self, sys.exc_info()) |
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
98 |
|
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
99 |
try: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
100 |
self.tearDown() |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
101 |
except KeyboardInterrupt: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
102 |
raise |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
103 |
except: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
104 |
result.addError(self, sys.exc_info()) |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
105 |
ok = False |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
106 |
|
436 | 107 |
if ok: |
108 |
result.addSuccess(self) |
|
435
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
109 |
finally: |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
110 |
result.stopTest(self) |
461b1c739de9
1753 An exception thrown during the teardown phase of the test program leaves pkg.depot running.
Brock Pytlik <bpytlik@sun.com>
parents:
430
diff
changeset
|
111 |
|
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
112 |
class _Pkg5TestResult(unittest._TextTestResult): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
113 |
baseline = None |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
114 |
machsep = "|" |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
115 |
def __init__(self, stream, output, baseline, bailonfail=False): |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
116 |
unittest.TestResult.__init__(self) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
117 |
self.stream = stream |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
118 |
self.output = output |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
119 |
self.baseline = baseline |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
120 |
self.success = [] |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
121 |
self.bailonfail = bailonfail |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
122 |
|
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
123 |
def addSuccess(self, test): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
124 |
unittest.TestResult.addSuccess(self, test) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
125 |
bresult = self.baseline.handleresult(str(test), "pass") |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
126 |
if self.output == OUTPUT_VERBOSE or \ |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
127 |
self.output == OUTPUT_PARSEABLE: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
128 |
res = "" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
129 |
if bresult == True: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
130 |
res = "pass" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
131 |
else: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
132 |
res = "pass (FAIL)" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
133 |
self.stream.writeln(res) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
134 |
elif self.output == OUTPUT_DOTS: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
135 |
self.stream.write('.') |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
136 |
self.success.append(test) |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
137 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
138 |
def addError(self, test, err): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
139 |
unittest.TestResult.addError(self, test, err) |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
140 |
bresult = self.baseline.handleresult(str(test), "error") |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
141 |
if self.output == OUTPUT_VERBOSE or \ |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
142 |
self.output == OUTPUT_PARSEABLE: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
143 |
self.stream.writeln("ERROR") |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
144 |
if self.output == OUTPUT_VERBOSE: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
145 |
self.stream.writeln("%s" % self.errors[-1][1]) |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
146 |
elif self.output == OUTPUT_DOTS: |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
147 |
if bresult: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
148 |
self.stream.write('e') |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
149 |
else: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
150 |
self.stream.write('E') |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
151 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
152 |
def addFailure(self, test, err): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
153 |
unittest.TestResult.addFailure(self, test, err) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
154 |
bresult = self.baseline.handleresult(str(test), "fail") |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
155 |
if self.output == OUTPUT_VERBOSE or \ |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
156 |
self.output == OUTPUT_PARSEABLE: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
157 |
res = "" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
158 |
if bresult == True: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
159 |
res = "FAIL (pass)" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
160 |
else: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
161 |
res = "FAIL" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
162 |
self.stream.writeln(res) |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
163 |
if self.output == OUTPUT_VERBOSE: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
164 |
self.stream.writeln("%s" % self.failures[-1][1]) |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
165 |
elif self.output == OUTPUT_DOTS: |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
166 |
if bresult: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
167 |
self.stream.write('f') |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
168 |
else: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
169 |
self.stream.write('F') |
1434
f7e5a99f4692
12126 -x stops at first failure (regardless of baseline status)
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1421
diff
changeset
|
170 |
if self.bailonfail and not bresult: |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
171 |
tdf = test.getTeardownFunc()[1] |
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
172 |
tdf() |
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
173 |
if test.persistent_depot: |
1505
cc598d70bbbe
4425 pkg install should deal w/ complex dependency changes in one install
Bart Smaalders <Bart.Smaalders@Sun.COM>
parents:
1434
diff
changeset
|
174 |
test.reallytearDown() |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
175 |
raise |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
176 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
177 |
def getDescription(self, test): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
178 |
return str(test) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
179 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
180 |
def startTest(self, test): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
181 |
unittest.TestResult.startTest(self, test) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
182 |
if not self.output == OUTPUT_DOTS: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
183 |
self.stream.write( |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
184 |
string.ljust(self.getDescription(test), 60)) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
185 |
if self.output == OUTPUT_VERBOSE: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
186 |
self.stream.write(" ") |
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
187 |
elif self.output == OUTPUT_PARSEABLE: |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
188 |
self.stream.write(" | ") |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
189 |
self.stream.flush() |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
190 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
191 |
def printErrors(self): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
192 |
self.stream.writeln() |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
193 |
self.printErrorList('ERROR', self.errors) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
194 |
self.printErrorList('FAIL', self.failures) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
195 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
196 |
def printErrorList(self, flavour, errors): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
197 |
for test, err in errors: |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
198 |
self.stream.writeln(self.separator1) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
199 |
self.stream.writeln("%s: %s" % |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
200 |
(flavour, self.getDescription(test))) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
201 |
self.stream.writeln(self.separator2) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
202 |
self.stream.writeln("%s" % err) |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
203 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
204 |
class Pkg5TestRunner(unittest.TextTestRunner): |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
205 |
"""TestRunner for test suites that we want to be able to compare |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
206 |
against a result baseline.""" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
207 |
baseline = None |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
208 |
sep1 = '=' * 70 |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
209 |
sep2 = '-' * 70 |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
210 |
|
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
211 |
def __init__(self, baseline, stream=sys.stderr, output=OUTPUT_DOTS, |
1513
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
212 |
timing_file=None, bailonfail=False, coverage=None): |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
213 |
"""Set up the runner, creating a baseline object that has |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
214 |
a name of 'suite'_baseline.pkl, where suite is 'cli', 'api', |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
215 |
etc.""" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
216 |
# output is one of "dots", "verbose", "machine" |
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
217 |
super(Pkg5TestRunner, self).__init__(stream) |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
218 |
self.baseline = baseline |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
219 |
self.output = output |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
220 |
self.timing_file = timing_file |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
221 |
self.bailonfail = bailonfail |
1513
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
222 |
self.coverage = coverage |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
223 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
224 |
def _makeResult(self): |
1421
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
225 |
return _Pkg5TestResult(self.stream, self.output, self.baseline, |
d6733f41c267
3961 Test suite should have a flag that makes it stop after encountering a failure
John Sonnenschein <John.Sonnenschein@Sun.COM>
parents:
1101
diff
changeset
|
226 |
bailonfail=self.bailonfail) |
430
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
227 |
|
3fb945d3bff3
2627 Test suite should have support for baselining
Brad Hall <bhall@eng.sun.com>
parents:
diff
changeset
|
228 |
def run(self, test): |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
229 |
"Run the given test case or test suite." |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
230 |
result = self._makeResult() |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
231 |
startTime = time.time() |
1513
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
232 |
result.coverage = self.coverage |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
233 |
test.run(result) |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
234 |
stopTime = time.time() |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
235 |
timeTaken = stopTime - startTime |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
236 |
timing = {} |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
237 |
lst = [] |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
238 |
for t in test._tests: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
239 |
for (cname, mname), secs in t.timing.items(): |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
240 |
lst.append((secs, cname, mname)) |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
241 |
if cname not in timing: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
242 |
timing[cname] = 0 |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
243 |
timing[cname] += secs |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
244 |
lst.sort() |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
245 |
clst = sorted((secs, cname) for cname, secs in timing.items()) |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
246 |
|
1002
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
247 |
if self.output != OUTPUT_VERBOSE: |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
248 |
result.printErrors() |
56ab3be9a8b7
3488 Test suite should supress expected failures (or notate them more clearly)
Danek Duvall <danek.duvall@sun.com>
parents:
683
diff
changeset
|
249 |
self.stream.writeln(result.separator2) |
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
250 |
run = result.testsRun |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
251 |
self.stream.writeln("Ran %d test%s in %.3fs" % |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
252 |
(run, run != 1 and "s" or "", timeTaken)) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
253 |
self.stream.writeln() |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
254 |
if not result.wasSuccessful(): |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
255 |
self.stream.write("FAILED (") |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
256 |
success, failed, errored, mismatches = map(len, |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
257 |
(result.success, result.failures, result.errors, |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
258 |
self.baseline.getfailures())) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
259 |
self.stream.write("successes=%d, " % success) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
260 |
self.stream.write("failures=%d, " % failed) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
261 |
self.stream.write("errors=%d, " % errored) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
262 |
self.stream.write("mismatches=%d" % mismatches) |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
263 |
self.stream.writeln(")") |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
264 |
else: |
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
265 |
self.stream.writeln("OK") |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
266 |
|
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
267 |
if self.timing_file: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
268 |
try: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
269 |
fh = open(self.timing_file, "ab+") |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
270 |
opened = True |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
271 |
except KeyboardInterrupt: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
272 |
raise |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
273 |
except Exception: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
274 |
fh = sys.stderr |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
275 |
opened = False |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
276 |
self.__write_timing_info(fh, clst, lst) |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
277 |
if opened: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
278 |
fh.close() |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
279 |
|
448
bcfa99ac18b0
2664 test suite baseline enhancements
Brad Hall <bhall@eng.sun.com>
parents:
436
diff
changeset
|
280 |
return result |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
281 |
|
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
282 |
@staticmethod |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
283 |
def __write_timing_info(stream, class_list, method_list): |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
284 |
if not class_list and not method_list: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
285 |
return |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
286 |
tot = 0 |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
287 |
for secs, cname in class_list: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
288 |
print >> stream, "%6.2f %s" % (secs, cname) |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
289 |
tot += secs |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
290 |
print >> stream, "%6.2f Total time" % tot |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
291 |
print >> stream, "=" * 60 |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
292 |
for secs, cname, mname in method_list: |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
293 |
print >> stream, "%6.2f %s %s" % (secs, cname, mname) |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
294 |
print >> stream, "=" * 60 |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
295 |
print >> stream, "=" * 60 + "\n" * 4 |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
296 |
|
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
297 |
class Pkg5TestSuite(unittest.TestSuite): |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
298 |
"""Test suite that handles persistent depot tests. Persistent depot |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
299 |
tests are ones that are able to only call their setUp/tearDown |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
300 |
functions once per class, instead of before and after every test case. |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
301 |
Aside from actually running the test it defers the majority of its |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
302 |
work to unittest.TestSuite. |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
303 |
|
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
304 |
To make a test class into a persistent depot one, add this class |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
305 |
variable declaration: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
306 |
persistent_depot = True |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
307 |
""" |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
308 |
|
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
309 |
def run(self, result): |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
310 |
self.timing = {} |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
311 |
inst = None |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
312 |
tdf = None |
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
313 |
try: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
314 |
persistent_depot = getattr(self._tests[0], |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
315 |
"persistent_depot", False) |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
316 |
except IndexError: |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
317 |
# No tests, thats ok. |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
318 |
return |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
319 |
|
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
320 |
if persistent_depot: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
321 |
inst, tdf = self._tests[0].getTeardownFunc() |
1513
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
322 |
if result.coverage: |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
323 |
inst.coverage_cmd, inst.coverage_env = result.coverage |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
324 |
else: |
3ece69a26155
5032 make coverage should only report on the files in our gate
Danek Duvall <danek.duvall@sun.com>
parents:
1505
diff
changeset
|
325 |
inst.coverage_cmd, inst.coverage_env = "", {} |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
326 |
try: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
327 |
inst.setUp() |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
328 |
except KeyboardInterrupt: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
329 |
raise |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
330 |
except: |
516
53da6297fb65
3039 Install and Image-update should check index consistency with installed packages
Brock Pytlik <bpytlik@sun.com>
parents:
502
diff
changeset
|
331 |
result.addError(inst, sys.exc_info()) |
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
332 |
def donothing(): |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
333 |
pass |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
334 |
for test in self._tests: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
335 |
if result.shouldStop: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
336 |
break |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
337 |
real_test_name = test._Pkg5TestCase__testMethodName |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
338 |
cname = test.__class__.__name__ |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
339 |
# Populate test with the data from the instance |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
340 |
# already constructed, but update the method name. |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
341 |
# We need to do this so that we have all the state |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
342 |
# that the object is populated with when setUp() is |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
343 |
# called (depot controller list, etc). |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
344 |
if persistent_depot: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
345 |
name = test._Pkg5TestCase__testMethodName |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
346 |
test = copy.copy(inst) |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
347 |
test._Pkg5TestCase__testMethodName = name |
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
348 |
# For test classes with persistent_depot set, |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
349 |
# make their setup/teardown methods do nothing |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
350 |
# since we are calling them here. |
1505
cc598d70bbbe
4425 pkg install should deal w/ complex dependency changes in one install
Bart Smaalders <Bart.Smaalders@Sun.COM>
parents:
1434
diff
changeset
|
351 |
test.reallytearDown = tdf |
571
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
352 |
test.setUp = donothing |
1a2452c33cb9
3476 Most cli test cases can't be run standalone anymore
Brad Hall <bhall@eng.sun.com>
parents:
516
diff
changeset
|
353 |
test.tearDown = donothing |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
354 |
test_start = time.time() |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
355 |
test(result) |
1101
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
356 |
test_end = time.time() |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
357 |
self.timing[cname, real_test_name] = \ |
35f360e2ee19
5247 Test suite should provide timing info
Brock Pytlik <bpytlik@sun.com>
parents:
1002
diff
changeset
|
358 |
test_end - test_start |
502
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
359 |
if persistent_depot: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
360 |
try: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
361 |
tdf() |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
362 |
except KeyboardInterrupt: |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
363 |
raise |
3f98e94acc4a
2690 test suite depot starting/stopping
Brad Hall <bhall@eng.sun.com>
parents:
448
diff
changeset
|
364 |
except: |
516
53da6297fb65
3039 Install and Image-update should check index consistency with installed packages
Brock Pytlik <bpytlik@sun.com>
parents:
502
diff
changeset
|
365 |
result.addError(inst, sys.exc_info()) |