20808505 gmake system-test
authorJohn Beck <John.Beck@Oracle.COM>
Thu, 09 Apr 2015 18:28:09 -0700
changeset 4091 fca9099ab398
parent 4089 1788e52b3086
child 4092 376ba5d29e51
20808505 gmake system-test
components/python/python34/Makefile
doc/testing.txt
make-rules/ant.mk
make-rules/attpackagemake.mk
make-rules/configure.mk
make-rules/justmake.mk
make-rules/makemaker.mk
make-rules/setup.py.mk
make-rules/shared-macros.mk
--- a/components/python/python34/Makefile	Tue Apr 07 05:51:44 2015 -0700
+++ b/components/python/python34/Makefile	Thu Apr 09 18:28:09 2015 -0700
@@ -173,6 +173,11 @@
 COMPONENT_TEST_ENV += PATH="$(SPRO_VROOT)/bin:$(PATH)"
 # Prevent the tests from getting stuck waiting for input.
 COMPONENT_TEST_TARGETS = test < /dev/null
+# Some different values for system testing.
+COMPONENT_SYSTEM_TEST_ENV = EXTRATESTOPTS="-v -uall,-network $(TESTOPTS_PYTHON_TEST)"
+COMPONENT_SYSTEM_TEST_TARGETS = 
+COMPONENT_SYSTEM_TEST_CMD=	$(PYTHON.3.4)
+COMPONENT_SYSTEM_TEST_ARGS=	/usr/lib/python3.4/test/regrtest.py -v -uall,-network
 
 # The test output contains details from each test, in whatever order they
 # complete.  The default _TRANSFORMER is not powerful enough to deal with
@@ -182,6 +187,7 @@
 COMPONENT_TEST_TRANSFORMS =	"'/tests OK./ {results = 1}; /Re-running failed tests in verbose mode/ {results = 0} {if (results) print $0 } '"
 
 test:				$(TEST_64)
+system-test:			$(SYSTEM_TEST_64)
 
 REQUIRED_PACKAGES += compress/bzip2
 REQUIRED_PACKAGES += compress/xz
--- a/doc/testing.txt	Tue Apr 07 05:51:44 2015 -0700
+++ b/doc/testing.txt	Thu Apr 09 18:28:09 2015 -0700
@@ -9,7 +9,8 @@
 'gmake test' is often run when a component is upgraded or otherwise
 intentionally changed, but sometimes it would be useful to rerun the tests
 after something else has changed (such as the system being upgraded, or a
-change in compilers) and see if that has affected the tests.
+change in compilers; see the 'System Test' section below) and see if that
+has affected the tests.
 
 We do this by having a 'master test file' that contains the expected results,
 and having a compare target that runs the tests and compares them with the
@@ -208,3 +209,53 @@
 
 When your master test file(s) are in good shape, then you should "hg add"
 them to your workspace.
+
+
+System Test
+-----------
+All of the above discusses how 'gmake test' works.  This is all good, but
+has the limitation that objects under test are those that were just built
+and installed into the component's proto area.  We also want to be able
+to test whatever is installed on the actual system.  This has the benefits
+of not requiring anything to be built, and allowing non-Userland people
+to test our bits.  (Think of people in ON changing libc or the linker and
+wanting to make sure they don't break anything.)
+
+To this end, we have added 'gmake system-test'.  It works just like 'gmake
+test', reusing some of the same variables:
+* COMPONENT_TEST_BUILD_DIR
+* COMPONENT_TEST_COMPARE
+* COMPONENT_TEST_CREATE_TRANSFORMS
+* COMPONENT_TEST_OUTPUT
+* COMPONENT_TEST_PERFORM_TRANSFORM
+but with its own "_SYSTEM" instance of other variables:
+* COMPONENT_POST_SYSTEM_TEST_ACTION
+* COMPONENT_PRE_SYSTEM_TEST_ACTION
+* COMPONENT_SYSTEM_TEST_ARGS
+* COMPONENT_SYSTEM_TEST_CLEANUP
+* COMPONENT_SYSTEM_TEST_CMD
+* COMPONENT_SYSTEM_TEST_DIR
+* COMPONENT_SYSTEM_TEST_ENV
+* COMPONENT_SYSTEM_TEST_ENV_CMD
+* COMPONENT_SYSTEM_TEST_RESULTS_DIR
+* COMPONENT_SYSTEM_TEST_TARGETS
+
+In the ideal case, only Makefile variables would need to be modified to
+take a component where 'gmake test' works and extend it so that 'gmake
+system-test' works; see components/python34/Makefile for such an example.
+
+The next level up from that is the case where some simple patching is needed.
+In this case, simple means:
+* tweaking LD_LIBRARY_PATH
+* changing the path to a binary to run
+* modifying a Makefile so tests are compiled as part of 'gmake build'
+* modifying a Makefile so tests are installed as part of 'gmake install'
+* modifying a pkg(5) manifest so tests are published as part of 'gmake publish'
+  (preferably under the "optional.test" facet)
+
+If more extensive patching is needed, a judgement call needs to be made:
+Are these changes that upstream would welcome?  If not, then we would need
+to keep such changes around indefinitely, which would constitute a fork,
+which we try very hard to avoid.  When 'gmake system-test' was initially
+implemented, changes to both fetchmail and python/decorator were begun but
+ultimately abandoned for this reason; see 20808505 for details.
--- a/make-rules/ant.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/ant.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -50,6 +50,10 @@
 COMPONENT_TEST_ENV += JAVA_HOME="$(JAVA_HOME)"
 COMPONENT_TEST_CMD = $(ANT)
 
+COMPONENT_SYSTEM_TEST_ENV_CMD = $(ENV)
+COMPONENT_SYSTEM_TEST_ENV += JAVA_HOME="$(JAVA_HOME)"
+COMPONENT_SYSTEM_TEST_CMD = $(ANT)
+
 # test the built source
 $(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
@@ -77,6 +81,33 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(SOURCE_DIR)/.prep
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(SOURCE_DIR)/.prep
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/attpackagemake.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/attpackagemake.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -60,9 +60,11 @@
 
 # The install and test process needs the same environment as the build
 COMPONENT_INSTALL_ENV = $(COMPONENT_BUILD_ENV)
+COMPONENT_INSTALL_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
 COMPONENT_TEST_ENV = $(COMPONENT_BUILD_ENV)
-COMPONENT_INSTALL_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
 COMPONENT_TEST_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
+COMPONENT_SYSTEM_TEST_ENV = $(COMPONENT_BUILD_ENV)
+COMPONENT_SYSTEM_TEST_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
 
 # build the configured source
 $(BUILD_DIR)/%/.built:	$(SOURCE_DIR)/.prep
@@ -115,6 +117,33 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(SOURCE_DIR)/.prep
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		bin/package test $(COMPONENT_SYSTEM_TEST_TARGETS) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(SOURCE_DIR)/.prep
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		bin/package test $(COMPONENT_SYSTEM_TEST_TARGETS) \
+		$(COMPONENT_SYSTEM_TEST_ARGS)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/configure.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/configure.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -161,6 +161,33 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(SOURCE_DIR)/.prep
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(SOURCE_DIR)/.prep
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/justmake.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/justmake.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -94,6 +94,33 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(SOURCE_DIR)/.prep
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(SOURCE_DIR)/.prep
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/makemaker.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/makemaker.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -55,6 +55,20 @@
 $(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared:	PERL_VERSION=5.16
 $(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared:	BITS=64
 
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested:	PERL_VERSION=5.12
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested:	BITS=32
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested:	PERL_VERSION=5.12-mt
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested:	BITS=32
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested:	PERL_VERSION=5.16
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested:	BITS=64
+
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared:	PERL_VERSION=5.12
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared:	BITS=32
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared:	PERL_VERSION=5.12-mt
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared:	BITS=32
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared:	PERL_VERSION=5.16
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared:	BITS=64
+
 BUILD_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.built
 BUILD_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.built
 BUILD_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.built
@@ -106,7 +120,9 @@
 COMPONENT_TEST_TRANSFORM_CMD = $(COMPONENT_TEST_BUILD_DIR)/transform-$(PERL_VERSION)-$(BITS)-results
 
 COMPONENT_TEST_TARGETS =	check
-COMPONENT_TEST_ENV +=	$(COMMON_PERL_ENV)
+COMPONENT_TEST_ENV +=		$(COMMON_PERL_ENV)
+COMPONENT_SYSTEM_TEST_TARGETS =	check
+COMPONENT_SYSTEM_TEST_ENV +=	$(COMMON_PERL_ENV)
 
 # determine the type of tests we want to run.
 ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
@@ -119,6 +135,16 @@
 TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared
 endif
 
+ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.system-tested
+SYSTEM_TEST_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested
+SYSTEM_TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.system-tested
+else
+SYSTEM_TEST_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared
+SYSTEM_TEST_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared
+SYSTEM_TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared
+endif
+
 # test the built source
 $(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
@@ -146,6 +172,33 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(SOURCE_DIR)/.prep
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(SOURCE_DIR)/.prep
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) \
+		$(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/setup.py.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/setup.py.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -71,6 +71,7 @@
 COMPONENT_BUILD_ENV += $(PYTHON_ENV)
 COMPONENT_INSTALL_ENV += $(PYTHON_ENV)
 COMPONENT_TEST_ENV += $(PYTHON_ENV)
+COMPONENT_SYSTEM_TEST_ENV += $(PYTHON_ENV)
 
 # If we are building Python 2.7 or 3.4 support, build them and install them
 # before Python 2.6, so 2.6 is installed last and is the canonical version.
@@ -135,6 +136,13 @@
 COMPONENT_TEST_CMD =	$(PYTHON)
 COMPONENT_TEST_ARGS +=	./runtests.py
 
+COMPONENT_SYSTEM_TEST_DEP =	$(SOURCE_DIR)/.prep
+COMPONENT_SYSTEM_TEST_DIR =	$(COMPONENT_SRC)/test
+COMPONENT_SYSTEM_TEST_ENV_CMD =	$(ENV)
+COMPONENT_SYSTEM_TEST_ENV +=	PYTHONPATH=$(PROTO_DIR)$(PYTHON_VENDOR_PACKAGES)
+COMPONENT_SYSTEM_TEST_CMD =	$(PYTHON)
+COMPONENT_SYSTEM_TEST_ARGS +=	./runtests.py
+
 # determine the type of tests we want to run.
 ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
 TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.tested)
@@ -148,6 +156,18 @@
 ifeq ($(PYTHON_VERSION),3.4)
 TEST_32_and_64 = $(TEST_64)
 endif
+ifeq ($(strip $(wildcard $(COMPONENT_SYSTEM_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.system-tested)
+SYSTEM_TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.system-tested)
+SYSTEM_TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.system-tested)
+else
+SYSTEM_TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.system-tested-and-compared)
+SYSTEM_TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.system-tested-and-compared)
+SYSTEM_TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.system-tested-and-compared)
+endif
+ifeq ($(PYTHON_VERSION),3.4)
+SYSTEM_TEST_32_and_64 = $(SYSTEM_TEST_64)
+endif
 
 # test the built source
 $(BUILD_DIR)/%/.tested-and-compared:    $(COMPONENT_TEST_DEP)
@@ -174,6 +194,31 @@
 	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
+# test the installed system
+$(BUILD_DIR)/%/.system-tested-and-compared:    $(COMPONENT_SYSTEM_TEST_DEP)
+	$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+	$(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	-(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) $(COMPONENT_SYSTEM_TEST_ARGS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.system-tested:    $(COMPONENT_SYSTEM_TEST_DEP)
+	$(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+	(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+		$(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+		$(COMPONENT_SYSTEM_TEST_CMD) $(COMPONENT_SYSTEM_TEST_ARGS))
+	$(COMPONENT_POST_SYSTEM_TEST_ACTION)
+	$(COMPONENT_SYSTEM_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
 parfait: build
 else
--- a/make-rules/shared-macros.mk	Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/shared-macros.mk	Thu Apr 09 18:28:09 2015 -0700
@@ -240,6 +240,7 @@
 
 # set the default master test results directory
 COMPONENT_TEST_RESULTS_DIR =	$(COMPONENT_DIR)/test
+COMPONENT_SYSTEM_TEST_RESULTS_DIR =	$(COMPONENT_DIR)/test
 
 # set the default master test results file
 COMPONENT_TEST_MASTER =		$(COMPONENT_TEST_RESULTS_DIR)/results-$(BITS).master
@@ -320,17 +321,31 @@
 
 # set the default env command to use for test of the component
 COMPONENT_TEST_ENV_CMD =        $(ENV)
+COMPONENT_SYSTEM_TEST_ENV_CMD =	$(ENV)
 
 # set the default command to use for test of the component
 COMPONENT_TEST_CMD =		$(GMAKE)
+COMPONENT_SYSTEM_TEST_CMD =	$(GMAKE)
 
 # set the default target for test of the component
 COMPONENT_TEST_TARGETS =	check
+COMPONENT_SYSTEM_TEST_TARGETS =	check
 
 # set the default directory for test of the component
 COMPONENT_TEST_DIR =		$(@D)
+COMPONENT_SYSTEM_TEST_DIR =	$(@D)
 
 # determine the type of tests we want to run.
+ifeq ($(strip $(wildcard $(COMPONENT_SYSTEM_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_NO_ARCH =		$(BUILD_DIR_NO_ARCH)/.system-tested
+SYSTEM_TEST_32 =		$(BUILD_DIR_32)/.system-tested
+SYSTEM_TEST_64 =		$(BUILD_DIR_64)/.system-tested
+else
+SYSTEM_TEST_NO_ARCH =		$(BUILD_DIR_NO_ARCH)/.system-tested-and-compared
+SYSTEM_TEST_32 =		$(BUILD_DIR_32)/.system-tested-and-compared
+SYSTEM_TEST_64 =		$(BUILD_DIR_64)/.system-tested-and-compared
+endif
+SYSTEM_TEST_32_and_64 =	$(TEST_32) $(TEST_64)
 ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
 TEST_NO_ARCH =		$(BUILD_DIR_NO_ARCH)/.tested
 TEST_32 =		$(BUILD_DIR_32)/.tested
@@ -351,12 +366,18 @@
 
 SKIP_TEST_AT_TOP_LEVEL = $(eval include $(WS_MAKE_RULES)/skip-test.mk)
 
-$(BUILD_DIR_NO_ARCH)/.tested:		BITS=32
-$(BUILD_DIR_32)/.tested:		BITS=32
-$(BUILD_DIR_64)/.tested:		BITS=64
-$(BUILD_DIR_NO_ARCH)/.tested-and-compared:	BITS=32
-$(BUILD_DIR_32)/.tested-and-compared:	BITS=32
-$(BUILD_DIR_64)/.tested-and-compared:	BITS=64
+$(BUILD_DIR_NO_ARCH)/.system-tested:			BITS=32
+$(BUILD_DIR_32)/.system-tested:				BITS=32
+$(BUILD_DIR_64)/.system-tested:				BITS=64
+$(BUILD_DIR_NO_ARCH)/.system-tested-and-compared:	BITS=32
+$(BUILD_DIR_32)/.system-tested-and-compared:		BITS=32
+$(BUILD_DIR_64)/.system-tested-and-compared:		BITS=64
+$(BUILD_DIR_NO_ARCH)/.tested:				BITS=32
+$(BUILD_DIR_32)/.tested:				BITS=32
+$(BUILD_DIR_64)/.tested:				BITS=64
+$(BUILD_DIR_NO_ARCH)/.tested-and-compared:		BITS=32
+$(BUILD_DIR_32)/.tested-and-compared:			BITS=32
+$(BUILD_DIR_64)/.tested-and-compared:			BITS=64
 
 
 # BUILD_TOOLS is the root of all tools not normally installed on the system.
@@ -447,6 +468,10 @@
 PYTHON_VENDOR_PACKAGES.64 = /usr/lib/python$(PYTHON_VERSION)/vendor-packages/64
 PYTHON_VENDOR_PACKAGES = $(PYTHON_VENDOR_PACKAGES.$(BITS))
 
+PYTHON.2.6.TEST = /usr/lib/python2.6/test
+PYTHON.2.7.TEST = /usr/lib/python2.7/test
+PYTHON.3.4.TEST = /usr/lib/python3.4/test
+
 USRBIN.32 =	/usr/bin
 USRBIN.64 =	/usr/bin/$(MACH64)
 USRBIN =	$(USRBIN.$(BITS))
@@ -465,7 +490,7 @@
 # than the 64-bit path.
 PYTHON.3.4.32 =	$(USRBIN.32)/python3.4
 PYTHON.3.4.64 =	$(USRBIN.32)/python3.4
-PYTHON.3.4 =	$(USRBIN)/python3.4
+PYTHON.3.4 =	$(USRBIN.32)/python3.4
 
 PYTHON.32 =	$(PYTHON.$(PYTHON_VERSION).32)
 PYTHON.64 =	$(PYTHON.$(PYTHON_VERSION).64)