20222479 Need a method to compare test results against a master in Userland
authorRich Burridge <rich.burridge@oracle.com>
Wed, 17 Dec 2014 15:33:37 -0800
changeset 3558 2cec274f17fc
parent 3557 39d5a85cfd14
child 3561 8806d147c2d5
20222479 Need a method to compare test results against a master in Userland
doc/makefile-variables.txt
make-rules/ant.mk
make-rules/attpackagemake.mk
make-rules/configure.mk
make-rules/justmake.mk
make-rules/makemaker.mk
make-rules/setup.py.mk
make-rules/shared-macros.mk
--- a/doc/makefile-variables.txt	Wed Dec 17 13:31:44 2014 -0800
+++ b/doc/makefile-variables.txt	Wed Dec 17 15:33:37 2014 -0800
@@ -72,6 +72,19 @@
 * COMPONENT_TEST_ARGS is little used.
 * COMPONENT_TEST_ENV is mainly used for altering PATH and friends.
 
+If your component needs to do some kind of cleanup after a "gmake test" run,
+such as kill processes after doing a "gmake test" run, then this can be done
+by setting COMPONENT_TEST_CLEANUP.
+
+If you have created master test results file(s) for your component in the
+COMPONENT_TEST_RESULTS_DIR directory, then in order to successfully compare
+your test results against that master results file, you might need to
+normalize some of the test output lines. This is done via a set of regular
+expressions that are applied to the test results. There are some global
+default ones in the COMPONENT_TEST_TRANSFORMS definition in shared-macros.mk,
+but your component Makefile might have to += some more for specific transforms
+that need to be applied to the test output for just this component.
+
 * COMPONENT_POST_UNPACK_ACTION is for making minor alterations to the unpacked
   source directory before any patching has taken place.  It should almost never
   be used.
--- a/make-rules/ant.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/ant.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -46,15 +46,33 @@
 	$(COMPONENT_POST_INSTALL_ACTION)
 	$(TOUCH) [email protected]
 
+COMPONENT_TEST_ENV_CMD = $(ENV)
 COMPONENT_TEST_ENV += JAVA_HOME="$(JAVA_HOME)"
 COMPONENT_TEST_CMD = $(ANT)
+
 # test the built source
-$(BUILD_DIR)/%/.tested: $(BUILD_DIR)/%/.built
+$(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(COMPONENT_PRE_TEST_ACTION)
-	(cd $(COMPONENT_TEST_DIR) ; $(ENV) $(COMPONENT_TEST_ENV) \
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
 		$(COMPONENT_TEST_CMD) \
-			$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(BUILD_DIR)/%/.built
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) \
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/attpackagemake.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/attpackagemake.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -89,13 +89,28 @@
 	$(TOUCH) [email protected]
 
 # test the built source
-$(BUILD_DIR)/%/.tested: $(BUILD_DIR)/%/.built
+$(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(COMPONENT_PRE_TEST_ACTION)
-	cd $(COMPONENT_TEST_DIR); $(COMPONENT_TEST_ENV_CMD) \
-		$(COMPONENT_TEST_ENV) \
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		bin/package test $(COMPONENT_TEST_TARGETS) \
+		$(COMPONENT_TEST_ARGS) \
+		&> $(COMPONENT_TEST_OUTPUT)
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(BUILD_DIR)/%/.built
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
 		bin/package test $(COMPONENT_TEST_TARGETS) \
 		$(COMPONENT_TEST_ARGS)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/configure.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/configure.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -135,13 +135,28 @@
 	$(TOUCH) [email protected]
 
 # test the built source
-$(BUILD_DIR)/%/.tested:	$(BUILD_DIR)/%/.built
+$(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(COMPONENT_PRE_TEST_ACTION)
-	(cd $(COMPONENT_TEST_DIR) ; $(COMPONENT_TEST_ENV_CMD) \
-		$(COMPONENT_TEST_ENV) \
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
 		$(COMPONENT_TEST_CMD) \
-			$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(BUILD_DIR)/%/.built
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) \
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/justmake.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/justmake.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -67,15 +67,29 @@
 	$(COMPONENT_POST_INSTALL_ACTION)
 	$(TOUCH) [email protected]
 
-
 # test the built source
-$(BUILD_DIR)/%/.tested: $(BUILD_DIR)/%/.built
+$(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(COMPONENT_PRE_TEST_ACTION)
-	(cd $(COMPONENT_TEST_DIR) ; $(COMPONENT_TEST_ENV_CMD) \
-		$(COMPONENT_TEST_ENV) \
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
 		$(COMPONENT_TEST_CMD) \
-			$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(BUILD_DIR)/%/.built
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) \
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/makemaker.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/makemaker.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -48,6 +48,13 @@
 $(BUILD_DIR)/$(MACH64)-5.16/.tested:	PERL_VERSION=5.16
 $(BUILD_DIR)/$(MACH64)-5.16/.tested:	BITS=64
 
+$(BUILD_DIR)/$(MACH32)-5.12/.tested-and-compared:	PERL_VERSION=5.12
+$(BUILD_DIR)/$(MACH32)-5.12/.tested-and-compared:	BITS=32
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.tested-and-compared:	PERL_VERSION=5.12-mt
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.tested-and-compared:	BITS=32
+$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared:	PERL_VERSION=5.16
+$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared:	BITS=64
+
 BUILD_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.built
 BUILD_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.built
 BUILD_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.built
@@ -56,10 +63,6 @@
 INSTALL_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.installed
 INSTALL_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.installed
 
-TEST_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.tested
-TEST_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.tested
-TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.tested
-
 COMPONENT_CONFIGURE_ENV +=	$(COMMON_PERL_ENV)
 COMPONENT_CONFIGURE_ENV +=	PERL="$(PERL)"
 $(BUILD_DIR)/%/.configured:	$(SOURCE_DIR)/.prep
@@ -95,15 +98,50 @@
 	$(TOUCH) [email protected]
 
 
+# Define bit specific and Perl version specific filenames.
+COMPONENT_TEST_MASTER = $(COMPONENT_TEST_RESULTS_DIR)/results-$(PERL_VERSION)-$(BITS).master
+COMPONENT_TEST_OUTPUT = $(COMPONENT_TEST_RESULTS_DIR)/test-$(PERL_VERSION)-$(BITS)-results
+COMPONENT_TEST_DIFFS =  $(COMPONENT_TEST_RESULTS_DIR)/test-$(PERL_VERSION)-$(BITS)-diffs
+COMPONENT_TEST_SNAPSHOT = $(COMPONENT_TEST_RESULTS_DIR)/results-$(PERL_VERSION)-$(BITS).snapshot
+COMPONENT_TEST_TRANSFORM_CMD = $(COMPONENT_TEST_RESULTS_DIR)/transform-$(PERL_VERSION)-$(BITS)-results
+
 COMPONENT_TEST_TARGETS =	check
 COMPONENT_TEST_ENV +=	$(COMMON_PERL_ENV)
-$(BUILD_DIR)/%/.tested:	$(BUILD_DIR)/%/.built
+
+# determine the type of tests we want to run.
+ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
+TEST_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.tested
+TEST_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.tested
+TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.tested
+else
+TEST_32 =	$(BUILD_DIR)/$(MACH32)-5.12/.tested-and-compared
+TEST_32 +=	$(BUILD_DIR)/$(MACH32)-5.12-mt/.tested-and-compared
+TEST_64 =	$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared
+endif
+
+# test the built source
+$(BUILD_DIR)/%/.tested-and-compared:    $(BUILD_DIR)/%/.built
 	$(COMPONENT_PRE_TEST_ACTION)
-	(cd $(COMPONENT_TEST_DIR) ; $(COMPONENT_TEST_ENV_CMD) \
-		$(COMPONENT_TEST_ENV) \
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
 		$(COMPONENT_TEST_CMD) \
-			$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(BUILD_DIR)/%/.built
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) \
+		$(COMPONENT_TEST_ARGS) $(COMPONENT_TEST_TARGETS))
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/setup.py.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/setup.py.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -39,6 +39,12 @@
 $(BUILD_DIR)/$(MACH32)-%/.tested:	BITS=32
 $(BUILD_DIR)/$(MACH64)-%/.tested:	BITS=64
 
+$(BUILD_DIR)/%-2.6/.tested-and-compared:	PYTHON_VERSION=2.6
+$(BUILD_DIR)/%-2.7/.tested-and-compared:	PYTHON_VERSION=2.7
+$(BUILD_DIR)/%-3.4/.tested-and-compared:	PYTHON_VERSION=3.4
+$(BUILD_DIR)/$(MACH32)-%/.tested-and-compared:	BITS=32
+$(BUILD_DIR)/$(MACH64)-%/.tested-and-compared:	BITS=64
+
 BUILD_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.built)
 BUILD_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.built)
 BUILD_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.built)
@@ -53,13 +59,6 @@
 INSTALL_32_and_64 = $(INSTALL_64)
 endif
 
-TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.tested)
-TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.tested)
-TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.tested)
-ifeq ($(PYTHON_VERSION),3.4)
-TEST_32_and_64 = $(TEST_64)
-endif
-
 PYTHON_ENV =	CC="$(CC)"
 PYTHON_ENV +=	CFLAGS="$(CFLAGS)"
 PYTHON_ENV +=	PATH="$(PATH):$(SPRO_VROOT)/bin"
@@ -117,6 +116,13 @@
 	$(COMPONENT_POST_INSTALL_ACTION)
 	$(TOUCH) [email protected]
 
+# Define bit specific and Python version specific filenames.
+COMPONENT_TEST_MASTER =	$(COMPONENT_TEST_RESULTS_DIR)/results-$(PYTHON_VERSION)-$(BITS).master
+COMPONENT_TEST_OUTPUT =	$(COMPONENT_TEST_RESULTS_DIR)/test-$(PYTHON_VERSION)-$(BITS)-results
+COMPONENT_TEST_DIFFS =	$(COMPONENT_TEST_RESULTS_DIR)/test-$(PYTHON_VERSION)-$(BITS)-diffs
+COMPONENT_TEST_SNAPSHOT = $(COMPONENT_TEST_RESULTS_DIR)/results-$(PYTHON_VERSION)-$(BITS).snapshot
+COMPONENT_TEST_TRANSFORM_CMD = $(COMPONENT_TEST_RESULTS_DIR)/transform-$(PYTHON_VERSION)-$(BITS)-results
+
 COMPONENT_TEST_DEP =	$(BUILD_DIR)/%/.installed
 COMPONENT_TEST_DIR =	$(COMPONENT_SRC)/test
 COMPONENT_TEST_ENV_CMD =	$(ENV)
@@ -124,13 +130,41 @@
 COMPONENT_TEST_CMD =	$(PYTHON)
 COMPONENT_TEST_ARGS +=	./runtests.py
 
+# determine the type of tests we want to run.
+ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
+TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.tested)
+TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.tested)
+TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.tested)
+else
+TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.tested-and-compared)
+TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.tested-and-compared)
+TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.tested-and-compared)
+endif
+ifeq ($(PYTHON_VERSION),3.4)
+TEST_32_and_64 = $(TEST_64)
+endif
+
 # test the built source
-$(BUILD_DIR)/%/.tested:	$(COMPONENT_TEST_DEP)
+$(BUILD_DIR)/%/.tested-and-compared:    $(COMPONENT_TEST_DEP)
 	$(COMPONENT_PRE_TEST_ACTION)
-	(cd $(COMPONENT_TEST_DIR); $(COMPONENT_TEST_ENV_CMD) \
-		$(COMPONENT_TEST_ENV) \
-		$(COMPONENT_TEST_CMD) $(COMPONENT_TEST_ARGS) )
+	-(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) $(COMPONENT_TEST_ARGS)) \
+		&> $(COMPONENT_TEST_OUTPUT)
 	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CREATE_TRANSFORMS)
+	$(COMPONENT_TEST_PERFORM_TRANSFORM)
+	$(COMPONENT_TEST_COMPARE)
+	$(COMPONENT_TEST_CLEANUP)
+	$(TOUCH) [email protected]
+
+$(BUILD_DIR)/%/.tested:    $(COMPONENT_TEST_DEP)
+	$(COMPONENT_PRE_TEST_ACTION)
+	(cd $(COMPONENT_TEST_DIR) ; \
+		$(COMPONENT_TEST_ENV_CMD) $(COMPONENT_TEST_ENV) \
+		$(COMPONENT_TEST_CMD) $(COMPONENT_TEST_ARGS))
+	$(COMPONENT_POST_TEST_ACTION)
+	$(COMPONENT_TEST_CLEANUP)
 	$(TOUCH) [email protected]
 
 ifeq   ($(strip $(PARFAIT_BUILD)),yes)
--- a/make-rules/shared-macros.mk	Wed Dec 17 13:31:44 2014 -0800
+++ b/make-rules/shared-macros.mk	Wed Dec 17 15:33:37 2014 -0800
@@ -224,11 +224,73 @@
 # set the default target for installation of the component
 COMPONENT_INSTALL_TARGETS =	install
 
-TEST_32 =		$(BUILD_DIR_32)/.tested
-TEST_64 =		$(BUILD_DIR_64)/.tested
-TEST_32_and_64 =	$(TEST_32) $(TEST_64)
-$(BUILD_DIR_32)/.tested:       BITS=32
-$(BUILD_DIR_64)/.tested:       BITS=64
+# set the default test results directory
+COMPONENT_TEST_RESULTS_DIR =	$(COMPONENT_DIR)/test
+
+# set the default master test results file
+COMPONENT_TEST_MASTER =		$(COMPONENT_TEST_RESULTS_DIR)/results-$(BITS).master
+
+# set the default test results output file
+COMPONENT_TEST_OUTPUT =		$(COMPONENT_TEST_RESULTS_DIR)/test-$(BITS)-results
+
+# set the default test results comparison diffs file
+COMPONENT_TEST_DIFFS =		$(COMPONENT_TEST_RESULTS_DIR)/test-$(BITS)-diffs
+
+# set the default test snapshot file
+COMPONENT_TEST_SNAPSHOT =	$(COMPONENT_TEST_RESULTS_DIR)/results-$(BITS).snapshot
+
+# The set of default transforms to be applied to the test results to try
+# to normalize them.
+COMPONENT_TEST_TRANSFORMS = \
+	'-e "s|$(@D)|\\$$(@D)|g" ' \
+	'-e "s|$(PERL)|\\$$(PERL)|g" ' \
+	'-e "s|$(SOURCE_DIR)|\\$$(SOURCE_DIR)|g" '
+
+# set the default commands used to generate the file containing the set
+# of transforms to be applied to the test results to try to normalize them.
+COMPONENT_TEST_CREATE_TRANSFORMS = \
+	if [ -e $(COMPONENT_TEST_MASTER) ]; \
+	then \
+		print "\#!/bin/sh" > $(COMPONENT_TEST_TRANSFORM_CMD); \
+        	print '$(GSED) ' \
+			$(COMPONENT_TEST_TRANSFORMS) \
+                	' \\' >> $(COMPONENT_TEST_TRANSFORM_CMD); \
+        	print '$(COMPONENT_TEST_OUTPUT) \\' \
+                	>> $(COMPONENT_TEST_TRANSFORM_CMD); \
+        	print '> $(COMPONENT_TEST_SNAPSHOT)' \
+                	>> $(COMPONENT_TEST_TRANSFORM_CMD); \
+	fi
+
+# set the default command for performing any test result munging
+COMPONENT_TEST_TRANSFORM_CMD =	$(COMPONENT_TEST_RESULTS_DIR)/transform-$(BITS)-results
+
+# set the default operation to run to perform test result normalization
+COMPONENT_TEST_PERFORM_TRANSFORM = \
+	if [ -e $(COMPONENT_TEST_MASTER) ]; \
+	then \
+		$(SHELL) $(COMPONENT_TEST_TRANSFORM_CMD); \
+	fi
+
+# set the default command used to compare the master results with the snapshot
+COMPONENT_TEST_COMPARE_CMD =	$(GDIFF) -uN
+
+# set the default way that master and snapshot test results are compared
+COMPONENT_TEST_COMPARE = \
+	if [ -e $(COMPONENT_TEST_MASTER) ]; \
+	then \
+		$(COMPONENT_TEST_COMPARE_CMD) \
+			$(COMPONENT_TEST_MASTER) $(COMPONENT_TEST_SNAPSHOT) \
+			> $(COMPONENT_TEST_DIFFS); \
+		print "Test results in $(COMPONENT_TEST_OUTPUT)"; \
+		if [ -s $(COMPONENT_TEST_DIFFS) ]; \
+		then \
+			print "Differences found."; \
+			$(CAT) $(COMPONENT_TEST_DIFFS); \
+			exit 2; \
+		else \
+			print "No differences found."; \
+		fi \
+	fi
 
 # set the default env command to use for test of the component
 COMPONENT_TEST_ENV_CMD =        $(ENV)
@@ -242,6 +304,21 @@
 # set the default directory for test of the component
 COMPONENT_TEST_DIR =		$(@D)
 
+# determine the type of tests we want to run.
+ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
+TEST_32 =		$(BUILD_DIR_32)/.tested
+TEST_64 =		$(BUILD_DIR_64)/.tested
+else
+TEST_32 =		$(BUILD_DIR_32)/.tested-and-compared
+TEST_64 =		$(BUILD_DIR_64)/.tested-and-compared
+endif
+TEST_32_and_64 =	$(TEST_32) $(TEST_64)
+$(BUILD_DIR_32)/.tested:		BITS=32
+$(BUILD_DIR_64)/.tested:		BITS=64
+$(BUILD_DIR_32)/.tested-and-compared:	BITS=32
+$(BUILD_DIR_64)/.tested-and-compared:	BITS=64
+
+
 # BUILD_TOOLS is the root of all tools not normally installed on the system.
 BUILD_TOOLS ?=	/ws/on12-tools
 
@@ -404,6 +481,8 @@
 GPATCH_BACKUP =	--backup --version-control=numbered
 GPATCH_FLAGS =	-p$(PATCH_LEVEL) $(GPATCH_BACKUP)
 GSED =		/usr/gnu/bin/sed
+GDIFF =		/usr/gnu/bin/diff
+GSORT =		/usr/gnu/bin/sort
 
 PKGREPO =	/usr/bin/pkgrepo
 PKGSEND =	/usr/bin/pkgsend
@@ -424,6 +503,7 @@
 CP =		/bin/cp -f
 MV =		/bin/mv -f
 LN =		/bin/ln
+CAT =		/bin/cat
 SYMLINK =	/bin/ln -s
 ENV =		/usr/bin/env
 INSTALL =	/usr/bin/ginstall