--- a/components/python/python34/Makefile Tue Apr 07 05:51:44 2015 -0700
+++ b/components/python/python34/Makefile Thu Apr 09 18:28:09 2015 -0700
@@ -173,6 +173,11 @@
COMPONENT_TEST_ENV += PATH="$(SPRO_VROOT)/bin:$(PATH)"
# Prevent the tests from getting stuck waiting for input.
COMPONENT_TEST_TARGETS = test < /dev/null
+# Some different values for system testing.
+COMPONENT_SYSTEM_TEST_ENV = EXTRATESTOPTS="-v -uall,-network $(TESTOPTS_PYTHON_TEST)"
+COMPONENT_SYSTEM_TEST_TARGETS =
+COMPONENT_SYSTEM_TEST_CMD= $(PYTHON.3.4)
+COMPONENT_SYSTEM_TEST_ARGS= /usr/lib/python3.4/test/regrtest.py -v -uall,-network
# The test output contains details from each test, in whatever order they
# complete. The default _TRANSFORMER is not powerful enough to deal with
@@ -182,6 +187,7 @@
COMPONENT_TEST_TRANSFORMS = "'/tests OK./ {results = 1}; /Re-running failed tests in verbose mode/ {results = 0} {if (results) print $0 } '"
test: $(TEST_64)
+system-test: $(SYSTEM_TEST_64)
REQUIRED_PACKAGES += compress/bzip2
REQUIRED_PACKAGES += compress/xz
--- a/doc/testing.txt Tue Apr 07 05:51:44 2015 -0700
+++ b/doc/testing.txt Thu Apr 09 18:28:09 2015 -0700
@@ -9,7 +9,8 @@
'gmake test' is often run when a component is upgraded or otherwise
intentionally changed, but sometimes it would be useful to rerun the tests
after something else has changed (such as the system being upgraded, or a
-change in compilers) and see if that has affected the tests.
+change in compilers; see the 'System Test' section below) and see if that
+has affected the tests.
We do this by having a 'master test file' that contains the expected results,
and having a compare target that runs the tests and compares them with the
@@ -208,3 +209,53 @@
When your master test file(s) are in good shape, then you should "hg add"
them to your workspace.
+
+
+System Test
+-----------
+All of the above discusses how 'gmake test' works. This is all good, but
+has the limitation that objects under test are those that were just built
+and installed into the component's proto area. We also want to be able
+to test whatever is installed on the actual system. This has the benefits
+of not requiring anything to be built, and allowing non-Userland people
+to test our bits. (Think of people in ON changing libc or the linker and
+wanting to make sure they don't break anything.)
+
+To this end, we have added 'gmake system-test'. It works just like 'gmake
+test', reusing some of the same variables:
+* COMPONENT_TEST_BUILD_DIR
+* COMPONENT_TEST_COMPARE
+* COMPONENT_TEST_CREATE_TRANSFORMS
+* COMPONENT_TEST_OUTPUT
+* COMPONENT_TEST_PERFORM_TRANSFORM
+but with its own "_SYSTEM" instance of other variables:
+* COMPONENT_POST_SYSTEM_TEST_ACTION
+* COMPONENT_PRE_SYSTEM_TEST_ACTION
+* COMPONENT_SYSTEM_TEST_ARGS
+* COMPONENT_SYSTEM_TEST_CLEANUP
+* COMPONENT_SYSTEM_TEST_CMD
+* COMPONENT_SYSTEM_TEST_DIR
+* COMPONENT_SYSTEM_TEST_ENV
+* COMPONENT_SYSTEM_TEST_ENV_CMD
+* COMPONENT_SYSTEM_TEST_RESULTS_DIR
+* COMPONENT_SYSTEM_TEST_TARGETS
+
+In the ideal case, only Makefile variables would need to be modified to
+take a component where 'gmake test' works and extend it so that 'gmake
+system-test' works; see components/python34/Makefile for such an example.
+
+The next level up from that is the case where some simple patching is needed.
+In this case, simple means:
+* tweaking LD_LIBRARY_PATH
+* changing the path to a binary to run
+* modifying a Makefile so tests are compiled as part of 'gmake build'
+* modifying a Makefile so tests are installed as part of 'gmake install'
+* modifying a pkg(5) manifest so tests are published as part of 'gmake publish'
+ (preferably under the "optional.test" facet)
+
+If more extensive patching is needed, a judgement call needs to be made:
+Are these changes that upstream would welcome? If not, then we would need
+to keep such changes around indefinitely, which would constitute a fork,
+which we try very hard to avoid. When 'gmake system-test' was initially
+implemented, changes to both fetchmail and python/decorator were begun but
+ultimately abandoned for this reason; see 20808505 for details.
--- a/make-rules/ant.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/ant.mk Thu Apr 09 18:28:09 2015 -0700
@@ -50,6 +50,10 @@
COMPONENT_TEST_ENV += JAVA_HOME="$(JAVA_HOME)"
COMPONENT_TEST_CMD = $(ANT)
+COMPONENT_SYSTEM_TEST_ENV_CMD = $(ENV)
+COMPONENT_SYSTEM_TEST_ENV += JAVA_HOME="$(JAVA_HOME)"
+COMPONENT_SYSTEM_TEST_CMD = $(ANT)
+
# test the built source
$(BUILD_DIR)/%/.tested-and-compared: $(BUILD_DIR)/%/.built
$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
@@ -77,6 +81,33 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared: $(SOURCE_DIR)/.prep
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(SOURCE_DIR)/.prep
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ (cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/attpackagemake.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/attpackagemake.mk Thu Apr 09 18:28:09 2015 -0700
@@ -60,9 +60,11 @@
# The install and test process needs the same environment as the build
COMPONENT_INSTALL_ENV = $(COMPONENT_BUILD_ENV)
+COMPONENT_INSTALL_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
COMPONENT_TEST_ENV = $(COMPONENT_BUILD_ENV)
-COMPONENT_INSTALL_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
COMPONENT_TEST_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
+COMPONENT_SYSTEM_TEST_ENV = $(COMPONENT_BUILD_ENV)
+COMPONENT_SYSTEM_TEST_ARGS = HOSTTYPE="$(HOSTTYPE$(BITS))"
# build the configured source
$(BUILD_DIR)/%/.built: $(SOURCE_DIR)/.prep
@@ -115,6 +117,33 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared: $(SOURCE_DIR)/.prep
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ bin/package test $(COMPONENT_SYSTEM_TEST_TARGETS) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(SOURCE_DIR)/.prep
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ bin/package test $(COMPONENT_SYSTEM_TEST_TARGETS) \
+ $(COMPONENT_SYSTEM_TEST_ARGS)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/configure.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/configure.mk Thu Apr 09 18:28:09 2015 -0700
@@ -161,6 +161,33 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared: $(SOURCE_DIR)/.prep
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(SOURCE_DIR)/.prep
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ (cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/justmake.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/justmake.mk Thu Apr 09 18:28:09 2015 -0700
@@ -94,6 +94,33 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared: $(SOURCE_DIR)/.prep
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(SOURCE_DIR)/.prep
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ (cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/makemaker.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/makemaker.mk Thu Apr 09 18:28:09 2015 -0700
@@ -55,6 +55,20 @@
$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared: PERL_VERSION=5.16
$(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared: BITS=64
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested: PERL_VERSION=5.12
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested: BITS=32
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested: PERL_VERSION=5.12-mt
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested: BITS=32
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested: PERL_VERSION=5.16
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested: BITS=64
+
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared: PERL_VERSION=5.12
+$(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared: BITS=32
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared: PERL_VERSION=5.12-mt
+$(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared: BITS=32
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared: PERL_VERSION=5.16
+$(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared: BITS=64
+
BUILD_32 = $(BUILD_DIR)/$(MACH32)-5.12/.built
BUILD_32 += $(BUILD_DIR)/$(MACH32)-5.12-mt/.built
BUILD_64 = $(BUILD_DIR)/$(MACH64)-5.16/.built
@@ -106,7 +120,9 @@
COMPONENT_TEST_TRANSFORM_CMD = $(COMPONENT_TEST_BUILD_DIR)/transform-$(PERL_VERSION)-$(BITS)-results
COMPONENT_TEST_TARGETS = check
-COMPONENT_TEST_ENV += $(COMMON_PERL_ENV)
+COMPONENT_TEST_ENV += $(COMMON_PERL_ENV)
+COMPONENT_SYSTEM_TEST_TARGETS = check
+COMPONENT_SYSTEM_TEST_ENV += $(COMMON_PERL_ENV)
# determine the type of tests we want to run.
ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
@@ -119,6 +135,16 @@
TEST_64 = $(BUILD_DIR)/$(MACH64)-5.16/.tested-and-compared
endif
+ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_32 = $(BUILD_DIR)/$(MACH32)-5.12/.system-tested
+SYSTEM_TEST_32 += $(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested
+SYSTEM_TEST_64 = $(BUILD_DIR)/$(MACH64)-5.16/.system-tested
+else
+SYSTEM_TEST_32 = $(BUILD_DIR)/$(MACH32)-5.12/.system-tested-and-compared
+SYSTEM_TEST_32 += $(BUILD_DIR)/$(MACH32)-5.12-mt/.system-tested-and-compared
+SYSTEM_TEST_64 = $(BUILD_DIR)/$(MACH64)-5.16/.system-tested-and-compared
+endif
+
# test the built source
$(BUILD_DIR)/%/.tested-and-compared: $(BUILD_DIR)/%/.built
$(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
@@ -146,6 +172,33 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed packages
+$(BUILD_DIR)/%/.system-tested-and-compared: $(SOURCE_DIR)/.prep
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS)) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(SOURCE_DIR)/.prep
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ (cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) \
+ $(COMPONENT_SYSTEM_TEST_ARGS) $(COMPONENT_SYSTEM_TEST_TARGETS))
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/setup.py.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/setup.py.mk Thu Apr 09 18:28:09 2015 -0700
@@ -71,6 +71,7 @@
COMPONENT_BUILD_ENV += $(PYTHON_ENV)
COMPONENT_INSTALL_ENV += $(PYTHON_ENV)
COMPONENT_TEST_ENV += $(PYTHON_ENV)
+COMPONENT_SYSTEM_TEST_ENV += $(PYTHON_ENV)
# If we are building Python 2.7 or 3.4 support, build them and install them
# before Python 2.6, so 2.6 is installed last and is the canonical version.
@@ -135,6 +136,13 @@
COMPONENT_TEST_CMD = $(PYTHON)
COMPONENT_TEST_ARGS += ./runtests.py
+COMPONENT_SYSTEM_TEST_DEP = $(SOURCE_DIR)/.prep
+COMPONENT_SYSTEM_TEST_DIR = $(COMPONENT_SRC)/test
+COMPONENT_SYSTEM_TEST_ENV_CMD = $(ENV)
+COMPONENT_SYSTEM_TEST_ENV += PYTHONPATH=$(PROTO_DIR)$(PYTHON_VENDOR_PACKAGES)
+COMPONENT_SYSTEM_TEST_CMD = $(PYTHON)
+COMPONENT_SYSTEM_TEST_ARGS += ./runtests.py
+
# determine the type of tests we want to run.
ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.tested)
@@ -148,6 +156,18 @@
ifeq ($(PYTHON_VERSION),3.4)
TEST_32_and_64 = $(TEST_64)
endif
+ifeq ($(strip $(wildcard $(COMPONENT_SYSTEM_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.system-tested)
+SYSTEM_TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.system-tested)
+SYSTEM_TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.system-tested)
+else
+SYSTEM_TEST_32 = $(PYTHON2_VERSIONS:%=$(BUILD_DIR)/$(MACH32)-%/.system-tested-and-compared)
+SYSTEM_TEST_64 = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH64)-%/.system-tested-and-compared)
+SYSTEM_TEST_NO_ARCH = $(PYTHON_VERSIONS:%=$(BUILD_DIR)/$(MACH)-%/.system-tested-and-compared)
+endif
+ifeq ($(PYTHON_VERSION),3.4)
+SYSTEM_TEST_32_and_64 = $(SYSTEM_TEST_64)
+endif
# test the built source
$(BUILD_DIR)/%/.tested-and-compared: $(COMPONENT_TEST_DEP)
@@ -174,6 +194,31 @@
$(COMPONENT_TEST_CLEANUP)
$(TOUCH) $@
+# test the installed system
+$(BUILD_DIR)/%/.system-tested-and-compared: $(COMPONENT_SYSTEM_TEST_DEP)
+ $(RM) -rf $(COMPONENT_TEST_BUILD_DIR)
+ $(MKDIR) $(COMPONENT_TEST_BUILD_DIR)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ -(cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) $(COMPONENT_SYSTEM_TEST_ARGS)) \
+ &> $(COMPONENT_TEST_OUTPUT)
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_TEST_CREATE_TRANSFORMS)
+ $(COMPONENT_TEST_PERFORM_TRANSFORM)
+ $(COMPONENT_TEST_COMPARE)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
+$(BUILD_DIR)/%/.system-tested: $(COMPONENT_SYSTEM_TEST_DEP)
+ $(COMPONENT_PRE_SYSTEM_TEST_ACTION)
+ (cd $(COMPONENT_SYSTEM_TEST_DIR) ; \
+ $(COMPONENT_SYSTEM_TEST_ENV_CMD) $(COMPONENT_SYSTEM_TEST_ENV) \
+ $(COMPONENT_SYSTEM_TEST_CMD) $(COMPONENT_SYSTEM_TEST_ARGS))
+ $(COMPONENT_POST_SYSTEM_TEST_ACTION)
+ $(COMPONENT_SYSTEM_TEST_CLEANUP)
+ $(TOUCH) $@
+
ifeq ($(strip $(PARFAIT_BUILD)),yes)
parfait: build
else
--- a/make-rules/shared-macros.mk Tue Apr 07 05:51:44 2015 -0700
+++ b/make-rules/shared-macros.mk Thu Apr 09 18:28:09 2015 -0700
@@ -240,6 +240,7 @@
# set the default master test results directory
COMPONENT_TEST_RESULTS_DIR = $(COMPONENT_DIR)/test
+COMPONENT_SYSTEM_TEST_RESULTS_DIR = $(COMPONENT_DIR)/test
# set the default master test results file
COMPONENT_TEST_MASTER = $(COMPONENT_TEST_RESULTS_DIR)/results-$(BITS).master
@@ -320,17 +321,31 @@
# set the default env command to use for test of the component
COMPONENT_TEST_ENV_CMD = $(ENV)
+COMPONENT_SYSTEM_TEST_ENV_CMD = $(ENV)
# set the default command to use for test of the component
COMPONENT_TEST_CMD = $(GMAKE)
+COMPONENT_SYSTEM_TEST_CMD = $(GMAKE)
# set the default target for test of the component
COMPONENT_TEST_TARGETS = check
+COMPONENT_SYSTEM_TEST_TARGETS = check
# set the default directory for test of the component
COMPONENT_TEST_DIR = $(@D)
+COMPONENT_SYSTEM_TEST_DIR = $(@D)
# determine the type of tests we want to run.
+ifeq ($(strip $(wildcard $(COMPONENT_SYSTEM_TEST_RESULTS_DIR)/results-*.master)),)
+SYSTEM_TEST_NO_ARCH = $(BUILD_DIR_NO_ARCH)/.system-tested
+SYSTEM_TEST_32 = $(BUILD_DIR_32)/.system-tested
+SYSTEM_TEST_64 = $(BUILD_DIR_64)/.system-tested
+else
+SYSTEM_TEST_NO_ARCH = $(BUILD_DIR_NO_ARCH)/.system-tested-and-compared
+SYSTEM_TEST_32 = $(BUILD_DIR_32)/.system-tested-and-compared
+SYSTEM_TEST_64 = $(BUILD_DIR_64)/.system-tested-and-compared
+endif
+SYSTEM_TEST_32_and_64 = $(TEST_32) $(TEST_64)
ifeq ($(strip $(wildcard $(COMPONENT_TEST_RESULTS_DIR)/results-*.master)),)
TEST_NO_ARCH = $(BUILD_DIR_NO_ARCH)/.tested
TEST_32 = $(BUILD_DIR_32)/.tested
@@ -351,12 +366,18 @@
SKIP_TEST_AT_TOP_LEVEL = $(eval include $(WS_MAKE_RULES)/skip-test.mk)
-$(BUILD_DIR_NO_ARCH)/.tested: BITS=32
-$(BUILD_DIR_32)/.tested: BITS=32
-$(BUILD_DIR_64)/.tested: BITS=64
-$(BUILD_DIR_NO_ARCH)/.tested-and-compared: BITS=32
-$(BUILD_DIR_32)/.tested-and-compared: BITS=32
-$(BUILD_DIR_64)/.tested-and-compared: BITS=64
+$(BUILD_DIR_NO_ARCH)/.system-tested: BITS=32
+$(BUILD_DIR_32)/.system-tested: BITS=32
+$(BUILD_DIR_64)/.system-tested: BITS=64
+$(BUILD_DIR_NO_ARCH)/.system-tested-and-compared: BITS=32
+$(BUILD_DIR_32)/.system-tested-and-compared: BITS=32
+$(BUILD_DIR_64)/.system-tested-and-compared: BITS=64
+$(BUILD_DIR_NO_ARCH)/.tested: BITS=32
+$(BUILD_DIR_32)/.tested: BITS=32
+$(BUILD_DIR_64)/.tested: BITS=64
+$(BUILD_DIR_NO_ARCH)/.tested-and-compared: BITS=32
+$(BUILD_DIR_32)/.tested-and-compared: BITS=32
+$(BUILD_DIR_64)/.tested-and-compared: BITS=64
# BUILD_TOOLS is the root of all tools not normally installed on the system.
@@ -447,6 +468,10 @@
PYTHON_VENDOR_PACKAGES.64 = /usr/lib/python$(PYTHON_VERSION)/vendor-packages/64
PYTHON_VENDOR_PACKAGES = $(PYTHON_VENDOR_PACKAGES.$(BITS))
+PYTHON.2.6.TEST = /usr/lib/python2.6/test
+PYTHON.2.7.TEST = /usr/lib/python2.7/test
+PYTHON.3.4.TEST = /usr/lib/python3.4/test
+
USRBIN.32 = /usr/bin
USRBIN.64 = /usr/bin/$(MACH64)
USRBIN = $(USRBIN.$(BITS))
@@ -465,7 +490,7 @@
# than the 64-bit path.
PYTHON.3.4.32 = $(USRBIN.32)/python3.4
PYTHON.3.4.64 = $(USRBIN.32)/python3.4
-PYTHON.3.4 = $(USRBIN)/python3.4
+PYTHON.3.4 = $(USRBIN.32)/python3.4
PYTHON.32 = $(PYTHON.$(PYTHON_VERSION).32)
PYTHON.64 = $(PYTHON.$(PYTHON_VERSION).64)