6771457 explore compiler performance improvements for python
authorShawn Walker <shawn.walker@oracle.com>
Tue, 20 Dec 2011 17:19:09 -0800
changeset 634 bd9c94f9f6e0
parent 633 29eebbeefc09
child 635 d4b487f99c39
6771457 explore compiler performance improvements for python
components/python/python26/Makefile
components/python/python26/patches/Python26-13-cflags.patch
components/python/python26/patches/Python26-24-symbolic.patch
components/python/python26/patches/Python26-25-studio-profile.patch
components/python/python26/patches/Python26-26-pybench.patch
components/python/python27/Makefile
components/python/python27/patches/10-cflags.patch
components/python/python27/patches/11-symbolic.patch
components/python/python27/patches/12-studio-profile.patch
--- a/components/python/python26/Makefile	Tue Dec 20 10:39:45 2011 -0800
+++ b/components/python/python26/Makefile	Tue Dec 20 17:19:09 2011 -0800
@@ -50,6 +50,18 @@
 	 perl -pi -e 's/^(\#define _XOPEN_SOURCE_EXTENDED.*)/\/* $$1 *\//' \
 			pyconfig.h)
 
+# The python build is profile-guided for studio; to see the benefits of that,
+# Python must be compiled with -xO5 and a different build target must be used.
+# Use of xprofile requires that the same options be used during compilation and
+# linking.  The targets chosen are based on Solaris 11 minimum supported system
+# requirements.
+COMPONENT_BUILD_TARGETS = profile-opt
+XPROFILE_DIR = $(BUILD_DIR_$(BITS))/.profile
+PYFLAGS.i386 = -xtarget=opteron -xarch=sse2 -xcache=generic
+PYFLAGS.sparc =
+CFLAGS += -xO5 $(PYFLAGS.$(MACH))
+LDFLAGS += -xO5 $(PYFLAGS.$(MACH))
+
 # we don't want to leak $(CC_BITS) into BASECFLAGS as it causes problems with
 # python-config
 CC +=	$(CFLAGS)
@@ -69,7 +81,6 @@
 
 CONFIGURE_OPTIONS  +=		--infodir=$(CONFIGURE_INFODIR)
 CONFIGURE_OPTIONS  +=		--enable-shared
-CONFIGURE_OPTIONS  +=		--disable-static
 CONFIGURE_OPTIONS  +=		--with-system-ffi
 CONFIGURE_OPTIONS  +=		--without-gcc
 CONFIGURE_OPTIONS  +=		ac_cv_opt_olimit_ok=no
@@ -80,7 +91,9 @@
 CONFIGURE_OPTIONS  +=		CXX="$(CXX)"
 CONFIGURE_OPTIONS  +=		CXXFLAGS="$(CXXFLAGS)"
 CONFIGURE_OPTIONS  +=		DFLAGS="-$(BITS)"
+CONFIGURE_OPTIONS  +=		XPROFILE_DIR="$(XPROFILE_DIR)"
 COMPONENT_BUILD_ENV  +=		DFLAGS="-$(BITS)"
+COMPONENT_BUILD_ENV  +=		XPROFILE_DIR="$(XPROFILE_DIR)"
 
 COMPONENT_TEST_TARGETS =	test
 
--- a/components/python/python26/patches/Python26-13-cflags.patch	Tue Dec 20 10:39:45 2011 -0800
+++ b/components/python/python26/patches/Python26-13-cflags.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -1,6 +1,14 @@
-diff --git Python-2.6.4/configure.in Python-2.6.4/configure.in
---- Python-2.6.4/configure.in
-+++ Python-2.6.4/configure.in
+--- Python-2.6.4/configure.in.old	Fri Nov  4 12:03:26 2011
++++ Python-2.6.4/configure.in	Fri Nov  4 12:04:05 2011
[email protected]@ -919,7 +919,7 @@
+ 	;;
+ 
+     *)
+-	OPT="-O"
++	OPT=""
+ 	;;
+     esac
+ 
 @@ -1110,6 +1110,8 @@
  	# environment?
          Darwin*)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python26/patches/Python26-24-symbolic.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,26 @@
+The use of -Bsymbolic here is not required for a profile-guided build, but
+provides noticeable performance improvements for Python applications.  This
+should be an acceptable change as runtime interposition on the Python
+interpreter library is generally undesirable.
+--- Python-2.6.4/configure.in.old	Wed Nov  2 15:49:33 2011
++++ Python-2.6.4/configure.in	Wed Nov  2 15:58:07 2011
[email protected]@ -1712,8 +1712,8 @@
+ 	IRIX*/6*) LDSHARED="ld ${SGI_ABI} -shared -all";;
+ 	SunOS/5*) 
+ 		if test "$GCC" = "yes"
+-		then LDSHARED='$(CC) -shared'
+-		else LDSHARED='$(CC) -G';
++		then LDSHARED='$(CC) -shared -Wl,-Bsymbolic'
++		else LDSHARED='$(CC) -G -Wl,-Bsymbolic';
+ 		fi ;;
+ 	hp*|HP*)
+ 		if test "$GCC" = "yes"
[email protected]@ -1882,7 +1882,7 @@
+ 		  *gcc*)
+ 		    if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null
+ 		    then
+-			LINKFORSHARED="-Xlinker --export-dynamic"
++			LINKFORSHARED="-Xlinker --export-dynamic -Xlinker -Bsymbolic"
+ 		    fi;;
+ 		  esac;;
+ 	CYGWIN*)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python26/patches/Python26-25-studio-profile.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,77 @@
+This changes Python's standard profile-guided build target to work with
+Studio instead of gcc.  The unfortunate repetition of options seen below
+is a workaround for build peculiarities and to meet Studio's requirement
+that the profiling options be used at all steps of the build -- not just
+compilation.
+--- Python-2.6.4/Makefile.pre.in.old	Thu Nov  3 15:14:03 2011
++++ Python-2.6.4/Makefile.pre.in	Thu Nov  3 15:14:11 2011
[email protected]@ -360,9 +360,9 @@
+ profile-opt:
+ 	@echo "Building with support for profile generation:"
+ 	$(MAKE) clean
++	$(MAKE) profile-removal
+ 	$(MAKE) build_all_generate_profile
+ 	@echo "Running benchmark to generate profile data:"
+-	$(MAKE) profile-removal
+ 	$(MAKE) run_profile_task
+ 	@echo "Rebuilding with profile guided optimizations:"
+ 	$(MAKE) clean
[email protected]@ -369,18 +369,27 @@
+ 	$(MAKE) build_all_use_profile
+ 
+ build_all_generate_profile:
+-	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov"
++	$(MAKE) all CC="$(CC) -xprofile=collect:$(XPROFILE_DIR)" \
++	    CFLAGS="$(CFLAGS) -xprofile=collect:$(XPROFILE_DIR)" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=collect:$(XPROFILE_DIR)" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=collect:$(XPROFILE_DIR)"
+ 
+ run_profile_task:
+-	./$(BUILDPYTHON) $(PROFILE_TASK)
++	LD_LIBRARY_PATH=. ./$(BUILDPYTHON) $(PROFILE_TASK)
+ 
+ build_all_use_profile:
+-	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use"
++	$(MAKE) all CC="$(CC) -xprofile=use:$(XPROFILE_DIR)" \
++	    CFLAGS="$(CFLAGS) -xprofile=use:$(XPROFILE_DIR)" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=use:$(XPROFILE_DIR)" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=use:$(XPROFILE_DIR)"
+ 
+ coverage:
+ 	@echo "Building with support for coverage checking:"
+ 	$(MAKE) clean
+-	$(MAKE) all CFLAGS="$(CFLAGS) -O0 -pg -fprofile-arcs -ftest-coverage" LIBS="$(LIBS) -lgcov"
++	$(MAKE) all CC="$(CC) -xprofile=tcov" \
++	    CFLAGS="$(CFLAGS) -xO1 -xprofile=tcov" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=tcov" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=tcov"
+ 
+ 
+ # Build the interpreter
[email protected]@ -395,10 +404,7 @@
+ 
+ # Build the shared modules
+ sharedmods: $(BUILDPYTHON)
+-	@case $$MAKEFLAGS in \
+-	*s*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py -q build;; \
+-	*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py build;; \
+-	esac
++	$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py build
+ 
+ # Build static library
+ # avoid long command lines, same as LIBRARY_OBJS
[email protected]@ -1168,12 +1174,13 @@
+ clean: pycremoval
+ 	find . -name '*.o' -exec rm -f {} ';'
+ 	find . -name '*.s[ol]' -exec rm -f {} ';'
++	find . -name '*.s[ol].*' -exec rm -f {} ';'
+ 	find $(srcdir)/build -name 'fficonfig.h' -exec rm -f {} ';' || true
+ 	find $(srcdir)/build -name 'fficonfig.py' -exec rm -f {} ';' || true
+ 	-rm -f Lib/lib2to3/*Grammar*.pickle
+ 
+ profile-removal:
+-	find . -name '*.gc??' -exec rm -f {} ';'
++	find . -name '.profile' -exec rm -rf {} ';'
+ 
+ clobber: clean profile-removal
+ 	-rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python26/patches/Python26-26-pybench.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,46 @@
+This updates pybench for Python 2.6 with the fixes found in Python 2.7's
+version.
+--- Python-2.6.4/Tools/pybench/pybench.py.orig	Wed Nov  2 17:09:28 2011
++++ Python-2.6.4/Tools/pybench/pybench.py	Sat Jun 11 08:46:28 2011
[email protected]@ -1,4 +1,4 @@
+-#!/usr/bin/python2.6
++#!/usr/local/bin/python -O
+ 
+ """ A Python Benchmark Suite
+ 
[email protected]@ -230,7 +230,7 @@
+                 raise ValueError('at least one calibration run is required')
+             self.calibration_runs = calibration_runs
+         if timer is not None:
+-            timer = timer
++            self.timer = timer
+ 
+         # Init variables
+         self.times = []
[email protected]@ -278,7 +278,7 @@
+             for i in calibration_loops:
+                 pass
+             t = timer() - t
+-            prep_times.append(t)
++            prep_times.append(t / CALIBRATION_LOOPS)
+         min_prep_time = min(prep_times)
+         if _debug:
+             print
[email protected]@ -689,7 +689,7 @@
+                 other_total_avg_time = other_total_avg_time + other_avg_time
+                 if (benchmarks_compatible and
+                     test.compatible(other)):
+-                    # Both benchmark and tests are comparible
++                    # Both benchmark and tests are comparable
+                     min_diff = ((min_time * self.warp) /
+                                 (other_min_time * other.warp) - 1.0)
+                     avg_diff = ((avg_time * self.warp) /
[email protected]@ -703,7 +703,7 @@
+                     else:
+                         avg_diff = '%+5.1f%%' % (avg_diff * PERCENT)
+                 else:
+-                    # Benchmark or tests are not comparible
++                    # Benchmark or tests are not comparable
+                     min_diff, avg_diff = 'n/a', 'n/a'
+                     tests_compatible = 0
+             print '%30s: %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % \
--- a/components/python/python27/Makefile	Tue Dec 20 10:39:45 2011 -0800
+++ b/components/python/python27/Makefile	Tue Dec 20 17:19:09 2011 -0800
@@ -54,6 +54,18 @@
 # build pic
 CFLAGS +=	$(CC_PIC)
 
+# The python build is profile-guided for studio; to see the benefits of that,
+# Python must be compiled with -xO5 and a different build target must be used.
+# Use of xprofile requires that the same options be used during compilation and
+# linking.  The targets chosen are based on Solaris 11 minimum supported system
+# requirements.
+COMPONENT_BUILD_TARGETS = profile-opt
+XPROFILE_DIR = $(BUILD_DIR_$(BITS))/.profile
+PYFLAGS.i386 = -xtarget=opteron -xarch=sse2 -xcache=generic
+PYFLAGS.sparc =
+CFLAGS += -xO5 $(PYFLAGS.$(MACH))
+LDFLAGS += -xO5 $(PYFLAGS.$(MACH))
+
 # add /usr/gnu/lib to the library search/run path
 LDFLAGS.32 = -R/usr/gnu/lib -L/usr/gnu/lib
 LDFLAGS.64 = -R/usr/gnu/lib/$(MACH64) -L/usr/gnu/lib/$(MACH64)
@@ -74,8 +86,10 @@
 CONFIGURE_OPTIONS  +=		BASECFLAGS="$(CFLAGS)"
 CONFIGURE_OPTIONS  +=		LDFLAGS="$(LDFLAGS)"
 CONFIGURE_OPTIONS  +=		DFLAGS="-$(BITS)"
+CONFIGURE_OPTIONS  +=		XPROFILE_DIR="$(XPROFILE_DIR)"
 
 COMPONENT_BUILD_ENV  +=		DFLAGS="-$(BITS)"
+COMPONENT_BUILD_ENV  +=		XPROFILE_DIR="$(XPROFILE_DIR)"
 
 # patches/00-bits.patch removes some AC_CHECK_SIZEOF tests in order to
 # generate a pyconfig.h that is both 32 and 64 bit compatible.  We add this
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python27/patches/10-cflags.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,20 @@
+--- Python-2.6.4/configure.in.orig	Fri Nov  4 12:03:26 2011
++++ Python-2.6.4/configure.in	Fri Nov  4 12:04:05 2011
[email protected]@ -919,7 +919,7 @@
+ 	;;
+ 
+     *)
+-	OPT="-O"
++	OPT=""
+ 	;;
+     esac
+ 
[email protected]@ -1110,6 +1110,8 @@
+ 	# environment?
+         Darwin*)
+ 	    ;;
++	SunOS*)
++	    ;;
+         *)
+ 	    BASECFLAGS="$BASECFLAGS -OPT:Olimit=0"
+ 	    ;;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python27/patches/11-symbolic.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,31 @@
+The use of -Bsymbolic here is not required for a profile-guided build, but
+provides noticeable performance improvements for Python applications.  This
+should be an acceptable change as runtime interposition on the Python
+interpreter library is generally undesirable.
+--- Python-2.7.1/configure.in.orig	2011-11-07 10:05:49.959873160 -0800
++++ Python-2.7.1/configure.in	2011-11-07 10:05:38.932222244 -0800
[email protected]@ -1812,11 +1812,11 @@
+ 	IRIX*/6*) LDSHARED="ld ${SGI_ABI} -shared -all";;
+ 	SunOS/5*) 
+ 		if test "$GCC" = "yes" ; then
+-			LDSHARED='$(CC) -shared'
+-			LDCXXSHARED='$(CXX) -shared'
++			LDSHARED='$(CC) -shared -Wl,-Bsymbolic'
++			LDCXXSHARED='$(CXX) -shared -Wl,-Bsymbolic'
+ 		else
+-			LDSHARED='$(CC) -G'
+-			LDCXXSHARED='$(CXX) -G'
++			LDSHARED='$(CC) -G -Wl,-Bsymbolic'
++			LDCXXSHARED='$(CXX) -G -Wl,-Bsymbolic'
+ 		fi ;;
+ 	hp*|HP*)
+ 		if test "$GCC" = "yes" ; then
[email protected]@ -2008,7 +2008,7 @@
+ 		  *gcc*)
+ 		    if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null
+ 		    then
+-			LINKFORSHARED="-Xlinker --export-dynamic"
++			LINKFORSHARED="-Xlinker --export-dynamic -Xlinker -Bsymbolic"
+ 		    fi;;
+ 		  esac;;
+ 	CYGWIN*)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/python/python27/patches/12-studio-profile.patch	Tue Dec 20 17:19:09 2011 -0800
@@ -0,0 +1,71 @@
+This changes Python's standard profile-guided build target to work with
+Studio instead of gcc.  The unfortunate repetition of options seen below
+is a workaround for build peculiarities and to meet Studio's requirement
+that the profiling options be used at all steps of the build -- not just
+compilation.
+--- Python-2.6.4/Makefile.pre.in.orig	Thu Nov  3 15:14:03 2011
++++ Python-2.6.4/Makefile.pre.in	Thu Nov  3 15:14:11 2011
[email protected]@ -360,9 +360,9 @@
+ profile-opt:
+ 	@echo "Building with support for profile generation:"
+ 	$(MAKE) clean
++	$(MAKE) profile-removal
+ 	$(MAKE) build_all_generate_profile
+ 	@echo "Running benchmark to generate profile data:"
+-	$(MAKE) profile-removal
+ 	$(MAKE) run_profile_task
+ 	@echo "Rebuilding with profile guided optimizations:"
+ 	$(MAKE) clean
[email protected]@ -369,18 +369,27 @@
+ 	$(MAKE) build_all_use_profile
+ 
+ build_all_generate_profile:
+-	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov"
++	$(MAKE) all CC="$(CC) -xprofile=collect:$(XPROFILE_DIR)" \
++	    CFLAGS="$(CFLAGS) -xprofile=collect:$(XPROFILE_DIR)" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=collect:$(XPROFILE_DIR)" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=collect:$(XPROFILE_DIR)"
+ 
+ run_profile_task:
+-	./$(BUILDPYTHON) $(PROFILE_TASK)
++	LD_LIBRARY_PATH=. ./$(BUILDPYTHON) $(PROFILE_TASK)
+ 
+ build_all_use_profile:
+-	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use"
++	$(MAKE) all CC="$(CC) -xprofile=use:$(XPROFILE_DIR)" \
++	    CFLAGS="$(CFLAGS) -xprofile=use:$(XPROFILE_DIR)" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=use:$(XPROFILE_DIR)" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=use:$(XPROFILE_DIR)"
+ 
+ coverage:
+ 	@echo "Building with support for coverage checking:"
+ 	$(MAKE) clean
+-	$(MAKE) all CFLAGS="$(CFLAGS) -O0 -pg -fprofile-arcs -ftest-coverage" LIBS="$(LIBS) -lgcov"
++	$(MAKE) all CC="$(CC) -xprofile=tcov" \
++	    CFLAGS="$(CFLAGS) -xO1 -xprofile=tcov" \
++	    LDFLAGS="$(LDFLAGS) -xprofile=tcov" \
++	    BLDSHARED="$(BLDSHARED) -xprofile=tcov"
+ 
+ 
+ # Build the interpreter
[email protected]@ -395,10 +404,7 @@
+ 
+ # Build the shared modules
+ sharedmods: $(BUILDPYTHON)
+-	@case $$MAKEFLAGS in \
+-	*s*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py -q build;; \
+-	*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py build;; \
+-	esac
++	$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py build
+ 
+ # Build static library
+ # avoid long command lines, same as LIBRARY_OBJS
[email protected]@ -1173,7 +1179,7 @@
+ 	-rm -f Lib/lib2to3/*Grammar*.pickle
+ 
+ profile-removal:
+-	find . -name '*.gc??' -exec rm -f {} ';'
++	find . -name '.profile' -exec rm -rf {} ';'
+ 
+ clobber: clean profile-removal
+ 	-rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \