450 RFE: Do a small illumos.sh cleanup
authorRoland Mainz <roland.mainz@nrubsig.org>
Fri, 21 Jan 2011 13:13:50 +0100
changeset 13272 f7d89c5253a8
parent 13271 5aca6ad7a5d9
child 13273 44d974917227
450 RFE: Do a small illumos.sh cleanup Reviewed by: [email protected] Reviewed by: [email protected] Reviewed by: [email protected] Approved by: [email protected]
usr/src/tools/env/illumos.sh
usr/src/tools/scripts/bldenv.sh
usr/src/tools/scripts/nightly.sh
--- a/usr/src/tools/env/illumos.sh	Thu Jan 20 21:42:44 2011 -0800
+++ b/usr/src/tools/env/illumos.sh	Fri Jan 21 13:13:50 2011 +0100
@@ -19,51 +19,95 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2010, 2011 Nexenta Systems, Inc.  All rights reserved.
 #
 
-#	Configuration variables for the runtime environment of the nightly
-# build script and other tools for construction and packaging of releases.
-# This script is sourced by 'nightly' and 'bldenv' to set up the environment
-# for the build. This example is suitable for building an OpenSolaris
-# workspace, which will contain the resulting archives. It is based
-# off the onnv release. It sets NIGHTLY_OPTIONS to make nightly do:
-#	DEBUG build only (-D, -F)
-#	do not bringover from the parent (-n)
-#	runs 'make check' (-C)
-#	runs lint in usr/src (-l plus the LINTDIRS variable)
-#	sends mail on completion (-m and the MAILTO variable)
-#	creates packages for PIT/RE (-p)
-#	checks for changes in ELF runpaths (-r)
-#	build and use this workspace's tools in $SRC/tools (-t)
+# Configuration variables for the runtime environment of the nightly
+# build script and other tools for construction and packaging of
+# releases.
+# This example is suitable for building an illumos workspace, which
+# will contain the resulting archives. It is based off the onnv
+# release. It sets NIGHTLY_OPTIONS to make nightly do:
+#       DEBUG build only (-D, -F)
+#       do not bringover from the parent (-n)
+#       runs 'make check' (-C)
+#       runs lint in usr/src (-l plus the LINTDIRS variable)
+#       sends mail on completion (-m and the MAILTO variable)
+#       creates packages for PIT/RE (-p)
+#       checks for changes in ELF runpaths (-r)
+#       build and use this workspace's tools in $SRC/tools (-t)
 #
-NIGHTLY_OPTIONS="-FnCDlmprt";		export NIGHTLY_OPTIONS
+# - This file is sourced by "bldenv.sh" and "nightly.sh" and should not 
+#   be executed directly.
+# - This script is only interpreted by ksh93 and explicitly allows the
+#   use of ksh93 language extensions.
+#
+export NIGHTLY_OPTIONS='-FnCDlmprt'
+
+#
+# -- PLEASE READ THIS --
+#
+# The variables  GATE and CODEMGR_WS must always be customised to
+# match your workspace/gate location!!
+#
+# -- PLEASE READ THIS --
+#
 
 # This is a variable for the rest of the script - GATE doesn't matter to
 # nightly itself
-GATE=testws;			export GATE
+export GATE='testws'
 
 # CODEMGR_WS - where is your workspace at (or what should nightly name it)
-CODEMGR_WS="/export/$GATE";			export CODEMGR_WS
+export CODEMGR_WS="$HOME/ws/$GATE"
 
 # Maximum number of dmake jobs.  The recommended number is 2 + NCPUS,
 # where NCPUS is the number of logical CPUs on your build system.
-maxjobs() {
-	ncpu=`kstat -p cpu_info:::state | grep -c on-line`
-	expr $ncpu + 2
+function maxjobs
+{
+	nameref maxjobs=$1
+	integer ncpu
+	integer -r min_mem_per_job=512 # minimum amount of memory for a job
+
+	ncpu=$(builtin getconf ; getconf 'NPROCESSORS_ONLN')
+	(( maxjobs=ncpu + 2 ))
+	
+	# Throttle number of parallel jobs launched by dmake to a value which
+	# gurantees that all jobs have enough memory. This was added to avoid
+	# excessive paging/swapping in cases of virtual machine installations
+	# which have lots of CPUs but not enough memory assigned to handle
+	# that many parallel jobs
+	if [[ $(/usr/sbin/prtconf 2>'/dev/null') == ~(E)Memory\ size:\ ([[:digit:]]+)\ Megabytes ]] ; then
+		integer max_jobs_per_memory # parallel jobs which fit into physical memory
+		integer physical_memory # physical memory installed
+
+		# The array ".sh.match" contains the contents of capturing
+		# brackets in the last regex, .sh.match[1] will contain
+		# the value matched by ([[:digit:]]+), i.e. the amount of
+		# memory installed
+		physical_memory="10#${.sh.match[1]}"
+		
+		((
+			max_jobs_per_memory=round(physical_memory/min_mem_per_job) ,
+			maxjobs=fmax(2, fmin(maxjobs, max_jobs_per_memory))
+		))
+	fi
+
+	return 0
 }
-DMAKE_MAX_JOBS=`maxjobs`;			export DMAKE_MAX_JOBS
+
+maxjobs DMAKE_MAX_JOBS # "DMAKE_MAX_JOBS" passed as ksh(1) name reference
+export DMAKE_MAX_JOBS
 
 # path to onbld tool binaries
-ONBLD_BIN="/opt/onbld/bin"
+ONBLD_BIN='/opt/onbld/bin'
 
 # PARENT_WS is used to determine the parent of this workspace. This is
 # for the options that deal with the parent workspace (such as where the
 # proto area will go).
-PARENT_WS="";			export PARENT_WS
+export PARENT_WS=''
 
 # CLONE_WS is the workspace nightly should do a bringover from.
-CLONE_WS="http://hg.illumos.org/illumos-gate"
-export CLONE_WS
+export CLONE_WS='ssh://[email protected]/illumos-gate'
 
 # The bringover, if any, is done as STAFFER.
 # Set STAFFER to your own login as gatekeeper or developer
@@ -71,33 +115,33 @@
 # workspace as root.
 # Some scripts optionally send mail messages to MAILTO.
 #
-STAFFER=nobody;				export STAFFER
-MAILTO=$STAFFER;			export MAILTO
+export STAFFER="$LOGNAME"
+export MAILTO="$STAFFER"
 
 # The project (see project(4)) under which to run this build.  If not
 # specified, the build is simply run in a new task in the current project.
-BUILD_PROJECT=;				export BUILD_PROJECT
+export BUILD_PROJECT=''
 
 # You should not need to change the next four lines
-LOCKNAME="`basename $CODEMGR_WS`_nightly.lock"; export LOCKNAME
-ATLOG="$CODEMGR_WS/log";			export ATLOG
-LOGFILE="$ATLOG/nightly.log";			export LOGFILE
-MACH=`uname -p`;				export MACH
+export LOCKNAME="$(basename -- "$CODEMGR_WS")_nightly.lock"
+export ATLOG="$CODEMGR_WS/log"
+export LOGFILE="$ATLOG/nightly.log"
+export MACH="$(uname -p)"
 
 #
 #  The following two macros are the closed/crypto binaries.  Once
 #  Illumos has totally freed itself, we can remove these references.
 #
 # Location of encumbered binaries.
-ON_CLOSED_BINS="$CODEMGR_WS/closed";		export ON_CLOSED_BINS
+export ON_CLOSED_BINS="$CODEMGR_WS/closed"
 # Location of signed cryptographic binaries.
-ON_CRYPTO_BINS="$CODEMGR_WS/on-crypto.$MACH.tar.bz2"; export ON_CRYPTO_BINS
+export ON_CRYPTO_BINS="$CODEMGR_WS/on-crypto.$MACH.tar.bz2"
 
 # REF_PROTO_LIST - for comparing the list of stuff in your proto area
 # with. Generally this should be left alone, since you want to see differences
 # from your parent (the gate).
 #
-REF_PROTO_LIST=$PARENT_WS/usr/src/proto_list_${MACH}; export REF_PROTO_LIST
+export REF_PROTO_LIST="$PARENT_WS/usr/src/proto_list_${MACH}"
 
 #
 #	build environment variables, including version info for mcs, motd,
@@ -105,24 +149,24 @@
 # when the release slips (nah) or you move an environment file to a new
 # release
 #
-ROOT="$CODEMGR_WS/proto/root_${MACH}";	export ROOT
-SRC="$CODEMGR_WS/usr/src";         	export SRC
-VERSION="$GATE";			export VERSION
+export ROOT="$CODEMGR_WS/proto/root_${MACH}"
+export SRC="$CODEMGR_WS/usr/src"
+export VERSION="$GATE"
 
 #
 # the RELEASE and RELEASE_DATE variables are set in Makefile.master;
 # there might be special reasons to override them here, but that
 # should not be the case in general
 #
-# RELEASE="5.11";			export RELEASE
-# RELEASE_DATE="October 2007";		export RELEASE_DATE
+# export RELEASE='5.11'
+# export RELEASE_DATE='October 2007'
 
 # proto area in parent for optionally depositing a copy of headers and
 # libraries corresponding to the protolibs target
 # not applicable given the NIGHTLY_OPTIONS
 #
-PARENT_ROOT=$PARENT_WS/proto/root_$MACH; export PARENT_ROOT
-PARENT_TOOLS_ROOT=$PARENT_WS/usr/src/tools/proto/root_$MACH-nd; export PARENT_TOOLS_ROOT
+export PARENT_ROOT="$PARENT_WS/proto/root_$MACH"
+export PARENT_TOOLS_ROOT="$PARENT_WS/usr/src/tools/proto/root_$MACH-nd"
 
 # Package creation variables.  You probably shouldn't change these,
 # either.
@@ -131,16 +175,16 @@
 #
 # PKGPUBLISHER_REDIST controls the publisher setting for the repository.
 #
-PKGARCHIVE="${CODEMGR_WS}/packages/${MACH}/nightly";	export PKGARCHIVE
-# PKGPUBLISHER_REDIST="on-redist";			export PKGPUBLISHER_REDIST
+export PKGARCHIVE="${CODEMGR_WS}/packages/${MACH}/nightly"
+# export PKGPUBLISHER_REDIST='on-redist'
 
 # we want make to do as much as it can, just in case there's more than
 # one problem.
-MAKEFLAGS=k;	export MAKEFLAGS
+export MAKEFLAGS='k'
 
 # Magic variable to prevent the devpro compilers/teamware from sending
 # mail back to devpro on every use.
-UT_NO_USAGE_TRACKING="1"; export UT_NO_USAGE_TRACKING
+export UT_NO_USAGE_TRACKING='1'
 
 # Build tools - don't change these unless you know what you're doing.  These
 # variables allows you to get the compilers and onbld files locally or
@@ -148,25 +192,25 @@
 # Alternately, you can set ONBLD_TOOLS to where you keep the contents of
 # SUNWonbld and SPRO_ROOT to where you keep the compilers.  SPRO_VROOT
 # exists to make it easier to test new versions of the compiler.
-BUILD_TOOLS=/opt;				export BUILD_TOOLS
-#ONBLD_TOOLS=/opt/onbld;			export ONBLD_TOOLS
-SPRO_ROOT=/opt/SUNWspro;			export SPRO_ROOT
-SPRO_VROOT=$SPRO_ROOT;				export SPRO_VROOT
+export BUILD_TOOLS='/opt'
+#export ONBLD_TOOLS='/opt/onbld'
+export SPRO_ROOT='/opt/SUNWspro'
+export SPRO_VROOT="$SPRO_ROOT"
 
 # This goes along with lint - it is a series of the form "A [y|n]" which
 # means "go to directory A and run 'make lint'" Then mail me (y) the
 # difference in the lint output. 'y' should only be used if the area you're
 # linting is actually lint clean or you'll get lots of mail.
 # You shouldn't need to change this though.
-#LINTDIRS="$SRC y";	export LINTDIRS
+#export LINTDIRS="$SRC y"
 
 # Set this flag to 'n' to disable the automatic validation of the dmake
 # version in use.  The default is to check it.
-#CHECK_DMAKE=y
+#CHECK_DMAKE='y'
 
 # Set this flag to 'n' to disable the use of 'checkpaths'.  The default,
 # if the 'N' option is not specified, is to run this test.
-#CHECK_PATHS=y
+#CHECK_PATHS='y'
 
 # POST_NIGHTLY can be any command to be run at the end of nightly.  See
 # nightly(1) for interactions between environment variables and this command.
--- a/usr/src/tools/scripts/bldenv.sh	Thu Jan 20 21:42:44 2011 -0800
+++ b/usr/src/tools/scripts/bldenv.sh	Fri Jan 21 13:13:50 2011 +0100
@@ -22,12 +22,19 @@
 
 #
 # Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
 #
 # Uses supplied "env" file, based on /opt/onbld/etc/env, to set shell variables
 # before spawning a shell for doing a release-style builds interactively
 # and incrementally.
 #
 
+function fatal_error
+{
+	print -u2 "${progname}: $*"
+	exit 1
+}
+
 function usage
 {
     OPTIND=0
@@ -166,7 +173,7 @@
 	)
 )
 
-typeset progname="$(basename "${0}")"
+typeset progname="$(basename -- "${0}")"
 
 OPTIND=1
 SUFFIX="-nd"
@@ -258,6 +265,11 @@
 # STDENV_START
 # STDENV_END
 
+# Check if we have sufficient data to continue...
+[[ -v CODEMGR_WS ]] || fatal_error "Error: Variable CODEMGR_WS not set."
+[[ -d "${CODEMGR_WS}" ]] || fatal_error "Error: ${CODEMGR_WS} is not a directory."
+[[ -f "${CODEMGR_WS}/usr/src/Makefile" ]] || fatal_error "Error: ${CODEMGR_WS}/usr/src/Makefile not found."
+
 #MACH=$(uname -p)
 
 # must match the getopts in nightly.sh
@@ -283,7 +295,7 @@
 	RELEASE_DATE=$(LC_ALL=C date +"%B %Y")
 fi
 BUILD_DATE=$(LC_ALL=C date +%Y-%b-%d)
-BASEWSDIR=$(basename $CODEMGR_WS)
+BASEWSDIR=$(basename -- "${CODEMGR_WS}")
 DEV_CM="\"@(#)SunOS Internal Development: $LOGNAME $BUILD_DATE [$BASEWSDIR]\""
 export DEV_CM RELEASE_DATE POUND_SIGN
 
@@ -332,7 +344,7 @@
 #
 # Keep track of this now, before we manipulate $PATH
 #
-WHICH_SCM=$(dirname $(whence $0))/which_scm
+WHICH_SCM="$(dirname -- "$(whence "$0")")/which_scm"
 if [[ ! -x $WHICH_SCM ]]; then
 	WHICH_SCM=which_scm
 fi
@@ -471,7 +483,7 @@
 /usr/bin/newtask -c $$ ${BUILD_PROJECT:+-p$BUILD_PROJECT}
 
 if [[ "${flags.c}" == "false" && -x "$SHELL" && \
-    "$(basename "${SHELL}")" != "csh" ]]; then
+    "$(basename -- "${SHELL}")" != "csh" ]]; then
 	# $SHELL is set, and it's not csh.
 
 	if "${flags.f}" ; then
--- a/usr/src/tools/scripts/nightly.sh	Thu Jan 20 21:42:44 2011 -0800
+++ b/usr/src/tools/scripts/nightly.sh	Fri Jan 21 13:13:50 2011 +0100
@@ -23,6 +23,7 @@
 #
 # Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright 2008, 2010, Richard Lowe
+# Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
 #
 # Based on the nightly script from the integration folks,
 # Mostly modified and owned by mike_s.
@@ -69,6 +70,12 @@
 	WHICH_SCM=which_scm
 fi
 
+function fatal_error
+{
+	print -u2 "nightly: $*"
+	exit 1
+}
+
 #
 # Function to do a DEBUG and non-DEBUG build. Needed because we might
 # need to do another for the source build, and since we only deliver DEBUG or
@@ -1276,6 +1283,11 @@
 # STDENV_START
 # STDENV_END
 
+# Check if we have sufficient data to continue...
+[[ -v CODEMGR_WS ]] || fatal_error "Error: Variable CODEMGR_WS not set."
+[[ -d "${CODEMGR_WS}" ]] || fatal_error "Error: ${CODEMGR_WS} is not a directory."
+[[ -f "${CODEMGR_WS}/usr/src/Makefile" ]] || fatal_error "Error: ${CODEMGR_WS}/usr/src/Makefile not found."
+
 #
 # place ourselves in a new task, respecting BUILD_PROJECT if set.
 #