18139749 userland-incorporator should allow generation of multiple incorporations
--- a/components/Makefile Tue Nov 18 17:39:37 2014 -0800
+++ b/components/Makefile Fri Oct 10 10:07:08 2014 -0700
@@ -134,15 +134,18 @@
$(BASS_O_MATIC) $(TEMPLATE_ZONE:%=--template-zone %) \
$(@:%=--component %) --make $(TARGET) $(LOG)
+# Turn userland-incorporator warnings into build failures
+INCORPORATE_WERROR ?= --Werror
+
incorporation:
$(PKGREPO) refresh -s $(PKG_REPO)
$(WS_TOOLS)/userland-incorporator --repository $(PKG_REPO) \
- -p pkg:/consolidation/$(CONSOLIDATION)/$(CONSOLIDATION)-incorporation@0.$(OS_VERSION),$(BUILD_VERSION) \
- -s "$(CONSOLIDATION) consolidation incorporation" \
- -d "This incorporation constrains packages from the $(CONSOLIDATION) consolidation" \
- -c $(CONSOLIDATION) >$(WS_LOGS)/$(CONSOLIDATION)-incorporation.p5m
- $(PKGSEND) -s $(PKG_REPO) publish --fmri-in-manifest \
- $(WS_LOGS)/$(CONSOLIDATION)-incorporation.p5m
+ --version=0.$(OS_VERSION),$(BUILD_VERSION) -p $(PUBLISHER) \
+ -c $(CONSOLIDATION) --destdir=$(WS_LOGS) $(INCORPORATE_WERROR)
+ for incorporation in $(WS_LOGS)/*.p5m ; do \
+ $(PKGSEND) -s $(PKG_REPO) publish --fmri-in-manifest \
+ $${incorporation} ; \
+ done
publish:
$(MAKE) incorporation
--- a/components/meta-packages/history/Makefile Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/Makefile Fri Oct 10 10:07:08 2014 -0700
@@ -51,7 +51,8 @@
$(MKDIR) $@
$(MANIFEST_BASE)-%.mogrified: %.p5m $(BUILD_DIR)
- $(PKGMOGRIFY) -D CONSOLIDATION=$(CONSOLIDATION) $< > $@
+ $(PKGMOGRIFY) -D CONSOLIDATION=$(CONSOLIDATION) $< \
+ $(WS_TOP)/transforms/incorporate > $@
$(MANIFEST_BASE)-%.published: $(MANIFEST_BASE)-%.mogrified $(BUILD_DIR)
$(PKGSEND) -s $(PKG_REPO) publish --fmri-in-manifest --no-catalog $<
--- a/components/meta-packages/history/SUNWIPython.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWIPython.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -26,5 +26,6 @@
#
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWmysql-python26.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWmysql-python26.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -26,5 +26,6 @@
#
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-cssutils.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-cssutils.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-imaging.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-imaging.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-simplejson.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-simplejson.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-twisted-web2.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-twisted-web2.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-twisted.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-twisted.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.1.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWpython26-zope-interface.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWpython26-zope-interface.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=library/python-2/[email protected] type=require
--- a/components/meta-packages/history/SUNWstunnel.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWstunnel.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=service/security/[email protected] type=require
--- a/components/meta-packages/history/SUNWwebalizer.p5m Tue Nov 18 17:39:37 2014 -0800
+++ b/components/meta-packages/history/SUNWwebalizer.p5m Fri Oct 10 10:07:08 2014 -0700
@@ -18,7 +18,7 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
@@ -27,4 +27,6 @@
set name=pkg.fmri value=pkg:/[email protected],5.11-0.175.0.0.0.0.0
set name=pkg.renamed value=true
+set name=org.opensolaris.consolidation value=$(CONSOLIDATION)
+
depend fmri=web/analytics/[email protected] type=require
--- a/doc/packaging.txt Tue Nov 18 17:39:37 2014 -0800
+++ b/doc/packaging.txt Fri Oct 10 10:07:08 2014 -0700
@@ -60,6 +60,35 @@
license actions should be placed in the canonical manifest.
+Manually generated actions
+ * com.oracle.info.description is a terse description of what utilities,
+ libraries and/or services the package provides. This should be short,
+ specific, concise text, identifying the technology covered by the
+ associated license(s). It should fit naturally in the sentence "This
+ package may contain XXX." For example, "XXX" might be "the tar command"
+ or "bzip2 compression software." When appropriate, this may begin with
+ "portions of" or another, more specific qualifying clause.
+ * com.oracle.info.tpno is the Oracle 3rd party license number.
+ * info.classification is "org.opensolaris.category.2008:FOO" where FOO
+ varies according to the sorts of utilities, libraries and/or services
+ that the package provides. Existing packages contain most useful
+ values; check them out to find the closest match. For a complete
+ list of allowed values, refer to the Solaris system file
+ /usr/share/lib/pkg/opensolaris.org.sections .
+ * org.opensolaris.arc-caseid is typically "PSARC/YYYY/###" and multiple
+ different values are allowed.
+ * pkg.summary is a short synopsis of what the package provides.
+ * org.opensolaris.consolidation is the name of the consolidation delivering
+ the package. In Userland, this is $(CONSOLIDATION) (which expands to
+ "Userland" during the build). Manifests in the Userland gate can also
+ decorate this package attribute with an 'incorporate={incorporation-name}'
+ decoration to specify where the package should be incorporated at the end
+ of the userland build. A special value of 'none' will cause the package
+ to be unincorporated and float freely from the rest of the rest of the
+ packages. Note that unincorporated packages don't automatically get
+ updated with the rest of the system when 'pkg update' is run unless the
+ unincorporated package(s) are specified on the command line.
+
Mogrified Manifest
The canonical manifest is combined with a set of the transforms
in $(WS_TOP)/transforms, and a set of macros to more complete
--- a/make-rules/ips.mk Tue Nov 18 17:39:37 2014 -0800
+++ b/make-rules/ips.mk Fri Oct 10 10:07:08 2014 -0700
@@ -70,6 +70,7 @@
PUBLISH_TRANSFORMS += $(WS_TOP)/transforms/docs
PUBLISH_TRANSFORMS += $(WS_TOP)/transforms/locale
PUBLISH_TRANSFORMS += $(PKGMOGRIFY_TRANSFORMS)
+PUBLISH_TRANSFORMS += $(WS_TOP)/transforms/incorporate
PUBLISH_TRANSFORMS += $(WS_TOP)/transforms/publish-cleanup
PKG_MACROS += MACH=$(MACH)
--- a/tools/userland-incorporator Tue Nov 18 17:39:37 2014 -0800
+++ b/tools/userland-incorporator Fri Oct 10 10:07:08 2014 -0700
@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/python2.7
#
# CDDL HEADER START
#
@@ -19,89 +19,153 @@
#
# CDDL HEADER END
#
-# Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
# incorporator - an utility to incorporate packages in a repo
#
-use Getopt::Long;
-
-sub enumerate_packages {
- local ($repository, $publisher, @fmris) = @_;
- my @packages = ();
+import subprocess
+import json
+import sys
+import getopt
+import re
+import os.path
- #printf "/usr/bin/pkg list -ng $repository @fmris\n";
- open($fp, "-|", "/usr/bin/pkgrepo", "list", "-H", "-s", $repository,
- "-p", $publisher, @fmris) ||
- die "pkg: $!";
- while (<$fp>) {
+Werror = False # set to true to exit with any warning
- # lines should be in the form:
- # publisher package [r|o] version,5.12-branch:timestamp
- # or lines should be in the form:
- # publisher package [r|o] version-branch:timestamp
- if ((/^(\S+)\s+(\S+)\s+\S?\s+([\d.]+),[\d.]+-([\d.]+):.+$/) ||
- (/^(\S+)\s+(\S+)\s+\S?\s+([\d.]+)-([\d.]+):.+$/)) {
- my ($package) = ();
+def warning(msg):
+ if Werror == True:
+ print >>sys.stderr, "ERROR: %s" % msg
+ sys.exit(1)
+ else:
+ print >>sys.stderr, "WARNING: %s" % msg
- $package->{publisher} = $1;
- $package->{name} = $2;
- $package->{version} = $3;
- $package->{branch} = $4;
+class Incorporation(object):
+ name = None
+ version = '5.12'
+ packages = {}
+
+ def __init__(self, name, version):
+ self.name = name
+ self.version = version
+ self.packages = {}
- if ($package->{name} !~ m/incorporation/) {
- push(@packages, $package);
- }
- } else {
- die "error: cannot handle: ", $_;
- }
- }
+ def __package_to_str(self, name, version):
+ # strip the :timestamp from the version string
+ version = version.split(':', 1)[0]
+ # strip the ,{build-release} from the version string
+ version = re.sub(",[\d\.]+", "", version)
- #printf "returning %s\n", $_->{name} for (@packages);
+ return "depend fmri=%s@%s facet.version-lock.%s=true type=incorporate" % (name, version, name)
- return @packages;
-}
+ def add_package(self, name, version):
+ self.packages[name] = version
-sub print_incorporate {
- local (%package) = @_;
- my $facet = "facet.version-lock.$package->{name}";
-
- printf "depend fmri=%s@%s-%s %s=true type=incorporate\n",
- $package->{name}, $package->{version}, $package->{branch},
- $facet;
-}
+ def __str__(self):
+ result = """
+set name=pkg.fmri value=pkg:/%s@%s
+set name=info.classification value="org.opensolaris.category.2008:Meta Packages/Incorporations"
+set name=org.opensolaris.consolidation value=userland
+set name=pkg.depend.install-hold value=core-os.userland
+set name=pkg.summary value="userland consolidation incorporation (%s)"
+set name=pkg.description value="This incorporation constrains packages from the userland consolidation"
+""" % (self.name, self.version, self.name)
-my ($repository, $fmri, $summary, $description, $consolidation) = ();
-my %seen = ();
-
-$consolidation = 'userland';
+ names = self.packages.keys()
+ names.sort()
+ for name in names:
+ result += (self.__package_to_str(name, self.packages[name]) + '\n')
-GetOptions("R|repository=s" => \$repository, "v|version=s" => \$version,
- "s|summary=s" => \$summary, "d|description=s" => \$description,
- "p|package=s" => \$fmri, "f|fmri=s" => \@fmris,
- "c|consolidation=s" => \$consolidation);
+ return result
#
-# print the incorporation
+# This should probably use the pkg APIs at some point, but this appears to be
+# a stable and less complicated interface to gathering information from the
+# manifests in the package repo.
#
-printf "set name=pkg.fmri value=%s\n", $fmri;
-printf "set name=pkg.summary value='%s'\n", $summary;
-printf "set name=pkg.description value='%s'\n", $description;
-printf "set name=org.opensolaris.consolidation value=%s\n",
- $consolidation;
-printf "set name=pkg.depend.install-hold value=core-os.%s\n",
- $consolidation;
-printf "set name=info.classification value='org.opensolaris.category.2008:Meta Packages/Incorporations'\n";
+def get_incorporations(repository, publisher, inc_version='5.12'):
+ tmp = subprocess.Popen(["/usr/bin/pkgrepo", "list", "-F", "json",
+ "-s", repository,
+ "-p", publisher],
+ stdout=subprocess.PIPE)
+ incorporations = {}
+ packages = json.load(tmp.stdout)
+
+ # Check for multiple versions of packages in the repo, but keep track of
+ # the latest one.
+ versions = {}
+ for package in packages:
+ pkg_name = package['name']
+ pkg_version = package['version']
+
+ if pkg_name in versions:
+ warning("%s is in the repo at multiple versions (%s, %s)" % (pkg_name, pkg_version, versions[pkg_name]))
+ pkg_version = max(pkg_version, versions[pkg_name])
+ versions[pkg_name] = pkg_version
+
+ for package in packages:
+ pkg_name = package['name']
+ pkg_version = package['version']
+
+ # skip older packages and those that don't want to be incorporated
+ if 'pkg.tmp.incorporate' not in package or pkg_version != versions[pkg_name]:
+ continue
+
+ # a dict inside a list inside a dict
+ incorporate = package['pkg.tmp.incorporate'][0]['value']
+
+ for inc_name in incorporate:
+ # if we haven't started to build this incorporation, create one.
+ if inc_name not in incorporations:
+ incorporations[inc_name] = Incorporation(inc_name, inc_version)
+ # find the incorporation and add the package
+ tmp = incorporations[inc_name]
+ tmp.add_package(pkg_name, pkg_version)
+ return incorporations
-@packages = enumerate_packages($repository, $consolidation, @fmris);
+def main_func():
+ global Werror
+
+ try:
+ opts, pargs = getopt.getopt(sys.argv[1:], "c:s:p:v:d:w",
+ ["repository=", "publisher=", "version=",
+ "consolidation=", "destdir=", "Werror"])
+ except getopt.GetoptError, e:
+ usage(_("illegal option: %s") % e.opt)
+
+ repository = None
+ publisher = None
+ version = None
+ destdir = None
+ consolidation = None
-for (@packages) {
- if ($seen->{$_->{name}} == 1) {
- die "error: duplicate package ", $_->{name};
- }
- printf "depend fmri=pkg:/%s@%s-%s %s=true type=incorporate\n",
- $_->{name}, $_->{version}, $_->{branch},
- "facet.version-lock.".$_->{name};
- $seen->{$_->{name}} = 1;
-}
+ for opt, arg in opts:
+ if opt in ("-s", "--repository"):
+ repository = arg
+ elif opt in ("-p", "--publisher"):
+ publisher = arg
+ elif opt in ("-v", "--version"):
+ version = arg
+ elif opt in ("-d", "--destdir"):
+ destdir = arg
+ elif opt in ("-c", "--consolidation"):
+ consolidation = arg
+ elif opt in ("-w", "--Werror"):
+ Werror = True
+
+ incorporations = get_incorporations(repository, publisher, version)
+
+ for incorporation_name in incorporations.keys():
+ filename = ''
+ if destdir != None:
+ filename = destdir + '/'
+ filename += os.path.basename(incorporation_name) + '.p5m'
+
+ print("Writing %s manifest to %s" % (incorporation_name, filename))
+ fd = open(filename, "w+")
+ fd.write(str(incorporations[incorporation_name]))
+ fd.close()
+
+if __name__ == "__main__":
+ main_func()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/transforms/incorporate Fri Oct 10 10:07:08 2014 -0700
@@ -0,0 +1,68 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+#
+
+#
+# We are overloading the org.opensolaris.consolidation package attribute to
+# maintain information about which incorporations a package should belong to.
+# This is done by including 'incorporation' decorations in the attribute.
+# Ex:
+# set name=org.opensolaris.consolidation value=userland \
+# incorporate=consolidation/userland/userland-incorporation
+#
+# In the process of package generation, we convert those decorations to
+# a separate package attribute (pkg.tmp.incorporate) and drop them from
+# the consolidation attribute. A special value of 'none' will cause the
+# package to be unincorporated and float freely from the rest of the
+# packages. Note that unincorporated packages don't automatically get
+# updated with the rest of the system when 'pkg update' is run unless the
+# unincorporated package(s) are specified on the command line.
+#
+
+#
+# Add a default incorporate decoration if none exists.
+#
+<transform set name=org.opensolaris.consolidation -> \
+ default incorporate consolidation/userland/userland-incorporation>
+
+#
+# convert the org.opensolaris.consolidation 'incorporate' attribute
+# to a package attribute containing the incorporations that should
+# be used in incorporating this package.
+#
+<transform set name=org.opensolaris.consolidation -> emit \
+ set name=pkg.tmp.incorporate \
+ value=%(incorporate;sep=' value=')>
+
+#
+# don't incorporate a package that is tagged for unincorporation.
+#
+<transform set name=pkg.tmp.incorporate value=none -> drop>
+
+#
+# Remove any incorporate decoration from the org.opensolaris.consolidation
+# package attribute.
+#
+<transform set name=org.opensolaris.consolidation -> \
+ delete incorporate .*>