Client-side filtering.
--- a/doc/filter.txt Thu Sep 20 15:45:09 2007 -0700
+++ b/doc/filter.txt Thu Sep 20 18:14:22 2007 -0700
@@ -2,9 +2,55 @@
pkg
FILTERING
-Client presents a filter construct. Platform, ISA (optional), Diskless
-(optional). (Diskless might be better as a restricted path list.)
+We start with all actions in a manifest.
+
+We apply each element in the filter chain in order. For each element, we
+eliminate all remaining actions which don't match the element.
+
+Simple elements have a single filter:
+
+ arch=i386
+
+means eliminate all actions which have an "arch" attribute that isn't
+"i386".
+
+More complicated elements can represent an intersection of filters:
+
+ arch=i386 & debug=true
+
+means eliminate all actions which have both "arch" and "debug" attributes,
+but whose values, respectively, aren't "i386" and "true" (e.g., eliminate
+all non-debug i386 actions and all sparc actions, debug and non-debug);
+
+ doc=true & locale=fr
-Server interprets filter. May offer challenges to assist in determining
-specific version. Version offerings are constrained by virtue of
-inability to change certain paths.
+means eliminate all actions which have both "doc" and "locale" attributes
+and which aren't "true" and "fr", respectively (no, I'm not sure what
+"doc=false" means, other than a reason to refactor). That is, strip out
+all documentation that isn't French (but keep other localized actions, such
+as message files, images, etc.).
+
+Elements can also represent a union of filters:
+
+ locale=fr | locale=sv
+
+means eliminate all actions which have a "locale" attribute that isn't
+either "fr" or "sv".
+
+We'll probably want to mix intersection and union, too:
+
+ doc=true & (locale=fr | locale=sv)
+
+And we want a fallback mechanism, allowing us to get the "best of" a
+particular attribute. Four examples:
+
+ locale=fr_FR.UTF-8;fr_FR;fr;C
+ platform=SUNW,Sun-Fire-V240;sun4u
+ debug=true;false
+ debug=false;true
+
+Any actions that haven't been eliminated by the end of the chain remain to
+be installed. At that point in time, if more than one action exists for
+any given object, the evaluation fails (can't have two copies of
+/usr/bin/ls just because you remembered to specify architecture but not
+debug-ness).
--- a/src/Makefile Thu Sep 20 15:45:09 2007 -0700
+++ b/src/Makefile Thu Sep 20 18:14:22 2007 -0700
@@ -177,6 +177,7 @@
$(PYTHON) $(LINKPYTHONPKG)/manifest.py
$(PYTHON) $(LINKPYTHONPKG)/smf.py
$(PYTHON) $(LINKPYTHONPKG)/client/imageconfig.py
+ $(PYTHON) $(LINKPYTHONPKG)/client/filter.py
$(ROOT) $(ROOTUSRBIN) $(ROOTUSRLIB):
mkdir -p $(ROOT)
--- a/src/brand/pkgcreatezone Thu Sep 20 15:45:09 2007 -0700
+++ b/src/brand/pkgcreatezone Thu Sep 20 18:14:22 2007 -0700
@@ -61,7 +61,7 @@
print "Preparing image"
mkdir -p -m 0700 $zoneroot
rootdir=$zoneroot/root
-pkg image -F $rootdir
+pkg image-create -z -F -a "localhost=http://localhost:10000" $rootdir
print "Retrieving catalog"
pkg -R $rootdir refresh
if [[ $? -ne 0 ]]; then
--- a/src/client.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/client.py Thu Sep 20 18:14:22 2007 -0700
@@ -148,9 +148,10 @@
error = 0
if len(args) > 0:
- opts, pargs = getopt.getopt(args, "Snv")
+ opts, pargs = getopt.getopt(args, "Snvf:")
strict = noexecute = verbose = False
+ filters = []
for opt, arg in opts:
if opt == "-S":
strict = True
@@ -158,10 +159,12 @@
noexecute = True
elif opt == "-v":
verbose = True
+ elif opt == "-f":
+ filters += [ arg ]
img.reload_catalogs()
- ip = imageplan.ImagePlan(img)
+ ip = imageplan.ImagePlan(img, filters = filters)
for ppat in pargs:
rpat = re.sub("\*", ".*", ppat)
@@ -321,6 +324,7 @@
if len(args) > 0:
opts, pargs = getopt.getopt(args, "FPUza:")
+ # XXX Can -z and -U be used at the same time?
for opt, arg in opts:
if opt == "-F":
type = image.IMG_ENTIRE
@@ -331,7 +335,7 @@
if opt == "-z":
is_zone = True
if opt == "-a":
- (auth_name, auth_url) = re.split("=", arg, maxsplit = 2)
+ auth_name, auth_url = arg.split("=", 1)
img.set_attrs(type, pargs[0], is_zone, auth_name, auth_url)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/modules/client/filter.py Thu Sep 20 18:14:22 2007 -0700
@@ -0,0 +1,141 @@
+#!/usr/bin/python
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+# Use is subject to license terms.
+
+from StringIO import StringIO
+import tokenize
+import token
+
+def compile_filter(filter):
+ def q(tup):
+ if tup[0] == token.NAME:
+ return "NAME", tup[0], tup[1]
+ else:
+ return tup[1], tup[0], tup[1]
+ tok_stream = [
+ q(i)
+ for i in tokenize.generate_tokens(StringIO(filter).readline)
+ ]
+
+ s = ""
+ attr = ""
+ want_attr = True
+ next_tok = ("(", "NAME")
+ for tok_str, tok_type, tok in tok_stream:
+ # print "%02s %-15s: %s" % (tok_type, tok, s)
+
+ if tok_str not in next_tok:
+ raise RuntimeError, \
+ "'%s' is not an allowable token %s" % \
+ (tok_str, next_tok)
+
+ if tok_type == token.NAME:
+ if want_attr:
+ attr += tok
+ next_tok = (".", "=")
+ else:
+ s += "'%s') == '%s'" % (tok, tok)
+ next_tok = ("&", "|", ")", "")
+ want_attr = True
+ elif tok_type == token.OP:
+ if tok == "=":
+ s += "d.get('%s', " % attr
+ next_tok = ("NAME",)
+ want_attr = False
+ attr = ""
+ elif tok == "&":
+ s += " and "
+ next_tok = ("NAME", "(")
+ elif tok == "|":
+ s += " or "
+ next_tok = ("NAME", "(")
+ elif tok == "(":
+ s += "("
+ next_tok = ("NAME", "(")
+ elif tok == ")":
+ s += ")"
+ next_tok = ("&", "|", ")", "")
+ elif tok == ".":
+ if want_attr:
+ attr += "."
+ next_tok = ("NAME",)
+
+ return s, compile(s, "<filter string>", "eval")
+
+def apply_filters(action, filters):
+ """Apply the filter chain to the action, returning the True if it's
+ not filtered out, or False if it is.
+
+ Filters operate on action attributes. A simple filter will eliminate
+ an action if the action has the attribute in the filter, but the value
+ is different. Simple filters can be chained together with AND and OR
+ logical operators. In addition, multiple filters may be applied; they
+ are effectively ANDed together.
+ """
+
+ if not action:
+ return False
+
+ # Evaluate each filter in turn. If a filter eliminates the action, we
+ # need check no further. If no filters eliminate the action, return
+ # True.
+ for filter, code in filters:
+ if not eval(code, {"d": action.attrs}):
+ return False
+ return True
+
+if __name__ == "__main__":
+ import sys
+ import pkg.actions
+
+ actionstr = """\
+ file path=/usr/bin/ls arch=i386 debug=true
+ file path=/usr/bin/ls arch=i386 debug=false
+ file path=/usr/bin/ls arch=sparc debug=true
+ file path=/usr/bin/ls arch=sparc debug=false
+ file path=/var/svc/manifest/intrd.xml opensolaris.zone=global
+ file path=/path/to/french/text doc=true locale=fr
+ file path=/path/to/swedish/text doc=true locale=sv
+ file path=/path/to/english/text doc=true locale=en
+ file path=/path/to/us-english/text doc=true locale=en_US"""
+
+ actions = [
+ pkg.actions.fromstr(s.strip())
+ for s in actionstr.splitlines()
+ ]
+
+ if len(sys.argv) > 1:
+ arg = sys.argv[1:]
+ else:
+ arg = [ "arch=i386 & debug=true" ]
+
+ filters = []
+ for filter in arg:
+ expr, comp_expr = compile_filter(filter)
+ print expr
+ filters.append((expr, comp_expr))
+
+ for a in actions:
+ d = a.attrs
+ print "%-5s" % apply_filters(a, filters), d
--- a/src/modules/client/image.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/modules/client/image.py Thu Sep 20 18:14:22 2007 -0700
@@ -177,7 +177,8 @@
self.cfg_cache = imageconfig.ImageConfig()
- self.cfg_cache.filters["opensolaris.zone"] = is_zone
+ if is_zone:
+ self.cfg_cache.filters["opensolaris.zone"] = "nonglobal"
self.cfg_cache.authorities[auth_name] = {}
self.cfg_cache.authorities[auth_name]["prefix"] = auth_name
--- a/src/modules/client/imageconfig.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/modules/client/imageconfig.py Thu Sep 20 18:14:22 2007 -0700
@@ -79,11 +79,11 @@
if re.match("filter", s):
for o in cp.options("filter"):
- self.policies[o] = cp.get("filter", o)
+ self.filters[o] = cp.get("filter", o)
# XXX Child images
- if self.policies.has_key("preferred-authority"):
+ if "preferred-authority" in self.policies:
self.preferred_authority = self.policies["preferred-authority"]
--- a/src/modules/client/imageplan.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/modules/client/imageplan.py Thu Sep 20 18:14:22 2007 -0700
@@ -33,6 +33,8 @@
import pkg.client.pkgplan as pkgplan
import pkg.client.retrieve as retrieve # XXX inventory??
+from pkg.client.filter import compile_filter
+
UNEVALUATED = 0
EVALUATED_OK = 1
EVALUATED_ERROR = 2
@@ -60,7 +62,7 @@
"pkg delete fmri; pkg install fmri@v(n - 1)", then we'd better have a
plan to identify when this operation is safe or unsafe."""
- def __init__(self, image, recursive_removal = False):
+ def __init__(self, image, recursive_removal = False, filters = []):
self.image = image
self.state = UNEVALUATED
self.recursive_removal = recursive_removal
@@ -69,6 +71,12 @@
self.target_rem_fmris = []
self.pkg_plans = []
+ ifilters = [
+ "%s = %s" % (k, v)
+ for k, v in image.cfg_cache.filters.iteritems()
+ ]
+ self.filters = [ compile_filter(f) for f in filters + ifilters ]
+
def __str__(self):
if self.state == UNEVALUATED:
s = "UNEVALUATED:\n"
@@ -214,7 +222,7 @@
print "pkg: %s already installed" % pfmri
return
- pp.evaluate()
+ pp.evaluate(self.filters)
self.pkg_plans.append(pp)
--- a/src/modules/client/pkgplan.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/modules/client/pkgplan.py Thu Sep 20 18:14:22 2007 -0700
@@ -89,7 +89,7 @@
def get_actions(self):
return []
- def evaluate(self):
+ def evaluate(self, filters = []):
"""Determine the actions required to transition the package."""
# if origin unset, determine if we're dealing with an previously
# installed version or if we're dealing with the null package
@@ -106,6 +106,35 @@
except LookupError:
pass
+ self.destination_filters = filters
+
+ # Try to load the filter used for the last install of the
+ # package.
+ self.origin_filters = []
+ if self.origin_fmri:
+ try:
+ f = file("%s/pkg/%s/filters" % \
+ (self.image.imgdir,
+ self.origin_fmri.get_dir_path()), "r")
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ else:
+ self.origin_filters = [
+ (l.strip(), compile(
+ l.strip(), "<filter string>", "eval"))
+ for l in f.readlines()
+ ]
+
+ self.destination_mfst.filter(self.destination_filters)
+ self.origin_mfst.filter(self.origin_filters)
+
+ # Assume that origin actions are unique, but make sure that
+ # destination ones are.
+ ddups = self.destination_mfst.duplicates()
+ if ddups:
+ raise RuntimeError, ["Duplicate actions", ddups]
+
self.actions = self.destination_mfst.difference(
self.origin_mfst)
@@ -121,6 +150,9 @@
os.unlink("%s/pkg/%s/installed" % (self.image.imgdir,
self.origin_fmri.get_dir_path()))
+ os.unlink("%s/pkg/%s/filters" % (self.image.imgdir,
+ self.origin_fmri.get_dir_path()))
+
for src, dest in self.actions:
if dest:
dest.preinstall(self.image, src)
@@ -146,7 +178,7 @@
dest.install(self.image, src)
except Exception, e:
print "Action install failed for '%s' (%s):\n %s: %s" % \
- (dest.attrs[dest.key_attr],
+ (dest.attrs.get(dest.key_attr, id(dest)),
self.destination_fmri.get_pkg_stem(),
e.__class__.__name__, e)
raise
@@ -167,15 +199,33 @@
else:
src.postremove(self.image)
+ # In the case of an upgrade, remove the installation turds from
+ # the origin's directory.
# XXX should this just go in preexecute?
if self.origin_fmri != None and self.destination_fmri != None:
os.unlink("%s/pkg/%s/installed" % (self.image.imgdir,
self.origin_fmri.get_dir_path()))
+ os.unlink("%s/pkg/%s/filters" % (self.image.imgdir,
+ self.origin_fmri.get_dir_path()))
+
if self.destination_fmri != None:
file("%s/pkg/%s/installed" % (self.image.imgdir,
self.destination_fmri.get_dir_path()), "w")
+ # Save the filters we used to install the package, so
+ # they can be referenced later.
+ if self.destination_filters:
+ f = file("%s/pkg/%s/filters" % \
+ (self.image.imgdir,
+ self.destination_fmri.get_dir_path()), "w")
+
+ f.writelines([
+ filter + "\n"
+ for filter, code in self.destination_filters
+ ])
+ f.close()
+
def make_indices(self):
"""Create the reverse index databases for a particular package.
--- a/src/modules/manifest.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/modules/manifest.py Thu Sep 20 18:14:22 2007 -0700
@@ -29,11 +29,13 @@
import shutil
import time
import urllib
+from itertools import groupby
import pkg.actions as actions
import pkg.fmri as fmri
import pkg.package as package
import pkg.client.retrieve as retrieve
+import pkg.client.filter as filter
# The type member is used for the ordering of actions.
ACTION_DIR = 10
@@ -175,6 +177,42 @@
else:
print "%s -> %s" % (src, dest)
+ def filter(self, filters):
+ """Filter out actions from the manifest based on filters."""
+
+ self.actions = [
+ a
+ for a in self.actions
+ if filter.apply_filters(a, filters)
+ ]
+
+ def duplicates(self):
+ """Find actions in the manifest which are duplicates (i.e.,
+ represent the same object) but which are not identical (i.e.,
+ have all the same attributes)."""
+
+ def fun(a):
+ """Return a key on which actions can be sorted."""
+ return a.name, a.attrs.get(a.key_attr, id(a))
+
+ def dup(a, b):
+ "Return whether or not two actions are duplicates."""
+ if not b:
+ return False
+ elif a.name == b.name and \
+ a.attrs.get(a.key_attr, id(a)) == \
+ b.attrs.get(b.key_attr, id(b)):
+ return True
+ else:
+ return False
+
+ dups = []
+ for k, g in groupby(sorted(self.actions, key = fun), fun):
+ gr = list(g)
+ if len(gr) > 1:
+ dups.append((k, gr))
+ return dups
+
def set_fmri(self, img, fmri):
self.img = img
self.fmri = fmri
--- a/src/publish.py Thu Sep 20 15:45:09 2007 -0700
+++ b/src/publish.py Thu Sep 20 18:14:22 2007 -0700
@@ -214,8 +214,10 @@
for i in range(len(attrs[args[0]])))
if args[0] == "file":
action = actions[args[0]](args[5], **kw)
+ extra = 6
else:
action = actions[args[0]](**kw)
+ extra = len(attrs[args[0]]) + 1
except KeyError, e:
if "add_" + e[0] in globals():
method = globals()["add_" + e[0]]
@@ -231,6 +233,14 @@
print 'pkgsend: not enough arguments for "%s" action' % args[0]
sys.exit(1)
+ # If we're presented with extra arguments, update the action's attribute
+ # dictionary.
+ if len(args) > extra:
+ action.attrs.update(dict(
+ s.split("=")
+ for s in args[extra:]
+ ))
+
t = trans.Transaction()
try:
t.add(config, trans_id, action)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tests/filters.ksh Thu Sep 20 18:14:22 2007 -0700
@@ -0,0 +1,41 @@
+#!/bin/ksh -px
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+
+# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+# Use is subject to license terms.
+
+eval `pkgsend open test/filter/[email protected]`
+if [ $? != 0 ]; then
+ echo \*\* script aborted: couldn\'t open test/upgrade/A
+ exit 1
+fi
+
+i386dir=/ws/onnv-gate/packages/i386/nightly
+sparcdir=/ws/onnv-gate/packages/sparc/nightly
+file=SUNWcsu/reloc/usr/bin/ls
+
+echo $PKG_TRANS_ID
+pkgsend add dir 0755 root sys /bin
+pkgsend add file 0755 root sys /bin/ls $i386dir/$file debug=true
+pkgsend add file 0755 root sys /bin/ls $i386dir-nd/$file debug=false
+pkgsend add file 0755 root sys /bin/ls $sparcdir/$file debug=true
+pkgsend add file 0755 root sys /bin/ls $sparcdir-nd/$file debug=false
+pkgsend close