12899 Need a nice way of comparing manifests from the command line to aid publication
authorBart Smaalders <Bart.Smaalders@Sun.COM>
Wed, 09 Dec 2009 10:14:26 -0800
changeset 1557 d242d38c27d9
parent 1556 9f92a327d343
child 1558 a6cfc654f575
12899 Need a nice way of comparing manifests from the command line to aid publication 12806 pkgmogrify.py referenced from wrong directory in scripts_other_unix
.hgignore
src/man/pkgdiff.1.txt
src/pkgdefs/SUNWipkg/prototype
src/setup.py
src/util/publish/pkgdiff.py
--- a/.hgignore	Wed Dec 09 16:07:38 2009 +0000
+++ b/.hgignore	Wed Dec 09 10:14:26 2009 -0800
@@ -26,7 +26,9 @@
 ^src/man/pkg.1$
 ^src/man/pkg.5$
 ^src/man/pkg.depotd.1m$
-^src/man/pkgdep.1$
+^src/man/pkgdepend.1$
+^src/man/pkgdiff.1$
+^src/man/pkgmogrify.1$
 ^src/man/pkgrecv.1$
 ^src/man/pkgsend.1$
 ^src/man/pm-updatemanager.1$
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/man/pkgdiff.1.txt	Wed Dec 09 10:14:26 2009 -0800
@@ -0,0 +1,70 @@
+User Commands                                            pkgdiff(1)
+
+
+NAME
+     pkgdiff - compare pkg manifests
+
+SYNOPSIS
+     /usr/bin/pkgdiff [-i attribute] [-o attribute] ...  file1 file2 
+
+
+DESCRIPTION
+     pkgdiff compares two package manifests and reports on differences.
+     pkgdiff sorts each manifest & action into a consistent order before
+     comparison.
+
+     Output is of the form:
+
+     + <complete action> if this action is in file2 but not in file1
+     - <complete action> if this action is in file1 but not in file2
+
+     actionname keyvalue <variant values, if any>
+       - attribute1=value1         if this attribute,value is in file1 
+                                   but not in file2
+       + attribute2=value2         if this attribute,value is in file2
+                                   but not in file1
+
+     Note that actions with different variants but the same type & 
+     key attribute value (usually path) are treated as separate actions
+     for purposes of comparison, so actions changing attributes will
+     appear in their complete form rather than as attribute changes.
+
+OPTIONS
+
+     -i attribute    Ignore this attribute if present during comparisons
+     	                  File hash values may be ignored with -i hash.  May not be used
+     	                  with -o option.
+
+     -o attribute   Only report differences in this attribute.   May not be
+     	                  used with -i option.  Will elide any action changes that don't 
+                          affect this attribute on an action.
+
+EXIT STATUS
+     The following exit values are returned:
+
+     0     No differences were found
+
+     1     Differences were found
+
+     >1    An error occurred
+
+
+ATTRIBUTES
+     See attributes(5) for descriptions of the  following  attri-
+     butes:
+     ____________________________________________________________
+    |       ATTRIBUTE TYPE        |       ATTRIBUTE VALUE       |
+    |_____________________________|_____________________________|
+    | Availability                |                             |
+    |_____________________________|_____________________________|
+
+SEE ALSO
+     pkg(5)
+
+NOTES
+     The image packaging system is an under-development feature.
+     Command names, invocation, formats, and operations are all subject
+     to change.  Development is hosted in the OpenSolaris community
+     at
+
+     http://opensolaris.org/os/project/pkg/
--- a/src/pkgdefs/SUNWipkg/prototype	Wed Dec 09 16:07:38 2009 +0000
+++ b/src/pkgdefs/SUNWipkg/prototype	Wed Dec 09 10:14:26 2009 -0800
@@ -14,6 +14,7 @@
 d none usr/bin 755 root bin
 f none usr/bin/pkg 755 root bin
 f none usr/bin/pkgdepend 755 root bin
+f none usr/bin/pkgdiff 755 root bin
 f none usr/bin/pkgmogrify 755 root bin
 f none usr/bin/pkgrecv 755 root bin
 f none usr/bin/pkgsend 755 root bin
@@ -298,6 +299,7 @@
 d none usr/share/man/cat1 755 root bin
 f none usr/share/man/cat1/pkg.1 444 root bin
 f none usr/share/man/cat1/pkgdepend.1 444 root bin
+f none usr/share/man/cat1/pkgdiff.1 444 root bin
 f none usr/share/man/cat1/pkgmogrify.1 444 root bin
 f none usr/share/man/cat1/pkgrecv.1 444 root bin
 f none usr/share/man/cat1/pkgsend.1 444 root bin
--- a/src/setup.py	Wed Dec 09 16:07:38 2009 +0000
+++ b/src/setup.py	Wed Dec 09 10:14:26 2009 -0800
@@ -173,6 +173,7 @@
         scripts_dir: [
                 ['client.py', 'pkg'],
                 ['pkgdep.py', 'pkgdepend'],
+                ['util/publish/pkgdiff.py', 'pkgdiff'],
                 ['util/publish/pkgmogrify.py', 'pkgmogrify'],
                 ['publish.py', 'pkgsend'],
                 ['pull.py', 'pkgrecv'],
@@ -209,7 +210,8 @@
         scripts_dir: [
                 ['client.py', 'client.py'],
                 ['pkgdep.py', 'pkgdep'],
-                ['pkgmogrify.py', 'pkgmogrify'],
+                ['util/publish/pkgdiff.py', 'pkgdiff'],
+                ['util/publish/pkgmogrify.py', 'pkgmogrify'],
                 ['pull.py', 'pull.py'],
                 ['publish.py', 'publish.py'],
                 ['scripts/pkg.sh', 'pkg'],
@@ -236,6 +238,7 @@
         'man/packagemanager.1',
         'man/pkg.1',
         'man/pkgdepend.1',
+        'man/pkgdiff.1',
         'man/pkgmogrify.1',
         'man/pkgsend.1',
         'man/pkgrecv.1',
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/util/publish/pkgdiff.py	Wed Dec 09 10:14:26 2009 -0800
@@ -0,0 +1,275 @@
+#!/usr/bin/python2.6
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+import getopt
+import gettext
+import os
+import re
+import shlex
+import sys
+
+import pkg.actions
+import pkg.manifest as manifest
+from pkg.misc import PipeError
+
+
+def usage(errmsg="", exitcode=2):
+        """Emit a usage message and optionally prefix it with a more specific
+        error message.  Causes program to exit."""
+
+        if errmsg:
+                print >> sys.stderr, "pkgdiff: %s" % errmsg
+        
+        print _(
+            "/usr/bin/pkgdiff [-i attribute] [-o attribute]  file1 file2")
+        sys.exit(exitcode)
+
+def error(text, exitcode=1):
+        """Emit an error message prefixed by the command name """
+
+        print >> sys.stderr, "pkgdiff: %s" % text
+
+        if exitcode != None:
+                sys.exit(exitcode)
+
+def main_func():
+        # /usr/lib/locale is OpenSolaris-specific.
+        gettext.install("pkgdiff", "/usr/lib/locale")
+
+        ignoreattrs = []
+        onlyattrs = []
+
+        try:
+                opts, pargs = getopt.getopt(sys.argv[1:], "i:o:", ["help"])
+                for opt, arg in opts:
+                        if opt == "-i":
+                                ignoreattrs.append(arg)
+                        if opt == "-o":
+                                onlyattrs.append(arg)
+                        if opt in ("--help", "-?"):
+                                usage(exitcode=0)
+     
+        except getopt.GetoptError, e:
+                usage(_("illegal global option -- %s") % e.opt)
+
+        if len(pargs) != 2:
+                usage(_("two file arguments are required"))
+
+        if ignoreattrs and onlyattrs:
+                usage(_("-i and -o options may not be used at the same time"))
+
+        ignoreattrs = set(ignoreattrs)
+        onlyattrs = set(onlyattrs)
+
+        try:
+                file1 = file(pargs[0]).read()
+                file2 = file(pargs[1]).read()
+        except IOError, e:
+                error(_("Cannot open or read file: %s") % e)
+
+        try:
+                manifest1 = manifest.Manifest()
+                manifest1.set_content(file1)
+        except pkg.actions.ActionError, e:
+                error(_("Action error in file %s: %s") % (pargs[0], e))
+
+        try:
+                manifest2 = manifest.Manifest()
+                manifest2.set_content(file2)
+        except pkg.actions.ActionError, e:
+                error(_("Action error in file %s: %s") % (pargs[1], e))                
+
+        # we need to be a little clever about variants, since
+        # we can have multiple actions w/ the same key attributes
+        # in each manifest in that case.  First, make sure any variants
+        # of the same name have the same values defined.
+        v1 = manifest1.get_all_variants()
+        v2 = manifest2.get_all_variants()
+        for k in set(v1.keys()) & set(v2.keys()):
+                if v1[k] != v2[k]:
+                        error(_("Manifests support different variants %s %s") % (v1, v2))
+
+        # Now, get a list of all possible variant values, including None
+        # across all variants and both manifests
+        v_values = dict()
+
+        for v in v1:
+                v1[v].add(None)
+                for a in v1[v]:
+                        v_values.setdefault(v, set()).add((v, a))
+
+        for v in v2:
+                v2[v].add(None)
+                for a in v2[v]:
+                        v_values.setdefault(v, set()).add((v, a))
+
+        diffs = []
+
+        for tup in product(*v_values.values()):
+                # build excludes closure to examine only actions exactly 
+                # matching current variant values... this is needed to
+                # avoid confusing manifest difference code w/ multiple
+                # actions w/ same key attribute values or getting dups
+                # in output
+                def allow(a):
+                        for k, v in tup:
+                                if v is not None:
+                                        if k not in a.attrs or a.attrs[k] != v:
+                                                return False
+                                elif k in a.attrs:
+                                        return False
+                        return True
+
+                a, c, r = manifest2.difference(manifest1, [allow], [allow])
+                diffs += a
+                diffs += c
+                diffs += r
+        # License action still causes spurious diffs... elide to get exit
+        # code correct
+        if not diffs or (len(diffs) == 1 and 
+            diffs[0][0] == diffs[0][0]): # no changes detected at all
+                return 0
+       
+        # define some ordering functions so that output is easily readable
+        # First, a human version of action comparison that works across
+        # variants and action changes...
+        def compare(a, b):                
+                if hasattr(a, "key_attr") and hasattr(b, "key_attr") and \
+                    a.key_attr == b.key_attr:
+                        res = cmp(a.attrs[a.key_attr], b.attrs[b.key_attr])
+                        if res:
+                                return res
+                        # sort by variant
+                        res = cmp(sorted(list(a.get_variants())), sorted(list(b.get_variants())))
+                        if res:
+                                return res
+                else:
+                        res = cmp(a.ord, b.ord)
+                        if res:
+                                return res
+                return cmp(str(a), str(b))
+
+        # and something to pull the relevant action out of the old value, new
+        # value tuples
+        def tuple_key(a):
+                if not a[0]:
+                        return a[1]
+                return a[0]
+
+        # sort and....
+        diffs = sorted(diffs, key=tuple_key, cmp=compare)
+
+        # handle list attributes
+        def attrval(attrs, k):                
+                def q(s):
+                        if " " in s or s == "":
+                                return '"%s"' % s
+                        else:
+                                return s
+
+                v = attrs[k]
+                if isinstance(v, list) or isinstance(v, set):
+                        out = " ".join(["%s=%s" % (k, q(lmt)) for lmt in v])
+                elif " " in v or v == "":
+                        out = k + "=\"" + v + "\""
+                else:
+                        out = k + "=" + v
+                return out
+
+        #figure out when to print diffs
+        def conditional_print(s, a):
+                if onlyattrs:
+                        if not set(a.attrs.keys()) & onlyattrs:
+                                return
+                elif ignoreattrs:
+                        if not set(a.attrs.keys()) - ignoreattrs:
+                                return
+                print "%s %s" % (s, a)               
+                
+        for old, new in diffs:
+                if not new:
+                        conditional_print("-", old)
+                elif not old:
+                        conditional_print("+", new)
+                else:
+                        s = []
+
+                        if not onlyattrs:
+                                if hasattr(old, "hash") and "hash" not in ignoreattrs:
+                                        if old.hash != new.hash:
+                                                s.append("  - %s" % new.hash)
+                                                s.append("  + %s" % old.hash)
+                                attrdiffs = set(new.differences(old)) - ignoreattrs
+                                attrsames = sorted(list(set(old.attrs.keys() + new.attrs.keys()) - 
+                                    set(new.differences(old))))
+                        else:
+                                if hasattr(old, "hash") and "hash"  in onlyattrs:
+                                        if old.hash != new.hash:
+                                                s.append("  - %s" % new.hash)
+                                                s.append("  + %s" % old.hash)
+                                attrdiffs = set(new.differences(old)) & onlyattrs
+                                attrsames = sorted(list(set(old.attrs.keys() + new.attrs.keys()) - 
+                                    set(new.differences(old))))                                
+
+                        for a in sorted(attrdiffs):
+                                if a in old.attrs:
+                                        s.append("  - %s" % attrval(old.attrs, a))
+                                if a in new.attrs:
+                                        s.append("  + %s" % attrval(new.attrs, a))
+                        # print out part of action that is the same
+                        if s:
+                                print "%s %s %s" % (old.name, 
+                                    attrval(old.attrs, old.key_attr), 
+                                    " ".join(("%s" % attrval(old.attrs,v) 
+                                    for v in attrsames if v != old.key_attr)))
+                                for l in s:
+                                        print l
+
+        sys.exit(1)
+
+def product(*args, **kwds):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        # from python 2.6 itertools
+        pools = map(tuple, args) * kwds.get('repeat', 1)
+        result = [[]]
+        for pool in pools:
+                result = [x+[y] for x in result for y in pool]
+        for prod in result:
+                yield tuple(prod)
+                        
+if __name__ == "__main__":
+        try:
+                exit_code = main_func()        
+        except (PipeError, KeyboardInterrupt):
+                exit_code = 1
+        except SystemExit, __e:
+                exit_code = __e
+        except Exception, __e: 
+                print >> sys.stderr, "pkgdiff: caught %s, %s" % (Exception, __e)
+                exit_code = 99
+
+        sys.exit(exit_code)
+