2589 pyc files generate lots of verify chaff
2726 pkg verify doesn't verify file or content hash by default
2680 packagemanager prototype files don't belong in the workspace
--- a/.hgignore Fri Aug 01 23:55:39 2008 -0700
+++ b/.hgignore Fri Aug 01 23:57:44 2008 -0700
@@ -21,11 +21,10 @@
^src/pkg$
^src/pkgrecv$
^src/pkg.depotd$
-^src/pkgdefs/SUNWipkg/prototype$
-^src/pkgdefs/SUNWpython-cherrypy/prototype$
-^src/*.tar.gz$
+^src/pkgdefs/.*/prototype$
^src/CherryPy-.*$
^src/pkgsend$
+^src/*.tar.gz$
^src/util/distro-import/SUNWfixes/
^src/util/distro-import/cluster.import$
^src/util/distro-import/ksh-wrapper$
@@ -34,3 +33,4 @@
^src/util/distro-import/slim_cluster$
^src/util/distro-import/.*/entire.incorporation$
^webrev
+
--- a/src/client.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/client.py Fri Aug 01 23:57:44 2008 -0700
@@ -266,8 +266,9 @@
def verify_image(img, args):
opts, pargs = getopt.getopt(args, "vfqH")
- quiet = forever = verbose = False
- display_headers = True
+ quiet = verbose = False
+ # for now, always check contents of files
+ forever = display_headers = True
for opt, arg in opts:
if opt == "-H":
--- a/src/man/pkg.1.txt Fri Aug 01 23:55:39 2008 -0700
+++ b/src/man/pkg.1.txt Fri Aug 01 23:57:44 2008 -0700
@@ -10,7 +10,7 @@
/usr/bin/pkg install [-nvq] pkg_fmri ...
/usr/bin/pkg uninstall [-nrvq] pkg_fmri ...
- /usr/bin/pkg verify [-fHvq] [pkg_fmri_pattern ...]
+ /usr/bin/pkg verify [-Hvq] [pkg_fmri_pattern ...]
/usr/bin/pkg info [-lr] [--license] [pkg_fmri_pattern ...]
/usr/bin/pkg contents [-Hmr] [-o attribute ...] [-s sort_key]
[-t action_type ... ] [pkg_fmri_pattern ...]
@@ -198,13 +198,11 @@
with newer versions available. With -v, report full package
FMRIs.
- verify [-fHqv] [pkg_fmri_pattern ...]
+ verify [-Hqv] [pkg_fmri_pattern ...]
Validate the installation of packages in the current image.
With -v, do more verbose reporting. With -q, print nothing, but
- return failure if there are any verification problems. With -f,
- fully validate the contents of files against the hash values in
- the package manifests, as well as checking for file and directory
- presence.
+ return failure if there are any verification problems. File hashes
+ are always checked.
The -H option causes the headers to be omitted.
--- a/src/man/pkgsend.1.txt Fri Aug 01 23:55:39 2008 -0700
+++ b/src/man/pkgsend.1.txt Fri Aug 01 23:57:44 2008 -0700
@@ -9,7 +9,7 @@
/usr/bin/pkgsend open [-en] pkg_fmri
/usr/bin/pkgsend add action arguments
- /usr/bin/pkgsend import bundlefile ...
+ /usr/bin/pkgsend import [-T pattern] bundlefile ...
/usr/bin/pkgsend include [-d basedir] manifest ...
/usr/bin/pkgsend close [-A]
@@ -44,9 +44,17 @@
transaction. Requires transaction context. See ACTIONS
below.
- import bundlefile ...
+ import [-T pattern] bundlefile ...
Add each given bundlefile (such as a SVr4 package) into the
- current transaction.
+ current transaction. If the basename of files in the bundle
+ match the optional pattern(s), the timestamp of the file is
+ added to the action for that file. The pattern uses shell
+ matching rules:
+
+ * matches everything
+ ? matches any single character
+ [seq] matches any character in seq
+ [!seq] matches any character not in seq
include [-d basedir] manifest ...
Add resources associated with the multiple actions present in
--- a/src/modules/actions/file.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/actions/file.py Fri Aug 01 23:57:44 2008 -0700
@@ -68,14 +68,9 @@
if not os.path.exists(os.path.dirname(final_path)):
self.makedirs(os.path.dirname(final_path), mode=0755)
- # If we're upgrading, extract the attributes from the old file.
- if orig:
- omode = int(orig.attrs["mode"], 8)
- oowner = pkgplan.image.get_user_by_name(
- orig.attrs["owner"])
- ogroup = pkgplan.image.get_group_by_name(
- orig.attrs["group"])
- ohash = orig.hash
+
+ # XXX If we're upgrading, do we need to preserve file perms from
+ # exiting file?
# If the action has been marked with a preserve attribute, and
# the file exists and has a contents hash different from what
@@ -92,7 +87,7 @@
# XXX We should save the originally installed file. It
# can be used as an ancestor for a three-way merge, for
# example. Where should it be stored?
- if not orig or chash != ohash:
+ if not orig or chash != orig.hash:
pres_type = self.attrs["preserve"]
if pres_type == "renameold":
old_path = final_path + ".old"
@@ -133,16 +128,9 @@
# This is safe even if temp == final_path.
portable.rename(temp, final_path)
- # XXX .pyc files are causing problems because they're not enough
- # newer than the .py files.... if we just installed a .pyc
- # file, move its modification time into the future to prevent
- # python commands running as root from updating these files
- # because they look out of date... the right fix is to fix
- # Solaris python to look at the entire timestamp.... pending.
- # in the mean time, this "accomodation" has to be made to
- # prevent pkg verify errors.
- if final_path.endswith(".pyc"):
- t = os.stat(final_path)[ST_MTIME] + 5 # magic
+ # Handle timestamp if specified
+ if "timestamp" in self.attrs:
+ t = misc.timestamp_to_time(self.attrs["timestamp"])
os.utime(final_path, (t, t))
def verify(self, img, **args):
@@ -185,6 +173,12 @@
errors.append("Mode: 0%.3o should be 0%.3o" % \
(S_IMODE(stat[ST_MODE]), mode))
+ if "timestamp" in self.attrs and stat[ST_MTIME] != \
+ misc.timestamp_to_time(self.attrs["timestamp"]):
+ errors.append("Timestamp: %s should be %s" %
+ (misc.time_to_timestamp(stat[ST_MTIME]),
+ self.attrs["timestamp"]))
+
# avoid checking pkg.size if elfhash present;
# different size files may have the same elfhash
if "preserve" not in self.attrs and \
--- a/src/modules/actions/generic.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/actions/generic.py Fri Aug 01 23:57:44 2008 -0700
@@ -346,8 +346,8 @@
return opener
def verify(self, img, **args):
- """returns True if correctly installed in the given image"""
- return ["verify method for action type %s unimplemented" % self.name]
+ """returns empty list if correctly installed in the given image"""
+ return []
def needsdata(self, orig):
"""Returns True if the action transition requires a
--- a/src/modules/bundle/SolarisPackageDatastreamBundle.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/bundle/SolarisPackageDatastreamBundle.py Fri Aug 01 23:57:44 2008 -0700
@@ -27,6 +27,7 @@
import os
import stat
+import pkg.misc as misc
from pkg.sysvpkg import SolarisPackage
from pkg.actions import *
@@ -106,24 +107,25 @@
mapline = pkgmap[path]
except KeyError:
# XXX Return an unknown instead of a missing, for now.
- return unknown.UnknownAction(path = path)
+ return unknown.UnknownAction(path=path)
if mapline.type in "fev":
return file.FileAction(ci.extractfile(),
- mode = mapline.mode, owner = mapline.owner,
- group = mapline.group, path = mapline.pathname)
+ mode=mapline.mode, owner=mapline.owner,
+ group=mapline.group, path=mapline.pathname,
+ timestamp=misc.time_to_timestamp(int(mapline.modtime)))
elif mapline.type in "dx":
return directory.DirectoryAction(mode = mapline.mode,
- owner = mapline.owner, group = mapline.group,
- path = mapline.pathname)
+ owner=mapline.owner, group=mapline.group,
+ path=mapline.pathname)
elif mapline.type == "s":
- return link.LinkAction(path = mapline.pathname,
- target = mapline.target)
+ return link.LinkAction(path=mapline.pathname,
+ target=mapline.target)
elif mapline.type == "l":
- return hardlink.HardLinkAction(path = mapline.pathname,
- target = mapline.target)
+ return hardlink.HardLinkAction(path=mapline.pathname,
+ target=mapline.target)
else:
- return unknown.UnknownAction(path = mapline.pathname)
+ return unknown.UnknownAction(path=mapline.pathname)
def test(filename):
if not os.path.isfile(filename):
--- a/src/modules/bundle/SolarisPackageDirBundle.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/bundle/SolarisPackageDirBundle.py Fri Aug 01 23:57:44 2008 -0700
@@ -26,6 +26,7 @@
#
import os
+import pkg.misc as misc
from pkg.sysvpkg import SolarisPackage
from pkg.cpiofile import CpioFile
from pkg.actions import *
@@ -95,7 +96,9 @@
if mapline.type in "fev":
return file.FileAction(data, mode=mapline.mode,
owner=mapline.owner, group=mapline.group,
- path=mapline.pathname)
+ path=mapline.pathname,
+ timestamp=misc.time_to_timestamp(int(mapline.modtime)))
+
elif mapline.type in "dx":
return directory.DirectoryAction(mode=mapline.mode,
owner=mapline.owner, group=mapline.group,
--- a/src/modules/bundle/TarBundle.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/bundle/TarBundle.py Fri Aug 01 23:57:44 2008 -0700
@@ -28,7 +28,7 @@
import os
import stat
import tarfile
-
+import pkg.misc as misc
from pkg.actions import *
class TarBundle(object):
@@ -49,14 +49,15 @@
def action(self, tarfile, tarinfo):
if tarinfo.isreg():
return file.FileAction(tarfile.extractfile(tarinfo),
- mode = oct(stat.S_IMODE(tarinfo.mode)),
- owner = tarinfo.uname, group = tarinfo.gname,
- path = tarinfo.name)
+ mode=oct(stat.S_IMODE(tarinfo.mode)),
+ owner=tarinfo.uname, group=tarinfo.gname,
+ path=tarinfo.name,
+ timestamp=misc.time_to_timestamp(tarinfo.mtime))
elif tarinfo.isdir():
return directory.DirectoryAction(
- mode = oct(stat.S_IMODE(tarinfo.mode)),
- owner = tarinfo.uname, group = tarinfo.gname,
- path = tarinfo.name)
+ mode=oct(stat.S_IMODE(tarinfo.mode)),
+ owner=tarinfo.uname, group=tarinfo.gname,
+ path=tarinfo.name)
def test(filename):
try:
--- a/src/modules/misc.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/modules/misc.py Fri Aug 01 23:57:44 2008 -0700
@@ -35,12 +35,23 @@
import urlparse
import sys
import zlib
-
+import time
+import calendar
import pkg.urlhelpers as urlhelpers
import pkg.portable as portable
from pkg.client.imagetypes import img_type_names, IMG_NONE
from pkg import VERSION
+def time_to_timestamp(t):
+ """ convert seconds since epoch to %Y%m%dT%H%M%SZ format"""
+ # XXX optimize?
+ return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(t))
+
+def timestamp_to_time(ts):
+ """ convert %Y%m%dT%H%M%SZ format to seconds since epoch"""
+ # XXX optimize?
+ return calendar.timegm(time.strptime(ts, "%Y%m%dT%H%M%SZ"))
+
def hash_file_name(f):
"""Return the two-level path fragment for the given filename, which is
assumed to be a content hash of at least 8 distinct characters."""
--- a/src/pkgdefs/SUNWipkg-gui-data/prototype Fri Aug 01 23:55:39 2008 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,5 +0,0 @@
-i pkginfo
-i copyright
-d none usr/share/package-manager/data 755 root bin
-f none usr/share/package-manager/data/opensolaris.org 444 root bin
-f none usr/share/package-manager/data/opensolaris.org.sections 444 root bin
--- a/src/pkgdefs/SUNWipkg-gui-l10n/prototype Fri Aug 01 23:55:39 2008 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,39 +0,0 @@
-i pkginfo
-i copyright
-d none usr/share/locale 755 root bin
-d none usr/share/locale/zh_CN 755 root bin
-d none usr/share/locale/zh_CN/LC_MESSAGES 755 root bin
-f none usr/share/locale/zh_CN/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/es 755 root bin
-d none usr/share/locale/es/LC_MESSAGES 755 root bin
-f none usr/share/locale/es/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/zh_TW 755 root bin
-d none usr/share/locale/zh_TW/LC_MESSAGES 755 root bin
-f none usr/share/locale/zh_TW/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/zh_HK 755 root bin
-d none usr/share/locale/zh_HK/LC_MESSAGES 755 root bin
-f none usr/share/locale/zh_HK/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/fr 755 root bin
-d none usr/share/locale/fr/LC_MESSAGES 755 root bin
-f none usr/share/locale/fr/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/pt_BR 755 root bin
-d none usr/share/locale/pt_BR/LC_MESSAGES 755 root bin
-f none usr/share/locale/pt_BR/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/de 755 root bin
-d none usr/share/locale/de/LC_MESSAGES 755 root bin
-f none usr/share/locale/de/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/ja 755 root bin
-d none usr/share/locale/ja/LC_MESSAGES 755 root bin
-f none usr/share/locale/ja/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/ko 755 root bin
-d none usr/share/locale/ko/LC_MESSAGES 755 root bin
-f none usr/share/locale/ko/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/sv 755 root bin
-d none usr/share/locale/sv/LC_MESSAGES 755 root bin
-f none usr/share/locale/sv/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/ru 755 root bin
-d none usr/share/locale/ru/LC_MESSAGES 755 root bin
-f none usr/share/locale/ru/LC_MESSAGES/packagemanager.mo 444 root bin
-d none usr/share/locale/it 755 root bin
-d none usr/share/locale/it/LC_MESSAGES 755 root bin
-f none usr/share/locale/it/LC_MESSAGES/packagemanager.mo 444 root bin
--- a/src/pkgdefs/SUNWipkg-gui/prototype Fri Aug 01 23:55:39 2008 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,26 +0,0 @@
-i pkginfo
-i copyright
-f none usr/bin/packagemanager 755 root bin
-d none usr/share/icons 755 root other
-d none usr/share/icons/package-manager 755 root other
-f none usr/share/icons/package-manager/new_update.png 444 root other
-f none usr/share/package-manager/packagemanager.glade 444 root bin
-d none usr/share/applications 755 root other
-f none usr/share/applications/packagemanager.desktop 444 root other
-d none usr/lib/python2.4/vendor-packages/pkg/gui 755 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/userrights.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/thread.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/installupdate.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/enumerations.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/imageinfo.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/filelist.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/installupdate.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/filelist.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/remove.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/__init__.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/remove.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/thread.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/userrights.pyc 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/enumerations.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/__init__.py 444 root bin
-f none usr/lib/python2.4/vendor-packages/pkg/gui/imageinfo.py 444 root bin
--- a/src/publish.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/publish.py Fri Aug 01 23:57:44 2008 -0700
@@ -45,6 +45,7 @@
import sys
import threading
import traceback
+import fnmatch
import pkg.bundle
import pkg.config as config
@@ -61,7 +62,7 @@
Packager subcommands:
pkgsend open [-en] pkg_fmri
pkgsend add action arguments
- pkgsend import bundlefile ...
+ pkgsend import [-T file_pattern] bundlefile ...
pkgsend include [-d basedir] manifest ...
pkgsend close [-A]
@@ -250,11 +251,26 @@
_("No transaction ID specified in $PKG_TRANS_ID")
sys.exit(1)
- for filename in args:
+ opts, pargs = getopt.getopt(args, "T:")
+
+ timestamp_files = []
+
+ for opt, arg in opts:
+ if opt == "-T":
+ timestamp_files.append(arg)
+
+ for filename in pargs:
bundle = pkg.bundle.make_bundle(filename)
t = trans.Transaction()
for action in bundle:
+ if action.name == "file":
+ basename = os.path.basename(action.attrs["path"])
+ for pattern in timestamp_files:
+ if fnmatch.fnmatch(basename, pattern):
+ break
+ else:
+ del action.attrs["timestamp"]
try:
status, msg, body = t.add(config, trans_id,
action)
--- a/src/tests/cli/t_pkg_install_basics.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/tests/cli/t_pkg_install_basics.py Fri Aug 01 23:57:44 2008 -0700
@@ -28,7 +28,9 @@
testutils.setup_environment("../../../proto")
import os
+import time
import unittest
+from stat import *
class TestPkgInstallBasics(testutils.SingleDepotTestCase):
@@ -39,8 +41,9 @@
foo11 = """
open [email protected],5.11-0
add dir mode=0755 owner=root group=bin path=/lib
- add file /tmp/libc.so.1 mode=0555 owner=root group=bin path=/lib/libc.so.1
+ add file /tmp/libc.so.1 mode=0555 owner=root group=bin path=/lib/libc.so.1 timestamp="20080731T024051Z"
close """
+ foo11_timestamp = 1217472051
foo12 = """
open [email protected],5.11-0
@@ -65,7 +68,7 @@
open [email protected],5.11-0
add depend type=require fmri=pkg:/[email protected]
add dir mode=0755 owner=root group=bin path=/bin
- add file /tmp/cat mode=0555 owner=root group=bin path=/bin/cat
+ add file /tmp/cat mode=0555 owner=root group=bin path=/bin/cat
close """
baz10 = """
@@ -137,6 +140,18 @@
self.pkg("search blah", exit = 1)
self.pkg("search -r blah", exit = 1)
+ # check to make sure timestamp was set to correct value
+
+ libc_path = os.path.join(self.get_img_path(), "lib/libc.so.1")
+ stat = os.stat(libc_path)
+
+ assert (stat[ST_MTIME] == self.foo11_timestamp)
+
+ # check that verify finds changes
+ now = time.time()
+ os.utime(libc_path, (now, now))
+ self.pkg("verify", exit=1)
+
self.pkg("uninstall foo")
self.pkg("verify")
self.pkg("list -a")
--- a/src/util/distro-import/Makefile Fri Aug 01 23:55:39 2008 -0700
+++ b/src/util/distro-import/Makefile Fri Aug 01 23:57:44 2008 -0700
@@ -42,7 +42,7 @@
TMPPKGS=SUNWfixes
EXTRA_OPTIONS=
-SOLARIS.PY=./solaris.py -b 0.$(BUILDID) $(EXTRA_OPTIONS)
+SOLARIS.PY=./solaris.py -b 0.$(BUILDID) $(EXTRA_OPTIONS) -T \*.py
#
# always remove the following (editable) files from packages we bulk import;
--- a/src/util/distro-import/README Fri Aug 01 23:55:39 2008 -0700
+++ b/src/util/distro-import/README Fri Aug 01 23:57:44 2008 -0700
@@ -77,6 +77,13 @@
for files included during pkg processing
May be used multiple times.
+ -T pattern Any file basenames matching this pattern will inherit
+ a timestamp attribute from their Svr4 package, and thus
+ will be installed w/ the same timestamp as when they
+ were packaged. The pattern uses shell-type matching:
-
+ * matches everything
+ ? matches any single character
+ [seq] matches any character in seq
+ [!seq] matches any character not in seq
--- a/src/util/distro-import/publish_ips Fri Aug 01 23:55:39 2008 -0700
+++ b/src/util/distro-import/publish_ips Fri Aug 01 23:57:44 2008 -0700
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
#
# CDDL HEADER START
#
@@ -27,5 +27,5 @@
export PATH=../../../proto/root_`uname -p`/usr/bin:$PATH
export PYTHONPATH=../../../proto/root_`uname -p`/usr/lib/python2.4/vendor-packages
eval `pkgsend open $1`
-pkgsend import $2
+pkgsend import -T \*.py $2
pkgsend close
--- a/src/util/distro-import/solaris.py Fri Aug 01 23:55:39 2008 -0700
+++ b/src/util/distro-import/solaris.py Fri Aug 01 23:57:44 2008 -0700
@@ -23,12 +23,12 @@
# Use is subject to license terms.
+import fnmatch
import getopt
import os
+import re
import shlex
import sys
-import fnmatch
-import re
from datetime import datetime
from itertools import groupby
@@ -503,6 +503,14 @@
f.attrs["owner"] = pathdict[path].owner
f.attrs["group"] = pathdict[path].group
f.attrs["mode"] = pathdict[path].mode
+
+ # is this a file for which we need a timestamp?
+ basename = os.path.basename(path)
+ for file_pattern in timestamp_files:
+ if fnmatch.fnmatch(basename, file_pattern):
+ break
+ else:
+ del f.attrs["timestamp"]
if pathdict[path].klass in preserve_dict.keys():
f.attrs["preserve"] = \
preserve_dict[pathdict[path].klass]
@@ -769,6 +777,7 @@
wos_path = []
include_path = []
branch_dict = {}
+timestamp_files = []
#
# files (by path) we always delete for bulk imports
@@ -787,7 +796,7 @@
" (/usr)", " - / filesystem", ",root(/)"]
try:
- opts, args = getopt.getopt(sys.argv[1:], "B:D:I:b:dns:v:w:j:")
+ opts, args = getopt.getopt(sys.argv[1:], "B:D:I:T:b:dns:v:w:j:")
except getopt.GetoptError, e:
print "unknown option", e.opt
sys.exit(1)
@@ -819,6 +828,8 @@
if len(bfargs) == 2:
branch_dict[bfargs[0]] = bfargs[1]
branch_file.close()
+ elif opt == "-T":
+ timestamp_files.append(arg)
if not def_branch:
print "need a branch id (build number)"