--- a/src/modules/client/imageplan.py Tue Nov 17 17:06:35 2009 -0600
+++ b/src/modules/client/imageplan.py Wed Nov 18 15:53:48 2009 -0800
@@ -25,23 +25,30 @@
# Use is subject to license terms.
#
+import errno
+import operator
import os
-import errno
import traceback
from pkg.client import global_settings
logger = global_settings.logger
import pkg.actions
-import pkg.client.actuator as actuator
+import pkg.catalog
+import pkg.client.actuator as actuator
+import pkg.client.indexer as indexer
import pkg.client.api_errors as api_errors
-import pkg.client.imagestate as imagestate
-import pkg.client.pkgplan as pkgplan
-import pkg.client.indexer as indexer
-import pkg.fmri as fmri
-import pkg.search_errors as se
+import pkg.client.pkgplan as pkgplan
+import pkg.client.pkg_solver as pkg_solver
+import pkg.fmri
+import pkg.manifest as manifest
+import pkg.search_errors as se
+import pkg.version
-from pkg.client.filter import compile_filter
+from pkg.client.debugvalues import DebugValues
+
+from pkg.misc import msg
+
UNEVALUATED = 0 # nothing done yet
EVALUATED_PKGS = 1 # established fmri changes
@@ -51,173 +58,307 @@
EXECUTED_OK = 5 # finished execution
EXECUTED_ERROR = 6 # failed
-class ImagePlan(object):
- """An image plan takes a list of requested packages, an Image (and its
- policy restrictions), and returns the set of package operations needed
- to transform the Image to the list of requested packages.
+PLANNED_NOTHING = "no-plan"
+PLANNED_INSTALL = "install"
+PLANNED_UNINSTALL = "uninstall"
+PLANNED_UPDATE = "image-update"
+PLANNED_FIX = "fix"
+PLANNED_VARIANT = "change-variant"
- Use of an ImagePlan involves the identification of the Image, the
- Catalogs (implicitly), and a set of complete or partial package FMRIs.
- The Image's policy, which is derived from its type and configuration
- will cause the formulation of the plan or an exception state.
- XXX In the current formulation, an ImagePlan can handle [null ->
- PkgFmri] and [PkgFmri@Version1 -> PkgFmri@Version2], for a set of
- PkgFmri objects. With a correct Action object definition, deletion
- should be able to be represented as [PkgFmri@V1 -> null].
+class ImagePlan(object):
+ """ImagePlan object contains the plan for changing the image...
+ there are separate routines for planning the various types of
+ image modifying operations; evaluation (comparing manifests
+ and buildig lists of removeal, install and update actions
+ and their execution is all common code"""
- XXX Should we allow downgrades? There's an "arrow of time" associated
- with the smf(5) configuration method, so it's better to direct
- manipulators to snapshot-based rollback, but if people are going to do
- "pkg delete fmri; pkg install fmri@v(n - 1)", then we'd better have a
- plan to identify when this operation is safe or unsafe."""
+ def __init__(self, image, progtrack, check_cancel, noexecute=False):
+ self.image = image
+ self.pkg_plans = []
- def __init__(self, image, progtrack, check_cancelation,
- recursive_removal=False, filters=None, variants=None,
- noexecute=False):
- if filters is None:
- filters = []
- self.image = image
self.state = UNEVALUATED
- self.recursive_removal = recursive_removal
- self.progtrack = progtrack
+ self.__progtrack = progtrack
+ self.__noexecute = noexecute
+
+ self.__fmri_changes = [] # install (None, fmri)
+ # update (oldfmri, newfmri)
+ # remove (oldfmri, None)
+ # reinstall(oldfmri, oldfmri)
- self.noexecute = noexecute
- if noexecute:
- self.__intent = imagestate.INTENT_EVALUATE
- else:
- self.__intent = imagestate.INTENT_PROCESS
+ self.update_actions = []
+ self.removal_actions = []
+ self.install_actions = []
- self.target_fmris = []
- self.target_rem_fmris = []
- self.pkg_plans = []
- self.target_insall_count = 0
- self.target_update_count = 0
+ self.__target_install_count = 0
+ self.__target_update_count = 0
+ self.__target_removal_count = 0
- self.__directories = None
+ self.__directories = None # implement ref counting
+ self.__symlinks = None # for dirs and links
self.__cached_actions = {}
- ifilters = [
- "%s = %s" % (k, v)
- for k, v in image.cfg_cache.filters.iteritems()
- ]
- self.filters = [ compile_filter(f) for f in filters + ifilters ]
+ self.__old_excludes = image.list_excludes()
+ self.__new_excludes = self.__old_excludes
- self.old_excludes = image.list_excludes()
- self.new_excludes = image.list_excludes(variants)
+ self.__check_cancelation = check_cancel
- self.check_cancelation = check_cancelation
-
- self.actuators = None
+ self.__actuators = None
self.update_index = True
- self.preexecuted_indexing_error = None
+ self.__preexecuted_indexing_error = None
+ self.__planned_op = PLANNED_NOTHING
+ self.__pkg_solver = None
+ self.__new_variants = None
+ self.__new_facets = None
+ self.__variant_change = False
+ self.__references = {} # dict of fmri -> pattern
def __str__(self):
+
if self.state == UNEVALUATED:
s = "UNEVALUATED:\n"
- for t in self.target_fmris:
- s = s + "+%s\n" % t
- for t in self.target_rem_fmris:
- s = s + "-%s\n" % t
+ return s
+
+ s = "%s\n" % self.__pkg_solver
+
+ if self.state < EVALUATED_PKGS:
+ return s
+
+ s += "Package version changes:\n"
+
+ for pp in self.pkg_plans:
+ s += "%s -> %s\n" % (pp.origin_fmri, pp.destination_fmri)
+
+ if self.__actuators:
+ s = s + "Actuators:\n%s\n" % self.__actuators
+
+ if self.__old_excludes != self.__new_excludes:
+ s = s + "Variants/Facet changes: %s -> %s\n" % (self.__old_excludes,
+ self.__new_excludes)
+
+ return s
+
+ def __verbose_str(self):
+ s = str(self)
+
+ if self.state == EVALUATED_PKGS:
return s
- s = "Package changes:\n"
- for pp in self.pkg_plans:
- s = s + "%s\n" % pp
+ s = s + "Actions being removed:\n"
+ for pplan, o_action, ignore in self.removal_actions:
+ s = s + "\t%s:%s\n" % ( pplan.origin_fmri, o_action)
+
+ s = s + "\nActions being updated:\n"
+ for pplan, o_action, d_action in self.update_actions:
+ s = s + "\t%s:%s -> %s%s\n" % (
+ pplan.origin_fmri, o_action,
+ pplan.destination_fmri, d_action )
+
+ s = s + "\nActions being installed:\n"
+ for pplan, ignore, d_action in self.removal_actions:
+ s = s + "\t%s:%s\n" % ( pplan.destination_fmri, d_action)
+
+ return s
+
+ def show_failure(self, verbose):
+ """Here's where extensive messaging needs to go"""
+
+ if self.__pkg_solver:
+ logger.info(_("Planning for %s failed: %s\n") %
+ (self.__planned_op, self.__pkg_solver.gen_failure_report(verbose)))
+
+ def plan_install(self, pkgs_to_install):
+ """Determine the fmri changes needed to install the specified pkgs"""
+ self.__planned_op = PLANNED_INSTALL
+
+ # get ranking of publishers
+ pub_ranks = self.image.get_publisher_ranks()
+
+ # build installed dict
+ installed_dict = ImagePlan.__fmris2dict(self.image.gen_installed_pkgs())
+
+ # build installed publisher dictionary
+ installed_pubs = dict((
+ (f.pkg_name, f.get_publisher())
+ for f in installed_dict.values()
+ ))
+
+ proposed_dict, self.__references = self.match_user_fmris(pkgs_to_install,
+ True, pub_ranks, installed_pubs)
+
+ # instantiate solver
+ self.__pkg_solver = pkg_solver.PkgSolver(
+ self.image.get_catalog(self.image.IMG_CATALOG_KNOWN),
+ installed_dict,
+ pub_ranks,
+ self.image.get_variants(),
+ self.__progtrack)
+
+ # Solve... will raise exceptions if no solution is found
+ new_vector = self.__pkg_solver.solve_install([], proposed_dict,
+ self.__new_excludes)
+
+ self.__fmri_changes = [
+ (a, b)
+ for a, b in ImagePlan.__dicts2fmrichanges(installed_dict,
+ ImagePlan.__fmris2dict(new_vector))
+ if a != b
+ ]
+
+ self.state = EVALUATED_PKGS
+
+ def plan_uninstall(self, pkgs_to_uninstall, recursive_removal=False):
+ self.__planned_op = PLANNED_UNINSTALL
+ proposed_dict, self.__references = self.match_user_fmris(pkgs_to_uninstall,
+ False, None, None)
+ # merge patterns together
+ proposed_removals = set([
+ f
+ for each in proposed_dict.values()
+ for f in each
+ ])
+
+
+ # compute removal of packages; until we implement require
+ # either A or B type dependencies no solver is needed
+ if recursive_removal:
+ needs_processing = proposed_removals
+ already_processed = set()
+ while needs_processing:
+ pfmri = needs_processing.pop()
+ already_processed.add(pfmri)
+ needs_processing |= set(self.image.get_dependents(pfmri,
+ self.__progtrack)) - already_processed
+ proposed_removals = already_processed
+
+ for pfmri in proposed_removals:
+ self.__progtrack.evaluate_progress(pfmri)
+ dependents = set(self.image.get_dependents(pfmri,
+ self.__progtrack))
+ if dependents - proposed_removals:
+ raise api_errors.NonLeafPackageException(pfmri,
+ dependents)
+
+ self.__fmri_changes = [(f, None) for f in proposed_removals]
- s = s + "Actuators:\n%s" % self.actuators
+ self.state = EVALUATED_PKGS
+
+ @staticmethod
+ def __fmris2dict(fmri_list):
+ return dict([
+ (f.pkg_name, f)
+ for f in fmri_list
+ ])
+
+ @staticmethod
+ def __dicts2fmrichanges(olddict, newdict):
+ return [
+ (olddict.get(k, None), newdict.get(k, None))
+ for k in set(olddict.keys() + newdict.keys())
+ ]
+
+ def plan_update(self):
+ """Determine the fmri changes needed to update all
+ pkgs"""
+ self.__planned_op = PLANNED_UPDATE
+
+ # build installed dict
+ installed_dict = dict([
+ (f.pkg_name, f)
+ for f in self.image.gen_installed_pkgs()
+ ])
+
+ # instantiate solver
+ self.__pkg_solver = pkg_solver.PkgSolver(
+ self.image.get_catalog(self.image.IMG_CATALOG_KNOWN),
+ installed_dict,
+ self.image.get_publisher_ranks(),
+ self.image.get_variants(),
+ self.__progtrack)
+ #
+ new_vector = self.__pkg_solver.solve_update([], self.__new_excludes)
+
+ self.__fmri_changes = [
+ (a, b)
+ for a, b in ImagePlan.__dicts2fmrichanges(installed_dict,
+ ImagePlan.__fmris2dict(new_vector))
+ if a != b
+ ]
+
+ self.state = EVALUATED_PKGS
+
- s = s + "Variants: %s -> %s\n" % (self.old_excludes, self.new_excludes)
- return s
+ def plan_fix(self, pkgs_to_fix):
+ """Create the list of pkgs to fix"""
+ self.__planned_op = PLANNED_FIX
+ # XXX complete this
+
+ def plan_change_varcets(self, variants, facets):
+ """Determine the fmri changes needed to change
+ the specified variants/facets"""
+ self.__planned_op = PLANNED_VARIANT
+
+ if variants == None and facets == None: # nothing to do
+ self.state = EVALUATED_PKGS
+ return
+
+ self.__variant_change = True
+
+ # build installed dict
+ installed_dict = dict([
+ (f.pkg_name, f)
+ for f in self.image.gen_installed_pkgs()
+ ])
+
+ # instantiate solver
+ self.__pkg_solver = pkg_solver.PkgSolver(
+ self.image.get_catalog(self.image.IMG_CATALOG_KNOWN),
+ installed_dict,
+ self.image.get_publisher_ranks(),
+ self.image.get_variants(),
+ self.__progtrack)
+
+ self.__new_excludes = self.image.list_excludes(variants, facets)
+
+ new_vector = self.__pkg_solver.solve_change_varcets([],
+ variants, facets, self.__new_excludes)
+
+ self.__new_variants = variants
+ self.__new_facets = facets
+
+ self.__fmri_changes = [
+ (a, b)
+ for a, b in ImagePlan.__dicts2fmrichanges(installed_dict,
+ ImagePlan.__fmris2dict(new_vector))
+ ]
+
+ self.state = EVALUATED_PKGS
+ return
+
+ def reboot_needed(self):
+ """Check if evaluated imageplan requires a reboot"""
+ assert self.state >= EVALUATED_OK
+ return self.__actuators.reboot_needed()
+
def get_plan(self, full=True):
if full:
return str(self)
output = ""
- for pp in self.pkg_plans:
- output += "%s -> %s\n" % (pp.origin_fmri,
- pp.destination_fmri)
-
+
+ for t in self.__fmri_changes:
+ output += "%s -> %s\n" % t
return output
def display(self):
- for pp in self.pkg_plans:
- logger.info("%s -> %s" % (pp.origin_fmri, pp.destination_fmri))
- logger.info("Actuators:\n%s" % self.actuators)
-
- def is_proposed_fmri(self, pfmri):
- for pf in self.target_fmris:
- if pfmri.is_same_pkg(pf):
- return not pfmri.is_successor(pf)
- return False
-
- def is_proposed_rem_fmri(self, pfmri):
- for pf in self.target_rem_fmris:
- if pfmri.is_same_pkg(pf):
- return True
- return False
-
- def propose_fmri(self, pfmri):
- # is a version of fmri.stem in the inventory?
- if self.image.has_version_installed(pfmri) and \
- self.old_excludes == self.new_excludes:
- return
-
- # is there a freeze or incorporation statement?
- # do any of them eliminate this fmri version?
- # discard
-
- #
- # update so that we meet any optional dependencies
- #
-
- pfmri = self.image.constraints.apply_constraints_to_fmri(pfmri)
- self.image.fmri_set_default_publisher(pfmri)
+ if DebugValues["plan"]:
+ logger.info(self.__verbose_str())
+ else:
+ logger.info(str(self))
- # Add fmri to target list only if it (or a successor) isn't
- # there already.
- for i, p in enumerate(self.target_fmris):
- if pfmri.is_successor(p):
- self.target_fmris[i] = pfmri
- break
- if p.is_successor(pfmri):
- break
- else:
- self.target_fmris.append(pfmri)
- return
-
- def get_proposed_version(self, pfmri):
- """ Return version of fmri already proposed, or None
- if not proposed yet."""
- for p in self.target_fmris:
- if pfmri.get_name() == p.get_name():
- return p
- else:
- return None
-
- def older_version_proposed(self, pfmri):
- # returns true if older version of this pfmri has been proposed
- # already
- for p in self.target_fmris:
- if pfmri.is_successor(p):
- return True
- return False
-
- # XXX Need to make sure that the same package isn't being added and
- # removed in the same imageplan.
- def propose_fmri_removal(self, pfmri):
- if not self.image.has_version_installed(pfmri):
- return
-
- for i, p in enumerate(self.target_rem_fmris):
- if pfmri.is_successor(p):
- self.target_rem_fmris[i] = pfmri
- break
- else:
- self.target_rem_fmris.append(pfmri)
def gen_new_installed_pkgs(self):
""" generates all the fmris in the new set of installed pkgs"""
@@ -232,17 +373,19 @@
def gen_new_installed_actions(self):
"""generates actions in new installed image"""
+ assert self.state >= EVALUATED_PKGS
for pfmri in self.gen_new_installed_pkgs():
m = self.image.get_manifest(pfmri)
- for act in m.gen_actions(self.new_excludes):
+ for act in m.gen_actions(self.__new_excludes):
yield act
def gen_new_installed_actions_bytype(self, atype):
"""generates actions in new installed image"""
+ assert self.state >= EVALUATED_PKGS
for pfmri in self.gen_new_installed_pkgs():
m = self.image.get_manifest(pfmri)
for act in m.gen_actions_by_type(atype,
- self.new_excludes):
+ self.__new_excludes):
yield act
def get_directories(self):
@@ -254,13 +397,22 @@
"var/pkg",
"var/sadm",
"var/sadm/install"])
- for fmri in self.gen_new_installed_pkgs():
- m = self.image.get_manifest(fmri)
- for d in m.get_directories(self.new_excludes):
+ for pfmri in self.gen_new_installed_pkgs():
+ m = self.image.get_manifest(pfmri)
+ for d in m.get_directories(self.__new_excludes):
dirs.add(os.path.normpath(d))
self.__directories = dirs
return self.__directories
+ def __get_symlinks(self):
+ """ return a set of all symlinks in target image"""
+ if self.__symlinks == None:
+ self.__symlinks = set((
+ a.attrs["path"]
+ for a in self.gen_new_installed_actions_bytype("link")
+ ))
+ return self.__symlinks
+
def get_actions(self, name, key=None):
"""Return a dictionary of actions of the type given by 'name'
describing the target image. If 'key' is given and not None,
@@ -282,209 +434,111 @@
self.__cached_actions[(name, key)] = d
return self.__cached_actions[(name, key)]
- def evaluate_fmri(self, pfmri):
- self.progtrack.evaluate_progress(pfmri)
-
- if self.check_cancelation():
- raise api_errors.CanceledException()
-
- ppub = self.image.get_preferred_publisher()
- self.image.fmri_set_default_publisher(pfmri)
-
- cat = self.image.get_catalog(self.image.IMG_CATALOG_KNOWN)
-
- # check to make sure package is not tagged as being only
- # for other architecture(s)
- supported = cat.get_entry_variants(pfmri, "variant.arch")
- if supported and self.image.get_arch() not in supported:
- raise api_errors.PlanCreationException(badarch=(pfmri,
- supported, self.image.get_arch()))
+ def __get_manifest(self, pfmri, intent):
+ """Return manifest for pfmri"""
+ if pfmri:
+ return self.image.get_manifest(pfmri,
+ all_variants=self.__variant_change, intent=intent)
+ else:
+ return manifest.NullCachedManifest
- # build list of (action, fmri, constraint) of dependencies
- a_list = [
- (a,) + a.parse(self.image, pfmri.get_name())
- for a in cat.get_entry_actions(pfmri, [cat.DEPENDENCY],
- excludes=self.new_excludes)
- if a.name == "depend"
- ]
+ def __create_intent(self, old_fmri, new_fmri):
+ """Return intent strings (or None). Given a pair
+ of fmris describing a package operation, this
+ routines returns intent strings to be passed to
+ originating publisher describing manifest
+ operations. We never send publisher info to
+ prevent cross-publisher leakage of info."""
- # Update constraints first to avoid problems w/ depth first
- # traversal of dependencies; we may violate an existing
- # constraint here.
- if self.image.constraints.start_loading(pfmri):
- for a, f, constraint in a_list:
- self.image.constraints.update_constraints(
- constraint)
- self.image.constraints.finish_loading(pfmri)
-
- # now check what work is required
- for a, f, constraint in a_list:
-
- # discover if we have an installed or proposed
- # version of this pkg already; proposed fmris
- # will always be newer
- ref_fmri = self.get_proposed_version(f)
- if not ref_fmri:
- ref_fmri = self.image.get_version_installed(f)
+ if self.__noexecute:
+ return None, None
- # check if new constraint requires us to make any
- # changes to already proposed pkgs or existing ones.
- if not constraint.check_for_work(ref_fmri):
- continue
- # Apply any active optional/incorporation constraints
- # from other packages
-
- cf = self.image.constraints.apply_constraints_to_fmri(f)
-
- # This will be the newest version of the specified
- # dependency package. Package names specified in
- # dependencies are treated as exact. Matches from the
- # preferred publisher are used first, then matches from
- # the same publisher as the evaluated fmri, and then
- # first available. Callers can override this behavior
- # by specifying the publisher prefix as part of the FMRI.
- matches = list(self.image.inventory([cf], all_known=True,
- matcher=fmri.exact_name_match, preferred=True,
- ordered=False))
+ if new_fmri:
+ reference = self.__references.get(new_fmri, None)
+ # don't leak prev. version info across publishers
+ if old_fmri:
+ if old_fmri.get_publisher() != \
+ new_fmri.get_publisher():
+ old_fmri = "unknown"
+ else:
+ old_fmri = old_fmri.get_fmri(anarchy=True)
+ new_fmri = new_fmri.get_fmri(anarchy=True)# don't send pub
+ else:
+ reference = self.__references.get(old_fmri, None)
+ old_fmri = old_fmri.get_fmri(anarchy=True)# don't send pub
- cf = matches[0][0]
- cs = matches[0][1]
- evalpub = pfmri.get_publisher()
- if len(matches) > 1 and not cf.get_publisher() == ppub \
- and cf.get_publisher() != evalpub:
- # If more than one match was returned, and it
- # wasn't for the preferred publisher or for the
- # same publisher as the fmri being evaluated,
- # then try to find a match that has the same
- # publisher as the fmri being evaluated.
- for f, st in matches[1:]:
- if f.get_publisher() == evalpub:
- cf = f
- cs = st
- break
+ info = {
+ "operation": self.__planned_op,
+ "old_fmri" : old_fmri,
+ "new_fmri" : new_fmri,
+ "reference": reference
+ }
- if cs["obsolete"]:
- # Depending on an obsolete package is an error,
- # unless the dependency is just there to move
- # the package forward.
- if constraint.presence == constraint.ALWAYS:
- raise api_errors.PlanCreationException(
- obsolete=((pfmri, cf),))
-
- vi = self.image.get_version_installed(cf)
- if vi:
- self.evaluate_fmri_removal(vi)
- else:
- self.propose_fmri(cf)
- self.evaluate_fmri(cf)
+ s = "(%s)" % ";".join([
+ "%s=%s" % (key, info[key]) for key in info
+ if info[key] is not None
+ ])
- def add_pkg_plan(self, pfmri):
- """add a pkg plan to imageplan for fully evaluated frmi"""
- m = self.image.get_manifest(pfmri)
- pp = pkgplan.PkgPlan(self.image, self.progtrack, \
- self.check_cancelation)
-
- if self.old_excludes != self.new_excludes:
- if self.image.is_pkg_installed(pfmri):
- pp.propose_reinstall(pfmri, m)
- else:
- pp.propose_destination(pfmri, m)
- else:
- try:
- pp.propose_destination(pfmri, m)
- except RuntimeError:
- logger.info("pkg: %s already installed" % pfmri)
- return
-
- pp.evaluate(self.old_excludes, self.new_excludes)
-
- if pp.origin_fmri:
- self.target_update_count += 1
- else:
- self.target_insall_count += 1
+ if new_fmri:
+ return None, s # only report new on upgrade
+ return s, None # handle uninstall
+
+ def evaluate(self, verbose=False):
+ """Given already determined fmri changes,
+ build pkg plans and figure out exact impact of
+ proposed changes"""
- self.pkg_plans.append(pp)
+ assert self.state == EVALUATED_PKGS, self
- def evaluate_fmri_removal(self, pfmri):
- # prob. needs breaking up as well
- assert self.image.has_manifest(pfmri)
-
- self.progtrack.evaluate_progress(pfmri)
-
- dependents = set(self.image.get_dependents(pfmri,
- self.progtrack))
+ if self.__noexecute and not verbose:
+ return # optimize performance if no one cares
- # Don't consider those dependencies already being removed in
- # this imageplan transaction.
- dependents = dependents.difference(self.target_rem_fmris)
-
- if dependents and not self.recursive_removal:
- raise api_errors.NonLeafPackageException(pfmri,
- dependents)
-
- pp = pkgplan.PkgPlan(self.image, self.progtrack, \
- self.check_cancelation)
-
- self.image.state.set_target(pfmri, self.__intent)
- m = self.image.get_manifest(pfmri)
+ #prefetch manifests
+
+ prefetch_list = [] # manifest, intents to be prefetched
+ eval_list = [] # oldfmri, oldintent, newfmri, newintent
+ # prefetched intents omitted
- try:
- pp.propose_removal(pfmri, m)
- except RuntimeError:
- self.image.state.set_target()
- logger.info("pkg %s not installed" % pfmri)
- return
-
- pp.evaluate([], self.old_excludes)
+ for oldfmri, newfmri in self.__fmri_changes:
+ self.__progtrack.evaluate_progress(oldfmri)
+ old_in, new_in = self.__create_intent(oldfmri, newfmri)
+ if oldfmri:
+ if not self.image.has_manifest(oldfmri):
+ prefetch_list.append((oldfmri, old_in))
+ old_in = None # so we don't send it twice
+ if newfmri:
+ if not self.image.has_manifest(newfmri):
+ prefetch_list.append((newfmri, new_in))
+ new_in = None
+ eval_list.append((oldfmri, old_in, newfmri, new_in))
- for d in dependents:
- if self.is_proposed_rem_fmri(d):
- continue
- if not self.image.has_version_installed(d):
- continue
- self.target_rem_fmris.append(d)
- self.progtrack.evaluate_progress(d)
- self.evaluate_fmri_removal(d)
-
- # Post-order append will ensure topological sorting for acyclic
- # dependency graphs. Cycles need to be arbitrarily broken, and
- # are done so in the loop above.
- self.pkg_plans.append(pp)
- self.image.state.set_target()
+ self.image.transport.prefetch_manifests(prefetch_list,
+ progtrack=self.__progtrack,
+ ccancel=self.__check_cancelation)
- def evaluate(self):
- assert self.state == UNEVALUATED
+ for oldfmri, old_in, newfmri, new_in in eval_list:
+ pp = pkgplan.PkgPlan(self.image, self.__progtrack,
+ self.__check_cancelation)
- outstring = ""
+ pp.propose(oldfmri, self.__get_manifest(oldfmri, old_in),
+ newfmri, self.__get_manifest(newfmri, new_in))
+
+ pp.evaluate(self.__old_excludes, self.__new_excludes)
- # Operate on a copy, as it will be modified in flight.
- for f in self.target_fmris[:]:
- self.progtrack.evaluate_progress(f)
- try:
- self.evaluate_fmri(f)
- except KeyError, e:
- outstring += "Attempting to install %s " \
- "causes:\n\t%s\n" % (f.get_name(), e)
- if outstring:
- raise RuntimeError("No packages were installed because "
- "package dependencies could not be satisfied\n" +
- outstring)
+ if pp.origin_fmri and pp.destination_fmri:
+ self.__target_update_count += 1
+ elif pp.destination_fmri:
+ self.__target_install_count += 1
+ elif pp.origin_fmri:
+ self.__target_removal_count += 1
- for f in self.target_fmris:
- self.add_pkg_plan(f)
- self.progtrack.evaluate_progress(f)
+ self.pkg_plans.append(pp)
- for f in self.target_rem_fmris[:]:
- self.evaluate_fmri_removal(f)
- self.progtrack.evaluate_progress(f)
-
- # we now have a workable set of packages to add/upgrade/remove
+ # we now have a workable set of pkgplans to add/upgrade/remove
# now combine all actions together to create a synthetic single
# step upgrade operation, and handle editable files moving from
# package to package. See theory comment in execute, below.
- self.state = EVALUATED_PKGS
-
self.removal_actions = [
(p, src, dest)
for p in self.pkg_plans
@@ -503,9 +557,9 @@
for src, dest in p.gen_install_actions()
]
- self.progtrack.evaluate_progress()
+ self.__progtrack.evaluate_progress()
- self.actuators = actuator.Actuator()
+ self.__actuators = actuator.Actuator()
# iterate over copy of removals since we're modding list
# keep track of deletion count so later use of index works
@@ -519,6 +573,14 @@
del self.removal_actions[i - deletions]
deletions += 1
continue
+ # remove link removal if link is still in final image
+ # (implement reference count on removal due to borked pkgs)
+ if a[1].name == "link" and \
+ os.path.normpath(a[1].attrs["path"]) in \
+ self.__get_symlinks():
+ del self.removal_actions[i - deletions]
+ deletions += 1
+ continue
# store names of files being removed under own name
# or original name if specified
if a[1].name == "file":
@@ -530,9 +592,9 @@
(i - deletions,
id(self.removal_actions[i-deletions][1]))
- self.actuators.scan_removal(a[1].attrs)
+ self.__actuators.scan_removal(a[1].attrs)
- self.progtrack.evaluate_progress()
+ self.__progtrack.evaluate_progress()
for a in self.install_actions:
# In order to handle editable files that move their path
@@ -553,9 +615,9 @@
"save_file"] = cache_name
a[2].attrs["save_file"] = cache_name
- self.actuators.scan_install(a[2].attrs)
+ self.__actuators.scan_install(a[2].attrs)
- self.progtrack.evaluate_progress()
+ self.__progtrack.evaluate_progress()
# Go over update actions
l_actions = self.get_actions("hardlink",
lambda a: a.get_target_path())
@@ -575,8 +637,8 @@
# scan both old and new actions
# repairs may result in update action w/o orig action
if a[1]:
- self.actuators.scan_update(a[1].attrs)
- self.actuators.scan_update(a[2].attrs)
+ self.__actuators.scan_update(a[1].attrs)
+ self.__actuators.scan_update(a[2].attrs)
self.update_actions.extend(l_refresh)
# sort actions to match needed processing order
@@ -584,7 +646,6 @@
self.update_actions.sort(key = lambda obj:obj[2])
self.install_actions.sort(key = lambda obj:obj[2])
- remove_npkgs = len(self.target_rem_fmris)
npkgs = 0
nfiles = 0
nbytes = 0
@@ -599,17 +660,22 @@
# install.
npkgs += 1
- self.progtrack.download_set_goal(npkgs, nfiles, nbytes)
+ self.__progtrack.download_set_goal(npkgs, nfiles, nbytes)
- self.progtrack.evaluate_done(self.target_insall_count, \
- self.target_update_count, remove_npkgs)
+ self.__progtrack.evaluate_done(self.__target_install_count, \
+ self.__target_update_count, self.__target_removal_count)
self.state = EVALUATED_OK
+
def nothingtodo(self):
""" Test whether this image plan contains any work to do """
- return not self.pkg_plans
+ # handle case w/ -n no verbose
+ if self.state == EVALUATED_PKGS:
+ return not self.__fmri_changes
+ elif self.state >= EVALUATED_OK:
+ return not self.pkg_plans
def preexecute(self):
"""Invoke the evaluated image plan
@@ -636,14 +702,14 @@
ind = indexer.Indexer(self.image,
self.image.get_manifest,
self.image.get_manifest_path,
- progtrack=self.progtrack,
- excludes=self.old_excludes)
+ progtrack=self.__progtrack,
+ excludes=self.__old_excludes)
if ind.check_index_existence():
try:
ind.check_index_has_exactly_fmris(
self.image.gen_installed_pkg_names())
except se.IncorrectIndexFileHash, e:
- self.preexecuted_indexing_error = \
+ self.__preexecuted_indexing_error = \
api_errors.WrapSuccessfulIndexingException(
e,
traceback.format_exc(),
@@ -659,7 +725,7 @@
# there's a problem updating the index on the
# new image, that error needs to be
# communicated to the user.
- self.preexecuted_indexing_error = \
+ self.__preexecuted_indexing_error = \
api_errors.WrapSuccessfulIndexingException(
e, traceback.format_exc(),
traceback.format_stack())
@@ -684,7 +750,7 @@
e.filename)
raise
- self.progtrack.download_done()
+ self.__progtrack.download_done()
except:
self.state = PREEXECUTED_ERROR
raise
@@ -745,47 +811,47 @@
# It's necessary to do this check here because the state of the
# image before the current operation is performed is desired.
- empty_image = self.is_image_empty()
+ empty_image = self.__is_image_empty()
- self.actuators.exec_prep(self.image)
+ self.__actuators.exec_prep(self.image)
- self.actuators.exec_pre_actuators(self.image)
+ self.__actuators.exec_pre_actuators(self.image)
try:
try:
# execute removals
- self.progtrack.actions_set_goal(
+ self.__progtrack.actions_set_goal(
_("Removal Phase"),
len(self.removal_actions))
for p, src, dest in self.removal_actions:
p.execute_removal(src, dest)
- self.progtrack.actions_add_progress()
- self.progtrack.actions_done()
+ self.__progtrack.actions_add_progress()
+ self.__progtrack.actions_done()
# execute installs
- self.progtrack.actions_set_goal(
+ self.__progtrack.actions_set_goal(
_("Install Phase"),
len(self.install_actions))
for p, src, dest in self.install_actions:
p.execute_install(src, dest)
- self.progtrack.actions_add_progress()
- self.progtrack.actions_done()
+ self.__progtrack.actions_add_progress()
+ self.__progtrack.actions_done()
# execute updates
- self.progtrack.actions_set_goal(
+ self.__progtrack.actions_set_goal(
_("Update Phase"),
len(self.update_actions))
for p, src, dest in self.update_actions:
p.execute_update(src, dest)
- self.progtrack.actions_add_progress()
+ self.__progtrack.actions_add_progress()
- self.progtrack.actions_done()
+ self.__progtrack.actions_done()
# handle any postexecute operations
for p in self.pkg_plans:
@@ -795,7 +861,11 @@
self.image.save_pkg_state()
# write out variant changes to the image config
- self.image.image_config_update()
+ if self.__variant_change:
+ self.image.image_config_update(
+ self.__new_variants,
+ self.__new_facets)
+
except EnvironmentError, e:
if e.errno == errno.EACCES or \
e.errno == errno.EPERM:
@@ -805,24 +875,21 @@
raise api_errors.ReadOnlyFileSystemException(e.filename)
raise
except:
- self.actuators.exec_fail_actuators(self.image)
+ self.__actuators.exec_fail_actuators(self.image)
raise
else:
- self.actuators.exec_post_actuators(self.image)
+ self.__actuators.exec_post_actuators(self.image)
self.state = EXECUTED_OK
# reduce memory consumption
- del self.removal_actions
- del self.update_actions
- del self.install_actions
-
- del self.target_rem_fmris
- del self.target_fmris
- del self.__directories
-
- del self.actuators
+ self.removal_actions = []
+ self.update_actions = []
+ self.install_actions = []
+ self.__fmri_changes = []
+ self.__directories = []
+ self.__actuators = []
# Perform the incremental update to the search indexes
# for all changed packages
@@ -833,19 +900,19 @@
in self.pkg_plans
]
del self.pkg_plans
- self.progtrack.actions_set_goal(_("Index Phase"),
+ self.__progtrack.actions_set_goal(_("Index Phase"),
len(plan_info))
self.image.update_index_dir()
ind = indexer.Indexer(self.image,
self.image.get_manifest,
self.image.get_manifest_path,
- progtrack=self.progtrack,
- excludes=self.new_excludes)
+ progtrack=self.__progtrack,
+ excludes=self.__new_excludes)
try:
if empty_image:
ind.setup()
if empty_image or ind.check_index_existence():
- ind.client_update_index((self.filters,
+ ind.client_update_index(([],
plan_info), self.image)
except KeyboardInterrupt:
raise
@@ -873,8 +940,8 @@
ind = indexer.Indexer(self.image,
self.image.get_manifest,
self.image.get_manifest_path,
- progtrack=self.progtrack,
- excludes=self.new_excludes)
+ progtrack=self.__progtrack,
+ excludes=self.__new_excludes)
ind.rebuild_index_from_scratch(
self.image.gen_installed_pkgs())
except Exception, e:
@@ -885,12 +952,237 @@
api_errors.WrapSuccessfulIndexingException(
e, traceback.format_exc(),
traceback.format_stack())
- if self.preexecuted_indexing_error is not None:
- raise self.preexecuted_indexing_error
+ if self.__preexecuted_indexing_error is not None:
+ raise self.__preexecuted_indexing_error
- def is_image_empty(self):
+ def __is_image_empty(self):
try:
self.image.gen_installed_pkg_names().next()
return False
except StopIteration:
return True
+
+ def match_user_fmris(self, patterns, all_known, pub_ranks, installed_pubs):
+ """Given a user-specified list of patterns, return a dictionary
+ of matching fmris:
+
+ {pkgname: [fmri1, fmri2, ...]
+ pkgname: [fmri1, fmri2, ...],
+ ...
+ }
+
+ Constraint used is always AUTO as per expected UI behavior.
+ If all_known is true, matching is done against all known package,
+ otherwise just all installed pkgs.
+
+ Note that patterns starting w/ pkg:/ require an exact match; patterns
+ containing '*' will using fnmatch rules; the default trailing match
+ rules are used for remaining patterns.
+
+ Exactly duplicated patterns are ignored.
+
+ Routine raises PlanCreationException if errors occur:
+ it is illegal to specify multiple different pattens that match
+ the same pkg name. Only patterns that contain wildcards are allowed
+ to match multiple packages.
+
+ Fmri lists are trimmed by publisher, either by pattern specification,
+ installed version or publisher ranking, in that order when all_known
+ is True.
+ """
+
+ # problems we check for
+ illegals = []
+ nonmatch = []
+ multimatch = []
+ not_installed = []
+ multispec = []
+ wrongpub = []
+
+ matchers = []
+ fmris = []
+ pubs = []
+ versions = []
+
+ wildcard_patterns = []
+
+ renamed_fmris = {}
+ obsolete_fmris = []
+
+ # ignore dups
+ patterns = list(set(patterns))
+ # print patterns, all_known, pub_ranks, installed_pubs
+
+ # figure out which kind of matching rules to employ
+ try:
+ for pat in patterns:
+ if "*" in pat or "?" in pat:
+ matcher = pkg.fmri.glob_match
+ fmri = pkg.fmri.MatchingPkgFmri(
+ pat, "5.11")
+ wildcard_patterns.append(pat)
+ elif pat.startswith("pkg:/"):
+ matcher = pkg.fmri.exact_name_match
+ fmri = pkg.fmri.PkgFmri(pat,
+ "5.11")
+ else:
+ matcher = pkg.fmri.fmri_match
+ fmri = pkg.fmri.PkgFmri(pat,
+ "5.11")
+
+ matchers.append(matcher)
+ pubs.append(fmri.get_publisher())
+ versions.append(fmri.version)
+ fmris.append(fmri)
+
+ except pkg.fmri.IllegalFmri, e:
+ illegals.append(e)
+
+ # Create a dictionary of patterns, with each value being
+ # a dictionary of pkg names & fmris that match that pattern.
+ ret = dict(zip(patterns, [dict() for i in patterns]))
+
+ # keep track of publishers we reject due to implict selection of
+ # installed publisher to produce better error message.
+ rejected_pubs = {}
+
+ if all_known:
+ cat = self.image.get_catalog(self.image.IMG_CATALOG_KNOWN)
+ info_needed = [pkg.catalog.Catalog.DEPENDENCY]
+ else:
+ cat = self.image.get_catalog(self.image.IMG_CATALOG_INSTALLED)
+ info_needed = []
+
+ for name in cat.names():
+ for pat, matcher, fmri, version, pub in \
+ zip(patterns, matchers, fmris, versions, pubs):
+ if not matcher(name, fmri.pkg_name):
+ continue # name doesn't match
+ for ver, entries in cat.entries_by_version(name,
+ info_needed=info_needed):
+ if version and not ver.is_successor(version,
+ pkg.version.CONSTRAINT_AUTO):
+ continue # version doesn't match
+ for f, metadata in entries:
+ fpub = f.get_publisher()
+ if pub and pub != fpub:
+ continue # specified pubs conflict
+ elif not pub and all_known and \
+ name in installed_pubs and \
+ pub_ranks[installed_pubs[name]][1] \
+ == True and installed_pubs[name] != \
+ fpub:
+ rejected_pubs.setdefault(pat,
+ set()).add(fpub)
+ continue # installed sticky pub
+ ret[pat].setdefault(f.pkg_name,
+ []).append(f)
+ states = metadata["metadata"]["states"]
+ if self.image.PKG_STATE_OBSOLETE in states:
+ obsolete_fmris.append(f)
+ if self.image.PKG_STATE_RENAMED in states and \
+ "actions" in metadata:
+ renamed_fmris[f] = metadata["actions"]
+
+ # remove multiple matches if all versions are obsolete
+ for p in patterns:
+ if len(ret[p]) > 1 and p not in wildcard_patterns:
+ # create dictionary of obsolete status vs pkg_name
+ obsolete = dict([
+ (pkg_name, reduce(operator.or_,
+ [f in obsolete_fmris for f in ret[p][pkg_name]]))
+ for pkg_name in ret[p]
+ ])
+ # remove all obsolete match if non-obsolete match also exists
+ if set([True, False]) == set(obsolete.values()):
+ for pkg_name in obsolete:
+ if obsolete[pkg_name]:
+ del ret[p][pkg_name]
+
+ # remove newer multiple match if renamed version exists
+ for p in patterns:
+ if len(ret[p]) > 1 and p not in wildcard_patterns:
+ targets = []
+ renamed_matches = (
+ pfmri
+ for pkg_name in ret[p]
+ for pfmri in ret[p][pkg_name]
+ if pfmri in renamed_fmris
+ )
+ for f in renamed_matches:
+ for a in renamed_fmris[f]:
+ a = pkg.actions.fromstr(a)
+ if a.name != "depend":
+ continue
+ if a.attrs["type"] != "require":
+ continue
+ targets.append(pkg.fmri.PkgFmri(
+ a.attrs["fmri"], "5.11"
+ ).pkg_name)
+
+ for pkg_name in ret[p].keys():
+ if pkg_name in targets:
+ del ret[p][pkg_name]
+
+ matchdict = {}
+ for p in patterns:
+ l = len(ret[p])
+ if l == 0: # no matches at all
+ if not all_known or p not in rejected_pubs:
+ nonmatch.append(p)
+ elif p in rejected_pubs:
+ wrongpub.append((p, rejected_pubs[p]))
+ elif l > 1 and p not in wildcard_patterns: # multiple matches
+ multimatch.append((p, [n for n in ret[p]]))
+ else: # single match or wildcard
+ for k in ret[p].keys(): # for each matching package name
+ matchdict.setdefault(k, []).append(p)
+
+ for name in matchdict:
+ if len(matchdict[name]) > 1: # different pats, same pkg
+ multispec.append(tuple([name] + matchdict[name]))
+
+ if not all_known:
+ not_installed, nonmatch = nonmatch, not_installed
+
+ if illegals or nonmatch or multimatch or not_installed or \
+ multispec or wrongpub:
+ raise api_errors.PlanCreationException(unmatched_fmris=nonmatch,
+ multiple_matches=multimatch, illegal=illegals,
+ missing_matches=not_installed, multispec=multispec, wrong_publishers=wrongpub)
+ # merge patterns together now that there are no conflicts
+ proposed_dict = {}
+ for d in ret.values():
+ proposed_dict.update(d)
+
+ # eliminate lower ranked publishers
+
+ if all_known: # no point for installed pkgs....
+ for pkg_name in proposed_dict:
+ pubs_found = set([
+ f.get_publisher()
+ for f in proposed_dict[pkg_name]
+ ])
+ # 1000 is hack for installed but unconfigured publishers
+ best_pub = sorted([
+ (pub_ranks.get(p, (1000, True))[0], p)
+ for p in pubs_found
+ ])[0][1]
+
+ proposed_dict[pkg_name] = [
+ f
+ for f in proposed_dict[pkg_name]
+ if f.get_publisher() == best_pub
+ ]
+
+ # construct references so that we can know which pattern
+ # generated which fmris...
+
+ references = dict([
+ (f, p)
+ for p in ret.keys()
+ for flist in ret[p].values()
+ for f in flist
+ ])
+
+ return proposed_dict, references