#!/usr/bin/python

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

#

# Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved.

#

import M2Crypto as m2

import atexit

import copy

import datetime

import errno

import os

import platform

import shutil

import simplejson as json

import stat

import tempfile

import time

import urllib

from contextlib import contextmanager

from pkg.client import global_settings

logger = global_settings.logger

import pkg.actions

import pkg.catalog

import pkg.client.api_errors            as apx

import pkg.client.bootenv               as bootenv

import pkg.client.history               as history

import pkg.client.imageconfig           as imageconfig

import pkg.client.imageplan             as imageplan

import pkg.client.linkedimage           as li

import pkg.client.pkgdefs               as pkgdefs

import pkg.client.pkgplan               as pkgplan

import pkg.client.progress              as progress

import pkg.client.publisher             as publisher

import pkg.client.sigpolicy             as sigpolicy

import pkg.client.transport.transport   as transport

import pkg.config                       as cfg

import pkg.fmri

import pkg.lockfile                     as lockfile

import pkg.manifest                     as manifest

import pkg.misc                         as misc

import pkg.nrlock

import pkg.pkgsubprocess                as subprocess

import pkg.portable                     as portable

import pkg.server.catalog

import pkg.smf                          as smf

import pkg.version

from pkg.client.debugvalues import DebugValues

from pkg.client.imagetypes import IMG_USER, IMG_ENTIRE

from pkg.misc import EmptyI, EmptyDict

img_user_prefix = ".org.opensolaris,pkg"

img_root_prefix = "var/pkg"

IMG_PUB_DIR = "publisher"

class Image(object):

        """An Image object is a directory tree containing the laid-down contents

        of a self-consistent graph of Packages.

        An Image has a root path.

        An Image of type IMG_ENTIRE does not have a parent Image.  Other Image

        types must have a parent Image.  The external state of the parent Image

        must be accessible from the Image's context, or duplicated within the

        Image (IMG_PARTIAL for zones, for instance).

        The parent of a user Image can be a partial Image.  The parent of a

        partial Image must be an entire Image.

        An Image of type IMG_USER stores its external state at self.root +

        ".org.opensolaris,pkg".

        An Image of type IMG_ENTIRE or IMG_PARTIAL stores its external state at

        self.root + "/var/pkg".

        An Image needs to be able to have a different repository set than the

        system's root Image.

        For image format details, see section 5.3 of doc/on-disk-format.txt

        in the pkg(5) gate.

        """

        # Class constants

        CURRENT_VERSION = 4

        IMG_CATALOG_KNOWN = "known"

        IMG_CATALOG_INSTALLED = "installed"

        # This is a transitory state used for temporary package sources to

        # indicate that the package entry should be removed if it does not

        # also have PKG_STATE_INSTALLED.  This state must not be written

        # to disk.

        PKG_STATE_ALT_SOURCE = 99

        # Please note that the values of these PKG_STATE constants should not

        # be changed as it would invalidate existing catalog data stored in the

        # image.  This means that if a constant is removed, the values of the

        # other constants should not change, etc.

        # This state indicates that a package is present in a repository

        # catalog.

        PKG_STATE_KNOWN = 0

        # This is a transitory state used to indicate that a package is no

        # longer present in a repository catalog; it is only used to clear

        # PKG_STATE_KNOWN.

        PKG_STATE_UNKNOWN = 1

        # This state indicates that a package is installed.

        PKG_STATE_INSTALLED = 2

        # This is a transitory state used to indicate that a package is no

        # longer installed; it is only used to clear PKG_STATE_INSTALLED.

        PKG_STATE_UNINSTALLED = 3

        PKG_STATE_UPGRADABLE = 4

        # These states are used to indicate the package's related catalog

        # version.  This is helpful to consumers of the catalog data so that

        # they can be aware of what metadata may not immediately available

        # (require manifest retrieval) based on the catalog version.

        PKG_STATE_V0 = 6

        PKG_STATE_V1 = 7

        PKG_STATE_OBSOLETE = 8

        PKG_STATE_RENAMED = 9

        # These states are used to indicate why a package was rejected and

        # is not available for packaging operations.

        PKG_STATE_UNSUPPORTED = 10      # Package contains invalid or

                                        # unsupported metadata.

        def __init__(self, root, user_provided_dir=False, progtrack=None,

            should_exist=True, imgtype=None, force=False,

            augment_ta_from_parent_image=True, allow_ondisk_upgrade=None,

            allow_ambiguous=False, props=misc.EmptyDict, cmdpath=None,

            runid=-1):

                if should_exist:

                        assert(imgtype is None)

                        assert(not force)

                else:

                        assert(imgtype is not None)

                # Alternate package sources.

                self.__alt_pkg_pub_map = None

                self.__alt_pubs = None

                self.__alt_known_cat = None

                self.__alt_pkg_sources_loaded = False

                if (runid < 0):

                        runid = os.getpid()

                self.runid = runid

                # Determine identity of client executable if appropriate.

                if cmdpath == None:

                        cmdpath = misc.api_cmdpath()

                self.cmdpath = cmdpath

                if self.cmdpath != None:

                        self.__cmddir = os.path.dirname(cmdpath)

                # prevent brokeness in the test suite

                if self.cmdpath and \

                    "PKG_NO_RUNPY_CMDPATH" in os.environ and \

                    self.cmdpath.endswith(os.sep + "run.py"):

                        raise RuntimeError, """

An Image object was allocated from within ipkg test suite and

cmdpath was not explicitly overridden.  Please make sure to

explicitly set cmdpath when allocating an Image object, or

override cmdpath when allocating an Image object by setting PKG_CMDPATH

in the environment or by setting simulate_cmdpath in DebugValues."""

                self.linked = None

                # Indicates whether automatic image format upgrades of the

                # on-disk format are allowed.

                self.allow_ondisk_upgrade = allow_ondisk_upgrade

                self.allow_ambiguous = allow_ambiguous

                self.__upgraded = False

                # Must happen after upgraded assignment.

                self.__init_catalogs()

                self.attrs = { "Build-Release": "5.11" } # XXX real data needed

                self.blocking_locks = False

                self.cfg = None

                self.history = history.History()

                self.imageplan = None

                self.img_prefix = None

                self.imgdir = None

                self.index_dir = None

                self.plandir = None

                self.root = root

                self.version = -1

                # Can have multiple read cache dirs...

                self.__read_cache_dirs = []

                # ...but only one global write cache dir and incoming write dir.

                self.__write_cache_dir = None

                self.__user_cache_dir = None

                self._incoming_cache_dir = None

                # Set if write_cache is actually a tree like /var/pkg/publisher

                # instead of a flat cache.

                self.__write_cache_root = None

                self.__lock = pkg.nrlock.NRLock()

                self.__lockfile = None

                self.__sig_policy = None

                self.__trust_anchors = None

                # cache for presence of boot-archive

                self.__boot_archive = None

                # When users and groups are added before their database files

                # have been installed, the actions store them temporarily in the

                # image, in these members.

                self._users = set()

                self._groups = set()

                self._usersbyname = {}

                self._groupsbyname = {}

                # Set of pkg stems being avoided

                self.__avoid_set = None

                self.__avoid_set_altered = False

                # set of pkg stems subject to group

                # dependency but removed because obsolete

                self.__group_obsolete = None

                self.__property_overrides = { "property": props }

                # Transport operations for this image

                self.transport = transport.Transport(

                    transport.ImageTransportCfg(self))

                self.linked = li.LinkedImage(self)

                if should_exist:

                        self.find_root(self.root, user_provided_dir,

                            progtrack)

                else:

                        if not force and self.image_type(self.root) != None:

                                raise apx.ImageAlreadyExists(self.root)

                        if not force and os.path.exists(self.root):

                                # ignore .zfs snapdir if it's present

                                snapdir = os.path.join(self.root, ".zfs")

                                listdir = set(os.listdir(self.root))

                                if os.path.isdir(snapdir):

                                        listdir -= set([".zfs"])

                                if len(listdir) > 0:

                                        raise apx.CreatingImageInNonEmptyDir(

                                            self.root)

                        self.__set_dirs(root=self.root, imgtype=imgtype,

                            progtrack=progtrack, purge=True)

                # right now we don't explicitly set dir/file modes everywhere;

                # set umask to proper value to prevent problems w/ overly

                # locked down umask.

                os.umask(0022)

                self.augment_ta_from_parent_image = augment_ta_from_parent_image

        @staticmethod

        def alloc(*args, **kwargs):

                return Image(*args, **kwargs)

        def __catalog_loaded(self, name):

                """Returns a boolean value indicating whether the named catalog

                has already been loaded.  This is intended to be used as an

                optimization function to determine which catalog to request."""

                return name in self.__catalogs

        def __init_catalogs(self):

                """Initializes default catalog state.  Actual data is provided

                on demand via get_catalog()"""

                if self.__upgraded and self.version < 3:

                        # Ignore request; transformed catalog data only exists

                        # in memory and can't be reloaded from disk.

                        return

                # This is used to cache image catalogs.

                self.__catalogs = {}

                self.__alt_pkg_sources_loaded = False

        @property

        def signature_policy(self):

                """Returns the signature policy for this image."""

                if self.__sig_policy is not None:

                        return self.__sig_policy

                txt = self.cfg.get_policy_str(imageconfig.SIGNATURE_POLICY)

                names = self.cfg.get_property("property",

                    "signature-required-names")

                self.__sig_policy = sigpolicy.Policy.policy_factory(txt, names)

                return self.__sig_policy

        @property

        def trust_anchors(self):

                """Return a dictionary mapping subject hashes for certificates

                this image trusts to those certs.  The image trusts those

                trust anchors in its trust_anchor_dir and those in the from

                which pkg was run."""

                if self.__trust_anchors is not None:

                        return self.__trust_anchors

                user_set_ta_loc = True

                rel_dir = self.get_property("trust-anchor-directory")

                if rel_dir[0] == "/":

                        rel_dir = rel_dir[1:]

                trust_anchor_loc = os.path.join(self.root, rel_dir)

                loc_is_dir = os.path.isdir(trust_anchor_loc)

                pkg_trust_anchors = {}

                if self.__cmddir and self.augment_ta_from_parent_image:

                        pkg_trust_anchors = Image(self.__cmddir,

                            augment_ta_from_parent_image=False,

                            allow_ambiguous=True,

                            cmdpath=self.cmdpath).trust_anchors

                if not loc_is_dir and os.path.exists(trust_anchor_loc):

                        raise apx.InvalidPropertyValue(_("The trust "

                            "anchors for the image were expected to be found "

                            "in %s, but that is not a directory.  Please set "

                            "the image property 'trust-anchor-directory' to "

                            "the correct path.") % trust_anchor_loc)

                self.__trust_anchors = {}

                if loc_is_dir:

                        for fn in os.listdir(trust_anchor_loc):

                                pth = os.path.join(trust_anchor_loc, fn)

                                if os.path.islink(pth):

                                        continue

                                trusted_ca = m2.X509.load_cert(pth)

                                # M2Crypto's subject hash doesn't match

                                # openssl's subject hash so recompute it so all

                                # hashes are in the same universe.

                                s = trusted_ca.get_subject().as_hash()

                                self.__trust_anchors.setdefault(s, [])

                                self.__trust_anchors[s].append(trusted_ca)

                for s in pkg_trust_anchors:

                        if s not in self.__trust_anchors:

                                self.__trust_anchors[s] = pkg_trust_anchors[s]

                return self.__trust_anchors

        @property

        def locked(self):

                """Returns a boolean value indicating whether the image is

                currently locked."""

                return self.__lock and self.__lock.locked

        @contextmanager

        def locked_op(self, op, allow_unprivileged=False, new_history_op=True):

                """Helper method for executing an image-modifying operation

                that needs locking.  It automatically handles calling

                log_operation_start and log_operation_end by default.  Locking

                behaviour is controlled by the blocking_locks image property.

                'allow_unprivileged' is an optional boolean value indicating

                that permissions-related exceptions should be ignored when

                attempting to obtain the lock as the related operation will

                still work correctly even though the image cannot (presumably)

                be modified.

                'new_history_op' indicates whether we should handle history

                operations.

                """

                error = None

                self.lock(allow_unprivileged=allow_unprivileged)

                try:

                        be_name, be_uuid = \

                            bootenv.BootEnv.get_be_name(self.root)

                        if new_history_op:

                                self.history.log_operation_start(op,

                                    be_name=be_name, be_uuid=be_uuid)

                        yield

                except apx.ImageLockedError, e:

                        # Don't unlock the image if the call failed to

                        # get the lock.

                        error = e

                        raise

                except Exception, e:

                        error = e

                        self.unlock()

                        raise

                else:

                        self.unlock()

                finally:

                        if new_history_op:

                                self.history.log_operation_end(error=error)

        def lock(self, allow_unprivileged=False):

                """Locks the image in preparation for an image-modifying

                operation.  Raises an ImageLockedError exception on failure.

                Locking behaviour is controlled by the blocking_locks image

                property.

                'allow_unprivileged' is an optional boolean value indicating

                that permissions-related exceptions should be ignored when

                attempting to obtain the lock as the related operation will

                still work correctly even though the image cannot (presumably)

                be modified.

                """

                blocking = self.blocking_locks

                # First, attempt to obtain a thread lock.

                if not self.__lock.acquire(blocking=blocking):

                        raise apx.ImageLockedError()

                try:

                        # Attempt to obtain a file lock.

                        self.__lockfile.lock(blocking=blocking)

                except EnvironmentError, e:

                        exc = None

                        if e.errno == errno.ENOENT:

                                return

                        if e.errno == errno.EACCES:

                                exc = apx.PermissionsException(e.filename)

                        elif e.errno == errno.EROFS:

                                exc = apx.ReadOnlyFileSystemException(

                                    e.filename)

                        else:

                                self.__lock.release()

                                raise

                        if exc and not allow_unprivileged:

                                self.__lock.release()

                                raise exc

                except:

                        # If process lock fails, ensure thread lock is released.

                        self.__lock.release()

                        raise

        def unlock(self):

                """Unlocks the image."""

                try:

                        if self.__lockfile:

                                self.__lockfile.unlock()

                finally:

                        self.__lock.release()

        def image_type(self, d):

                """Returns the type of image at directory: d; or None"""

                rv = None

                def is_image(sub_d, prefix):

                        # First check for new image configuration file.

                        if os.path.isfile(os.path.join(sub_d, prefix,

                            "pkg5.image")):

                                # Regardless of directory structure, assume

                                # this is an image for now.

                                return True

                        if not os.path.isfile(os.path.join(sub_d, prefix,

                            "cfg_cache")):

                                # For older formats, if configuration is

                                # missing, this can't be an image.

                                return False