#!/usr/bin/python

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

#

# Copyright 2010 Sun Microsystems, Inc.  All rights reserved.

# Use is subject to license terms.

#

import datetime

import errno

import fcntl

import os

import platform

import shutil

import tempfile

import time

import urllib

from contextlib import contextmanager

from pkg.client import global_settings

logger = global_settings.logger

import pkg.catalog

import pkg.client.api_errors            as api_errors

import pkg.client.history               as history

import pkg.client.imageconfig           as imageconfig

import pkg.client.imageplan             as imageplan

import pkg.client.pkgplan               as pkgplan

import pkg.client.progress              as progress

import pkg.client.publisher             as publisher

import pkg.client.transport.transport   as transport

import pkg.fmri

import pkg.manifest                     as manifest

import pkg.misc                         as misc

import pkg.portable                     as portable

import pkg.server.catalog

import pkg.version

from pkg.client.debugvalues import DebugValues

from pkg.client.imagetypes import IMG_USER, IMG_ENTIRE

from pkg.misc import CfgCacheError, EmptyI, EmptyDict

img_user_prefix = ".org.opensolaris,pkg"

img_root_prefix = "var/pkg"

IMG_PUB_DIR = "publisher"

class Image(object):

        """An Image object is a directory tree containing the laid-down contents

        of a self-consistent graph of Packages.

        An Image has a root path.

        An Image of type IMG_ENTIRE does not have a parent Image.  Other Image

        types must have a parent Image.  The external state of the parent Image

        must be accessible from the Image's context, or duplicated within the

        Image (IMG_PARTIAL for zones, for instance).

        The parent of a user Image can be a partial Image.  The parent of a

        partial Image must be an entire Image.

        An Image of type IMG_USER stores its external state at self.root +

        ".org.opensolaris,pkg".

        An Image of type IMG_ENTIRE or IMG_PARTIAL stores its external state at

        self.root + "/var/pkg".

        An Image needs to be able to have a different repository set than the

        system's root Image.

        Directory layout

          $IROOT/file

               Directory containing file hashes of installed packages.

          $IROOT/pkg

               Directory containing manifests.

          $IROOT/index

               Directory containing search indices.

          $IROOT/cfg_cache

               File containing image's cached configuration.

          $IROOT/opaque

               File containing image's opaque state.

          $IROOT/publisher

                Directory containing publisher metadata.

        All of these directories and files other than state are considered

        essential for any image to be complete. To add a new essential file or

        subdirectory, the following steps should be done.

        If it's a directory, add it to the image_subdirs list below and it will

        be created automatically. The programmer must fill the directory as

        needed. If a file is added, the programmer is responsible for creating

        that file during image creation at an appropriate place and time.

        If a directory is required to be present in order for an image to be

        identifiable as such, it should go into required_subdirs instead.

        However, upgrade issues abound; this list should probably not change.

        Once those steps have been carried out, the change should be added

        to the test suite for image corruption (t_pkg_install_corrupt_image.py).

        This will likely also involve a change to

        SingleDepotTestCaseCorruptImage in testutils.py. Each of these files

        outline what must be updated.

        XXX Root path probably can't be absolute, so that we can combine or

        reuse Image contents.

        XXX Image file format?  Image file manipulation API?"""

        # Class constants

        IMG_CATALOG_KNOWN = "known"

        IMG_CATALOG_INSTALLED = "installed"

        # Please note that the values of these PKG_STATE constants should not

        # be changed as it would invalidate existing catalog data stored in the

        # image.  This means that if a constant is removed, the values of the

        # other constants should not change, etc.

        # This state indicates that a package is present in a repository

        # catalog.

        PKG_STATE_KNOWN = 0

        # This is a transitory state used to indicate that a package is no

        # longer present in a repository catalog; it is only used to clear

        # PKG_STATE_KNOWN.

        PKG_STATE_UNKNOWN = 1

        # This state indicates that a package is installed.

        PKG_STATE_INSTALLED = 2

        # This is a transitory state used to indicate that a package is no

        # longer installed; it is only used to clear PKG_STATE_INSTALLED.

        PKG_STATE_UNINSTALLED = 3

        PKG_STATE_UPGRADABLE = 4

        # These states are used to indicate the package's related catalog

        # version.  This is helpful to consumers of the catalog data so that

        # they can be aware of what metadata may not immediately available

        # (require manifest retrieval) based on the catalog version.

        PKG_STATE_V0 = 6

        PKG_STATE_V1 = 7

        PKG_STATE_OBSOLETE = 8

        PKG_STATE_RENAMED = 9

        # Class properties

        required_subdirs = [ "file", "pkg" ]

        image_subdirs = required_subdirs + [ "index", IMG_PUB_DIR,

            "state/installed", "state/known" ]

        def __init__(self, root, user_provided_dir=False, progtrack=None,

            should_exist=True, imgtype=None, force=False):

                if should_exist:

                        assert(imgtype is None)

                        assert(not force)

                else:

                        assert(imgtype is not None)

                self.__init_catalogs()

                self.__upgraded = False

                self.attrs = {

                    "Policy-Require-Optional": False,

                    "Policy-Pursue-Latest": True

                }

                self.blocking_locks = False

                self.cfg_cache = None

                self.dl_cache_dir = None

                self.dl_cache_incoming = None

                self.history = history.History()

                self.imageplan = None # valid after evaluation succeeds

                self.img_prefix = None

                self.imgdir = None

                self.index_dir = None

                self.is_user_cache_dir = False

                self.pkgdir = None

                self.root = root

                self.__lock = pkg.nrlock.NRLock()

                self.__locked = False

                self.__lockf = None

                self.__req_dependents = None

                # Transport operations for this image

                self.transport = transport.Transport(self)

                if should_exist:

                        self.find_root(self.root, user_provided_dir,

                            progtrack)

                else:

                        if not force and self.image_type(self.root) != None:

                                raise api_errors.ImageAlreadyExists(self.root)

                        if not force and os.path.exists(self.root) and \

                            len(os.listdir(self.root)) > 0:

                                raise api_errors.CreatingImageInNonEmptyDir(

                                    self.root)

                        self.__set_dirs(root=self.root, imgtype=imgtype,

                            progtrack=progtrack)

                # a place to keep info about saved_files; needed by file action

                self.saved_files = {}

                # right now we don't explicitly set dir/file modes everywhere;

                # set umask to proper value to prevent problems w/ overly

                # locked down umask.

                os.umask(0022)

        def _check_subdirs(self, sub_d, prefix):

                for n in self.required_subdirs:

                        if not os.path.isdir(os.path.join(sub_d, prefix, n)):

                                return False

                return True

        def __catalog_loaded(self, name):

                """Returns a boolean value indicating whether the named catalog

                has already been loaded.  This is intended to be used as an

                optimization function to determine which catalog to request."""

                return name in self.__catalogs

        def __init_catalogs(self):

                """Initializes default catalog state.  Actual data is provided

                on demand via get_catalog()"""

                # This is used to cache image catalogs.

                self.__catalogs = {}

                # This is used to keep track of what packages have been added

                # to IMG_CATALOG_KNOWN by set_pkg_state().

                self.__catalog_new_installs = set()

        @property

        def locked(self):

                """Returns a boolean value indicating whether the image is

                currently locked."""

                return self.__locked

        @contextmanager

        def locked_op(self, op, allow_unprivileged=False):

                """Helper method for executing an image-modifying operation

                that needs locking.  It also automatically handles calling

                log_operation_start and log_operation_end.  Locking behaviour

                is controlled by the blocking_locks image property.

                'allow_unprivileged' is an optional boolean value indicating

                that permissions-related exceptions should be ignored when

                attempting to obtain the lock as the related operation will

                still work correctly even though the image cannot (presumably)

                be modified.

                """

                error = None

                self.lock(allow_unprivileged=allow_unprivileged)

                try:

                        self.history.log_operation_start(op)

                        yield

                except Exception, e:

                        error = e

                        raise

                finally:

                        self.history.log_operation_end(error=error)

                        self.unlock()

        def lock(self, allow_unprivileged=False):

                """Locks the image in preparation for an image-modifying

                operation.  Raises an ImageLockedError exception on failure.

                Locking behaviour is controlled by the blocking_locks image

                property.

                'allow_unprivileged' is an optional boolean value indicating

                that permissions-related exceptions should be ignored when

                attempting to obtain the lock as the related operation will

                still work correctly even though the image cannot (presumably)

                be modified.

                """

                blocking = self.blocking_locks

                # First, attempt to obtain a thread lock.

                if not self.__lock.acquire(blocking=blocking):

                        raise api_errors.ImageLockedError()

                self.__locked = True

                try:

                        # Attempt to obtain a file lock.

                        self.__lock_process()

                except api_errors.PermissionsException:

                        if not allow_unprivileged:

                                self.__lock.release()

                                raise

                except:

                        # If process lock fails, ensure thread lock is released.

                        self.__lock.release()

                        raise

        def __lock_process(self):

                """Locks the image to prevent modification by other

                processes."""

                if not os.path.exists(self.imgdir):

                        # Image structure doesn't exist yet so a file lock

                        # cannot be obtained.  This path should only happen

                        # during image-create.

                        return

                # Attempt to obtain a file lock for the image.

                lfpath = os.path.join(self.imgdir, "lock")

                lock_type = fcntl.LOCK_EX

                if not self.blocking_locks:

                        lock_type |= fcntl.LOCK_NB

                # Attempt an initial open of the lock file.

                lf = None

                try:

                        lf = open(lfpath, "ab+")

                except EnvironmentError, e:

                        if e.errno == errno.EACCES:

                                raise api_errors.PermissionsException(

                                    e.filename)

                        if e.errno == errno.EROFS:

                                raise api_errors.ReadOnlyFileSystemException(

                                    e.filename)

                        raise

                # Attempt to lock the file.

                try:

                        fcntl.lockf(lf, lock_type)

                except IOError, e:

                        if e.errno not in (errno.EAGAIN, errno.EACCES):

                                raise

                        # If the lock failed (because it is likely contended),

                        # then extract the information about the lock acquirer

                        # and raise an exception.

                        pid_data = lf.read().strip()

                        pid, pid_name, hostname, lock_ts = \

                            pid_data.split("\n", 4)

                        raise api_errors.ImageLockedError(pid=pid,

                            pid_name=pid_name, hostname=hostname)

                # Store lock time as ISO-8601 basic UTC timestamp in lock file.

                lock_ts = pkg.catalog.now_to_basic_ts()

                # Store information about the lock acquirer and write it.

                try:

                        lf.truncate(0)

                        lf.write("\n".join((str(os.getpid()),

                            global_settings.client_name,

                            platform.node(), lock_ts, "\n")))

                        lf.flush()

                        self.__lockf = lf

                except EnvironmentError, e:

                        if e.errno == errno.EACCES:

                                raise api_errors.PermissionsException(

                                    e.filename)

                        if e.errno == errno.EROFS:

                                raise api_errors.ReadOnlyFileSystemException(

                                    e.filename)

                        raise

        def unlock(self):

                """Unlocks the image."""

                if self.__lockf:

                        # To avoid race conditions with the next caller waiting

                        # for the lock file, it is simply truncated instead of

                        # removed.

                        self.__lockf.truncate(0)

                        self.__lockf.close()

                        self.__lockf = None

                self.__locked = False

                self.__lock.release()

        def image_type(self, d):

                """Returns the type of image at directory: d; or None"""

                rv = None

                if os.path.isdir(os.path.join(d, img_user_prefix)) and \

                        os.path.isfile(os.path.join(d, img_user_prefix,

                            imageconfig.CFG_FILE)) and \

                            self._check_subdirs(d, img_user_prefix):

                        rv = IMG_USER

                elif os.path.isdir(os.path.join(d, img_root_prefix)) \

                         and os.path.isfile(os.path.join(d,

                             img_root_prefix, imageconfig.CFG_FILE)) and \

                             self._check_subdirs(d, img_root_prefix):

                        rv = IMG_ENTIRE

                return rv

        def find_root(self, d, exact_match=False, progtrack=None):

                # Ascend from the given directory d to find first

                # encountered image.  If exact_match is true, if the

                # image found doesn't match startd, raise an

                # ImageNotFoundException.

                startd = d

                # eliminate problem if relative path such as "." is passed in

                d = os.path.realpath(d)

                while True:

                        imgtype = self.image_type(d)

                        if imgtype == IMG_USER:

                                # XXX Should look at image file to determine

                                # repo URIs.

                                if exact_match and \

                                    os.path.realpath(startd) != \

                                    os.path.realpath(d):

                                        raise api_errors.ImageNotFoundException(

                                            exact_match, startd, d)

                                self.__set_dirs(imgtype=imgtype, root=d,

                                    progtrack=progtrack)

                                self.attrs["Build-Release"] = "5.11"

                                return

                        elif imgtype == IMG_ENTIRE:

                                # XXX Look at image file to determine

                                # repo URIs.

                                # XXX Look at image file to determine if this

                                # image is a partial image.

                                if exact_match and \

                                    os.path.realpath(startd) != \

                                    os.path.realpath(d):

                                        raise api_errors.ImageNotFoundException(

                                            exact_match, startd, d)

                                self.__set_dirs(imgtype=imgtype, root=d,

                                    progtrack=progtrack)

                                self.attrs["Build-Release"] = "5.11"

                                return

                        # XXX follow symlinks or not?

                        oldpath = d

                        d = os.path.normpath(os.path.join(d, os.path.pardir))

                        # Make sure we are making progress and aren't in an

                        # infinite loop.

                        #

                        # (XXX - Need to deal with symlinks here too)

                        if d == oldpath:

                                raise api_errors.ImageNotFoundException(

                                    exact_match, startd, d)

        def __load_config(self):

                """Load this image's cached configuration from the default

                location."""

                # XXX Incomplete with respect to doc/image.txt description of

                # configuration.

                if self.root == None:

                        raise RuntimeError, "self.root must be set"

                ic = imageconfig.ImageConfig(self.root,

                    self._get_publisher_meta_dir())

                ic.read(self.imgdir)

                self.cfg_cache = ic

                for pub in self.gen_publishers(inc_disabled=True):

                        pub.transport = self.transport

        def save_config(self):

                # First, create the image directories if they haven't been, so

                # the cfg_cache can be written.

                self.mkdirs()

                self.cfg_cache.write(self.imgdir)

        # XXX mkdirs and set_attrs() need to be combined into a create

        # operation.

        def mkdirs(self):

                for sd in self.image_subdirs: