#!/usr/bin/python2.4

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

#

# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

# Use is subject to license terms.

#

import datetime as dt

import errno

import os

import platform

import shutil

import tempfile

import time

import urllib

import pkg.Uuid25

import pkg.catalog             as catalog

import pkg.client.api_errors   as api_errors

import pkg.client.constraint   as constraint

import pkg.client.history      as history

import pkg.client.imageconfig  as imageconfig

import pkg.client.imageplan    as imageplan

import pkg.client.imagestate   as imagestate

import pkg.client.pkgplan      as pkgplan

import pkg.client.progress     as progress

import pkg.client.publisher    as publisher

import pkg.client.retrieve     as retrieve

import pkg.client.variant      as variant

import pkg.fmri

import pkg.manifest            as manifest

import pkg.misc                as misc

import pkg.portable            as portable

import pkg.version

from pkg.actions import MalformedActionError

from pkg.client import global_settings

from pkg.client.api_errors import InvalidDepotResponseException

from pkg.client.imagetypes import IMG_USER, IMG_ENTIRE

from pkg.misc import CfgCacheError

from pkg.misc import EmptyI, EmptyDict

from pkg.misc import msg, emsg

from pkg.misc import TransportException

from pkg.misc import TransportFailures

CATALOG_CACHE_FILE = "catalog_cache"

img_user_prefix = ".org.opensolaris,pkg"

img_root_prefix = "var/pkg"

PKG_STATE_INSTALLED = "installed"

PKG_STATE_KNOWN = "known"

class Image(object):

        """An Image object is a directory tree containing the laid-down contents

        of a self-consistent graph of Packages.

        An Image has a root path.

        An Image of type IMG_ENTIRE does not have a parent Image.  Other Image

        types must have a parent Image.  The external state of the parent Image

        must be accessible from the Image's context, or duplicated within the

        Image (IMG_PARTIAL for zones, for instance).

        The parent of a user Image can be a partial Image.  The parent of a

        partial Image must be an entire Image.

        An Image of type IMG_USER stores its external state at self.root +

        ".org.opensolaris,pkg".

        An Image of type IMG_ENTIRE or IMG_PARTIAL stores its external state at

        self.root + "/var/pkg".

        An Image needs to be able to have a different repository set than the

        system's root Image.

        Directory layout

          $IROOT/catalog

               Directory containing catalogs for URIs of interest.  Filename is

               the escaped URI of the catalog.

          $IROOT/file

               Directory containing file hashes of installed packages.

          $IROOT/pkg

               Directory containing manifests and states of installed packages.

          $IROOT/index

               Directory containing reverse-index databases.

          $IROOT/cfg_cache

               File containing image's cached configuration.

          $IROOT/opaque

               File containing image's opaque state.

          $IROOT/state/installed

               Directory containing files whose names identify the installed

               packages.

        All of these directories and files other than state are considered

        essential for any image to be complete. To add a new essential file or

        subdirectory, the following steps should be done.

        If it's a directory, add it to the image_subdirs list below and it will

        be created automatically. The programmer must fill the directory as

        needed. If a file is added, the programmer is responsible for creating

        that file during image creation at an appropriate place and time.

        If a directory is required to be present in order for an image to be

        identifiable as such, it should go into required_subdirs instead.

        However, upgrade issues abound; this list should probably not change.

        Once those steps have been carried out, the change should be added

        to the test suite for image corruption (t_pkg_install_corrupt_image.py).

        This will likely also involve a change to

        SingleDepotTestCaseCorruptImage in testutils.py. Each of these files

        outline what must be updated.

        XXX Root path probably can't be absolute, so that we can combine or

        reuse Image contents.

        XXX Image file format?  Image file manipulation API?"""

        required_subdirs = [ "catalog", "file", "pkg" ]

        image_subdirs = required_subdirs + [ "index", "state/installed" ]

        def __init__(self):

                self.cfg_cache = None

                self.type = None

                self.root = None

                self.history = history.History()

                self.imgdir = None

                self.pkgdir = None

                self.img_prefix = None

                self.index_dir = None

                self.repo_uris = []

                self.filter_tags = {}

                self.__catalogs = {}

                self._catalog = {}

                self.__pkg_states = None

                self.dl_cache_dir = None

                self.dl_cache_incoming = None

                self.is_user_cache_dir = False

                self.state = imagestate.ImageState()

                self.attrs = {

                    "Policy-Require-Optional": False,

                    "Policy-Pursue-Latest": True

                }

                self.__catalog_cache_mod_time = None

                self.imageplan = None # valid after evaluation succeeds

                self.constraints = constraint.ConstraintSet()

                # a place to keep info about saved_files; needed by file action

                self.saved_files = {}

                # A place to keep track of which manifests (based on fmri and

                # operation) have already provided intent information.

                self.__touched_manifests = {}

                self.__manifest_cache = {}

                # right now we don't explicitly set dir/file modes everywhere;

                # set umask to proper value to prevent problems w/ overly

                # locked down umask.

                os.umask(0022)

        def _check_subdirs(self, sub_d, prefix):

                for n in self.required_subdirs:

                        if not os.path.isdir(os.path.join(sub_d, prefix, n)):

                                return False

                return True

        def image_type(self, d):

                """Returns the type of image at directory: d; or None"""

                rv = None

                if os.path.isdir(os.path.join(d, img_user_prefix)) and \

                        os.path.isfile(os.path.join(d, img_user_prefix,

                            imageconfig.CFG_FILE)) and \

                            self._check_subdirs(d, img_user_prefix):

                        rv = IMG_USER

                elif os.path.isdir(os.path.join(d, img_root_prefix)) \

                         and os.path.isfile(os.path.join(d,

                             img_root_prefix, imageconfig.CFG_FILE)) and \

                             self._check_subdirs(d, img_root_prefix):

                        rv = IMG_ENTIRE

                return rv

        def find_root(self, d, exact_match=False):

                # Ascend from the given directory d to find first

                # encountered image. If exact_match is true, if the

                # image found doesn't match startd, raise an

                # ImageNotFoundException.

                startd = d

                # eliminate problem if relative path such as "." is passed in

                d = os.path.realpath(d)

                while True:

                        imgtype = self.image_type(d)

                        if imgtype == IMG_USER:

                                # XXX Look at image file to determine filter

                                # tags and repo URIs.

                                if exact_match and \

                                    os.path.realpath(startd) != \

                                    os.path.realpath(d):

                                        raise api_errors.ImageNotFoundException(

                                            exact_match, startd, d)

                                self.__set_dirs(imgtype=imgtype, root=d)

                                self.attrs["Build-Release"] = "5.11"

                                return

                        elif imgtype == IMG_ENTIRE:

                                # XXX Look at image file to determine filter

                                # tags and repo URIs.

                                # XXX Look at image file to determine if this

                                # image is a partial image.

                                if exact_match and \

                                    os.path.realpath(startd) != \

                                    os.path.realpath(d):

                                        raise api_errors.ImageNotFoundException(

                                            exact_match, startd, d)

                                self.__set_dirs(imgtype=imgtype, root=d)

                                self.attrs["Build-Release"] = "5.11"

                                return

                        # XXX follow symlinks or not?

                        oldpath = d

                        d = os.path.normpath(os.path.join(d, os.path.pardir))

                        # Make sure we are making progress and aren't in an

                        # infinite loop.

                        #

                        # (XXX - Need to deal with symlinks here too)

                        if d == oldpath:

                                raise api_errors.ImageNotFoundException(

                                    exact_match, startd, d)

        def load_config(self):

                """Load this image's cached configuration from the default

                location."""

                # XXX Incomplete with respect to doc/image.txt description of

                # configuration.

                if self.root == None:

                        raise RuntimeError, "self.root must be set"

                ic = imageconfig.ImageConfig()

                ic.read(self.imgdir)

                self.cfg_cache = ic

                # make sure we define architecture variant; upgrade config

                # file if possible.

                if "variant.arch" not in self.cfg_cache.variants:

                        self.cfg_cache.variants["variant.arch"] = \

                            platform.processor()

                        try:

                                self.save_config()

                        except api_errors.PermissionsException:

                                pass

                # make sure we define zone variant; upgrade config if possible

                if "variant.opensolaris.zone" not in self.cfg_cache.variants:

                        zone = self.cfg_cache.filters.get("opensolaris.zone",

                            "")

                        if zone == "nonglobal":

                                self.cfg_cache.variants[

                                    "variant.opensolaris.zone"] = "nonglobal"

                        else:

                                self.cfg_cache.variants[

                                    "variant.opensolaris.zone"] = "global"

                        try:

                                self.save_config()

                        except api_errors.PermissionsException:

                                pass

        def save_config(self):

                self.cfg_cache.write(self.imgdir)

        # XXX mkdirs and set_attrs() need to be combined into a create

        # operation.

        def mkdirs(self):

                for sd in self.image_subdirs:

                        if not os.path.isdir(os.path.join(self.imgdir, sd)):

                                os.makedirs(os.path.join(self.imgdir, sd))

        def __set_dirs(self, imgtype, root):

                self.type = imgtype

                self.root = root

                if self.type == IMG_USER:

                        self.img_prefix = img_user_prefix

                else:

                        self.img_prefix = img_root_prefix

                # Change directory to the root of the image so that we can

                # remove any directories beneath us.  If we're changing the

                # image, don't chdir, as we're likely changing to a new BE

                # and want to be able to unmount it later.

                if not self.imgdir and os.path.isdir(root):

                        os.chdir(root)

                self.imgdir = os.path.join(self.root, self.img_prefix)

                self.pkgdir = os.path.join(self.imgdir, "pkg")

                self.history.root_dir = self.imgdir

                if "PKG_CACHEDIR" in os.environ:

                        self.dl_cache_dir = os.path.normpath( \

                            os.environ["PKG_CACHEDIR"])

                        self.is_user_cache_dir = True

                else:

                        self.dl_cache_dir = os.path.normpath( \

                            os.path.join(self.imgdir, "download"))

                self.dl_cache_incoming = os.path.normpath(os.path.join(

                    self.dl_cache_dir, "incoming-%d" % os.getpid()))

        def set_attrs(self, imgtype, root, is_zone, prefix, pub_url,

            ssl_key=None, ssl_cert=None, variants=EmptyDict,

            refresh_allowed=True, progtrack=None):

                self.__set_dirs(imgtype=imgtype, root=root)

                # Create the publisher object before creating the image...

                repo = publisher.Repository()

                repo.add_origin(pub_url, ssl_cert=ssl_cert, ssl_key=ssl_key)

                newpub = publisher.Publisher(prefix,

                    meta_root=self._get_publisher_meta_root(prefix),

                    repositories=[repo])

                # Initialize and store the configuration object.

                self.cfg_cache = imageconfig.ImageConfig()

                # ...so that if creation of the Publisher object fails, an

                # empty, useless image won't be left behind.

                if not os.path.exists(os.path.join(self.imgdir,

                    imageconfig.CFG_FILE)):

                        self.history.log_operation_start("image-create")

                else:

                        self.history.log_operation_start("image-set-attributes")

                # Determine and add the default variants for the image.

                if is_zone:

                        self.cfg_cache.filters["opensolaris.zone"] = "nonglobal"

                        self.cfg_cache.variants[

                            "variant.opensolaris.zone"] = "nonglobal"

                else:

                        self.cfg_cache.variants[

                            "variant.opensolaris.zone"] = "global"

                self.cfg_cache.variants["variant.arch"] = \

                    variants.get("variant.arch", platform.processor())

                # After setting up the default variants, add any overrides or

                # additional variants specified.

                self.cfg_cache.variants.update(variants)

                # Now everything is ready for publisher configuration.

                self.cfg_cache.preferred_publisher = newpub.prefix

                self.add_publisher(newpub, refresh_allowed=refresh_allowed,

                    progtrack=progtrack)

                # Next, create the image directories if they haven't been

                # (add_publisher will normally do this if it needs to write

                # something to disk).  Waiting until after add_publisher is

                # called avoids leaving an empty, useless set of directories

                # for the image if the add fails.

                self.mkdirs()

                # Finally, since all operations were successful, save the image

                # configuration.

                self.save_config()

                self.history.log_operation_end()

        def is_liveroot(self):

                return self.root == "/"

        def is_zone(self):

                return self.cfg_cache.variants[

                    "variant.opensolaris.zone"] == "nonglobal"

        def get_arch(self):

                return self.cfg_cache.variants["variant.arch"]

        def get_root(self):

                return self.root

        def gen_publishers(self, inc_disabled=False):

                if not self.cfg_cache:

                        raise CfgCacheError, "empty ImageConfig"

                for p in self.cfg_cache.publishers:

                        pub = self.cfg_cache.publishers[p]

                        if inc_disabled or not pub.disabled:

                                yield self.cfg_cache.publishers[p]

        def get_url_by_publisher(self, prefix=None):

                """Return the URL prefix associated with the given prefix.

                For the undefined case, represented by None, return the

                preferred publisher."""

                # XXX This function is a possible location to insert one or more

                # policies regarding use of mirror responses, etc.

                if prefix is None:

                        prefix = self.cfg_cache.preferred_publisher

                try:

                        o = self.cfg_cache.publishers[prefix]["origin"]

                except KeyError:

                        # If the publisher that we're trying to get no longer

                        # exists, fall back to preferred publisher.

                        prefix = self.cfg_cache.preferred_publisher

                        o = self.cfg_cache.publishers[prefix]["origin"]

                return o.rstrip("/")

        def gen_depot_status(self):

                """Walk all publishers and return all depot status

                objects for both mirrors and primary publishers."""

                pubs = self.cfg_cache.publishers

                # return depot status objects in publisher order

                for pub in pubs.keys():

                        # first yield publisher origin

                        yield self.cfg_cache.publisher_status[pub]

                        # then return mirrors

                        for ds in self.cfg_cache.mirror_status[pub]:

                                yield ds

        def num_mirrors(self, pub):

                """Return the number of mirrors configured for the

                given publisher."""

                if pub == None:

                        pub = self.cfg_cache.preferred_publisher

                try:

                        num = len(self.cfg_cache.mirror_status[pub])

                except KeyError:

                        # pub isn't in the list of mirrors, return 0

                        num = 0

                return num

        def select_mirror(self, pub = None, chosen_set = None):

                """For the given publisher, look through the status of

                the mirrors.  Pick the best one.  This method returns

                a DepotStatus object or None.  The chosen_set argument

                contains a set object that lists the mirrors that were

                previously chosen.  This allows us to choose both

                by depot status statistics and ensures we don't

                always pick the same depot."""

                if pub == None:

                        pub = self.cfg_cache.preferred_publisher

                try:

                        slst = self.cfg_cache.mirror_status[pub]

                except KeyError:

                        # If the publisher that we're trying to get no longer

                        # exists, fall back to preferred publisher.

                        pub = self.cfg_cache.preferred_publisher

                        slst = self.cfg_cache.mirror_status[pub]

                if len(slst) == 0:

                        if pub in self.cfg_cache.publisher_status:

                                return self.cfg_cache.publisher_status[pub]