#!/usr/bin/python2.4

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.

# Use is subject to license terms.

import cPickle

import errno

import os

import socket

import urllib

import urllib2

import httplib

import shutil

import tempfile

import time

import datetime

import calendar

import OpenSSL.crypto as osc

from pkg.misc import msg, emsg

# import uuid           # XXX interesting 2.5 module

import pkg.catalog as catalog

import pkg.updatelog as updatelog

import pkg.fmri

import pkg.manifest as manifest

import pkg.misc as misc

import pkg.version

import pkg.client.history as history

import pkg.client.imageconfig as imageconfig

import pkg.client.imageplan as imageplan

import pkg.client.retrieve as retrieve

import pkg.portable as portable

import pkg.client.query_engine as query_e

import pkg.client.indexer as indexer

import pkg.search_errors as search_errors

from pkg.misc import versioned_urlopen

from pkg.misc import TransferTimedOutException

from pkg.misc import CLIENT_DEFAULT_MEM_USE_KB

from pkg.client.imagetypes import *

img_user_prefix = ".org.opensolaris,pkg"

img_root_prefix = "var/pkg"

PKG_STATE_INSTALLED = "installed"

PKG_STATE_KNOWN = "known"

# Minimum number of days to issue warning before a certificate expires

MIN_WARN_DAYS = datetime.timedelta(days=30)

class CatalogRefreshException(Exception):

        def __init__(self, failed, total, succeeded):

                Exception.__init__(self)

                self.failed = failed

                self.total = total

                self.succeeded = succeeded

class InventoryException(Exception):

        def __init__(self, notfound=None, illegal=None):

                Exception.__init__(self)

                if notfound is None:

                        self.notfound = []

                else:

                        self.notfound = notfound

                if illegal is None:

                        self.illegal = []

                else:

                        self.illegal = illegal

                assert(self.notfound or self.illegal)

        def __str__(self):

                outstr = ""

                for x in self.illegal:

                        # Illegal FMRIs have their own __str__ method

                        outstr += "%s\n" % x

                for x in self.notfound:

                        outstr += "Not found in inventory: %s\n" % str(x)

                return outstr

class Image(object):

        """An Image object is a directory tree containing the laid-down contents

        of a self-consistent graph of Packages.

        An Image has a root path.

        An Image of type IMG_ENTIRE does not have a parent Image.  Other Image

        types must have a parent Image.  The external state of the parent Image

        must be accessible from the Image's context, or duplicated within the

        Image (IMG_PARTIAL for zones, for instance).

        The parent of a user Image can be a partial Image.  The parent of a

        partial Image must be an entire Image.

        An Image of type IMG_USER stores its external state at self.root +

        ".org.opensolaris,pkg".

        An Image of type IMG_ENTIRE or IMG_PARTIAL stores its external state at

        self.root + "/var/pkg".

        An Image needs to be able to have a different repository set than the

        system's root Image.

        Directory layout

          $IROOT/catalog

               Directory containing catalogs for URIs of interest.  Filename is

               the escaped URI of the catalog.

          $IROOT/file

               Directory containing file hashes of installed packages.

          $IROOT/pkg

               Directory containing manifests and states of installed packages.

          $IROOT/index

               Directory containing reverse-index databases.

          $IROOT/cfg_cache

               File containing image's cached configuration.

          $IROOT/opaque

               File containing image's opaque state.

          $IROOT/state/installed

               Directory containing files whose names identify the installed

               packages.

        All of these directories and files other than state are considered

        essential for any image to be complete. To add a new essential file or

        subdirectory, the following steps should be done.

        If it's a directory, add it to the image_subdirs list below and it will

        be created automatically. The programmer must fill the directory as

        needed. If a file is added, the programmer is responsible for creating

        that file during image creation at an appropriate place and time.

        If a directory is required to be present in order for an image to be

        identifiable as such, it should go into required_subdirs instead.

        However, upgrade issues abound; this list should probably not change.

        Once those steps have been carried out, the change should be added

        to the test suite for image corruption (t_pkg_install_corrupt_image.py).

        This will likely also involve a change to

        SingleDepotTestCaseCorruptImage in testutils.py. Each of these files

        outline what must be updated.

        XXX Root path probably can't be absolute, so that we can combine or

        reuse Image contents.

        XXX Image file format?  Image file manipulation API?"""

        required_subdirs = [ "catalog", "file", "pkg" ]

        image_subdirs = required_subdirs + [ "index", "state/installed" ]

        def __init__(self):

                self.cfg_cache = None

                self.type = None

                self.root = None

                self.history = history.History()

                self.imgdir = None

                self.img_prefix = None

                self.index_dir = None

                self.repo_uris = []

                self.filter_tags = {}

                self.catalogs = {}

                self._catalog = {}

                self.pkg_states = None

                self.attrs = {}

                self.attrs["Policy-Require-Optional"] = False

                self.attrs["Policy-Pursue-Latest"] = True

                self.imageplan = None # valid after evaluation succceds

                # contains a dictionary w/ key = pkgname, value is miminum

                # frmi.XXX  Needs rewrite using graph follower

                self.optional_dependencies = {}

                # a place to keep info about saved_files; needed by file action

                self.saved_files = {}

        def find_root(self, d):

                def check_subdirs(sub_d, prefix):

                        for n in self.required_subdirs:

                                if not os.path.isdir(

                                    os.path.join(sub_d, prefix, n)):

                                        return False

                        return True

                # Ascend from the given directory d to find first

                # encountered image.

                startd = d

                # eliminate problem if relative path such as "." is passed in

                d = os.path.realpath(d)

                while True:

                        if os.path.isdir(os.path.join(d, img_user_prefix)) and \

                            os.path.isfile(os.path.join(d, img_user_prefix,

                                "cfg_cache")) and \

                            check_subdirs(d, img_user_prefix):

                                # XXX Look at image file to determine filter

                                # tags and repo URIs.

                                self.type = IMG_USER

                                self.root = d

                                self.img_prefix = img_user_prefix

                                self.imgdir = os.path.join(d, self.img_prefix)

                                self.history.root_dir = self.imgdir

                                self.attrs["Build-Release"] = "5.11"

                                return

                        elif os.path.isdir(os.path.join(d, img_root_prefix)) \

                              and os.path.isfile(os.path.join(d,

                              img_root_prefix,"cfg_cache")) and \

                              check_subdirs(d, img_root_prefix):

                                # XXX Look at image file to determine filter

                                # tags and repo URIs.

                                # XXX Look at image file to determine if this

                                # image is a partial image.

                                self.type = IMG_ENTIRE

                                self.root = d

                                self.img_prefix = img_root_prefix

                                self.imgdir = os.path.join(d, self.img_prefix)

                                self.history.root_dir = self.imgdir

                                self.attrs["Build-Release"] = "5.11"

                                return

                        # XXX follow symlinks or not?

                        oldpath = d

                        d = os.path.normpath(os.path.join(d, os.path.pardir))

                        # Make sure we are making progress and aren't in an

                        # infinite loop.

                        #

                        # (XXX - Need to deal with symlinks here too)

                        if d == oldpath:

                                raise ValueError, "directory %s not contained within an image" % startd

        def load_config(self):

                """Load this image's cached configuration from the default

                location."""

                # XXX Incomplete with respect to doc/image.txt description of

                # configuration.

                if self.root == None:

                        raise RuntimeError, "self.root must be set"

                ic = imageconfig.ImageConfig()

                if os.path.isfile("%s/cfg_cache" % self.imgdir):

                        ic.read("%s/cfg_cache" % self.imgdir)

                self.cfg_cache = ic

        # XXX mkdirs and set_attrs() need to be combined into a create

        # operation.

        def mkdirs(self):

                for sd in self.image_subdirs:

                        if not os.path.isdir(os.path.join(self.imgdir, sd)):

                                os.makedirs(os.path.join(self.imgdir, sd))

        def set_attrs(self, type, root, is_zone, auth_name, auth_url,

            ssl_key = None, ssl_cert = None):

                self.type = type

                self.root = root

                if self.type == IMG_USER:

                        self.img_prefix = img_user_prefix

                else:

                        self.img_prefix = img_root_prefix

                self.imgdir = os.path.join(self.root, self.img_prefix)

                self.history.root_dir = self.imgdir

                if not os.path.exists(os.path.join(self.imgdir, "cfg_cache")):

                        self.history.operation_name = "image-create"

                else:

                        self.history.operation_name = "image-set-attributes"

                self.mkdirs()

                self.cfg_cache = imageconfig.ImageConfig()

                if is_zone:

                        self.cfg_cache.filters["opensolaris.zone"] = "nonglobal"

                self.cfg_cache.authorities[auth_name] = {}

                self.cfg_cache.authorities[auth_name]["prefix"] = auth_name

                self.cfg_cache.authorities[auth_name]["origin"] = \

                    misc.url_affix_trailing_slash(auth_url)

                self.cfg_cache.authorities[auth_name]["mirrors"] = []

                self.cfg_cache.authorities[auth_name]["ssl_key"] = ssl_key

                self.cfg_cache.authorities[auth_name]["ssl_cert"] = ssl_cert

                self.cfg_cache.preferred_authority = auth_name

                self.cfg_cache.write("%s/cfg_cache" % self.imgdir)

                self.history.operation_result = history.RESULT_SUCCEEDED

        def is_liveroot(self):

                return self.root == "/"

        def is_zone(self):

                zone = self.cfg_cache.filters.get("opensolaris.zone", "")

                return zone == "nonglobal"

        def get_root(self):

                return self.root

        def gen_authorities(self):

                if not self.cfg_cache:

                        raise RuntimeError, "empty ImageConfig"

                if not self.cfg_cache.authorities:

                        raise RuntimeError, "no defined authorities"

                for a in self.cfg_cache.authorities:

                        yield self.cfg_cache.authorities[a]

        def get_url_by_authority(self, authority = None):

                """Return the URL prefix associated with the given authority.

                For the undefined case, represented by None, return the

                preferred authority."""

                # XXX This function is a possible location to insert one or more

                # policies regarding use of mirror responses, etc.

                if authority == None:

                        authority = self.cfg_cache.preferred_authority

                try:

                        o = self.cfg_cache.authorities[authority]["origin"]

                except KeyError:

                        # If the authority that we're trying to get no longer

                        # exists, fall back to preferred authority.

                        authority = self.cfg_cache.preferred_authority

                        o = self.cfg_cache.authorities[authority]["origin"]

                return o.rstrip("/")

        def select_mirror(self, auth = None, chosen_set = None):

                """For the given authority, look through the status of

                the mirrors.  Pick the best one.  This method returns

                a DepotStatus object or None.  The chosen_set argument

                contains a set object that lists the mirrors that were

                previously chosen.  This allows us to choose both

                by depot status statistics and ensures we don't

                always pick the same depot."""

                if auth == None:

                        auth = self.cfg_cache.preferred_authority

                try:

                        slst = self.cfg_cache.mirror_status[auth]

                except KeyError:

                        # If the authority that we're trying to get no longer

                        # exists, fall back to preferred authority.

                        auth = self.cfg_cache.preferred_authority

                        slst = self.cfg_cache.mirror_status[auth]

                if len(slst) == 0:

                        if auth in self.cfg_cache.authority_status:

                                return self.cfg_cache.authority_status[auth]

                        else:

                                return None

                # Choose mirror with fewest errors.

                # If mirrors have same number of errors, choose mirror

                # with smaller number of good transactions.  Assume it's

                # being underused, not high-latency.

                #

                # XXX Will need to revisit the above assumption.

                def cmp_depotstatus(a, b):

                        res = cmp(a.errors, b.errors)

                        if res == 0:

                                return cmp(a.good_tx, b.good_tx)

                        return res                        

                slst.sort(cmp = cmp_depotstatus)

                if chosen_set and len(chosen_set) == len(slst):

                        chosen_set.clear()

                        chosen_set = None

                if chosen_set and slst[0] in chosen_set:

                        for ds in slst:

                                if ds not in chosen_set:

                                        return ds

                return slst[0]

        def get_ssl_credentials(self, authority = None, origin = None):

                """Return a tuple containing (ssl_key, ssl_cert) for the

                specified authority prefix.  If the authority isn't specified,

                attempt to determine the authority by the given origin.  If

                neither is specified, use the preferred authority.

                """

                if authority is None:

                        if origin is None:

                                authority = self.cfg_cache.preferred_authority

                        else:

                                auths = self.cfg_cache.authorities

                                for pfx, auth in auths.iteritems():

                                        if auth["origin"] == origin:

                                                authority = pfx

                                                break

                                else:

                                        return None

                try:

                        authent = self.cfg_cache.authorities[authority]

                except KeyError:

                        authority = self.cfg_cache.preferred_authority

                        authent = self.cfg_cache.authorities[authority]

                return (authent["ssl_key"], authent["ssl_cert"])

        @staticmethod

        def build_cert(path):

                """Take the file given in path, open it, and use it to create

                an X509 certificate object."""

                cf = file(path, "rb")

                certdata = cf.read()

                cf.close()

                cert = osc.load_certificate(osc.FILETYPE_PEM, certdata)

                return cert

        def check_cert_validity(self):

                """Look through the authorities defined for the image.  Print

                a message and exit with an error if one of the certificates

                has expired.  If certificates are getting close to expiration,

                print a warning instead."""

                for a in self.gen_authorities():

                        pfx, url, ssl_key, ssl_cert, dt, mir = \

                            self.split_authority(a)

                        if not ssl_cert:

                                continue

                        try:

                                cert = self.build_cert(ssl_cert)

                        except IOError, e:

                                if e.errno == errno.ENOENT:

                                        emsg(_("Certificate for authority %s" \

                                            " not found") % pfx)

                                        emsg(_("File was supposed to exist at" \

                                           "  path %s") % ssl_cert)

                                        return False 

                                else:

                                        raise

                        # OpenSSL.crypto.Error

                        except osc.Error, e:

                                emsg(_("Certificate for authority %s at" \

                                    " %s has an invalid format.") % \

                                    (pfx, ssl_cert))

                                return False

                        if cert.has_expired():

                                emsg(_("Certificate for authority %s" \

                                    " has expired") % pfx)

                                emsg(_("Please install a valid certificate"))

                                return False

                        now = datetime.datetime.utcnow()

                        nb = cert.get_notBefore()

                        t = time.strptime(nb, "%Y%m%d%H%M%SZ")

                        nbdt = datetime.datetime.utcfromtimestamp(