#!/usr/bin/python

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

# Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.

"""

Misc utility functions used by the packaging system.

"""

from __future__ import division

from __future__ import print_function

import OpenSSL.crypto as osc

import cStringIO

import calendar

import collections

import datetime

import errno

import fnmatch

import getopt

import itertools

import locale

import os

import platform

import re

import resource

import shutil

import signal

import simplejson as json

import socket

import struct

import sys

import threading

import time

import traceback

import urllib

import urlparse

import zlib

from collections import defaultdict

from stat import S_IFMT, S_IMODE, S_IRGRP, S_IROTH, S_IRUSR, S_IRWXU, \

    S_ISBLK, S_ISCHR, S_ISDIR, S_ISFIFO, S_ISLNK, S_ISREG, S_ISSOCK, \

    S_IWUSR, S_IXGRP, S_IXOTH

import pkg.client.api_errors as api_errors

import pkg.portable as portable

import pkg.digest as digest

from pkg import VERSION

from pkg.client import global_settings

from pkg.client.debugvalues import DebugValues

from pkg.client.imagetypes import img_type_names, IMG_NONE

from pkg.pkggzip import PkgGzipFile

# Default path where the temporary directories will be created.

DEFAULT_TEMP_PATH = "/var/tmp"

# Minimum number of days to issue warning before a certificate expires

MIN_WARN_DAYS = datetime.timedelta(days=30)

# Constant string used across many modules as a property name.

SIGNATURE_POLICY = "signature-policy"

# Bug URI Constants (deprecated)

# Line too long; pylint: disable=C0301

BUG_URI_CLI = "https://defect.opensolaris.org/bz/enter_bug.cgi?product=pkg&component=cli"

BUG_URI_GUI = "https://defect.opensolaris.org/bz/enter_bug.cgi?product=pkg&component=gui"

# pylint: enable=C0301

# Traceback message.

def get_traceback_message():

        """This function returns the standard traceback message.  A function

        is necessary since the _() call must be done at runtime after locale

        setup."""

        return _("""\n

This is an internal error in pkg(5) version {version}.  Please log a

Service Request about this issue including the information above and this

message.""").format(version=VERSION)

def get_release_notes_url():

        """Return a release note URL pointing to the correct release notes

           for this version"""

        # TBD: replace with a call to api.info() that can return a "release"

        # attribute of form YYYYMM against the SUNWsolnm package

        return "http://www.oracle.com/pls/topic/lookup?ctx=E23824&id=SERNS"

def time_to_timestamp(t):

        """convert seconds since epoch to %Y%m%dT%H%M%SZ format"""

        # XXX optimize?; pylint: disable=W0511

        return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(t))

def timestamp_to_time(ts):

        """convert %Y%m%dT%H%M%SZ format to seconds since epoch"""

        # XXX optimize?; pylint: disable=W0511

        return calendar.timegm(time.strptime(ts, "%Y%m%dT%H%M%SZ"))

def timestamp_to_datetime(ts):

        """convert %Y%m%dT%H%M%SZ format to a datetime object"""

        return datetime.datetime.strptime(ts,"%Y%m%dT%H%M%SZ")

def copyfile(src_path, dst_path):

        """copy a file, preserving attributes, ownership, etc. where possible"""

        fs = os.lstat(src_path)

        shutil.copy2(src_path, dst_path)

        try:

                portable.chown(dst_path, fs.st_uid, fs.st_gid)

        except OSError as e:

                if e.errno != errno.EPERM:

                        raise

def copytree(src, dst):

        """Rewrite of shutil.copytree() that can handle special files such as

        FIFOs, sockets, and device nodes.  It re-creates all symlinks rather

        than copying the data behind them, and supports neither the 'symlinks'

        nor the 'ignore' keyword arguments of the shutil version.

        """

        problem = None

        os.makedirs(dst, PKG_DIR_MODE)

        src_stat = os.stat(src)

        for name in sorted(os.listdir(src)):

                s_path = os.path.join(src, name)

                d_path = os.path.join(dst, name)

                s = os.lstat(s_path)

                if S_ISDIR(s.st_mode):

                        copytree(s_path, d_path)

                        os.chmod(d_path, S_IMODE(s.st_mode))

                        os.chown(d_path, s.st_uid, s.st_gid)

                        os.utime(d_path, (s.st_atime, s.st_mtime))

                elif S_ISREG(s.st_mode):

                        shutil.copyfile(s_path, d_path)

                        os.chmod(d_path, S_IMODE(s.st_mode))

                        os.chown(d_path, s.st_uid, s.st_gid)

                        os.utime(d_path, (s.st_atime, s.st_mtime))

                elif S_ISLNK(s.st_mode):

                        os.symlink(os.readlink(s_path), d_path)

                elif S_ISFIFO(s.st_mode):

                        os.mkfifo(d_path, S_IMODE(s.st_mode))

                        os.chown(d_path, s.st_uid, s.st_gid)

                        os.utime(d_path, (s.st_atime, s.st_mtime))

                elif S_ISSOCK(s.st_mode):

                        sock = socket.socket(socket.AF_UNIX)

                        # The s11 fcs version of python doesn't have os.mknod()

                        # but sock.bind has a path length limitation that we can

                        # hit when archiving the test suite.

                        # E1101 Module '{0}' has no '{1}' member

                        # pylint: disable=E1101

                        if hasattr(os, "mknod"):

                                os.mknod(d_path, s.st_mode, s.st_dev)

                        else:

                                try:

                                        sock.bind(d_path)

                                        sock.close()

                                except sock.error as _e:

                                        # Store original exception so that the

                                        # real cause of failure can be raised if

                                        # this fails.

                                        problem = sys.exc_info()

                                        continue

                        os.chown(d_path, s.st_uid, s.st_gid)

                        os.utime(d_path, (s.st_atime, s.st_mtime))

                elif S_ISCHR(s.st_mode) or S_ISBLK(s.st_mode):

                        # the s11 fcs version of python doesn't have os.mknod()

                        # E1101 Module '{0}' has no '{1}' member

                        # pylint: disable=E1101

                        if hasattr(os, "mknod"):

                                os.mknod(d_path, s.st_mode, s.st_dev)

                                os.chown(d_path, s.st_uid, s.st_gid)

                                os.utime(d_path, (s.st_atime, s.st_mtime))

                elif S_IFMT(s.st_mode) == 0xd000: # doors

                        pass

                elif S_IFMT(s.st_mode) == 0xe000: # event ports

                        pass

                else:

                        print("unknown file type:", oct(S_IFMT(s.st_mode)))

        os.chmod(dst, S_IMODE(src_stat.st_mode))

        os.chown(dst, src_stat.st_uid, src_stat.st_gid)

        os.utime(dst, (src_stat.st_atime, src_stat.st_mtime))

        if problem:

                raise problem[0], problem[1], problem[2]

def move(src, dst):

        """Rewrite of shutil.move() that uses our copy of copytree()."""

        # If dst is a directory, then we try to move src into it.

        if os.path.isdir(dst):

                dst = os.path.join(dst,

                    os.path.basename(src).rstrip(os.path.sep))

        try:

                os.rename(src, dst)

        except EnvironmentError as e:

                s = os.lstat(src)

                if e.errno == errno.EXDEV:

                        if S_ISDIR(s.st_mode):

                                copytree(src, dst)

                                shutil.rmtree(src)

                        else:

                                shutil.copyfile(src, dst)

                                os.chmod(dst, S_IMODE(s.st_mode))

                                os.chown(dst, s.st_uid, s.st_gid)

                                os.utime(dst, (s.st_atime, s.st_mtime))

                                os.unlink(src)

                elif e.errno == errno.EINVAL and S_ISDIR(s.st_mode):

                        raise shutil.Error("Cannot move a directory '{0}' "

                            "into itself '{1}'.".format(src, dst))

                elif e.errno == errno.ENOTDIR and S_ISDIR(s.st_mode):

                        raise shutil.Error("Destination path '{0}' already "

                            "exists".format(dst))

                else:

                        raise

def expanddirs(dirs):

        """given a set of directories, return expanded set that includes

        all components"""

        out = set()

        for d in dirs:

                p = d

                while p != "":

                        out.add(p)

                        p = os.path.dirname(p)

        return out

def url_affix_trailing_slash(u):

        """if 'u' donesn't have a trailing '/', append one."""

        if u[-1] != '/':

                u = u + '/'

        return u

_client_version = "pkg/{0} ({1} {2}; {3} {4}; {{0}}; {{1}})".format(

    VERSION, portable.util.get_canonical_os_name(), platform.machine(),

    portable.util.get_os_release(), platform.version())

def user_agent_str(img, client_name):

        """Return a string that can use to identify the client."""

        if not img or img.type is None:

                imgtype = IMG_NONE

        else:

                imgtype = img.type

        useragent = _client_version.format(img_type_names[imgtype], client_name)

        return useragent

# Valid hostname can be : HOSTNAME or IPv4 addr or IPV6 addr

_hostname_re = re.compile(r"""^(?:[a-zA-Z0-9\-]+[a-zA-Z0-9\-\.]*

                   |(?:\d{1,3}\.){3}\d{3}

                   |\[([a-fA-F0-9\.]*:){,7}[a-fA-F0-9\.]+\])$""", re.X)

_invalid_host_chars = re.compile(".*[^a-zA-Z0-9\-\.:\[\]]+")

_valid_proto = ["file", "http", "https"]

def valid_pub_prefix(prefix):

        """Verify that the publisher prefix only contains valid characters."""

        if not prefix:

                return False

        # This is a workaround for the the hostname_re being slow when

        # it comes to finding invalid characters in the prefix string.

        if _invalid_host_chars.match(prefix):

                # prefix bad chars

                return False

        if _hostname_re.match(prefix):

                return True

        return False

def valid_pub_url(url, proxy=False):

        """Verify that the publisher URL contains only valid characters.

        If 'proxy' is set to True, some checks are relaxed."""

        if not url:

                return False

        # First split the URL and check if the scheme is one we support

        o = urlparse.urlsplit(url)

        if not o[0] in _valid_proto:

                return False

        if o[0] == "file":

                path = urlparse.urlparse(url, "file", allow_fragments=0)[2]

                path = urllib.url2pathname(path)

                if not os.path.abspath(path):

                        return False

                # No further validation to be done.

                return True

        # Next verify that the network location is valid

        host = urllib.splitport(o[1])[0]

        if proxy:

                # We may have authentication details in the proxy URI, which

                # we must ignore when checking for hostname validity.

                host_parts = host.split("@")

                if len(host_parts) == 2:

                        host = host[1]

        if not host or _invalid_host_chars.match(host):

                return False

        if _hostname_re.match(host):

                return True

        return False

def gunzip_from_stream(gz, outfile, hash_func=None, hash_funcs=None,

    ignore_hash=False):

        """Decompress a gzipped input stream into an output stream.

        The argument 'gz' is an input stream of a gzipped file and 'outfile'

        is is an output stream.  gunzip_from_stream() decompresses data from

        'gz' and writes it to 'outfile', and returns the hexadecimal SHA sum

        of that data using the hash_func supplied.

        'hash_funcs', if supplied, is a list of hash functions which we should

        use to compute the hash. If 'hash_funcs' is supplied, a list of

        hexadecimal digests computed using those functions is returned. The

        returned list is in the same order as 'hash_funcs'.

        If 'ignore_hash' is True, we do not compute a hash when decompressing

        the content and do not return any value.

        """

        FHCRC = 2

        FEXTRA = 4

        FNAME = 8

        FCOMMENT = 16

        if not (hash_func or hash_funcs) and not ignore_hash:

                raise ValueError("no hash functions for gunzip_from_stream")

        # Read the header

        magic = gz.read(2)

        if magic != "\037\213":

                raise zlib.error("Not a gzipped file")

        method = ord(gz.read(1))

        if method != 8:

                raise zlib.error("Unknown compression method")

        flag = ord(gz.read(1))

        gz.read(6) # Discard modtime, extraflag, os

        # Discard an extra field

        if flag & FEXTRA:

                xlen = ord(gz.read(1))

                xlen = xlen + 256 * ord(gz.read(1))

                gz.read(xlen)

        # Discard a null-terminated filename

        if flag & FNAME:

                while True:

                        s = gz.read(1)

                        if not s or s == "\000":

                                break

        # Discard a null-terminated comment

        if flag & FCOMMENT:

                while True:

                        s = gz.read(1)

                        if not s or s == "\000":

                                break

        # Discard a 16-bit CRC

        if flag & FHCRC:

                gz.read(2)

        if ignore_hash:

                pass

        elif hash_funcs:

                shasums = []

                for f in hash_funcs:

                        shasums.append(digest.HASH_ALGS[f]())

        else:

                shasum = hash_func()

        dcobj = zlib.decompressobj(-zlib.MAX_WBITS)

        while True:

                buf = gz.read(64 * 1024)

                if buf == "":

                        ubuf = dcobj.flush()

                        if ignore_hash:

                                pass

                        elif hash_funcs:

                                for sha in shasums:

                                        sha.update(ubuf)

                        else:

                                shasum.update(ubuf) # pylint: disable=E1101

                        outfile.write(ubuf)

                        break

                ubuf = dcobj.decompress(buf)

                if ignore_hash:

                        pass

                elif hash_funcs:

                        for sha in shasums:

                                sha.update(ubuf)

                else:

                        shasum.update(ubuf) # pylint: disable=E1101

                outfile.write(ubuf)

        if ignore_hash:

                return

        elif hash_funcs:

                hexdigests = []

                for sha in shasums:

                        hexdigests.append(sha.hexdigest())

                return hexdigests

        return shasum.hexdigest()

class PipeError(Exception):

        """ Pipe exception. """

        def __init__(self, args=None):

                Exception.__init__(self)

                self._args = args

def msg(*text):

        """ Emit a message. """

        try:

                print(' '.join([str(l) for l in text]))

        except IOError as e:

                if e.errno == errno.EPIPE:

                        raise PipeError(e)

                raise

def emsg(*text):

        """ Emit a message to sys.stderr. """

        try:

                print(' '.join([str(l) for l in text]), file=sys.stderr)

        except IOError as e:

                if e.errno == errno.EPIPE:

                        raise PipeError(e)

                raise

def setlocale(category, loc=None, printer=None):

        """Wraps locale.setlocale(), falling back to the C locale if the desired

        locale is broken or unavailable.  The 'printer' parameter should be a

        function which takes a string and displays it.  If 'None' (the default),

        setlocale() will print the message to stderr."""

        if printer is None:

                printer = emsg

        try:

                locale.setlocale(category, loc)

                # Because of Python bug 813449, getdefaultlocale may fail

                # with a ValueError even if setlocale succeeds. So we call

                # it here to prevent having this error raised if it is

                # called later by other non-pkg(5) code.

                locale.getdefaultlocale()

        except (locale.Error, ValueError):

                try:

                        dl = " '{0}.{1}'".format(*locale.getdefaultlocale())

                except ValueError: