#!/usr/bin/python

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

#

# Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.

#

import pkg.client.api

import pkg.client.progress as progress

import pkg.lint.base as base

import pkg.lint.config

import pkg.fmri

from pkg.client.api_errors import ApiException

from pkg.version import DotSequence, Version

import ConfigParser

import logging

import os

import shutil

import sys

import urllib2

PKG_CLIENT_NAME = "pkglint"

pkg.client.global_settings.client_name = PKG_CLIENT_NAME

class LintEngineException(Exception):

        """An exception thrown when something fatal goes wrong with the engine,

        such that linting can no longer continue."""

        def __unicode__(self):

                # To workaround python issues 6108 and 2517, this provides a

                # a standard wrapper for this class' exceptions so that they

                # have a chance of being stringified correctly.

                return str(self)

class LintEngineSetupException(LintEngineException):

        """An exception thrown when the engine failed to complete its setup."""

        def __unicode__(self):

                # To workaround python issues 6108 and 2517, this provides a

                # a standard wrapper for this class' exceptions so that they

                # have a chance of being stringified correctly.

                return str(self)

class LintEngineCache():

        """This class provides two caches for the LintEngine.  A cache of the

        latest packages for one or more ImageInterface objects intended to be

        seeded at startup, and a generic manifest cache"""

        def __init__(self, version_pattern, release=None):

                self.latest_cache = {}

                self.misc_cache = {}

                self.logger = logging.getLogger("pkglint")

                self.seeded = False

                # release is a build number, eg. 150

                # version_pattern used by the engine is a regexp, intended

                # to be used when searching for images, combined with the

                # release - eg. "*,5.11-0."

                #

                self.version_pattern = version_pattern

                self.release = release

                if self.release:

                        combined = "%s%s" % \

                            (version_pattern.split(",")[1], release)

                        try:

                                self.branch = DotSequence(

                                    combined.split("-")[1])

                        except pkg.version.IllegalDotSequence:

                                raise LintEngineSetupException(

                                    _("Invalid release string: %s") %

                                    self.release)

        def seed_latest(self, api_inst, tracker, phase):

                """Builds a cache of latest manifests for this api_inst, using

                the provided progress tracker, phase and release.

                """

                search_type = pkg.client.api.ImageInterface.LIST_NEWEST

                pattern_list = ["*"]

                self.seeded = True

                # a dictionary of PkgFmri objects which we'll use to retrieve

                # manifests

                packages = {}

                # a dictionary of the latest packages for a given release

                self.latest_cache[api_inst] = {}

                # a dictionary of packages at other versions

                self.misc_cache[api_inst] = {}

                if not self.release:

                        for item in api_inst.get_pkg_list(

                            search_type, patterns=pattern_list, variants=True):

                                pub_name, name, version = item[0]

                                pub = api_inst.get_publisher(prefix=pub_name)

                                fmri ="pkg://%s/%s@%s" % (pub, name, version)

                                pfmri = pkg.fmri.PkgFmri(fmri)

                                # index with just the pkg name, allowing us to

                                # use this cache when searching for dependencies

                                packages["pkg:/%s" % name] = pfmri

                else:

                        # take a bit more time building up the latest version

                        # of all packages not greater than build_release

                        search_type = pkg.client.api.ImageInterface.LIST_ALL

                        for item in api_inst.get_pkg_list(

                            search_type, variants=True):

                                pub_name, name, version = item[0]

                                pub = api_inst.get_publisher(prefix=pub_name)

                                fmri ="pkg://%s/%s@%s" % (pub, name, version)

                                # obtain just the build branch, e.g. from

                                # 0.5.11,5.11-0.111:20090508T235707Z, return

                                # 0.111

                                branch = Version(version, None).branch

                                pfmri = pkg.fmri.PkgFmri(fmri)

                                key = "pkg:/%s" % name

                                if key not in packages and \

                                    branch <= self.branch:

                                        packages[key] = pfmri

                                # get_pkg_list returns results sorted by

                                # publisher, then sorted by version. We may find

                                # another publisher that has a more recent

                                # package available, so we need to respect

                                # timestamps in that case.

                                elif key in packages:

                                        prev = packages[key]

                                        if lint_fmri_successor(pfmri, prev,

                                            ignore_timestamps=False) and \

                                            branch <= self.branch:

                                                packages[key] = pfmri

                # now get the manifests

                tracker.manifest_fetch_start(len(packages))

                for item in packages:

                        self.latest_cache[api_inst][item] = \

                            api_inst.get_manifest(packages[item])

                        tracker.manifest_fetch_progress(completion=True)

                tracker.manifest_fetch_done()

        def gen_latest(self, api_inst, pattern):

                """ A generator function to return the latest version of the

                packages matching the supplied pattern from the publishers set

                for api_inst"""

                if not self.seeded:

                        raise LintEngineException("Cache has not been seeded")

                if api_inst in self.latest_cache:

                        for item in sorted(self.latest_cache[api_inst]):

                                mf = self.latest_cache[api_inst][item]

                                if pattern and pkg.fmri.glob_match(

                                    str(mf.fmri), pattern):

                                        yield mf

                                elif not pattern:

                                        yield mf

        def get_latest(self, api_inst, pkg_name):

                """ Return the package matching pkg_name from the publishers set

                for api_inst """

                if not self.seeded:

                        raise LintEngineException("Cache has not been seeded")

                if api_inst in self.latest_cache:

                        if pkg_name in self.latest_cache[api_inst]:

                                return self.latest_cache[api_inst][pkg_name]

                return None

        def count_latest(self, api_inst, pattern):

                """Returns the number of manifests in the given api_inst cache

                that match the provided pattern. If pattern is None,

                we return the length of the api_inst cache."""

                if not self.seeded:

                        raise LintEngineException("Cache has not been seeded")

                if not pattern:

                        return len(self.latest_cache[api_inst])

                count = 0

                for item in self.latest_cache[api_inst]:

                        mf = self.latest_cache[api_inst][item]

                        if pkg.fmri.glob_match(str(mf.fmri), pattern):

                                count = count + 1

                return count

        def add(self, api_inst, pkg_name, manifest):

                """Adds a given manifest to the cache for a given api_inst"""

                # we don't update latest_cache, since that should have been

                # pre-seeded on startup.

                self.misc_cache[api_inst][pkg_name] = manifest

        def get(self, api_inst, pkg_name):

                """Retrieves a given pkg_name entry from the cache.

                Can raise KeyError if the package isn't in the cache."""

                if not self.seeded:

                        raise LintEngineException("Cache has not been seeded")

                if pkg_name in self.latest_cache[api_inst]:

                        return self.latest_cache[api_inst][pkg_name]

                else:

                        return self.misc_cache[api_inst][pkg_name]

class LintEngine(object):

        """LintEngine is the main object used by pkglint to discover lint

        plugins, and execute lint checks on package manifests, retrieved

        from a repository or from local objects.

        Lint plugins are written as subclasses of pkg.lint.base.Checker

        and are discovered once the module containing them has been listed

        in either the shipped pkglintrc file, a user-supplied version, or

        the pkg.lint.config.PkglintConfig defaults.

        User-supplied manifests for linting are read directly as files

        provided on the command line.  For cross-checking against a

        reference repository, or linting the manifests in a repository, we

        create a reference or lint user-images in a provided cache location,

        used to obtain manifests from those repositories.

        Multiple repositories can be provided, however each must

        use a different publisher.prefix (as they are added as publishers

        to a single user image)

        Our reference image is stored in

        <cache>/ref_image/

        The image for linting is stored in

        <cache>/lint_image/

        We can also lint pkg.manifest.Manifest objects, passed as the

        lint_manifests list to the setup(..) method.

        The config object for the engine has the following keys:

        'log_level' The minimum level at which to emit lint messages. Lint

        messages lower than this level are discarded.

        By default, this is set to INFO. Log levels in order of least to most

        severe are: DEBUG, INFO, WARNING, ERROR, CRITICAL

        'do_pub_checks' Whether to perform checks which may  only  make  sense

        for published packages. Set to True by default.

        'pkglint.ext.*' Multiple keys, specifying modules to load which contain

        subclasses of pkg.lint.base.Checker.  The value of this property should

        be fully specified python module name, assumed to be in $PYTHONPATH

        'pkglint.exclude' A space-separated list of fully-specified Python

        modules, classes or function names which should be omitted from the set

        of checks performed. eg.

        'pkg.lint.opensolaris.ActionChecker' or

        'my.lint.module' or

        'my.lint.module.ManifestChecker.crosscheck_paths'

        'version.pattern' A version pattern, used when specifying a build number

        to lint against (-b). If not specified in the rcfile, this pattern is

        "*,5.11-0.", matching all components of the '5.11' build, with a branch

        prefix of '0.'

        'info_classification_path' A path the file used to check the values

        of info.classification attributes in manifests.

        'use_progress_tracker' Whether to use progress tracking.

        'ignore_different_publishers' Whether to ignore differences in publisher

        when comparing package FMRIs.

        The engine has support for a "/* LINTED */"-like functionality,

        see the comment for <LintEngine>.execute()"""

        def __init__(self, formatter, verbose=False, config_file=None,

            use_tracker=None):

                """Creates a lint engine a given pkg.lint.log.LogFormatter.

                'verbose' overrides any log_level settings in the config file

                to DEBUG

                'config_file' specifies a path to a pkglintrc configuration file

                'use_tracker' overrides any pkglintrc settings to either

                explicitly enable or disable the use of a tracker, set to None,

                we don't override the config file setting."""

                # the directory used to store our user-images.

                self.basedir = None

                # a pattern used to narrow searches in the lint image

                self.pattern = None

                self.release = None

                # a prefix for the pattern used to search for given releases

                self.version_pattern = "*,5.11-0."

                # lists of checker functions and excluded checker functions

                self.checkers = []

                self.excluded_checkers = []

                # A progress tracker, used during the lint run

                self.tracker = None

                # set up our python logger

                self.logger = logging.getLogger("pkglint")

                formatter.engine = self

                # the pkglint LogFormatters we are configured with

                self.logs = [formatter]

                # whether to run checks that may only be valid for published

                # manifests

                self.do_pub_checks = True

                # whether to ignore publisher differences when comparing vers

                self.ignore_pubs = True

                self.conf = self.load_config(config_file, verbose=verbose)

                # overrides config_file entry

                if use_tracker is not None:

                        self.use_tracker = use_tracker

                self.tracker_phase = 0

                self.in_setup = False

                formatter.tracker = self.get_tracker()

                self.ref_image = None

                self.lint_image = None

                self.lint_uris = []

                self.ref_uris = []

                # a reference to the pkg.client.api for our reference and lint

                # images

                self.ref_api_inst = None

                self.lint_api_inst = None

                # manifests presented to us for parsing on the command line

                self.lint_manifests = []

                self.mf_cache = None

        def _load_checker_module(self, name, config):

                """Dynamically loads a given checker module, returning new

                instances of the checker classes the module declares,

                assuming they haven't been excluded by the config object."""

                try:

                        self.logger.debug("Loading module %s" % name)

                        __import__(name, None, None, [], -1)

                        (checkers, excluded) = \

                            base.get_checkers(sys.modules[name], config)

                        return (checkers, excluded)

                except (KeyError, ImportError), err:

                        raise base.LintException(err)

        def _unique_checkers(self):

                """Ensure that the engine has unique names for all of the loaded

                checks."""

                unique_names = set()

                for checker in self.checkers:

                        if checker.name in unique_names:

                                raise LintEngineSetupException(

                                    _("loading extensions: "

                                    "duplicate checker name %(name)s: "

                                    "%(class)s") %

                                    {"name": checker.name,

                                    "class": checker})

                        unique_names.add(checker.name)

                        unique_methods = set()

                        for method, pkglint_id in checker.included_checks + \

                            checker.excluded_checks:

                                if pkglint_id in unique_methods:

                                        raise LintEngineSetupException(_(

                                            "loading extension "

                                            "%(checker)s: duplicate pkglint_id "

                                            "%(pkglint_id)s in %(method)s") %

                                            {"checker": checker.name,

                                            "pkglint_id": pkglint_id,

                                            "method": method})

                                unique_methods.add(pkglint_id)

        def load_config(self, config, verbose=False):

                """Loads configuration from supplied config file, allowing

                a verbosity override."""

                try:

                        conf = pkg.lint.config.PkglintConfig(

                            config_file=config).config

                except (pkg.lint.config.PkglintConfigException), err:

                        raise LintEngineSetupException(err)

                excl = []

                try:

                        excl = conf.get("pkglint", "pkglint.exclude").split()

                except ConfigParser.NoOptionError:

                        pass

                try:

                        self.version_pattern = conf.get("pkglint",

                            "version.pattern")

                except ConfigParser.NoOptionError:

                        pass

                for key, value in conf.items("pkglint"):

                        if "pkglint.ext" in key:

                                if value in excl:

                                        # want to exclude everything from here

                                        (checkers, exclude) = \

                                            self._load_checker_module(value,

                                            conf)

                                        self.excluded_checkers.extend(checkers)

                                        self.excluded_checkers.extend(exclude)

                                        continue

                                try:

                                        (checkers, exclude) = \

                                            self._load_checker_module(value,

                                            conf)

                                        self.checkers.extend(checkers)

                                        self.excluded_checkers.extend(exclude)

                                except base.LintException, err:

                                        raise LintEngineSetupException(

                                            _("Error parsing config value for "

                                            "%(key)s: %(err)s") % locals())

                self._unique_checkers()

                if verbose:

                        for lint_log in self.logs:

                                lint_log.level = "DEBUG"

                else:

                        for lint_log in self.logs:

                                lint_log.level = conf.get("pkglint",

                                    "log_level")

                try:

                        self.do_pub_checks = conf.getboolean("pkglint",

                            "do_pub_checks")

                except ConfigParser.NoOptionError:

                        pass

                try:

                        self.use_tracker = conf.get("pkglint",

                            "use_progress_tracker").lower() == "true"

                except ConfigParser.NoOptionError:

                        pass

                try:

                        self.ignore_pubs = conf.get("pkglint",

                            "ignore_different_publishers").lower() == "true"

                except ConfigParser.NoOptionError:

                        pass

                return conf

        def setup(self, lint_manifests=[], ref_uris=[], lint_uris=[],

            cache=None, pattern=None, release=None):

                """Starts a pkglint session, creates our image, pulls manifests,

                etc. from servers if necessary.

                'cache' An area where we create images to access repos for both

                reference and linting purposes

                'lint_manifests' An array of paths to manifests for linting

                'ref_uris' A list of repositories which will be added to the

                image used as a reference for linting

                'lint_uris' A list of repositories which will be added to th

                image we want to lint

                'pattern' A regexp to match the packages we want to lint, if

                empty, we match everything.  Note that this is only applied when

                searching for packages to lint: we lint against all packages for

                a given release in the reference repository (if configured)

                'release' A release value that narrows the set of packages we

                lint with.  This affects both the packages presented for linting

                as well as the packages in the repository we are linting

                against. If release if set to None, we lint with and against the

                latest available packages in the repositories."""

                self.ref_uris = ref_uris

                self.lint_uris = lint_uris

                self.lint_manifests = lint_manifests

                self.lint_manifests.sort(key=_manifest_sort_key)

                self.pattern = pattern