#!/usr/bin/python2.6

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

# Use is subject to license terms.

import fnmatch

import getopt

import gettext

import os

import pkg.fmri

import pkg.manifest            as manifest

import pkg.pkgsubprocess       as subprocess

import pkg.publish.transaction as trans

import pkg.server.catalog      as catalog

import pkg.variant             as variant

import pkg.version             as version

import platform

import re

import shlex

import shutil

import sys

import tempfile

import time

import urllib

from datetime import datetime

from pkg      import actions, elf

from pkg.bundle.SolarisPackageDirBundle import SolarisPackageDirBundle

from pkg.misc import versioned_urlopen, emsg

from tempfile import mkstemp

gettext.install("import", "/usr/lib/locale")

# rewrite of solaris.py to convert to actions as soon as possible;

# all chatters, etc. are performed before adding package contents

# to global name table. Actions are annotated to include svr4 source

# pkg & path

basename_dict = {}   # basenames to action lists

branch_dict = {}     # 

cons_dict = {}       # consolidation incorporation dictionaries

file_repo = False    #

curpkg = None        # which IPS package we're currently importing

def_branch = ""      # default branch

def_repo = "http://localhost:10000"

def_vers = "0.5.11"  # default package version

# default search path

def_wos_path =  ["/net/netinstall.eng/export/nv/x/latest/Solaris_11/Product"]

elided_files = {}    # always delete these files; not checked on specific import

extra_entire_contents = [] # additional entries to be added to "entire"

fmridict = {}        # all ips FMRIS known, indexed by name

global_includes = [] # include these for every package

include_path = []    # where to find inport files - searched in order

just_these_pkgs = [] # publish only thesee pkgs

macro_definitions = {} # list of macro substitutions

nopublish = False    # fake publication?

path_dict = {}       # map of paths to action lists

pkgdict = {}         # pkgdict contains Package objects we're importing by name

pkgpaths = {}        # where we found svr4 pkgs by name

pkgpath_dict = {}    # mapping of paths to ips pkg

print_pkg_names = False # jusr print package names seen

publish_all = False  # always publish all obsoleted and renamed packages?

reference_uris = []  # list of url@pkg specs to compute dependencies against

show_debug = False   # print voluminous debug output

summary_detritus = [", (usr)", ", (root)", " (usr)", " (root)", " (/usr)", \

    " - / filesystem", ",root(/)"] # remove from summaries

svr4pkgsseen = {}    #svr4 pkgs seen - pkgs indexed by name

timestamp_files = [] # patterns of files that retain timestamps from svr4 pkgs

wos_path = []        # list of search pathes for svr4 packages

class Package(object):

        def __init__(self, name):

                self.name = name

                self.depend = []        # require dependencies

                self.file_depend = []   # file dependencies

                self.undepend = []

                self.extra = []

                self.dropped_licenses = []

                self.nonhollow_dirs = {}

                self.srcpkgs = []

                self.classification = []

                self.desc = ""

                self.summary = ""

                self.version = ""

                self.consolidation = ""

                self.obsolete_branch = None

                self.rename_branch = None

                self.imppkg = None

                self.actions = []

                pkgdict[name] = self

        def fmristr(self):

                return "%s@%s" % (self.name, self.version)

        def import_pkg(self, imppkg_filename, line):

                exclude_files = line.split() + elided_files.keys()

                p = self.import_files_from_pkg(imppkg_filename,

                    [], exclude_files)

                if not self.version:

                        self.version = "%s-%s" % (def_vers,

                            get_branch(self.name))

                if not self.desc:

                        try:

                                self.desc = zap_strings(p.pkginfo["DESC"],

                                    summary_detritus)

                        except KeyError:

                                self.desc = None

                if not self.summary:

                        self.summary = zap_strings(p.pkginfo["NAME"],

                            summary_detritus)

        def add_svr4_src(self, imppkg):

                self.srcpkgs.append(imppkg)

        def import_files_from_pkg(self, imppkg_filename, includes, excludes):

                try:

                        ppath = pkg_path(imppkg_filename)

                except:

                        raise RuntimeError("No such package: '%s'" % imppkg_filename)

                bundle = SolarisPackageDirBundle(ppath, data=False)

                p = bundle.pkg

                imppkg_name = p.pkginfo["PKG.PLAT"]

                self.imppkg = bundle.pkg

                includes_seen = []

                # filename NOT always same as pkgname

                svr4pkgsseen[imppkg_name] = p

                if "SUNW_PKG_HOLLOW" in p.pkginfo and \

                    p.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true":

                        hollow = True

                else:

                        hollow = False

                for action in bundle:

                        if includes:

                                if action.name == "license":

                                        pass # always include license.

                                elif "path" not in action.attrs or \

                                    action.attrs["path"] not in includes:

                                        continue

                                else:

                                        includes_seen.append(

                                            action.attrs["path"])

                        elif not includes and "path" in action.attrs and \

                            action.attrs["path"] in excludes:

                                if show_debug:

                                        print "excluding %s from %s" % \

                                            (action.attrs["path"], imppkg_name)

                                continue

                        if action.name == "unknown":

                                continue

                        action.attrs["importer.source"] = "svr4pkg"

                        action.attrs["importer.svr4pkg"] = imppkg_name

                        action.attrs["importer.svr4path"] = action.attrs["path"]

                        if action.name == "license":

                                # The "path" attribute is confusing and

                                # unnecessary for licenses.

                                del action.attrs["path"]

                        # is this a file for which we need a timestamp?

                        if action.name == "file":

                                basename = os.path.basename(action.attrs["path"])

                                for file_pattern in timestamp_files:

                                        if fnmatch.fnmatch(basename, file_pattern):

                                                break

                                else:

                                        del action.attrs["timestamp"]

                        if hollow:

                                action.attrs["opensolaris.zone"] = "global"

                                action.attrs["variant.opensolaris.zone"] = "global"

                        self.check_perms(action)

                        self.actions.append(action)

                includes_missed = set(includes) - set(includes_seen)

                if includes_missed:

                        raise RuntimeError("pkg %s: Files specified in multi-line import from %s not seen: %s" %

                            (self.name, imppkg_name, " ".join(includes_missed)))

                self.add_svr4_src(imppkg_name)

                return p

        def import_files(self, imppkg_filename, filenames):

                self.import_files_from_pkg(imppkg_filename, filenames, [])

        def check_perms(self, action):

                if action.name not in ["file", "dir"]:

                        return

                orig = action.attrs.copy()

                if action.attrs["owner"] == "?":

                        action.attrs["owner"] = "root"

                if action.attrs["group"] == "?":

                        action.attrs["group"] = "bin"

                if action.attrs["mode"] == "?":

                        if action.name == "dir":

                                action.attrs["mode"] = "0755"

                        else:

                                action.attrs["mode"] = "0444"

                if orig != action.attrs:

                        for k in action.attrs:

                                if orig[k] != action.attrs[k]:

                                        print "File %s in pkg %s has %s == \"?\": mapped to %s" % \

                                            (

                                            action.attrs["path"],

                                            action.attrs["importer.svr4pkgname"],

                                            k,

                                            action.attrs[k]

                                            )

        def chattr(self, fname, line):

                matches = [

                    a 

                    for a in self.actions

                    if "path" in a.attrs and a.attrs["path"] == fname

                ]

                if not matches:

                        raise RuntimeError("No file '%s' in package '%s'" % \

                            (fname, curpkg.name))

                line = line.rstrip()

                # is this a deletion?

                if line.startswith("drop"):

                        for f in matches:

                                # deletion of existing attribute

                                for d in line.split()[1:]:

                                        if d in f.attrs:

                                                del f.attrs[d]

                                                if show_debug:

                                                        print "removing attribute \"%s\" on %s" % \

                                                            (d, fname)

                        return

                # handle insertion/modification case

                for f in matches:

                        # create attribute dictionary from line

                        new_attrs = actions.attrsfromstr(line.rstrip())

                        f.attrs.update(new_attrs)

                        if show_debug:

                                print "Updating attributes on " + \

                                    "'%s' in '%s' with '%s'" % \

                                    (f.attrs["path"], curpkg.name, new_attrs)

        # apply a chattr to wildcarded files/dirs in current package

        # also allows regexp edit of existing attrs

        def chattr_glob(self, glob, line):

                args = line.split()

                if args[0] == "type": # we care about type

                        args.pop(0)

                        type = args.pop(0)

                        line = " ".join(args)

                else:

                        type = None

                if args[0] == "edit": # we're doing regexp edit of attr

                        edit = True

                        args.pop(0)

                        target = args.pop(0)

                        regexp = re.compile(args.pop(0))

                        replace = args.pop(0)

                        line = " ".join(args)

                else:

                        edit = False

                        new_attrs = actions.attrsfromstr(line.rstrip())

                o = [

                        f

                        for f in self.actions

                        if "path" in f.attrs and 

                            fnmatch.fnmatchcase(f.attrs["path"], glob) and

                            (not type or type == f.name)

                     ]

                for f in o:

                        fname = f.attrs["path"]

                        if edit:

                                if target in f.attrs:

                                        old_value = f.attrs[target]

                                        new_value = regexp.sub(replace, \

                                            old_value)

                                        if old_value == new_value:

                                                continue

                                        f.attrs[target] = new_value

                                else:

                                        continue

                        else:

                                f.attrs.update(new_attrs)

                                if show_debug:

                                        print "Updating attributes on " + \

                                            "'%s' in '%s' with '%s'" % \

                                            (fname, curpkg.name, new_attrs)

def pkg_path(pkgname):

        name = os.path.basename(pkgname)

        if pkgname in pkgpaths:

                return pkgpaths[name]

        if "/" in pkgname:

                pkgpaths[name] = os.path.realpath(pkgname)

                return pkgname

        else:

                for each_path in wos_path:

                        if os.path.exists(each_path + "/" + pkgname):

                                pkgpaths[name] = each_path + "/" + pkgname

                                return pkgpaths[name]

                raise RuntimeError("package %s not found" % pkgname)

def check_pkg_actions(pkg):

        local_path_dict = {}

        # build dictionary of actions in pk by path

        for a in pkg.actions:

                if "path" in a.attrs:

                        local_path_dict.setdefault(a.attrs["path"], []).append(a)

        errors = check_pathdict_actions(local_path_dict, remove_dups=True)

        if errors:

                for e in errors:

                        print e

                raise RuntimeError("Package %s: errors occurred" % pkg.name)

        return local_path_dict

def check_pathdict_actions(my_path_dict, remove_dups=False):

        # investigate all paths w/ multiple actions

        errorlist = []

        for p in my_path_dict:

                # check to make sure all higher parts of path are indeed directories -

                # avoid publishing through symlinks

                tmp = p

                while True:

                        tmp = os.path.dirname(tmp)

                        if tmp:

                                if tmp in my_path_dict: # don't worry about implicit dirs for now

                                        a = my_path_dict[tmp][0]# just check the first one

                                        if a.name != "dir":

                                                errorlist.append("action %s path component %s is not directory: %s" %

                                                    (my_path_dict[p], tmp, a))

                        else:

                                break

                if len(my_path_dict[p]) == 1:

                        continue

                dups = my_path_dict[p]

                # make sure all are the same type

                if len(set((d.name for d in dups))) > 1:

                        errorlist.append("Multiple actions on different types with the same path:\n\t%s\n" %

                            ("\n\t".join(str(d) for d in dups)))

                        # disallow any duplicates that aren't directories

                        continue

                elif dups[0].name == "license": #XXX double check this

                        continue

                elif dups[0].name == "link":

                        targets = set((d.attrs["target"] for d in dups))

                        if len(targets) > 1:

                                errorlist.append("Multiple link actions with same path and different targets:\n\t%s\n" %

                                    ("\n\t".join(str(d) for d in dups)))

                        continue

                elif dups[0].name != "dir":

                        errorlist.append("Multiple actions with the same path that aren't directories:\n\t%s\n" %

                            ("\n\t".join(str(d) for d in dups)))

                        continue

                # construct glommed attrs dict; this check could be more thorough

                dkeys = set([

                    k

                    for d in dups

                    for k in d.attrs.keys()

                ])

                ga = dict(zip(dkeys, [set([d.attrs.get(k, None) for d in dups]) for k in dkeys]))

                for g in ga:

                        if len(ga[g]) == 1:

                                continue

                        if g in ["owner", "group", "mode"]:

                                errorlist.append("Multiple directory actions with the same path(%s) and different %s:\n\t%s\n" %

                                    (p, g, "\n\t".join(str(d) for d in dups)))

                        elif remove_dups and g.startswith("variant.") and None in ga[g]:

                                # remove any dirs that are zone variants if same dir w/o variant exists

                                for d in dups:

                                        if d.attrs.get(g) != None:

                                                d.attrs["importer.deleteme"] = "True"

                                                if 1 or show_debug:

                                                        print "removing %s as hollow dup" % d

        return errorlist

def start_package(pkgname):

        set_macro("PKGNAME", urllib.quote(pkgname, ""))

        return Package(pkgname)

def end_package(pkg):

        pkg_branch = get_branch(pkg.name)

        if not pkg.version:

                pkg.version = "%s-%s" % (def_vers, pkg_branch)

        elif "-" not in pkg.version:

                pkg.version += "-%s" % pkg_branch

       # add description actions

        if pkg.desc:

                pkg.actions.append( actions.attribute.AttributeAction(None,

                    name="pkg.description", value=pkg.desc))

        if pkg.summary:

                pkg.actions.extend([

                    actions.attribute.AttributeAction(None,

                        name="pkg.summary", value=pkg.summary),

                    actions.attribute.AttributeAction(None,

                        name="description", value=pkg.summary)

                ])

        if pkg.classification:

                pkg.actions.append(actions.attribute.AttributeAction(None,

                    name="info.classification", value=pkg.classification))

        # add dependency on consolidation incorporation if not obsolete

        if pkg.consolidation and not pkg.obsolete_branch:

                action = actions.fromstr(

                    "depend fmri=consolidation/%s/%s-incorporation "

                    "type=require importer.no-version=true" % 

                    (pkg.consolidation, pkg.consolidation))

                pkg.actions.append(action)

        # add legacy actions

        if pkg.name != "SUNWipkg":

                for p in pkg.srcpkgs:

                        try:

                                sp = svr4pkgsseen[p]

                        except KeyError:

                                continue

                        wanted_attrs = (

                                "PKG", "NAME", "ARCH", "VERSION", "CATEGORY",

                                "VENDOR", "DESC", "HOTLINE"

                                )

                        attrs = dict(

                                (k.lower(), v)

                                for k, v in sp.pkginfo.iteritems()

                                if k in wanted_attrs

                                )

                        attrs["pkg"] = sp.pkginfo["PKG.PLAT"]

                        pkg.actions.append(

                            actions.legacy.LegacyAction(None, **attrs))

        for action in pkg.actions[:]:

                action.attrs["importer.ipspkg"] = pkg.fmristr()

                if action.name == "license" and \

                    action.attrs["license"] in pkg.dropped_licenses:

                        del pkg.actions[pkg.actions.index(action)]

        # need to check for duplicate actions

        check_pkg_actions(pkg)

        # add to dictionary of known fmris

        fmridict[pkg.name] = pkg.fmristr()

        clear_macro("PKGNAME")

        print "Package '%s'" % pkg.name

        if not show_debug:

                return

        print "  Version:", pkg.version

        print "  Description:", pkg.desc

        print "  Summary:", pkg.summary

        print "  Classification: ", ",".join(pkg.classification)

def publish_action(t, pkg, a):

        # remove any temp attributes

        if show_debug:

                print "%s: %s" % (pkg.name, a)

        for k in a.attrs.keys():

                if k.startswith("importer."):

                        del a.attrs[k]

        try:

                t.add(a)

        except TypeError, e:

                print a.attrs

                print a.name

                raise

def publish_pkg(pkg):

        """ send this package to the repo """

        svr4_pkg_list = sorted(list(set([

            a.attrs["importer.svr4pkg"]

            for a in pkg.actions

            if "importer.svr4pkg" in a.attrs and