#!/usr/bin/python2.4

#

# CDDL HEADER START

#

# The contents of this file are subject to the terms of the

# Common Development and Distribution License (the "License").

# You may not use this file except in compliance with the License.

#

# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

# or http://www.opensolaris.org/os/licensing.

# See the License for the specific language governing permissions

# and limitations under the License.

#

# When distributing Covered Code, include this CDDL HEADER in each

# file and include the License file at usr/src/OPENSOLARIS.LICENSE.

# If applicable, add the following below this CDDL HEADER, with the

# fields enclosed by brackets "[]" replaced with your own identifying

# information: Portions Copyright [yyyy] [name of copyright owner]

#

# CDDL HEADER END

#

# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

# Use is subject to license terms.

import testutils

if __name__ == "__main__":

	testutils.setup_environment("../../../proto")

import unittest

import os

import sys

import copy

import re

import shutil

import difflib

import time

import pkg.client.api as api

import pkg.client.api_errors as api_errors

import pkg.client.query_parser as query_parser

import pkg.client.progress as progress

import pkg.fmri as fmri

import pkg.portable as portable

import pkg.search_storage as ss

API_VERSION = 15

PKG_CLIENT_NAME = "pkg"

class TestApiSearchBasics(testutils.SingleDepotTestCase):

        example_pkg10 = """

            open [email protected],5.11-0

            add dir mode=0755 owner=root group=bin path=/bin

            add dir mode=0755 owner=root group=bin path=/bin/example_dir

            add dir mode=0755 owner=root group=bin path=/usr/lib/python2.4/vendor-packages/OpenSSL

            add file /tmp/example_file mode=0555 owner=root group=bin path=/bin/example_path

            add set name=com.sun.service.incorporated_changes value="6556919 6627937"

            add set name=com.sun.service.random_test value=42 value=79

            add set name=com.sun.service.bug_ids value="4641790 4725245 4817791 4851433 4897491 4913776 6178339 6556919 6627937"

            add set name=com.sun.service.keywords value="sort null -n -m -t sort 0x86 separator"

            add set name=com.sun.service.info_url value=http://service.opensolaris.com/xml/pkg/[email protected],5.11-1:20080514I120000Z

            add set description='FOOO bAr O OO OOO'

            add set name='weirdness' value='] [ * ?'

            close """

        example_pkg11 = """

            open [email protected],5.11-0

            add dir mode=0755 owner=root group=bin path=/bin

            add file /tmp/example_file mode=0555 owner=root group=bin path=/bin/example_path11

            close """

        another_pkg10 = """

            open [email protected],5.11-0

            add dir mode=0755 owner=root group=bin path=/bin

            add dir mode=0755 owner=root group=bin path=/bin/another_dir

            add file /tmp/example_file mode=0555 owner=root group=bin path=/bin/another_path

            add set name=com.sun.service.incorporated_changes value="6556919 6627937"

            add set name=com.sun.service.random_test value=42 value=79

            add set name=com.sun.service.bug_ids value="4641790 4725245 4817791 4851433 4897491 4913776 6178339 6556919 6627937"

            add set name=com.sun.service.keywords value="sort null -n -m -t sort 0x86 separator"

            add set name=com.sun.service.info_url value=http://service.opensolaris.com/xml/pkg/[email protected],5.11-1:20080514I120000Z

            close """

        bad_pkg10 = """

            open [email protected],5.11-0

            add dir path=badfoo/ mode=0755 owner=root group=bin

            close """

        space_pkg10 = """

            open [email protected],5.11-0

            add file /tmp/example_file mode=0444 owner=nobody group=sys path='unique/with a space'

            add dir mode=0755 owner=root group=bin path=unique_dir

            close """

        cat_pkg10 = """

            open [email protected],5.11-0

            add set name=info.classification value=org.opensolaris.category.2008:System/Security value=org.random:Other/Category

            close """

        cat2_pkg10 = """

            open [email protected],5.11-0

            add set name=info.classification value="org.opensolaris.category.2008:Applications/Sound and Video" value=Developer/C

            close """

        cat3_pkg10 = """

            open [email protected],5.11-0

            add set name=info.classification value="org.opensolaris.category.2008:foo/bar/baz/bill/beam/asda"

            close """

        bad_cat_pkg10 = """

            open [email protected],5.11-0

            add set name=info.classification value="TestBad1/TestBad2"

            close """

        bad_cat2_pkg10 = """

            open [email protected],5.11-0

            add set name=info.classification value="org.opensolaris.category.2008:TestBad1:TestBad2"

            close """

        fat_pkg10 = """

open [email protected],5.11-0

add set name=variant.arch value=sparc value=i386

add set name=description value="i386 variant" variant.arch=i386

add set name=description value="sparc variant" variant.arch=sparc

close """

        bogus_pkg10 = """

set name=fmri value=pkg:/[email protected],5.11-0:20090326T233451Z

set name=description value=""validation with simple chains of constraints ""

set name=pkg.description value="pseudo-hashes as arrays tied to a "type" (list of fields)"

depend fmri=XML-Atom-Entry

set name=com.sun.service.incorporated_changes value="6556919 6627937"

"""

        bogus_fmri = fmri.PkgFmri("[email protected],5.11-0:20090326T233451Z")

        res_remote_path = set([

            ("pkg:/[email protected]", "basename","file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18")

        ])

        res_remote_bin = set([

            ("pkg:/[email protected]", "path", "dir group=bin mode=0755 owner=root path=bin")

        ])

        res_remote_openssl = set([

            ("pkg:/[email protected]", "basename", "dir group=bin mode=0755 owner=root path=usr/lib/python2.4/vendor-packages/OpenSSL")

        ])

        res_remote_bug_id = set([

            ("pkg:/[email protected]", "4851433", 'set name=com.sun.service.bug_ids value="4641790 4725245 4817791 4851433 4897491 4913776 6178339 6556919 6627937"')

        ])

        res_remote_bug_id_4725245 = set([

            ("pkg:/[email protected]", "4725245", 'set name=com.sun.service.bug_ids value="4641790 4725245 4817791 4851433 4897491 4913776 6178339 6556919 6627937"')

        ])

        res_remote_inc_changes = set([

            ("pkg:/[email protected]", "6556919", 'set name=com.sun.service.incorporated_changes value="6556919 6627937"'),

            ("pkg:/[email protected]", "6556919", 'set name=com.sun.service.bug_ids value="4641790 4725245 4817791 4851433 4897491 4913776 6178339 6556919 6627937"')

        ])

        res_remote_random_test = set([

            ("pkg:/[email protected]", "42", "set name=com.sun.service.random_test value=42 value=79")

        ])

        res_remote_random_test_79 = set([

            ("pkg:/[email protected]", "79", "set name=com.sun.service.random_test value=42 value=79")

        ])

        res_remote_keywords = set([

            ("pkg:/[email protected]", "separator", 'set name=com.sun.service.keywords value="sort null -n -m -t sort 0x86 separator"')

        ])

        res_remote_keywords_sort_phrase = set([

            ("pkg:/[email protected]", "sort 0x86", 'set name=com.sun.service.keywords value="sort null -n -m -t sort 0x86 separator"')

        ])

        res_remote_wildcard = res_remote_path.union(set([

            ('pkg:/[email protected]', 'basename', 'dir group=bin mode=0755 owner=root path=bin/example_dir')

        ]))

        res_remote_glob = set([

            ('pkg:/[email protected]', 'path', 'dir group=bin mode=0755 owner=root path=bin/example_dir'),

            ('pkg:/[email protected]', 'basename', 'dir group=bin mode=0755 owner=root path=bin/example_dir'),

            ('pkg:/[email protected]', 'path', 'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18')

        ]) | res_remote_path

        res_remote_foo = set([

            ('pkg:/[email protected]', 'FOOO', 'set name=description value="FOOO bAr O OO OOO"')

        ])

        res_remote_bar = set([

            ('pkg:/[email protected]', 'bAr', 'set name=description value="FOOO bAr O OO OOO"')

        ])

        res_remote_star = set([

            ('pkg:/[email protected]', '*', 'set name=weirdness value="] [ * ?"')

        ])

        res_remote_mark = set([

            ('pkg:/[email protected]', '?', 'set name=weirdness value="] [ * ?"')

        ])

        res_remote_left_brace = set([

            ('pkg:/[email protected]', '[', 'set name=weirdness value="] [ * ?"')

        ])

        res_remote_right_brace = set([

            ('pkg:/[email protected]', ']', 'set name=weirdness value="] [ * ?"')    

        ])

        local_fmri_string = \

            ('pkg:/[email protected]', 'example_pkg', '')

        res_local_pkg = set([

                local_fmri_string

                ])

        res_local_path = copy.copy(res_remote_path)

        res_local_bin = copy.copy(res_remote_bin)

        res_local_bug_id = copy.copy(res_remote_bug_id)

        res_local_inc_changes = copy.copy(res_remote_inc_changes)

        res_local_random_test = copy.copy(res_remote_random_test)

        res_local_keywords = copy.copy(res_remote_keywords)

        res_local_wildcard = copy.copy(res_remote_wildcard)

        res_local_wildcard.add(local_fmri_string)

        res_local_glob = copy.copy(res_remote_glob)

        res_local_glob.add(local_fmri_string)

        res_local_foo = copy.copy(res_remote_foo)

        res_local_bar = copy.copy(res_remote_bar)

        res_local_openssl = copy.copy(res_remote_openssl)

        res_local_path_example11 = set([

            ("pkg:/[email protected]", "basename", "file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path11 pkg.csize=38 pkg.size=18")

        ])

        res_local_bin_example11 = set([

            ("pkg:/[email protected]", "path", "dir group=bin mode=0755 owner=root path=bin")

        ])

        res_local_wildcard_example11 = set([

            ("pkg:/[email protected]", "basename", "file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path11 pkg.csize=38 pkg.size=18"),

            ("pkg:/[email protected]", "example_pkg", "")

       ])

        res_local_pkg_example11 = set([

            ("pkg:/[email protected]", "example_pkg", "")

        ])

        res_cat_pkg10 = set([

            ('pkg:/[email protected]', 'System/Security', 'set name=info.classification value=org.opensolaris.category.2008:System/Security value=org.random:Other/Category')

        ])

        res_cat_pkg10_2 = set([

            ('pkg:/[email protected]', 'Other/Category', 'set name=info.classification value=org.opensolaris.category.2008:System/Security value=org.random:Other/Category')

        ])

        res_cat2_pkg10 = set([

            ('pkg:/[email protected]', 'Applications/Sound and Video', 'set name=info.classification value="org.opensolaris.category.2008:Applications/Sound and Video" value=Developer/C')

        ])

        res_cat2_pkg10_2 = set([

            ('pkg:/[email protected]', 'Developer/C', 'set name=info.classification value="org.opensolaris.category.2008:Applications/Sound and Video" value=Developer/C')

        ])

        res_cat3_pkg10 = set([

            ('pkg:/[email protected]', 'foo/bar/baz/bill/beam/asda', 'set name=info.classification value=org.opensolaris.category.2008:foo/bar/baz/bill/beam/asda')

        ])

        res_fat10_i386 = set([

            ('pkg:/[email protected]', 'i386', 'set name=description value="i386 variant" variant.arch=i386'),

            ('pkg:/[email protected]', 'variant', 'set name=description value="i386 variant" variant.arch=i386'),

            ('pkg:/[email protected]', 'i386', 'set name=variant.arch value=sparc value=i386'),

        ])

        res_fat10_sparc = set([

            ('pkg:/[email protected]', 'sparc', 'set name=description value="sparc variant" variant.arch=sparc'),

            ('pkg:/[email protected]', 'sparc', 'set name=variant.arch value=sparc value=i386'),

            ('pkg:/[email protected]', 'variant', 'set name=description value="sparc variant" variant.arch=sparc')

        ])

        res_remote_fat10_star = res_fat10_sparc | res_fat10_i386

        res_local_fat10_i386_star = res_fat10_i386.union(set([

            ('pkg:/[email protected]', 'test', 'set name=publisher value=test'),

            ('pkg:/[email protected]', 'fat', ''),

            ('pkg:/[email protected]', 'sparc', 'set name=variant.arch value=sparc value=i386')

        ]))

        res_local_fat10_sparc_star = res_fat10_sparc.union(set([

            ('pkg:/[email protected]', 'test', 'set name=publisher value=test'),

            ('pkg:/[email protected]', 'fat', ''),

            ('pkg:/[email protected]', 'i386', 'set name=variant.arch value=sparc value=i386')

        ]))

        res_space_with_star = set([

            ('pkg:/[email protected]', 'basename', 'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=sys mode=0444 owner=nobody path="unique/with a space" pkg.csize=38 pkg.size=18')

        ])

        res_space_space_star = set([

            ('pkg:/[email protected]', 'basename', 'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=sys mode=0444 owner=nobody path="unique/with a space" pkg.csize=38 pkg.size=18'), ('pkg:/[email protected]', 'path', 'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=sys mode=0444 owner=nobody path="unique/with a space" pkg.csize=38 pkg.size=18')

        ])

        res_space_unique = set([

            ('pkg:/[email protected]', 'basename', 'dir group=bin mode=0755 owner=root path=unique_dir')

        ])

        misc_files = ['/tmp/example_file']

        # This is a copy of the 3.81%2C5.11-0.89%3A20080527T163123Z version of

        # SUNWgmake from ipkg with the file and liscense actions changed so

        # that they all take /tmp/example file when sending.

        bug_983_manifest = """

open [email protected],5.11-0.89

add dir group=sys mode=0755 owner=root path=usr

add dir group=bin mode=0755 owner=root path=usr/bin

add dir group=bin mode=0755 owner=root path=usr/gnu

add dir group=bin mode=0755 owner=root path=usr/gnu/bin

add link path=usr/gnu/bin/make target=../../bin/gmake

add dir group=sys mode=0755 owner=root path=usr/gnu/share

add dir group=bin mode=0755 owner=root path=usr/gnu/share/man

add dir group=bin mode=0755 owner=root path=usr/gnu/share/man/man1

add link path=usr/gnu/share/man/man1/make.1 target=../../../../share/man/man1/gmake.1

add dir group=bin mode=0755 owner=root path=usr/sfw

add dir group=bin mode=0755 owner=root path=usr/sfw/bin

add link path=usr/sfw/bin/gmake target=../../bin/gmake

add dir group=bin mode=0755 owner=root path=usr/sfw/share

add dir group=bin mode=0755 owner=root path=usr/sfw/share/man

add dir group=bin mode=0755 owner=root path=usr/sfw/share/man/man1

add link path=usr/sfw/share/man/man1/gmake.1 target=../../../../share/man/man1/gmake.1

add dir group=sys mode=0755 owner=root path=usr/share

add dir group=bin mode=0755 owner=root path=usr/share/info

add dir group=bin mode=0755 owner=root path=usr/share/man

add dir group=bin mode=0755 owner=root path=usr/share/man/man1

add file /tmp/example_file elfarch=i386 elfbits=32 elfhash=68cca393e816e6adcbac1e8ffe9c618de70413e0 group=bin mode=0555 owner=root path=usr/bin/gmake pkg.size=18

add file /tmp/example_file group=bin mode=0444 owner=root path=usr/share/info/make.info pkg.size=18

add file /tmp/example_file group=bin mode=0444 owner=root path=usr/share/info/make.info-1 pkg.size=18

add file /tmp/example_file group=bin mode=0444 owner=root path=usr/share/info/make.info-2 pkg.size=18

add file /tmp/example_file group=bin mode=0444 owner=root path=usr/share/man/man1/gmake.1 pkg.size=18

add license /tmp/example_file license=SUNWgmake.copyright pkg.size=18 transaction_id=1211931083_pkg%3A%2FSUNWgmake%403.81%2C5.11-0.89%3A20080527T163123Z

add depend fmri=pkg:/[email protected] type=require

add depend [email protected] type=require

add depend [email protected] type=incorporate

add set name=description value="gmake - GNU make"

add legacy arch=i386 category=system desc="GNU make - A utility used to build software (gmake) 3.81" hotline="Please contact your local service provider" name="gmake - GNU make" pkg=SUNWgmake vendor="Sun Microsystems, Inc." version=11.11.0,REV=2008.04.29.02.08

close

"""

        res_bug_983 = set([

            ("pkg:/[email protected]", "basename", "link path=usr/sfw/bin/gmake target=../../bin/gmake"),

            ('pkg:/[email protected]', 'basename', 'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 elfarch=i386 elfbits=32 elfhash=68cca393e816e6adcbac1e8ffe9c618de70413e0 group=bin mode=0555 owner=root path=usr/bin/gmake pkg.csize=38 pkg.size=18'),

            ('pkg:/[email protected]', 'gmake', 'set name=description value="gmake - GNU make"')

        ])

        res_983_csl_dependency = set([

            ('pkg:/[email protected]', 'require', 'depend fmri=pkg:/[email protected] type=require')

        ])

        res_983_bar_dependency = set([

            ('pkg:/[email protected]', 'require', 'depend [email protected] type=require')

        ])

        res_983_foo_dependency = set([

            ('pkg:/[email protected]', 'incorporate', 'depend [email protected] type=incorporate')

        ])

        res_local_pkg_ret_pkg = set([

            "pkg:/[email protected]"

        ])

        res_remote_pkg_ret_pkg = set([

            "pkg:/[email protected]"

        ])

        res_remote_file = set([

            ('pkg:/[email protected]',

             'path',

             'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18'),

            ('pkg:/[email protected]',

             '820157a2043e3135f342b238129b556aade20347',

             'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18')

        ]) | res_remote_path

        res_remote_url = set([

            ('pkg:/[email protected]',

            'http://service.opensolaris.com/xml/pkg/[email protected],5.11-1:20080514I120000Z',

            'set name=com.sun.service.info_url value=http://service.opensolaris.com/xml/pkg/[email protected],5.11-1:20080514I120000Z')

        ])

        res_remote_path_extra = set([

            ('pkg:/[email protected]',

             'basename',

             'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18'),

            ('pkg:/[email protected]',

             'path',

             'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18'),

            ('pkg:/[email protected]',

             '820157a2043e3135f342b238129b556aade20347',

             'file 820157a2043e3135f342b238129b556aade20347 chash=bfa46fc98d1ca97f1260090797d35a35e76096a3 group=bin mode=0555 owner=root path=bin/example_path pkg.csize=38 pkg.size=18')

        ])

        res_bad_pkg = set([

            ('pkg:/[email protected]', 'basename',

             'dir group=bin mode=0755 owner=root path=badfoo/')

        ])

        debug_features = None

        def setUp(self):

                for p in self.misc_files:

                        f = open(p, "w")

                        # Write the name of the file into the file, so that

                        # all files have differing contents.

                        f.write(p + "\n")

                        f.close()

                testutils.SingleDepotTestCase.setUp(self,

                    debug_features=self.debug_features)

                tp = self.get_test_prefix()

                self.testdata_dir = os.path.join(tp, "search_results")

                os.mkdir(self.testdata_dir)

                self._dir_restore_functions = [self._restore_dir,

                    self._restore_dir_preserve_hash]

        def tearDown(self):

                testutils.SingleDepotTestCase.tearDown(self)

                for p in self.misc_files:

                        os.remove(p)

                shutil.rmtree(self.testdata_dir)

        def _check(self, proposed_answer, correct_answer):

                if correct_answer == proposed_answer:

                        return True

                else:

                        print "Proposed Answer: " + str(proposed_answer)

                        print "Correct Answer : " + str(correct_answer)

                        if isinstance(correct_answer, set) and \

                            isinstance(proposed_answer, set):

                                print >> sys.stderr, "Missing: " + \

                                    str(correct_answer - proposed_answer)

                                print >> sys.stderr, "Extra  : " + \

                                    str(proposed_answer - correct_answer)

                        self.assert_(correct_answer == proposed_answer)

        @staticmethod

        def _replace_act(act):

                if act.startswith('set name=fmri'):

                        return ''

                else:

                        return act.strip()

        @staticmethod

        def _extract_action_from_res(it):

                return (

                    (fmri.PkgFmri(str(pkg_name)).get_short_fmri(), piece,

                    TestApiSearchBasics._replace_act(act))

                    for query_num, auth, (version, return_type,

                    (pkg_name, piece, act))

                    in it

                )

        @staticmethod

        def _extract_package_from_res(it):

                return (

                    (fmri.PkgFmri(str(pkg_name)).get_short_fmri())

                    for query_num, auth, (version, return_type, pkg_name)

                    in it

                )

        def _search_op(self, api_obj, remote, token, test_value,

            case_sensitive=False, return_actions=True, num_to_return=None,

            start_point=None, servers=None):

                search_func = api_obj.local_search

                if remote:

                        search_func = lambda x: api_obj.remote_search(x,

                            servers=servers)

                query = api.Query(token, case_sensitive, return_actions,

                    num_to_return, start_point)

                res = search_func([query])

                if return_actions:

                        res = self._extract_action_from_res(res)

                else:

                        res = self._extract_package_from_res(res)

                res = set(res)

                self._check(set(res), test_value)

        def _search_op_slow(self, api_obj, remote, token, test_value,

            case_sensitive=False, return_actions=True, num_to_return=None,

            start_point=None):

                search_func = api_obj.local_search

                if remote:

                        search_func = api_obj.remote_search

                query = api.Query(token, case_sensitive, return_actions,

                    num_to_return, start_point)

                tmp = search_func([query])

                res = []

                ssu = False

                try:

                        for i in tmp:

                                res.append(i)