src/modules/misc.py
author Danek Duvall <danek.duvall@sun.com>
Fri, 18 Apr 2008 16:17:09 -0700
changeset 342 5e1f4d8429bf
parent 327 6c6bd07efe8d
child 351 9301d4d85349
permissions -rw-r--r--
669 Need method to print package licenses before installation
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
46
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     1
#!/usr/bin/python
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     2
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     3
# CDDL HEADER START
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     4
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     5
# The contents of this file are subject to the terms of the
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     6
# Common Development and Distribution License (the "License").
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     7
# You may not use this file except in compliance with the License.
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     8
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
     9
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    10
# or http://www.opensolaris.org/os/licensing.
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    11
# See the License for the specific language governing permissions
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    12
# and limitations under the License.
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    13
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    14
# When distributing Covered Code, include this CDDL HEADER in each
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    15
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    16
# If applicable, add the following below this CDDL HEADER, with the
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    17
# fields enclosed by brackets "[]" replaced with your own identifying
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    18
# information: Portions Copyright [yyyy] [name of copyright owner]
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    19
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    20
# CDDL HEADER END
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    21
#
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    22
260
a1b77322abb2 255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents: 181
diff changeset
    23
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
46
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    24
# Use is subject to license terms.
119
537d69114be4 Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents: 108
diff changeset
    25
537d69114be4 Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents: 108
diff changeset
    26
import os
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    27
import urllib
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    28
import urllib2
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    29
import urlparse
260
a1b77322abb2 255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents: 181
diff changeset
    30
import httplib
319
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    31
import platform
327
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
    32
import re
342
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
    33
import sha
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
    34
import zlib
46
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    35
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    36
import pkg.urlhelpers as urlhelpers
319
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    37
import pkg.portable as portable
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    38
from pkg.client.imagetypes import img_type_names, IMG_NONE
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    39
from pkg import VERSION
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    40
46
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    41
def hash_file_name(f):
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    42
        """Return the two-level path fragment for the given filename, which is
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    43
        assumed to be a content hash of at least 8 distinct characters."""
119
537d69114be4 Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents: 108
diff changeset
    44
        return os.path.join("%s" % f[0:2], "%s" % f[2:8], "%s" % f)
46
86a10bea7bb6 commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff changeset
    45
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    46
def url_affix_trailing_slash(u):
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    47
        if u[-1] != '/':
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    48
                u = u + '/'
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    49
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    50
        return u
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    51
319
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    52
_client_version = "pkg/%s (%s %s; %s %s; %%s)" % \
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    53
    (VERSION, portable.util.get_canonical_os_name(), platform.machine(),
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    54
    portable.util.get_os_release(), platform.version())
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    55
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    56
def versioned_urlopen(base_uri, operation, versions = [], tail = None,
319
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    57
    data = None, headers = {}, ssl_creds = None, imgtype = IMG_NONE):
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    58
        """Open the best URI for an operation given a set of versions.
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    59
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    60
        Both the client and the server may support multiple versions of
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    61
        the protocol of a particular operation.  The client will pass
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    62
        this method an ordered array of versions it understands, along
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    63
        with the base URI and the operation it wants.  This method will
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    64
        open the URL corresponding to the best version both the client
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    65
        and the server understand, returning a tuple of the open URL and
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    66
        the version used on success, and throwing an exception if no
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    67
        matching version can be found.
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    68
        """
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    69
        # Ignore http_proxy for localhost case, by overriding
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    70
        # default proxy behaviour of urlopen().
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    71
        netloc = urlparse.urlparse(base_uri)[1]
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    72
285
538afd6a9d3e 763 pkg image-create /foo , pkg image-create /tmp/foo both yield (different) tracebacks
Danek Duvall <danek.duvall@sun.com>
parents: 260
diff changeset
    73
        if not netloc:
538afd6a9d3e 763 pkg image-create /foo , pkg image-create /tmp/foo both yield (different) tracebacks
Danek Duvall <danek.duvall@sun.com>
parents: 260
diff changeset
    74
                raise ValueError, "Malformed URL: %s" % base_uri
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    75
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    76
        if urllib.splitport(netloc)[0] == "localhost":
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    77
                # XXX cache this opener?
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    78
                proxy_handler = urllib2.ProxyHandler({})
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    79
                opener_dir = urllib2.build_opener(proxy_handler)
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    80
                url_opener = opener_dir.open
310
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    81
        elif ssl_creds and ssl_creds != (None, None):
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    82
                cert_handler = urlhelpers.HTTPSCertHandler(
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    83
                    key_file = ssl_creds[0], cert_file = ssl_creds[1])
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    84
                opener_dir = urllib2.build_opener(cert_handler)
8fa744df663a 975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents: 285
diff changeset
    85
                url_opener = opener_dir.open
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    86
        else:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    87
                url_opener = urllib2.urlopen
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    88
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    89
        for version in versions:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    90
                if tail:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    91
                        uri = urlparse.urljoin(base_uri, "%s/%s/%s" % \
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    92
                            (operation, version, tail))
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    93
                else:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    94
                        uri = urlparse.urljoin(base_uri, "%s/%s" % \
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    95
                            (operation, version))
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    96
319
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    97
                headers["User-Agent"] = \
39b8b5c538bf 1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents: 310
diff changeset
    98
                    _client_version % img_type_names[imgtype]
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
    99
                req = urllib2.Request(url = uri, headers = headers)
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   100
                if data is not None:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   101
                        req.add_data(data)
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   102
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   103
                try:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   104
                        c = url_opener(req)
153
05d90faecc40 Fix versioned_urlopen() to catch the right error, and catch URLError higher up.
Danek Duvall <danek.duvall@sun.com>
parents: 145
diff changeset
   105
                except urllib2.HTTPError, e:
260
a1b77322abb2 255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents: 181
diff changeset
   106
                        if e.code != httplib.NOT_FOUND or e.msg != "Version not supported":
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   107
                                raise
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   108
                        continue
260
a1b77322abb2 255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents: 181
diff changeset
   109
                # XXX catch BadStatusLine and convert to INTERNAL_SERVER_ERROR?
145
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   110
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   111
                return c, version
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   112
        else:
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   113
                # Couldn't find a version that we liked.
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   114
                raise RuntimeError, \
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   115
                    "%s doesn't speak a known version of %s operation" % \
08bee7fd13f6 Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents: 119
diff changeset
   116
                    (base_uri, operation)
327
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   117
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   118
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   119
_hostname_re = re.compile("^[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9]+\.?)*$")
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   120
_invalid_host_chars = re.compile(".*[^a-zA-Z0-9\-\.]+")
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   121
_valid_proto = ["http", "https"]
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   122
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   123
def valid_auth_prefix(prefix):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   124
        """Verify that the authority prefix only contains valid characters."""
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   125
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   126
        # This is a workaround for the the hostname_re being slow when
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   127
        # it comes to finding invalid characters in the prefix string.
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   128
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   129
        if _invalid_host_chars.match(prefix):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   130
                # prefix bad chars
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   131
                return False
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   132
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   133
        if _hostname_re.match(prefix):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   134
                return True
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   135
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   136
        return False
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   137
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   138
def valid_auth_url(url):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   139
        """Verify that the authority URL contains only valid characters."""
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   140
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   141
        # First split the URL and check if the scheme is one we support
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   142
        o = urlparse.urlsplit(url)
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   143
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   144
        if not o[0] in _valid_proto:
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   145
                return False
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   146
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   147
        # Next verify that the network location is valid
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   148
        host, port = urllib.splitport(o[1])
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   149
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   150
        if not host or _invalid_host_chars.match(host):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   151
                return False
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   152
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   153
        if _hostname_re.match(host):
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   154
                return True
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   155
6c6bd07efe8d 1031 authority prefix needs validation
johansen <johansen@sun.com>
parents: 319
diff changeset
   156
        return False
342
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   157
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   158
def gunzip_from_stream(gz, outfile):
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   159
        """Decompress a gzipped input stream into an output stream.
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   160
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   161
        The argument 'gz' is an input stream of a gzipped file (XXX make it do
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   162
        either a gzipped file or raw zlib compressed data), and 'outfile' is is
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   163
        an output stream.  gunzip_from_stream() decompresses data from 'gz' and
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   164
        writes it to 'outfile', and returns the hexadecimal SHA-1 sum of that
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   165
        data.
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   166
        """
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   167
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   168
        FHCRC = 2
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   169
        FEXTRA = 4
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   170
        FNAME = 8
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   171
        FCOMMENT = 16
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   172
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   173
        # Read the header
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   174
        magic = gz.read(2)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   175
        if magic != "\037\213":
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   176
                raise IOError, "Not a gzipped file"
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   177
        method = ord(gz.read(1))
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   178
        if method != 8:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   179
                raise IOError, "Unknown compression method"
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   180
        flag = ord(gz.read(1))
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   181
        gz.read(6) # Discard modtime, extraflag, os
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   182
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   183
        # Discard an extra field
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   184
        if flag & FEXTRA:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   185
                xlen = ord(gz.read(1))
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   186
                xlen = xlen + 256 * ord(gz.read(1))
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   187
                gz.read(xlen)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   188
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   189
        # Discard a null-terminated filename
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   190
        if flag & FNAME:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   191
                while True:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   192
                        s = gz.read(1)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   193
                        if not s or s == "\000":
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   194
                                break
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   195
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   196
        # Discard a null-terminated comment
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   197
        if flag & FCOMMENT:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   198
                while True:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   199
                        s = gz.read(1)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   200
                        if not s or s == "\000":
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   201
                                break
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   202
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   203
        # Discard a 16-bit CRC
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   204
        if flag & FHCRC:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   205
                gz.read(2)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   206
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   207
        shasum = sha.new()
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   208
        dcobj = zlib.decompressobj(-zlib.MAX_WBITS)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   209
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   210
        while True:
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   211
                buf = gz.read(64 * 1024)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   212
                if buf == "":
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   213
                        ubuf = dcobj.flush()
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   214
                        shasum.update(ubuf)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   215
                        outfile.write(ubuf)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   216
                        break
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   217
                ubuf = dcobj.decompress(buf)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   218
                shasum.update(ubuf)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   219
                outfile.write(ubuf)
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   220
5e1f4d8429bf 669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents: 327
diff changeset
   221
        return shasum.hexdigest()