author | Danek Duvall <danek.duvall@sun.com> |
Fri, 18 Apr 2008 16:17:09 -0700 | |
changeset 342 | 5e1f4d8429bf |
parent 327 | 6c6bd07efe8d |
child 351 | 9301d4d85349 |
permissions | -rw-r--r-- |
46
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/python |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
2 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
3 |
# CDDL HEADER START |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
4 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
5 |
# The contents of this file are subject to the terms of the |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
6 |
# Common Development and Distribution License (the "License"). |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
7 |
# You may not use this file except in compliance with the License. |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
8 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
10 |
# or http://www.opensolaris.org/os/licensing. |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
11 |
# See the License for the specific language governing permissions |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
12 |
# and limitations under the License. |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
13 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
19 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
20 |
# CDDL HEADER END |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
21 |
# |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
22 |
|
260
a1b77322abb2
255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents:
181
diff
changeset
|
23 |
# Copyright 2008 Sun Microsystems, Inc. All rights reserved. |
46
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
24 |
# Use is subject to license terms. |
119
537d69114be4
Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents:
108
diff
changeset
|
25 |
|
537d69114be4
Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents:
108
diff
changeset
|
26 |
import os |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
27 |
import urllib |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
28 |
import urllib2 |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
29 |
import urlparse |
260
a1b77322abb2
255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents:
181
diff
changeset
|
30 |
import httplib |
319
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
31 |
import platform |
327
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
32 |
import re |
342
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
33 |
import sha |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
34 |
import zlib |
46
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
35 |
|
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
36 |
import pkg.urlhelpers as urlhelpers |
319
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
37 |
import pkg.portable as portable |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
38 |
from pkg.client.imagetypes import img_type_names, IMG_NONE |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
39 |
from pkg import VERSION |
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
40 |
|
46
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
41 |
def hash_file_name(f): |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
42 |
"""Return the two-level path fragment for the given filename, which is |
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
43 |
assumed to be a content hash of at least 8 distinct characters.""" |
119
537d69114be4
Implement bundled file downloads using filelist
johansen <johansen@sun.com>
parents:
108
diff
changeset
|
44 |
return os.path.join("%s" % f[0:2], "%s" % f[2:8], "%s" % f) |
46
86a10bea7bb6
commonize file/ prefix handling; sketch manifest difference algorithm
Stephen Hahn <sch@sun.com>
parents:
diff
changeset
|
45 |
|
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
46 |
def url_affix_trailing_slash(u): |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
47 |
if u[-1] != '/': |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
48 |
u = u + '/' |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
49 |
|
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
50 |
return u |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
51 |
|
319
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
52 |
_client_version = "pkg/%s (%s %s; %s %s; %%s)" % \ |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
53 |
(VERSION, portable.util.get_canonical_os_name(), platform.machine(), |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
54 |
portable.util.get_os_release(), platform.version()) |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
55 |
|
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
56 |
def versioned_urlopen(base_uri, operation, versions = [], tail = None, |
319
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
57 |
data = None, headers = {}, ssl_creds = None, imgtype = IMG_NONE): |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
58 |
"""Open the best URI for an operation given a set of versions. |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
59 |
|
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
60 |
Both the client and the server may support multiple versions of |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
61 |
the protocol of a particular operation. The client will pass |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
62 |
this method an ordered array of versions it understands, along |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
63 |
with the base URI and the operation it wants. This method will |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
64 |
open the URL corresponding to the best version both the client |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
65 |
and the server understand, returning a tuple of the open URL and |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
66 |
the version used on success, and throwing an exception if no |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
67 |
matching version can be found. |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
68 |
""" |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
69 |
# Ignore http_proxy for localhost case, by overriding |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
70 |
# default proxy behaviour of urlopen(). |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
71 |
netloc = urlparse.urlparse(base_uri)[1] |
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
72 |
|
285
538afd6a9d3e
763 pkg image-create /foo , pkg image-create /tmp/foo both yield (different) tracebacks
Danek Duvall <danek.duvall@sun.com>
parents:
260
diff
changeset
|
73 |
if not netloc: |
538afd6a9d3e
763 pkg image-create /foo , pkg image-create /tmp/foo both yield (different) tracebacks
Danek Duvall <danek.duvall@sun.com>
parents:
260
diff
changeset
|
74 |
raise ValueError, "Malformed URL: %s" % base_uri |
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
75 |
|
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
76 |
if urllib.splitport(netloc)[0] == "localhost": |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
77 |
# XXX cache this opener? |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
78 |
proxy_handler = urllib2.ProxyHandler({}) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
79 |
opener_dir = urllib2.build_opener(proxy_handler) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
80 |
url_opener = opener_dir.open |
310
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
81 |
elif ssl_creds and ssl_creds != (None, None): |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
82 |
cert_handler = urlhelpers.HTTPSCertHandler( |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
83 |
key_file = ssl_creds[0], cert_file = ssl_creds[1]) |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
84 |
opener_dir = urllib2.build_opener(cert_handler) |
8fa744df663a
975 Catalogs need to know their origins
johansen <johansen@sun.com>
parents:
285
diff
changeset
|
85 |
url_opener = opener_dir.open |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
86 |
else: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
87 |
url_opener = urllib2.urlopen |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
88 |
|
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
89 |
for version in versions: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
90 |
if tail: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
91 |
uri = urlparse.urljoin(base_uri, "%s/%s/%s" % \ |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
92 |
(operation, version, tail)) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
93 |
else: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
94 |
uri = urlparse.urljoin(base_uri, "%s/%s" % \ |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
95 |
(operation, version)) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
96 |
|
319
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
97 |
headers["User-Agent"] = \ |
39b8b5c538bf
1104 want custom user-agent string
Danek Duvall <danek.duvall@sun.com>
parents:
310
diff
changeset
|
98 |
_client_version % img_type_names[imgtype] |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
99 |
req = urllib2.Request(url = uri, headers = headers) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
100 |
if data is not None: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
101 |
req.add_data(data) |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
102 |
|
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
103 |
try: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
104 |
c = url_opener(req) |
153
05d90faecc40
Fix versioned_urlopen() to catch the right error, and catch URLError higher up.
Danek Duvall <danek.duvall@sun.com>
parents:
145
diff
changeset
|
105 |
except urllib2.HTTPError, e: |
260
a1b77322abb2
255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents:
181
diff
changeset
|
106 |
if e.code != httplib.NOT_FOUND or e.msg != "Version not supported": |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
107 |
raise |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
108 |
continue |
260
a1b77322abb2
255 Symbolic HTTP response codes should be used
Shawn Walker <swalker@opensolaris.org>
parents:
181
diff
changeset
|
109 |
# XXX catch BadStatusLine and convert to INTERNAL_SERVER_ERROR? |
145
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
110 |
|
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
111 |
return c, version |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
112 |
else: |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
113 |
# Couldn't find a version that we liked. |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
114 |
raise RuntimeError, \ |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
115 |
"%s doesn't speak a known version of %s operation" % \ |
08bee7fd13f6
Add versioning to the protocols
Danek Duvall <danek.duvall@sun.com>
parents:
119
diff
changeset
|
116 |
(base_uri, operation) |
327
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
117 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
118 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
119 |
_hostname_re = re.compile("^[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9]+\.?)*$") |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
120 |
_invalid_host_chars = re.compile(".*[^a-zA-Z0-9\-\.]+") |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
121 |
_valid_proto = ["http", "https"] |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
122 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
123 |
def valid_auth_prefix(prefix): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
124 |
"""Verify that the authority prefix only contains valid characters.""" |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
125 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
126 |
# This is a workaround for the the hostname_re being slow when |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
127 |
# it comes to finding invalid characters in the prefix string. |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
128 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
129 |
if _invalid_host_chars.match(prefix): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
130 |
# prefix bad chars |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
131 |
return False |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
132 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
133 |
if _hostname_re.match(prefix): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
134 |
return True |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
135 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
136 |
return False |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
137 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
138 |
def valid_auth_url(url): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
139 |
"""Verify that the authority URL contains only valid characters.""" |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
140 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
141 |
# First split the URL and check if the scheme is one we support |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
142 |
o = urlparse.urlsplit(url) |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
143 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
144 |
if not o[0] in _valid_proto: |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
145 |
return False |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
146 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
147 |
# Next verify that the network location is valid |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
148 |
host, port = urllib.splitport(o[1]) |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
149 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
150 |
if not host or _invalid_host_chars.match(host): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
151 |
return False |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
152 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
153 |
if _hostname_re.match(host): |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
154 |
return True |
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
155 |
|
6c6bd07efe8d
1031 authority prefix needs validation
johansen <johansen@sun.com>
parents:
319
diff
changeset
|
156 |
return False |
342
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
157 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
158 |
def gunzip_from_stream(gz, outfile): |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
159 |
"""Decompress a gzipped input stream into an output stream. |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
160 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
161 |
The argument 'gz' is an input stream of a gzipped file (XXX make it do |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
162 |
either a gzipped file or raw zlib compressed data), and 'outfile' is is |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
163 |
an output stream. gunzip_from_stream() decompresses data from 'gz' and |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
164 |
writes it to 'outfile', and returns the hexadecimal SHA-1 sum of that |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
165 |
data. |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
166 |
""" |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
167 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
168 |
FHCRC = 2 |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
169 |
FEXTRA = 4 |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
170 |
FNAME = 8 |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
171 |
FCOMMENT = 16 |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
172 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
173 |
# Read the header |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
174 |
magic = gz.read(2) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
175 |
if magic != "\037\213": |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
176 |
raise IOError, "Not a gzipped file" |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
177 |
method = ord(gz.read(1)) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
178 |
if method != 8: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
179 |
raise IOError, "Unknown compression method" |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
180 |
flag = ord(gz.read(1)) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
181 |
gz.read(6) # Discard modtime, extraflag, os |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
182 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
183 |
# Discard an extra field |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
184 |
if flag & FEXTRA: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
185 |
xlen = ord(gz.read(1)) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
186 |
xlen = xlen + 256 * ord(gz.read(1)) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
187 |
gz.read(xlen) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
188 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
189 |
# Discard a null-terminated filename |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
190 |
if flag & FNAME: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
191 |
while True: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
192 |
s = gz.read(1) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
193 |
if not s or s == "\000": |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
194 |
break |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
195 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
196 |
# Discard a null-terminated comment |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
197 |
if flag & FCOMMENT: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
198 |
while True: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
199 |
s = gz.read(1) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
200 |
if not s or s == "\000": |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
201 |
break |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
202 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
203 |
# Discard a 16-bit CRC |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
204 |
if flag & FHCRC: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
205 |
gz.read(2) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
206 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
207 |
shasum = sha.new() |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
208 |
dcobj = zlib.decompressobj(-zlib.MAX_WBITS) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
209 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
210 |
while True: |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
211 |
buf = gz.read(64 * 1024) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
212 |
if buf == "": |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
213 |
ubuf = dcobj.flush() |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
214 |
shasum.update(ubuf) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
215 |
outfile.write(ubuf) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
216 |
break |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
217 |
ubuf = dcobj.decompress(buf) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
218 |
shasum.update(ubuf) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
219 |
outfile.write(ubuf) |
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
220 |
|
5e1f4d8429bf
669 Need method to print package licenses before installation
Danek Duvall <danek.duvall@sun.com>
parents:
327
diff
changeset
|
221 |
return shasum.hexdigest() |