54 if block == '': |
56 if block == '': |
55 break |
57 break |
56 |
58 |
57 return m.hexdigest() == value |
59 return m.hexdigest() == value |
58 |
60 |
59 def download(url, filename = None, hash = None, verbose = False): |
61 def download(url, filename = None): |
60 from urllib import splittype, urlopen |
62 src = None |
61 import hashlib |
|
62 |
63 |
63 src = None |
64 try: |
64 filename = None |
65 src = urlopen(url) |
65 |
66 except IOError: |
66 #print "Downloading %s from %s" % (filename, url) |
|
67 |
|
68 # open the download source |
|
69 if splittype(url)[0]: |
|
70 try: |
|
71 src = urlopen(url) |
|
72 except IOError: |
|
73 return None |
|
74 remote_name = src.geturl().split('/')[-1] |
|
75 elif os.path.isfile(url): |
|
76 os.symlink(url, filename) |
|
77 return filename, filehash(filename) |
|
78 src = open(url, 'r') |
|
79 remote_name = url.split('/')[-1] |
|
80 else: |
|
81 return None |
67 return None |
82 |
68 |
83 if filename == None: |
69 if filename == None: |
84 filename = remote_name |
70 filename = src.geturl().split('/')[-1] |
85 |
71 |
86 # if we have a source to download, open a destination and copy the data |
72 try: |
87 if src: |
|
88 dst = open(filename, 'wb'); |
73 dst = open(filename, 'wb'); |
89 if verbose: |
74 except IOError: |
90 print "Downloading %s from %s" % (filename, url) |
75 src.close() |
91 while True: |
76 return None |
92 block = src.read() |
77 |
93 if block == '': |
78 while True: |
94 break; |
79 block = src.read() |
95 dst.write(block) |
80 if block == '': |
96 dst.close() |
81 break; |
|
82 dst.write(block) |
|
83 |
|
84 src.close() |
|
85 dst.close() |
97 |
86 |
98 # return the name of the file that we downloaded the data to. |
87 # return the name of the file that we downloaded the data to. |
99 return filename |
88 return filename |
100 |
89 |
101 def download_paths(search, filename): |
90 def download_paths(search, filename, url): |
102 tmp = os.getenv('DOWNLOAD_SEARCH_PATH') |
|
103 if tmp: |
|
104 search += tmp.split(' ') |
|
105 base = os.path.basename(filename) |
|
106 |
|
107 urls = list() |
91 urls = list() |
108 |
92 |
109 for path in search: |
93 if filename != None: |
110 urls.append(path+'/'+base) |
94 tmp = os.getenv('DOWNLOAD_SEARCH_PATH') |
|
95 if tmp: |
|
96 search += tmp.split(' ') |
|
97 |
|
98 file = os.path.basename(filename) |
|
99 |
|
100 urls = [ base + '/' + file for base in search ] |
|
101 |
|
102 # filename should always be first |
|
103 if filename in urls: |
|
104 urls.remove(filename) |
|
105 urls.insert(0, filename) |
|
106 |
|
107 # command line url is a fallback, so it's last |
|
108 if url != None and url not in urls: |
|
109 urls.append(url) |
111 |
110 |
112 return urls |
111 return urls |
113 |
112 |
114 def usage(): |
113 def usage(): |
115 print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1]) |
114 print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1]) |
116 sys.exit(1) |
115 sys.exit(1) |
117 |
116 |
118 def main(): |
117 def main(): |
119 from urllib import splittype, urlopen |
|
120 import getopt |
118 import getopt |
121 import sys |
|
122 |
119 |
123 # FLUSH STDOUT |
120 # FLUSH STDOUT |
124 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
121 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
125 |
122 |
126 verbose = False |
123 file_arg = None |
127 filename = None |
124 link_arg = False |
128 url = None |
125 hash_arg = None |
129 hash = None |
126 url_arg = None |
130 search = list() |
127 search_list = list() |
131 |
128 |
132 try: |
129 try: |
133 opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v", |
130 opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:", |
134 ["file=", "hash=", "search=", "url=", "verbose"]) |
131 ["file=", "link", "hash=", "search=", "url="]) |
135 except getopt.GetoptError, err: |
132 except getopt.GetoptError, err: |
136 print str(err) |
133 print str(err) |
137 usage() |
134 usage() |
138 |
135 |
139 for opt, arg in opts: |
136 for opt, arg in opts: |
140 if opt in [ "-f", "--file" ]: |
137 if opt in [ "-f", "--file" ]: |
141 filename = arg |
138 file_arg = arg |
|
139 elif opt in [ "-l", "--link" ]: |
|
140 link_arg = True |
142 elif opt in [ "-h", "--hash" ]: |
141 elif opt in [ "-h", "--hash" ]: |
143 hash = arg |
142 hash_arg = arg |
144 elif opt in [ "-s", "--search" ]: |
143 elif opt in [ "-s", "--search" ]: |
145 search.append(arg) |
144 search_list.append(arg) |
146 elif opt in [ "-u", "--url" ]: |
145 elif opt in [ "-u", "--url" ]: |
147 url = arg |
146 url_arg = arg |
148 elif opt in [ "-v", "--verbose" ]: |
|
149 verbose = True |
|
150 else: |
147 else: |
151 assert False, "unknown option" |
148 assert False, "unknown option" |
152 |
149 |
153 if url == None: |
150 if url_arg == None: |
154 usage() |
151 usage() |
155 |
152 |
156 print "Fetching %s" % filename |
153 for url in download_paths(search_list, file_arg, url_arg): |
|
154 print "Source %s..." % url, |
157 |
155 |
158 if validate(filename, hash): |
156 scheme, path = splittype(url) |
159 print "\tcurrent copy is ok, nothing to do" |
157 name = file_arg |
160 sys.exit(0) |
|
161 |
158 |
162 # generate a list of URLS to search for a suitable download |
159 if scheme in [ None, 'file' ]: |
163 urls = download_paths(search, filename) |
160 if os.path.exists(path) == False: |
164 urls.append(url) |
161 print "not found, skipping" |
165 |
162 continue |
166 for url in urls: |
163 elif name != path: |
167 print "\tTrying %s..." % url, |
164 if link_arg == False: |
168 |
165 print "copying...", |
169 if os.path.isfile(url): |
166 shutil.copy2(path, name) |
170 os.symlink(url, filename) |
167 else: |
171 elif splittype(url)[0]: |
168 print "linking...", |
172 if download(url, filename) == None: |
169 os.symlink(path, name) |
|
170 else: |
|
171 pass |
|
172 elif scheme in [ 'http', 'https', 'ftp' ]: |
|
173 print "downloading...", |
|
174 name = download(url, file_arg) |
|
175 if name == None: |
173 print "failed" |
176 print "failed" |
174 continue |
177 continue |
175 |
178 |
176 print "retrieved...", |
179 print "validating...", |
177 |
180 if validate(name, hash_arg): |
178 if validate(filename, hash): |
181 print "ok" |
179 print "validated" |
|
180 sys.exit(0) |
182 sys.exit(0) |
181 else: |
183 else: |
182 print "corrupt" |
184 print "corrupt" |
183 |
185 |
184 try: |
186 try: |
185 os.remove(filename) |
187 os.remove(name) |
186 except OSError: |
188 except OSError: |
187 pass |
189 pass |
188 |
190 |
189 sys.exit(1) |
191 sys.exit(1) |
190 |
192 |