--- a/tools/userland-fetch Thu Nov 04 21:06:30 2010 -0500
+++ b/tools/userland-fetch Thu Nov 04 21:10:58 2010 -0500
@@ -30,6 +30,8 @@
import os
import sys
+import shutil
+from urllib import splittype, urlopen
def validate(filename, hash):
import hashlib
@@ -56,133 +58,133 @@
return m.hexdigest() == value
-def download(url, filename = None, hash = None, verbose = False):
- from urllib import splittype, urlopen
- import hashlib
-
+def download(url, filename = None):
src = None
- filename = None
-
- #print "Downloading %s from %s" % (filename, url)
- # open the download source
- if splittype(url)[0]:
- try:
- src = urlopen(url)
- except IOError:
- return None
- remote_name = src.geturl().split('/')[-1]
- elif os.path.isfile(url):
- os.symlink(url, filename)
- return filename, filehash(filename)
- src = open(url, 'r')
- remote_name = url.split('/')[-1]
- else:
+ try:
+ src = urlopen(url)
+ except IOError:
return None
if filename == None:
- filename = remote_name
+ filename = src.geturl().split('/')[-1]
- # if we have a source to download, open a destination and copy the data
- if src:
+ try:
dst = open(filename, 'wb');
- if verbose:
- print "Downloading %s from %s" % (filename, url)
- while True:
- block = src.read()
- if block == '':
- break;
- dst.write(block)
- dst.close()
+ except IOError:
+ src.close()
+ return None
+
+ while True:
+ block = src.read()
+ if block == '':
+ break;
+ dst.write(block)
+
+ src.close()
+ dst.close()
# return the name of the file that we downloaded the data to.
return filename
-def download_paths(search, filename):
- tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
- if tmp:
- search += tmp.split(' ')
- base = os.path.basename(filename)
-
+def download_paths(search, filename, url):
urls = list()
- for path in search:
- urls.append(path+'/'+base)
+ if filename != None:
+ tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
+ if tmp:
+ search += tmp.split(' ')
+
+ file = os.path.basename(filename)
+
+ urls = [ base + '/' + file for base in search ]
+
+ # filename should always be first
+ if filename in urls:
+ urls.remove(filename)
+ urls.insert(0, filename)
+
+ # command line url is a fallback, so it's last
+ if url != None and url not in urls:
+ urls.append(url)
return urls
def usage():
- print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1])
+ print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
sys.exit(1)
def main():
- from urllib import splittype, urlopen
import getopt
- import sys
# FLUSH STDOUT
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
- verbose = False
- filename = None
- url = None
- hash = None
- search = list()
+ file_arg = None
+ link_arg = False
+ hash_arg = None
+ url_arg = None
+ search_list = list()
try:
- opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v",
- ["file=", "hash=", "search=", "url=", "verbose"])
+ opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
+ ["file=", "link", "hash=", "search=", "url="])
except getopt.GetoptError, err:
print str(err)
usage()
for opt, arg in opts:
if opt in [ "-f", "--file" ]:
- filename = arg
+ file_arg = arg
+ elif opt in [ "-l", "--link" ]:
+ link_arg = True
elif opt in [ "-h", "--hash" ]:
- hash = arg
+ hash_arg = arg
elif opt in [ "-s", "--search" ]:
- search.append(arg)
+ search_list.append(arg)
elif opt in [ "-u", "--url" ]:
- url = arg
- elif opt in [ "-v", "--verbose" ]:
- verbose = True
+ url_arg = arg
else:
assert False, "unknown option"
- if url == None:
+ if url_arg == None:
usage()
- print "Fetching %s" % filename
+ for url in download_paths(search_list, file_arg, url_arg):
+ print "Source %s..." % url,
- if validate(filename, hash):
- print "\tcurrent copy is ok, nothing to do"
- sys.exit(0)
+ scheme, path = splittype(url)
+ name = file_arg
- # generate a list of URLS to search for a suitable download
- urls = download_paths(search, filename)
- urls.append(url)
-
- for url in urls:
- print "\tTrying %s..." % url,
-
- if os.path.isfile(url):
- os.symlink(url, filename)
- elif splittype(url)[0]:
- if download(url, filename) == None:
+ if scheme in [ None, 'file' ]:
+ if os.path.exists(path) == False:
+ print "not found, skipping"
+ continue
+ elif name != path:
+ if link_arg == False:
+ print "copying...",
+ shutil.copy2(path, name)
+ else:
+ print "linking...",
+ os.symlink(path, name)
+ else:
+ pass
+ elif scheme in [ 'http', 'https', 'ftp' ]:
+ print "downloading...",
+ name = download(url, file_arg)
+ if name == None:
print "failed"
continue
- print "retrieved...",
-
- if validate(filename, hash):
- print "validated"
+ print "validating...",
+ if validate(name, hash_arg):
+ print "ok"
sys.exit(0)
else:
print "corrupt"
try:
- os.remove(filename)
+ os.remove(name)
except OSError:
pass