tools/userland-fetch
author Norm Jacobs <Norm.Jacobs@Oracle.COM>
Wed, 23 Feb 2011 10:37:11 -0800
changeset 99 c15c9099bb44
parent 58 d8024c042a00
child 135 5c6feb0253fa
permissions -rwxr-xr-x
6841644 OpenSolaris Python should support gdbm 6991811 Migrate Mercurial (Hg) to Python 2.6 7009592 PyOpenSSL should be updated from 0.8 to 0.11 7016735 move mercurial from sfw to userland 7018363 Userland LD_OPTIONS should include non-executable data mapfiles as well 7019410 python 2.6 should move to userland 7020772 userland build should provide more macros for common flags 7021204 userland runpath checking should relax slightly.

#!/usr/bin/python2.6
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
#
#
# fetch.py - a file download utility
#
#  A simple program similiar to wget(1), but handles local file copy, ignores
#  directories, and verifies file hashes.
#

import os
import sys
import shutil
from urllib import splittype, urlopen

def validate(filename, hash):
	import hashlib

	try:
		file = open(filename, 'r')
	except IOError:
		return False

	algorithm, hashvalue = hash.split(':')
	try:
		m = hashlib.new(algorithm)
	except ValueError:
		return False

	while True:
		block = file.read()
		m.update(block)
		if block == '':
			break

	return "%s:%s" % (algorithm, m.hexdigest())

def download(url, filename = None):
	src = None

	try:
		src = urlopen(url)
	except IOError:
		return None

	if filename == None:
		filename = src.geturl().split('/')[-1]

	try:
		dst = open(filename, 'wb');
	except IOError:
		src.close()
		return None

	while True:
		block = src.read()
		if block == '':
			break;
		dst.write(block)

	src.close()
	dst.close()

	# return the name of the file that we downloaded the data to.
	return filename

def download_paths(search, filename, url):
	urls = list()

	if filename != None:
		tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
		if tmp:
			search += tmp.split(' ')

		file = os.path.basename(filename)

		urls = [ base + '/' + file for base in search ]

		# filename should always be first
		if filename in urls:
			urls.remove(filename)
		urls.insert(0, filename)

	# command line url is a fallback, so it's last
	if url != None and url not in urls:
		urls.append(url)

	return urls

def usage():
	print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
	sys.exit(1)

def main():
	import getopt

	# FLUSH STDOUT 
	sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

	file_arg = None
	link_arg = False
	hash_arg = None
	url_arg = None
	search_list = list()

	try:
		opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
			["file=", "link", "hash=", "search=", "url="])
	except getopt.GetoptError, err:
		print str(err)
		usage()

	for opt, arg in opts:
		if opt in [ "-f", "--file" ]:
			file_arg = arg
		elif opt in [ "-l", "--link" ]:
			link_arg = True
		elif opt in [ "-h", "--hash" ]:
			hash_arg = arg
		elif opt in [ "-s", "--search" ]:
			search_list.append(arg)
		elif opt in [ "-u", "--url" ]:
			url_arg = arg
		else:
			assert False, "unknown option"

	if url_arg == None:
		usage()

	for url in download_paths(search_list, file_arg, url_arg):
		print "Source %s..." % url,

		scheme, path = splittype(url)
		name = file_arg

		if scheme in [ None, 'file' ]:
			if os.path.exists(path) == False:
				print "not found, skipping file copy"
				continue
			elif name != path:
				if link_arg == False:
					print "\n    copying..."
					shutil.copy2(path, name)
				else:
					print "\n    linking..."
					os.symlink(path, name)
			else:
				pass
		elif scheme in [ 'http', 'https', 'ftp' ]:
			print "\n    downloading...",
			name = download(url, file_arg)
			if name == None:
				print "failed"
				continue

		print "\n    validating...",
		if hash_arg == None:
			print "skipping (no hash)"
			sys.exit(0)
			
		realhash = validate(name, hash_arg)
		if realhash == hash_arg:
			print "ok"
			sys.exit(0)
		else:
			print "corruption detected"
			print "    expected: %s" % hash_arg
			print "    actual:   %s" % realhash

		try:
			os.remove(name)
		except OSError:
			pass

	sys.exit(1)

if __name__ == "__main__":
	main()