Add a fetch utility to do more than just download like wget does
authorNorm Jacobs <Norm.Jacobs@Sun.COM>
Thu, 10 Jun 2010 15:16:47 -0500
changeset 18 6da6bd662127
parent 17 56b936d4786f
child 19 a82b10a74fd1
Add a fetch utility to do more than just download like wget does
components/file/Makefile
components/lcms/Makefile
components/pycups/Makefile
make-rules/prep.mk
tools/fetch.py
--- a/components/file/Makefile	Thu Jun 10 15:08:47 2010 -0500
+++ b/components/file/Makefile	Thu Jun 10 15:16:47 2010 -0500
@@ -30,7 +30,7 @@
 COMPONENT_SRC=		$(COMPONENT_NAME)-5.04
 COMPONENT_COPYRIGHT=	$(COMPONENT_SRC)/COPYING
 COMPONENT_ARCHIVE=	$(COMPONENT_SRC).tar.gz
-COMPONENT_ARCHIVE_HASH=	56ddf7135471aa656334ed8fefe1112bcccc2cc3
+COMPONENT_ARCHIVE_HASH=	sha1:56ddf7135471aa656334ed8fefe1112bcccc2cc3
 COMPONENT_ARCHIVE_URL=	ftp://ftp.fu-berlin.de/unix/tools/file/$(COMPONENT_ARCHIVE)
 
 include ../../make-rules/prep.mk
--- a/components/lcms/Makefile	Thu Jun 10 15:08:47 2010 -0500
+++ b/components/lcms/Makefile	Thu Jun 10 15:16:47 2010 -0500
@@ -30,7 +30,7 @@
 COMPONENT_SRC=		$(COMPONENT_NAME)-$(COMPONENT_VERSION)
 COMPONENT_COPYRIGHT=	$(COMPONENT_SRC)/COPYING
 COMPONENT_ARCHIVE=	$(COMPONENT_SRC).tar.gz
-COMPONENT_ARCHIVE_HASH=	d5b075ccffc0068015f74f78e4bc39138bcfe2d4
+COMPONENT_ARCHIVE_HASH=	sha1:d5b075ccffc0068015f74f78e4bc39138bcfe2d4
 COMPONENT_ARCHIVE_URL=	http://downloads.sourceforge.net/project/lcms/$(COMPONENT_NAME)/$(COMPONENT_VERSION)/$(COMPONENT_ARCHIVE)?use_mirror=voxel
 
 include ../../make-rules/prep.mk
--- a/components/pycups/Makefile	Thu Jun 10 15:08:47 2010 -0500
+++ b/components/pycups/Makefile	Thu Jun 10 15:16:47 2010 -0500
@@ -30,7 +30,7 @@
 COMPONENT_SRC=		$(COMPONENT_NAME)-$(COMPONENT_VERSION)
 COMPONENT_COPYRIGHT=	$(COMPONENT_SRC)/COPYING
 COMPONENT_ARCHIVE=	$(COMPONENT_SRC).tar.bz2
-COMPONENT_ARCHIVE_HASH=	3d58a0f7b4a1b6a2880baf9ef5e14eae461d0982
+COMPONENT_ARCHIVE_HASH=	sha1:3d58a0f7b4a1b6a2880baf9ef5e14eae461d0982
 COMPONENT_ARCHIVE_URL=	http://cyberelk.net/tim/data/pycups/$(COMPONENT_ARCHIVE)
 
 include ../../make-rules/prep.mk
--- a/make-rules/prep.mk	Thu Jun 10 15:08:47 2010 -0500
+++ b/make-rules/prep.mk	Thu Jun 10 15:16:47 2010 -0500
@@ -22,13 +22,14 @@
 #
 
 UNPACK =	$(TOOLS)/unpack.py
+FETCH =		$(TOOLS)/fetch.py
 
 ARCHIVES += $(COMPONENT_ARCHIVE)
 CLEAN_PATHS += $(COMPONENT_SRC)
 CLOBBER_PATHS += $(COMPONENT_ARCHIVE)
 
 PATCHES =	$(shell find . -type f -name '*.patch' | \
-			 sed -e 's;^\./;;' | sort)
+			 sed -e 's;^\./;;' | grep -v $(COMPONENT_SRC) | sort)
 STAMPS =	$(PATCHES:%=$(COMPONENT_SRC)/.%ed)
 
 $(COMPONENT_SRC)/.%ed:	%
@@ -36,7 +37,9 @@
 	$(TOUCH) $@
 
 $(COMPONENT_ARCHIVE):
-	wget $(COMPONENT_ARCHIVE_URL)
+	$(FETCH) --file $@ \
+		$(COMPONENT_ARCHIVE_URL:%=--url %) \
+		$(COMPONENT_ARCHIVE_HASH:%=--hash %)
 
 $(COMPONENT_SRC)/.unpacked:	$(COMPONENT_ARCHIVE)
 	$(UNPACK) $(UNPACK_ARGS) $(COMPONENT_ARCHIVE)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/fetch.py	Thu Jun 10 15:16:47 2010 -0500
@@ -0,0 +1,192 @@
+#!/usr/bin/python2.6
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+# Copyright (c) 2010, Oracle and/or it's affiliates.  All rights reserved.
+#
+#
+# fetch.py - a file download utility
+#
+#  A simple program similiar to wget(1), but handles local file copy, ignores
+#  directories, and verifies file hashes.
+#
+
+import os
+import sys
+
+def validate(filename, hash):
+	import hashlib
+
+	try:
+		file = open(filename, 'r')
+	except IOError:
+		return False
+
+	if (hash == None):
+		return True
+
+	algorithm, value = hash.split(':')
+	try:
+		m = hashlib.new(algorithm)
+	except ValueError:
+		return False
+
+	while True:
+		block = file.read()
+		m.update(block)
+		if block == '':
+			break
+
+	return m.hexdigest() == value
+
+def download(url, filename = None, hash = None, verbose = False):
+	from urllib import splittype, urlopen
+	import hashlib
+
+	src = None
+	filename = None
+
+	#print "Downloading %s from %s" % (filename, url)
+
+	# open the download source
+	if splittype(url)[0]:
+		try:
+			src = urlopen(url)
+		except IOError:
+			return None
+		remote_name = src.geturl().split('/')[-1]
+	elif os.path.isfile(url):
+		os.symlink(url, filename)
+		return filename, filehash(filename)
+		src = open(url, 'r')
+		remote_name = url.split('/')[-1]
+	else:
+		return None
+
+	if filename == None:
+		filename = remote_name
+
+	# if we have a source to download, open a destination and copy the data
+	if src:
+		dst = open(filename, 'wb');
+		if verbose:
+			print "Downloading %s from %s" % (filename, url)
+		while True:
+			block = src.read()
+			if block == '':
+				break;
+			dst.write(block)
+		dst.close()
+
+	# return the name of the file that we downloaded the data to.
+	return filename
+
+def download_paths(search, filename):
+	tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
+	if tmp:
+		search += tmp.split(' ')
+	base = os.path.basename(filename)
+
+	urls = list()
+
+	for path in search:
+		urls.append(path+'/'+base)
+
+	return urls
+
+def usage():
+	print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1])
+	sys.exit(1)
+
+def main():
+	from urllib import splittype, urlopen
+	import getopt
+	import sys
+
+	# FLUSH STDOUT 
+	sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+	verbose = False
+	filename = None
+	url = None
+	hash = None
+	search = list()
+
+	try:
+		opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v",
+			["file=", "hash=", "search=", "url=", "verbose"])
+	except getopt.GetoptError, err:
+		print str(err)
+		usage()
+
+	for opt, arg in opts:
+		if opt in [ "-f", "--file" ]:
+			filename = arg
+		elif opt in [ "-h", "--hash" ]:
+			hash = arg
+		elif opt in [ "-s", "--search" ]:
+			search.append(arg)
+		elif opt in [ "-u", "--url" ]:
+			url = arg
+		elif opt in [ "-v", "--verbose" ]:
+			verbose = True
+		else:
+			assert False, "unknown option"
+
+	if url == None:
+		usage()
+
+	print "Fetching %s" % filename
+
+	if validate(filename, hash):
+		print "\tcurrent copy is ok, nothing to do"
+		sys.exit(0)
+
+	# generate a list of URLS to search for a suitable download
+	urls = download_paths(search, filename)
+	urls.append(url)
+
+	for url in urls:
+		print "\tTrying %s..." % url,
+
+		if os.path.isfile(url):
+			os.symlink(url, filename)
+		elif splittype(url)[0]:
+			if download(url, filename) == None:
+				print "failed"
+				continue
+
+		print "retrieved...",
+
+		if validate(filename, hash):
+			print "validated"
+			sys.exit(0)
+		else:
+			print "corrupt"
+
+		try:
+			os.remove(filename)
+		except OSError:
+			pass
+
+	sys.exit(1)
+
+if __name__ == "__main__":
+	main()