20183619 userland should support PGP signatures
authorDanek Duvall <danek.duvall@oracle.com>
Tue, 09 Dec 2014 11:43:38 -0800
changeset 3533 0b8107a40da7
parent 3531 18f175f98e0e
child 3534 75e5dba8a315
20183619 userland should support PGP signatures
components/mercurial/Makefile
components/openstack/cinder/Makefile
components/openstack/glance/Makefile
components/openstack/heat/Makefile
components/openstack/horizon/Makefile
components/openstack/keystone/Makefile
components/openstack/neutron/Makefile
components/openstack/nova/Makefile
components/openstack/swift/Makefile
doc/makefile-variables.txt
make-rules/prep.mk
tools/.gnupg/gpg.conf
tools/.gnupg/pubring.gpg
tools/userland-fetch
--- a/components/mercurial/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/mercurial/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:ed387cc0e9754ec59bd4a390639b5a4ea11698ae4413243f8b4a2d6d48b3b7d6
 COMPONENT_ARCHIVE_URL=	http://www.selenic.com/mercurial/release/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_BUGDB=	utility/hg
 
 # Fails to build: not Python 3 ready.
--- a/components/openstack/cinder/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/cinder/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:a2740f0a0481139ae21cdb0868bebcce01b9f19832439b7f3056435e75791194
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/cinder
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/glance/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/glance/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:8766f8d198ec513c46519f1c44f99a4845ba3c04e7b7c41893cb3d5a7c2a9a28
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/glance
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/heat/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/heat/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:5e1f437f75fd831bc6cbd23986d8e60a4008cf7a0a775e7a7405d56b335b1800
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/heat
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/horizon/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/horizon/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:de9b87ee62d8b28792399be0fc867ba99618eaaad289cf9842b5c7084e12620f
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	utility/horizon
 IPS_COMPONENT_VERSION=  0.$(COMPONENT_VERSION)
--- a/components/openstack/keystone/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/keystone/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:0d27a32c6c211706f8b13aafe2fd51c7ddbea97897be90663fd8c2527ef56032
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/keystone
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/neutron/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/neutron/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:116cc2ce9f2f5b2dcbd5a314d78a496b180a148dadd02a076ff664b0f3c20cd3
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/neutron
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/nova/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/nova/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:02902cb65b5adb0419c69cdb03ea2a0cfdfe8f7df342be44f3760d66cdecb61e
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUGDB=	service/nova
 IPS_COMPONENT_VERSION=	0.$(COMPONENT_VERSION) 
--- a/components/openstack/swift/Makefile	Tue Dec 09 18:47:43 2014 +0100
+++ b/components/openstack/swift/Makefile	Tue Dec 09 11:43:38 2014 -0800
@@ -32,6 +32,7 @@
 COMPONENT_ARCHIVE_HASH=	\
     sha256:9a841225c3a00a93a15a160102d3f7116f2f1ba98ebffedfe641747844e14889
 COMPONENT_ARCHIVE_URL=	http://launchpad.net/$(COMPONENT_NAME)/$(COMPONENT_CODENAME)/$(COMPONENT_VERSION)/+download/$(COMPONENT_ARCHIVE)
+COMPONENT_SIG_URL=	$(COMPONENT_ARCHIVE_URL).asc
 COMPONENT_PROJECT_URL=	http://www.openstack.org/
 COMPONENT_BUG_DB=	service/swift
 
--- a/doc/makefile-variables.txt	Tue Dec 09 18:47:43 2014 +0100
+++ b/doc/makefile-variables.txt	Tue Dec 09 11:43:38 2014 -0800
@@ -14,6 +14,16 @@
   field of `sha256sum $(COMPONENT_ARCHIVE)`.
 * COMPONENT_ARCHIVE_URL is where the archive can be downloaded from.  This is
   typically constructed from $(COMPONENT_PROJECT_URL) and $(COMPONENT_ARCHIVE).
+* COMPONENT_SIG_URL is the URL where the PGP signature for $(COMPONENT_ARCHIVE)
+  can be found.  This can be used in addition to the hash in
+  $(COMPONENT_ARCHIVE_HASH) to verify the correctness of the archive.  If
+  COMPONENT_SIG_URL is present, then COMPONENT_ARCHIVE_HASH needn't be, but its
+  presence is strongly encouraged to ensure that the archive contents don't
+  change silently.  If the signature results in a new key being added to
+  tools/.gnupg/pubring.pgp, then as part of your code review, please show the
+  diffs of the text version of the file by running
+      gpg2 --homedir $WS/tools/.gnupg --fingerprint
+  both before and after the change.
 * COMPONENT_BUGDB is the lower-case rendering of the BugDB cat/subcat.
 
 These two are both initialized in make-rules/shared-macros.mk rather than any
--- a/make-rules/prep.mk	Tue Dec 09 18:47:43 2014 +0100
+++ b/make-rules/prep.mk	Tue Dec 09 11:43:38 2014 -0800
@@ -18,7 +18,7 @@
 #
 # CDDL HEADER END
 #
-# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
 #
 
 # One must do all unpack and patch in sequence.
@@ -67,7 +67,8 @@
 $$(USERLAND_ARCHIVES)$$(COMPONENT_ARCHIVE$(1)):	Makefile
 	$$(FETCH) --file [email protected] \
 		$$(COMPONENT_ARCHIVE_URL$(1):%=--url %) \
-		$$(COMPONENT_ARCHIVE_HASH$(1):%=--hash %)
+		$$(COMPONENT_ARCHIVE_HASH$(1):%=--hash %) \
+		$$(COMPONENT_SIG_URL$(1):%=--sigurl %)
 	$$(TOUCH) [email protected]
 endef
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/.gnupg/gpg.conf	Tue Dec 09 11:43:38 2014 -0800
@@ -0,0 +1,9 @@
+# When verifying a signature made from a subkey, ensure that the cross
+# certification "back signature" on the subkey is present and valid.
+# This protects against a subtle attack against subkeys that can sign.
+# Defaults to --no-require-cross-certification.  However for new
+# installations it should be enabled.
+require-cross-certification
+
+keyserver hkp://keys.gnupg.net
+keyserver-options honor-http-proxy auto-key-retrieve
Binary file tools/.gnupg/pubring.gpg has changed
--- a/tools/userland-fetch	Tue Dec 09 18:47:43 2014 +0100
+++ b/tools/userland-fetch	Tue Dec 09 11:43:38 2014 -0800
@@ -19,18 +19,20 @@
 #
 # CDDL HEADER END
 #
-# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
 #
 #
-# fetch.py - a file download utility
+# userland-fetch - a file download utility
 #
 #  A simple program similiar to wget(1), but handles local file copy, ignores
 #  directories, and verifies file hashes.
 #
 
+import errno
 import os
 import sys
 import shutil
+import subprocess
 from urllib import splittype
 from urllib2 import urlopen
 import hashlib
@@ -43,9 +45,51 @@
 		print str(message) + " (" + str(code) + ")"
 	except:
 		print str(e)
+
+def validate_signature(path, signature):
+	"""Given paths to a file and a detached PGP signature, verify that
+	the signature is valid for the file.  Current configuration allows for
+	unrecognized keys to be downloaded as necessary."""
+
+	# Find the root of the repo so that we can point GnuPG at the right
+	# configuration and keyring.
+	proc = subprocess.Popen(["hg", "root"], stdout=subprocess.PIPE)
+	proc.wait()
+	if proc.returncode != 0:
+		return False
+	out, err = proc.communicate()
+	gpgdir = os.path.join(out.strip(), "tools", ".gnupg")
+
+        # Skip the permissions warning: none of the information here is private,
+        # so not having to worry about getting mercurial keeping the directory
+        # unreadable is just simplest.
+	try:
+		proc = subprocess.Popen(["gpg2", "--verify",
+		    "--no-permission-warning", "--homedir", gpgdir, signature,
+		    path], stdin=open("/dev/null"),
+		    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	except OSError as e:
+		# If the executable simply couldn't be found, just skip the
+		# validation.
+		if e.errno == errno.ENOENT:
+			return False
+		raise
+
+        proc.wait()
+        if proc.returncode != 0:
+		# Only print GnuPG's output when there was a problem.
+                print proc.stdout.read()
+                return False
+        return True
 	
 def validate(file, hash):
-	algorithm, hashvalue = hash.split(':')
+	"""Given a file-like object and a hash string, verify that the hash
+	matches the file contents."""
+
+	try:
+		algorithm, hashvalue = hash.split(':')
+	except:
+		algorithm = "sha256"
 
 	# force migration away from sha1
 	if algorithm == "sha1":
@@ -69,6 +113,9 @@
 	return "%s:%s" % (algorithm, m.hexdigest())
 
 def validate_container(filename, hash):
+	"""Given a file path and a hash string, verify that the hash matches the
+	file contents."""
+
 	try:
 		file = open(filename, 'r')
 	except IOError as e:
@@ -78,6 +125,9 @@
 
 
 def validate_payload(filename, hash):
+	"""Given a file path and a hash string, verify that the hash matches the
+	payload (uncompressed content) of the file."""
+
 	import re
 	import gzip
 	import bz2
@@ -101,18 +151,25 @@
 	return validate(file, hash)
 
 
-def download(url, filename = None):
+def download(url, filename=None, quiet=False):
+	"""Download the content at the given URL to the given filename
+	(defaulting to the basename of the URL if not given.  If 'quiet' is
+	True, throw away any error messages.  Returns the name of the file to
+	which the content was donloaded."""
+
 	src = None
 
 	try:
 		src = urlopen(url)
 	except IOError as e:
-		printIOError(e, "Can't open url " + url)
+		if not quiet:
+			printIOError(e, "Can't open url " + url)
 		return None
 
 	# 3xx, 4xx and 5xx (f|ht)tp codes designate unsuccessfull action
 	if 3 <= int(src.getcode()/100) <= 5:
-		print "Error code: " + str(src.getcode())
+		if not quiet:
+			print "Error code: " + str(src.getcode())
 		return None
 
 	if filename == None:
@@ -121,7 +178,8 @@
 	try:
 		dst = open(filename, 'wb');
 	except IOError as e:
-		printIOError(e, "Can't open file " + filename + " for writing")
+		if not quiet:
+			printIOError(e, "Can't open file " + filename + " for writing")
 		src.close()
 		return None
 
@@ -138,6 +196,12 @@
 	return filename
 
 def download_paths(search, filename, url):
+	"""Returns a list of URLs where the file 'filename' might be found,
+	using 'url', 'search', and $DOWNLOAD_SEARCH_PATH as places to look.
+
+	If 'filename' is None, then the list will simply contain 'url'.
+	"""
+
 	urls = list()
 
 	if filename != None:
@@ -160,8 +224,52 @@
 
 	return urls
 
+def download_from_paths(search_list, file_arg, url, link_arg, quiet=False):
+	"""Attempts to download a file from a number of possible locations.
+	Generates a list of paths where the file ends up on the local
+	filesystem.  This is a generator because while a download might be
+	successful, the signature or hash may not validate, and the caller may
+	want to try again from the next location.  The 'link_arg' argument is a
+	boolean which, when True, specifies that if the source is not a remote
+	URL and not already found where it should be, to make a symlink to the
+	source rather than copying it.
+	"""
+	for url in download_paths(search_list, file_arg, url):
+		if not quiet:
+			print "Source %s..." % url,
+
+		scheme, path = splittype(url)
+		name = file_arg
+
+		if scheme in [ None, 'file' ]:
+			if os.path.exists(path) == False:
+				if not quiet:
+					print "not found, skipping file copy"
+				continue
+			elif name and name != path:
+				if link_arg == False:
+					if not quiet:
+						print "\n    copying..."
+					shutil.copy2(path, name)
+				else:
+					if not quiet:
+						print "\n    linking..."
+					os.symlink(path, name)
+		elif scheme in [ 'http', 'https', 'ftp' ]:
+			if not quiet:
+				print "\n    downloading...",
+			name = download(url, file_arg, quiet)
+			if name == None:
+				if not quiet:
+					print "failed"
+				continue
+
+		yield name
+
 def usage():
-	print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
+	print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] " \
+          "[-s|--search (search-dir)] [-S|--sigurl (signature-url)] --url (url)" % \
+          (sys.argv[0].split('/')[-1])
 	sys.exit(1)
 
 def main():
@@ -174,11 +282,12 @@
 	link_arg = False
 	hash_arg = None
 	url_arg = None
+	sig_arg = None
 	search_list = list()
 
 	try:
-		opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
-			["file=", "link", "hash=", "search=", "url="])
+                opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:S:u:",
+			["file=", "link", "hash=", "search=", "sigurl=", "url="])
 	except getopt.GetoptError, err:
 		print str(err)
 		usage()
@@ -192,6 +301,8 @@
 			hash_arg = arg
 		elif opt in [ "-s", "--search" ]:
 			search_list.append(arg)
+		elif opt in [ "-S", "--sigurl" ]:
+			sig_arg = arg
 		elif opt in [ "-u", "--url" ]:
 			url_arg = arg
 		else:
@@ -200,56 +311,69 @@
 	if url_arg == None:
 		usage()
 
-	for url in download_paths(search_list, file_arg, url_arg):
-		print "Source %s..." % url,
-
-		scheme, path = splittype(url)
-		name = file_arg
+	for name in download_from_paths(search_list, file_arg, url_arg, link_arg):
+		print "\n    validating signature...",
 
-		if scheme in [ None, 'file' ]:
-			if os.path.exists(path) == False:
-				print "not found, skipping file copy"
-				continue
-			elif name != path:
-				if link_arg == False:
-					print "\n    copying..."
-					shutil.copy2(path, name)
+		sig_valid = False
+		if not sig_arg:
+			print "skipping (no signature URL)"
+		else:
+			# Put the signature file in the same directory as the
+			# file we're downloading.
+			sig_file = os.path.join(
+			    os.path.dirname(file_arg),
+			    os.path.basename(sig_arg))
+			# Validate with the first signature we find.
+			for sig_file in download_from_paths(search_list, sig_file,
+			    sig_arg, link_arg, True):
+				if sig_file:
+					if validate_signature(name, sig_file):
+						print "ok"
+						sig_valid = True
+					else:
+						print "failed"
+					break
 				else:
-					print "\n    linking..."
-					os.symlink(path, name)
+					continue
 			else:
-				pass
-		elif scheme in [ 'http', 'https', 'ftp' ]:
-			print "\n    downloading...",
-			name = download(url, file_arg)
-			if name == None:
-				print "failed"
-				continue
+				print "failed (couldn't fetch signature)"
+
+		print "    validating hash...",
+		realhash = validate_container(name, hash_arg)
 
-		print "\n    validating...",
-		if hash_arg == None:
+		if not hash_arg:
 			print "skipping (no hash)"
-			sys.exit(0)
-			
-		realhash = validate_container(name, hash_arg)
-		if realhash == hash_arg:
+			print "hash is: %s" % realhash
+		elif realhash == hash_arg:
 			print "ok"
-			sys.exit(0)
 		else:
 			payloadhash = validate_payload(name, hash_arg)
 			if payloadhash == hash_arg:
 				print "ok"
-				sys.exit(0)
-			print "corruption detected"
-			print "    expected: %s" % hash_arg
-			print "    actual:   %s" % realhash
-			print "    payload:  %s" % payloadhash
+			else:
+				# If the signature validated, then we assume
+				# that the expected hash is just a typo, but we
+				# warn just in case.
+				if sig_valid:
+					print "invalid hash!"
+				else:
+					print "corruption detected"
 
-		try:
-			os.remove(name)
-		except OSError:
-			pass
+				print "    expected: %s" % hash_arg
+				print "    actual:   %s" % realhash
+				print "    payload:  %s" % payloadhash
 
+				# An invalid hash shouldn't cause us to remove
+				# the target file if the signature was valid.
+				if not sig_valid:
+					try:
+						os.remove(name)
+					except OSError:
+						pass
+
+					continue
+
+		sys.exit(0)
 	sys.exit(1)
 
 if __name__ == "__main__":