# HG changeset patch # User Danek Duvall # Date 1461792517 25200 # Node ID ce31ce66da02bbf2a0b32ddd4e8529aeca482373 # Parent 720358c56a5ddda27d366f34962ac6cecf7edd8d 23183311 pypi archive url schemes have changed incompatibly diff -r 720358c56a5d -r ce31ce66da02 components/openstack/horizon/Makefile --- a/components/openstack/horizon/Makefile Thu Apr 21 12:46:25 2016 -0400 +++ b/components/openstack/horizon/Makefile Wed Apr 27 14:28:37 2016 -0700 @@ -75,185 +75,185 @@ # components are delivered in the Horizon package itself. # COMPONENT_NAME_1 = XStatic -VERSION_XSTATIC = 1.0.1 -COMPONENT_SRC_1 = XStatic-$(VERSION_XSTATIC) +COMPONENT_VERSION_1 = 1.0.1 +COMPONENT_SRC_1 = XStatic-$(COMPONENT_VERSION_1) COMPONENT_ARCHIVE_1 = $(COMPONENT_SRC_1).tar.gz COMPONENT_ARCHIVE_HASH_1 = \ sha256:0ec93d7c66ebb9e0d31b664753437dc8634cb66e13310cb47c9eb1e0bc66d726 COMPONENT_ARCHIVE_URL_1 = $(call pypi_url,1) COMPONENT_NAME_2 = XStatic-Angular -VERSION_ANGULAR = 1.3.7.0 -COMPONENT_SRC_2 = XStatic-Angular-$(VERSION_ANGULAR) +COMPONENT_VERSION_2 = 1.3.7.0 +COMPONENT_SRC_2 = XStatic-Angular-$(COMPONENT_VERSION_2) COMPONENT_ARCHIVE_2 = $(COMPONENT_SRC_2).tar.gz COMPONENT_ARCHIVE_HASH_2 = \ sha256:7cad216b226399cbe8d2909ed5b9f28d724907b9c9e1e078e6e25d320a3d5dd7 COMPONENT_ARCHIVE_URL_2 = $(call pypi_url,2) -PKG_VARS += VERSION_ANGULAR +PKG_MACROS += VERSION_ANGULAR=$(COMPONENT_VERSION_2) COMPONENT_NAME_3 = XStatic-Bootstrap-Datepicker -VERSION_BOOTSTRAP_DP= 1.3.1.0 -COMPONENT_SRC_3 = XStatic-Bootstrap-Datepicker-$(VERSION_BOOTSTRAP_DP) +COMPONENT_VERSION_3 = 1.3.1.0 +COMPONENT_SRC_3 = XStatic-Bootstrap-Datepicker-$(COMPONENT_VERSION_3) COMPONENT_ARCHIVE_3 = $(COMPONENT_SRC_3).tar.gz COMPONENT_ARCHIVE_HASH_3 = \ sha256:9edc9b77501001fcee9fbf4bf0a722c263efd928ef928b40081a8269fdd9a944 COMPONENT_ARCHIVE_URL_3 = $(call pypi_url,3) -PKG_VARS += VERSION_BOOTSTRAP_DP +PKG_MACROS += VERSION_BOOTSTRAP_DP=$(COMPONENT_VERSION_3) COMPONENT_NAME_4 = XStatic-Bootstrap-SCSS -VERSION_BOOTSTRAP_SCSS= 3.2.0.0 -COMPONENT_SRC_4 = XStatic-Bootstrap-SCSS-$(VERSION_BOOTSTRAP_SCSS) +COMPONENT_VERSION_4 = 3.2.0.0 +COMPONENT_SRC_4 = XStatic-Bootstrap-SCSS-$(COMPONENT_VERSION_4) COMPONENT_ARCHIVE_4 = $(COMPONENT_SRC_4).tar.gz COMPONENT_ARCHIVE_HASH_4 = \ sha256:7e9858737e2e6aac921ec7a3fca627e522901c3061611e154ebc0b8a892c7018 COMPONENT_ARCHIVE_URL_4 = $(call pypi_url,4) -PKG_VARS += VERSION_BOOTSTRAP_SCSS +PKG_MACROS += VERSION_BOOTSTRAP_SCSS=$(COMPONENT_VERSION_4) COMPONENT_NAME_5 = XStatic-D3 -VERSION_D3 = 3.1.6.2 -COMPONENT_SRC_5 = XStatic-D3-$(VERSION_D3) +COMPONENT_VERSION_5 = 3.1.6.2 +COMPONENT_SRC_5 = XStatic-D3-$(COMPONENT_VERSION_5) COMPONENT_ARCHIVE_5 = $(COMPONENT_SRC_5).tar.gz COMPONENT_ARCHIVE_HASH_5 = \ sha256:46fe521f8dad99f5e20f6702180510c37b81d11f1d78119d308fcec31381f90e COMPONENT_ARCHIVE_URL_5 = $(call pypi_url,5) -PKG_VARS += VERSION_D3 +PKG_MACROS += VERSION_D3=$(COMPONENT_VERSION_5) COMPONENT_NAME_6 = XStatic-Font-Awesome -VERSION_FONT_AWESOME = 4.3.0.0 -COMPONENT_SRC_6 = XStatic-Font-Awesome-$(VERSION_FONT_AWESOME) +COMPONENT_VERSION_6 = 4.3.0.0 +COMPONENT_SRC_6 = XStatic-Font-Awesome-$(COMPONENT_VERSION_6) COMPONENT_ARCHIVE_6 = $(COMPONENT_SRC_6).tar.gz COMPONENT_ARCHIVE_HASH_6 = \ sha256:d2c0548990a22744ad20013e1d095de8ba1ea56b1878d8393a721205b03885c3 COMPONENT_ARCHIVE_URL_6 = $(call pypi_url,6) -PKG_VARS += VERSION_FONT_AWESOME +PKG_MACROS += VERSION_FONT_AWESOME=$(COMPONENT_VERSION_6) COMPONENT_NAME_7 = XStatic-Hogan -VERSION_HOGAN = 2.0.0.2 -COMPONENT_SRC_7 = XStatic-Hogan-$(VERSION_HOGAN) +COMPONENT_VERSION_7 = 2.0.0.2 +COMPONENT_SRC_7 = XStatic-Hogan-$(COMPONENT_VERSION_7) COMPONENT_ARCHIVE_7 = $(COMPONENT_SRC_7).tar.gz COMPONENT_ARCHIVE_HASH_7 = \ sha256:5941bc7fb2a09916b8837848e6fc2a13b2dfc271811e9b522c61e1337d5fc2bd COMPONENT_ARCHIVE_URL_7 = $(call pypi_url,7) -PKG_VARS += VERSION_HOGAN +PKG_MACROS += VERSION_HOGAN=$(COMPONENT_VERSION_7) COMPONENT_NAME_8 = XStatic-jQuery -VERSION_JQUERY = 1.10.2.1 -COMPONENT_SRC_8 = XStatic-jQuery-$(VERSION_JQUERY) +COMPONENT_VERSION_8 = 1.10.2.1 +COMPONENT_SRC_8 = XStatic-jQuery-$(COMPONENT_VERSION_8) COMPONENT_ARCHIVE_8 = $(COMPONENT_SRC_8).tar.gz COMPONENT_ARCHIVE_HASH_8 = \ sha256:83416a6bb86e8534858c4d1ddca45e881c87639da6f78000c28c3a193fe91305 COMPONENT_ARCHIVE_URL_8 = $(call pypi_url,8) -PKG_VARS += VERSION_JQUERY +PKG_MACROS += VERSION_JQUERY=$(COMPONENT_VERSION_8) COMPONENT_NAME_9 = XStatic-JQuery-Migrate -VERSION_JQUERY_MIGRATE= 1.2.1.1 -COMPONENT_SRC_9 = XStatic-JQuery-Migrate-$(VERSION_JQUERY_MIGRATE) +COMPONENT_VERSION_9 = 1.2.1.1 +COMPONENT_SRC_9 = XStatic-JQuery-Migrate-$(COMPONENT_VERSION_9) COMPONENT_ARCHIVE_9 = $(COMPONENT_SRC_9).tar.gz COMPONENT_ARCHIVE_HASH_9 = \ sha256:e2959b3df49afdddb00d36b74cca727a91b994b9f4edb993d7264731a750900e COMPONENT_ARCHIVE_URL_9 = $(call pypi_url,9) -PKG_VARS += VERSION_JQUERY_MIGRATE +PKG_MACROS += VERSION_JQUERY_MIGRATE=$(COMPONENT_VERSION_9) COMPONENT_NAME_10 = XStatic-jquery-ui -VERSION_JQUERY_UI = 1.11.0.1 -COMPONENT_SRC_10 = XStatic-jquery-ui-$(VERSION_JQUERY_UI) +COMPONENT_VERSION_10 = 1.11.0.1 +COMPONENT_SRC_10 = XStatic-jquery-ui-$(COMPONENT_VERSION_10) COMPONENT_ARCHIVE_10 = $(COMPONENT_SRC_10).tar.gz COMPONENT_ARCHIVE_HASH_10 = \ sha256:099b1836eb0d91b8dc98f5b8a6b856a2631d43af0d47f33ef90ee72ed37bda58 COMPONENT_ARCHIVE_URL_10 = $(call pypi_url,10) -PKG_VARS += VERSION_JQUERY_UI +PKG_MACROS += VERSION_JQUERY_UI=$(COMPONENT_VERSION_10) COMPONENT_NAME_11 = XStatic-JQuery.quicksearch -VERSION_JQUERY_QS = 2.0.3.1 -COMPONENT_SRC_11 = XStatic-JQuery.quicksearch-$(VERSION_JQUERY_QS) +COMPONENT_VERSION_11 = 2.0.3.1 +COMPONENT_SRC_11 = XStatic-JQuery.quicksearch-$(COMPONENT_VERSION_11) COMPONENT_ARCHIVE_11 = $(COMPONENT_SRC_11).tar.gz COMPONENT_ARCHIVE_HASH_11 = \ sha256:1271571b420417add56c274fd935e81bfc79e0d54a03559d6ba5ef369f358477 COMPONENT_ARCHIVE_URL_11 = $(call pypi_url,11) -PKG_VARS += VERSION_JQUERY_QS +PKG_MACROS += VERSION_JQUERY_QS=$(COMPONENT_VERSION_11) COMPONENT_NAME_12 = XStatic-JQuery.TableSorter -VERSION_JQUERY_TS = 2.14.5.1 -COMPONENT_SRC_12 = XStatic-JQuery.TableSorter-$(VERSION_JQUERY_TS) +COMPONENT_VERSION_12 = 2.14.5.1 +COMPONENT_SRC_12 = XStatic-JQuery.TableSorter-$(COMPONENT_VERSION_12) COMPONENT_ARCHIVE_12 = $(COMPONENT_SRC_12).tar.gz COMPONENT_ARCHIVE_HASH_12 = \ sha256:3ba24aecd9a3dc71a79dd4096fa5a8a041c3a7b892c61d05e6e46de0605070f0 COMPONENT_ARCHIVE_URL_12 = $(call pypi_url,12) -PKG_VARS += VERSION_JQUERY_TS +PKG_MACROS += VERSION_JQUERY_TS=$(COMPONENT_VERSION_12) COMPONENT_NAME_13 = XStatic-JSEncrypt -VERSION_JSENCRYPT = 2.0.0.2 -COMPONENT_SRC_13 = XStatic-JSEncrypt-$(VERSION_JSENCRYPT) +COMPONENT_VERSION_13 = 2.0.0.2 +COMPONENT_SRC_13 = XStatic-JSEncrypt-$(COMPONENT_VERSION_13) COMPONENT_ARCHIVE_13 = $(COMPONENT_SRC_13).tar.gz COMPONENT_ARCHIVE_HASH_13 = \ sha256:5852892afc6f80c7848f4110b6dad190a54aeb908271d67aaeae9d966f4a26b5 COMPONENT_ARCHIVE_URL_13 = $(call pypi_url,13) -PKG_VARS += VERSION_JSENCRYPT +PKG_MACROS += VERSION_JSENCRYPT=$(COMPONENT_VERSION_13) COMPONENT_NAME_14 = XStatic-Rickshaw -VERSION_RICKSHAW = 1.5.0.0 -COMPONENT_SRC_14 = XStatic-Rickshaw-$(VERSION_RICKSHAW) +COMPONENT_VERSION_14 = 1.5.0.0 +COMPONENT_SRC_14 = XStatic-Rickshaw-$(COMPONENT_VERSION_14) COMPONENT_ARCHIVE_14 = $(COMPONENT_SRC_14).tar.gz COMPONENT_ARCHIVE_HASH_14 = \ sha256:147574228757254442700a9eea5150f14acb1224ef0612f896b663ab58406de8 COMPONENT_ARCHIVE_URL_14 = $(call pypi_url,14) -PKG_VARS += VERSION_RICKSHAW +PKG_MACROS += VERSION_RICKSHAW=$(COMPONENT_VERSION_14) COMPONENT_NAME_15 = XStatic-Spin -VERSION_SPIN = 1.2.5.2 -COMPONENT_SRC_15 = XStatic-Spin-$(VERSION_SPIN) +COMPONENT_VERSION_15 = 1.2.5.2 +COMPONENT_SRC_15 = XStatic-Spin-$(COMPONENT_VERSION_15) COMPONENT_ARCHIVE_15 = $(COMPONENT_SRC_15).tar.gz COMPONENT_ARCHIVE_HASH_15 = \ sha256:7f46ef0e45e047019ba6eda22c432fb96f681b97bbe7f1749aa9209e07727192 COMPONENT_ARCHIVE_URL_15 = $(call pypi_url,15) -PKG_VARS += VERSION_SPIN - -PKG_VARS += COMPONENT_BE_VERSION +PKG_MACROS += VERSION_SPIN=$(COMPONENT_VERSION_15) COMPONENT_NAME_16 = XStatic-Angular-lrdragndrop -VERSION_LR_DND = 1.0.2.2 -COMPONENT_SRC_16 = XStatic-Angular-lrdragndrop-$(VERSION_LR_DND) -COMPONENT_ARCHIVE_16 = XStatic-Angular-lrdragndrop-$(VERSION_LR_DND).tar.gz +COMPONENT_VERSION_16 = 1.0.2.2 +COMPONENT_SRC_16 = XStatic-Angular-lrdragndrop-$(COMPONENT_VERSION_16) +COMPONENT_ARCHIVE_16 = $(COMPONENT_SRC_16).tar.gz COMPONENT_ARCHIVE_HASH_16 = \ sha256:1cf04495981db5dfd5536441e17ec69bb18d624f847ddc203f3259d81b10a77e COMPONENT_ARCHIVE_URL_16 = $(call pypi_url,16) -PKG_MACROS += VERSION_ANGULAR_LR_DND=$(VERSION_LR_DND) +PKG_MACROS += VERSION_LR_DND=$(COMPONENT_VERSION_16) COMPONENT_NAME_17 = XStatic-Magic-Search -VERSION_MAGIC_SEARCH = 0.2.5.1 -COMPONENT_SRC_17 = XStatic-Magic-Search-$(VERSION_MAGIC_SEARCH) -COMPONENT_ARCHIVE_17 = XStatic-Magic-Search-$(VERSION_MAGIC_SEARCH).tar.gz +COMPONENT_VERSION_17 = 0.2.5.1 +COMPONENT_SRC_17 = XStatic-Magic-Search-$(COMPONENT_VERSION_17) +COMPONENT_ARCHIVE_17 = $(COMPONENT_SRC_17).tar.gz COMPONENT_ARCHIVE_HASH_17 = \ sha256:9b2f35a5792f4e763e6dc319036e3676f3e18f46153096f3ab5e507177ec007e COMPONENT_ARCHIVE_URL_17 = $(call pypi_url,17) -PKG_MACROS += VERSION_MAGIC_SEARCH=$(VERSION_MAGIC_SEARCH) +PKG_MACROS += VERSION_MAGIC_SEARCH=$(COMPONENT_VERSION_17) COMPONENT_NAME_18 = XStatic-smart-table -VERSION_SMART_TABLE = 1.4.5.3 -COMPONENT_SRC_18 = XStatic-smart-table-$(VERSION_SMART_TABLE) -COMPONENT_ARCHIVE_18 = XStatic-smart-table-$(VERSION_SMART_TABLE).tar.gz +COMPONENT_VERSION_18 = 1.4.5.3 +COMPONENT_SRC_18 = XStatic-smart-table-$(COMPONENT_VERSION_18) +COMPONENT_ARCHIVE_18 = $(COMPONENT_SRC_18).tar.gz COMPONENT_ARCHIVE_HASH_18 = \ sha256:573bdff0b1ec88dd81b7f92c1b46fda4dd1b92cde94817837d61e62c9b20a8b6 COMPONENT_ARCHIVE_URL_18 = $(call pypi_url,18) -PKG_MACROS += VERSION_SMART_TABLE=$(VERSION_SMART_TABLE) +PKG_MACROS += VERSION_SMART_TABLE=$(COMPONENT_VERSION_18) COMPONENT_NAME_19 = XStatic-term.js -VERSION_TERM_JS = 0.0.4.2 -COMPONENT_SRC_19 = XStatic-term.js-$(VERSION_TERM_JS) -COMPONENT_ARCHIVE_19 = XStatic-term.js-$(VERSION_TERM_JS).tar.gz +COMPONENT_VERSION_19 = 0.0.4.2 +COMPONENT_SRC_19 = XStatic-term.js-$(COMPONENT_VERSION_19) +COMPONENT_ARCHIVE_19 = $(COMPONENT_SRC_19).tar.gz COMPONENT_ARCHIVE_HASH_19 = \ sha256:1ed5c1cd4de60d6f290a032bfc7cdc4261d8d36cb7788b2b0a610551bbda15ec COMPONENT_ARCHIVE_URL_19 = $(call pypi_url,19) -PKG_MACROS += VERSION_TERM_JS=$(VERSION_TERM_JS) +PKG_MACROS += VERSION_TERM_JS=$(COMPONENT_VERSION_19) COMPONENT_NAME_20 = XStatic-Angular-Bootstrap -VERSION_ANGULAR_BS = 0.11.0.2 -COMPONENT_SRC_20 = XStatic-Angular-Bootstrap-$(VERSION_ANGULAR_BS) -COMPONENT_ARCHIVE_20 = XStatic-Angular-Bootstrap-$(VERSION_ANGULAR_BS).tar.gz +COMPONENT_VERSION_20 = 0.11.0.2 +COMPONENT_SRC_20 = XStatic-Angular-Bootstrap-$(COMPONENT_VERSION_20) +COMPONENT_ARCHIVE_20 = $(COMPONENT_SRC_20).tar.gz COMPONENT_ARCHIVE_HASH_20 = \ sha256:cbe428bf04c000460776b521f6ace0455e9f3f20135499e9aa2f4af693dc7b3e COMPONENT_ARCHIVE_URL_20 = $(call pypi_url,20) -PKG_MACROS += VERSION_ANGULAR_BS=$(VERSION_ANGULAR_BS) +PKG_MACROS += VERSION_ANGULAR_BS=$(COMPONENT_VERSION_20) + +PKG_VARS += COMPONENT_BE_VERSION include $(WS_MAKE_RULES)/prep.mk include $(WS_MAKE_RULES)/setup.py.mk diff -r 720358c56a5d -r ce31ce66da02 make-rules/setup.py.mk --- a/make-rules/setup.py.mk Thu Apr 21 12:46:25 2016 -0400 +++ b/make-rules/setup.py.mk Wed Apr 27 14:28:37 2016 -0700 @@ -255,10 +255,9 @@ # Make it easy to construct a URL for a pypi source download. This # construct supports an optional call to a number from # NUM_EXTRA_ARCHIVES for multiple archive downloads. -PYPI_BASE = http://pypi.python.org/packages/source -pypi_url_multi = $(shell echo $(COMPONENT_NAME_$(1)) | cut -c1)/$(COMPONENT_NAME_$(1))/$(COMPONENT_ARCHIVE_$(1)) -pypi_url_single = $(shell echo $(COMPONENT_NAME) | cut -c1)/$(COMPONENT_NAME)/$(COMPONENT_ARCHIVE) -pypi_url = $(PYPI_BASE)/$(if $(COMPONENT_NAME_$(1)),$(pypi_url_multi),$(pypi_url_single)) +pypi_url_multi = pypi:///$(COMPONENT_NAME_$(1))==$(COMPONENT_VERSION_$(1)) +pypi_url_single = pypi:///$(COMPONENT_NAME)==$(COMPONENT_VERSION) +pypi_url = $(if $(COMPONENT_NAME_$(1)),$(pypi_url_multi),$(pypi_url_single)) ifneq ($(findstring 2.7, $(PYTHON_VERSIONS)),) REQUIRED_PACKAGES += runtime/python-27 diff -r 720358c56a5d -r ce31ce66da02 tools/userland-fetch --- a/tools/userland-fetch Thu Apr 21 12:46:25 2016 -0400 +++ b/tools/userland-fetch Wed Apr 27 14:28:37 2016 -0700 @@ -32,11 +32,13 @@ import errno import os +import re import sys import shutil +import json import subprocess -from urllib import splittype -from urllib2 import urlopen +from urllib import splittype, splithost +from urllib2 import urlopen, HTTPError import hashlib def printIOError(e, txt): @@ -130,7 +132,6 @@ """Given a file path and a hash string, verify that the hash matches the payload (uncompressed content) of the file.""" - import re import gzip import bz2 @@ -152,7 +153,6 @@ return False return validate(file, hash) - def download(url, timeout, filename=None, quiet=False): """Download the content at the given URL to the given filename (defaulting to the basename of the URL if not given. If 'quiet' is @@ -197,6 +197,65 @@ # return the name of the file that we downloaded the data to. return filename +def pypi_url(url, filename): + """Given a pypi: URL, return the real URL for that component/version. + + The pypi scheme has a host (with an empty host defaulting to + pypi.python.org), and a path that should be of the form + "component==version". Other specs could be supported, but == is the + only thing that makes sense in this context. + + The filename argument is the name of the expected file to download, so + that when pypi gives us multiple archives to choose from, we can pick + the right one. + """ + + host, path = splithost(splittype(url)[1]) + + # We have to use ==; anything fancier would require pkg_resources, but + # really that's the only thing that makes sense in this context. + try: + name, version = re.match("/(.*)==(.*)$", path).groups() + except AttributeError: + print "PyPI URLs must be of the form 'pypi:///component==version'" + return None + + if not host: + jsurl = "http://pypi.python.org/pypi/%s/json" % name + else: + jsurl = "http://%s/pypi/%s/json" % (host, name) + + try: + f = urlopen(jsurl) + except HTTPError as e: + if e.getcode() == 404: + print "Unknown component '%s'" % name + else: + printIOError(e, "Can't open PyPI JSON url %s" % url) + return None + except IOError as e: + printIOError(e, "Can't open PyPI JSON url %s" % url) + return None + + js = json.load(f) + try: + verblock = js["releases"][version] + except KeyError: + print "Unknown version '%s'" % version + return None + + urls = [ d["url"] for d in verblock ] + for archiveurl in urls: + if archiveurl.endswith("/%s" % filename): + return archiveurl + + if urls: + print "None of the following URLs delivers '%s':" % filename + print " " + "\n ".join(urls) + else: + print "Couldn't find any suitable URLs" + return None + def download_paths(search, filename, url): """Returns a list of URLs where the file 'filename' might be found, using 'url', 'search', and $DOWNLOAD_SEARCH_PATH as places to look. @@ -222,7 +281,11 @@ # command line url is a fallback, so it's last if url != None and url not in urls: - urls.append(url) + scheme, path = splittype(url) + if scheme == "pypi": + url = pypi_url(url, filename) + if url != None and url not in urls: + urls.append(url) return urls