509
|
1 |
#!/usr/bin/python2.6
|
|
2 |
#
|
|
3 |
# CDDL HEADER START
|
|
4 |
#
|
|
5 |
# The contents of this file are subject to the terms of the
|
|
6 |
# Common Development and Distribution License (the "License").
|
|
7 |
# You may not use this file except in compliance with the License.
|
|
8 |
#
|
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
|
10 |
# or http://www.opensolaris.org/os/licensing.
|
|
11 |
# See the License for the specific language governing permissions
|
|
12 |
# and limitations under the License.
|
|
13 |
#
|
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each
|
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
16 |
# If applicable, add the following below this CDDL HEADER, with the
|
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying
|
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner]
|
|
19 |
#
|
|
20 |
# CDDL HEADER END
|
|
21 |
#
|
|
22 |
|
|
23 |
#
|
|
24 |
# Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
|
|
25 |
#
|
|
26 |
|
|
27 |
import sys
|
|
28 |
import os
|
|
29 |
import traceback
|
|
30 |
import getopt
|
|
31 |
import urllib
|
|
32 |
import tempfile
|
|
33 |
import gettext
|
|
34 |
import shutil
|
|
35 |
import warnings
|
|
36 |
|
|
37 |
import pkg.fmri
|
|
38 |
import pkg.client.api_errors as apx
|
|
39 |
import pkg.client.publisher as publisher
|
|
40 |
import pkg.client.transport.transport as transport
|
|
41 |
import pkg.actions as actions
|
|
42 |
import pkg.manifest as manifest
|
|
43 |
import pkg.version as version
|
|
44 |
|
|
45 |
from pkg.misc import PipeError
|
|
46 |
from pkg.client import global_settings
|
|
47 |
|
|
48 |
pub = None
|
|
49 |
tmpdirs = []
|
|
50 |
xport = None
|
|
51 |
xport_cfg = None
|
|
52 |
|
|
53 |
def pname():
|
|
54 |
return os.path.basename(sys.argv[0])
|
|
55 |
|
|
56 |
def usage(usage_error = None):
|
|
57 |
|
|
58 |
if usage_error:
|
|
59 |
error(usage_error)
|
|
60 |
|
|
61 |
print >> sys.stderr, _("""\
|
|
62 |
Usage:
|
|
63 |
%s -r [-d dir] [-n] -v varname,url -v varname,url [-v varname,url ...] variant_type pkgname [pkgname ...]
|
|
64 |
|
|
65 |
example:
|
|
66 |
|
|
67 |
%s -r -d /tmp/merge -n -v sparc,http://server1 -v i386,http://server2 arch entire
|
|
68 |
""" % (pname(), pname()))
|
|
69 |
|
|
70 |
sys.exit(2)
|
|
71 |
|
|
72 |
def error(error):
|
|
73 |
""" Emit an error message prefixed by the command name """
|
|
74 |
|
|
75 |
print >> sys.stderr, pname() + ": " + error
|
|
76 |
|
|
77 |
def fetch_files_byaction(repouri, actions, pkgdir):
|
|
78 |
"""Given a list of files named by content hash, download from
|
|
79 |
repouri into pkgdir."""
|
|
80 |
|
|
81 |
mfile = xport.multi_file_ni(repouri, pkgdir, decompress=True)
|
|
82 |
|
|
83 |
for a in actions:
|
|
84 |
mfile.add_action(a)
|
|
85 |
|
|
86 |
mfile.wait_files()
|
|
87 |
|
|
88 |
manifest_cache = {}
|
|
89 |
null_manifest = manifest.Manifest()
|
|
90 |
|
|
91 |
def get_manifest(repouri, fmri):
|
|
92 |
if not fmri: # no matching fmri
|
|
93 |
return null_manifest
|
|
94 |
|
|
95 |
key = "%s->%s" % (repouri.uri, fmri)
|
|
96 |
if key not in manifest_cache:
|
|
97 |
manifest_cache[key] = fetch_manifest(repouri, fmri)
|
|
98 |
return manifest_cache[key]
|
|
99 |
|
|
100 |
def fetch_manifest(repouri, fmri):
|
|
101 |
"""Fetch the manifest for package-fmri 'fmri' from the server
|
|
102 |
in 'server_url'... return as Manifest object."""
|
|
103 |
|
|
104 |
mfst_str = xport.get_manifest(fmri, pub=repouri, content_only=True)
|
|
105 |
m = manifest.Manifest(fmri)
|
|
106 |
m.set_content(content=mfst_str)
|
|
107 |
return m
|
|
108 |
|
|
109 |
def fetch_catalog(repouri):
|
|
110 |
"""Fetch the catalog from the server_url."""
|
|
111 |
|
|
112 |
if not pub.meta_root:
|
|
113 |
# Create a temporary directory for catalog.
|
|
114 |
cat_dir = tempfile.mkdtemp()
|
|
115 |
tmpdirs.append(cat_dir)
|
|
116 |
pub.meta_root = cat_dir
|
|
117 |
|
|
118 |
pub.transport = xport
|
|
119 |
# Pull catalog only from this host
|
|
120 |
pub.selected_repository.origins = [repouri]
|
|
121 |
pub.refresh(True, True)
|
|
122 |
|
|
123 |
cat = pub.catalog
|
|
124 |
|
|
125 |
return cat
|
|
126 |
|
|
127 |
catalog_dict = {}
|
|
128 |
def load_catalog(repouri):
|
|
129 |
c = fetch_catalog(repouri)
|
|
130 |
d = {}
|
|
131 |
for f in c.fmris():
|
|
132 |
if f.pkg_name in d:
|
|
133 |
d[f.pkg_name].append(f)
|
|
134 |
else:
|
|
135 |
d[f.pkg_name] = [f]
|
|
136 |
for k in d.keys():
|
|
137 |
d[k].sort(reverse = True)
|
|
138 |
catalog_dict[repouri.uri] = d
|
|
139 |
|
|
140 |
def expand_fmri(repouri, fmri_string, constraint=version.CONSTRAINT_AUTO):
|
|
141 |
""" from specified server, find matching fmri using CONSTRAINT_AUTO
|
|
142 |
cache for performance. Returns None if no matching fmri is found """
|
|
143 |
if repouri.uri not in catalog_dict:
|
|
144 |
load_catalog(repouri)
|
|
145 |
|
|
146 |
fmri = pkg.fmri.PkgFmri(fmri_string, "5.11")
|
|
147 |
|
|
148 |
for f in catalog_dict[repouri.uri].get(fmri.pkg_name, []):
|
|
149 |
if not fmri.version or f.version.is_successor(fmri.version, constraint):
|
|
150 |
return f
|
|
151 |
return None
|
|
152 |
|
|
153 |
def get_all_pkg_names(repouri):
|
|
154 |
""" return all the pkg_names in this catalog """
|
|
155 |
if repouri.uri not in catalog_dict:
|
|
156 |
load_catalog(repouri)
|
|
157 |
return catalog_dict[repouri.uri].keys()
|
|
158 |
|
|
159 |
def get_dependencies(repouri, fmri_list):
|
|
160 |
s = set()
|
|
161 |
for f in fmri_list:
|
|
162 |
fmri = expand_fmri(repouri, f)
|
|
163 |
_get_dependencies(s, repouri, fmri)
|
|
164 |
return s
|
|
165 |
|
|
166 |
def _get_dependencies(s, repouri, fmri):
|
|
167 |
""" recursive incorp expansion"""
|
|
168 |
s.add(fmri)
|
|
169 |
for a in get_manifest(repouri, fmri).gen_actions_by_type("depend"):
|
|
170 |
if a.attrs["type"] == "incorporate":
|
|
171 |
new_fmri = expand_fmri(repouri, a.attrs["fmri"])
|
|
172 |
if new_fmri and new_fmri not in s:
|
|
173 |
_get_dependencies(s, repouri, new_fmri)
|
|
174 |
return s
|
|
175 |
|
|
176 |
def cleanup():
|
|
177 |
"""To be called at program finish."""
|
|
178 |
|
|
179 |
for d in tmpdirs:
|
|
180 |
shutil.rmtree(d, True)
|
|
181 |
|
|
182 |
def main_func():
|
|
183 |
|
|
184 |
global pub, xport, xport_cfg
|
|
185 |
basedir = None
|
|
186 |
newfmri = False
|
|
187 |
incomingdir = None
|
|
188 |
|
|
189 |
gettext.install("pkg", "/usr/share/locale")
|
|
190 |
|
|
191 |
global_settings.client_name = "pkgmerge"
|
|
192 |
|
|
193 |
try:
|
|
194 |
opts, pargs = getopt.getopt(sys.argv[1:], "d:nrv:")
|
|
195 |
except getopt.GetoptError, e:
|
|
196 |
usage(_("Illegal option -- %s") % e.opt)
|
|
197 |
|
|
198 |
varlist = []
|
|
199 |
recursive = False
|
|
200 |
get_files = True
|
|
201 |
|
|
202 |
for opt, arg in opts:
|
|
203 |
if opt == "-d":
|
|
204 |
basedir = arg
|
|
205 |
if opt == "-v":
|
|
206 |
varlist.append(arg)
|
|
207 |
if opt == "-r":
|
|
208 |
recursive = True
|
|
209 |
if opt == "-n":
|
|
210 |
get_files = False
|
|
211 |
|
|
212 |
|
|
213 |
if len(varlist) < 2:
|
|
214 |
usage(_("at least two -v arguments needed to merge"))
|
|
215 |
|
|
216 |
if not basedir:
|
|
217 |
basedir = os.getcwd()
|
|
218 |
|
|
219 |
incomingdir = os.path.normpath(os.path.join(basedir,
|
|
220 |
"incoming-%d" % os.getpid()))
|
|
221 |
os.makedirs(incomingdir)
|
|
222 |
tmpdirs.append(incomingdir)
|
|
223 |
|
|
224 |
server_list = [
|
|
225 |
publisher.RepositoryURI(v.split(",", 1)[1])
|
|
226 |
for v in varlist
|
|
227 |
]
|
|
228 |
|
|
229 |
xport, xport_cfg = transport.setup_transport()
|
|
230 |
xport_cfg.incoming_root = incomingdir
|
|
231 |
pub = transport.setup_publisher(server_list, "merge", xport, xport_cfg,
|
|
232 |
remote_prefix=True)
|
|
233 |
|
|
234 |
if len(pargs) == 1:
|
|
235 |
recursive = False
|
|
236 |
overall_set = set()
|
|
237 |
for s in server_list:
|
|
238 |
for name in get_all_pkg_names(s):
|
|
239 |
overall_set.add(name)
|
|
240 |
fmri_arguments = list(overall_set)
|
|
241 |
|
|
242 |
else:
|
|
243 |
fmri_arguments = pargs[1:]
|
|
244 |
|
|
245 |
if not pargs:
|
|
246 |
usage(_("you must specify a variant"))
|
|
247 |
|
|
248 |
variant = "variant.%s" % pargs[0]
|
|
249 |
|
|
250 |
variant_list = [
|
|
251 |
v.split(",", 1)[0]
|
|
252 |
for v in varlist
|
|
253 |
]
|
|
254 |
|
|
255 |
fmri_expansions = []
|
|
256 |
|
|
257 |
if recursive:
|
|
258 |
overall_set = set()
|
|
259 |
for s in server_list:
|
|
260 |
deps = get_dependencies(s, fmri_arguments)
|
|
261 |
for d in deps:
|
|
262 |
if d:
|
|
263 |
q = str(d).rsplit(":", 1)[0]
|
|
264 |
overall_set.add(q)
|
|
265 |
fmri_arguments = list(overall_set)
|
|
266 |
|
|
267 |
fmri_arguments.sort()
|
|
268 |
print "Processing %d packages" % len(fmri_arguments)
|
|
269 |
|
|
270 |
for fmri in fmri_arguments:
|
|
271 |
try:
|
|
272 |
fmri_list = [
|
|
273 |
expand_fmri(s, fmri)
|
|
274 |
for s in server_list
|
|
275 |
]
|
|
276 |
if len(set([
|
|
277 |
str(f).rsplit(":", 1)[0]
|
|
278 |
for f in fmri_list
|
|
279 |
if f
|
|
280 |
])) != 1:
|
|
281 |
error("fmris at different versions: %s" % fmri_list)
|
|
282 |
continue
|
|
283 |
|
|
284 |
except pkg.fmri.IllegalFmri:
|
|
285 |
error(_("pkgfmri error"))
|
|
286 |
return 1
|
|
287 |
|
|
288 |
for f in fmri_list:
|
|
289 |
if f:
|
|
290 |
basename = f.get_name()
|
|
291 |
break
|
|
292 |
else:
|
|
293 |
error("No package of name %s in specified catalogs %s; ignoring." %\
|
|
294 |
(fmri, server_list))
|
|
295 |
continue
|
|
296 |
|
|
297 |
merge_fmris(server_list, fmri_list, variant_list, variant, basedir, basename, get_files)
|
|
298 |
cleanup()
|
|
299 |
|
|
300 |
return 0
|
|
301 |
|
|
302 |
def merge_fmris(server_list, fmri_list, variant_list, variant, basedir,
|
|
303 |
basename, get_files):
|
|
304 |
|
|
305 |
manifest_list = [
|
|
306 |
get_manifest(s, f)
|
|
307 |
for s, f in zip(server_list, fmri_list)
|
|
308 |
]
|
|
309 |
|
|
310 |
# remove variant tags and package variant metadata
|
|
311 |
# from manifests since we're reassigning...
|
|
312 |
# this allows merging pre-tagged packages
|
|
313 |
for m in manifest_list:
|
|
314 |
for i, a in enumerate(m.actions[:]):
|
|
315 |
if variant in a.attrs:
|
|
316 |
del a.attrs[variant]
|
|
317 |
if a.name == "set" and a.attrs["name"] == variant:
|
|
318 |
del m.actions[i]
|
|
319 |
|
|
320 |
action_lists = manifest.Manifest.comm(*tuple(manifest_list))
|
|
321 |
|
|
322 |
# set fmri actions require special merge logic.
|
|
323 |
set_fmris = []
|
|
324 |
for l in action_lists:
|
|
325 |
for i, a in enumerate(l):
|
|
326 |
if not (a.name == "set" and
|
|
327 |
a.attrs["name"] == "pkg.fmri"):
|
|
328 |
continue
|
|
329 |
|
|
330 |
set_fmris.append(a)
|
|
331 |
del l[i]
|
|
332 |
|
|
333 |
# If set fmris are present, then only the most recent one
|
|
334 |
# and add it back to the last action list.
|
|
335 |
if set_fmris:
|
|
336 |
def order(a, b):
|
|
337 |
f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11")
|
|
338 |
f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11")
|
|
339 |
return cmp(f1, f2)
|
|
340 |
set_fmris.sort(cmp=order)
|
|
341 |
action_lists[-1].insert(0, set_fmris[-1])
|
|
342 |
|
|
343 |
for a_list, v in zip(action_lists[0:-1], variant_list):
|
|
344 |
for a in a_list:
|
|
345 |
a.attrs[variant] = v
|
|
346 |
|
|
347 |
# combine actions into single list
|
|
348 |
allactions = reduce(lambda a, b: a + b, action_lists)
|
|
349 |
|
|
350 |
# figure out which variants are actually there for this pkg
|
|
351 |
actual_variant_list = [
|
|
352 |
v
|
|
353 |
for m, v in zip(manifest_list, variant_list)
|
|
354 |
if m != null_manifest
|
|
355 |
]
|
|
356 |
print "Merging %s for %s" % (basename, actual_variant_list)
|
|
357 |
|
|
358 |
# add set action to document which variants are supported
|
|
359 |
allactions.append(actions.fromstr("set name=%s %s" % (variant,
|
|
360 |
" ".join(["value=%s" % a
|
|
361 |
for a in actual_variant_list
|
|
362 |
]))))
|
|
363 |
|
|
364 |
allactions.sort()
|
|
365 |
|
|
366 |
m = manifest.Manifest()
|
|
367 |
m.set_content(content=allactions)
|
|
368 |
|
|
369 |
# urlquote to avoid problems w/ fmris w/ '/' character in name
|
|
370 |
basedir = os.path.join(basedir, urllib.quote(basename, ""))
|
|
371 |
if not os.path.exists(basedir):
|
|
372 |
os.makedirs(basedir)
|
|
373 |
|
|
374 |
m_path = os.path.join(basedir, "manifest")
|
|
375 |
m.store(m_path)
|
|
376 |
|
|
377 |
for f in fmri_list:
|
|
378 |
if f:
|
|
379 |
fmri = str(f).rsplit(":", 1)[0]
|
|
380 |
break
|
|
381 |
f_file = file(os.path.join(basedir, "fmri"), "w")
|
|
382 |
f_file.write(fmri)
|
|
383 |
f_file.close()
|
|
384 |
|
|
385 |
|
|
386 |
if get_files:
|
|
387 |
# generate list of hashes for each server; last is commom
|
|
388 |
already_seen = {}
|
|
389 |
def repeated(a, d):
|
|
390 |
if a in d:
|
|
391 |
return True
|
|
392 |
d[a] = 1
|
|
393 |
return False
|
|
394 |
|
|
395 |
action_sets = [
|
|
396 |
set(
|
|
397 |
[
|
|
398 |
a
|
|
399 |
for a in action_list
|
|
400 |
if hasattr(a, "hash") and not \
|
|
401 |
repeated(a.hash, already_seen)
|
|
402 |
]
|
|
403 |
)
|
|
404 |
for action_list in action_lists
|
|
405 |
]
|
|
406 |
# remove duplicate files (save time)
|
|
407 |
|
|
408 |
for server, action_set in zip(server_list + [server_list[0]],
|
|
409 |
action_sets):
|
|
410 |
if len(action_set) > 0:
|
|
411 |
fetch_files_byaction(server, action_set,
|
|
412 |
basedir)
|
|
413 |
|
|
414 |
return 0
|
|
415 |
|
|
416 |
|
|
417 |
if __name__ == "__main__":
|
|
418 |
|
|
419 |
# Make all warnings be errors.
|
|
420 |
warnings.simplefilter('error')
|
|
421 |
|
|
422 |
try:
|
|
423 |
ret = main_func()
|
|
424 |
except (apx.InvalidDepotResponseException, apx.TransportError,
|
|
425 |
apx.BadRepositoryURI, apx.UnsupportedRepositoryURI), e:
|
|
426 |
cleanup()
|
|
427 |
print >> sys.stderr, e
|
|
428 |
sys.exit(1)
|
|
429 |
except SystemExit, e:
|
|
430 |
cleanup()
|
|
431 |
raise e
|
|
432 |
except (PipeError, KeyboardInterrupt):
|
|
433 |
# We don't want to display any messages here to prevent
|
|
434 |
# possible further broken pipe (EPIPE) errors.
|
|
435 |
cleanup()
|
|
436 |
sys.exit(1)
|
|
437 |
except:
|
|
438 |
traceback.print_exc()
|
|
439 |
cleanup()
|
|
440 |
sys.exit(99)
|
|
441 |
sys.exit(ret)
|
|
442 |
|