upstream/oracle/pkg-gate: comparison src/util/distro-import/importer.py

equal deleted inserted replaced

-:f9716f0b9c0a
+:21ae45e8dfcf
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 #
-# Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
 #
 import fnmatch
 import getopt
 import gettext
 from pkg      import actions, elf
 from pkg.bundle.SolarisPackageDirBundle import SolarisPackageDirBundle
 from pkg.misc import emsg
 from pkg.portable import PD_LOCAL_PATH, PD_PROTO_DIR, PD_PROTO_DIR_LIST
-CLIENT_API_VERSION = 48
+CLIENT_API_VERSION = 49
 PKG_CLIENT_NAME = "importer.py"
 pkg.client.global_settings.client_name = PKG_CLIENT_NAME
 from tempfile import mkstemp
 # to global name table. Actions are annotated to include svr4 source
 # pkg & path
 basename_dict = {}   # basenames to action lists
 branch_dict = {}     #
 cons_dict = {}       # consolidation incorporation dictionaries
 file_repo = False    #
 curpkg = None        # which IPS package we're currently importing
 def_branch = ""      # default branch
 def_pub = None
 action.attrs["path"] in excludes:
 if show_debug:
 print "excluding %s from %s" % \
 (action.attrs["path"], imppkg_name)
 continue
 if action.name == "unknown":
 continue
 action.attrs["importer.source"] = "svr4pkg"
 action.attrs["importer.svr4pkg"] = imppkg_name
 if action.name == "license":
 # The "path" attribute is confusing and
 # unnecessary for licenses.
 del action.attrs["path"]
 if action.name == "file":
 # is this a file for which we need a timestamp?
 basename = os.path.basename(action.attrs["path"])
 for file_pattern in timestamp_files:
 if fnmatch.fnmatch(basename, file_pattern):
 # is this file likely to be an SMF manifest? If so,
 # save a copy of the file to use for dependency analysis
 if smf_manifest.has_smf_manifest_dir(action.attrs["path"]):
 fetch_file(action, local_smf_manifests)
 if hollow:
 action.attrs["variant.opensolaris.zone"] = "global"
 self.check_perms(action)
 self.actions.append(action)
 action.attrs[k]
 )
 def chattr(self, fname, line):
 matches = [
 a
 for a in self.actions
 if "path" in a.attrs and a.attrs["path"] == fname
 ]
 if not matches:
 new_attrs = actions.attrsfromstr(line.rstrip())
 o = [
 f
 for f in self.actions
 if "path" in f.attrs and
 fnmatch.fnmatchcase(f.attrs["path"], glob) and
 (not type or type == f.name)
 ]
 for f in o:
 targets = set((d.attrs["target"] for d in dups))
 if len(targets) > 1:
 errorlist.append("Multiple %s actions with same path and different targets:\n\t%s\n" %
 (dups[0].name, "\n\t".join(str(d) for d in dups)))
 continue
 elif dups[0].name != "dir":
 errorlist.append("Multiple actions with the same path that aren't directories:\n\t%s\n" %
 ("\n\t".join(str(d) for d in dups)))
 continue
 (p, g, "\n\t".join(str(d) for d in dups))
 if allow_dir_goofs:
 print >> sys.stderr, "%s\n" % dir_error
 else:
 errorlist.append(dir_error)
 elif remove_dups and g.startswith("variant.") and None in ga[g]:
 # remove any dirs that are zone variants if same dir w/o variant exists
 for d in dups:
 if d.attrs.get(g) != None:
 d.attrs["importer.deleteme"] = "True"
 # add dependency on consolidation incorporation if not obsolete or renamed
 if pkg.consolidation and not pkg.obsolete_branch and not pkg.rename_branch:
 action = actions.fromstr(
 "depend fmri=consolidation/%s/%s-incorporation "
 "type=require importer.no-version=true" %
 (pkg.consolidation, pkg.consolidation))
 pkg.actions.append(action)
 # add legacy actions
 if pkg.name != "SUNWipkg":
 try:
 t.add(a)
 except TypeError, e:
 print a.attrs
 print a.name
 raise
 def publish_pkg(pkg, proto_dir):
 """ send this package to the repo """
 smf_fmris = []
 svr4_pkg_list = sorted(list(set([
 a.attrs["importer.svr4pkg"]
 for a in pkg.actions
 if "importer.svr4pkg" in a.attrs and
 a.name in ["license", "file"]
 np = link.attrs["path"]
 nt = link.attrs["target"]
 newpath = os.path.normpath(
 os.path.join(os.path.split(np)[0], nt))
 assert path.startswith(np)
 ret = [pkgpath_dict[p][0]]
 next = search_dicts(path.replace(np, newpath))
 if next:
 ret += next
 return ret
 else:
 print "unexpected action %s in path %s" % (path_dict[p][0], path)
 return []
 def get_smf_fmris(file, action_path):
 """ pull the delivered SMF FMRIs from file, associated with action_path """
 if smf_manifest.has_smf_manifest_dir(action_path):
 instance_mf, instance_deps = smf_manifest.parse_smf_manifest(file)
 if instance_mf:
 return instance_mf.keys()
 elif "perl" in l or path.endswith(".pl"):
 pass # and here
 # handle smf manifests
 if smf_manifest.has_smf_manifest_dir(path):
 # pkg.flavor.* used by pkgdepend wants PD_LOCAL_PATH, PD_PROTO_DIR
 # and PD_PROTO_DIR_LIST set
 action.attrs[PD_LOCAL_PATH] = fname
 action.attrs[PD_PROTO_DIR] = proto_dir
 action.attrs[PD_PROTO_DIR_LIST] = [proto_dir]
 except elf.ElfError:
 deps = []
 rp = []
 else:
 deps = [
 a
 for d in ed.get("deps", [])
 for a in d[0].split()
 ]
 rp = ed.get("runpath", "").split(":")
 if len(rp) == 1 and rp[0] == "":
 def get_manifest(server_pub, fmri):
 if not fmri: # no matching fmri
 return null_manifest
 return manifest_cache.setdefault((server_pub, fmri),
 fetch_manifest(server_pub, fmri))
 def fetch_manifest(server_pub, fmri):
 """Fetch the manifest for package-fmri 'fmri' from the server
 in 'server_url'... return as Manifest object.... needs
 d.setdefault(f.pkg_name, []).append(f)
 for k in d:
 d[k].sort(reverse=True)
 catalog_dict[server_pub] = d
 def expand_fmri(server_pub, fmri_string, constraint=version.CONSTRAINT_AUTO):
 """ from specified server, find matching fmri using CONSTRAINT_AUTO
 cache for performance.  Returns None if no matching fmri is found """
 if server_pub not in catalog_dict:
 load_catalog(server_pub)
 fmri = pkg.fmri.PkgFmri(fmri_string, "5.11")
 for f in catalog_dict[server_pub].get(fmri.pkg_name, []):
 if not fmri.version or f.version.is_successor(fmri.version, constraint):
 return f
 return None
 def _get_dependencies(s, server_pub, fmri):
 """ recursive incorp expansion"""
 s.add(fmri)
 for a in get_manifest(server_pub, fmri).gen_actions_by_type("depend"):
 if a.attrs["type"] == "incorporate":
 new_fmri = expand_fmri(server_pub, a.attrs["fmri"])
 if new_fmri and new_fmri not in s: # ignore missing, already planned
 _get_dependencies(s, server_pub, new_fmri)
 def get_smf_packages(server_url, manifest_locations, filter):
 """ Performs a search against server_url looking for packages which contain
 continue
 if filter in pfmri.get_fmri():
 fmris.add(pfmri.get_fmri())
 return [pkg.fmri.PkgFmri(pfmri) for pfmri in fmris]
 def zap_strings(instr, strings):
 """takes an input string and a list of strings to be removed, ignoring
 case"""
 for s in strings:
 ls = s.lower()
 li = instr.lower()
 i = li.find(ls)
 if i < 0:
 break
 instr = instr[0:i] + instr[i + len(ls):]
 return instr
 def get_branch(name):
 return branch_dict.get(name, def_branch)
 def set_macro(key, value):
 macro_definitions.update([("$(%s)" % key, value)])
 def clear_macro(key):
 del macro_definitions["$(%s)" % key]
 def get_arch(): # use value of arch macro or platform
 return macro_definitions.get("$ARCH", platform.processor())
 def read_full_line(lexer, continuation='\\'):
 """Read a complete line, allowing for the possibility of it being
 continued over multiple lines.  Returns a single joined line, with
 class tokenlexer(shlex.shlex):
 def read_token(self):
 """ simple replacement of $(ARCH) with a non-special
 value defined on the command line is trivial.  Since
 shlex's read_token routine also strips comments and
 white space, this read_token cannot return either
 one so any macros that translate to either spaces or
 # (comment) need to be removed from the token stream."""
 while True:
 s = apply_macros(shlex.shlex.read_token(self))
 raise RuntimeError("Error: unknown token '%s' "
 "(%s:%s)" % (token, lexer.infile, lexer.lineno))
 def repo_add_content(path_to_repo, path_to_proto):
 """Fire up depo to add content and rebuild search index"""
 cmdname = os.path.join(path_to_proto, "usr/bin/pkgrepo")
 argstr = "%s -s %s refresh" % (cmdname, path_to_repo)
 print "Adding content & rebuilding search indicies synchronously...."
 print "%s" % str(argstr)
 try:
 global not_these_consolidations
 global curpkg
 global xport
 global xport_cfg
 try:
 _opts, _args = getopt.getopt(sys.argv[1:], "AB:C:D:E:I:J:G:NR:T:b:dj:m:ns:v:w:p:")
 except getopt.GetoptError, _e:
 print "unknown option", _e.opt
 sys.exit(1)
 try:
 del pkgdict[pkg]
 except KeyError:
 print "excluded package %s not in pkgdict" % pkg
 # Unless we are publishing all obsolete and renamed packages
 # (-A command line option), remove obsolete and renamed packages
 # that weren't obsoleted or renamed at this branch and create
 # a dictionary (called or_pkgs_per_con) of obsoleted and renamed
 # packages per consolidation.  The version portion of the fmri
 # will contain the branch that the package was obsoleted or renamed at.
 or_pkgs_per_con = {}
 obs_or_renamed_pkgs = {}
 for pkg in pkgdict.keys():
 if action.name != "depend":
 continue
 if action.attrs["type"] == "require" and "fmri" in action.attrs:
 fmri = action.attrs["fmri"].split("@")[0] # remove version
 if fmri.startswith("pkg:/"): # remove pkg:/ if exists
 fmri = fmri[5:]
 if fmri in obs_or_renamed_pkgs:
 tup = obs_or_renamed_pkgs[fmri]
 s = "Pkg %s has 'require' dependency on pkg %s, which is %s" % (
 (pack.fmristr(),) + tup)
 if tup[1] == "obsolete":
 curpkg = None
 # Generate entire consolidation if we're generating any consolidation incorps
 if consolidation_incorporations:
 curpkg = start_package("entire")
 curpkg.summary = "incorporation to lock all system packages to same build"
 curpkg.desc = "This package constrains " \
 "system package versions to the same build.  WARNING: Proper " \
 "system update and correct package selection depend on the " \
 "presence of this incorporation.  Removing this package will " \
 "result in an unsupported system."
 incorporated_pkgs = set([
 f
 for l in cons_dict.values()
 for f in l
 ])
 incorporated_pkgs |= set(consolidation_incorporations)
 incorporated_pkgs |= set(["entire", "redistributable"])
 incorporated_pkgs |= set(obsoleted_renamed_pkgs)
 unincorps = set(pkgdict.keys()) - incorporated_pkgs
 if unincorps:
 # look through these; if they have only set actions they're
 # ancient obsoleted pkgs - ignore them.
 for f in unincorps.copy():
 else:
 unincorps.remove(f)
 print "The following non-empty unincorporated pkgs are not part of any consolidation"
 for f in unincorps:
 print f
 if just_these_pkgs:
 newpkgs = set(pkgdict[name]
 for name in pkgdict.keys()
 if name in just_these_pkgs
 )
 if code:
 cleanup()
 sys.exit(code)
 print "Done:", datetime.now()
 elapsed = time.clock() - start_time
 print "publication took %d:%.2d" % (elapsed/60, elapsed % 60)
 cleanup()
 sys.exit(0)
 if __name__ == "__main__":
 main_func()

changeset 2183	21ae45e8dfcf
parent 2158	5a688b5b92a3
child 2215	b4355e8c5097