1 #!/usr/bin/python

     2 #

     3 # CDDL HEADER START

     4 #

     5 # The contents of this file are subject to the terms of the

     6 # Common Development and Distribution License (the "License").

     7 # You may not use this file except in compliance with the License.

     8 #

     9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

    10 # or http://www.opensolaris.org/os/licensing.

    11 # See the License for the specific language governing permissions

    12 # and limitations under the License.

    13 #

    14 # When distributing Covered Code, include this CDDL HEADER in each

    15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.

    16 # If applicable, add the following below this CDDL HEADER, with the

    17 # fields enclosed by brackets "[]" replaced with your own identifying

    18 # information: Portions Copyright [yyyy] [name of copyright owner]

    19 #

    20 # CDDL HEADER END

    21 #

    22 

    23 #

    24 # Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.

    25 #

    26 

    27 import atexit

    28 import collections

    29 import errno

    30 import tarfile as tf

    31 import pkg.pkggzip

    32 import pkg.pkgtarfile as ptf

    33 import os

    34 import pkg

    35 import pkg.client.api_errors as apx

    36 import pkg.client.publisher

    37 import pkg.fmri

    38 import pkg.misc

    39 import pkg.portable

    40 import pkg.p5i

    41 import shutil

    42 import tempfile

    43 import urllib

    44 

    45 

    46 class ArchiveErrors(apx.ApiException):

    47         """Base exception class for archive class errors."""

    48 

    49 

    50 class InvalidArchiveIndex(ArchiveErrors):

    51         """Used to indicate that the specified index is in a format not

    52         supported or recognized by this version of the pkg(5) ArchiveIndex

    53         class."""

    54 

    55         def __init__(self, arc_name):

    56                 ArchiveErrors.__init__(self)

    57                 self.__name = arc_name

    58 

    59         def __str__(self):

    60                 return _("%s is not in a supported or recognizable archive "

    61                     "index format.") % self.__name

    62 

    63 

    64 class ArchiveIndex(object):

    65         """Class representing a pkg(5) archive table of contents and a set of

    66         interfaces to populate and retrieve entries.

    67 

    68         Entries in this file are written in the following format:

    69 

    70             <name>NUL<offset>NUL<entry_size>NUL<size>NUL<typeflag>NULNL

    71 

    72             <name> is a string containing the pathname of the file in the

    73             archive.  It can be up to 65,535 bytes in length.

    74 

    75             <offset> is an unsigned long long integer containing the relative

    76             offset in bytes of the first header block for the file in the

    77             archive.  The offset is relative to the end of the last block of

    78             the first file in the archive.

    79 

    80             <entry_size> is an unsigned long long integer containing the size of

    81             the file's entry in bytes in the archive (including archive

    82             headers and trailers for the entry).

    83 

    84             <size> is an unsigned long long integer containing the size of the

    85             file in bytes in the archive.

    86 

    87             <typeflag> is a single character representing the type of the file

    88             in the archive.  Possible values are:

    89                 0 Regular File

    90                 1 Hard Link

    91                 2 Symbolic Link

    92                 5 Directory or subdirectory"""

    93 

    94         version = None

    95         CURRENT_VERSION = 0

    96         COMPATIBLE_VERSIONS = 0,

    97         ENTRY_FORMAT = "%s\0%d\0%d\0%d\0%c\0\n"

    98 

    99         def __init__(self, name, mode="r", version=None):

   100                 """Open a pkg(5) archive table of contents file.

   101 

   102                 'name' should be the absolute path of the file to use when

   103                 reading or writing index data.

   104 

   105                 'mode' indicates whether the index is being used for reading

   106                 or writing, and can be 'r' or 'w'.  Appending to or updating

   107                 a table of contents file is not supported.

   108 

   109                 'version' is an optional integer value specifying the version

   110                 of the index to be read or written.  If not specified, the

   111                 current version is assumed.

   112                 """

   113 

   114                 assert os.path.isabs(name)

   115                 if version is None:

   116                         version = self.CURRENT_VERSION

   117                 if version not in self.COMPATIBLE_VERSIONS:

   118                         raise InvalidArchiveIndex(name)

   119 

   120                 self.__closed = False

   121                 self.__name = name

   122                 self.__mode = mode + "b"

   123                 try:

   124                         self.__file = pkg.pkggzip.PkgGzipFile(self.__name,

   125                             self.__mode)

   126                 except IOError, e:

   127                         if e.errno:

   128                                 raise

   129                         # Underlying gzip library raises this exception if the

   130                         # file isn't a valid gzip file.  So, assume that if

   131                         # errno isn't set, this is a gzip error instead.

   132                         raise InvalidArchiveIndex(name)

   133 

   134                 self.version = version

   135 

   136         def __exit__(self, exc_type, exc_value, exc_tb):

   137                 """Context handler that ensures archive is automatically closed

   138                 in a non-error condition scenario.  This enables 'with' usage.

   139                 """

   140                 if exc_type or exc_value or exc_tb:

   141                         # Only close filehandles in an error condition.

   142                         self.__close_fh()

   143                 else:

   144                         # Close archive normally in all other cases.

   145                         self.close()

   146 

   147         @property

   148         def pathname(self):

   149                 """The absolute path of the archive index file."""

   150                 return self.__name

   151 

   152         def add(self, name, offset, entry_size, size, typeflag):

   153                 """Add an entry for the given archive file to the table of

   154                 contents."""

   155 

   156                 self.__file.write(self.ENTRY_FORMAT % (name, offset, entry_size,

   157                     size, typeflag))

   158 

   159         def offsets(self):

   160                 """Returns a generator that yields tuples of the form (name,

   161                 offset) for each file in the index."""

   162 

   163                 self.__file.seek(0)

   164                 l = None

   165                 try:

   166                         for line in self.__file:

   167                                 if line[-2] != "\0":

   168                                         # Filename contained newline.

   169                                         if l is None:

   170                                                 l = line

   171                                         else:

   172                                                 l += "\n"

   173                                                 l += line

   174                                         continue

   175                                 elif l is None:

   176                                         l = line

   177 

   178                                 name, offset, ignored = l.split("\0", 2)

   179                                 yield name, long(offset)

   180                                 l = None

   181                 except ValueError:

   182                         raise InvalidArchiveIndex(self.__name)

   183                 except IOError, e:

   184                         if e.errno:

   185                                 raise

   186                         # Underlying gzip library raises this exception if the

   187                         # file isn't a valid gzip file.  So, assume that if

   188                         # errno isn't set, this is a gzip error instead.

   189                         raise InvalidArchiveIndex(self.__name)

   190 

   191         def close(self):

   192                 """Close the index.  No further operations can be performed

   193                 using this object once closed."""

   194 

   195                 if self.__closed:

   196                         return

   197                 if self.__file:

   198                         self.__file.close()

   199                         self.__file = None

   200                 self.__closed = True

   201 

   202 

   203 class InvalidArchive(ArchiveErrors):

   204         """Used to indicate that the specified archive is in a format not

   205         supported or recognized by this version of the pkg(5) Archive class.

   206         """

   207 

   208         def __init__(self, arc_name):

   209                 ArchiveErrors.__init__(self)

   210                 self.arc_name = arc_name

   211 

   212         def __str__(self):

   213                 return _("Archive %s is missing, unsupported, or corrupt.") % \

   214                     self.arc_name

   215 

   216 

   217 class CorruptArchiveFiles(ArchiveErrors):

   218         """Used to indicate that the specified file(s) could not be found in the

   219         archive.

   220         """

   221 

   222         def __init__(self, arc_name, files):

   223                 ArchiveErrors.__init__(self)

   224                 self.arc_name = arc_name

   225                 self.files = files

   226 

   227         def __str__(self):

   228                 return _("Package archive %(arc_name)s contains corrupt "

   229                     "entries for the requested package file(s):\n%(files)s.") % {

   230                     "arc_name": self.arc_name,

   231                     "files": "\n".join(self.files) }

   232 

   233 

   234 class UnknownArchiveFiles(ArchiveErrors):

   235         """Used to indicate that the specified file(s) could not be found in the

   236         archive.

   237         """

   238 

   239         def __init__(self, arc_name, files):

   240                 ArchiveErrors.__init__(self)

   241                 self.arc_name = arc_name

   242                 self.files = files

   243 

   244         def __str__(self):

   245                 return _("Package archive %(arc_name)s does not contain the "

   246                     "requested package file(s):\n%(files)s.") % {

   247                     "arc_name": self.arc_name,

   248                     "files": "\n".join(self.files) }

   249 

   250 

   251 class UnknownPackageManifest(ArchiveErrors):

   252         """Used to indicate that a manifest for the specified package could not

   253         be found in the archive.

   254         """

   255 

   256         def __init__(self, arc_name, pfmri):

   257                 ArchiveErrors.__init__(self)

   258                 self.arc_name = arc_name

   259                 self.pfmri = pfmri

   260 

   261         def __str__(self):

   262                 return _("No package manifest for package '%(pfmri)s' exists "

   263                     "in archive %(arc_name)s.") % self.__dict__

   264 

   265 

   266 class Archive(object):

   267         """Class representing a pkg(5) archive and a set of interfaces to

   268         populate it and retrieve data from it.

   269 

   270         This class stores package data in pax archives in version 4 repository

   271         format.  Encoding the structure of a repository into the archive is

   272         necessary to enable easy composition of package archive contents with

   273         existing repositories and to enable consumers to access the contents of

   274         a package archive the same as they would a repository.

   275 

   276         This class can be used to access or extract the contents of almost any

   277         tar archive, except for those that are compressed.

   278         """

   279 

   280         __idx_pfx = "pkg5.index."

   281         __idx_sfx = ".gz"

   282         __idx_name = "pkg5.index.%s.gz"

   283         __idx_ver = ArchiveIndex.CURRENT_VERSION

   284         __index = None

   285         __arc_tfile = None

   286         __arc_file = None

   287         version = None

   288 

   289         # If the repository format changes, then the version of the package

   290         # archive format should be rev'd and this updated.  (Although that isn't

   291         # strictly necessary, as the Repository class should remain backwards

   292         # compatible with this format.)

   293         CURRENT_VERSION = 0

   294         COMPATIBLE_VERSIONS = (0,)

   295 

   296         def __init__(self, pathname, mode="r"):

   297                 """'pathname' is the absolute path of the archive file to create

   298                 or read from.

   299 

   300                 'mode' is a string used to indicate whether the archive is being

   301                 opened for reading or writing, which is indicated by 'r' and 'w'

   302                 respectively.  An archive opened for writing may not be used for

   303                 any extraction operations, and must not already exist.

   304                 """

   305 

   306                 assert os.path.isabs(pathname)

   307                 self.__arc_name = pathname

   308                 self.__closed = False

   309                 self.__mode = mode

   310                 self.__temp_dir = tempfile.mkdtemp()

   311 

   312                 # Used to cache publisher objects.

   313                 self.__pubs = None

   314 

   315                 # Used to cache location of publisher catalog data.

   316                 self.__catalogs = {}

   317 

   318                 arc_mode = mode + "b"

   319                 mode += ":"

   320 

   321                 assert "r" in mode or "w" in mode

   322                 assert "a" not in mode

   323                 if "w" in mode:

   324                         # Don't allow overwrite of existing archive.

   325                         assert not os.path.exists(self.__arc_name)

   326 

   327                 try:

   328                         self.__arc_file = open(self.__arc_name, arc_mode,

   329                             128*1024)

   330                 except EnvironmentError, e:

   331                         if e.errno in (errno.ENOENT, errno.EISDIR):

   332                                 raise InvalidArchive(self.__arc_name)

   333                         raise apx._convert_error(e)

   334 

   335                 self.__queue_offset = 0

   336                 self.__queue = collections.deque()

   337 

   338                 # Ensure cleanup is performed on exit if the archive is not

   339                 # explicitly closed.

   340                 def arc_cleanup():

   341                         if not self.__closed:

   342                                 self.__close_fh()

   343                         self.__cleanup()

   344                         return

   345                 atexit.register(arc_cleanup)

   346 

   347                 # Open the pax archive for the package.

   348                 try:

   349                         self.__arc_tfile = ptf.PkgTarFile.open(mode=mode,

   350                             fileobj=self.__arc_file, format=tf.PAX_FORMAT)

   351                 except EnvironmentError, e:

   352                         raise apx._convert_error(e)

   353                 except Exception:

   354                         # Likely not an archive or the archive is corrupt.

   355                         raise InvalidArchive(self.__arc_name)

   356 

   357                 self.__extract_offsets = {}

   358                 if "r" in mode:

   359                         # Opening the tarfile loaded the first member, which

   360                         # should be the archive index file.

   361                         member = self.__arc_tfile.firstmember

   362                         if not member:

   363                                 # Archive is empty.

   364                                 raise InvalidArchive(self.__arc_name)

   365 

   366                         if not member.name.startswith(self.__idx_pfx) or \

   367                             not member.name.endswith(self.__idx_sfx):

   368                                 return

   369                         else:

   370                                 self.__idx_name = member.name

   371 

   372                         comment = member.pax_headers.get("comment", "")

   373                         if not comment.startswith("pkg5.archive.version."):

   374                                 return

   375 

   376                         try:

   377                                 self.version = int(comment.rsplit(".", 1)[-1])

   378                         except (IndexError, ValueError):

   379                                 raise InvalidArchive(self.__arc_name)

   380 

   381                         if self.version not in self.COMPATIBLE_VERSIONS:

   382                                 raise InvalidArchive(self.__arc_name)

   383 

   384                         # Create a temporary file to extract the index to,

   385                         # and then extract it from the archive.

   386                         fobj, idxfn = self.__mkstemp()

   387                         fobj.close()

   388                         try:

   389                                 self.__arc_tfile.extract_to(member,

   390                                     path=self.__temp_dir,

   391                                     filename=os.path.basename(idxfn))

   392                         except tf.TarError:

   393                                 # Read error encountered.

   394                                 raise InvalidArchive(self.__arc_name)

   395                         except EnvironmentError, e:

   396                                 raise apx._convert_error(e)

   397 

   398                         # After extraction, the current archive file offset

   399                         # is the base that will be used for all other

   400                         # extractions.

   401                         index_offset = self.__arc_tfile.offset

   402 

   403                         # Load archive index.

   404                         try:

   405                                 self.__index = ArchiveIndex(idxfn, mode="r",

   406                                     version=self.__idx_ver)

   407                                 for name, offset in self.__index.offsets():

   408                                         self.__extract_offsets[name] = \

   409                                             index_offset + offset

   410                         except InvalidArchiveIndex:

   411                                 # Index is corrupt; rather than driving on

   412                                 # and failing later, bail now.

   413                                 os.unlink(idxfn)

   414                                 raise InvalidArchive(self.__arc_name)

   415                         except EnvironmentError, e:

   416                                 raise apx._convert_error(e)

   417 

   418                 elif "w" in mode:

   419                         self.__pubs = {}

   420 

   421                         # Force normalization of archive member mode and

   422                         # ownership information during archive creation.

   423                         def gettarinfo(*args, **kwargs):

   424                                 ti = ptf.PkgTarFile.gettarinfo(self.__arc_tfile,

   425                                     *args, **kwargs)

   426                                 if ti.isreg():

   427                                         ti.mode = pkg.misc.PKG_FILE_MODE

   428                                 elif ti.isdir():

   429                                         ti.mode = pkg.misc.PKG_DIR_MODE

   430                                 if ti.name == "pkg5.index.0.gz":

   431                                         ti.pax_headers["comment"] = \

   432                                             "pkg5.archive.version.%d" % \

   433                                             self.CURRENT_VERSION

   434                                 ti.uid = 0

   435                                 ti.gid = 0

   436                                 ti.uname = "root"

   437                                 ti.gname = "root"

   438                                 return ti

   439                         self.__arc_tfile.gettarinfo = gettarinfo

   440 

   441                         self.__idx_name = self.__idx_name % self.__idx_ver

   442 

   443                         # Create a temporary file to write the index to,

   444                         # and then create the index.

   445                         fobj, idxfn = self.__mkstemp()

   446                         fobj.close()

   447                         self.__index = ArchiveIndex(idxfn, mode=arc_mode)

   448 

   449                         # Used to determine what the default publisher will be

   450                         # for the archive file at close().

   451                         self.__default_pub = ""

   452 

   453                         # Used to keep track of which package files have already

   454                         # been added to archive.

   455                         self.__processed_pfiles = set()

   456 

   457                         # Always create archives using current version.

   458                         self.version = self.CURRENT_VERSION

   459 

   460                         # Always add base publisher directory to start; tarfile

   461                         # requires an actual filesystem object to do this, so

   462                         # re-use an existing directory to do so.

   463                         self.add("/", arcname="publisher")

   464 

   465         def __exit__(self, exc_type, exc_value, exc_tb):

   466                 """Context handler that ensures archive is automatically closed

   467                 in a non-error condition scenario.  This enables 'with' usage.

   468                 """

   469 

   470                 if exc_type or exc_value or exc_tb:

   471                         # Only close file objects; don't actually write anything

   472                         # out in an error condition.

   473                         self.__close_fh()

   474                         return

   475 

   476                 # Close and/or write out archive as needed.

   477                 self.close()

   478 

   479         def __find_extract_offsets(self):

   480                 """Private helper method to find offsets for individual archive

   481                 member extraction.

   482                 """

   483 

   484                 if self.__extract_offsets:

   485                         return

   486 

   487                 # This causes the entire archive to be read, but is the only way

   488                 # to find the offsets to extract everything.

   489                 try:

   490                         for member in self.__arc_tfile.getmembers():

   491                                 self.__extract_offsets[member.name] = \

   492                                     member.offset

   493                 except tf.TarError:

   494                         # Read error encountered.

   495                         raise InvalidArchive(self.__arc_name)

   496                 except EnvironmentError, e:

   497                         raise apx._convert_error(e)

   498 

   499         def __mkdtemp(self):

   500                 """Creates a temporary directory for use during archive

   501                 operations, and return its absolute path.  The temporary

   502                 directory will be removed after the archive is closed.

   503                 """

   504 

   505                 try:

   506                         return tempfile.mkdtemp(dir=self.__temp_dir)

   507                 except EnvironmentError, e:

   508                         raise apx._convert_error(e)

   509 

   510         def __mkstemp(self):

   511                 """Creates a temporary file for use during archive operations,

   512                 and returns a file object for it and its absolute path.  The

   513                 temporary file will be removed after the archive is closed.

   514                 """

   515                 try:

   516                         fd, fn = tempfile.mkstemp(dir=self.__temp_dir)

   517                         fobj = os.fdopen(fd, "wb")

   518                 except EnvironmentError, e:

   519                         raise apx._convert_error(e)

   520                 return fobj, fn

   521 

   522         def add(self, pathname, arcname=None):

   523                 """Queue the specified object for addition to the archive.

   524                 The archive will be created and the object added to it when the

   525                 close() method is called.  The target object must not change

   526                 after this method is called while the archive is open.  The

   527                 item being added must not already exist in the archive.

   528 

   529                 'pathname' is an optional string specifying the absolute path

   530                 of a file to add to the archive.  The file may be a regular

   531                 file, directory, symbolic link, or hard link. 

   532 

   533                 'arcname' is an optional string specifying an alternative name

   534                 for the file in the archive.  If not given, the full pathname

   535                 provided will be used.

   536                 """

   537 

   538                 assert not self.__closed and "w" in self.__mode

   539                 tfile = self.__arc_tfile

   540                 ti = tfile.gettarinfo(pathname, arcname=arcname)

   541                 buf = ti.tobuf(tfile.format, tfile.encoding, tfile.errors)

   542 

   543                 # Pre-calculate size of archive entry by determining where

   544                 # in the archive the entry would be added.

   545                 entry_sz = len(buf)

   546                 blocks, rem = divmod(ti.size, tf.BLOCKSIZE)

   547                 if rem > 0:

   548                         blocks += 1

   549                 entry_sz += blocks * tf.BLOCKSIZE

   550 

   551                 # Record name, offset, entry_size, size type for each file.

   552                 self.__index.add(ti.name, self.__queue_offset, entry_sz,

   553                     ti.size, ti.type)

   554                 self.__queue_offset += entry_sz

   555                 self.__queue.append((pathname, ti.name))

   556 

   557                 # Discard tarinfo; it would be more efficient to keep these in

   558                 # memory, but at a significant memory footprint cost.

   559                 ti.tarfile = None

   560                 del ti

   561 

   562         def __add_publisher_files(self, root, file_dir, hashes, fpath=None,

   563             repo=None):

   564                 """Private helper function for adding package files."""

   565 

   566                 if file_dir not in self.__processed_pfiles:

   567                         # Directory entry needs to be added

   568                         # for package files.

   569                         self.add(root, arcname=file_dir)

   570                         self.__processed_pfiles.add(file_dir)

   571 

   572                 for fhash in hashes:

   573                         hash_dir = os.path.join(file_dir, fhash[:2])

   574                         if hash_dir not in self.__processed_pfiles:

   575                                 # Directory entry needs to be added

   576                                 # for hash directory.

   577                                 self.add(root, arcname=hash_dir)

   578                                 self.__processed_pfiles.add(hash_dir)

   579 

   580                         hash_fname = os.path.join(hash_dir, fhash)

   581                         if hash_fname in self.__processed_pfiles:

   582                                 # Already added for a different

   583                                 # package.

   584                                 continue

   585 

   586                         if repo:

   587                                 src = repo.file(fhash)

   588                         else:

   589                                 src = os.path.join(fpath, fhash)

   590                         self.add(src, arcname=hash_fname)

   591 

   592                         # A bit expensive potentially in terms of

   593                         # memory usage, but necessary to prevent

   594                         # duplicate archive entries.

   595                         self.__processed_pfiles.add(hash_fname)

   596 

   597         def __add_package(self, pfmri, mpath, fpath=None, repo=None):

   598                 """Private helper function that queues a package for addition to

   599                 the archive.

   600 

   601                 'mpath' is the absolute path of the package manifest file.

   602 

   603                 'fpath' is an optional directory containing the package files

   604                 stored by hash.

   605 

   606                 'repo' is an optional Repository object to use to retrieve the

   607                 data for the package to be added to the archive.

   608 

   609                 'fpath' or 'repo' must be provided.

   610                 """

   611 

   612                 assert not self.__closed and "w" in self.__mode

   613                 assert mpath

   614                 assert not (fpath and repo)

   615                 assert fpath or repo

   616 

   617                 if not self.__default_pub:

   618                         self.__default_pub = pfmri.publisher

   619 

   620                 m = pkg.manifest.Manifest(pfmri)

   621                 m.set_content(pathname=mpath)

   622 

   623                 # Throughout this function, the archive root directory is used

   624                 # as a template to add other directories that should be present

   625                 # in the archive.  This is necessary as the tarfile class does

   626                 # not support adding arbitrary archive entries without a real

   627                 # filesystem object as a source.

   628                 root = os.path.dirname(self.__arc_name)

   629                 pub_dir = os.path.join("publisher", pfmri.publisher)

   630                 pkg_dir = os.path.join(pub_dir, "pkg")

   631                 for d in pub_dir, pkg_dir:

   632                         if d not in self.__processed_pfiles:

   633                                 self.add(root, arcname=d)

   634                                 self.__processed_pfiles.add(d)

   635 

   636                 # After manifest has been loaded, assume it's ok to queue the

   637                 # manifest itself for addition to the archive.

   638                 arcname = os.path.join(pkg_dir, pfmri.get_dir_path())

   639 

   640                 # Entry may need to be added for manifest directory.

   641                 man_dir = os.path.dirname(arcname)

   642                 if man_dir not in self.__processed_pfiles:

   643                         self.add(root, arcname=man_dir)

   644                         self.__processed_pfiles.add(man_dir)

   645 

   646                 # Entry needs to be added for manifest file.

   647                 self.add(mpath, arcname=arcname)

   648 

   649                 # Now add any files to the archive for every action that has a

   650                 # payload.  (That payload can consist of multiple files.)

   651                 file_dir = os.path.join(pub_dir, "file")

   652                 for a in m.gen_actions():

   653                         if not a.has_payload or not a.hash:

   654                                 # Nothing to archive.

   655                                 continue

   656 

   657                         payloads = set([a.hash])

   658 

   659                         # Signature actions require special handling.

   660                         if a.name == "signature":

   661                                 payloads.update(a.attrs.get("chain",

   662                                     "").split())

   663 

   664                                 if repo:

   665                                         # This bit of logic only possible if

   666                                         # package source is a repository.

   667                                         pub = self.__pubs.get(pfmri.publisher,

   668                                             None)

   669                                         if not pub:

   670                                                 self.__pubs[pfmri.publisher] = \

   671                                                     pub = repo.get_publisher(

   672                                                     pfmri.publisher)

   673                                                 assert pub

   674 

   675                                         payloads.update(pub.signing_ca_certs)

   676                                         payloads.update(pub.intermediate_certs)

   677 

   678                         if not payloads:

   679                                 # Nothing more to do.

   680                                 continue

   681 

   682                         self.__add_publisher_files(root, file_dir, payloads,

   683                              fpath=fpath, repo=repo)

   684 

   685         def add_package(self, pfmri, mpath, fpath):

   686                 """Queues the specified package for addition to the archive.

   687                 The archive will be created and the package added to it when

   688                 the close() method is called.  The package contents must not

   689                 change after this method is called while the archive is open.

   690                 Please note that, for signed packages, signing certificates

   691                 used by the publisher are not automatically added to the

   692                 archive.

   693 

   694                 'pfmri' is the FMRI string or object identifying the package to

   695                 add.

   696 

   697                 'mpath' is the absolute path of the package manifest file.

   698 

   699                 'fpath' is the directory containing the package files stored

   700                 by hash.

   701                 """

   702 

   703                 assert pfmri and mpath and fpath

   704                 if isinstance(pfmri, basestring):

   705                         pfmri = pkg.fmri.PkgFmri(pfmri)

   706                 assert pfmri.publisher

   707                 self.__add_package(pfmri, mpath, fpath=fpath)

   708 

   709         def add_signing_certs(self, pub, hashes, ca):

   710                 """Queues the specified publisher certs for addition to the

   711                 archive. The archive will be created and the certs added to it

   712                 when the close() method is called.  The cert contents must not

   713                 change after this method is called while the archive is open.

   714 

   715                 'pub' is the prefix of the publisher to store the package

   716                 files for.

   717 

   718                 'hashes' is the list of certificate hash files to store.

   719                 (The certificate files must be in the same compressed format

   720                 that the Repository class stores them in.)

   721 

   722                 'ca' is a boolean indicating whether the certs are added as

   723                 as CA certificates or intermediate certificates.

   724                 """

   725 

   726                 root = os.path.dirname(self.__arc_name)

   727                 pub_dir = os.path.join("publisher", pub)

   728                 file_dir = os.path.join(pub_dir, "file")

   729 

   730                 pubobj = self.__pubs.get(pub, None) 

   731                 if not pubobj:

   732                         self.__pubs[pub] = pubobj = \

   733                             pkg.client.publisher.Publisher(pub)

   734 

   735                 for fname in hashes:

   736                         hsh = os.path.basename(fname)

   737                         self.__add_publisher_files(root, file_dir, [hsh],

   738                             fpath=os.path.dirname(fname))

   739                         if ca:

   740                                 pubobj.signing_ca_certs.append(hsh)

   741                         else:

   742                                 pubobj.intermediate_certs.append(hsh)

   743 

   744         def add_repo_package(self, pfmri, repo):

   745                 """Queues the specified package in a repository for addition to

   746                 the archive. The archive will be created and the package added

   747                 to it when the close() method is called.  The package contents

   748                 must not change after this method is called while the archive is

   749                 open.

   750 

   751                 'pfmri' is the FMRI string or object identifying the package to

   752                 add.

   753 

   754                 'repo' is the Repository object to use to retrieve the data for

   755                 the package to be added to the archive.

   756                 """

   757 

   758                 assert pfmri and repo

   759                 if isinstance(pfmri, basestring):

   760                         pfmri = pkg.fmri.PkgFmri(pfmri)

   761                 assert pfmri.publisher

   762                 self.__add_package(pfmri, repo.manifest(pfmri), repo=repo)

   763 

   764         def extract_catalog1(self, part, path, pub=None):

   765                 """Extract the named v1 catalog part to the specified directory.

   766 

   767                 'part' is the name of the catalog file part.

   768 

   769                 'path' is the absolute path of the directory to extract the

   770                 file to.  It will be created automatically if it does not

   771                 exist.

   772 

   773                 'pub' is an optional publisher prefix.  If not provided, the

   774                 first publisher catalog found in the archive will be used.

   775                 """

   776 

   777                 # If the extraction index doesn't exist, scan the

   778                 # complete archive and build one.

   779                 self.__find_extract_offsets()

   780 

   781                 pubs = [

   782                     p for p in self.get_publishers()

   783                     if not pub or p.prefix == pub

   784                 ]

   785                 if not pubs:

   786                         raise UnknownArchiveFiles(self.__arc_name, [part])

   787 

   788                 if not pub:

   789                         # Default to first known publisher.

   790                         pub = pubs[0].prefix

   791 

   792                 # Expected locations in archive for various metadata.

   793                 # A trailing slash is appended so that archive entry

   794                 # comparisons skip the entries for the directory.

   795                 pubpath = os.path.join("publisher", pub) + os.path.sep

   796                 catpath = os.path.join(pubpath, "catalog") + os.path.sep

   797                 partpath = os.path.join(catpath, part)

   798 

   799                 if pub in self.__catalogs:

   800                         # Catalog file requested for this publisher before.

   801                         croot = self.__catalogs[pub]

   802                         if croot:

   803                                 # Catalog data is cached because it was

   804                                 # generated on demand, so just copy it

   805                                 # from there to the destination.

   806                                 src = os.path.join(croot, part)

   807                                 if not os.path.exists(src):

   808                                         raise UnknownArchiveFiles(

   809                                             self.__arc_name, [partpath])

   810 

   811                                 try:

   812                                         pkg.portable.copyfile(

   813                                             os.path.join(croot, part),

   814                                             os.path.join(path, part))

   815                                 except EnvironmentError, e:

   816                                         raise apx._convert_error(e)

   817                         else:

   818                                 # Use default extraction logic.

   819                                 self.extract_to(partpath, path, filename=part)

   820                         return

   821 

   822                 # Determine whether any catalog files are present for this

   823                 # publisher in the archive.

   824                 for name in self.__extract_offsets:

   825                         if name.startswith(catpath):

   826                                 # Any catalog file at all means this publisher

   827                                 # should be marked as being known to have one

   828                                 # and then the request passed on to extract_to.

   829                                 self.__catalogs[pub] = None

   830                                 return self.extract_to(partpath, path,

   831                                     filename=part)

   832 

   833                 # No catalog data found for publisher; construct a catalog

   834                 # in memory based on packages found for publisher.

   835                 cat = pkg.catalog.Catalog(batch_mode=True, sign=False)

   836                 manpath = os.path.join(pubpath, "pkg") + os.path.sep

   837                 for name in self.__extract_offsets:

   838                         if name.startswith(manpath) and name.count("/") == 4:

   839                                 ignored, stem, ver = name.rsplit("/", 2)

   840                                 stem = urllib.unquote(stem)

   841                                 ver = urllib.unquote(ver)

   842                                 pfmri = pkg.fmri.PkgFmri("%s@%s" % (stem, ver),

   843                                     publisher=pub)

   844 

   845                                 fobj = self.get_file(name)

   846                                 m = pkg.manifest.Manifest(pfmri=pfmri)

   847                                 m.set_content(content=fobj.read(),

   848                                     signatures=True)

   849                                 cat.add_package(pfmri, manifest=m)

   850 

   851                 # Store catalog in a temporary directory and mark publisher

   852                 # as having catalog data cached.

   853                 croot = self.__mkdtemp()

   854                 cat.meta_root = croot

   855                 cat.batch_mode = False

   856                 cat.finalize()

   857                 cat.save()

   858                 self.__catalogs[pub] = croot

   859 

   860                 # Finally, copy requested file to destination.

   861                 try:

   862                         pkg.portable.copyfile(os.path.join(croot, part),

   863                             os.path.join(path, part))

   864                 except EnvironmentError, e:

   865                         raise apx._convert_error(e)

   866 

   867         def extract_package_files(self, hashes, path, pub=None):

   868                 """Extract one or more package files from the archive.

   869 

   870                 'hashes' is a list of the files to extract named by their hash.

   871 

   872                 'path' is the absolute path of the directory to extract the

   873                 files to.  It will be created automatically if it does not

   874                 exist.

   875 

   876                 'pub' is the prefix (name) of the publisher that the package

   877                 files are associated with.  If not provided, the first file

   878                 named after the given hash found in the archive will be used.

   879                 (This will be noticeably slower depending on the size of the

   880                 archive.)

   881                 """

   882 

   883                 assert not self.__closed and "r" in self.__mode

   884                 assert hashes

   885 

   886                 # If the extraction index doesn't exist, scan the complete

   887                 # archive and build one.

   888                 self.__find_extract_offsets()

   889 

   890                 if not pub:

   891                         # Scan extract offsets index for the first instance of

   892                         # any package file seen for each hash and extract the

   893                         # file as each is found.

   894                         hashes = set(hashes)

   895 

   896                         for name in self.__extract_offsets:

   897                                 for fhash in hashes:

   898                                         hash_fname = os.path.join("file",

   899                                             fhash[:2], fhash)

   900                                         if name.endswith(hash_fname):

   901                                                 self.extract_to(name, path,

   902                                                     filename=fhash)

   903                                                 hashes.discard(fhash)

   904                                                 break

   905                                 if not hashes:

   906                                         break

   907 

   908                         if hashes:

   909                                 # Any remaining hashes are for package files

   910                                 # that couldn't be found.

   911                                 raise UnknownArchiveFiles(self.__arc_name,

   912                                     hashes)

   913                         return

   914 

   915                 for fhash in hashes:

   916                         arcname = os.path.join("publisher", pub, "file",

   917                             fhash[:2], fhash)

   918                         self.extract_to(arcname, path, filename=fhash)

   919 

   920         def extract_package_manifest(self, pfmri, path, filename=""):

   921                 """Extract a package manifest from the archive.

   922 

   923                 'pfmri' is the FMRI string or object identifying the package

   924                 manifest to extract.

   925 

   926                 'path' is the absolute path of the directory to extract the

   927                 manifest to.  It will be created automatically if it does not

   928                 exist.

   929 

   930                 'filename' is an optional name to use for the extracted file.

   931                 If not provided, the default behaviour is to create a directory

   932                 named after the package stem in 'path' and a file named after

   933                 the version in that directory; both components will be URI

   934                 encoded.

   935                 """

   936 

   937                 assert not self.__closed and "r" in self.__mode

   938                 assert pfmri and path

   939                 if isinstance(pfmri, basestring):

   940                         pfmri = pkg.fmri.PkgFmri(pfmri)

   941                 assert pfmri.publisher

   942 

   943                 if not filename:

   944                         filename = pfmri.get_dir_path()

   945 

   946                 arcname = os.path.join("publisher", pfmri.publisher, "pkg",

   947                     pfmri.get_dir_path())

   948                 try:

   949                         self.extract_to(arcname, path, filename=filename)

   950                 except UnknownArchiveFiles:

   951                         raise UnknownPackageManifest(self.__arc_name, pfmri)

   952 

   953         def extract_to(self, src, path, filename=""):

   954                 """Extract a member from the archive.

   955 

   956                 'src' is the pathname of the archive file to extract.

   957 

   958                 'path' is the absolute path of the directory to extract the file

   959                 to.

   960 

   961                 'filename' is an optional string indicating the name to use for

   962                 the extracted file.  If not provided, the full member name in

   963                 the archive will be used.

   964                 """

   965 

   966                 assert not self.__closed and "r" in self.__mode

   967 

   968                 # Get the offset in the archive for the given file, and then

   969                 # seek to it.

   970                 offset = self.__extract_offsets.get(src, None)

   971                 tfile = self.__arc_tfile

   972                 if offset is not None:

   973                         # Prepare the tarfile object for extraction by telling

   974                         # it where to look for the file.

   975                         self.__arc_file.seek(offset)

   976                         tfile.offset = offset

   977 

   978                         # Get the tarinfo object needed to extract the file.

   979                         try:

   980                                 member = tf.TarInfo.fromtarfile(tfile)

   981                         except tf.TarError:

   982                                 # Read error encountered.

   983                                 raise InvalidArchive(self.__arc_name)

   984                         except EnvironmentError, e:

   985                                 raise apx._convert_error(e)

   986 

   987                         if member.name != src:

   988                                 # Index must be invalid or tarfile has gone off

   989                                 # the rails trying to read the archive.

   990                                 raise InvalidArchive(self.__arc_name)

   991 

   992                 elif self.__extract_offsets:

   993                         # Assume there is no such archive member if extract

   994                         # offsets are known, but the item can't be found.

   995                         raise UnknownArchiveFiles(self.__arc_name, [src])

   996                 else:

   997                         # No archive index; fallback to retrieval by name.

   998                         member = src

   999 

  1000                 # Extract the file to the specified location.

  1001                 try:

  1002                         self.__arc_tfile.extract_to(member, path=path,

  1003                             filename=filename)

  1004                 except KeyError:

  1005                         raise UnknownArchiveFiles(self.__arc_name, [src])

  1006                 except tf.TarError:

  1007                         # Read error encountered.

  1008                         raise InvalidArchive(self.__arc_name)

  1009                 except EnvironmentError, e:

  1010                         raise apx._convert_error(e)

  1011 

  1012                 if not isinstance(member, tf.TarInfo):

  1013                         # Nothing more to do.

  1014                         return

  1015 

  1016                 # If possible, validate the size of the extracted object.

  1017                 try:

  1018                         if not filename:

  1019                                 filename = member.name

  1020                         dest = os.path.join(path, filename)

  1021                         if os.stat(dest).st_size != member.size:

  1022                                 raise CorruptArchiveFiles(self.__arc_name,

  1023                                     [src])

  1024                 except EnvironmentError, e:

  1025                         raise apx._convert_error(e)

  1026 

  1027         def get_file(self, src):

  1028                 """Returns an archive member as a file object.  If the matching

  1029                 member is a regular file, a file-like object will be returned.

  1030                 If it is a link, a file-like object is constructed from the

  1031                 link's target.  In all other cases, None will be returned.  The

  1032                 file-like object is read-only and provides methods: read(),

  1033                 readline(), readlines(), seek() and tell().  The returned object

  1034                 must be closed before the archive is, and must not be used after

  1035                 the archive is closed.

  1036 

  1037                 'src' is the pathname of the archive file to return.

  1038                 """

  1039 

  1040                 assert not self.__closed and "r" in self.__mode

  1041 

  1042                 # Get the offset in the archive for the given file, and then

  1043                 # seek to it.

  1044                 offset = self.__extract_offsets.get(src, None)

  1045                 tfile = self.__arc_tfile

  1046                 if offset is not None:

  1047                         # Prepare the tarfile object for extraction by telling

  1048                         # it where to look for the file.

  1049                         self.__arc_file.seek(offset)

  1050                         tfile.offset = offset

  1051 

  1052                         # Get the tarinfo object needed to extract the file.

  1053                         member = tf.TarInfo.fromtarfile(tfile)

  1054                 elif self.__extract_offsets:

  1055                         # Assume there is no such archive member if extract

  1056                         # offsets are known, but the item can't be found.

  1057                         raise UnknownArchiveFiles(self.__arc_name, [src])

  1058                 else:

  1059                         # No archive index; fallback to retrieval by name.

  1060                         member = src

  1061 

  1062                 # Finally, return the object for the matching archive member.

  1063                 try:

  1064                         return tfile.extractfile(member)

  1065                 except KeyError:

  1066                         raise UnknownArchiveFiles(self.__arc_name, [src])

  1067 

  1068         def get_package_file(self, fhash, pub=None):

  1069                 """Returns the first package file matching the given hash as a

  1070                 file-like object. The file-like object is read-only and provides

  1071                 methods: read(), readline(), readlines(), seek() and tell().

  1072                 The returned object  must be closed before the archive is, and

  1073                 must not be used after the archive is closed.

  1074 

  1075                 'fhash' is the hash name of the file to return.

  1076 

  1077                 'pub' is the prefix (name) of the publisher that the package

  1078                 files are associated with.  If not provided, the first file

  1079                 named after the given hash found in the archive will be used.

  1080                 (This will be noticeably slower depending on the size of the

  1081                 archive.)

  1082                 """

  1083 

  1084                 assert not self.__closed and "r" in self.__mode

  1085 

  1086                 if not self.__extract_offsets:

  1087                         # If the extraction index doesn't exist, scan the

  1088                         # complete archive and build one.

  1089                         self.__find_extract_offsets()

  1090 

  1091                 if not pub:

  1092                         # Scan extract offsets index for the first instance of

  1093                         # any package file seen for the hash and extract it.

  1094                         hash_fname = os.path.join("file", fhash[:2], fhash)

  1095                         for name in self.__extract_offsets:

  1096                                 if name.endswith(hash_fname):

  1097                                         return self.get_file(name)

  1098                         raise UnknownArchiveFiles(self.__arc_name, [fhash])

  1099 

  1100                 return self.get_file(os.path.join("publisher", pub, "file",

  1101                     fhash[:2], fhash))

  1102 

  1103         def get_package_manifest(self, pfmri, raw=False):

  1104                 """Returns a package manifest from the archive.

  1105 

  1106                 'pfmri' is the FMRI string or object identifying the package

  1107                 manifest to extract.

  1108 

  1109                 'raw' is an optional boolean indicating whether the raw

  1110                 content of the Manifest should be returned.  If True,

  1111                 a file-like object containing the content of the manifest.

  1112                 If False, a Manifest object will be returned.

  1113                 """

  1114 

  1115                 assert not self.__closed and "r" in self.__mode

  1116                 assert pfmri

  1117                 if isinstance(pfmri, basestring):

  1118                         pfmri = pkg.fmri.PkgFmri(pfmri)

  1119                 assert pfmri.publisher

  1120 

  1121                 arcname = os.path.join("publisher", pfmri.publisher, "pkg",

  1122                     pfmri.get_dir_path())

  1123 

  1124                 try:

  1125                         fobj = self.get_file(arcname)

  1126                 except UnknownArchiveFiles:

  1127                         raise UnknownPackageManifest(self.__arc_name, pfmri)

  1128 

  1129                 if raw:

  1130                         return fobj

  1131 

  1132                 m = pkg.manifest.Manifest(pfmri=pfmri)

  1133                 m.set_content(content=fobj.read(), signatures=True)

  1134                 return m

  1135