# HG changeset patch

# User Michael Gerdts <[email protected]>

# Date 1412600364 25200

#      Mon Oct 06 05:59:24 2014 -0700

# Node ID ba834c48cee26e4c43976af8477dd34863b40f18

# Parent  1debb63439545fd0b30153eb68e884623d06c531

parallel uncompress - developed by Oracle

Offered to upstream at https://github.com/mgerdts/pigz

 - Branch mt-uncompress-2.2 forked from https://github.com/madler/pigz v. 2.2.6

 - Branch mt-uncompress forked from https://github.com/madler/pigz branch master

diff -r 1debb6343954 -r ba834c48cee2 Makefile

--- a/Makefile

+++ b/Makefile

@@ -44,6 +44,15 @@

 	  compress -f < pigz.c | ./unpigz | cmp - pigz.c ;\

 	fi

 	@rm -f pigz.c.gz pigz.c.zz pigz.c.zip

+	@rm -rf d/1 d/2

+	(mkdir -p d/1; cd d/1; tar xzf ../../../../pigz-2.2.5.tar.gz; \

+	  cd ..; cp -pr 1 2; ../pigz -rp 4 --index %z 1; \

+	  ../pigz -drp 4 --index %z 1; diff -r 1 2)

+	@rm -rf d/1 d/2

+	(mkdir -p d/1; cd d/1; tar xzf ../../../../pigz-2.2.5.tar.gz; \

+	  cd ..; cp -pr 1 2; ../pigz -zrp 4 -X %f.idx 1; \

+	  ../pigz -dzrp 4 -X %f.idx 1; diff -r 1 2)

+	@rm -rf d/1 d/2

 tests: dev test

 	./pigzn -kf pigz.c ; ./pigz -t pigz.c.gz

diff -r 1debb6343954 -r ba834c48cee2 pigz.1

--- a/pigz.1

+++ b/pigz.1

@@ -180,6 +180,14 @@

 .B -V --version

 Show the version of pigz.

 .TP

+.B -X --index file

+During compression, create an index that can be used for parallel

+decompression.  During decompression, use the specified index file for parallel

+decompression.  Each occurrence of %f and %z are replaced by the uncompressed

+and compressed file names, respectively.  If the index file is the same file as

+the compressed file, the index is written to or read from the end of the

+compressed file.

+.TP

 .B -z --zlib

 Compress to zlib (.zz) instead of gzip format.

 .TP

diff -r 1debb6343954 -r ba834c48cee2 pigz.c

--- a/pigz.c

+++ b/pigz.c

@@ -191,13 +191,27 @@

    effectiveness of deflating in a single thread.  This can be turned off using

    the --independent or -i option, so that the blocks can be decompressed

    independently for partial error recovery or for random access.

-

-   Decompression can't be parallelized, at least not without specially prepared

-   deflate streams for that purpose.  As a result, pigz uses a single thread

-   (the main thread) for decompression, but will create three other threads for

-   reading, writing, and check calculation, which can speed up decompression

-   under some circumstances.  Parallel decompression can be turned off by

-   specifying one process (-dp 1 or -tp 1).

+   

+   The --index or -X option causes the generation of a block index which can be

+   used for parallel decompression.  The block index can be appended onto the

+   compressed output or it may be stored in a separate file.  The uncompressed

+   size, compressed size, checksum of each block are stored in the index,

+   allowing future applications to perform random reads of the compressed file.

+   Streams generated with -X are readable by legacy versions of pigz and gzip.

+

+   Decompression can be parallelized, but only if a block index is available.

+   If a block index is not present, pigz uses a single thread (the main thread)

+   for decompression, but will create three other threads for reading, writing,

+   and check calculation, which can speed up decompression under some

+   circumstances.  Parallel decompression can be turned off by specifying one

+   process (-dp 1 or -tp 1).

+

+   If the block index is present, the main thread reads the input file and

+   dispatches each block to an uncompress thread.  The uncompress thread

+   uncompresses the block, verifies the block checksum, and passes the block

+   off to a writer thread.  The writer thread writes the blocks in order,

+   and combines the individual block checksums into a per-file checksum.  The

+   per-file checksum is compared to the checksum in the stream's trailer.

    pigz requires zlib 1.2.1 or later to allow setting the dictionary when doing

    raw deflate.  Since zlib 1.2.3 corrects security vulnerabilities in zlib

@@ -259,13 +273,14 @@

    can't get way ahead of the write thread and build up a large backlog of

    unwritten compressed data.  The write thread will write the compressed data,

    drop the output buffer, and then wait for the check value to be unlocked

-   by the compress thread.  Then the write thread combines the check value for

-   this chunk with the total check value for eventual use in the trailer.  If

-   this is not the last chunk, the write thread then goes back to look for the

-   next output chunk in sequence.  After the last chunk, the write thread

-   returns and joins the main thread.  Unlike the compress threads, a new write

-   thread is launched for each input stream.  The write thread writes the

-   appropriate header and trailer around the compressed data.

+   by the compress thread.  Then the write thread writes an index entry (if -X)

+   and combines the check value for this chunk with the total check value for

+   eventual use in the trailer.  If this is not the last chunk, the write thread

+   then goes back to look for the next output chunk in sequence.  After the last

+   chunk, the write thread returns and joins the main thread.  Unlike the

+   compress threads, a new write thread is launched for each input stream.  The

+   write thread writes the appropriate header and trailer around the compressed

+   data.

    The input and output buffers are reused through their collection in pools.

    Each buffer has a use count, which when decremented to zero returns the

@@ -313,6 +328,9 @@

 #if __STDC_VERSION__-0 >= 199901L || __GNUC__-0 >= 3

 #  include <inttypes.h> /* intmax_t */

 #endif

+#include <stddef.h>     /* offsetof() */

+#include <sys/mman.h>   /* mmap() */

+#include <netinet/in.h> /* htonl() */

 #ifdef __hpux

 #  include <sys/param.h>

@@ -420,8 +438,10 @@

 local char *prog;           /* name by which pigz was invoked */

 local int ind;              /* input file descriptor */

 local int outd;             /* output file descriptor */

+local int idxd;             /* index file descriptor */

 local char in[PATH_MAX+1];  /* input file name (accommodate recursion) */

 local char *out = NULL;     /* output file name (allocated if not NULL) */

+local char *index = NULL;   /* index file name template (may have %f, %z) */

 local int verbosity;        /* 0 = quiet, 1 = normal, 2 = verbose, 3 = trace */

 local int headis;           /* 1 to store name, 2 to store date, 3 both */

 local int pipeout;          /* write output to stdout even if file */

@@ -467,9 +487,12 @@

     return 0;

 }

+local void idx_abort(void);

+

 /* exit with error, delete output file if in the middle of writing it */

 local int bail(char *why, char *what)

 {

+    idx_abort();

     if (outd != -1 && out != NULL)

         unlink(out);

     complain("abort: %s%s", why, what);

@@ -684,11 +707,23 @@

     return dos;

 }

-/* put a 4-byte integer into a byte array in LSB order or MSB order */

+/* put integers into a byte array in LSB order or MSB order */

 #define PUT2L(a,b) (*(a)=(b)&0xff,(a)[1]=(b)>>8)

 #define PUT4L(a,b) (PUT2L(a,(b)&0xffff),PUT2L((a)+2,(b)>>16))

+#define PUT8L(a,b) (PUT4L(a,(b)&0xffffffff),PUT4L((a)+4,(b)>>32))

 #define PUT4M(a,b) (*(a)=(b)>>24,(a)[1]=(b)>>16,(a)[2]=(b)>>8,(a)[3]=(b))

+/* pull LSB order or MSB order integers from an unsigned char buffer */

+#define PULL2L(p) ((p)[0] + ((unsigned)((p)[1]) << 8))

+#define PULL4L(p) (PULL2L(p) + ((unsigned long)(PULL2L((p) + 2)) << 16))

+#define PULL8L(p) ((uint64_t)((p)[0]) | ((uint64_t)((p)[1]) << 8) | \

+                   ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \

+                   ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \

+                   ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))

+#define PULL2M(p) (((unsigned)((p)[0]) << 8) + (p)[1])

+#define PULL4M(p) (((unsigned long)(PULL2M(p)) << 16) + PULL2M((p) + 2))

+

+

 /* write a gzip, zlib, or zip header using the information in the globals */

 local unsigned long put_header(void)

 {

@@ -982,7 +1017,7 @@

 /* get a space from a pool -- the use count is initially set to one, so there

    is no need to call use_space() for the first use */

-local struct space *get_space(struct pool *pool)

+local struct space *get_space_size(struct pool *pool, size_t size)

 {

     struct space *space;

@@ -995,6 +1030,15 @@

     if (pool->head != NULL) {

         space = pool->head;

         possess(space->use);

+        /* If there's not enough space, free and malloc rather than realloc to

+           avoid the potential of an unnecessary memory copy. */

+        if (space->size < size) {

+            free(space->buf);

+            space->buf = malloc(size);

+            if (space->buf == NULL)

+                bail("not enough memory", "");

+            space->size = size;

+        }

         pool->head = space->next;

         twist(pool->have, BY, -1);      /* one less in pool */

         twist(space->use, TO, 1);       /* initially one user */

@@ -1012,15 +1056,20 @@

     if (space == NULL)

         bail("not enough memory", "");

     space->use = new_lock(1);           /* initially one user */

-    space->buf = malloc(pool->size);

+    space->buf = malloc(size);

     if (space->buf == NULL)

         bail("not enough memory", "");

-    space->size = pool->size;

+    space->size = size;

     space->len = 0;

     space->pool = pool;                 /* remember the pool this belongs to */

     return space;

 }

+local struct space *get_space(struct pool *pool)

+{

+    return get_space_size(pool, pool->size);

+}

+

 /* compute next size up by multiplying by about 2**(1/3) and round to the next

    power of 2 if we're close (so three applications results in doubling) -- if

    small, go up to at least 16, if overflow, go to max size_t value */

@@ -1109,17 +1158,35 @@

     return count;

 }

+/* prompt for permission to overwrite a file */

+local int allow_overwrite(const char *path)

+{

+    char ch;

+    int reply = -1;

+

+    fprintf(stderr, "%s exists -- overwrite (y/n)? ", path);

+    fflush(stderr);

+    do {

+        ch = getchar();

+        if (reply < 0 && ch != ' ' && ch != '\t')

+            reply = ch == 'y' || ch == 'Y' ? 1 : 0;

+    } while (ch != EOF && ch != '\n' && ch != '\r');

+    return reply;

+}

+

 /* input and output buffer pools */

 local struct pool in_pool;

 local struct pool out_pool;

 local struct pool dict_pool;

 local struct pool lens_pool;

+local struct pool idx_pool;

 /* -- parallel compression -- */

 /* compress or write job (passed from compress list to write list) -- if seq is

    equal to -1, compress_thread is instructed to return; if more is false then

-   this is the last chunk, which after writing tells write_thread to return */

+   this is the last chunk, which after writing tells compress_write_thread to

+   return */

 struct job {

     long seq;                   /* sequence number */

     int more;                   /* true if this is not the last chunk */

@@ -1166,6 +1233,7 @@

     new_pool(&out_pool, OUTPOOL(size), -1);

     new_pool(&dict_pool, DICT, -1);

     new_pool(&lens_pool, size >> (RSYNCBITS - 1), -1);

+    new_pool(&idx_pool, 1, -1);

 }

 /* command the compress threads to all return, then join them all (call from

@@ -1202,6 +1270,8 @@

     Trace(("-- freed %d output buffers", caught));

     caught = free_pool(&in_pool);

     Trace(("-- freed %d input buffers", caught));

+    caught = free_pool(&idx_pool);

+    Trace(("-- freed %d index buffers", caught));

     free_lock(write_first);

     free_lock(compress_have);

     compress_have = NULL;

@@ -1395,18 +1465,483 @@

     (void)deflateEnd(&strm);

 }

+/* Block Index

+  

+   The block index is an array of idx_entry structs followed by an idx_trailer

+   struct.  They are written to the file in LSB order.  The block index can

+   exist as a standalone file or be appended onto the compressed files.

+

+   The trailer is used to identify a block index.  The beginning of the trailer

+   contains a magic number that is a value too large to be confused with a valid

+   block length.  Aside from backwards P's the magic number looks kinda like

+   "0xf pigzip 0xf". */

+#define IDXMAGIC 0xf916219f

+

+struct idx_trailer {

+    uint32_t    magic;

+    uint64_t    count;

+};

+

+struct idx_entry {

+    uint32_t    infsz;          /* inflated size of the block */

+    uint32_t    defsz;          /* deflated size of the block */

+    uint32_t    check;          /* adler32 or crc32 checksum of the block */

+};

+

+local struct {

+    int             valid;      /* Do the rest of these fields mean anything? */

+

+    /* An array of entries.  References address in space or map */

+    struct idx_entry *ents;     /* not in right byte order, used for offset */

+    uint64_t        seq;        /* current entry */

+    int64_t         eof;        /* has the last entry been retrieved? */

+

+    /* When compressing and appending, entries are stored in space->buf.  */

+    int             append;     /* is the index at end of compressed file? */

+    struct space    *space;     /* space for storage of index */

+

+    /* The following are valid only when mmap is used. */

+    uchar_t         *map;       /* mmap'd region containing ents */

+    size_t          mapsz;      /* size of mmap'd region at map */

+    off_t           mapoff;     /* bytes between map and ents */

+

+    /* Index path, after %f and %z are replaced. */

+    char            path[PATH_MAX+1];

+} idx;

+

+/* determines if the two paths refer to the same extant file */

+local int same_file(const char *f1, const char *f2)

+{

+    struct stat s1;

+    struct stat s2;

+

+    return (stat(f1, &s1) == 0 && stat(f2, &s2) == 0 &&

+            s1.st_dev == s2.st_dev && s1.st_ino == s2.st_ino);

+}

+

+/* Remove the index file, but only if it is not the same as in or out.

+   We don't worry about a full cleanup, as this should only be called in an

+   error path just before exiting. */

+local void idx_abort(void)

+{

+    if (!idx.valid)

+        return;

+    if (idx.path[0] == '\0' || idx.append)

+        return;

+    (void) unlink(idx.path);

+}

+

+/* If 0 is returned, a trailer was found and read.  Non-zero return means

+   there was no trailer. Does not exit. Does not change file pointer for fd. */

+local int idx_read_trailer(int fd, char *path, struct idx_trailer *trail)

+{

+    uchar_t buf[sizeof(*trail)];

+    off_t off;

+    struct stat st;

+

+    if (fd < 0) {

+        Trace(("%s: index file descriptor %d not valid", path, fd));

+        return -1;

+    }

+    if (fstat(fd, &st) != 0 || !S_ISREG(st.st_mode)) {

+        Trace(("%s: index appended to non-regular file", path));

+        return -1;

+    }

+    off = st.st_size - sizeof(*trail);

+    if (off < 0) {

+        Trace(("%s: index file too short for header", path));

+        return -1;

+    }

+    if (pread(fd, buf, sizeof(buf), off) != sizeof(buf)) {

+        Trace(("%s: unable to read index trailer", path));

+        return -1;

+    }

+    trail->magic = PULL4L(buf);

+    trail->count = PULL8L(buf + 4);

+

+    if (trail->magic != IDXMAGIC) {

+        Trace(("%s: invalid pigz index magic", path));

+        return -1;

+    }

+    return 0;

+}

+

+/* Expand a path pattern containing %f and/or %z tokens into a full path.

+ * Result is stored in idx.path. */

+local int expand_pathpat(char *pathpat)

+{

+    char *copy = NULL;              /* points to in or out global */

+    char *suf = NULL;               /* suffix (.zz, .gz, etc.) */

+    int chop_suffix;

+    int len;

+    int i;

+    int j;

+    int nag;

+

+    /* Be quiet when opportunistic index use check is being done. */

+    nag = ((index == NULL) && strcmp(pathpat, "%z"));

+

+    for (i = 0, j = 0; pathpat[i] && j < sizeof(idx.path); i++) {

+        if (pathpat[i] != '%') {

+            idx.path[j++] = pathpat[i];

+            continue;

+        }

+        i++;

+        switch (pathpat[i]) {

+        case '%':               /* %% is replaced by % */

+            idx.path[j++] = '%';

+            continue;

+        case 'f':               /* %f is replaced by uncompressed file name */

+            if (decode) {

+                if (strcmp(out, "<stdout>") != 0) {

+                    copy = out;                     /* uncompressed file */

+                    chop_suffix = 0;

+                    break;

+                }

+                if (strcmp(in, "<stdin>") != 0) {

+                    copy = in;                      /* compressed file */

+                    chop_suffix = 1;

+                    suf = strrchr(in, '.');

+                    break;

+                }

+                if (nag)

+                    complain("file name for %%f unknown");

+                return -1;

+            }

+

+            if (strcmp(out, "<stdout>") != 0) {

+                copy = out;                         /* compressed file */

+                chop_suffix = 1;

+                suf = strrchr(out, '.');

+                break;

+            }

+            if (strcmp(in, "<stdin>") != 0) {

+                copy = in;                          /* uncompressed file */

+                chop_suffix = 0;

+                break;

+            }

+            if (nag)

+                complain("file name for %%f unknown");

+            return -1;

+        case 'z':               /* %z is replaced by compressed file name */

+            chop_suffix = 0;

+            if (decode) {

+                if (strcmp(in, "<stdin>") == 0) {

+                    if (nag)

+                        complain("file name for %%z unknown");

+                    return -1;

+                }

+                copy = in;

+                break;

+            }

+            if (strcmp(pathpat, "%z") == 0) {

+                /* index will be appended onto stdout */

+                copy = NULL;

+                idx.append = 1;

+                break;

+            }

+            if (strcmp(out, "<stdout>") == 0) {

+                if (nag)

+                    complain("file name for %%z unknown");

+                return -1;

+            }

+            copy = out;

+            break;

+        default:

+            if (nag) {

+                complain("invalid %% sequence in index file pattern %s",

+                         pathpat);

+            }

+            return -1;

+        }

+

+        /* pathpat is "%z" and out is stdout */

+        if (copy == NULL)

+            break;

+

+        len = strlen(&idx.path[j]) + strlen(copy);

+        if (chop_suffix)

+            len -= strlen(suf);

+        if (len >= (sizeof(idx.path) - j)) {

+            if (nag)

+                complain("index file name too long");

+            return -1;

+        }

+        (void)strncpy(&idx.path[j], copy, sizeof(idx.path) - j);

+        j += len;

+        assert(j <= sizeof(idx.path));

+    }

+    if (j == sizeof(idx.path)) {

+        idx.path[j-1] = '\0';

+        if (nag)

+            complain("index file \"%s...\" name too long", idx.path);

+        return -1;

+    }

+    idx.path[j] = '\0';

+

+    if (copy == NULL && idx.append) {

+            (void)strncpy(idx.path, out, sizeof(idx.path));

+            idx.path[sizeof(idx.path) - 1] = '\0';

+    }

+    else {

+        if (same_file(decode ? out : in, idx.path)) {

+            if (nag)

+                complain("index file %s must not be same as uncompressed file",

+                        idx.path);

+            return -1;

+        }

+

+        idx.append = same_file(decode ? in : out, idx.path);

+    }

+

+    if (verbosity > 1)

+        (void) fprintf(stderr, "index %s ", idx.path);

+

+    return 0;

+}

+

+/* open the index file associated with the current input or output file. */

+local int idx_open(char *pathpat)

+{

+    int ret;

+    struct stat st;

+

+    assert(pathpat != NULL);

+

+    memset(&idx, 0, sizeof(idx));

+

+    setup_jobs();

+

+    idxd = -1;

+

+    if (expand_pathpat(pathpat) != 0)

+        return -1;

+

+    if (decode) {                               /* Uncompress */

+        int64_t sz;

+        int64_t off;