compress -f < pigz.c | ./unpigz | cmp - pigz.c ;\

 	fi

 	@rm -f pigz.c.gz pigz.c.zz pigz.c.zip

+	@rm -rf d/1 d/2

+	  cd ..; cp -pr 1 2; ../pigz -rp 4 --index %z 1; \

+	  ../pigz -drp 4 --index %z 1; diff -r 1 2)

+	@rm -rf d/1 d/2

+	  cd ..; cp -pr 1 2; ../pigz -zrp 4 -X %f.idx 1; \

+	  ../pigz -dzrp 4 -X %f.idx 1; diff -r 1 2)

+	@rm -rf d/1 d/2

 tests: dev test

 	./pigzn -kf pigz.c ; ./pigz -t pigz.c.gz

 .B -V --version

 Show the version of pigz.

 .TP

+.B -X --index file

+During compression, create an index that can be used for parallel

+decompression.  During decompression, use the specified index file for parallel

+decompression.  Each occurrence of %f and %z are replaced by the uncompressed

+and compressed file names, respectively.  If the index file is the same file as

+the compressed file, the index is written to or read from the end of the

+compressed file.

+.TP

 .B -z --zlib

 Compress to zlib (.zz) instead of gzip format.

 .TP

    the --independent or -i option, so that the blocks can be decompressed

    independently for partial error recovery or for random access.

+   The --index or -X option causes the generation of a block index which can be

+   used for parallel decompression.  The block index can be appended onto the

+   compressed output or it may be stored in a separate file.  The uncompressed

+   size, compressed size, checksum of each block are stored in the index,

+   allowing future applications to perform random reads of the compressed file.

+   Streams generated with -X are readable by legacy versions of pigz and gzip.

+

+   Decompression can be parallelized, but only if a block index is available.

+   If a block index is not present, pigz uses a single thread (the main thread)

+   for decompression, but will create three other threads for reading, writing,

+   and check calculation, which can speed up decompression under some

+   If the block index is present, the main thread reads the input file and

+   dispatches each block to an uncompress thread.  The uncompress thread

+   uncompresses the block, verifies the block checksum, and passes the block

+   off to a writer thread.  The writer thread writes the blocks in order,

+   and combines the individual block checksums into a per-file checksum.  The

+   per-file checksum is compared to the checksum in the stream's trailer.

    pigz requires zlib 1.2.1 or later to allow setting the dictionary when doing

    raw deflate.  Since zlib 1.2.3 corrects security vulnerabilities in zlib

    can't get way ahead of the write thread and build up a large backlog of

    unwritten compressed data.  The write thread will write the compressed data,

    drop the output buffer, and then wait for the check value to be unlocked

-   by the compress thread.  Then the write thread combines the check value for

-   this chunk with the total check value for eventual use in the trailer.  If

-   this is not the last chunk, the write thread then goes back to look for the

-   next output chunk in sequence.  After the last chunk, the write thread

-   returns and joins the main thread.  Unlike the compress threads, a new write

-   thread is launched for each input stream.  The write thread writes the

-   appropriate header and trailer around the compressed data.

+   by the compress thread.  Then the write thread writes an index entry (if -X)

+   and combines the check value for this chunk with the total check value for

+   eventual use in the trailer.  If this is not the last chunk, the write thread

+   then goes back to look for the next output chunk in sequence.  After the last

+   chunk, the write thread returns and joins the main thread.  Unlike the

+   compress threads, a new write thread is launched for each input stream.  The

+   write thread writes the appropriate header and trailer around the compressed

+   data.

    The input and output buffers are reused through their collection in pools.

    Each buffer has a use count, which when decremented to zero returns the

 #if __STDC_VERSION__-0 >= 199901L || __GNUC__-0 >= 3

 #  include <inttypes.h> /* intmax_t */

 #endif

+#include <stddef.h>     /* offsetof() */

+#include <sys/mman.h>   /* mmap() */

+#include <netinet/in.h> /* htonl() */

 }

+local void idx_abort(void);

+

 {

+    idx_abort();

     return dos;

 }

-/* put a 4-byte integer into a byte array in LSB order or MSB order */

+/* put integers into a byte array in LSB order or MSB order */

 #define PUT2L(a,b) (*(a)=(b)&0xff,(a)[1]=(b)>>8)

 #define PUT4L(a,b) (PUT2L(a,(b)&0xffff),PUT2L((a)+2,(b)>>16))

+#define PUT8L(a,b) (PUT4L(a,(b)&0xffffffff),PUT4L((a)+4,(b)>>32))

 #define PUT4M(a,b) (*(a)=(b)>>24,(a)[1]=(b)>>16,(a)[2]=(b)>>8,(a)[3]=(b))

+/* pull LSB order or MSB order integers from an unsigned char buffer */

+#define PULL2L(p) ((p)[0] + ((unsigned)((p)[1]) << 8))

+#define PULL4L(p) (PULL2L(p) + ((unsigned long)(PULL2L((p) + 2)) << 16))

+#define PULL8L(p) ((uint64_t)((p)[0]) | ((uint64_t)((p)[1]) << 8) | \

+                   ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \

+                   ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \

+                   ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))

+#define PULL2M(p) (((unsigned)((p)[0]) << 8) + (p)[1])

+#define PULL4M(p) (((unsigned long)(PULL2M(p)) << 16) + PULL2M((p) + 2))

+

+

 /* write a gzip, zlib, or zip header using the information in the globals */

 local unsigned long put_header(void)

 {

 /* get a space from a pool -- the use count is initially set to one, so there

    is no need to call use_space() for the first use */

-local struct space *get_space(struct pool *pool)

+local struct space *get_space_size(struct pool *pool, size_t size)

 {

     struct space *space;

     if (pool->head != NULL) {

         space = pool->head;

         possess(space->use);

+        /* If there's not enough space, free and malloc rather than realloc to

+           avoid the potential of an unnecessary memory copy. */

+        if (space->size < size) {

+            free(space->buf);

+            space->buf = malloc(size);

+            if (space->buf == NULL)

+            space->size = size;

+        }

         pool->head = space->next;

         twist(pool->have, BY, -1);      /* one less in pool */

         twist(space->use, TO, 1);       /* initially one user */

     space->use = new_lock(1);           /* initially one user */

-    space->size = pool->size;

+    space->size = size;

     space->len = 0;

     space->pool = pool;                 /* remember the pool this belongs to */

     return space;

 }

+local struct space *get_space(struct pool *pool)

+{

+    return get_space_size(pool, pool->size);

+}

+

     return count;

 }

+/* prompt for permission to overwrite a file */

+local int allow_overwrite(const char *path)

+{

+    char ch;

+    int reply = -1;

+

+    fprintf(stderr, "%s exists -- overwrite (y/n)? ", path);

+    fflush(stderr);

+    do {

+        ch = getchar();

+        if (reply < 0 && ch != ' ' && ch != '\t')

+            reply = ch == 'y' || ch == 'Y' ? 1 : 0;

+    } while (ch != EOF && ch != '\n' && ch != '\r');

+    return reply;

+}

+

 /* input and output buffer pools */

 local struct pool in_pool;

 local struct pool out_pool;

 local struct pool dict_pool;

 local struct pool lens_pool;

+local struct pool idx_pool;

 /* -- parallel compression -- */

 /* compress or write job (passed from compress list to write list) -- if seq is

    equal to -1, compress_thread is instructed to return; if more is false then

-   this is the last chunk, which after writing tells write_thread to return */

+   this is the last chunk, which after writing tells compress_write_thread to

+   return */

 struct job {

     long seq;                   /* sequence number */

     int more;                   /* true if this is not the last chunk */

     new_pool(&dict_pool, DICT, -1);

+    new_pool(&idx_pool, 1, -1);

 }

 /* command the compress threads to all return, then join them all (call from

     Trace(("-- freed %d output buffers", caught));

     caught = free_pool(&in_pool);

     Trace(("-- freed %d input buffers", caught));

+    caught = free_pool(&idx_pool);

+    Trace(("-- freed %d index buffers", caught));

     free_lock(write_first);

     free_lock(compress_have);

     compress_have = NULL;

 }

+/* Block Index

+   The block index is an array of idx_entry structs followed by an idx_trailer

+   struct.  They are written to the file in LSB order.  The block index can

+   exist as a standalone file or be appended onto the compressed files.

+

+   The trailer is used to identify a block index.  The beginning of the trailer

+   contains a magic number that is a value too large to be confused with a valid

+   block length.  Aside from backwards P's the magic number looks kinda like

+   "0xf pigzip 0xf". */

+#define IDXMAGIC 0xf916219f

+

+struct idx_trailer {

+    uint32_t    magic;

+    uint64_t    count;

+};

+

+struct idx_entry {

+    uint32_t    infsz;          /* inflated size of the block */

+    uint32_t    defsz;          /* deflated size of the block */

+    uint32_t    check;          /* adler32 or crc32 checksum of the block */

+};

+

+local struct {

+    int             valid;      /* Do the rest of these fields mean anything? */

+

+    /* An array of entries.  References address in space or map */

+    struct idx_entry *ents;     /* not in right byte order, used for offset */

+    uint64_t        seq;        /* current entry */

+    int64_t         eof;        /* has the last entry been retrieved? */

+

+    /* When compressing and appending, entries are stored in space->buf.  */

+    int             append;     /* is the index at end of compressed file? */

+    struct space    *space;     /* space for storage of index */

+

+    /* The following are valid only when mmap is used. */

+    uchar_t         *map;       /* mmap'd region containing ents */

+    size_t          mapsz;      /* size of mmap'd region at map */

+    off_t           mapoff;     /* bytes between map and ents */

+

+    /* Index path, after %f and %z are replaced. */

+    char            path[PATH_MAX+1];

+} idx;

+

+/* determines if the two paths refer to the same extant file */

+local int same_file(const char *f1, const char *f2)

+{

+    struct stat s1;

+    struct stat s2;

+

+    return (stat(f1, &s1) == 0 && stat(f2, &s2) == 0 &&

+            s1.st_dev == s2.st_dev && s1.st_ino == s2.st_ino);

+}

+

+/* Remove the index file, but only if it is not the same as in or out.

+   We don't worry about a full cleanup, as this should only be called in an

+   error path just before exiting. */

+local void idx_abort(void)

+{

+    if (!idx.valid)

+        return;

+    if (idx.path[0] == '\0' || idx.append)

+        return;

+    (void) unlink(idx.path);

+}

+

+/* If 0 is returned, a trailer was found and read.  Non-zero return means

+   there was no trailer. Does not exit. Does not change file pointer for fd. */

+local int idx_read_trailer(int fd, char *path, struct idx_trailer *trail)

+{

+    uchar_t buf[sizeof(*trail)];

+    off_t off;

+    struct stat st;

+

+    if (fd < 0) {

+        Trace(("%s: index file descriptor %d not valid", path, fd));

+        return -1;

+    }

+    if (fstat(fd, &st) != 0 || !S_ISREG(st.st_mode)) {

+        Trace(("%s: index appended to non-regular file", path));

+        return -1;

+    }

+    off = st.st_size - sizeof(*trail);

+    if (off < 0) {

+        Trace(("%s: index file too short for header", path));

+        return -1;

+    }

+    if (pread(fd, buf, sizeof(buf), off) != sizeof(buf)) {

+        Trace(("%s: unable to read index trailer", path));

+        return -1;

+    }

+    trail->magic = PULL4L(buf);

+    trail->count = PULL8L(buf + 4);

+

+    if (trail->magic != IDXMAGIC) {

+        Trace(("%s: invalid pigz index magic", path));

+        return -1;

+    }

+    return 0;

+}

+

+/* Expand a path pattern containing %f and/or %z tokens into a full path.

+ * Result is stored in idx.path. */

+local int expand_pathpat(char *pathpat)

+{

+    char *copy = NULL;              /* points to in or out global */

+    char *suf = NULL;               /* suffix (.zz, .gz, etc.) */

+    int chop_suffix;

+    int len;

+    int i;

+    int j;

+    int nag;

+

+    /* Be quiet when opportunistic index use check is being done. */

+

+    for (i = 0, j = 0; pathpat[i] && j < sizeof(idx.path); i++) {

+        if (pathpat[i] != '%') {

+            idx.path[j++] = pathpat[i];

+            continue;

+        }

+        i++;

+        switch (pathpat[i]) {

+        case '%':               /* %% is replaced by % */

+            idx.path[j++] = '%';

+            continue;

+        case 'f':               /* %f is replaced by uncompressed file name */

+                    chop_suffix = 0;

+                    break;

+                }

+                    chop_suffix = 1;

+                    break;

+                }

+                if (nag)

+                    complain("file name for %%f unknown");

+                return -1;

+            }

+

+                chop_suffix = 1;

+                break;

+            }

+                chop_suffix = 0;

+                break;

+            }

+            if (nag)

+                complain("file name for %%f unknown");

+            return -1;

+        case 'z':               /* %z is replaced by compressed file name */

+            chop_suffix = 0;

+                    if (nag)

+                        complain("file name for %%z unknown");

+                    return -1;

+                }

+                break;

+            }

+            if (strcmp(pathpat, "%z") == 0) {

+                /* index will be appended onto stdout */

+                copy = NULL;

+                idx.append = 1;

+                break;

+            }

+                if (nag)

+                    complain("file name for %%z unknown");

+                return -1;

+            }

+            break;

+        default:

+            if (nag) {

+                complain("invalid %% sequence in index file pattern %s",

+                         pathpat);

+            }

+            return -1;

+        }

+

+        /* pathpat is "%z" and out is stdout */

+        if (copy == NULL)

+            break;

+

+        len = strlen(&idx.path[j]) + strlen(copy);

+        if (chop_suffix)

+            len -= strlen(suf);

+        if (len >= (sizeof(idx.path) - j)) {

+            if (nag)

+                complain("index file name too long");

+            return -1;

+        }

+        (void)strncpy(&idx.path[j], copy, sizeof(idx.path) - j);

+        j += len;

+        assert(j <= sizeof(idx.path));

+    }

+    if (j == sizeof(idx.path)) {

+        idx.path[j-1] = '\0';

+        if (nag)

+            complain("index file \"%s...\" name too long", idx.path);

+        return -1;

+    }

+    idx.path[j] = '\0';

+

+    if (copy == NULL && idx.append) {

+            idx.path[sizeof(idx.path) - 1] = '\0';

+    }

+    else {

+            if (nag)

+                complain("index file %s must not be same as uncompressed file",

+                        idx.path);

+            return -1;

+        }

+

+    }

+

+        (void) fprintf(stderr, "index %s ", idx.path);

+

+    return 0;