Try to decrease the performance hit of seeking during the bid phase.

Tell each bidder about the best bid so far so it can decide
to do nothing.  Reorder support_format_all so formats
with relatively inexpensive bidders will run first.

SVN-Revision: 3822

diff --git a/libarchive/archive_read.c b/libarchive/archive_read.c
index b64547d79b9a4f779a076acad126701a69abbf9e..1fa407ee6412ad6e7f3963e3c5b9faa6b1ff8e4e 100644 (file)
--- a/libarchive/archive_read.c
+++ b/libarchive/archive_read.c
@@ -537,7 +537,7 @@ choose_format(struct archive_read *a)
        a->format = &(a->formats[0]);
        for (i = 0; i < slots; i++, a->format++) {
                if (a->format->bid) {
-                       bid = (a->format->bid)(a);
+                       bid = (a->format->bid)(a, best_bid);
                        if (bid == ARCHIVE_FATAL)
                                return (ARCHIVE_FATAL);
                        if (a->filter->position != 0)
@@ -920,7 +920,7 @@ int
 __archive_read_register_format(struct archive_read *a,
     void *format_data,
     const char *name,
-    int (*bid)(struct archive_read *),
+    int (*bid)(struct archive_read *, int),
     int (*options)(struct archive_read *, const char *, const char *),
     int (*read_header)(struct archive_read *, struct archive_entry *),
     int (*read_data)(struct archive_read *, const void **, size_t *, int64_t *),

diff --git a/libarchive/archive_read_private.h b/libarchive/archive_read_private.h
index 5fbb07d3215e53725d35e4e63856e8b6343f8d75..76d0b91d9e75c05f4e307f3e49227fd02db25050 100644 (file)
--- a/libarchive/archive_read_private.h
+++ b/libarchive/archive_read_private.h
@@ -169,7 +169,7 @@ struct archive_read {
        struct archive_format_descriptor {
                void     *data;
                const char *name;
-               int     (*bid)(struct archive_read *);
+               int     (*bid)(struct archive_read *, int best_bid);
                int     (*options)(struct archive_read *, const char *key,
                    const char *value);
                int     (*read_header)(struct archive_read *, struct archive_entry *);
@@ -189,7 +189,7 @@ struct archive_read {
 int    __archive_read_register_format(struct archive_read *a,
            void *format_data,
            const char *name,
-           int (*bid)(struct archive_read *),
+           int (*bid)(struct archive_read *, int),
            int (*options)(struct archive_read *, const char *, const char *),
            int (*read_header)(struct archive_read *, struct archive_entry *),
            int (*read_data)(struct archive_read *, const void **, size_t *, int64_t *),

diff --git a/libarchive/archive_read_support_format_all.c b/libarchive/archive_read_support_format_all.c
index 7d0f6efff0e7a5cf9350e196ff69ebe7095ca9b7..270a0420dbb98ab0f6bf369d7b0104c9473055cb 100644 (file)
--- a/libarchive/archive_read_support_format_all.c
+++ b/libarchive/archive_read_support_format_all.c
@@ -35,16 +35,44 @@ archive_read_support_format_all(struct archive *a)
        archive_check_magic(a, ARCHIVE_READ_MAGIC,
            ARCHIVE_STATE_NEW, "archive_read_support_format_all");
 
+       /* TODO: It would be nice to compute the ordering
+        * here automatically so that people who enable just
+        * a few formats can still get the benefits.  That
+        * may just require the format registration to include
+        * a "maximum read-ahead" value (anything that uses seek
+        * would be essentially infinite read-ahead).  The core
+        * bid management can then sort the bidders before calling
+        * them.
+        *
+        * If you implement the above, please return the list below
+        * to alphabetic order.
+        */
+
+       /*
+        * These bidders are all pretty cheap; they just examine a
+        * small initial part of the archive.  If one of these bids
+        * high, we can maybe avoid running any of the more expensive
+        * bidders below.
+        */
        archive_read_support_format_ar(a);
-       archive_read_support_format_cab(a);
        archive_read_support_format_cpio(a);
        archive_read_support_format_empty(a);
-       archive_read_support_format_iso9660(a);
        archive_read_support_format_lha(a);
        archive_read_support_format_mtree(a);
-       archive_read_support_format_rar(a);
        archive_read_support_format_tar(a);
        archive_read_support_format_xar(a);
+
+       /*
+        * Install expensive bidders last.  By doing them last, we
+        * increase the chance that a high bid from someone else will
+        * make it unnecessary for these to do anything at all.
+        */
+       /* These three have potentially large look-ahead. */
+       archive_read_support_format_cab(a);
+       archive_read_support_format_rar(a);
+       archive_read_support_format_iso9660(a);
+       /* Seek is really bad, since it forces the read-ahead
+        * logic to discard buffered data. */
        archive_read_support_format_zip(a);
 
        /* Note: We always return ARCHIVE_OK here, even if some of the

diff --git a/libarchive/archive_read_support_format_ar.c b/libarchive/archive_read_support_format_ar.c
index 772354057847c3c079c907a882eb9f25575291cf..9feb547dbbeaa134f31a58d1d4e0edc1f86658cb 100644 (file)
--- a/libarchive/archive_read_support_format_ar.c
+++ b/libarchive/archive_read_support_format_ar.c
@@ -81,7 +81,7 @@ struct ar {
 #define AR_fmag_offset 58
 #define AR_fmag_size 2
 
-static int     archive_read_format_ar_bid(struct archive_read *a);
+static int     archive_read_format_ar_bid(struct archive_read *a, int);
 static int     archive_read_format_ar_cleanup(struct archive_read *a);
 static int     archive_read_format_ar_read_data(struct archive_read *a,
                    const void **buff, size_t *size, int64_t *offset);
@@ -144,14 +144,11 @@ archive_read_format_ar_cleanup(struct archive_read *a)
 }
 
 static int
-archive_read_format_ar_bid(struct archive_read *a)
+archive_read_format_ar_bid(struct archive_read *a, int best_bid)
 {
        const void *h;
 
-       if (a->archive.archive_format != 0 &&
-           (a->archive.archive_format & ARCHIVE_FORMAT_BASE_MASK) !=
-           ARCHIVE_FORMAT_AR)
-               return(0);
+       (void)best_bid; /* UNUSED */
 
        /*
         * Verify the 8-byte file signature.

diff --git a/libarchive/archive_read_support_format_cab.c b/libarchive/archive_read_support_format_cab.c
index ff87dcef8adb4676073f24e95441850b897b84a4..f932fadf870f37f5b73a02ce48196d254d1c3fe3 100644 (file)
--- a/libarchive/archive_read_support_format_cab.c
+++ b/libarchive/archive_read_support_format_cab.c
@@ -313,7 +313,7 @@ struct cab {
        struct lzx_stream        xstrm;
 };
 
-static int     archive_read_format_cab_bid(struct archive_read *);
+static int     archive_read_format_cab_bid(struct archive_read *, int);
 static int     archive_read_format_cab_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_cab_read_header(struct archive_read *,
@@ -415,11 +415,16 @@ find_cab_magic(const char *p)
 }
 
 static int
-archive_read_format_cab_bid(struct archive_read *a)
+archive_read_format_cab_bid(struct archive_read *a, int best_bid)
 {
        const char *p;
        ssize_t bytes_avail, offset, window;
 
+       /* If there's already a better bid than we can ever
+          make, don't bother testing. */
+       if (best_bid > 64)
+               return (-1);
+
        if ((p = __archive_read_ahead(a, 8, NULL)) == NULL)
                return (-1);
 

diff --git a/libarchive/archive_read_support_format_cpio.c b/libarchive/archive_read_support_format_cpio.c
index e30a0b484f131ab5e0198dc7d06d914a3d06df6b..5ae73d7700ba01edcc518a12aa578b08904e3948 100644 (file)
--- a/libarchive/archive_read_support_format_cpio.c
+++ b/libarchive/archive_read_support_format_cpio.c
@@ -189,7 +189,7 @@ struct cpio {
 
 static int64_t atol16(const char *, unsigned);
 static int64_t atol8(const char *, unsigned);
-static int     archive_read_format_cpio_bid(struct archive_read *);
+static int     archive_read_format_cpio_bid(struct archive_read *, int);
 static int     archive_read_format_cpio_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_cpio_cleanup(struct archive_read *);
@@ -251,12 +251,14 @@ archive_read_support_format_cpio(struct archive *_a)
 
 
 static int
-archive_read_format_cpio_bid(struct archive_read *a)
+archive_read_format_cpio_bid(struct archive_read *a, int best_bid)
 {
        const unsigned char *p;
        struct cpio *cpio;
        int bid;
 
+       (void)best_bid; /* UNUSED */
+
        cpio = (struct cpio *)(a->format->data);
 
        if ((p = __archive_read_ahead(a, 6, NULL)) == NULL)

diff --git a/libarchive/archive_read_support_format_empty.c b/libarchive/archive_read_support_format_empty.c
index 5186b11774fc1128efca1b022a21be49e1c1daae..3dc2196cb99ece94a8494b068996257c03238e1b 100644 (file)
--- a/libarchive/archive_read_support_format_empty.c
+++ b/libarchive/archive_read_support_format_empty.c
@@ -31,7 +31,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_empty.c 1915
 #include "archive_private.h"
 #include "archive_read_private.h"
 
-static int     archive_read_format_empty_bid(struct archive_read *);
+static int     archive_read_format_empty_bid(struct archive_read *, int);
 static int     archive_read_format_empty_read_data(struct archive_read *,
                    const void **, size_t *, int64_t *);
 static int     archive_read_format_empty_read_header(struct archive_read *,
@@ -60,14 +60,11 @@ archive_read_support_format_empty(struct archive *_a)
 
 
 static int
-archive_read_format_empty_bid(struct archive_read *a)
+archive_read_format_empty_bid(struct archive_read *a, int best_bid)
 {
-       ssize_t avail;
-
-       (void)__archive_read_ahead(a, 1, &avail);
-       if (avail != 0)
-               return (-1);
-       return (1);
+       if (best_bid < 1 && __archive_read_ahead(a, 1, NULL) == NULL)
+               return (1);
+       return (-1);
 }
 
 static int

diff --git a/libarchive/archive_read_support_format_iso9660.c b/libarchive/archive_read_support_format_iso9660.c
index 3b97b6454beabfc0c0e17617e4c6a63222f7510b..a2ef9299ae1c1b1cd14910d0a227ed31808a49ea 100644 (file)
--- a/libarchive/archive_read_support_format_iso9660.c
+++ b/libarchive/archive_read_support_format_iso9660.c
@@ -376,7 +376,7 @@ struct iso9660 {
        size_t           utf16be_previous_path_len;
 };
 
-static int     archive_read_format_iso9660_bid(struct archive_read *);
+static int     archive_read_format_iso9660_bid(struct archive_read *, int);
 static int     archive_read_format_iso9660_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_iso9660_cleanup(struct archive_read *);
@@ -486,13 +486,18 @@ archive_read_support_format_iso9660(struct archive *_a)
 
 
 static int
-archive_read_format_iso9660_bid(struct archive_read *a)
+archive_read_format_iso9660_bid(struct archive_read *a, int best_bid)
 {
        struct iso9660 *iso9660;
        ssize_t bytes_read;
        const unsigned char *p;
        int seenTerminator;
 
+       /* If there's already a better bid than we can ever
+          make, don't bother testing. */
+       if (best_bid > 48)
+               return (-1);
+
        iso9660 = (struct iso9660 *)(a->format->data);
 
        /*

diff --git a/libarchive/archive_read_support_format_lha.c b/libarchive/archive_read_support_format_lha.c
index f573b3e3ee7253ad2a675fa06a131c3beed4128b..673feb92a0ad922d863c65d41cfa580012d2f7ee 100644 (file)
--- a/libarchive/archive_read_support_format_lha.c
+++ b/libarchive/archive_read_support_format_lha.c
@@ -253,7 +253,7 @@ static const uint16_t crc16tbl[256] = {
        0x8201,0x42C0,0x4380,0x8341,0x4100,0x81C1,0x8081,0x4040
 };
 
-static int      archive_read_format_lha_bid(struct archive_read *);
+static int      archive_read_format_lha_bid(struct archive_read *, int);
 static int      archive_read_format_lha_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_lha_read_header(struct archive_read *,
@@ -384,13 +384,18 @@ lha_check_header_format(const void *h)
 }
 
 static int
-archive_read_format_lha_bid(struct archive_read *a)
+archive_read_format_lha_bid(struct archive_read *a, int best_bid)
 {
        const char *p;
        const void *buff;
        ssize_t bytes_avail, offset, window;
        size_t next;
 
+       /* If there's already a better bid than we can ever
+          make, don't bother testing. */
+       if (best_bid > 30)
+               return (-1);
+
        if ((p = __archive_read_ahead(a, H_SIZE, NULL)) == NULL)
                return (-1);
 

diff --git a/libarchive/archive_read_support_format_mtree.c b/libarchive/archive_read_support_format_mtree.c
index e2f7d711b07b5640b48ff59ea32d7d813aaff9cc..60c6997decf2d41c5dc0818d452b041f2b9492a4 100644 (file)
--- a/libarchive/archive_read_support_format_mtree.c
+++ b/libarchive/archive_read_support_format_mtree.c
@@ -103,7 +103,7 @@ struct mtree {
 };
 
 static int     cleanup(struct archive_read *);
-static int     mtree_bid(struct archive_read *);
+static int     mtree_bid(struct archive_read *, int);
 static int     parse_file(struct archive_read *, struct archive_entry *,
                    struct mtree *, struct mtree_entry *, int *);
 static void    parse_escapes(char *, struct mtree_entry *);
@@ -521,7 +521,7 @@ bid_entry(const char *p, ssize_t len)
 #define MAX_BID_ENTRY  3
 
 static int
-mtree_bid(struct archive_read *a)
+mtree_bid(struct archive_read *a, int best_bid)
 {
        const char *signature = "#mtree";
        const char *p;
@@ -529,6 +529,8 @@ mtree_bid(struct archive_read *a)
        ssize_t len, nl;
        int detected_bytes = 0, entry_cnt = 0, multiline = 0;
 
+       (void)best_bid; /* UNUSED */
+
        /* Now let's look at the actual header and see if it matches. */
        p = __archive_read_ahead(a, strlen(signature), &avail);
        if (p == NULL)

diff --git a/libarchive/archive_read_support_format_rar.c b/libarchive/archive_read_support_format_rar.c
index fabbcbc3e189b244a40141e1bed46449462f5467..b778de470dd9794e26759aba20c9ea353da220e0 100644 (file)
--- a/libarchive/archive_read_support_format_rar.c
+++ b/libarchive/archive_read_support_format_rar.c
@@ -293,7 +293,7 @@ struct rar
   } br;
 };
 
-static int archive_read_format_rar_bid(struct archive_read *);
+static int archive_read_format_rar_bid(struct archive_read *, int);
 static int archive_read_format_rar_options(struct archive_read *,
     const char *, const char *);
 static int archive_read_format_rar_read_header(struct archive_read *,
@@ -650,10 +650,14 @@ archive_read_support_format_rar(struct archive *_a)
 }
 
 static int
-archive_read_format_rar_bid(struct archive_read *a)
+archive_read_format_rar_bid(struct archive_read *a, int best_bid)
 {
   const char *p;
 
+  /* If there's already a bid > 30, we'll never win. */
+  if (best_bid > 30)
+         return (-1);
+
   if ((p = __archive_read_ahead(a, 7, NULL)) == NULL)
     return (-1);
 

diff --git a/libarchive/archive_read_support_format_raw.c b/libarchive/archive_read_support_format_raw.c
index afad7f88cab4a446a5618b4e3252e38a54b34ef3..df2c00c96c326c1350bac6f2fe3d2ed428eede66 100644 (file)
--- a/libarchive/archive_read_support_format_raw.c
+++ b/libarchive/archive_read_support_format_raw.c
@@ -44,7 +44,7 @@ struct raw_info {
        int     end_of_file;
 };
 
-static int     archive_read_format_raw_bid(struct archive_read *);
+static int     archive_read_format_raw_bid(struct archive_read *, int);
 static int     archive_read_format_raw_cleanup(struct archive_read *);
 static int     archive_read_format_raw_read_data(struct archive_read *,
                    const void **, size_t *, int64_t *);
@@ -91,12 +91,11 @@ archive_read_support_format_raw(struct archive *_a)
  * include "raw" as part of support_format_all().
  */
 static int
-archive_read_format_raw_bid(struct archive_read *a)
+archive_read_format_raw_bid(struct archive_read *a, int best_bid)
 {
-
-       if (__archive_read_ahead(a, 1, NULL) == NULL)
-               return (-1);
-       return (1);
+       if (best_bid < 1 && __archive_read_ahead(a, 1, NULL) != NULL)
+               return (1);
+       return (-1);
 }
 
 /*

diff --git a/libarchive/archive_read_support_format_tar.c b/libarchive/archive_read_support_format_tar.c
index 6452633115e61038b97a0e00d5d06f4a6ec8555b..89a1d4f2a53226f1cae51f922c500bb0796e3891 100644 (file)
--- a/libarchive/archive_read_support_format_tar.c
+++ b/libarchive/archive_read_support_format_tar.c
@@ -189,7 +189,7 @@ static int  header_ustar(struct archive_read *, struct tar *,
                    struct archive_entry *, const void *h);
 static int     header_gnutar(struct archive_read *, struct tar *,
                    struct archive_entry *, const void *h, size_t *);
-static int     archive_read_format_tar_bid(struct archive_read *);
+static int     archive_read_format_tar_bid(struct archive_read *, int);
 static int     archive_read_format_tar_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_tar_cleanup(struct archive_read *);
@@ -286,12 +286,14 @@ archive_read_format_tar_cleanup(struct archive_read *a)
 
 
 static int
-archive_read_format_tar_bid(struct archive_read *a)
+archive_read_format_tar_bid(struct archive_read *a, int best_bid)
 {
        int bid;
        const char *h;
        const struct archive_entry_header_ustar *header;
 
+       (void)best_bid; /* UNUSED */
+
        bid = 0;
 
        /* Now let's look at the actual header and see if it matches. */
@@ -1126,7 +1128,7 @@ header_common(struct archive_read *a, struct tar *tar,
                        /* Old-style or GNU tar: we must ignore the size. */
                        archive_entry_set_size(entry, 0);
                        tar->entry_bytes_remaining = 0;
-               } else if (archive_read_format_tar_bid(a) > 50) {
+               } else if (archive_read_format_tar_bid(a, 50) > 50) {
                        /*
                         * We don't know if it's pax: If the bid
                         * function sees a valid ustar header

diff --git a/libarchive/archive_read_support_format_xar.c b/libarchive/archive_read_support_format_xar.c
index aa34486afca22f4cd0773566259daac6f0067b76..26aff2cd114690652cbd74aba6f5b17c192d9f99 100644 (file)
--- a/libarchive/archive_read_support_format_xar.c
+++ b/libarchive/archive_read_support_format_xar.c
@@ -374,7 +374,7 @@ struct xmlattr_list {
        struct xmlattr  **last;
 };
 
-static int     xar_bid(struct archive_read *);
+static int     xar_bid(struct archive_read *, int);
 static int     xar_read_header(struct archive_read *,
                    struct archive_entry *);
 static int     xar_read_data(struct archive_read *,
@@ -475,11 +475,13 @@ archive_read_support_format_xar(struct archive *_a)
 }
 
 static int
-xar_bid(struct archive_read *a)
+xar_bid(struct archive_read *a, int best_bid)
 {
        const unsigned char *b;
        int bid;
 
+       (void)best_bid; /* UNUSED */
+
        b = __archive_read_ahead(a, HEADER_SIZE, NULL);
        if (b == NULL)
                return (-1);

diff --git a/libarchive/archive_read_support_format_zip.c b/libarchive/archive_read_support_format_zip.c
index 43bc55582f5fbbaf808c0d35d27e541db037c897..7493e00a26046538676709320874dd27078bc251 100644 (file)
--- a/libarchive/archive_read_support_format_zip.c
+++ b/libarchive/archive_read_support_format_zip.c
@@ -115,8 +115,8 @@ struct zip {
 #define ZIP_LENGTH_AT_END      8
 #define ZIP_UTF8_NAME          (1<<11) 
 
-static int     archive_read_format_zip_streamable_bid(struct archive_read *);
-static int     archive_read_format_zip_seekable_bid(struct archive_read *);
+static int     archive_read_format_zip_streamable_bid(struct archive_read *, int);
+static int     archive_read_format_zip_seekable_bid(struct archive_read *, int);
 static int     archive_read_format_zip_options(struct archive_read *,
                    const char *, const char *);
 static int     archive_read_format_zip_cleanup(struct archive_read *);
@@ -225,12 +225,17 @@ archive_read_support_format_zip(struct archive *a)
  * seeking if it knows it's going to lose anyway.
  */
 static int
-archive_read_format_zip_seekable_bid(struct archive_read *a)
+archive_read_format_zip_seekable_bid(struct archive_read *a, int best_bid)
 {
        struct zip *zip = (struct zip *)a->format->data;
        int64_t filesize;
        const char *p;
 
+       /* If someone has already bid more than 32, then avoid
+          trashing the look-ahead buffers with a seek. */
+       if (best_bid > 32)
+               return (-1);
+
        filesize = __archive_read_seek(a, -22, SEEK_END);
        /* If we can't seek, then we can't bid. */
        if (filesize <= 0)
@@ -365,10 +370,12 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a,
 }
 
 static int
-archive_read_format_zip_streamable_bid(struct archive_read *a)
+archive_read_format_zip_streamable_bid(struct archive_read *a, int best_bid)
 {
        const char *p;
 
+       (void)best_bid; /* UNUSED */
+
        if ((p = __archive_read_ahead(a, 4, NULL)) == NULL)
                return (-1);