From: Tim Kientzle Date: Mon, 21 Nov 2011 00:52:32 +0000 (-0500) Subject: Try to decrease the performance hit of seeking during the bid phase. X-Git-Tag: v3.0.1b~34 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bb44a73ac7ea4f0efee36ad41b2a3ec6e43ea512;p=thirdparty%2Flibarchive.git Try to decrease the performance hit of seeking during the bid phase. Tell each bidder about the best bid so far so it can decide to do nothing. Reorder support_format_all so formats with relatively inexpensive bidders will run first. SVN-Revision: 3822 --- diff --git a/libarchive/archive_read.c b/libarchive/archive_read.c index b64547d79..1fa407ee6 100644 --- a/libarchive/archive_read.c +++ b/libarchive/archive_read.c @@ -537,7 +537,7 @@ choose_format(struct archive_read *a) a->format = &(a->formats[0]); for (i = 0; i < slots; i++, a->format++) { if (a->format->bid) { - bid = (a->format->bid)(a); + bid = (a->format->bid)(a, best_bid); if (bid == ARCHIVE_FATAL) return (ARCHIVE_FATAL); if (a->filter->position != 0) @@ -920,7 +920,7 @@ int __archive_read_register_format(struct archive_read *a, void *format_data, const char *name, - int (*bid)(struct archive_read *), + int (*bid)(struct archive_read *, int), int (*options)(struct archive_read *, const char *, const char *), int (*read_header)(struct archive_read *, struct archive_entry *), int (*read_data)(struct archive_read *, const void **, size_t *, int64_t *), diff --git a/libarchive/archive_read_private.h b/libarchive/archive_read_private.h index 5fbb07d32..76d0b91d9 100644 --- a/libarchive/archive_read_private.h +++ b/libarchive/archive_read_private.h @@ -169,7 +169,7 @@ struct archive_read { struct archive_format_descriptor { void *data; const char *name; - int (*bid)(struct archive_read *); + int (*bid)(struct archive_read *, int best_bid); int (*options)(struct archive_read *, const char *key, const char *value); int (*read_header)(struct archive_read *, struct archive_entry *); @@ -189,7 +189,7 @@ struct archive_read { int __archive_read_register_format(struct archive_read *a, void *format_data, const char *name, - int (*bid)(struct archive_read *), + int (*bid)(struct archive_read *, int), int (*options)(struct archive_read *, const char *, const char *), int (*read_header)(struct archive_read *, struct archive_entry *), int (*read_data)(struct archive_read *, const void **, size_t *, int64_t *), diff --git a/libarchive/archive_read_support_format_all.c b/libarchive/archive_read_support_format_all.c index 7d0f6efff..270a0420d 100644 --- a/libarchive/archive_read_support_format_all.c +++ b/libarchive/archive_read_support_format_all.c @@ -35,16 +35,44 @@ archive_read_support_format_all(struct archive *a) archive_check_magic(a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, "archive_read_support_format_all"); + /* TODO: It would be nice to compute the ordering + * here automatically so that people who enable just + * a few formats can still get the benefits. That + * may just require the format registration to include + * a "maximum read-ahead" value (anything that uses seek + * would be essentially infinite read-ahead). The core + * bid management can then sort the bidders before calling + * them. + * + * If you implement the above, please return the list below + * to alphabetic order. + */ + + /* + * These bidders are all pretty cheap; they just examine a + * small initial part of the archive. If one of these bids + * high, we can maybe avoid running any of the more expensive + * bidders below. + */ archive_read_support_format_ar(a); - archive_read_support_format_cab(a); archive_read_support_format_cpio(a); archive_read_support_format_empty(a); - archive_read_support_format_iso9660(a); archive_read_support_format_lha(a); archive_read_support_format_mtree(a); - archive_read_support_format_rar(a); archive_read_support_format_tar(a); archive_read_support_format_xar(a); + + /* + * Install expensive bidders last. By doing them last, we + * increase the chance that a high bid from someone else will + * make it unnecessary for these to do anything at all. + */ + /* These three have potentially large look-ahead. */ + archive_read_support_format_cab(a); + archive_read_support_format_rar(a); + archive_read_support_format_iso9660(a); + /* Seek is really bad, since it forces the read-ahead + * logic to discard buffered data. */ archive_read_support_format_zip(a); /* Note: We always return ARCHIVE_OK here, even if some of the diff --git a/libarchive/archive_read_support_format_ar.c b/libarchive/archive_read_support_format_ar.c index 772354057..9feb547db 100644 --- a/libarchive/archive_read_support_format_ar.c +++ b/libarchive/archive_read_support_format_ar.c @@ -81,7 +81,7 @@ struct ar { #define AR_fmag_offset 58 #define AR_fmag_size 2 -static int archive_read_format_ar_bid(struct archive_read *a); +static int archive_read_format_ar_bid(struct archive_read *a, int); static int archive_read_format_ar_cleanup(struct archive_read *a); static int archive_read_format_ar_read_data(struct archive_read *a, const void **buff, size_t *size, int64_t *offset); @@ -144,14 +144,11 @@ archive_read_format_ar_cleanup(struct archive_read *a) } static int -archive_read_format_ar_bid(struct archive_read *a) +archive_read_format_ar_bid(struct archive_read *a, int best_bid) { const void *h; - if (a->archive.archive_format != 0 && - (a->archive.archive_format & ARCHIVE_FORMAT_BASE_MASK) != - ARCHIVE_FORMAT_AR) - return(0); + (void)best_bid; /* UNUSED */ /* * Verify the 8-byte file signature. diff --git a/libarchive/archive_read_support_format_cab.c b/libarchive/archive_read_support_format_cab.c index ff87dcef8..f932fadf8 100644 --- a/libarchive/archive_read_support_format_cab.c +++ b/libarchive/archive_read_support_format_cab.c @@ -313,7 +313,7 @@ struct cab { struct lzx_stream xstrm; }; -static int archive_read_format_cab_bid(struct archive_read *); +static int archive_read_format_cab_bid(struct archive_read *, int); static int archive_read_format_cab_options(struct archive_read *, const char *, const char *); static int archive_read_format_cab_read_header(struct archive_read *, @@ -415,11 +415,16 @@ find_cab_magic(const char *p) } static int -archive_read_format_cab_bid(struct archive_read *a) +archive_read_format_cab_bid(struct archive_read *a, int best_bid) { const char *p; ssize_t bytes_avail, offset, window; + /* If there's already a better bid than we can ever + make, don't bother testing. */ + if (best_bid > 64) + return (-1); + if ((p = __archive_read_ahead(a, 8, NULL)) == NULL) return (-1); diff --git a/libarchive/archive_read_support_format_cpio.c b/libarchive/archive_read_support_format_cpio.c index e30a0b484..5ae73d770 100644 --- a/libarchive/archive_read_support_format_cpio.c +++ b/libarchive/archive_read_support_format_cpio.c @@ -189,7 +189,7 @@ struct cpio { static int64_t atol16(const char *, unsigned); static int64_t atol8(const char *, unsigned); -static int archive_read_format_cpio_bid(struct archive_read *); +static int archive_read_format_cpio_bid(struct archive_read *, int); static int archive_read_format_cpio_options(struct archive_read *, const char *, const char *); static int archive_read_format_cpio_cleanup(struct archive_read *); @@ -251,12 +251,14 @@ archive_read_support_format_cpio(struct archive *_a) static int -archive_read_format_cpio_bid(struct archive_read *a) +archive_read_format_cpio_bid(struct archive_read *a, int best_bid) { const unsigned char *p; struct cpio *cpio; int bid; + (void)best_bid; /* UNUSED */ + cpio = (struct cpio *)(a->format->data); if ((p = __archive_read_ahead(a, 6, NULL)) == NULL) diff --git a/libarchive/archive_read_support_format_empty.c b/libarchive/archive_read_support_format_empty.c index 5186b1177..3dc2196cb 100644 --- a/libarchive/archive_read_support_format_empty.c +++ b/libarchive/archive_read_support_format_empty.c @@ -31,7 +31,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_empty.c 1915 #include "archive_private.h" #include "archive_read_private.h" -static int archive_read_format_empty_bid(struct archive_read *); +static int archive_read_format_empty_bid(struct archive_read *, int); static int archive_read_format_empty_read_data(struct archive_read *, const void **, size_t *, int64_t *); static int archive_read_format_empty_read_header(struct archive_read *, @@ -60,14 +60,11 @@ archive_read_support_format_empty(struct archive *_a) static int -archive_read_format_empty_bid(struct archive_read *a) +archive_read_format_empty_bid(struct archive_read *a, int best_bid) { - ssize_t avail; - - (void)__archive_read_ahead(a, 1, &avail); - if (avail != 0) - return (-1); - return (1); + if (best_bid < 1 && __archive_read_ahead(a, 1, NULL) == NULL) + return (1); + return (-1); } static int diff --git a/libarchive/archive_read_support_format_iso9660.c b/libarchive/archive_read_support_format_iso9660.c index 3b97b6454..a2ef9299a 100644 --- a/libarchive/archive_read_support_format_iso9660.c +++ b/libarchive/archive_read_support_format_iso9660.c @@ -376,7 +376,7 @@ struct iso9660 { size_t utf16be_previous_path_len; }; -static int archive_read_format_iso9660_bid(struct archive_read *); +static int archive_read_format_iso9660_bid(struct archive_read *, int); static int archive_read_format_iso9660_options(struct archive_read *, const char *, const char *); static int archive_read_format_iso9660_cleanup(struct archive_read *); @@ -486,13 +486,18 @@ archive_read_support_format_iso9660(struct archive *_a) static int -archive_read_format_iso9660_bid(struct archive_read *a) +archive_read_format_iso9660_bid(struct archive_read *a, int best_bid) { struct iso9660 *iso9660; ssize_t bytes_read; const unsigned char *p; int seenTerminator; + /* If there's already a better bid than we can ever + make, don't bother testing. */ + if (best_bid > 48) + return (-1); + iso9660 = (struct iso9660 *)(a->format->data); /* diff --git a/libarchive/archive_read_support_format_lha.c b/libarchive/archive_read_support_format_lha.c index f573b3e3e..673feb92a 100644 --- a/libarchive/archive_read_support_format_lha.c +++ b/libarchive/archive_read_support_format_lha.c @@ -253,7 +253,7 @@ static const uint16_t crc16tbl[256] = { 0x8201,0x42C0,0x4380,0x8341,0x4100,0x81C1,0x8081,0x4040 }; -static int archive_read_format_lha_bid(struct archive_read *); +static int archive_read_format_lha_bid(struct archive_read *, int); static int archive_read_format_lha_options(struct archive_read *, const char *, const char *); static int archive_read_format_lha_read_header(struct archive_read *, @@ -384,13 +384,18 @@ lha_check_header_format(const void *h) } static int -archive_read_format_lha_bid(struct archive_read *a) +archive_read_format_lha_bid(struct archive_read *a, int best_bid) { const char *p; const void *buff; ssize_t bytes_avail, offset, window; size_t next; + /* If there's already a better bid than we can ever + make, don't bother testing. */ + if (best_bid > 30) + return (-1); + if ((p = __archive_read_ahead(a, H_SIZE, NULL)) == NULL) return (-1); diff --git a/libarchive/archive_read_support_format_mtree.c b/libarchive/archive_read_support_format_mtree.c index e2f7d711b..60c6997de 100644 --- a/libarchive/archive_read_support_format_mtree.c +++ b/libarchive/archive_read_support_format_mtree.c @@ -103,7 +103,7 @@ struct mtree { }; static int cleanup(struct archive_read *); -static int mtree_bid(struct archive_read *); +static int mtree_bid(struct archive_read *, int); static int parse_file(struct archive_read *, struct archive_entry *, struct mtree *, struct mtree_entry *, int *); static void parse_escapes(char *, struct mtree_entry *); @@ -521,7 +521,7 @@ bid_entry(const char *p, ssize_t len) #define MAX_BID_ENTRY 3 static int -mtree_bid(struct archive_read *a) +mtree_bid(struct archive_read *a, int best_bid) { const char *signature = "#mtree"; const char *p; @@ -529,6 +529,8 @@ mtree_bid(struct archive_read *a) ssize_t len, nl; int detected_bytes = 0, entry_cnt = 0, multiline = 0; + (void)best_bid; /* UNUSED */ + /* Now let's look at the actual header and see if it matches. */ p = __archive_read_ahead(a, strlen(signature), &avail); if (p == NULL) diff --git a/libarchive/archive_read_support_format_rar.c b/libarchive/archive_read_support_format_rar.c index fabbcbc3e..b778de470 100644 --- a/libarchive/archive_read_support_format_rar.c +++ b/libarchive/archive_read_support_format_rar.c @@ -293,7 +293,7 @@ struct rar } br; }; -static int archive_read_format_rar_bid(struct archive_read *); +static int archive_read_format_rar_bid(struct archive_read *, int); static int archive_read_format_rar_options(struct archive_read *, const char *, const char *); static int archive_read_format_rar_read_header(struct archive_read *, @@ -650,10 +650,14 @@ archive_read_support_format_rar(struct archive *_a) } static int -archive_read_format_rar_bid(struct archive_read *a) +archive_read_format_rar_bid(struct archive_read *a, int best_bid) { const char *p; + /* If there's already a bid > 30, we'll never win. */ + if (best_bid > 30) + return (-1); + if ((p = __archive_read_ahead(a, 7, NULL)) == NULL) return (-1); diff --git a/libarchive/archive_read_support_format_raw.c b/libarchive/archive_read_support_format_raw.c index afad7f88c..df2c00c96 100644 --- a/libarchive/archive_read_support_format_raw.c +++ b/libarchive/archive_read_support_format_raw.c @@ -44,7 +44,7 @@ struct raw_info { int end_of_file; }; -static int archive_read_format_raw_bid(struct archive_read *); +static int archive_read_format_raw_bid(struct archive_read *, int); static int archive_read_format_raw_cleanup(struct archive_read *); static int archive_read_format_raw_read_data(struct archive_read *, const void **, size_t *, int64_t *); @@ -91,12 +91,11 @@ archive_read_support_format_raw(struct archive *_a) * include "raw" as part of support_format_all(). */ static int -archive_read_format_raw_bid(struct archive_read *a) +archive_read_format_raw_bid(struct archive_read *a, int best_bid) { - - if (__archive_read_ahead(a, 1, NULL) == NULL) - return (-1); - return (1); + if (best_bid < 1 && __archive_read_ahead(a, 1, NULL) != NULL) + return (1); + return (-1); } /* diff --git a/libarchive/archive_read_support_format_tar.c b/libarchive/archive_read_support_format_tar.c index 645263311..89a1d4f2a 100644 --- a/libarchive/archive_read_support_format_tar.c +++ b/libarchive/archive_read_support_format_tar.c @@ -189,7 +189,7 @@ static int header_ustar(struct archive_read *, struct tar *, struct archive_entry *, const void *h); static int header_gnutar(struct archive_read *, struct tar *, struct archive_entry *, const void *h, size_t *); -static int archive_read_format_tar_bid(struct archive_read *); +static int archive_read_format_tar_bid(struct archive_read *, int); static int archive_read_format_tar_options(struct archive_read *, const char *, const char *); static int archive_read_format_tar_cleanup(struct archive_read *); @@ -286,12 +286,14 @@ archive_read_format_tar_cleanup(struct archive_read *a) static int -archive_read_format_tar_bid(struct archive_read *a) +archive_read_format_tar_bid(struct archive_read *a, int best_bid) { int bid; const char *h; const struct archive_entry_header_ustar *header; + (void)best_bid; /* UNUSED */ + bid = 0; /* Now let's look at the actual header and see if it matches. */ @@ -1126,7 +1128,7 @@ header_common(struct archive_read *a, struct tar *tar, /* Old-style or GNU tar: we must ignore the size. */ archive_entry_set_size(entry, 0); tar->entry_bytes_remaining = 0; - } else if (archive_read_format_tar_bid(a) > 50) { + } else if (archive_read_format_tar_bid(a, 50) > 50) { /* * We don't know if it's pax: If the bid * function sees a valid ustar header diff --git a/libarchive/archive_read_support_format_xar.c b/libarchive/archive_read_support_format_xar.c index aa34486af..26aff2cd1 100644 --- a/libarchive/archive_read_support_format_xar.c +++ b/libarchive/archive_read_support_format_xar.c @@ -374,7 +374,7 @@ struct xmlattr_list { struct xmlattr **last; }; -static int xar_bid(struct archive_read *); +static int xar_bid(struct archive_read *, int); static int xar_read_header(struct archive_read *, struct archive_entry *); static int xar_read_data(struct archive_read *, @@ -475,11 +475,13 @@ archive_read_support_format_xar(struct archive *_a) } static int -xar_bid(struct archive_read *a) +xar_bid(struct archive_read *a, int best_bid) { const unsigned char *b; int bid; + (void)best_bid; /* UNUSED */ + b = __archive_read_ahead(a, HEADER_SIZE, NULL); if (b == NULL) return (-1); diff --git a/libarchive/archive_read_support_format_zip.c b/libarchive/archive_read_support_format_zip.c index 43bc55582..7493e00a2 100644 --- a/libarchive/archive_read_support_format_zip.c +++ b/libarchive/archive_read_support_format_zip.c @@ -115,8 +115,8 @@ struct zip { #define ZIP_LENGTH_AT_END 8 #define ZIP_UTF8_NAME (1<<11) -static int archive_read_format_zip_streamable_bid(struct archive_read *); -static int archive_read_format_zip_seekable_bid(struct archive_read *); +static int archive_read_format_zip_streamable_bid(struct archive_read *, int); +static int archive_read_format_zip_seekable_bid(struct archive_read *, int); static int archive_read_format_zip_options(struct archive_read *, const char *, const char *); static int archive_read_format_zip_cleanup(struct archive_read *); @@ -225,12 +225,17 @@ archive_read_support_format_zip(struct archive *a) * seeking if it knows it's going to lose anyway. */ static int -archive_read_format_zip_seekable_bid(struct archive_read *a) +archive_read_format_zip_seekable_bid(struct archive_read *a, int best_bid) { struct zip *zip = (struct zip *)a->format->data; int64_t filesize; const char *p; + /* If someone has already bid more than 32, then avoid + trashing the look-ahead buffers with a seek. */ + if (best_bid > 32) + return (-1); + filesize = __archive_read_seek(a, -22, SEEK_END); /* If we can't seek, then we can't bid. */ if (filesize <= 0) @@ -365,10 +370,12 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a, } static int -archive_read_format_zip_streamable_bid(struct archive_read *a) +archive_read_format_zip_streamable_bid(struct archive_read *a, int best_bid) { const char *p; + (void)best_bid; /* UNUSED */ + if ((p = __archive_read_ahead(a, 4, NULL)) == NULL) return (-1);