++count;
// Prevent an unlikely integer overflow.
- if (count > SIZE_MAX / sizeof(uint64_t) - 1)
+ if (count > SIZE_MAX / sizeof(block_list_entry) - 1)
message_fatal(_("%s: Too many arguments to --block-list"),
str);
// Allocate memory to hold all the sizes specified.
// If --block-list was specified already, its value is forgotten.
free(opt_block_list);
- opt_block_list = xmalloc((count + 1) * sizeof(uint64_t));
+ opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry));
for (size_t i = 0; i < count; ++i) {
// Locate the next comma and replace it with \0.
if (p != NULL)
*p = '\0';
+ // Use the default filter chain unless overridden.
+ opt_block_list[i].filters_index = 0;
+
+ // To specify a filter chain, the block list entry may be
+ // prepended with "[filter-chain-number]:". The size is
+ // still required for every block.
+ // For instance:
+ // --block-list=2:10MiB,1:5MiB,,8MiB,0:0
+ //
+ // Translates to:
+ // 1. Block of 10 MiB using filter chain 2
+ // 2. Block of 5 MiB using filter chain 1
+ // 3. Block of 5 MiB using filter chain 1
+ // 4. Block of 8 MiB using the default filter chain
+ // 5. The last block uses the default filter chain
+ //
+ // The block list:
+ // --block-list=2:MiB,1:,0
+ //
+ // Is not allowed because the second block does not specify
+ // the block size, only the filter chain.
+ if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') {
+ if (str[2] == '\0')
+ message_fatal(_("In --block-list, block "
+ "size is missing after "
+ "filter chain number `%c:'"),
+ str[0]);
+
+ int filter_num = str[0] - '0';
+ opt_block_list[i].filters_index =
+ (uint32_t)filter_num;
+ str += 2;
+ }
+
if (str[0] == '\0') {
// There is no string, that is, a comma follows
// another comma. Use the previous value.
assert(i > 0);
opt_block_list[i] = opt_block_list[i - 1];
} else {
- opt_block_list[i] = str_to_uint64("block-list", str,
- 0, UINT64_MAX);
+ opt_block_list[i].size = str_to_uint64("block-list",
+ str, 0, UINT64_MAX);
// Zero indicates no more new Blocks.
- if (opt_block_list[i] == 0) {
+ if (opt_block_list[i].size == 0) {
if (i + 1 != count)
message_fatal(_("0 can only be used "
"as the last element "
"in --block-list"));
- opt_block_list[i] = UINT64_MAX;
+ opt_block_list[i].size = UINT64_MAX;
}
}
}
// Terminate the array.
- opt_block_list[count] = 0;
+ opt_block_list[count].size = 0;
free(str_start);
return;
{
enum {
OPT_FILTERS = INT_MIN,
+ OPT_FILTERS1,
+ OPT_FILTERS2,
+ OPT_FILTERS3,
+ OPT_FILTERS4,
+ OPT_FILTERS5,
+ OPT_FILTERS6,
+ OPT_FILTERS7,
+ OPT_FILTERS8,
+ OPT_FILTERS9,
+
OPT_X86,
OPT_POWERPC,
OPT_IA64,
// Filters
{ "filters", optional_argument, NULL, OPT_FILTERS},
+ { "filters1", optional_argument, NULL, OPT_FILTERS1},
+ { "filters2", optional_argument, NULL, OPT_FILTERS2},
+ { "filters3", optional_argument, NULL, OPT_FILTERS3},
+ { "filters4", optional_argument, NULL, OPT_FILTERS4},
+ { "filters5", optional_argument, NULL, OPT_FILTERS5},
+ { "filters6", optional_argument, NULL, OPT_FILTERS6},
+ { "filters7", optional_argument, NULL, OPT_FILTERS7},
+ { "filters8", optional_argument, NULL, OPT_FILTERS8},
+ { "filters9", optional_argument, NULL, OPT_FILTERS9},
+
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
{ "x86", optional_argument, NULL, OPT_X86 },
coder_add_filters_from_str(optarg);
break;
+ // --filters1...--filters9
+ case OPT_FILTERS1:
+ case OPT_FILTERS2:
+ case OPT_FILTERS3:
+ case OPT_FILTERS4:
+ case OPT_FILTERS5:
+ case OPT_FILTERS6:
+ case OPT_FILTERS7:
+ case OPT_FILTERS8:
+ case OPT_FILTERS9:
+ coder_add_block_filters(optarg,
+ (size_t)(c - OPT_FILTERS));
+ break;
+
case OPT_X86:
coder_add_filter(LZMA_FILTER_X86,
options_bcj(optarg));
bool opt_auto_adjust = true;
bool opt_single_stream = false;
uint64_t opt_block_size = 0;
-uint64_t *opt_block_list = NULL;
-
+block_list_entry *opt_block_list = NULL;
/// Stream used to communicate with liblzma
static lzma_stream strm = LZMA_STREAM_INIT;
-/// Filters needed for all encoding all formats, and also decoding in raw data
-static lzma_filter filters[LZMA_FILTERS_MAX + 1];
+/// Maximum number of filter chains. The first filter chain is the default,
+/// and 9 other filter chains can be specified with --filtersX.
+#define NUM_FILTER_CHAIN_MAX 10
+
+/// The default filter chain is in filters[0]. It is used for encoding
+/// in all supported formats and also for decdoing raw streams. The other
+/// filter chains are set by --filtersX to support changing filters with
+/// the --block-list option.
+static lzma_filter filters[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1];
+
+/// Bit mask representing the filters specified through --filtersX. This
+/// is needed to verify that an entry in the --block-list option does not
+/// try to reference a filter chain that was not initialized.
+static uint32_t filters_init_mask = 1;
/// Input and output buffers
static io_buf in_buf;
static io_buf out_buf;
-/// Number of filters. Zero indicates that we are using a preset.
+/// Number of filters in the default filter chain. Zero indicates that
+/// we are using a preset.
static uint32_t filters_count = 0;
/// Number of the preset (0-9)
static uint32_t preset_number = LZMA_PRESET_DEFAULT;
-/// True if the current filter chain was set using the --filters option.
-/// The filter chain is reset if a preset option (like -9) or an old-style
-/// filter option (like --lzma2) is used after a --filters option.
+/// True if the current default filter chain was set using the --filters
+/// option. The filter chain is reset if a preset option (like -9) or an
+/// old-style filter option (like --lzma2) is used after a --filters option.
static bool string_to_filter_used = false;
/// Integrity check type
static lzma_mt mt_options = {
.flags = 0,
.timeout = 300,
- .filters = filters,
};
#endif
// Setting a preset or using --filters makes us forget
// the earlier custom filter chain (if any).
if (filters_count > 0) {
- lzma_filters_free(filters, NULL);
+ lzma_filters_free(filters[0], NULL);
filters_count = 0;
}
if (string_to_filter_used)
forget_filter_chain();
- filters[filters_count].id = id;
- filters[filters_count].options = options;
+ filters[0][filters_count].id = id;
+ filters[0][filters_count].options = options;
// Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
// implementation of forget_filter_chain().
- filters[++filters_count].id = LZMA_VLI_UNKNOWN;
+ filters[0][++filters_count].id = LZMA_VLI_UNKNOWN;
// Setting a custom filter chain makes us forget the preset options.
// This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
static void
-str_to_filter(const char *str, lzma_filter *filter, uint32_t flags)
+str_to_filters(const char *str, uint32_t index, uint32_t flags)
{
int error_pos;
- const char *err = lzma_str_to_filters(str, &error_pos, filter,
- flags, NULL);
+ const char *err = lzma_str_to_filters(str, &error_pos,
+ filters[index], flags, NULL);
if (err != NULL) {
+ char filter_num[2] = "";
+ if (index > 0)
+ filter_num[0] = '0' + index;
+
// FIXME? The message in err isn't translated.
// Including the translations in the xz translations is
// slightly ugly but possible. Creating a new domain for
// liblzma might not be worth it especially since on some
// OSes it adds extra dependencies to translation libraries.
- message(V_ERROR, _("Error in --filters=FILTERS option:"));
+ message(V_ERROR, _("Error in --filters%s=FILTERS option:"),
+ filter_num);
message(V_ERROR, "%s", str);
message(V_ERROR, "%*s^", error_pos, "");
message_fatal("%s", err);
string_to_filter_used = true;
// Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
- str_to_filter(filter_str, filters, LZMA_STR_ALL_FILTERS);
+ str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS);
// Set the filters_count to be the number of filters converted from
// the string.
- for (filters_count = 0; filters[filters_count].id != LZMA_VLI_UNKNOWN;
+ for (filters_count = 0; filters[0][filters_count].id
+ != LZMA_VLI_UNKNOWN;
++filters_count) ;
assert(filters_count > 0);
}
+extern void
+coder_add_block_filters(const char *str, size_t slot)
+{
+ // Free old filters first, if they were previously allocated.
+ if (filters_init_mask & (1 << slot))
+ lzma_filters_free(filters[slot], NULL);
+
+ str_to_filters(str, slot, 0);
+
+ filters_init_mask |= 1 << slot;
+}
+
+
static void lzma_attribute((__noreturn__))
memlimit_too_small(uint64_t memory_usage)
{
}
+// For a given opt_block_list index, validate that the filter has been
+// set. If it has not been set, we must exit with error to avoid using
+// an uninitialized filter chain.
+static void
+validate_block_list_filter(const uint32_t filter_num)
+{
+ if (!(filters_init_mask & (1 << filter_num)))
+ message_fatal(_("filter chain %u used by --block-list, but "
+ "not specified with --filters%u="),
+ (unsigned)filter_num, (unsigned)filter_num);
+}
extern void
coder_set_compression_settings(void)
{
assert(opt_format != FORMAT_LZIP);
#endif
+ if (opt_block_list != NULL)
+ for (uint32_t i = 0; opt_block_list[i].size != 0; i++)
+ validate_block_list_filter(
+ opt_block_list[i].filters_index);
+
// The default check type is CRC64, but fallback to CRC32
// if CRC64 isn't supported by the copy of liblzma we are
// using. CRC32 is always supported.
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
+ // The first filter in the filters[] array is for the default
+ // filter chain.
+ lzma_filter *default_filters = filters[0];
+
if (filters_count == 0) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
- filters[0].id = opt_format == FORMAT_LZMA
+ default_filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
- filters[0].options = &opt_lzma;
+ default_filters[0].options = &opt_lzma;
filters_count = 1;
// Terminate the filter options array.
- filters[1].id = LZMA_VLI_UNKNOWN;
+ default_filters[1].id = LZMA_VLI_UNKNOWN;
}
// If we are using the .lzma format, allow exactly one filter
// which has to be LZMA1.
if (opt_format == FORMAT_LZMA && (filters_count != 1
- || filters[0].id != LZMA_FILTER_LZMA1))
+ || default_filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// filter to prevent LZMA_PROG_ERROR.
if (opt_format == FORMAT_XZ)
for (size_t i = 0; i < filters_count; ++i)
- if (filters[i].id == LZMA_FILTER_LZMA1)
+ if (default_filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
- // Print the selected filter chain.
- message_filters_show(V_DEBUG, filters);
+ // Print the selected default filter chain.
+ message_filters_show(V_DEBUG, default_filters);
// The --flush-timeout option requires LZMA_SYNC_FLUSH support
- // from the filter chain. Currently threaded encoder doesn't support
- // LZMA_SYNC_FLUSH so single-threaded mode must be used.
+ // from the filter chain. Currently the threaded encoder doesn't
+ // support LZMA_SYNC_FLUSH so single-threaded mode must be used.
if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
for (size_t i = 0; i < filters_count; ++i) {
- switch (filters[i].id) {
+ switch (default_filters[i].id) {
case LZMA_FILTER_LZMA2:
case LZMA_FILTER_DELTA:
break;
} else
# endif
{
- memory_usage = lzma_raw_encoder_memusage(filters);
+ memory_usage = lzma_raw_encoder_memusage(default_filters);
}
#endif
} else {
#ifdef HAVE_DECODERS
- memory_usage = lzma_raw_decoder_memusage(filters);
+ memory_usage = lzma_raw_decoder_memusage(default_filters);
#endif
}
message_mem_needed(V_DEBUG, memory_usage);
#ifdef HAVE_DECODERS
if (opt_mode == MODE_COMPRESS) {
- const uint64_t decmem = lzma_raw_decoder_memusage(filters);
+ const uint64_t decmem = lzma_raw_decoder_memusage(default_filters);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
// the multithreaded mode but the output
// is also different.
hardware_threads_set(1);
- memory_usage = lzma_raw_encoder_memusage(filters);
+ memory_usage = lzma_raw_encoder_memusage(default_filters);
message(V_WARNING, _("Switching to single-threaded mode "
"to not exceed the memory usage limit of %s MiB"),
uint64_to_str(round_up_to_mib(memory_limit), 0));
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;
- while (filters[i].id != LZMA_FILTER_LZMA2
- && filters[i].id != LZMA_FILTER_LZMA1) {
- if (filters[i].id == LZMA_VLI_UNKNOWN)
+ while (default_filters[i].id != LZMA_FILTER_LZMA2
+ && default_filters[i].id != LZMA_FILTER_LZMA1) {
+ if (default_filters[i].id == LZMA_VLI_UNKNOWN)
memlimit_too_small(memory_usage);
++i;
// Decrease the dictionary size until we meet the memory
// usage limit. First round down to full mebibytes.
- lzma_options_lzma *opt = filters[i].options;
+ lzma_options_lzma *opt = default_filters[i].options;
const uint32_t orig_dict_size = opt->dict_size;
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
while (true) {
if (opt->dict_size < (UINT32_C(1) << 20))
memlimit_too_small(memory_usage);
- memory_usage = lzma_raw_encoder_memusage(filters);
+ memory_usage = lzma_raw_encoder_memusage(default_filters);
if (memory_usage == UINT64_MAX)
message_bug();
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
"from %s MiB to %s MiB to not exceed "
"the memory usage limit of %s MiB"),
- filters[i].id == LZMA_FILTER_LZMA2
+ default_filters[i].id == LZMA_FILTER_LZMA2
? '2' : '1',
uint64_to_str(orig_dict_size >> 20, 0),
uint64_to_str(opt->dict_size >> 20, 1),
// These will be handled later in this function.
allow_trailing_input = false;
+ // Set the first filter chain. If the --block-list option is not
+ // used then use the default filter chain (filters[0]).
+ // Otherwise, use first filter chain from the block list.
+ lzma_filter *active_filters = opt_block_list == NULL
+ ? filters[0]
+ : filters[opt_block_list[0].filters_index];
+
if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
switch (opt_format) {
case FORMAT_XZ:
# ifdef MYTHREAD_ENABLED
+ mt_options.filters = active_filters;
if (hardware_threads_is_mt())
ret = lzma_stream_encoder_mt(
&strm, &mt_options);
else
# endif
ret = lzma_stream_encoder(
- &strm, filters, check);
+ &strm, active_filters, check);
break;
case FORMAT_LZMA:
- ret = lzma_alone_encoder(&strm, filters[0].options);
+ ret = lzma_alone_encoder(&strm,
+ active_filters[0].options);
break;
# ifdef HAVE_LZIP_DECODER
# endif
case FORMAT_RAW:
- ret = lzma_raw_encoder(&strm, filters);
+ ret = lzma_raw_encoder(&strm, active_filters);
break;
}
#endif
case FORMAT_RAW:
// Memory usage has already been checked in
// coder_set_compression_settings().
- ret = lzma_raw_decoder(&strm, filters);
+ ret = lzma_raw_decoder(&strm, active_filters);
break;
}
} else {
// The Block at *list_pos has been finished. Go to the next
- // entry in the list. If the end of the list has been reached,
- // reuse the size of the last Block.
- if (opt_block_list[*list_pos + 1] != 0)
+ // entry in the list. If the end of the list has been
+ // reached, reuse the size and filters of the last Block.
+ if (opt_block_list[*list_pos + 1].size != 0) {
++*list_pos;
- *block_remaining = opt_block_list[*list_pos];
+ // Update the filters if needed.
+ if (opt_block_list[*list_pos - 1].filters_index
+ != opt_block_list[*list_pos].filters_index) {
+ const uint32_t filter_idx = opt_block_list
+ [*list_pos].filters_index;
+ const lzma_filter *next = filters[filter_idx];
+ const lzma_ret ret = lzma_filters_update(
+ &strm, next);
+
+ if (ret != LZMA_OK) {
+ // This message is only possible if
+ // the filter chain has unsupported
+ // options since the filter chain is
+ // validated using
+ // lzma_raw_encoder_memusage() or
+ // lzma_stream_encoder_mt_memusage().
+ // Some options are not validated until
+ // the encoders are initialized.
+ message_fatal(
+ _("Error changing to "
+ "filter chain %u: %s"),
+ (unsigned)filter_idx,
+ message_strm(ret));
+ }
+ }
+ }
+
+ *block_remaining = opt_block_list[*list_pos].size;
// If in single-threaded mode, split up the Block if needed.
// This is not needed in multi-threaded mode because liblzma
// output is still not identical because in single-threaded
// mode the size info isn't written into Block Headers.
if (opt_block_list != NULL) {
- if (block_remaining < opt_block_list[list_pos]) {
+ if (block_remaining < opt_block_list[list_pos].size) {
assert(!hardware_threads_is_mt());
- next_block_remaining = opt_block_list[list_pos]
+ next_block_remaining =
+ opt_block_list[list_pos].size
- block_remaining;
} else {
- block_remaining = opt_block_list[list_pos];
+ block_remaining =
+ opt_block_list[list_pos].size;
}
}
}
projects / thirdparty / xz.git / commitdiff
