liblzma: Fix infinite loop in LZMA encoder init with dict_size >= 2 GiB.

The encoder doesn't support dictionary sizes larger than 1536 MiB.
This is validated, for example, when calculating the memory usage
via lzma_raw_encoder_memusage(). It is also enforced by the LZ
part of the encoder initialization. However, LZMA encoder with
LZMA_MODE_NORMAL did an unsafe calculation with dict_size before
such validation and that results in an infinite loop if dict_size
was 2 << 30 or greater.

diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c
index c1552f194451783d9baa4597b050ead4b04aa177..bbbb5d2d7cdfadfb737e049dfb5c22760f17d5a4 100644 (file)
--- a/src/liblzma/lzma/lzma_encoder.c
+++ b/src/liblzma/lzma/lzma_encoder.c
@@ -559,10 +559,9 @@ lzma_lzma_encoder_create(void **coder_ptr,
 
        lzma_lzma1_encoder *coder = *coder_ptr;
 
-       // Set compression mode. We haven't validates the options yet,
-       // but it's OK here, since nothing bad happens with invalid
-       // options in the code below, and they will get rejected by
-       // lzma_lzma_encoder_reset() call at the end of this function.
+       // Set compression mode. Note that we haven't validated the options
+       // yet. Invalid options will get rejected by lzma_lzma_encoder_reset()
+       // call at the end of this function.
        switch (options->mode) {
                case LZMA_MODE_FAST:
                        coder->fast_mode = true;
@@ -573,6 +572,18 @@ lzma_lzma_encoder_create(void **coder_ptr,
 
                        // Set dist_table_size.
                        // Round the dictionary size up to next 2^n.
+                       //
+                       // Currently the maximum encoder dictionary size
+                       // is 1.5 GiB due to lz_encoder.c and here we need
+                       // to be below 2 GiB to make the rounded up value
+                       // fit in an uint32_t and avoid an infite while-loop
+                       // (and undefined behavior due to a too large shift).
+                       // So do the same check as in LZ encoder,
+                       // limiting to 1.5 GiB.
+                       if (options->dict_size > (UINT32_C(1) << 30)
+                                       + (UINT32_C(1) << 29))
+                               return LZMA_OPTIONS_ERROR;
+
                        uint32_t log_size = 0;
                        while ((UINT32_C(1) << log_size) < options->dict_size)
                                ++log_size;