liblzma: Fix lzma_memlimit_set(strm, 0).

The 0 got treated specially in a buggy way and as a result
the function did nothing. The API doc said that 0 was supposed
to return LZMA_PROG_ERROR but it didn't.

Now 0 is treated as if 1 had been specified. This is done because
0 is already used to indicate an error from lzma_memlimit_get()
and lzma_memusage().

In addition, lzma_memlimit_set() no longer checks that the new
limit is at least LZMA_MEMUSAGE_BASE. It's counter-productive
for the Index decoder and was actually needed only by the
auto decoder. Auto decoder has now been modified to check for
LZMA_MEMUSAGE_BASE.

diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h
index 7a31a4205136c868e85ee6f018f6fa84350fffad..a6005accc93d813042af29a5561f95153f4b1854 100644 (file)
--- a/src/liblzma/api/lzma/base.h
+++ b/src/liblzma/api/lzma/base.h
@@ -644,11 +644,16 @@ extern LZMA_API(uint64_t) lzma_memlimit_get(const lzma_stream *strm)
  * This function is supported only when *strm has been initialized with
  * a function that takes a memlimit argument.
  *
+ * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes
+ * this function to do nothing (leaving the limit unchanged) and still
+ * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so
+ * lzma_memlimit_get() will return 1 even if you specify 0 here).
+ *
  * \return      - LZMA_OK: New memory usage limit successfully set.
  *              - LZMA_MEMLIMIT_ERROR: The new limit is too small.
  *                The limit was not changed.
  *              - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't
- *                support memory usage limit or memlimit was zero.
+ *                support memory usage limit.
  */
 extern LZMA_API(lzma_ret) lzma_memlimit_set(
                lzma_stream *strm, uint64_t memlimit) lzma_nothrow;

diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c
index 479b150f866e9fe082ac2b0391321e817befcc53..6895c7ccf7b55c21adf0cdc4492944321112c37a 100644 (file)
--- a/src/liblzma/common/auto_decoder.c
+++ b/src/liblzma/common/auto_decoder.c
@@ -139,7 +139,10 @@ auto_decoder_memconfig(void *coder_ptr, uint64_t *memusage,
                // the current memory usage.
                *memusage = LZMA_MEMUSAGE_BASE;
                *old_memlimit = coder->memlimit;
+
                ret = LZMA_OK;
+               if (new_memlimit != 0 && new_memlimit < *memusage)
+                       ret = LZMA_MEMLIMIT_ERROR;
        }
 
        if (ret == LZMA_OK && new_memlimit != 0)

diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c
index 28aa2b7142f47051a5d31ee3b1b0c8570c8cd3e2..57e3f8ebd62a64a3325b2b9ace1ab7c9470cc89c 100644 (file)
--- a/src/liblzma/common/common.c
+++ b/src/liblzma/common/common.c
@@ -435,8 +435,10 @@ lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit)
                        || strm->internal->next.memconfig == NULL)
                return LZMA_PROG_ERROR;
 
-       if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE)
-               return LZMA_MEMLIMIT_ERROR;
+       // Zero is a special value that cannot be used as an actual limit.
+       // If 0 was specified, use 1 instead.
+       if (new_memlimit == 0)
+               new_memlimit = 1;
 
        return strm->internal->next.memconfig(strm->internal->next.coder,
                        &memusage, &old_memlimit, new_memlimit);