liblzma: Fix handling of memlimit == 0 in lzma_index_decoder().

It returned LZMA_PROG_ERROR, which was done to avoid zero as
the limit (because it's a special value elsewhere), but using
LZMA_PROG_ERROR is simply inconvenient and can cause bugs.

The fix/workaround is to treat 0 as if it were 1 byte. It's
effectively the same thing. The only weird consequence is
that then lzma_memlimit_get() will return 1 even when 0 was
specified as the limit.

This fixes a very rare corner case in xz --list where a specific
memory usage limit and a multi-stream file could print the
error message "Internal error (bug)" instead of saying that
the memory usage limit is too low.

2 files changed, 13 insertions, 9 deletions

diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h
index dda60ec1..3dac6fb8 100644
--- a/src/liblzma/api/lzma/index.h
+++ b/src/liblzma/api/lzma/index.h
@@ -586,8 +586,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup(
  * \param       i           Pointer to lzma_index which should be encoded.
  *
  * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH.
- * It is enough to use only one of them (you can choose freely; use LZMA_RUN
- * to support liblzma versions older than 5.0.0).
+ * It is enough to use only one of them (you can choose freely).
  *
  * \return      - LZMA_OK: Initialization succeeded, continue with lzma_code().
  *              - LZMA_MEM_ERROR
@@ -610,16 +609,21 @@ extern LZMA_API(lzma_ret) lzma_index_encoder(
  *                          to a new lzma_index, which the application
  *                          has to later free with lzma_index_end().
  * \param       memlimit    How much memory the resulting lzma_index is
- *                          allowed to require.
+ *                          allowed to require. liblzma 5.2.3 and earlier
+ *                          don't allow 0 here and return LZMA_PROG_ERROR;
+ *                          later versions treat 0 as if 1 had been specified.
  *
- * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH.
- * It is enough to use only one of them (you can choose freely; use LZMA_RUN
- * to support liblzma versions older than 5.0.0).
+ * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH.
+ * There is no need to use LZMA_FINISH, but it's allowed because it may
+ * simplify certain types of applications.
  *
  * \return      - LZMA_OK: Initialization succeeded, continue with lzma_code().
  *              - LZMA_MEM_ERROR
- *              - LZMA_MEMLIMIT_ERROR
  *              - LZMA_PROG_ERROR
+ *
+ *              liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here
+ *              but that error code has never been possible from this
+ *              initialization function.
  */
 extern LZMA_API(lzma_ret) lzma_index_decoder(
 		lzma_stream *strm, lzma_index **i, uint64_t memlimit)
diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c
index 1e33f0b0..cc07a1b8 100644
--- a/src/liblzma/common/index_decoder.c
+++ b/src/liblzma/common/index_decoder.c
@@ -256,7 +256,7 @@ index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator,
 
 	// Initialize the rest.
 	coder->sequence = SEQ_INDICATOR;
-	coder->memlimit = memlimit;
+	coder->memlimit = my_max(1, memlimit);
 	coder->count = 0; // Needs to be initialized due to _memconfig().
 	coder->pos = 0;
 	coder->crc32 = 0;
@@ -271,7 +271,7 @@ index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 {
 	lzma_next_coder_init(&index_decoder_init, next, allocator);
 
-	if (i == NULL || memlimit == 0)
+	if (i == NULL)
 		return LZMA_PROG_ERROR;
 
 	lzma_index_coder *coder = next->coder;