liblzma: Fix uint64_t vs. size_t confusion.

This broke 32-bit builds due to a pointer type mismatch.

This bug was introduced with the output-size-limited encoding
in 625f4c7c99b2fcc4db9e7ab2deb4884790e2e17c.

Thanks to huangqinjin for the bug report.

1 files changed, 7 insertions, 4 deletions

diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h
index 5e2afafe..d794eabb 100644
--- a/src/liblzma/rangecoder/range_encoder.h
+++ b/src/liblzma/rangecoder/range_encoder.h
@@ -160,9 +160,12 @@ rc_shift_low(lzma_range_encoder *rc,
 }
 
 
+// NOTE: The last two arguments are uint64_t instead of size_t because in
+// the dummy version these refer to the size of the whole range-encoded
+// output stream, not just to the currently available output buffer space.
 static inline bool
 rc_shift_low_dummy(uint64_t *low, uint64_t *cache_size, uint8_t *cache,
-		size_t *out_pos, size_t out_size)
+		uint64_t *out_pos, uint64_t out_size)
 {
 	if ((uint32_t)(*low) < (uint32_t)(0xFF000000)
 			|| (uint32_t)(*low >> 32) != 0) {
@@ -262,7 +265,7 @@ rc_encode(lzma_range_encoder *rc,
 
 
 static inline bool
-rc_encode_dummy(const lzma_range_encoder *rc, size_t out_size)
+rc_encode_dummy(const lzma_range_encoder *rc, uint64_t out_limit)
 {
 	assert(rc->count <= RC_SYMBOLS_MAX);
 
@@ -278,7 +281,7 @@ rc_encode_dummy(const lzma_range_encoder *rc, size_t out_size)
 		// Normalize
 		if (range < RC_TOP_VALUE) {
 			if (rc_shift_low_dummy(&low, &cache_size, &cache,
-					&out_pos, out_size))
+					&out_pos, out_limit))
 				return true;
 
 			range <<= RC_SHIFT_BITS;
@@ -330,7 +333,7 @@ rc_encode_dummy(const lzma_range_encoder *rc, size_t out_size)
 	// the flushing that will be done at the end of the stream.
 	for (pos = 0; pos < 5; ++pos) {
 		if (rc_shift_low_dummy(&low, &cache_size,
-				&cache, &out_pos, out_size))
+				&cache, &out_pos, out_limit))
 			return true;
 	}