VLI encoder and decoder cleanups. Made encoder return

LZMA_PROG_ERROR in single-call mode if there's no output
space.

1 files changed, 24 insertions, 7 deletions

diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c
index 4bb4d351..0d8e7f1a 100644
--- a/src/liblzma/common/vli_encoder.c
+++ b/src/liblzma/common/vli_encoder.c
@@ -27,30 +27,47 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos,
 {
 	// If we haven't been given vli_pos, work in single-call mode.
 	size_t vli_pos_internal = 0;
-	if (vli_pos == NULL)
+	if (vli_pos == NULL) {
 		vli_pos = &vli_pos_internal;
 
+		// In single-call mode, we expect that the caller has
+		// reserved enough output space.
+		if (*out_pos >= out_size)
+			return LZMA_PROG_ERROR;
+	} else {
+		// This never happens when we are called by liblzma, but
+		// may happen if called directly from an application.
+		if (*out_pos >= out_size)
+			return LZMA_BUF_ERROR;
+	}
+
 	// Validate the arguments.
 	if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_MAX)
 		return LZMA_PROG_ERROR;
 
-	if (*out_pos >= out_size)
-		return LZMA_BUF_ERROR;
+	// Shift vli so that the next bits to encode are the lowest. In
+	// single-call mode this never changes vli since *vli_pos is zero.
+	vli >>= *vli_pos * 7;
 
 	// Write the non-last bytes in a loop.
-	while ((vli >> (*vli_pos * 7)) >= 0x80) {
-		out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7)) | 0x80;
-
+	while (vli >= 0x80) {
+		// We don't need *vli_pos during this function call anymore,
+		// but update it here so that it is ready if we need to
+		// return before the whole integer has been decoded.
 		++*vli_pos;
 		assert(*vli_pos < LZMA_VLI_BYTES_MAX);
 
+		// Write the next byte.
+		out[*out_pos] = (uint8_t)(vli) | 0x80;
+		vli >>= 7;
+
 		if (++*out_pos == out_size)
 			return vli_pos == &vli_pos_internal
 					? LZMA_PROG_ERROR : LZMA_OK;
 	}
 
 	// Write the last byte.
-	out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7));
+	out[*out_pos] = (uint8_t)(vli);
 	++*out_pos;
 	++*vli_pos;