Fix a buffer overflow in the LZMA encoder. It was due to my

misunderstanding of the code. There's no tiny fix for this
problem, so I also cleaned up the code in general.

This reduces the speed of the encoder 2-5 % in the fastest
compression mode ("lzma -1"). High compression modes should
have no noticeable performance difference.

This commit breaks things (especially LZMA_SYNC_FLUSH) but I
will fix them once the new format and LZMA2 has been roughly
implemented. Plain LZMA won't support LZMA_SYNC_FLUSH at all
and won't be supported in the new .lzma format. This may
change still but this is what it looks like now.

Support for known uncompressed size (that is, LZMA or LZMA2
without EOPM) is likely to go away. This means there will
be API changes.

1 files changed, 2 insertions, 16 deletions

diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h
index b13e4b83..da0e0804 100644
--- a/src/liblzma/lz/lz_encoder.h
+++ b/src/liblzma/lz/lz_encoder.h
@@ -24,32 +24,19 @@
 #include "common.h"
 
 
-#define LZMA_LZ_TEMP_SIZE 64
-
-
 typedef struct lzma_lz_encoder_s lzma_lz_encoder;
 struct lzma_lz_encoder_s {
 	enum {
-		SEQ_START,
 		SEQ_RUN,
 		SEQ_FLUSH,
-		SEQ_FLUSH_END,
 		SEQ_FINISH,
-		SEQ_END
 	} sequence;
 
+	/// Function to do the actual encoding from the sliding input window
+	/// to the output stream.
 	bool (*process)(lzma_coder *coder, uint8_t *restrict out,
 			size_t *restrict out_pos, size_t out_size);
 
-	/// Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN if using EOPM. We need
-	/// to track Uncompressed Size to prevent writing flush marker to the
-	/// very end of stream that doesn't use EOPM.
-	lzma_vli uncompressed_size;
-
-	/// Temporary buffer for range encoder.
-	uint8_t temp[LZMA_LZ_TEMP_SIZE];
-	size_t temp_size;
-
 	///////////////
 	// In Window //
 	///////////////
@@ -145,7 +132,6 @@ extern lzma_ret lzma_lz_encoder_reset(lzma_lz_encoder *lz,
 		lzma_allocator *allocator,
 		bool (*process)(lzma_coder *coder, uint8_t *restrict out,
 			size_t *restrict out_pos, size_t out_size),
-		lzma_vli uncompressed_size,
 		size_t history_size, size_t additional_buffer_before,
 		size_t match_max_len, size_t additional_buffer_after,
 		lzma_match_finder match_finder, uint32_t match_finder_cycles,