Sort of garbage collection commit. :-| Many things are still

broken. API has changed a lot and it will still change a
little more here and there. The command line tool doesn't
have all the required changes to reflect the API changes, so
it's easy to get "internal error" or trigger assertions.

5 files changed, 112 insertions, 186 deletions

diff --git a/src/lzma/args.c b/src/lzma/args.c
index a4764032..30df4522 100644
--- a/src/lzma/args.c
+++ b/src/lzma/args.c
@@ -39,8 +39,8 @@ bool opt_force = false;
 bool opt_keep_original = false;
 bool opt_preserve_name = false;
 
-lzma_check_type opt_check = LZMA_CHECK_CRC64;
-lzma_options_filter opt_filters[8];
+lzma_check opt_check = LZMA_CHECK_CRC64;
+lzma_filter opt_filters[8];
 
 // We don't modify or free() this, but we need to assign it in some
 // non-const pointers.
@@ -61,6 +61,7 @@ enum {
 	OPT_SPARC,
 	OPT_DELTA,
 	OPT_LZMA,
+	OPT_LZMA2,
 
 	OPT_FILES,
 	OPT_FILES0,
@@ -108,6 +109,7 @@ static const struct option long_opts[] = {
 	{ "sparc",              no_argument,       NULL,   OPT_SPARC },
 	{ "delta",              optional_argument, NULL,   OPT_DELTA },
 	{ "lzma",               optional_argument, NULL,   OPT_LZMA },
+	{ "lzma2",              optional_argument, NULL,   OPT_LZMA2 },
 
 	// Other
 	{ "format",             required_argument, NULL,   'F' },
@@ -141,6 +143,7 @@ add_filter(lzma_vli id, const char *opt_str)
 		break;
 
 	case LZMA_FILTER_LZMA:
+	case LZMA_FILTER_LZMA2:
 		opt_filters[filter_count].options
 				= parse_options_lzma(opt_str);
 		break;
@@ -301,6 +304,10 @@ parse_real(int argc, char **argv)
 			add_filter(LZMA_FILTER_LZMA, optarg);
 			break;
 
+		case OPT_LZMA2:
+			add_filter(LZMA_FILTER_LZMA2, optarg);
+			break;
+
 		// Other
 
 		// --format
@@ -445,7 +452,8 @@ static void
 set_compression_settings(void)
 {
 	if (filter_count == 0) {
-		opt_filters[0].id = LZMA_FILTER_LZMA;
+		opt_filters[0].id = opt_header == HEADER_ALONE
+				? LZMA_FILTER_LZMA : LZMA_FILTER_LZMA2;
 		opt_filters[0].options = (lzma_options_lzma *)(
 				lzma_preset_lzma + preset_number);
 		filter_count = 1;
@@ -463,13 +471,15 @@ set_compression_settings(void)
 		my_exit(ERROR);
 	}
 
-	const uint32_t memory_limit = opt_memory / (1024 * 1024) + 1;
-	uint32_t memory_usage = lzma_memory_usage(opt_filters, true);
+	uint64_t memory_usage = lzma_memusage_encoder(opt_filters);
+			/* opt_mode == MODE_COMPRESS
+			? lzma_memusage_encoder(opt_filters)
+			: lzma_memusage_decoder(opt_filters); */
 
 	// Don't go over the memory limits when the default
 	// setting is used.
 	if (preset_default) {
-		while (memory_usage > memory_limit) {
+		while (memory_usage > opt_memory) {
 			if (preset_number == 0) {
 				errmsg(V_ERROR, _("Memory usage limit is too "
 						"small for any internal "
@@ -481,11 +491,10 @@ set_compression_settings(void)
 			opt_filters[0].options = (lzma_options_lzma *)(
 					lzma_preset_lzma
 					+ preset_number);
-			memory_usage = lzma_memory_usage(opt_filters,
-					true);
+			memory_usage = lzma_memusage_encoder(opt_filters);
 		}
 	} else {
-		if (memory_usage > memory_limit) {
+		if (memory_usage > opt_memory) {
 			errmsg(V_ERROR, _("Memory usage limit is too small "
 					"for the given filter setup"));
 			my_exit(ERROR);
@@ -494,12 +503,8 @@ set_compression_settings(void)
 
 	// Limit the number of worked threads so that memory usage
 	// limit isn't exceeded.
-	// FIXME: Probably should use bytes instead of mebibytes for
-	// memory_usage and memory_limit.
-	if (memory_usage == 0)
-		memory_usage = 1;
-
-	size_t thread_limit = memory_limit / memory_usage;
+	assert(memory_usage > 0);
+	size_t thread_limit = opt_memory / memory_usage;
 	if (thread_limit == 0)
 		thread_limit = 1;
 
diff --git a/src/lzma/args.h b/src/lzma/args.h
index c6098558..b60e7545 100644
--- a/src/lzma/args.h
+++ b/src/lzma/args.h
@@ -52,8 +52,8 @@ extern bool opt_preserve_name;
 extern enum tool_mode opt_mode;
 extern enum header_type opt_header;
 
-extern lzma_check_type opt_check;
-extern lzma_options_filter opt_filters[8];
+extern lzma_check opt_check;
+extern lzma_filter opt_filters[8];
 
 extern const char *stdin_filename;
 
diff --git a/src/lzma/options.c b/src/lzma/options.c
index c82cb1a0..b2ec200e 100644
--- a/src/lzma/options.c
+++ b/src/lzma/options.c
@@ -299,9 +299,9 @@ extern lzma_options_lzma *
 parse_options_lzma(const char *str)
 {
 	static const name_id_map modes[] = {
-		{ "fast", LZMA_MODE_FAST },
-		{ "best", LZMA_MODE_BEST },
-		{ NULL,   0 }
+		{ "fast",   LZMA_MODE_FAST },
+		{ "normal", LZMA_MODE_NORMAL },
+		{ NULL,     0 }
 	};
 
 	static const name_id_map mfs[] = {
@@ -317,9 +317,9 @@ parse_options_lzma(const char *str)
 		{ "dict", NULL,   LZMA_DICTIONARY_SIZE_MIN,
 				LZMA_DICTIONARY_SIZE_MAX },
 		{ "lc",   NULL,   LZMA_LITERAL_CONTEXT_BITS_MIN,
-				  LZMA_LITERAL_CONTEXT_BITS_MAX },
+				LZMA_LITERAL_CONTEXT_BITS_MAX },
 		{ "lp",   NULL,   LZMA_LITERAL_POS_BITS_MIN,
-				  LZMA_LITERAL_POS_BITS_MAX },
+				LZMA_LITERAL_POS_BITS_MAX },
 		{ "pb",   NULL,   LZMA_POS_BITS_MIN, LZMA_POS_BITS_MAX },
 		{ "mode", modes,  0, 0 },
 		{ "fb",   NULL,   LZMA_FAST_BYTES_MIN, LZMA_FAST_BYTES_MAX },
@@ -334,7 +334,9 @@ parse_options_lzma(const char *str)
 		.literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT,
 		.literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT,
 		.pos_bits = LZMA_POS_BITS_DEFAULT,
-		.mode = LZMA_MODE_BEST,
+		.preset_dictionary =  NULL,
+		.persistent = false,
+		.mode = LZMA_MODE_NORMAL,
 		.fast_bytes = LZMA_FAST_BYTES_DEFAULT,
 		.match_finder = LZMA_MF_BT4,
 		.match_finder_cycles = 0,
diff --git a/src/lzma/process.c b/src/lzma/process.c
index c180caf7..b4387709 100644
--- a/src/lzma/process.c
+++ b/src/lzma/process.c
@@ -63,12 +63,7 @@ process_init(void)
 	}
 
 	for (size_t i = 0; i < opt_threads; ++i)
-		threads[i] = (thread_data){
-			.strm = LZMA_STREAM_INIT_VAR,
-			.options = NULL,
-			.pair = NULL,
-			.in_use = false,
-		};
+		memzero(&threads[i], sizeof(threads[0]));
 
 	if (pthread_attr_init(&thread_attr)
 			|| pthread_attr_setdetachstate(
@@ -169,7 +164,9 @@ single_init(thread_data *t)
 		}
 	} else {
 		// TODO Restrict file format if requested on the command line.
-		ret = lzma_auto_decoder(&t->strm);
+		ret = lzma_auto_decoder(&t->strm, opt_memory,
+				LZMA_WARN_UNSUPPORTED_CHECK
+					| LZMA_CONCATENATED);
 	}
 
 	if (ret != LZMA_OK) {
@@ -185,36 +182,6 @@ single_init(thread_data *t)
 }
 
 
-static lzma_ret
-single_skip_padding(thread_data *t, uint8_t *in_buf)
-{
-	// Handle decoding of concatenated Streams. There can be arbitrary
-	// number of nul-byte padding between the Streams, which must be
-	// ignored.
-	//
-	// NOTE: Concatenating LZMA_Alone files works only if at least
-	// one of lc, lp, and pb is non-zero. Using the concatenation
-	// on LZMA_Alone files is strongly discouraged.
-	while (true) {
-		while (t->strm.avail_in > 0) {
-			if (*t->strm.next_in != '\0')
-				return LZMA_OK;
-
-			++t->strm.next_in;
-			--t->strm.avail_in;
-		}
-
-		if (t->pair->src_eof)
-			return LZMA_STREAM_END;
-
-		t->strm.next_in = in_buf;
-		t->strm.avail_in = io_read(t->pair, in_buf, BUFSIZ);
-		if (t->strm.avail_in == SIZE_MAX)
-			return LZMA_DATA_ERROR;
-	}
-}
-
-
 static void *
 single(thread_data *t)
 {
@@ -227,10 +194,11 @@ single(thread_data *t)
 	uint8_t in_buf[BUFSIZ];
 	uint8_t out_buf[BUFSIZ];
 	lzma_action action = LZMA_RUN;
-	lzma_ret ret;
 	bool success = false;
 
 	t->strm.avail_in = 0;
+	t->strm.next_out = out_buf;
+	t->strm.avail_out = BUFSIZ;
 
 	while (!user_abort) {
 		if (t->strm.avail_in == 0 && !t->pair->src_eof) {
@@ -239,48 +207,36 @@ single(thread_data *t)
 
 			if (t->strm.avail_in == SIZE_MAX)
 				break;
-			else if (t->pair->src_eof
-					&& opt_mode == MODE_COMPRESS)
+
+			if (t->pair->src_eof)
 				action = LZMA_FINISH;
 		}
 
-		t->strm.next_out = out_buf;
-		t->strm.avail_out = BUFSIZ;
-
-		ret = lzma_code(&t->strm, action);
+		const lzma_ret ret = lzma_code(&t->strm, action);
 
-		if (opt_mode != MODE_TEST)
+		if ((t->strm.avail_out == 0 || ret != LZMA_OK)
+				&& opt_mode != MODE_TEST) {
 			if (io_write(t->pair, out_buf,
 					BUFSIZ - t->strm.avail_out))
 				break;
 
+			t->strm.next_out = out_buf;
+			t->strm.avail_out = BUFSIZ;
+		}
+
 		if (ret != LZMA_OK) {
 			if (ret == LZMA_STREAM_END) {
-				if (opt_mode == MODE_COMPRESS) {
-					assert(t->pair->src_eof);
-					success = true;
-					break;
-				}
-
-				// Support decoding concatenated .lzma files.
-				ret = single_skip_padding(t, in_buf);
-
-				if (ret == LZMA_STREAM_END) {
-					assert(t->pair->src_eof);
-					success = true;
-					break;
-				}
-
-				if (ret == LZMA_OK && !single_init(t))
-					continue;
-
-				break;
-
+				// FIXME !!! This doesn't work when decoding
+				// LZMA_Alone files, because LZMA_Alone decoder
+				// doesn't wait for LZMA_FINISH.
+				assert(t->pair->src_eof);
+				success = true;
 			} else {
 				errmsg(V_ERROR, "%s: %s", t->pair->src_name,
 						str_strm_error(ret));
-				break;
 			}
+
+			break;
 		}
 	}
 
diff --git a/src/lzmadec/lzmadec.c b/src/lzmadec/lzmadec.c
index 1fc561b7..ed5947ad 100644
--- a/src/lzmadec/lzmadec.c
+++ b/src/lzmadec/lzmadec.c
@@ -65,14 +65,7 @@ static uint8_t out_buf[BUFSIZ];
 static lzma_stream strm = LZMA_STREAM_INIT;
 
 /// Number of bytes to use memory at maximum
-static size_t mem_limit;
-
-/// Memory allocation hooks
-static lzma_allocator allocator = {
-	.alloc = (void *(*)(void *, size_t, size_t))(&lzma_memlimit_alloc),
-	.free = (void (*)(void *, void *))(&lzma_memlimit_free),
-	.opaque = NULL,
-};
+static size_t memlimit;
 
 /// Program name to be shown in error messages
 static const char *argv0;
@@ -116,7 +109,7 @@ help(void)
 		" MiB of memory at maximum.\n"
 "\n"
 "Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n",
-		argv0, ((uint64_t)(mem_limit) + 512 * 1024) / (1024 * 1024));
+		argv0, ((uint64_t)(memlimit) + 512 * 1024) / (1024 * 1024));
 		// Using PRIu64 above instead of %zu to support pre-C99 libc.
 	exit(0);
 }
@@ -148,7 +141,7 @@ version(void)
 /// Finds out the amount of physical memory in the system, and sets
 /// a default memory usage limit.
 static void
-set_default_mem_limit(void)
+set_default_memlimit(void)
 {
 	uint64_t mem = physmem();
 	if (mem != 0) {
@@ -159,10 +152,10 @@ set_default_mem_limit(void)
 			mem = SIZE_MAX;
 #endif
 
-		mem_limit = mem / 3;
+		memlimit = mem / 3;
 	} else {
 		// Cannot autodetect, use 10 MiB as the default limit.
-		mem_limit = (1U << 23) + (1U << 21);
+		memlimit = (1U << 23) + (1U << 21);
 	}
 
 	return;
@@ -272,7 +265,7 @@ parse_options(int argc, char **argv)
 			break;
 
 		case 'M':
-			mem_limit = str_to_size(optarg);
+			memlimit = str_to_size(optarg);
 			break;
 
 		case 'h':
@@ -309,19 +302,20 @@ parse_options(int argc, char **argv)
 static void
 init(void)
 {
+	const uint32_t flags = LZMA_WARN_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
 	lzma_ret ret;
 
 	switch (format_type) {
 	case FORMAT_AUTO:
-		ret = lzma_auto_decoder(&strm);
+		ret = lzma_auto_decoder(&strm, memlimit, flags);
 		break;
 
 	case FORMAT_NATIVE:
-		ret = lzma_stream_decoder(&strm);
+		ret = lzma_stream_decoder(&strm, memlimit, flags);
 		break;
 
 	case FORMAT_ALONE:
-		ret = lzma_alone_decoder(&strm);
+		ret = lzma_alone_decoder(&strm, memlimit);
 		break;
 
 	default:
@@ -345,50 +339,6 @@ init(void)
 
 
 static void
-read_input(void)
-{
-	strm.next_in = in_buf;
-	strm.avail_in = fread(in_buf, 1, BUFSIZ, file);
-
-	if (ferror(file)) {
-		// POSIX says that fread() sets errno if an error occurred.
-		// ferror() doesn't touch errno.
-		fprintf(stderr, "%s: %s: Error reading input file: %s\n",
-				argv0, filename, strerror(errno));
-		exit(ERROR);
-	}
-
-	return;
-}
-
-
-static bool
-skip_padding(void)
-{
-	// Handle concatenated Streams. There can be arbitrary number of
-	// nul-byte padding between the Streams, which must be ignored.
-	//
-	// NOTE: Concatenating LZMA_Alone files works only if at least
-	// one of lc, lp, and pb is non-zero. Using the concatenation
-	// on LZMA_Alone files is strongly discouraged.
-	while (true) {
-		while (strm.avail_in > 0) {
-			if (*strm.next_in != '\0')
-				return true;
-
-			++strm.next_in;
-			--strm.avail_in;
-		}
-
-		if (feof(file))
-			return false;
-
-		read_input();
-	}
-}
-
-
-static void
 uncompress(void)
 {
 	if (file == stdin && !force && isatty(STDIN_FILENO)) {
@@ -400,44 +350,65 @@ uncompress(void)
 	}
 
 	init();
+
 	strm.avail_in = 0;
+	strm.next_out = out_buf;
+	strm.avail_out = BUFSIZ;
+
+	lzma_action action = LZMA_RUN;
 
 	while (true) {
-		if (strm.avail_in == 0)
-			read_input();
+		if (strm.avail_in == 0) {
+			strm.next_in = in_buf;
+			strm.avail_in = fread(in_buf, 1, BUFSIZ, file);
+
+			if (ferror(file)) {
+				// POSIX says that fread() sets errno if
+				// an error occurred. ferror() doesn't
+				// touch errno.
+				fprintf(stderr, "%s: %s: Error reading "
+						"input file: %s\n",
+						argv0, filename,
+						strerror(errno));
+				exit(ERROR);
+			}
 
-		strm.next_out = out_buf;
-		strm.avail_out = BUFSIZ;
+			if (feof(file))
+				action = LZMA_FINISH;
+		}
 
-		const lzma_ret ret = lzma_code(&strm, LZMA_RUN);
+		const lzma_ret ret = lzma_code(&strm, action);
 
 		// Write and check write error before checking decoder error.
 		// This way as much data as possible gets written to output
-		// even if decoder detected an error. Checking write error
-		// needs to be done before checking decoder error due to
-		// how concatenated Streams are handled a few lines later.
-		const size_t write_size = BUFSIZ - strm.avail_out;
-		if (fwrite(out_buf, 1, write_size, stdout) != write_size) {
-			// Wouldn't be a surprise if writing to stderr would
-			// fail too but at least try to show an error message.
-			fprintf(stderr, "%s: Cannot write to "
-					"standard output: %s\n", argv0,
-					strerror(errno));
-			exit(ERROR);
+		// even if decoder detected an error.
+		if (strm.avail_out == 0 || ret != LZMA_OK) {
+			const size_t write_size = BUFSIZ - strm.avail_out;
+
+			if (fwrite(out_buf, 1, write_size, stdout)
+					!= write_size) {
+				// Wouldn't be a surprise if writing to stderr
+				// would fail too but at least try to show an
+				// error message.
+				fprintf(stderr, "%s: Cannot write to "
+						"standard output: %s\n", argv0,
+						strerror(errno));
+				exit(ERROR);
+			}
+
+			strm.next_out = out_buf;
+			strm.avail_out = BUFSIZ;
 		}
 
 		if (ret != LZMA_OK) {
-			if (ret == LZMA_STREAM_END) {
-				if (skip_padding()) {
-					init();
-					continue;
-				}
-
+			// FIXME !!! Doesn't work with LZMA_Alone for the
+			// same reason as in process.c.
+			if (ret == LZMA_STREAM_END)
 				return;
-			}
 
 			fprintf(stderr, "%s: %s: ", argv0, filename);
 
+			// FIXME Add LZMA_*_CHECK and LZMA_FORMAT_ERROR.
 			switch (ret) {
 			case LZMA_DATA_ERROR:
 				fprintf(stderr, "File is corrupt\n");
@@ -452,6 +423,11 @@ uncompress(void)
 				fprintf(stderr, "%s\n", strerror(ENOMEM));
 				exit(ERROR);
 
+			case LZMA_MEMLIMIT_ERROR:
+				fprintf(stderr, "Memory usage limit "
+						"reached\n");
+				exit(ERROR);
+
 			case LZMA_BUF_ERROR:
 				fprintf(stderr, "Unexpected end of input\n");
 				exit(ERROR);
@@ -479,23 +455,12 @@ main(int argc, char **argv)
 {
 	argv0 = argv[0];
 
-	set_default_mem_limit();
+	set_default_memlimit();
 
 	parse_options(argc, argv);
 
 	lzma_init_decoder();
 
-	lzma_memlimit *mem_limiter = lzma_memlimit_create(mem_limit);
-	if (mem_limiter == NULL) {
-		fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM));
-		exit(ERROR);
-	}
-
-	assert(lzma_memlimit_count(mem_limiter) == 0);
-
-	allocator.opaque = mem_limiter;
-	strm.allocator = &allocator;
-
 #ifdef WIN32
 	setmode(fileno(stdin), O_BINARY);
 	setmode(fileno(stdout), O_BINARY);
@@ -531,8 +496,6 @@ main(int argc, char **argv)
 	// Free the memory only when debugging. Freeing wastes some time,
 	// but allows detecting possible memory leaks with Valgrind.
 	lzma_end(&strm);
-	assert(lzma_memlimit_count(mem_limiter) == 0);
-	lzma_memlimit_end(mem_limiter, false);
 #endif
 
 	return exit_status;