Sort of garbage collection commit. :-| Many things are still

broken. API has changed a lot and it will still change a
little more here and there. The command line tool doesn't
have all the required changes to reflect the API changes, so
it's easy to get "internal error" or trigger assertions.

7 files changed, 267 insertions, 14 deletions

diff --git a/debug/Makefile.am b/debug/Makefile.am
index 71ca7e4c..6ed5bc99 100644
--- a/debug/Makefile.am
+++ b/debug/Makefile.am
@@ -16,7 +16,10 @@ noinst_PROGRAMS = \
 	repeat \
 	sync_flush \
 	full_flush \
-	memusage
+	memusage \
+	crc32 \
+	known_sizes \
+	hex2bin
 
 AM_CPPFLAGS = \
 	-I@top_srcdir@/src/common \
diff --git a/debug/crc32.c b/debug/crc32.c
new file mode 100644
index 00000000..4052a863
--- /dev/null
+++ b/debug/crc32.c
@@ -0,0 +1,45 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       crc32.c
+/// \brief      Primitive CRC32 calculation tool
+//
+//  Copyright (C) 2008 Lasse Collin
+//
+//  This library is free software; you can redistribute it and/or
+//  modify it under the terms of the GNU Lesser General Public
+//  License as published by the Free Software Foundation; either
+//  version 2.1 of the License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+//  Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "sysdefs.h"
+#include <stdio.h>
+
+
+int
+main(void)
+{
+	uint32_t crc = 0;
+
+	do {
+		uint8_t buf[BUFSIZ];
+		const size_t size = fread(buf, 1, sizeof(buf), stdin);
+		crc = lzma_crc32(buf, size, crc);
+	} while (!ferror(stdin) && !feof(stdin));
+
+	//printf("%08" PRIX32 "\n", crc);
+
+	// I want it little endian so it's easy to work with hex editor.
+	printf("%02" PRIX32 " ", crc & 0xFF);
+	printf("%02" PRIX32 " ", (crc >> 8) & 0xFF);
+	printf("%02" PRIX32 " ", (crc >> 16) & 0xFF);
+	printf("%02" PRIX32 " ", crc >> 24);
+	printf("\n");
+
+	return 0;
+}
diff --git a/debug/full_flush.c b/debug/full_flush.c
index fd775ce3..db82a60a 100644
--- a/debug/full_flush.c
+++ b/debug/full_flush.c
@@ -72,18 +72,24 @@ main(int argc, char **argv)
 
 	file_in = argc > 1 ? fopen(argv[1], "rb") : stdin;
 
+
 	// Config
-	lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
-	filters[0].id = LZMA_FILTER_SUBBLOCK;
-	filters[0].options = NULL;
+	lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+	filters[0].id = LZMA_FILTER_LZMA2;
+	filters[0].options = (void *)&lzma_preset_lzma[0];
 	filters[1].id = LZMA_VLI_VALUE_UNKNOWN;
 
 	// Init
-	if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) {
+	if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_SHA256) != LZMA_OK) {
 		fprintf(stderr, "init failed\n");
 		exit(1);
 	}
 
+// 	if (lzma_easy_encoder(&strm, 1)) {
+// 		fprintf(stderr, "init failed\n");
+// 		exit(1);
+// 	}
+
 	// Encoding
 	encode(0, LZMA_FULL_FLUSH);
 	encode(6, LZMA_FULL_FLUSH);
diff --git a/debug/hex2bin.c b/debug/hex2bin.c
new file mode 100644
index 00000000..ebfc289f
--- /dev/null
+++ b/debug/hex2bin.c
@@ -0,0 +1,54 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       hex2bin.c
+/// \brief      Converts hexadecimal input strings to binary
+//
+//  This code has been put into the public domain.
+//
+//  This library is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "sysdefs.h"
+#include <stdio.h>
+#include <ctype.h>
+
+
+static int
+getbin(int x)
+{
+	if (x >= '0' && x <= '9')
+		return x - '0';
+
+	if (x >= 'A' && x <= 'F')
+		return x - 'A' + 10;
+
+	return x - 'a' + 10;
+}
+
+
+int
+main(void)
+{
+	while (true) {
+		int byte = getchar();
+		if (byte == EOF)
+			return 0;
+		if (!isxdigit(byte))
+			continue;
+
+		const int digit = getchar();
+		if (digit == EOF || !isxdigit(digit)) {
+			fprintf(stderr, "Invalid input\n");
+			return 1;
+		}
+
+		byte = (getbin(byte) << 4) | getbin(digit);
+		if (putchar(byte) == EOF) {
+			perror(NULL);
+			return 1;
+		}
+	}
+}
diff --git a/debug/known_sizes.c b/debug/known_sizes.c
new file mode 100644
index 00000000..571c105e
--- /dev/null
+++ b/debug/known_sizes.c
@@ -0,0 +1,135 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       known_sizes.c
+/// \brief      Encodes .lzma Stream with sizes known in Block Header
+///
+/// The input file is encoded in RAM, and the known Compressed Size
+/// and/or Uncompressed Size values are stored in the Block Header.
+/// As of writing there's no such Stream encoder in liblzma.
+//
+//  Copyright (C) 2008 Lasse Collin
+//
+//  This library is free software; you can redistribute it and/or
+//  modify it under the terms of the GNU Lesser General Public
+//  License as published by the Free Software Foundation; either
+//  version 2.1 of the License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+//  Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "sysdefs.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/unistd.h>
+#include <stdio.h>
+
+
+// Support file sizes up to 1 MiB. We use this for output space too, so files
+// close to 1 MiB had better compress at least a little or we have a buffer
+// overflow.
+#define BUFFER_SIZE (1U << 20)
+
+
+int
+main(void)
+{
+	// Allocate the buffers.
+	uint8_t *in = malloc(BUFFER_SIZE);
+	uint8_t *out = malloc(BUFFER_SIZE);
+	if (in == NULL || out == NULL)
+		return 1;
+
+	// Fill the input buffer.
+	const size_t in_size = fread(in, 1, BUFFER_SIZE, stdin);
+
+	// Filter setup
+	lzma_filter filters[] = {
+		{
+			.id = LZMA_FILTER_LZMA2,
+			.options = (void *)(&lzma_preset_lzma[0])
+		},
+		{
+			.id = LZMA_VLI_VALUE_UNKNOWN
+		}
+	};
+
+	lzma_block block = {
+		.check = LZMA_CHECK_CRC32,
+		.compressed_size = BUFFER_SIZE, // Worst case reserve
+		.uncompressed_size = in_size,
+		.filters = filters,
+	};
+
+	// FIXME Insane paranoia in liblzma.
+	if (lzma_block_header_size(&block) != LZMA_OK)
+		return 1;
+
+	// We don't actually know the compressed size, so don't tell it to
+	// Block encoder.
+	block.compressed_size = LZMA_VLI_VALUE_UNKNOWN;
+
+	lzma_stream strm = LZMA_STREAM_INIT;
+	if (lzma_block_encoder(&strm, &block) != LZMA_OK)
+		return 1;
+
+	// Reserve space for Stream Header and Block Header.
+	size_t out_size = LZMA_STREAM_HEADER_SIZE + block.header_size;
+
+	strm.next_in = in;
+	strm.avail_in = in_size;
+	strm.next_out = out + out_size;
+	strm.avail_out = BUFFER_SIZE - out_size;
+
+	if (lzma_code(&strm, LZMA_FINISH) != LZMA_STREAM_END)
+		return 1;
+
+	out_size += strm.total_out;
+
+	if (lzma_block_header_encode(&block, out + LZMA_STREAM_HEADER_SIZE)
+			!= LZMA_OK)
+		return 1;
+
+	lzma_index *idx = lzma_index_init(NULL, NULL);
+	if (idx == NULL)
+		return 1;
+
+	if (lzma_index_append(idx, NULL, block.header_size + strm.total_out,
+			strm.total_in) != LZMA_OK)
+		return 1;
+
+	if (lzma_index_encoder(&strm, idx) != LZMA_OK)
+		return 1;
+
+	if (lzma_code(&strm, LZMA_RUN) != LZMA_STREAM_END)
+		return 1;
+
+	out_size += strm.total_out;
+
+	lzma_end(&strm);
+
+	lzma_index_end(idx, NULL);
+
+	// Encode the Stream Header and Stream Footer. backwards_size is
+	// needed only for the Stream Footer.
+	lzma_stream_flags sf = {
+		.backward_size = strm.total_out,
+		.check = block.check,
+	};
+
+	if (lzma_stream_header_encode(&sf, out) != LZMA_OK)
+		return 1;
+
+	if (lzma_stream_footer_encode(&sf, out + out_size) != LZMA_OK)
+		return 1;
+
+	out_size += LZMA_STREAM_HEADER_SIZE;
+
+	// Write out the file.
+	fwrite(out, 1, out_size, stdout);
+
+	return 0;
+}
diff --git a/debug/memusage.c b/debug/memusage.c
index 0716f5a2..eaf81f95 100644
--- a/debug/memusage.c
+++ b/debug/memusage.c
@@ -23,6 +23,7 @@
 int
 main(void)
 {
+	lzma_init();
 
 	lzma_options_lzma lzma = {
 		.dictionary_size = (1 << 27) + (1 << 26),
@@ -31,7 +32,7 @@ main(void)
 		.pos_bits = 2,
 		.preset_dictionary = NULL,
 		.preset_dictionary_size = 0,
-		.mode = LZMA_MODE_BEST,
+		.mode = LZMA_MODE_NORMAL,
 		.fast_bytes = 48,
 		.match_finder = LZMA_MF_BT4,
 		.match_finder_cycles = 0,
@@ -44,12 +45,13 @@ main(void)
 		{ UINT64_MAX, NULL }
 	};
 */
-	lzma_options_filter filters[] = {
+	lzma_filter filters[] = {
 		{ LZMA_FILTER_LZMA, &lzma },
 		{ UINT64_MAX, NULL }
 	};
 
-	printf("%u MiB\n", lzma_memory_usage(filters, true));
+	printf("Encoder: %10" PRIu64 " B\n", lzma_memusage_encoder(filters));
+	printf("Decoder: %10" PRIu64 " B\n", lzma_memusage_decoder(filters));
 
 	return 0;
 }
diff --git a/debug/sync_flush.c b/debug/sync_flush.c
index 03dfdd7d..eb6efef4 100644
--- a/debug/sync_flush.c
+++ b/debug/sync_flush.c
@@ -79,9 +79,10 @@ main(int argc, char **argv)
 		.literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT,
 		.pos_bits = LZMA_POS_BITS_DEFAULT,
 		.preset_dictionary = NULL,
-		.mode = LZMA_MODE_BEST,
+		.persistent = true,
+		.mode = LZMA_MODE_NORMAL,
 		.fast_bytes = 32,
-		.match_finder = LZMA_MF_BT3,
+		.match_finder = LZMA_MF_HC3,
 		.match_finder_cycles = 0,
 	};
 
@@ -101,24 +102,31 @@ main(int argc, char **argv)
 	opt_subblock.subfilter_options.id = LZMA_FILTER_DELTA;
 	opt_subblock.subfilter_options.options = &opt_delta;
 
-	lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
-	filters[0].id = LZMA_FILTER_LZMA;
+	lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+	filters[0].id = LZMA_FILTER_LZMA2;
 	filters[0].options = &opt_lzma;
 	filters[1].id = LZMA_VLI_VALUE_UNKNOWN;
 
 	// Init
-	if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_NONE) != LZMA_OK) {
+	if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) {
 		fprintf(stderr, "init failed\n");
 		exit(1);
 	}
 
 	// Encoding
+/*
 	encode(0, LZMA_SYNC_FLUSH);
 	encode(6, LZMA_SYNC_FLUSH);
 	encode(0, LZMA_SYNC_FLUSH);
-	encode(6, LZMA_SYNC_FLUSH);
+	encode(7, LZMA_SYNC_FLUSH);
 	encode(0, LZMA_SYNC_FLUSH);
 	encode(0, LZMA_FINISH);
+*/
+	encode(53, LZMA_SYNC_FLUSH);
+// 	opt_lzma.literal_context_bits = 2;
+// 	opt_lzma.literal_pos_bits = 1;
+// 	opt_lzma.pos_bits = 0;
+	encode(404, LZMA_FINISH);
 
 	// Clean up
 	lzma_end(&strm);