liblzma: Add lzma_stream_encoder_mt() for threaded compression.

This is the simplest method to do threading, which splits
the uncompressed data into blocks and compresses them
independently from each other. There's room for improvement
especially to reduce the memory usage, but nevertheless,
this is a good start.

1 files changed, 163 insertions, 0 deletions

diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h
index 7a9ffc64..e68c8940 100644
--- a/src/liblzma/api/lzma/container.h
+++ b/src/liblzma/api/lzma/container.h
@@ -61,6 +61,127 @@
 
 
 /**
+ * \brief       Multithreading options
+ */
+typedef struct {
+	/**
+	 * \brief       Flags
+	 *
+	 * Set this to zero if no flags are wanted.
+	 *
+	 * No flags are currently supported.
+	 */
+	uint32_t flags;
+
+	/**
+	 * \brief       Number of worker threads to use
+	 */
+	uint32_t threads;
+
+	/**
+	 * \brief       Maximum uncompressed size of a Block
+	 *
+	 * The encoder will start a new .xz Block every block_size bytes.
+	 * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code()
+	 * the caller may tell liblzma to start a new Block earlier.
+	 *
+	 * With LZMA2, a recommended block size is 2-4 times the LZMA2
+	 * dictionary size. With very small dictionaries, it is recommended
+	 * to use at least 1 MiB block size for good compression ratio, even
+	 * if this is more than four times the dictionary size. Note that
+	 * these are only recommendations for typical use cases; feel free
+	 * to use other values. Just keep in mind that using a block size
+	 * less than the LZMA2 dictionary size is waste of RAM.
+	 *
+	 * Set this to 0 to let liblzma choose the block size depending
+	 * on the compression options. For LZMA2 it will be 3*dict_size
+	 * or 1 MiB, whichever is more.
+	 */
+	uint64_t block_size;
+
+	/**
+	 * \brief       Timeout to allow lzma_code() to return early
+	 *
+	 * Multithreading can make liblzma to consume input and produce
+	 * output in a very bursty way: it may first read a lot of input
+	 * to fill internal buffers, then no input or output occurs for
+	 * a while.
+	 *
+	 * In single-threaded mode, lzma_code() won't return until it has
+	 * either consumed all the input or filled the output buffer. If
+	 * this is done in multithreaded mode, it may cause a call
+	 * lzma_code() to take even tens of seconds, which isn't acceptable
+	 * in all applications.
+	 *
+	 * To avoid very long blocking times in lzma_code(), a timeout
+	 * (in milliseconds) may be set here. If lzma_code() would block
+	 * longer than this number of milliseconds, it will return with
+	 * LZMA_OK. Reasonable values are 100 ms or more. The xz command
+	 * line tool uses 300 ms.
+	 *
+	 * If long blocking times are fine for you, set timeout to a special
+	 * value of 0, which will disable the timeout mechanism and will make
+	 * lzma_code() block until all the input is consumed or the output
+	 * buffer has been filled.
+	 *
+	 * \note        Even with a timeout, lzma_code() might sometimes take
+	 *              somewhat long time to return. No timing guarantees
+	 *              are made.
+	 */
+	uint32_t timeout;
+
+	/**
+	 * \brief       Compression preset (level and possible flags)
+	 *
+	 * The preset is set just like with lzma_easy_encoder().
+	 * The preset is ignored if filters below is non-NULL.
+	 */
+	uint32_t preset;
+
+	/**
+	 * \brief       Filter chain (alternative to a preset)
+	 *
+	 * If this is NULL, the preset above is used. Otherwise the preset
+	 * is ignored and the filter chain specified here is used.
+	 */
+	const lzma_filter *filters;
+
+	/**
+	 * \brief       Integrity check type
+	 *
+	 * See check.h for available checks. The xz command line tool
+	 * defaults to LZMA_CHECK_CRC64, which is a good choice if you
+	 * are unsure.
+	 */
+	lzma_check check;
+
+	/*
+	 * Reserved space to allow possible future extensions without
+	 * breaking the ABI. You should not touch these, because the names
+	 * of these variables may change. These are and will never be used
+	 * with the currently supported options, so it is safe to leave these
+	 * uninitialized.
+	 */
+	lzma_reserved_enum reserved_enum1;
+	lzma_reserved_enum reserved_enum2;
+	lzma_reserved_enum reserved_enum3;
+	uint32_t reserved_int1;
+	uint32_t reserved_int2;
+	uint32_t reserved_int3;
+	uint32_t reserved_int4;
+	uint64_t reserved_int5;
+	uint64_t reserved_int6;
+	uint64_t reserved_int7;
+	uint64_t reserved_int8;
+	void *reserved_ptr1;
+	void *reserved_ptr2;
+	void *reserved_ptr3;
+	void *reserved_ptr4;
+
+} lzma_mt;
+
+
+/**
  * \brief       Calculate approximate memory usage of easy encoder
  *
  * This function is a wrapper for lzma_raw_encoder_memusage().
@@ -191,6 +312,48 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm,
 
 
 /**
+ * \brief       Calculate approximate memory usage of multithreaded .xz encoder
+ *
+ * Since doing the encoding in threaded mode doesn't affect the memory
+ * requirements of single-threaded decompressor, you can use
+ * lzma_easy_decoder_memusage(options->preset) or
+ * lzma_raw_decoder_memusage(options->filters) to calculate
+ * the decompressor memory requirements.
+ *
+ * \param       options Compression options
+ *
+ * \return      Number of bytes of memory required for encoding with the
+ *              given options. If an error occurs, for example due to
+ *              unsupported preset or filter chain, UINT64_MAX is returned.
+ */
+extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage(
+		const lzma_mt *options) lzma_nothrow lzma_attr_pure;
+
+
+/**
+ * \brief       Initialize multithreaded .xz Stream encoder
+ *
+ * This provides the functionality of lzma_easy_encoder() and
+ * lzma_stream_encoder() as a single function for multithreaded use.
+ *
+ * TODO: For lzma_code(), only LZMA_RUN and LZMA_FINISH are currently
+ * supported. Support for other actions has been planned.
+ *
+ * \param       strm    Pointer to properly prepared lzma_stream
+ * \param       options Pointer to multithreaded compression options
+ *
+ * \return      - LZMA_OK
+ *              - LZMA_MEM_ERROR
+ *              - LZMA_UNSUPPORTED_CHECK
+ *              - LZMA_OPTIONS_ERROR
+ *              - LZMA_PROG_ERROR
+ */
+extern LZMA_API(lzma_ret) lzma_stream_encoder_mt(
+		lzma_stream *strm, const lzma_mt *options)
+		lzma_nothrow lzma_attr_warn_unused_result;
+
+
+/**
  * \brief       Initialize .lzma encoder (legacy file format)
  *
  * The .lzma format is sometimes called the LZMA_Alone format, which is the