xz: Add a default soft memory usage limit for --threads=0.

This is a soft limit in sense that it only affects the number of
threads. It never makes xz fail and it never makes xz change
settings that would affect the compressed output.

The idea is to make -T0 have more reasonable behavior when
the system has very many cores or when a memory-hungry
compression options are used. This also helps with 32-bit xz,
preventing it from running out of address space.

The downside of this commit is that now the number of threads
might become too low compared to what the user expected. I
hope this to be an acceptable compromise as the old behavior
has been a source of well-argued complaints for a long time.

1 files changed, 29 insertions, 9 deletions

diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index 18eee7ec..2cc3f4f2 100644
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -29,6 +29,13 @@ static uint64_t memlimit_decompress = 0;
 
 /// Default memory usage for multithreaded modes:
 ///
+///   - Default value for --memlimit-compress when automatic number of threads
+///     is used. However, if the limit wouldn't allow even one thread then
+///     the limit is ignored in coder.c and one thread will be used anyway.
+///     This mess is a compromise: we wish to prevent -T0 from using too
+///     many threads but we also don't want xz to give an error due to
+///     a memlimit that the user didn't explicitly set.
+///
 ///   - Default value for --memlimit-mt-decompress
 ///
 /// This value is caluclated in hardware_init() and cannot be changed later.
@@ -151,15 +158,12 @@ hardware_memlimit_set(uint64_t new_memlimit,
 extern uint64_t
 hardware_memlimit_get(enum operation_mode mode)
 {
-	// Zero is a special value that indicates the default. Currently
-	// the default simply disables the limit. Once there is threading
-	// support, this might be a little more complex, because there will
-	// probably be a special case where a user asks for "optimal" number
-	// of threads instead of a specific number (this might even become
-	// the default mode). Each thread may use a significant amount of
-	// memory. When there are no memory usage limits set, we need some
-	// default soft limit for calculating the "optimal" number of
-	// threads.
+	// 0 is a special value that indicates the default.
+	// It disables the limit in single-threaded mode.
+	//
+	// NOTE: For multithreaded decompression, this is the hard limit
+	// (memlimit_stop). hardware_memlimit_mtdec_get() gives the
+	// soft limit (memlimit_threaded).
 	const uint64_t memlimit = mode == MODE_COMPRESS
 			? memlimit_compress : memlimit_decompress;
 	return memlimit != 0 ? memlimit : UINT64_MAX;
@@ -167,6 +171,22 @@ hardware_memlimit_get(enum operation_mode mode)
 
 
 extern uint64_t
+hardware_memlimit_mtenc_get(void)
+{
+	return memlimit_compress == 0 && threads_are_automatic
+			? memlimit_mt_default
+			: hardware_memlimit_get(MODE_COMPRESS);
+}
+
+
+extern bool
+hardware_memlimit_mtenc_is_default(void)
+{
+	return memlimit_compress == 0 && threads_are_automatic;
+}
+
+
+extern uint64_t
 hardware_memlimit_mtdec_get(void)
 {
 	uint64_t m = memlimit_mtdec != 0