xz: Add a default soft memory usage limit for --threads=0.

This is a soft limit in sense that it only affects the number of
threads. It never makes xz fail and it never makes xz change
settings that would affect the compressed output.

The idea is to make -T0 have more reasonable behavior when
the system has very many cores or when a memory-hungry
compression options are used. This also helps with 32-bit xz,
preventing it from running out of address space.

The downside of this commit is that now the number of threads
might become too low compared to what the user expected. I
hope this to be an acceptable compromise as the old behavior
has been a source of well-argued complaints for a long time.

3 files changed, 82 insertions, 11 deletions

diff --git a/src/xz/coder.c b/src/xz/coder.c
index a2699a9b..224c2d39 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -220,12 +220,16 @@ coder_set_compression_settings(void)
 
 	// Get the memory usage. Note that if --format=raw was used,
 	// we can be decompressing.
-	const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
+	//
+	// If multithreaded .xz compression is done, this value will be
+	// replaced.
+	uint64_t memory_limit = hardware_memlimit_get(opt_mode);
 	uint64_t memory_usage = UINT64_MAX;
 	if (opt_mode == MODE_COMPRESS) {
 #ifdef HAVE_ENCODERS
 #	ifdef MYTHREAD_ENABLED
 		if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
+			memory_limit = hardware_memlimit_mtenc_get();
 			mt_options.threads = hardware_threads_get();
 			mt_options.block_size = opt_block_size;
 			mt_options.check = check;
@@ -304,6 +308,27 @@ coder_set_compression_settings(void)
 			}
 		}
 
+		// If the memory usage limit is only a soft limit (automatic
+		// number of threads and no --memlimit-compress), the limit
+		// is only used to reduce the number of threads and once at
+		// just one thread, the limit is completely ignored. This
+		// way -T0 won't use insane amount of memory but at the same
+		// time the soft limit will never make xz fail and never make
+		// xz change settings that would affect the compressed output.
+		if (hardware_memlimit_mtenc_is_default()) {
+			message(V_WARNING, _("Reduced the number of threads "
+				"from %s to one. The automatic memory usage "
+				"limit of %s MiB is still being exceeded. "
+				"%s MiB of memory is required. "
+				"Continuing anyway."),
+				uint64_to_str(hardware_threads_get(), 0),
+				uint64_to_str(
+					round_up_to_mib(memory_limit), 1),
+				uint64_to_str(
+					round_up_to_mib(memory_usage), 2));
+			return;
+		}
+
 		// If --no-adjust was used, we cannot drop to single-threaded
 		// mode since it produces different compressed output.
 		//
@@ -321,7 +346,6 @@ coder_set_compression_settings(void)
 		message(V_WARNING, _("Switching to single-threaded mode "
 			"to not exceed the memory usage limit of %s MiB"),
 			uint64_to_str(round_up_to_mib(memory_limit), 0));
-
 	}
 #	endif
 
diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index 18eee7ec..2cc3f4f2 100644
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -29,6 +29,13 @@ static uint64_t memlimit_decompress = 0;
 
 /// Default memory usage for multithreaded modes:
 ///
+///   - Default value for --memlimit-compress when automatic number of threads
+///     is used. However, if the limit wouldn't allow even one thread then
+///     the limit is ignored in coder.c and one thread will be used anyway.
+///     This mess is a compromise: we wish to prevent -T0 from using too
+///     many threads but we also don't want xz to give an error due to
+///     a memlimit that the user didn't explicitly set.
+///
 ///   - Default value for --memlimit-mt-decompress
 ///
 /// This value is caluclated in hardware_init() and cannot be changed later.
@@ -151,15 +158,12 @@ hardware_memlimit_set(uint64_t new_memlimit,
 extern uint64_t
 hardware_memlimit_get(enum operation_mode mode)
 {
-	// Zero is a special value that indicates the default. Currently
-	// the default simply disables the limit. Once there is threading
-	// support, this might be a little more complex, because there will
-	// probably be a special case where a user asks for "optimal" number
-	// of threads instead of a specific number (this might even become
-	// the default mode). Each thread may use a significant amount of
-	// memory. When there are no memory usage limits set, we need some
-	// default soft limit for calculating the "optimal" number of
-	// threads.
+	// 0 is a special value that indicates the default.
+	// It disables the limit in single-threaded mode.
+	//
+	// NOTE: For multithreaded decompression, this is the hard limit
+	// (memlimit_stop). hardware_memlimit_mtdec_get() gives the
+	// soft limit (memlimit_threaded).
 	const uint64_t memlimit = mode == MODE_COMPRESS
 			? memlimit_compress : memlimit_decompress;
 	return memlimit != 0 ? memlimit : UINT64_MAX;
@@ -167,6 +171,22 @@ hardware_memlimit_get(enum operation_mode mode)
 
 
 extern uint64_t
+hardware_memlimit_mtenc_get(void)
+{
+	return memlimit_compress == 0 && threads_are_automatic
+			? memlimit_mt_default
+			: hardware_memlimit_get(MODE_COMPRESS);
+}
+
+
+extern bool
+hardware_memlimit_mtenc_is_default(void)
+{
+	return memlimit_compress == 0 && threads_are_automatic;
+}
+
+
+extern uint64_t
 hardware_memlimit_mtdec_get(void)
 {
 	uint64_t m = memlimit_mtdec != 0
diff --git a/src/xz/hardware.h b/src/xz/hardware.h
index 1a5a7a67..2cd6aa23 100644
--- a/src/xz/hardware.h
+++ b/src/xz/hardware.h
@@ -37,9 +37,36 @@ extern void hardware_memlimit_set(uint64_t new_memlimit,
 		bool is_percentage);
 
 /// Get the current memory usage limit for compression or decompression.
+/// This is a hard limit that will not be exceeded. This is obeyed in
+/// both single-threaded and multithreaded modes.
 extern uint64_t hardware_memlimit_get(enum operation_mode mode);
 
+/// This returns a system-specific default value if all of the following
+/// conditions are true:
+///
+///   - An automatic number of threads was requested (--threads=0).
+///
+///   - --memlimit-compress wasn't used or it was reset to the default
+///     value by setting it to 0.
+///
+/// Otherwise this is identical to hardware_memlimit_get(MODE_COMPRESS).
+///
+/// The idea is to keep automatic thread count reasonable so that too
+/// high memory usage is avoided and, with 32-bit xz, running out of
+/// address space is avoided.
+extern uint64_t hardware_memlimit_mtenc_get(void);
+
+/// Returns true if the value returned by hardware_memlimit_mtenc_get() is
+/// a system-specific default value. coder.c uses this to ignore the default
+/// memlimit in case it's too small even for a single thread in multithreaded
+/// mode. This way the default limit will never make xz fail or affect the
+/// compressed output; it will only make xz reduce the number of threads.
+extern bool hardware_memlimit_mtenc_is_default(void);
+
 /// Get the current memory usage limit for multithreaded decompression.
+/// This is only used to reduce the number of threads. This limit can be
+/// exceeded if the number of threads are reduce to one. Then the value
+/// from hardware_memlimit_get() will be honored like in single-threaded mode.
 extern uint64_t hardware_memlimit_mtdec_get(void);
 
 /// Display the amount of RAM and memory usage limits and exit.