aboutsummaryrefslogtreecommitdiff
path: root/src/xz/coder.c
diff options
context:
space:
mode:
authorLasse Collin <lasse.collin@tukaani.org>2022-04-14 14:20:46 +0300
committerLasse Collin <lasse.collin@tukaani.org>2022-04-14 14:20:46 +0300
commitc77fe55ddb7752ed0fec46967c5ec9a72632ea0c (patch)
treee6365d3d05c78f08f7ac59a2b351e828d282a744 /src/xz/coder.c
parentxz: Make -T0 use multithreaded mode on single-core systems. (diff)
downloadxz-c77fe55ddb7752ed0fec46967c5ec9a72632ea0c.tar.xz
xz: Add a default soft memory usage limit for --threads=0.
This is a soft limit in sense that it only affects the number of threads. It never makes xz fail and it never makes xz change settings that would affect the compressed output. The idea is to make -T0 have more reasonable behavior when the system has very many cores or when a memory-hungry compression options are used. This also helps with 32-bit xz, preventing it from running out of address space. The downside of this commit is that now the number of threads might become too low compared to what the user expected. I hope this to be an acceptable compromise as the old behavior has been a source of well-argued complaints for a long time.
Diffstat (limited to '')
-rw-r--r--src/xz/coder.c28
1 files changed, 26 insertions, 2 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c
index a2699a9b..224c2d39 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -220,12 +220,16 @@ coder_set_compression_settings(void)
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
- const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
+ //
+ // If multithreaded .xz compression is done, this value will be
+ // replaced.
+ uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage = UINT64_MAX;
if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
# ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
+ memory_limit = hardware_memlimit_mtenc_get();
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
@@ -304,6 +308,27 @@ coder_set_compression_settings(void)
}
}
+ // If the memory usage limit is only a soft limit (automatic
+ // number of threads and no --memlimit-compress), the limit
+ // is only used to reduce the number of threads and once at
+ // just one thread, the limit is completely ignored. This
+ // way -T0 won't use insane amount of memory but at the same
+ // time the soft limit will never make xz fail and never make
+ // xz change settings that would affect the compressed output.
+ if (hardware_memlimit_mtenc_is_default()) {
+ message(V_WARNING, _("Reduced the number of threads "
+ "from %s to one. The automatic memory usage "
+ "limit of %s MiB is still being exceeded. "
+ "%s MiB of memory is required. "
+ "Continuing anyway."),
+ uint64_to_str(hardware_threads_get(), 0),
+ uint64_to_str(
+ round_up_to_mib(memory_limit), 1),
+ uint64_to_str(
+ round_up_to_mib(memory_usage), 2));
+ return;
+ }
+
// If --no-adjust was used, we cannot drop to single-threaded
// mode since it produces different compressed output.
//
@@ -321,7 +346,6 @@ coder_set_compression_settings(void)
message(V_WARNING, _("Switching to single-threaded mode "
"to not exceed the memory usage limit of %s MiB"),
uint64_to_str(round_up_to_mib(memory_limit), 0));
-
}
# endif