aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLasse Collin <lasse.collin@tukaani.org>2022-04-14 12:38:00 +0300
committerLasse Collin <lasse.collin@tukaani.org>2022-04-14 12:38:00 +0300
commit898faa97287a756231c663a3ed5165672b417207 (patch)
treef24f5d465d41ec36c06a793a114fd000acd830a4
parentxz: Add --memlimit-mt-decompress along with a default limit value. (diff)
downloadxz-898faa97287a756231c663a3ed5165672b417207.tar.xz
xz: Changes to --memlimit-compress and --no-adjust.
In single-threaded mode, --memlimit-compress can make xz scale down the LZMA2 dictionary size to meet the memory usage limit. This obviously affects the compressed output. However, if xz was in threaded mode, --memlimit-compress could make xz reduce the number of threads but it wouldn't make xz switch from multithreaded mode to single-threaded mode or scale down the LZMA2 dictionary size. This seemed illogical and there was even a "FIXME?" about it. Now --memlimit-compress can make xz switch to single-threaded mode if one thread in multithreaded mode uses too much memory. If memory usage is still too high, then the LZMA2 dictionary size can be scaled down too. The option --no-adjust was also changed so that it no longer prevents xz from scaling down the number of threads as that doesn't affect compressed output (only performance). After this commit --no-adjust only prevents adjustments that affect compressed output, that is, with --no-adjust xz won't switch from multithreaded mode to single-threaded mode and won't scale down the LZMA2 dictionary size. The man page wasn't updated yet.
-rw-r--r--src/xz/coder.c63
1 files changed, 43 insertions, 20 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c
index b5f7c392..268359a4 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -269,10 +269,9 @@ coder_set_compression_settings(void)
if (memory_usage <= memory_limit)
return;
- // If --no-adjust was used or we didn't find LZMA1 or
- // LZMA2 as the last filter, give an error immediately.
- // --format=raw implies --no-adjust.
- if (!opt_auto_adjust || opt_format == FORMAT_RAW)
+ // With --format=raw settings are never adjusted to meet
+ // the memory usage limit.
+ if (opt_format == FORMAT_RAW)
memlimit_too_small(memory_usage);
assert(opt_mode == MODE_COMPRESS);
@@ -282,34 +281,58 @@ coder_set_compression_settings(void)
if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
// Try to reduce the number of threads before
// adjusting the compression settings down.
- do {
- // FIXME? The real single-threaded mode has
- // lower memory usage, but it's not comparable
- // because it doesn't write the size info
- // into Block Headers.
- if (--mt_options.threads == 0)
- memlimit_too_small(memory_usage);
-
+ while (mt_options.threads > 1) {
+ // Reduce the number of threads by one and check
+ // the memory usage.
+ --mt_options.threads;
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage == UINT64_MAX)
message_bug();
- } while (memory_usage > memory_limit);
+ if (memory_usage <= memory_limit) {
+ // The memory usage is now low enough.
+ message(V_WARNING, _("Reduced the number of "
+ "threads from %s to %s to not exceed "
+ "the memory usage limit of %s MiB"),
+ uint64_to_str(
+ hardware_threads_get(), 0),
+ uint64_to_str(mt_options.threads, 1),
+ uint64_to_str(round_up_to_mib(
+ memory_limit), 2));
+ return;
+ }
+ }
+
+ // If --no-adjust was used, we cannot drop to single-threaded
+ // mode since it produces different compressed output.
+ //
+ // NOTE: In xz 5.2.x, --no-adjust also prevented reducing
+ // the number of threads. This changed in 5.3.3alpha.
+ if (!opt_auto_adjust)
+ memlimit_too_small(memory_usage);
+
+ // Switch to single-threaded mode. It uses
+ // less memory than using one thread in
+ // the multithreaded mode but the output
+ // is also different.
+ hardware_threads_set(1);
+ memory_usage = lzma_raw_encoder_memusage(filters);
+ message(V_WARNING, _("Switching to single-threaded mode "
+ "to not exceed the memory usage limit of %s MiB"),
+ uint64_to_str(round_up_to_mib(memory_limit), 0));
- message(V_WARNING, _("Adjusted the number of threads "
- "from %s to %s to not exceed "
- "the memory usage limit of %s MiB"),
- uint64_to_str(hardware_threads_get(), 0),
- uint64_to_str(mt_options.threads, 1),
- uint64_to_str(round_up_to_mib(
- memory_limit), 2));
}
# endif
if (memory_usage <= memory_limit)
return;
+ // Don't adjust LZMA2 or LZMA1 dictionary size if --no-adjust
+ // was specified as that would change the compressed output.
+ if (!opt_auto_adjust)
+ memlimit_too_small(memory_usage);
+
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;