aboutsummaryrefslogtreecommitdiff
path: root/src/xz
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/xz/coder.c28
-rw-r--r--src/xz/hardware.c38
-rw-r--r--src/xz/hardware.h27
3 files changed, 82 insertions, 11 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c
index a2699a9b..224c2d39 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -220,12 +220,16 @@ coder_set_compression_settings(void)
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
- const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
+ //
+ // If multithreaded .xz compression is done, this value will be
+ // replaced.
+ uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage = UINT64_MAX;
if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
# ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
+ memory_limit = hardware_memlimit_mtenc_get();
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
@@ -304,6 +308,27 @@ coder_set_compression_settings(void)
}
}
+ // If the memory usage limit is only a soft limit (automatic
+ // number of threads and no --memlimit-compress), the limit
+ // is only used to reduce the number of threads and once at
+ // just one thread, the limit is completely ignored. This
+ // way -T0 won't use insane amount of memory but at the same
+ // time the soft limit will never make xz fail and never make
+ // xz change settings that would affect the compressed output.
+ if (hardware_memlimit_mtenc_is_default()) {
+ message(V_WARNING, _("Reduced the number of threads "
+ "from %s to one. The automatic memory usage "
+ "limit of %s MiB is still being exceeded. "
+ "%s MiB of memory is required. "
+ "Continuing anyway."),
+ uint64_to_str(hardware_threads_get(), 0),
+ uint64_to_str(
+ round_up_to_mib(memory_limit), 1),
+ uint64_to_str(
+ round_up_to_mib(memory_usage), 2));
+ return;
+ }
+
// If --no-adjust was used, we cannot drop to single-threaded
// mode since it produces different compressed output.
//
@@ -321,7 +346,6 @@ coder_set_compression_settings(void)
message(V_WARNING, _("Switching to single-threaded mode "
"to not exceed the memory usage limit of %s MiB"),
uint64_to_str(round_up_to_mib(memory_limit), 0));
-
}
# endif
diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index 18eee7ec..2cc3f4f2 100644
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -29,6 +29,13 @@ static uint64_t memlimit_decompress = 0;
/// Default memory usage for multithreaded modes:
///
+/// - Default value for --memlimit-compress when automatic number of threads
+/// is used. However, if the limit wouldn't allow even one thread then
+/// the limit is ignored in coder.c and one thread will be used anyway.
+/// This mess is a compromise: we wish to prevent -T0 from using too
+/// many threads but we also don't want xz to give an error due to
+/// a memlimit that the user didn't explicitly set.
+///
/// - Default value for --memlimit-mt-decompress
///
/// This value is caluclated in hardware_init() and cannot be changed later.
@@ -151,15 +158,12 @@ hardware_memlimit_set(uint64_t new_memlimit,
extern uint64_t
hardware_memlimit_get(enum operation_mode mode)
{
- // Zero is a special value that indicates the default. Currently
- // the default simply disables the limit. Once there is threading
- // support, this might be a little more complex, because there will
- // probably be a special case where a user asks for "optimal" number
- // of threads instead of a specific number (this might even become
- // the default mode). Each thread may use a significant amount of
- // memory. When there are no memory usage limits set, we need some
- // default soft limit for calculating the "optimal" number of
- // threads.
+ // 0 is a special value that indicates the default.
+ // It disables the limit in single-threaded mode.
+ //
+ // NOTE: For multithreaded decompression, this is the hard limit
+ // (memlimit_stop). hardware_memlimit_mtdec_get() gives the
+ // soft limit (memlimit_threaded).
const uint64_t memlimit = mode == MODE_COMPRESS
? memlimit_compress : memlimit_decompress;
return memlimit != 0 ? memlimit : UINT64_MAX;
@@ -167,6 +171,22 @@ hardware_memlimit_get(enum operation_mode mode)
extern uint64_t
+hardware_memlimit_mtenc_get(void)
+{
+ return memlimit_compress == 0 && threads_are_automatic
+ ? memlimit_mt_default
+ : hardware_memlimit_get(MODE_COMPRESS);
+}
+
+
+extern bool
+hardware_memlimit_mtenc_is_default(void)
+{
+ return memlimit_compress == 0 && threads_are_automatic;
+}
+
+
+extern uint64_t
hardware_memlimit_mtdec_get(void)
{
uint64_t m = memlimit_mtdec != 0
diff --git a/src/xz/hardware.h b/src/xz/hardware.h
index 1a5a7a67..2cd6aa23 100644
--- a/src/xz/hardware.h
+++ b/src/xz/hardware.h
@@ -37,9 +37,36 @@ extern void hardware_memlimit_set(uint64_t new_memlimit,
bool is_percentage);
/// Get the current memory usage limit for compression or decompression.
+/// This is a hard limit that will not be exceeded. This is obeyed in
+/// both single-threaded and multithreaded modes.
extern uint64_t hardware_memlimit_get(enum operation_mode mode);
+/// This returns a system-specific default value if all of the following
+/// conditions are true:
+///
+/// - An automatic number of threads was requested (--threads=0).
+///
+/// - --memlimit-compress wasn't used or it was reset to the default
+/// value by setting it to 0.
+///
+/// Otherwise this is identical to hardware_memlimit_get(MODE_COMPRESS).
+///
+/// The idea is to keep automatic thread count reasonable so that too
+/// high memory usage is avoided and, with 32-bit xz, running out of
+/// address space is avoided.
+extern uint64_t hardware_memlimit_mtenc_get(void);
+
+/// Returns true if the value returned by hardware_memlimit_mtenc_get() is
+/// a system-specific default value. coder.c uses this to ignore the default
+/// memlimit in case it's too small even for a single thread in multithreaded
+/// mode. This way the default limit will never make xz fail or affect the
+/// compressed output; it will only make xz reduce the number of threads.
+extern bool hardware_memlimit_mtenc_is_default(void);
+
/// Get the current memory usage limit for multithreaded decompression.
+/// This is only used to reduce the number of threads. This limit can be
+/// exceeded if the number of threads are reduce to one. Then the value
+/// from hardware_memlimit_get() will be honored like in single-threaded mode.
extern uint64_t hardware_memlimit_mtdec_get(void);
/// Display the amount of RAM and memory usage limits and exit.