diff options
| -rw-r--r-- | src/xz/coder.c | 28 | ||||
| -rw-r--r-- | src/xz/hardware.c | 38 | ||||
| -rw-r--r-- | src/xz/hardware.h | 27 |
3 files changed, 82 insertions, 11 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c index a2699a9b..224c2d39 100644 --- a/src/xz/coder.c +++ b/src/xz/coder.c @@ -220,12 +220,16 @@ coder_set_compression_settings(void) // Get the memory usage. Note that if --format=raw was used, // we can be decompressing. - const uint64_t memory_limit = hardware_memlimit_get(opt_mode); + // + // If multithreaded .xz compression is done, this value will be + // replaced. + uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage = UINT64_MAX; if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS # ifdef MYTHREAD_ENABLED if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) { + memory_limit = hardware_memlimit_mtenc_get(); mt_options.threads = hardware_threads_get(); mt_options.block_size = opt_block_size; mt_options.check = check; @@ -304,6 +308,27 @@ coder_set_compression_settings(void) } } + // If the memory usage limit is only a soft limit (automatic + // number of threads and no --memlimit-compress), the limit + // is only used to reduce the number of threads and once at + // just one thread, the limit is completely ignored. This + // way -T0 won't use insane amount of memory but at the same + // time the soft limit will never make xz fail and never make + // xz change settings that would affect the compressed output. + if (hardware_memlimit_mtenc_is_default()) { + message(V_WARNING, _("Reduced the number of threads " + "from %s to one. The automatic memory usage " + "limit of %s MiB is still being exceeded. " + "%s MiB of memory is required. " + "Continuing anyway."), + uint64_to_str(hardware_threads_get(), 0), + uint64_to_str( + round_up_to_mib(memory_limit), 1), + uint64_to_str( + round_up_to_mib(memory_usage), 2)); + return; + } + // If --no-adjust was used, we cannot drop to single-threaded // mode since it produces different compressed output. // @@ -321,7 +346,6 @@ coder_set_compression_settings(void) message(V_WARNING, _("Switching to single-threaded mode " "to not exceed the memory usage limit of %s MiB"), uint64_to_str(round_up_to_mib(memory_limit), 0)); - } # endif diff --git a/src/xz/hardware.c b/src/xz/hardware.c index 18eee7ec..2cc3f4f2 100644 --- a/src/xz/hardware.c +++ b/src/xz/hardware.c @@ -29,6 +29,13 @@ static uint64_t memlimit_decompress = 0; /// Default memory usage for multithreaded modes: /// +/// - Default value for --memlimit-compress when automatic number of threads +/// is used. However, if the limit wouldn't allow even one thread then +/// the limit is ignored in coder.c and one thread will be used anyway. +/// This mess is a compromise: we wish to prevent -T0 from using too +/// many threads but we also don't want xz to give an error due to +/// a memlimit that the user didn't explicitly set. +/// /// - Default value for --memlimit-mt-decompress /// /// This value is caluclated in hardware_init() and cannot be changed later. @@ -151,15 +158,12 @@ hardware_memlimit_set(uint64_t new_memlimit, extern uint64_t hardware_memlimit_get(enum operation_mode mode) { - // Zero is a special value that indicates the default. Currently - // the default simply disables the limit. Once there is threading - // support, this might be a little more complex, because there will - // probably be a special case where a user asks for "optimal" number - // of threads instead of a specific number (this might even become - // the default mode). Each thread may use a significant amount of - // memory. When there are no memory usage limits set, we need some - // default soft limit for calculating the "optimal" number of - // threads. + // 0 is a special value that indicates the default. + // It disables the limit in single-threaded mode. + // + // NOTE: For multithreaded decompression, this is the hard limit + // (memlimit_stop). hardware_memlimit_mtdec_get() gives the + // soft limit (memlimit_threaded). const uint64_t memlimit = mode == MODE_COMPRESS ? memlimit_compress : memlimit_decompress; return memlimit != 0 ? memlimit : UINT64_MAX; @@ -167,6 +171,22 @@ hardware_memlimit_get(enum operation_mode mode) extern uint64_t +hardware_memlimit_mtenc_get(void) +{ + return memlimit_compress == 0 && threads_are_automatic + ? memlimit_mt_default + : hardware_memlimit_get(MODE_COMPRESS); +} + + +extern bool +hardware_memlimit_mtenc_is_default(void) +{ + return memlimit_compress == 0 && threads_are_automatic; +} + + +extern uint64_t hardware_memlimit_mtdec_get(void) { uint64_t m = memlimit_mtdec != 0 diff --git a/src/xz/hardware.h b/src/xz/hardware.h index 1a5a7a67..2cd6aa23 100644 --- a/src/xz/hardware.h +++ b/src/xz/hardware.h @@ -37,9 +37,36 @@ extern void hardware_memlimit_set(uint64_t new_memlimit, bool is_percentage); /// Get the current memory usage limit for compression or decompression. +/// This is a hard limit that will not be exceeded. This is obeyed in +/// both single-threaded and multithreaded modes. extern uint64_t hardware_memlimit_get(enum operation_mode mode); +/// This returns a system-specific default value if all of the following +/// conditions are true: +/// +/// - An automatic number of threads was requested (--threads=0). +/// +/// - --memlimit-compress wasn't used or it was reset to the default +/// value by setting it to 0. +/// +/// Otherwise this is identical to hardware_memlimit_get(MODE_COMPRESS). +/// +/// The idea is to keep automatic thread count reasonable so that too +/// high memory usage is avoided and, with 32-bit xz, running out of +/// address space is avoided. +extern uint64_t hardware_memlimit_mtenc_get(void); + +/// Returns true if the value returned by hardware_memlimit_mtenc_get() is +/// a system-specific default value. coder.c uses this to ignore the default +/// memlimit in case it's too small even for a single thread in multithreaded +/// mode. This way the default limit will never make xz fail or affect the +/// compressed output; it will only make xz reduce the number of threads. +extern bool hardware_memlimit_mtenc_is_default(void); + /// Get the current memory usage limit for multithreaded decompression. +/// This is only used to reduce the number of threads. This limit can be +/// exceeded if the number of threads are reduce to one. Then the value +/// from hardware_memlimit_get() will be honored like in single-threaded mode. extern uint64_t hardware_memlimit_mtdec_get(void); /// Display the amount of RAM and memory usage limits and exit. |