aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorJia Tan <jiat0218@gmail.com>2023-05-13 20:11:13 +0800
committerJia Tan <jiat0218@gmail.com>2023-07-17 23:34:55 +0800
commit183819bfd9efac8c184d9bf123325719b7eee30f (patch)
tree47ef3ab42bd839f9662395aab41b78058f825416 /src
parentxz: Validate --flush-timeout for all specified filter chains. (diff)
downloadxz-183819bfd9efac8c184d9bf123325719b7eee30f.tar.xz
xz: Set the Block size for mt encoding correctly.
When opt_block_size is not used, the Block size for mt encoder is derived from the minimum of the largest Block specified by --block-list and the recommended Block size on all filter chains calculated by lzma_mt_block_size(). This avoids using unnecessary memory and ensures that all Blocks are large enough for the most memory needy filter chain.
Diffstat (limited to 'src')
-rw-r--r--src/xz/coder.c68
1 files changed, 67 insertions, 1 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c
index 7cb286b2..6a99d8ce 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -50,6 +50,13 @@ static uint32_t filters_init_mask = 1;
/// Track the memory usage for all filter chains (default or --filtersX).
/// The memory usage may need to be scaled down depending on the memory limit.
static uint64_t filter_memusages[ARRAY_SIZE(filters)];
+
+# ifdef MYTHREAD_ENABLED
+/// Represents the largest Block size specified with --block-list. This
+/// is needed to help reduce the Block size in the multithreaded encoder
+/// so memory is not wasted.
+static uint64_t max_block_list_size = 0;
+# endif
#endif
/// Input and output buffers
@@ -290,6 +297,18 @@ filters_memusage_max(const lzma_mt *mt, bool encode)
return max_memusage;
}
+
+
+# ifdef MYTHREAD_ENABLED
+static void
+filter_chain_error(const uint32_t index, const char *msg)
+{
+ if (index == 0)
+ message_fatal(_("Error in the filter chain: %s"), msg);
+ else
+ message_fatal(_("Error in --filters%d: %s"), index, msg);
+}
+# endif
#endif
@@ -306,6 +325,11 @@ coder_set_compression_settings(void)
for (uint32_t i = 0; opt_block_list[i].size != 0; i++) {
validate_block_list_filter(
opt_block_list[i].filters_index);
+
+# ifdef MYTHREAD_ENABLED
+ if (opt_block_list[i].size > max_block_list_size)
+ max_block_list_size = opt_block_list[i].size;
+# endif
}
#endif
// The default check type is CRC64, but fallback to CRC32
@@ -420,7 +444,49 @@ coder_set_compression_settings(void)
if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
memory_limit = hardware_memlimit_mtenc_get();
mt_options.threads = hardware_threads_get();
- mt_options.block_size = opt_block_size;
+
+ uint64_t block_size = opt_block_size;
+ // If opt_block_size is not set, find the maximum
+ // recommended Block size based on the filter chains
+ if (block_size == 0) {
+ for (uint32_t i = 0; i < ARRAY_SIZE(filters);
+ i++) {
+ if (!(filters_init_mask & (1 << i)))
+ continue;
+
+ uint64_t size = lzma_mt_block_size(
+ filters[i]);
+
+ // If this returns an error, then one
+ // of the filter chains in use is
+ // invalid, so there is no point in
+ // progressing further.
+ if (size == UINT64_MAX)
+ filter_chain_error(i,
+ message_strm(
+ LZMA_OPTIONS_ERROR));
+
+ if (size > block_size)
+ block_size = size;
+ }
+
+ // If the largest block size specified
+ // with --block-list is less than the
+ // recommended Block size, then it is a waste
+ // of RAM to use a larger Block size. It may
+ // even allow more threads to be used in some
+ // situations. If the special 0 Block size is
+ // used (encode all remaining data in 1 Block)
+ // then max_block_list_size will be set to
+ // UINT64_MAX, so the recommended Block size
+ // will always be used in this case.
+ if (max_block_list_size > 0
+ && max_block_list_size
+ < block_size)
+ block_size = max_block_list_size;
+ }
+
+ mt_options.block_size = block_size;
mt_options.check = check;
memory_usage = filters_memusage_max(