diff options
Diffstat (limited to 'src/liblzma/common')
| -rw-r--r-- | src/liblzma/common/alone_decoder.c | 47 | ||||
| -rw-r--r-- | src/liblzma/common/auto_decoder.c | 29 | ||||
| -rw-r--r-- | src/liblzma/common/block_util.c | 52 | ||||
| -rw-r--r-- | src/liblzma/common/common.c | 58 | ||||
| -rw-r--r-- | src/liblzma/common/common.h | 9 | ||||
| -rw-r--r-- | src/liblzma/common/easy.c | 33 | ||||
| -rw-r--r-- | src/liblzma/common/filter_common.c | 2 | ||||
| -rw-r--r-- | src/liblzma/common/index.c | 11 | ||||
| -rw-r--r-- | src/liblzma/common/index_decoder.c | 46 | ||||
| -rw-r--r-- | src/liblzma/common/stream_decoder.c | 47 |
10 files changed, 270 insertions, 64 deletions
diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c index 7ff29289..32d44311 100644 --- a/src/liblzma/common/alone_decoder.c +++ b/src/liblzma/common/alone_decoder.c @@ -42,6 +42,9 @@ struct lzma_coder_s { /// Memory usage limit uint64_t memlimit; + /// Amount of memory actually needed (only an estimate) + uint64_t memusage; + /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; @@ -117,21 +120,16 @@ alone_decode(lzma_coder *coder, } ++*in_pos; - break; + + // Calculate the memory usage so that it is ready + // for SEQ_CODER_INIT. + coder->memusage = lzma_lzma_decoder_memusage(&coder->options) + + LZMA_MEMUSAGE_BASE; + + // Fall through case SEQ_CODER_INIT: { - // FIXME It is unfair that this doesn't add a fixed amount - // like lzma_memusage_common() does. - const uint64_t memusage - = lzma_lzma_decoder_memusage(&coder->options); - - // Use LZMA_PROG_ERROR since LZMA_Alone decoder cannot be - // built without LZMA support. - // FIXME TODO Make the above comment true. - if (memusage == UINT64_MAX) - return LZMA_PROG_ERROR; - - if (memusage > coder->memlimit) + if (coder->memusage > coder->memlimit) return LZMA_MEMLIMIT_ERROR; lzma_filter_info filters[2] = { @@ -153,10 +151,9 @@ alone_decode(lzma_coder *coder, coder->uncompressed_size); coder->sequence = SEQ_CODE; + break; } - // Fall through - case SEQ_CODE: { return coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, @@ -180,12 +177,30 @@ alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) } +static lzma_ret +alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + if (new_memlimit != 0 && new_memlimit < coder->memusage) + return LZMA_MEMLIMIT_ERROR; + + *memusage = coder->memusage; + *old_memlimit = coder->memlimit; + coder->memlimit = new_memlimit; + + return LZMA_OK; +} + + extern lzma_ret lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, uint64_t memlimit) { lzma_next_coder_init(lzma_alone_decoder_init, next, allocator); + if (memlimit == 0) + return LZMA_PROG_ERROR; + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -193,6 +208,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->code = &alone_decode; next->end = &alone_decoder_end; + next->memconfig = &alone_decoder_memconfig; next->coder->next = LZMA_NEXT_CODER_INIT; } @@ -201,6 +217,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->options.dict_size = 0; next->coder->uncompressed_size = 0; next->coder->memlimit = memlimit; + next->coder->memusage = LZMA_MEMUSAGE_BASE; return LZMA_OK; } diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c index dd108324..2520dc17 100644 --- a/src/liblzma/common/auto_decoder.c +++ b/src/liblzma/common/auto_decoder.c @@ -125,11 +125,39 @@ auto_decoder_get_check(const lzma_coder *coder) static lzma_ret +auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + lzma_ret ret; + + if (coder->next.memconfig != NULL) { + ret = coder->next.memconfig(coder->next.coder, + memusage, old_memlimit, new_memlimit); + assert(*old_memlimit == coder->memlimit); + } else { + // No coder is configured yet. Use the base value as + // the current memory usage. + *memusage = LZMA_MEMUSAGE_BASE; + *old_memlimit = coder->memlimit; + ret = LZMA_OK; + } + + if (ret == LZMA_OK && new_memlimit != 0) + coder->memlimit = new_memlimit; + + return ret; +} + + +static lzma_ret auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(auto_decoder_init, next, allocator); + if (memlimit == 0) + return LZMA_PROG_ERROR; + if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; @@ -141,6 +169,7 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->code = &auto_decode; next->end = &auto_decoder_end; next->get_check = &auto_decoder_get_check; + next->memconfig = &auto_decoder_memconfig; next->coder->next = LZMA_NEXT_CODER_INIT; } diff --git a/src/liblzma/common/block_util.c b/src/liblzma/common/block_util.c index 66e1cad9..2fa45841 100644 --- a/src/liblzma/common/block_util.c +++ b/src/liblzma/common/block_util.c @@ -22,31 +22,35 @@ extern LZMA_API lzma_ret -lzma_block_compressed_size(lzma_block *options, lzma_vli total_size) +lzma_block_compressed_size(lzma_block *block, lzma_vli total_size) { - // Validate. - if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN - || options->header_size > LZMA_BLOCK_HEADER_SIZE_MAX - || (options->header_size & 3) - || (unsigned)(options->check) > LZMA_CHECK_ID_MAX - || (total_size & 3)) + // Validate everything but Uncompressed Size and filters. + if (lzma_block_unpadded_size(block) == 0) return LZMA_PROG_ERROR; - const uint32_t container_size = options->header_size - + lzma_check_size(options->check); + const uint32_t container_size = block->header_size + + lzma_check_size(block->check); // Validate that Compressed Size will be greater than zero. if (container_size <= total_size) return LZMA_DATA_ERROR; - options->compressed_size = total_size - container_size; + // Calculate what Compressed Size is supposed to be. + // If Compressed Size was present in Block Header, + // compare that the new value matches it. + const lzma_vli compressed_size = total_size - container_size; + if (block->compressed_size != LZMA_VLI_UNKNOWN + && block->compressed_size != compressed_size) + return LZMA_DATA_ERROR; + + block->compressed_size = compressed_size; return LZMA_OK; } extern LZMA_API lzma_vli -lzma_block_unpadded_size(const lzma_block *options) +lzma_block_unpadded_size(const lzma_block *block) { // Validate the values that we are interested in i.e. all but // Uncompressed Size and the filters. @@ -54,23 +58,23 @@ lzma_block_unpadded_size(const lzma_block *options) // NOTE: This function is used for validation too, so it is // essential that these checks are always done even if // Compressed Size is unknown. - if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN - || options->header_size > LZMA_BLOCK_HEADER_SIZE_MAX - || (options->header_size & 3) - || !lzma_vli_is_valid(options->compressed_size) - || options->compressed_size == 0 - || (unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + if (block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN + || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX + || (block->header_size & 3) + || !lzma_vli_is_valid(block->compressed_size) + || block->compressed_size == 0 + || (unsigned int)(block->check) > LZMA_CHECK_ID_MAX) return 0; // If Compressed Size is unknown, return that we cannot know // size of the Block either. - if (options->compressed_size == LZMA_VLI_UNKNOWN) + if (block->compressed_size == LZMA_VLI_UNKNOWN) return LZMA_VLI_UNKNOWN; // Calculate Unpadded Size and validate it. - const lzma_vli unpadded_size = options->compressed_size - + options->header_size - + lzma_check_size(options->check); + const lzma_vli unpadded_size = block->compressed_size + + block->header_size + + lzma_check_size(block->check); assert(unpadded_size >= UNPADDED_SIZE_MIN); if (unpadded_size > UNPADDED_SIZE_MAX) @@ -81,11 +85,11 @@ lzma_block_unpadded_size(const lzma_block *options) extern LZMA_API lzma_vli -lzma_block_total_size(const lzma_block *options) +lzma_block_total_size(const lzma_block *block) { - lzma_vli unpadded_size = lzma_block_unpadded_size(options); + lzma_vli unpadded_size = lzma_block_unpadded_size(block); - if (unpadded_size != 0 && unpadded_size != LZMA_VLI_UNKNOWN) + if (unpadded_size != LZMA_VLI_UNKNOWN) unpadded_size = vli_ceil4(unpadded_size); return unpadded_size; diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c index c5f5039d..81f783da 100644 --- a/src/liblzma/common/common.c +++ b/src/liblzma/common/common.c @@ -301,5 +301,63 @@ lzma_end(lzma_stream *strm) extern LZMA_API lzma_check lzma_get_check(const lzma_stream *strm) { + // Return LZMA_CHECK_NONE if we cannot know the check type. + // It's a bug in the application if this happens. + if (strm->internal->next.get_check == NULL) + return LZMA_CHECK_NONE; + return strm->internal->next.get_check(strm->internal->next.coder); } + + +extern LZMA_API uint64_t +lzma_memusage(const lzma_stream *strm) +{ + uint64_t memusage; + uint64_t old_memlimit; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL + || strm->internal->next.memconfig( + strm->internal->next.coder, + &memusage, &old_memlimit, 0) != LZMA_OK) + return 0; + + return memusage; +} + + +extern LZMA_API uint64_t +lzma_memlimit_get(const lzma_stream *strm) +{ + uint64_t old_memlimit; + uint64_t memusage; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL + || strm->internal->next.memconfig( + strm->internal->next.coder, + &memusage, &old_memlimit, 0) != LZMA_OK) + return 0; + + return old_memlimit; +} + + +extern LZMA_API lzma_ret +lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) +{ + // Dummy variables to simplify memconfig functions + uint64_t old_memlimit; + uint64_t memusage; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL) + return LZMA_PROG_ERROR; + + if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) + return LZMA_MEMLIMIT_ERROR; + + return strm->internal->next.memconfig(strm->internal->next.coder, + &memusage, &old_memlimit, new_memlimit); +} diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h index 0ee8574c..ef8d0cbf 100644 --- a/src/liblzma/common/common.h +++ b/src/liblzma/common/common.h @@ -46,6 +46,12 @@ #define LZMA_BUFFER_SIZE 4096 +/// Starting value for memory usage estimates. Instead of calculating size +/// of _every_ structure and taking into accont malloc() overhead etc. we +/// add a base size to all memory usage estimates. It's not very accurate +/// but should be easily good enough. +#define LZMA_MEMUSAGE_BASE (UINT64_C(1) << 15) + /// Start of internal Filter ID space. These IDs must never be used /// in Streams. #define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62) @@ -134,7 +140,8 @@ struct lzma_next_coder_s { /// Pointer to function to get and/or change the memory usage limit. /// If memlimit == 0, the limit is not changed. - uint64_t (*memconfig)(lzma_coder *coder, uint64_t memlimit); + lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit); }; diff --git a/src/liblzma/common/easy.c b/src/liblzma/common/easy.c index d5e19525..769253f4 100644 --- a/src/liblzma/common/easy.c +++ b/src/liblzma/common/easy.c @@ -33,8 +33,11 @@ struct lzma_coder_s { static bool -easy_set_filters(lzma_coder *coder, uint32_t level) +easy_set_filters(lzma_coder *coder, uint32_t level, uint32_t flags) { + // FIXME + (void)flags; + bool error = false; if (level == 0) { @@ -43,7 +46,7 @@ easy_set_filters(lzma_coder *coder, uint32_t level) #ifdef HAVE_ENCODER_LZMA2 } else if (level <= 9) { - error = lzma_lzma_preset(&coder->opt_lzma, level - 1); + error = lzma_lzma_preset(&coder->opt_lzma, level); coder->filters[0].id = LZMA_FILTER_LZMA2; coder->filters[0].options = &coder->opt_lzma; coder->filters[1].id = LZMA_VLI_UNKNOWN; @@ -80,7 +83,7 @@ easy_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_easy_level level) + uint32_t level, uint32_t flags, lzma_check check) { lzma_next_coder_init(easy_encoder_init, next, allocator); @@ -95,18 +98,19 @@ easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->stream_encoder = LZMA_NEXT_CODER_INIT; } - if (easy_set_filters(next->coder, level)) + if (easy_set_filters(next->coder, level, flags)) return LZMA_OPTIONS_ERROR; return lzma_stream_encoder_init(&next->coder->stream_encoder, - allocator, next->coder->filters, LZMA_CHECK_CRC32); + allocator, next->coder->filters, check); } extern LZMA_API lzma_ret -lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) +lzma_easy_encoder(lzma_stream *strm, + uint32_t level, uint32_t flags, lzma_check check) { - lzma_next_strm_init(easy_encoder_init, strm, level); + lzma_next_strm_init(easy_encoder_init, strm, level, flags, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; @@ -118,11 +122,22 @@ lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) extern LZMA_API uint64_t -lzma_easy_memory_usage(lzma_easy_level level) +lzma_easy_encoder_memusage(uint32_t level, uint32_t flags) { lzma_coder coder; - if (easy_set_filters(&coder, level)) + if (easy_set_filters(&coder, level, flags)) return UINT32_MAX; return lzma_memusage_encoder(coder.filters); } + + +extern LZMA_API uint64_t +lzma_easy_decoder_memusage(uint32_t level, uint32_t flags) +{ + lzma_coder coder; + if (easy_set_filters(&coder, level, flags)) + return UINT32_MAX; + + return lzma_memusage_decoder(coder.filters); +} diff --git a/src/liblzma/common/filter_common.c b/src/liblzma/common/filter_common.c index 13a7cdd2..fe3c03a2 100644 --- a/src/liblzma/common/filter_common.c +++ b/src/liblzma/common/filter_common.c @@ -264,5 +264,5 @@ lzma_memusage_coder(lzma_filter_find coder_find, // Add some fixed amount of extra. It's to compensate memory usage // of Stream, Block etc. coders, malloc() overhead, stack etc. - return total + (1U << 15); + return total + LZMA_MEMUSAGE_BASE; } diff --git a/src/liblzma/common/index.c b/src/liblzma/common/index.c index 1fe65650..d7025fff 100644 --- a/src/liblzma/common/index.c +++ b/src/liblzma/common/index.c @@ -114,6 +114,17 @@ struct lzma_index_s { }; +extern LZMA_API lzma_vli +lzma_index_memusage(lzma_vli count) +{ + if (count > LZMA_VLI_MAX) + return UINT64_MAX; + + return sizeof(lzma_index) + (count + INDEX_GROUP_SIZE - 1) + / INDEX_GROUP_SIZE * sizeof(lzma_index_group); +} + + static void free_index_list(lzma_index *i, lzma_allocator *allocator) { diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c index 5faac161..e29e0b0d 100644 --- a/src/liblzma/common/index_decoder.c +++ b/src/liblzma/common/index_decoder.c @@ -25,6 +25,7 @@ struct lzma_coder_s { enum { SEQ_INDICATOR, SEQ_COUNT, + SEQ_MEMUSAGE, SEQ_UNPADDED, SEQ_UNCOMPRESSED, SEQ_PADDING_INIT, @@ -32,6 +33,9 @@ struct lzma_coder_s { SEQ_CRC32, } sequence; + /// Memory usage limit + uint64_t memlimit; + /// Target Index lzma_index *index; @@ -82,18 +86,27 @@ index_decode(lzma_coder *coder, lzma_allocator *allocator, coder->sequence = SEQ_COUNT; break; - case SEQ_COUNT: { + case SEQ_COUNT: ret = lzma_vli_decode(&coder->count, &coder->pos, in, in_pos, in_size); if (ret != LZMA_STREAM_END) goto out; - ret = LZMA_OK; coder->pos = 0; + coder->sequence = SEQ_MEMUSAGE; + + // Fall through + + case SEQ_MEMUSAGE: + if (lzma_index_memusage(coder->count) > coder->memlimit) { + ret = LZMA_MEMLIMIT_ERROR; + goto out; + } + + ret = LZMA_OK; coder->sequence = coder->count == 0 ? SEQ_PADDING_INIT : SEQ_UNPADDED; break; - } case SEQ_UNPADDED: case SEQ_UNCOMPRESSED: { @@ -197,12 +210,28 @@ index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret +index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + *memusage = lzma_index_memusage(coder->count); + + if (new_memlimit != 0 && new_memlimit < *memusage) + return LZMA_MEMLIMIT_ERROR; + + *old_memlimit = coder->memlimit; + coder->memlimit = new_memlimit; + + return LZMA_OK; +} + + +static lzma_ret index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_index **i) + lzma_index **i, uint64_t memlimit) { lzma_next_coder_init(index_decoder_init, next, allocator); - if (i == NULL) + if (i == NULL || memlimit == 0) return LZMA_PROG_ERROR; if (next->coder == NULL) { @@ -212,6 +241,7 @@ index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->code = &index_decode; next->end = &index_decoder_end; + next->memconfig = &index_decoder_memconfig; next->coder->index = NULL; } else { lzma_index_end(next->coder->index, allocator); @@ -224,7 +254,9 @@ index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the rest. next->coder->sequence = SEQ_INDICATOR; + next->coder->memlimit = memlimit; next->coder->index = *i; + next->coder->count = 0; // Needs to be initialized due to _memconfig(). next->coder->pos = 0; next->coder->crc32 = 0; @@ -233,9 +265,9 @@ index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API lzma_ret -lzma_index_decoder(lzma_stream *strm, lzma_index **i) +lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init(index_decoder_init, strm, i); + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c index 9be47893..956a08f3 100644 --- a/src/liblzma/common/stream_decoder.c +++ b/src/liblzma/common/stream_decoder.c @@ -50,6 +50,9 @@ struct lzma_coder_s { /// Memory usage limit uint64_t memlimit; + /// Amount of memory actually needed (only an estimate) + uint64_t memusage; + /// If true, LZMA_NO_CHECK is returned if the Stream has /// no integrity check. bool tell_no_check; @@ -204,14 +207,24 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, if (memusage == UINT64_MAX) { // One or more unknown Filter IDs. ret = LZMA_OPTIONS_ERROR; - } else if (memusage > coder->memlimit) { - // The chain would need too much memory. - ret = LZMA_MEMLIMIT_ERROR; } else { - // Memory usage is OK. Initialize the Block decoder. - ret = lzma_block_decoder_init( - &coder->block_decoder, - allocator, &coder->block_options); + // Now we can set coder->memusage since we know that + // the filter chain is valid. We don't want + // lzma_memusage() to return UINT64_MAX in case of + // invalid filter chain. + coder->memusage = memusage; + + if (memusage > coder->memlimit) { + // The chain would need too much memory. + ret = LZMA_MEMLIMIT_ERROR; + } else { + // Memory usage is OK. + // Initialize the Block decoder. + ret = lzma_block_decoder_init( + &coder->block_decoder, + allocator, + &coder->block_options); + } } // Free the allocated filter options since they are needed @@ -374,12 +387,30 @@ stream_decoder_get_check(const lzma_coder *coder) } +static lzma_ret +stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + if (new_memlimit != 0 && new_memlimit < coder->memusage) + return LZMA_MEMLIMIT_ERROR; + + *memusage = coder->memusage; + *old_memlimit = coder->memlimit; + coder->memlimit = new_memlimit; + + return LZMA_OK; +} + + extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(lzma_stream_decoder_init, next, allocator); + if (memlimit == 0) + return LZMA_PROG_ERROR; + if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; @@ -391,12 +422,14 @@ lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->code = &stream_decode; next->end = &stream_decoder_end; next->get_check = &stream_decoder_get_check; + next->memconfig = &stream_decoder_memconfig; next->coder->block_decoder = LZMA_NEXT_CODER_INIT; next->coder->index_hash = NULL; } next->coder->memlimit = memlimit; + next->coder->memusage = LZMA_MEMUSAGE_BASE; next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; next->coder->tell_unsupported_check = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; |