diff options
Diffstat (limited to '')
| -rw-r--r-- | src/liblzma/common/outqueue.h | 138 |
1 files changed, 97 insertions, 41 deletions
diff --git a/src/liblzma/common/outqueue.h b/src/liblzma/common/outqueue.h index 079634de..355e0ced 100644 --- a/src/liblzma/common/outqueue.h +++ b/src/liblzma/common/outqueue.h @@ -14,16 +14,27 @@ /// Output buffer for a single thread -typedef struct { - /// Pointer to the output buffer of lzma_outq.buf_size_max bytes - uint8_t *buf; - - /// Amount of data written to buf - size_t size; - - /// Additional size information - lzma_vli unpadded_size; - lzma_vli uncompressed_size; +typedef struct lzma_outbuf_s lzma_outbuf; +struct lzma_outbuf_s { + /// Pointer to the next buffer. This is used for the cached buffers. + /// The worker thread must not modify this. + lzma_outbuf *next; + + /// This initialized by lzma_outq_get_buf() and + /// is used by lzma_outq_enable_partial_output(). + /// The worker thread must not modify this. + void *worker; + + /// Amount of memory allocated for buf[]. + /// The worker thread must not modify this. + size_t allocated; + + /// Writing position in the worker thread or, in other words, the + /// amount of finished data written to buf[] which can be copied out + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + size_t pos; /// True when no more data will be written into this buffer. /// @@ -31,32 +42,44 @@ typedef struct { /// to this variable needs a mutex. bool finished; -} lzma_outbuf; + /// Additional size information. lzma_outq_read() may read these + /// when "finished" is true. + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + /// Buffer of "allocated" bytes + uint8_t buf[]; +}; -typedef struct { - /// Array of buffers that are used cyclically. - lzma_outbuf *bufs; - /// Memory allocated for all the buffers - uint8_t *bufs_mem; +typedef struct { + /// Linked list of buffers in use. The next output byte will be + /// read from the head and buffers for the next thread will be + /// appended to the tail. tail->next is always NULL. + lzma_outbuf *head; + lzma_outbuf *tail; - /// Amount of buffer space available in each buffer - size_t buf_size_max; + /// Number of bytes read from head->buf[] in lzma_outq_read() + size_t read_pos; - /// Number of buffers allocated - uint32_t bufs_allocated; + /// Linked list of allocated buffers that aren't currently used. + /// This way buffers of similar size can be reused and don't + /// need to be reallocated every time. For simplicity, all + /// cached buffers in the list have the same allocated size. + lzma_outbuf *cache; - /// Position in the bufs array. The next buffer to be taken - /// into use is bufs[bufs_pos]. - uint32_t bufs_pos; + /// Total amount of memory allocated for buffers + uint64_t memusage; - /// Number of buffers in use - uint32_t bufs_used; + /// Number of buffers in use in the head...tail list. If and only if + /// this is zero, the pointers head and tail above are NULL. + uint32_t bufs_in_use; - /// Position in the buffer in lzma_outq_read() - size_t read_pos; + /// Number of buffers allocated (in use + cached) + uint32_t bufs_allocated; + /// Maximum allowed number of allocated buffers + uint32_t bufs_limit; } lzma_outq; @@ -76,32 +99,50 @@ extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads); /// function knows that there are no previous /// allocations to free. /// \param allocator Pointer to allocator or NULL -/// \param buf_size_max Maximum amount of data that a single buffer -/// in the queue may need to store. /// \param threads Number of buffers that may be in use /// concurrently. Note that more than this number -/// of buffers will actually get allocated to +/// of buffers may actually get allocated to /// improve performance when buffers finish -/// out of order. +/// out of order. The actual maximum number of +/// allocated buffers is derived from the number +/// of threads. /// /// \return - LZMA_OK /// - LZMA_MEM_ERROR /// -extern lzma_ret lzma_outq_init( - lzma_outq *outq, const lzma_allocator *allocator, - uint64_t buf_size_max, uint32_t threads); +extern lzma_ret lzma_outq_init(lzma_outq *outq, + const lzma_allocator *allocator, uint32_t threads); /// \brief Free the memory associated with the output queue extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); +/// \brief Free all cached buffers that consume memory but aren't in use +extern void lzma_outq_clear_cache( + lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Preallocate a new buffer into cache +/// +/// Splitting the buffer allocation into a separate function makes it +/// possible to ensure that way lzma_outq_get_buf() cannot fail. +/// If the preallocated buffer isn't actually used (for example, some +/// other error occurs), the caller has to do nothing as the buffer will +/// be used later or cleared from the cache when not needed. +/// +/// \return LZMA_OK on success, LZMA_MEM_ERROR if allocation fails +/// +extern lzma_ret lzma_outq_prealloc_buf( + lzma_outq *outq, const lzma_allocator *allocator, size_t size); + + /// \brief Get a new buffer /// -/// lzma_outq_has_buf() must be used to check that there is a buffer +/// lzma_outq_prealloc_buf() must be used to ensure that there is a buffer /// available before calling lzma_outq_get_buf(). /// -extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq, void *worker); /// \brief Test if there is data ready to be read @@ -126,17 +167,32 @@ extern bool lzma_outq_is_readable(const lzma_outq *outq); /// \return - LZMA: All OK. Either no data was available or the buffer /// being read didn't become empty yet. /// - LZMA_STREAM_END: The buffer being read was finished. -/// *unpadded_size and *uncompressed_size were set. +/// *unpadded_size and *uncompressed_size were set if they +/// were not NULL. /// -/// \note This reads lzma_outbuf.finished variables and thus call -/// to this function needs to be protected with a mutex. +/// \note This reads lzma_outbuf.finished and .pos variables and thus +/// calls to this function need to be protected with a mutex. /// extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + const lzma_allocator *restrict allocator, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_vli *restrict unpadded_size, lzma_vli *restrict uncompressed_size); +/// \brief Enable partial output from a worker thread +/// +/// If the buffer at the head of the output queue isn't finished, +/// this will call enable_partial_output on the worker associated with +/// that output buffer. +/// +/// \note This reads a lzma_outbuf.finished variable and thus +/// calls to this function need to be protected with a mutex. +/// +extern void lzma_outq_enable_partial_output(lzma_outq *outq, + void (*enable_partial_output)(void *worker)); + + /// \brief Test if there is at least one buffer free /// /// This must be used before getting a new buffer with lzma_outq_get_buf(). @@ -144,7 +200,7 @@ extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, static inline bool lzma_outq_has_buf(const lzma_outq *outq) { - return outq->bufs_used < outq->bufs_allocated; + return outq->bufs_in_use < outq->bufs_limit; } @@ -152,5 +208,5 @@ lzma_outq_has_buf(const lzma_outq *outq) static inline bool lzma_outq_is_empty(const lzma_outq *outq) { - return outq->bufs_used == 0; + return outq->bufs_in_use == 0; } |