diff options
| author | Lasse Collin <lasse.collin@tukaani.org> | 2008-08-28 22:53:15 +0300 |
|---|---|---|
| committer | Lasse Collin <lasse.collin@tukaani.org> | 2008-08-28 22:53:15 +0300 |
| commit | 3b34851de1eaf358cf9268922fa0eeed8278d680 (patch) | |
| tree | 7bab212af647541df64227a8d350d17a2e789f6b /src | |
| parent | Fix test_filter_flags to match the new restriction of lc+lp. (diff) | |
| download | xz-3b34851de1eaf358cf9268922fa0eeed8278d680.tar.xz | |
Sort of garbage collection commit. :-| Many things are still
broken. API has changed a lot and it will still change a
little more here and there. The command line tool doesn't
have all the required changes to reflect the API changes, so
it's easy to get "internal error" or trigger assertions.
Diffstat (limited to 'src')
147 files changed, 8453 insertions, 7082 deletions
diff --git a/src/common/integer.h b/src/common/integer.h index 136a0f8d..a6e43be2 100644 --- a/src/common/integer.h +++ b/src/common/integer.h @@ -15,7 +15,7 @@ #define LZMA_INTEGER_H // I'm aware of AC_CHECK_ALIGNED_ACCESS_REQUIRED from Autoconf archive, but -// it's not useful for us. We don't care if unaligned access is supported, +// it's not useful here. We don't care if unaligned access is supported, // we care if it is fast. Some systems can emulate unaligned access in // software, which is horribly slow; we want to use byte-by-byte access on // such systems but the Autoconf test would detect such a system as @@ -32,13 +32,13 @@ // that also allow unaligned access. Inline assembler could be OK for that. #ifdef WORDS_BIGENDIAN # include "bswap.h" -# define integer_convert_16(n) bswap_16(n) -# define integer_convert_32(n) bswap_32(n) -# define integer_convert_64(n) bswap_64(n) +# define integer_le_16(n) bswap_16(n) +# define integer_le_32(n) bswap_32(n) +# define integer_le_64(n) bswap_64(n) #else -# define integer_convert_16(n) (n) -# define integer_convert_32(n) (n) -# define integer_convert_64(n) (n) +# define integer_le_16(n) (n) +# define integer_le_32(n) (n) +# define integer_le_64(n) (n) #endif @@ -46,7 +46,7 @@ static inline uint16_t integer_read_16(const uint8_t buf[static 2]) { uint16_t ret = *(const uint16_t *)(buf); - return integer_convert_16(ret); + return integer_le_16(ret); } @@ -54,7 +54,7 @@ static inline uint32_t integer_read_32(const uint8_t buf[static 4]) { uint32_t ret = *(const uint32_t *)(buf); - return integer_convert_32(ret); + return integer_le_32(ret); } @@ -63,7 +63,7 @@ static inline uint64_t integer_read_64(const uint8_t buf[static 8]) { uint64_t ret = *(const uint64_t *)(buf); - return integer_convert_64(ret); + return integer_le_64(ret); } */ @@ -71,14 +71,14 @@ integer_read_64(const uint8_t buf[static 8]) static inline void integer_write_16(uint8_t buf[static 2], uint16_t num) { - *(uint16_t *)(buf) = integer_convert_16(num); + *(uint16_t *)(buf) = integer_le_16(num); } static inline void integer_write_32(uint8_t buf[static 4], uint32_t num) { - *(uint32_t *)(buf) = integer_convert_32(num); + *(uint32_t *)(buf) = integer_le_32(num); } @@ -86,7 +86,7 @@ integer_write_32(uint8_t buf[static 4], uint32_t num) static inline void integer_write_64(uint8_t buf[static 8], uint64_t num) { - *(uint64_t *)(buf) = integer_convert_64(num); + *(uint64_t *)(buf) = integer_le_64(num); } */ diff --git a/src/common/sysdefs.h b/src/common/sysdefs.h index 2c7fb6ff..7f935f67 100644 --- a/src/common/sysdefs.h +++ b/src/common/sysdefs.h @@ -31,12 +31,21 @@ # include <config.h> #endif -#include <sys/types.h> +// size_t and NULL +#include <stddef.h> #ifdef HAVE_INTTYPES_H # include <inttypes.h> #endif +// C99 says that inttypes.h always includes stdint.h, but some systems +// don't do that, and require including stdint.h separately. +#ifdef HAVE_STDINT_H +# include <stdint.h> +#endif + +// Some pre-C99 systems have SIZE_MAX in limits.h instead of stdint.h. The +// limits are also used to figure out some macros missing from pre-C99 systems. #ifdef HAVE_LIMITS_H # include <limits.h> #endif @@ -44,7 +53,12 @@ // Be more compatible with systems that have non-conforming inttypes.h. // We assume that int is 32-bit and that long is either 32-bit or 64-bit. // Full Autoconf test could be more correct, but this should work well enough. +// Note that this duplicates some code from lzma.h, but this is better since +// we can work without inttypes.h thanks to Autoconf tests. #ifndef UINT32_C +# if UINT_MAX != 4294967295U +# error UINT32_C is not defined and unsiged int is not 32-bit. +# endif # define UINT32_C(n) n ## U #endif #ifndef UINT32_MAX @@ -56,7 +70,8 @@ #ifndef PRIX32 # define PRIX32 "X" #endif -#if SIZEOF_UNSIGNED_LONG == 4 + +#if ULONG_MAX == 4294967295UL # ifndef UINT64_C # define UINT64_C(n) n ## ULL # endif @@ -80,16 +95,33 @@ #ifndef UINT64_MAX # define UINT64_MAX UINT64_C(18446744073709551615) #endif + +// The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX # if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# else +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX +# else +# error sizeof(size_t) is not 32-bit or 64-bit # endif #endif +#if SIZE_MAX != UINT32_MAX && SIZE_MAX != UINT64_MAX +# error sizeof(size_t) is not 32-bit or 64-bit +#endif #include <stdlib.h> +// Pre-C99 systems lack stdbool.h. All the code in LZMA Utils must be written +// so that it works with fake bool type, for example: +// +// bool foo = (flags & 0x100) != 0; +// bool bar = !!(flags & 0x100); +// +// This works with the real C99 bool but breaks with fake bool: +// +// bool baz = (flags & 0x100); +// #ifdef HAVE_STDBOOL_H # include <stdbool.h> #else @@ -108,11 +140,13 @@ typedef unsigned char _Bool; # ifdef NDEBUG # define assert(x) # else - // TODO: Pretty bad assert() macro. + // TODO: Pretty bad assert macro. # define assert(x) (!(x) && abort()) # endif #endif +// string.h should be enough but let's include strings.h and memory.h too if +// they exists, since that shouldn't do any harm, but may improve portability. #ifdef HAVE_STRING_H # include <string.h> #endif diff --git a/src/liblzma/Makefile.am b/src/liblzma/Makefile.am index 78a072f4..a234bfd5 100644 --- a/src/liblzma/Makefile.am +++ b/src/liblzma/Makefile.am @@ -22,11 +22,15 @@ liblzma_la_LIBADD = \ common/libcommon.la \ check/libcheck.la +if COND_FILTER_LZ +SUBDIRS += lz +liblzma_la_LIBADD += lz/liblz.la +endif + if COND_FILTER_LZMA -SUBDIRS += lz lzma rangecoder +SUBDIRS += lzma rangecoder liblzma_la_LIBADD += \ - lz/liblz.la \ - lzma/liblzma4.la \ + lzma/liblzma2.la \ rangecoder/librangecoder.la endif @@ -35,7 +39,12 @@ SUBDIRS += subblock liblzma_la_LIBADD += subblock/libsubblock.la endif -if COND_MAIN_SIMPLE +if COND_FILTER_DELTA +SUBDIRS += delta +liblzma_la_LIBADD += delta/libdelta.la +endif + +if COND_FILTER_SIMPLE SUBDIRS += simple liblzma_la_LIBADD += simple/libsimple.la endif diff --git a/src/liblzma/api/Makefile.am b/src/liblzma/api/Makefile.am index 194f85db..86ce5bda 100644 --- a/src/liblzma/api/Makefile.am +++ b/src/liblzma/api/Makefile.am @@ -15,22 +15,18 @@ nobase_include_HEADERS = \ lzma.h \ lzma/alignment.h \ - lzma/alone.h \ - lzma/auto.h \ lzma/base.h \ lzma/block.h \ lzma/check.h \ + lzma/container.h \ lzma/delta.h \ - lzma/easy.h \ lzma/filter.h \ lzma/index.h \ lzma/index_hash.h \ lzma/init.h \ lzma/lzma.h \ lzma/memlimit.h \ - lzma/raw.h \ lzma/simple.h \ - lzma/stream.h \ lzma/stream_flags.h \ lzma/subblock.h \ lzma/version.h \ diff --git a/src/liblzma/api/lzma.h b/src/liblzma/api/lzma.h index 9dec904f..0f109eb3 100644 --- a/src/liblzma/api/lzma.h +++ b/src/liblzma/api/lzma.h @@ -17,36 +17,103 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. + */ + +#ifndef LZMA_H +#define LZMA_H + +/***************************** + * Required standard headers * + *****************************/ + +/** + * liblzma API headers need some standard types and macros. To allow + * including lzma.h without requiring the application to include other + * headers first, lzma.h includes the required standard headers unless + * they already seem to be included. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Here's what types and macros are needed and from which headers: + * - stddef.h: size_t, NULL + * - stdint.h: uint8_t, uint32_t, uint64_t, UINT32_C(n), uint64_C(n), + * UINT32_MAX, UINT64_MAX * - * Before #including this file, you must make the following types available: - * - size_t - * - uint8_t - * - int32_t - * - uint32_t - * - int64_t - * - uint64_t + * However, inttypes.h is a little more portable than stdint.h, although + * inttypes.h declares some unneeded things compared to plain stdint.h. * - * Before #including this file, you must make the following macros available: - * - UINT32_C(n) - * - UINT64_C(n) - * - UINT32_MAX - * - UINT64_MAX + * The hacks below aren't perfect, specifically they assume that inttypes.h + * exists and that it typedefs at least uint8_t, uint32_t, and uint64_t, + * and that unsigned int is 32-bit. If your application already takes care + * of setting up all the types properly (for example by using gnulib's + * stdint.h or inttypes.h), feel free to define LZMA_MANUAL_HEADERS before + * including lzma.h. * - * Easiest way to achieve the above is to #include sys/types.h and inttypes.h - * before #including lzma.h. However, some pre-C99 libc headers don't provide - * all the required types in inttypes.h (that file may even be missing). - * Portable applications need to provide these types themselves. This way - * liblzma API can use the standard types instead of defining its own - * (e.g. lzma_uint32). + * Some could argue that liblzma API should provide all the required types, + * for example lzma_uint64, LZMA_UINT64_C(n), and LZMA_UINT64_MAX. This was + * seen unnecessary mess, since most systems already provide all the necessary + * types and macros in the standard headers. * - * Note that the API still has lzma_bool, because using stdbool.h would + * Note that liblzma API still has lzma_bool, because using stdbool.h would * break C89 and C++ programs on many systems. */ -#ifndef LZMA_H -#define LZMA_H +/* stddef.h even in C++ so that we get size_t in global namespace. */ +#include <stddef.h> + +#if !defined(UINT32_C) || !defined(UINT64_C) \ + || !defined(UINT32_MAX) || !defined(UINT64_MAX) +# ifdef __cplusplus + /* + * C99 sections 7.18.2 and 7.18.4 specify that in C++ + * implementations define the limit and constant macros only + * if specifically requested. Note that if you want the + * format macros too, you need to define __STDC_FORMAT_MACROS + * before including lzma.h, since re-including inttypes.h + * with __STDC_FORMAT_MACROS defined doesn't necessarily work. + */ +# ifndef __STDC_LIMIT_MACROS +# define __STDC_LIMIT_MACROS 1 +# endif +# ifndef __STDC_CONSTANT_MACROS +# define __STDC_CONSTANT_MACROS 1 +# endif +# endif + +# include <inttypes.h> + + /* + * Some old systems have only the typedefs in inttypes.h, and lack + * all the macros. For those systems, we need a few more hacks. + * We assume that unsigned int is 32-bit and unsigned long is either + * 32-bit or 64-bit. If these hacks aren't enough, the application + * has to use setup the types manually before including lzma.h. + */ +# ifndef UINT32_C +# define UINT32_C(n) n # U +# endif + +# ifndef UINT64_C + /* Get ULONG_MAX. */ +# ifndef __cplusplus +# include <limits.h> +# else +# include <climits> +# endif +# if ULONG_MAX == 4294967295UL +# define UINT64_C(n) n ## ULL +# else +# define UINT64_C(n) n ## UL +# endif +# endif + +# ifndef UINT32_MAX +# define UINT32_MAX (UINT32_C(4294967295)) +# endif + +# ifndef UINT64_MAX +# define UINT64_MAX (UINT64_C(18446744073709551615)) +# endif +#endif + /****************** * GCC extensions * @@ -57,20 +124,50 @@ * break anything if these are sometimes enabled and sometimes not, only * affects warnings and optimizations. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#if __GNUC__ >= 3 # ifndef lzma_attribute # define lzma_attribute(attr) __attribute__(attr) # endif + # ifndef lzma_restrict # define lzma_restrict __restrict__ # endif + + /* warn_unused_result was added in GCC 3.4. */ +# ifndef lzma_attr_warn_unused_result +# if __GNUC__ == 3 && __GNUC_MINOR__ < 4 +# define lzma_attr_warn_unused_result +# endif +# endif + #else # ifndef lzma_attribute # define lzma_attribute(attr) # endif + # ifndef lzma_restrict -# define lzma_restrict +# if __STDC_VERSION__ >= 199901L +# define lzma_restrict restrict +# else +# define lzma_restrict +# endif # endif + +# define lzma_attr_warn_unused_result +#endif + + +#ifndef lzma_attr_pure +# define lzma_attr_pure lzma_attribute((__pure__)) +#endif + +#ifndef lzma_attr_const +# define lzma_attr_const lzma_attribute((__const__)) +#endif + +#ifndef lzma_attr_warn_unused_result +# define lzma_attr_warn_unused_result \ + lzma_attribute((__warn_unused_result__)) #endif @@ -89,36 +186,30 @@ extern "C" { #define LZMA_H_INTERNAL 1 /* Basic features */ +#include "lzma/version.h" #include "lzma/init.h" #include "lzma/base.h" #include "lzma/vli.h" -#include "lzma/filter.h" #include "lzma/check.h" /* Filters */ +#include "lzma/filter.h" #include "lzma/subblock.h" #include "lzma/simple.h" #include "lzma/delta.h" #include "lzma/lzma.h" /* Container formats */ -#include "lzma/block.h" -#include "lzma/stream.h" -#include "lzma/alone.h" -#include "lzma/raw.h" -#include "lzma/auto.h" -#include "lzma/easy.h" +#include "lzma/container.h" /* Advanced features */ +#include "lzma/alignment.h" /* FIXME */ +#include "lzma/block.h" #include "lzma/index.h" #include "lzma/index_hash.h" -#include "lzma/alignment.h" #include "lzma/stream_flags.h" #include "lzma/memlimit.h" -/* Version number */ -#include "lzma/version.h" - /* * All subheaders included. Undefine LZMA_H_INTERNAL to prevent applications * re-including the subheaders. diff --git a/src/liblzma/api/lzma/alignment.h b/src/liblzma/api/lzma/alignment.h index 6672656c..008af690 100644 --- a/src/liblzma/api/lzma/alignment.h +++ b/src/liblzma/api/lzma/alignment.h @@ -27,7 +27,7 @@ * FIXME desc */ extern uint32_t lzma_alignment_input( - const lzma_options_filter *filters, uint32_t guess); + const lzma_filter *filters, uint32_t guess); /** @@ -36,7 +36,7 @@ extern uint32_t lzma_alignment_input( * Knowing the alignment of the output data is useful e.g. in the Block * encoder which tries to align the Compressed Data field optimally. * - * \param filters Pointer to lzma_options_filter array, whose last + * \param filters Pointer to lzma_filter array, whose last * member must have .id = LZMA_VLI_VALUE_UNKNOWN. * \param guess The value to return if the alignment of the output * is the same as the alignment of the input data. @@ -57,4 +57,4 @@ extern uint32_t lzma_alignment_input( * options), UINT32_MAX is returned. */ extern uint32_t lzma_alignment_output( - const lzma_options_filter *filters, uint32_t guess); + const lzma_filter *filters, uint32_t guess); diff --git a/src/liblzma/api/lzma/alone.h b/src/liblzma/api/lzma/alone.h deleted file mode 100644 index 72299773..00000000 --- a/src/liblzma/api/lzma/alone.h +++ /dev/null @@ -1,52 +0,0 @@ -/** - * \file lzma/alone.h - * \brief Handling of the legacy LZMA_Alone format - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use <lzma.h> instead. -#endif - - -/** - * \brief Initializes LZMA_Alone encoder - * - * LZMA_Alone files have the suffix .lzma like the .lzma Stream files. - * LZMA_Alone format supports only one filter, the LZMA filter. There is - * no support for integrity checks like CRC32. - * - * Use this format if and only if you need to create files readable by - * legacy LZMA tools. - * - * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_alone_encoder( - lzma_stream *strm, const lzma_options_lzma *options); - - -/** - * \brief Initializes decoder for LZMA_Alone file - * - * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - */ -extern lzma_ret lzma_alone_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/auto.h b/src/liblzma/api/lzma/auto.h deleted file mode 100644 index fd5bf7d2..00000000 --- a/src/liblzma/api/lzma/auto.h +++ /dev/null @@ -1,36 +0,0 @@ -/** - * \file lzma/auto.h - * \brief Decoder with automatic file format detection - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use <lzma.h> instead. -#endif - - -/** - * \brief Decode .lzma Streams and LZMA_Alone files with autodetection - * - * Autodetects between the .lzma Stream and LZMA_Alone formats, and - * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once - * the type of the file has been detected. - * - * \param strm Pointer to propertily prepared lzma_stream - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR: Cannot allocate memory. - */ -extern lzma_ret lzma_auto_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h index b0dfed95..cb614176 100644 --- a/src/liblzma/api/lzma/base.h +++ b/src/liblzma/api/lzma/base.h @@ -134,7 +134,7 @@ typedef enum { * \brief Unknown file format */ - LZMA_MEMLIMIT_ERROR = -9 + LZMA_MEMLIMIT_ERROR = -9, /** * \brief Memory usage limit was reached * @@ -143,6 +143,9 @@ typedef enum { * the memory usage limit has to be increased. See functions * lzma_memlimit_get() and lzma_memlimit_set(). */ + + LZMA_NO_CHECK = -10, + LZMA_SEE_CHECK = -11 } lzma_ret; @@ -229,11 +232,6 @@ typedef struct { /** * \brief Pointer to custom memory allocation function * - * Set this to point to your custom memory allocation function. - * It can be useful for example if you want to limit how much - * memory liblzma is allowed to use: for this, you may use - * a pointer to lzma_memory_alloc(). - * * If you don't want a custom allocator, but still want * custom free(), set this to NULL and liblzma will use * the standard malloc(). @@ -250,16 +248,19 @@ typedef struct { * size nmemb * size, or NULL if allocation fails * for some reason. When allocation fails, functions * of liblzma return LZMA_MEM_ERROR. + * + * For performance reasons, the allocator should not waste time + * zeroing the allocated buffers. This is not only about speed, but + * also memory usage, since the operating system kernel doesn't + * necessarily allocate the requested memory until it is actually + * used. With small input files liblzma may actually need only a + * fraction of the memory that it requested for allocation. */ void *(*alloc)(void *opaque, size_t nmemb, size_t size); /** * \brief Pointer to custom memory freeing function * - * Set this to point to your custom memory freeing function. - * If lzma_memory_alloc() is used as allocator, this should - * be set to lzma_memory_free(). - * * If you don't want a custom freeing function, but still * want a custom allocator, set this to NULL and liblzma * will use the standard free(). @@ -279,10 +280,6 @@ typedef struct { * and lzma_allocator.free(). This intended to ease implementing * custom memory allocation functions for use with liblzma. * - * When using lzma_memory_alloc() and lzma_memory_free(), opaque - * must point to lzma_memory_limiter structure allocated and - * initialized with lzma_memory_limiter_create(). - * * If you don't need this, you should set it to NULL. */ void *opaque; @@ -347,6 +344,17 @@ typedef struct { /** Internal state is not visible to outsiders. */ lzma_internal *internal; + /** + * Reserved space to allow possible future extensions without + * breaking the ABI. Excluding the initialization of this structure, + * you should not touch these, because the names of these variables + * may change. + */ + void *reserved_ptr1; + void *reserved_ptr2; + uint64_t reserved_int1; + uint64_t reserved_int2; + } lzma_stream; @@ -358,22 +366,18 @@ typedef struct { * has been allocated yet: * * lzma_stream strm = LZMA_STREAM_INIT; - */ -#define LZMA_STREAM_INIT { NULL, 0, 0, NULL, 0, 0, NULL, NULL } - - -/** - * \brief Initialization for lzma_stream * - * This is like LZMA_STREAM_INIT, but this can be used when the lzma_stream - * has already been allocated: + * If you need to initialize a dynamically allocatedlzma_stream, you can use + * memset(strm_pointer, 0, sizeof(lzma_stream)). Strictly speaking, this + * violates the C standard since NULL may have different internal + * representation than zero, but it should be portable enough in practice. + * Anyway, for maximum portability, you could use this: * - * lzma_stream *strm = malloc(sizeof(lzma_stream)); - * if (strm == NULL) - * return LZMA_MEM_ERROR; - * *strm = LZMA_STREAM_INIT_VAR; + * lzma_stream tmp = LZMA_STREAM_INIT; + * *strm = tmp; */ -extern const lzma_stream LZMA_STREAM_INIT_VAR; +#define LZMA_STREAM_INIT \ + { NULL, 0, 0, NULL, 0, 0, NULL, NULL, NULL, NULL, 0, 0 } /** @@ -409,7 +413,8 @@ extern const lzma_stream LZMA_STREAM_INIT_VAR; * - LZMA_PROG_ERROR: Invalid arguments or the internal state * of the coder is corrupt. */ -extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action); +extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action) + lzma_attr_warn_unused_result; /** diff --git a/src/liblzma/api/lzma/block.h b/src/liblzma/api/lzma/block.h index a8941165..45045815 100644 --- a/src/liblzma/api/lzma/block.h +++ b/src/liblzma/api/lzma/block.h @@ -36,12 +36,13 @@ typedef struct { * \brief Size of the Block Header * * Read by: + * - lzma_block_header_encode() + * - lzma_block_header_decode() * - lzma_block_encoder() * - lzma_block_decoder() * * Written by: * - lzma_block_header_size() - * - lzma_block_header_decode() */ uint32_t header_size; # define LZMA_BLOCK_HEADER_SIZE_MIN 8 @@ -54,10 +55,12 @@ typedef struct { * Header, thus its value must be provided also when decoding. * * Read by: + * - lzma_block_header_encode() + * - lzma_block_header_decode() * - lzma_block_encoder() * - lzma_block_decoder() */ - lzma_check_type check; + lzma_check check; /** * \brief Size of the Compressed Data in bytes @@ -134,17 +137,17 @@ typedef struct { * have LZMA_BLOCK_FILTERS_MAX + 1 members or the Block * Header decoder will overflow the buffer. */ - lzma_options_filter *filters; + lzma_filter *filters; # define LZMA_BLOCK_FILTERS_MAX 4 -} lzma_options_block; +} lzma_block; /** * \brief Decodes the Block Header Size field * * To decode Block Header using lzma_block_header_decode(), the size of the - * Block Header has to be known and stored into lzma_options_block.header_size. + * Block Header has to be known and stored into lzma_block.header_size. * The size can be calculated from the first byte of a Block using this macro. * Note that if the first byte is 0x00, it indicates beginning of Index; use * this macro only when the byte is not 0x00. @@ -164,7 +167,8 @@ typedef struct { * may return LZMA_OK even if lzma_block_header_encode() or * lzma_block_encoder() would fail. */ -extern lzma_ret lzma_block_header_size(lzma_options_block *options); +extern lzma_ret lzma_block_header_size(lzma_block *options) + lzma_attr_warn_unused_result; /** @@ -183,7 +187,8 @@ extern lzma_ret lzma_block_header_size(lzma_options_block *options); * - LZMA_PROG_ERROR */ extern lzma_ret lzma_block_header_encode( - const lzma_options_block *options, uint8_t *out); + const lzma_block *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -203,8 +208,9 @@ extern lzma_ret lzma_block_header_encode( * - LZMA_HEADER_ERROR: Invalid or unsupported options. * - LZMA_PROG_ERROR */ -extern lzma_ret lzma_block_header_decode(lzma_options_block *options, - lzma_allocator *allocator, const uint8_t *in); +extern lzma_ret lzma_block_header_decode(lzma_block *options, + lzma_allocator *allocator, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -227,7 +233,8 @@ extern lzma_ret lzma_block_header_decode(lzma_options_block *options, * options->header_size between 8 and 1024 inclusive. */ extern lzma_ret lzma_block_total_size_set( - lzma_options_block *options, lzma_vli total_size); + lzma_block *options, lzma_vli total_size) + lzma_attr_warn_unused_result; /** @@ -238,7 +245,8 @@ extern lzma_ret lzma_block_total_size_set( * * \return Total Size on success, or zero on error. */ -extern lzma_vli lzma_block_total_size_get(const lzma_options_block *options); +extern lzma_vli lzma_block_total_size_get(const lzma_block *options) + lzma_attr_pure; /** @@ -259,8 +267,8 @@ extern lzma_vli lzma_block_total_size_get(const lzma_options_block *options); * * lzma_code() can return FIXME */ -extern lzma_ret lzma_block_encoder( - lzma_stream *strm, lzma_options_block *options); +extern lzma_ret lzma_block_encoder(lzma_stream *strm, lzma_block *options) + lzma_attr_warn_unused_result; /** @@ -273,5 +281,5 @@ extern lzma_ret lzma_block_encoder( * - LZMA_PROG_ERROR * - LZMA_MEM_ERROR */ -extern lzma_ret lzma_block_decoder( - lzma_stream *strm, lzma_options_block *options); +extern lzma_ret lzma_block_decoder(lzma_stream *strm, lzma_block *options) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h index dcba8269..18394a86 100644 --- a/src/liblzma/api/lzma/check.h +++ b/src/liblzma/api/lzma/check.h @@ -56,7 +56,7 @@ typedef enum { * * Size of the Check field: 32 bytes */ -} lzma_check_type; +} lzma_check; /** @@ -74,31 +74,34 @@ typedef enum { /** - * \brief Check IDs supported by this liblzma build - * - * If lzma_available_checks[n] is true, the Check ID n is supported by this - * liblzma build. You can assume that LZMA_CHECK_NONE and LZMA_CHECK_CRC32 - * are always available. + * \brief Maximum size of a Check field */ -extern const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1]; +#define LZMA_CHECK_SIZE_MAX 64 /** - * \brief Size of the Check field with different Check IDs + * \brief Test if the given Check ID is supported * - * Although not all Check IDs have a check algorithm associated, the size of - * every Check is already frozen. This array contains the size (in bytes) of - * the Check field with specified Check ID. The values are taken from the - * section 2.1.1.2 of the .lzma file format specification: - * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 } + * Returns true if the given Check ID is supported by this liblzma build. + * Otherwise false is returned. It is safe to call this with a value that + * is not in the range [0, 15]; in that case the return value is always false. */ -extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1]; +extern lzma_bool lzma_check_is_supported(lzma_check check) + lzma_attr_const; /** - * \brief Maximum size of a Check field + * \brief Get the size of the Check field with given Check ID + * + * Although not all Check IDs have a check algorithm associated, the size of + * every Check is already frozen. This function returns the size (in bytes) of + * the Check field with the specified Check ID. The values are taken from the + * section 2.1.1.2 of the .lzma file format specification: + * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 } + * + * If the argument is not in the range [0, 15], UINT32_MAX is returned. */ -#define LZMA_CHECK_SIZE_MAX 64 +extern uint32_t lzma_check_size(lzma_check check) lzma_attr_const; /** @@ -115,7 +118,8 @@ extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1]; * \return Updated CRC value, which can be passed to this function * again to continue CRC calculation. */ -extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); +extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) + lzma_attr_pure; /** @@ -125,7 +129,8 @@ extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); * * This function is used similarly to lzma_crc32(). See its documentation. */ -extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); +extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) + lzma_attr_pure; /* diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h new file mode 100644 index 00000000..27014856 --- /dev/null +++ b/src/liblzma/api/lzma/container.h @@ -0,0 +1,252 @@ +/** + * \file lzma/FIXME.h + * \brief File formats + * + * \author Copyright (C) 1999-2008 Igor Pavlov + * \author Copyright (C) 2007-2008 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/************ + * Encoding * + ************/ + +/** + * \brief Compression level names for lzma_easy_* functions + * + * At the moment, all the compression levels support LZMA_SYNC_FLUSH. + * In future there may be levels that don't support LZMA_SYNC_FLUSH. + * However, the LZMA_SYNC_FLUSH support won't be removed from the + * existing compression levels. + * + * \note If liblzma is built without encoder support, or with some + * filters disabled, some of the compression levels may be + * unsupported. In that case, the initialization functions + * will return LZMA_HEADER_ERROR. + */ +typedef enum { + LZMA_EASY_COPY = 0, + /**< + * No compression; the data is just wrapped into .lzma + * container. + */ + + LZMA_EASY_LZMA2_1 = 1, + /**< + * LZMA2 filter with fast compression (fast in terms of LZMA2). + * If you are interested in the exact options used, see + * lzma_preset_lzma[0]. Note that the exact options may + * change between liblzma versions. + * + * At the moment, the command line tool uses these settings + * when `lzma -1' is used. In future, the command line tool + * may default to some more complex way to determine the + * settings used e.g. the type of files being compressed. + * + * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1] + * and so on. + */ + + LZMA_EASY_LZMA_2 = 2, + LZMA_EASY_LZMA_3 = 3, + LZMA_EASY_LZMA_4 = 4, + LZMA_EASY_LZMA_5 = 5, + LZMA_EASY_LZMA_6 = 6, + LZMA_EASY_LZMA_7 = 7, + LZMA_EASY_LZMA_8 = 8, + LZMA_EASY_LZMA_9 = 9, +} lzma_easy_level; + + +/** + * \brief Default compression level + * + * Data Blocks contain the actual compressed data. It's not straightforward + * to recommend a default level, because in some cases keeping the resource + * usage relatively low is more important that getting the maximum + * compression ratio. + */ +#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA2_7 + + +/** + * \brief Calculates rough memory requirements of a compression level + * + * This function is a wrapper for lzma_memory_usage(), which is declared + * in filter.h. + * + * \return Approximate memory usage of the encoder with the given + * compression level in mebibytes (value * 1024 * 1024 bytes). + * On error (e.g. compression level is not supported), + * UINT32_MAX is returned. + */ +extern uint32_t lzma_easy_memory_usage(lzma_easy_level level) + lzma_attr_pure; + + +/** + * \brief Initializes .lzma Stream encoder + * + * This function is intended for those who just want to use the basic features + * if liblzma (that is, most developers out there). Lots of assumptions are + * made, which are correct or at least good enough for most situations. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param level Compression level to use. This selects a set of + * compression settings from a list of compression + * presets. + * + * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to + * encode your data. + * - LZMA_MEM_ERROR: Memory allocation failed. All memory + * previously allocated for *strm is now freed. + * - LZMA_HEADER_ERROR: The given compression level is not + * supported by this build of liblzma. + * + * If initialization succeeds, use lzma_code() to do the actual encoding. + * Valid values for `action' (the second argument of lzma_code()) are + * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, + * there may be compression levels that don't support LZMA_SYNC_FLUSH. + */ +extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes .lzma Stream encoder + * + * \param strm Pointer to properly prepared lzma_stream + * \param filters Array of filters. This must be terminated with + * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must + * be 1-4 filters, but there are restrictions on how + * multiple filters can be combined. FIXME Tell where + * to find more information. + * \param check Type of the integrity check to calculate from + * uncompressed data. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_stream_encoder(lzma_stream *strm, + const lzma_filter *filters, lzma_check check) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes LZMA_Alone (deprecated file format) encoder + * + * LZMA_Alone files have the suffix .lzma like the .lzma Stream files. + * LZMA_Alone format supports only one filter, the LZMA filter. There is + * no support for integrity checks like CRC32. + * + * Use this format if and only if you need to create files readable by + * legacy LZMA tools such as LZMA Utils 4.32.x. + * + * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_alone_encoder( + lzma_stream *strm, const lzma_options_lzma *options) + lzma_attr_warn_unused_result; + + +/************ + * Decoding * + ************/ + +/** + * This flag makes lzma_code() return LZMA_NO_CHECK if the input stream + * being decoded has no integrity check. Note that when used with + * lzma_auto_decoder(), all LZMA_Alone files will cause trigger LZMA_NO_CHECK + * if LZMA_WARN_NO_CHECK is used. + */ +#define LZMA_WARN_NO_CHECK UINT32_C(0x01) + + +/** + * This flag makes lzma_code() return LZMA_UNSUPPORTED_CHECK if the input + * stream has an integrity check, but the type of the integrity check is not + * supported by this liblzma version or build. Such files can still be + * decoded, but the integrity check cannot be verified. + */ +#define LZMA_WARN_UNSUPPORTED_CHECK UINT32_C(0x02) + + +/** + * This flag makes lzma_code() return LZMA_READ_CHECK as soon as the type + * of the integrity check is known. The type can then be read with + * lzma_check_get(). + */ +#define LZMA_TELL_CHECK UINT32_C(0x04) + + +/** + * This flag makes lzma_code() decode concatenated .lzma files. + * FIXME Explain the changed API. + */ +#define LZMA_CONCATENATED UINT32_C(0x08) + + +/** + * \brief Initializes decoder for .lzma Stream + * + * \param strm Pointer to propertily prepared lzma_stream + * \param memlimit Rough memory usage limit as bytes + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_stream_decoder( + lzma_stream *strm, uint64_t memlimit, uint32_t flags) + lzma_attr_warn_unused_result; + + +/** + * \brief Decode .lzma Streams and LZMA_Alone files with autodetection + * + * Autodetects between the .lzma Stream and LZMA_Alone formats, and + * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once + * the type of the file has been detected. + * + * \param strm Pointer to propertily prepared lzma_stream + * \param memlimit Rough memory usage limit as bytes + * \param flags Bitwise-or of flags, or zero for no flags. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_auto_decoder( + lzma_stream *strm, uint64_t memlimit, uint32_t flags) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes decoder for LZMA_Alone file + * + * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. FIXME + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + */ +extern lzma_ret lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/delta.h b/src/liblzma/api/lzma/delta.h index 58afec18..740de97c 100644 --- a/src/liblzma/api/lzma/delta.h +++ b/src/liblzma/api/lzma/delta.h @@ -24,9 +24,21 @@ /** * \brief Filter ID * - * Filter ID of the Delta filter. This is used as lzma_options_filter.id. + * Filter ID of the Delta filter. This is used as lzma_filter.id. */ -#define LZMA_FILTER_DELTA LZMA_VLI_C(0x20) +#define LZMA_FILTER_DELTA LZMA_VLI_C(0x03) + + +/** + * \brief Type of the delta calculation + * + * Currently only byte-wise delta is supported. Other possible types could + * be, for example, delta of 16/32/64-bit little/big endian integers, but + * these are not currently planned since byte-wise delta is almost as good. + */ +typedef enum { + LZMA_DELTA_TYPE_BYTE +} lzma_delta_type; /** @@ -35,8 +47,14 @@ * These options are needed by both encoder and decoder. */ typedef struct { + /** For now, this must always be LZMA_DELTA_TYPE_BYTE. */ + lzma_delta_type type; + /** - * \brief Delta distance as bytes + * \brief Delta distance + * + * With the only currently supported type, LZMA_DELTA_TYPE_BYTE, + * the distance is as bytes. * * Examples: * - 16-bit stereo audio: distance = 4 bytes @@ -46,4 +64,16 @@ typedef struct { # define LZMA_DELTA_DISTANCE_MIN 1 # define LZMA_DELTA_DISTANCE_MAX 256 + /** + * \brief Reserved space for possible future extensions + * + * You should not touch these, because the names of these variables + * may change. These are and will never be used when type is + * LZMA_DELTA_TYPE_BYTE, so it is safe to leave these uninitialized. + */ + uint32_t reserved_int1; + uint32_t reserved_int2; + void *reserved_ptr1; + void *reserved_ptr2; + } lzma_options_delta; diff --git a/src/liblzma/api/lzma/easy.h b/src/liblzma/api/lzma/easy.h deleted file mode 100644 index d83a79a2..00000000 --- a/src/liblzma/api/lzma/easy.h +++ /dev/null @@ -1,121 +0,0 @@ -/** - * \file lzma/easy.h - * \brief Easy to use encoder initialization - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2008 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use <lzma.h> instead. -#endif - - -/** - * \brief Compression level names for lzma_easy_* functions - * - * At the moment, all the compression levels support LZMA_SYNC_FLUSH. - * In future there may be levels that don't support LZMA_SYNC_FLUSH. - * However, the LZMA_SYNC_FLUSH support won't be removed from the - * existing compression levels. - * - * \note If liblzma is built without encoder support, or with some - * filters disabled, some of the compression levels may be - * unsupported. In that case, the initialization functions - * will return LZMA_HEADER_ERROR. - */ -typedef enum { - LZMA_EASY_COPY, - /**< - * No compression; the data is just wrapped into .lzma - * container. - */ - - LZMA_EASY_LZMA_1, - /**< - * LZMA filter with fast compression (fast in terms of LZMA). - * If you are interested in the exact options used, see - * lzma_preset_lzma[0]. Note that the exact options may - * change between liblzma versions. - * - * At the moment, the command line tool uses these settings - * when `lzma -1' is used. In future, the command line tool - * may default to some more complex way to determine the - * settings used e.g. the type of files being compressed. - * - * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1] - * and so on. - */ - - LZMA_EASY_LZMA_2, - LZMA_EASY_LZMA_3, - LZMA_EASY_LZMA_4, - LZMA_EASY_LZMA_5, - LZMA_EASY_LZMA_6, - LZMA_EASY_LZMA_7, - LZMA_EASY_LZMA_8, - LZMA_EASY_LZMA_9, -} lzma_easy_level; - - -/** - * \brief Default compression level - * - * Data Blocks contain the actual compressed data. It's not straightforward - * to recommend a default level, because in some cases keeping the resource - * usage relatively low is more important that getting the maximum - * compression ratio. - */ -#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA_7 - - -/** - * \brief Calculates rough memory requirements of a compression level - * - * This function is a wrapper for lzma_memory_usage(), which is declared - * in lzma/filter.h. - * - * \return Approximate memory usage of the encoder with the given - * compression level in mebibytes (value * 1024 * 1024 bytes). - * On error (e.g. compression level is not supported), - * UINT32_MAX is returned. - */ -extern uint32_t lzma_easy_memory_usage(lzma_easy_level level); - - -/** - * \brief Initializes .lzma Stream encoder - * - * This function is intended for those who just want to use the basic LZMA - * features (that is, most developers out there). Lots of assumptions are - * made, which are correct or at least good enough for most situations. - * - * \param strm Pointer to lzma_stream that is at least initialized - * with LZMA_STREAM_INIT. - * \param level Compression level to use. This selects a set of - * compression settings from a list of compression - * presets. - * - * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to - * encode your data. - * - LZMA_MEM_ERROR: Memory allocation failed. All memory - * previously allocated for *strm is now freed. - * - LZMA_HEADER_ERROR: The given compression level is not - * supported by this build of liblzma. - * - * If initialization succeeds, use lzma_code() to do the actual encoding. - * Valid values for `action' (the second argument of lzma_code()) are - * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, - * there may be compression levels that don't support LZMA_SYNC_FLUSH. - */ -extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level); diff --git a/src/liblzma/api/lzma/filter.h b/src/liblzma/api/lzma/filter.h index 412c30e5..d5903f89 100644 --- a/src/liblzma/api/lzma/filter.h +++ b/src/liblzma/api/lzma/filter.h @@ -51,7 +51,7 @@ typedef struct { */ void *options; -} lzma_options_filter; +} lzma_filter; /** @@ -65,7 +65,7 @@ typedef struct { * encoding-specific functions are probably missing from the library * API/ABI completely. */ -extern const lzma_vli *const lzma_available_filter_encoders; +extern const lzma_vli *const lzma_filter_encoders; /** @@ -79,7 +79,7 @@ extern const lzma_vli *const lzma_available_filter_encoders; * decoding-specific functions are probably missing from the library * API/ABI completely. */ -extern const lzma_vli *const lzma_available_filter_decoders; +extern const lzma_vli *const lzma_filter_decoders; /** @@ -87,8 +87,6 @@ extern const lzma_vli *const lzma_available_filter_decoders; * * \param filters Array of filters terminated with * .id == LZMA_VLI_VALUE_UNKNOWN. - * \param is_encoder Set to true when calculating memory requirements - * of an encoder; false for decoder. * * \return Number of mebibytes (MiB i.e. 2^20) required for the given * encoder or decoder filter chain. @@ -98,8 +96,55 @@ extern const lzma_vli *const lzma_available_filter_decoders; * if calculating memory requirements of decoder, lzma_init() or * lzma_init_decoder() must have been called earlier. */ -extern uint32_t lzma_memory_usage( - const lzma_options_filter *filters, lzma_bool is_encoder); +// extern uint32_t lzma_memory_usage( +// const lzma_filter *filters, lzma_bool is_encoder); + +extern uint64_t lzma_memusage_encoder(const lzma_filter *filters) + lzma_attr_pure; + +extern uint64_t lzma_memusage_decoder(const lzma_filter *filters) + lzma_attr_pure; + + +/** + * \brief Initializes raw encoder + * + * This function may be useful when implementing custom file formats. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Array of lzma_filter structures. + * The end of the array must be marked with + * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum + * number of filters is one and the maximum is four. + * + * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * filter chain supports it), or LZMA_FINISH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_encoder( + lzma_stream *strm, const lzma_filter *options) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes raw decoder + * + * The initialization of raw decoder goes similarly to raw encoder. + * + * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_decoder( + lzma_stream *strm, const lzma_filter *options) + lzma_attr_warn_unused_result; /** @@ -119,10 +164,11 @@ extern uint32_t lzma_memory_usage( * - LZMA_PROG_ERROR: Invalid options * * \note If you need to calculate size of List of Filter Flags, - * you need to loop over every lzma_options_filter entry. + * you need to loop over every lzma_filter entry. */ extern lzma_ret lzma_filter_flags_size( - uint32_t *size, const lzma_options_filter *options); + uint32_t *size, const lzma_filter *options) + lzma_attr_warn_unused_result; /** @@ -143,8 +189,9 @@ extern lzma_ret lzma_filter_flags_size( * buffer space (you should have checked it with * lzma_filter_flags_size()). */ -extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, - size_t out_size, const lzma_options_filter *options); +extern lzma_ret lzma_filter_flags_encode(const lzma_filter *options, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_attr_warn_unused_result; /** @@ -163,5 +210,6 @@ extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, * - LZMA_PROG_ERROR */ extern lzma_ret lzma_filter_flags_decode( - lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *in, size_t *in_pos, size_t in_size); + lzma_filter *options, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h index 13cddf47..44be10b1 100644 --- a/src/liblzma/api/lzma/index.h +++ b/src/liblzma/api/lzma/index.h @@ -66,7 +66,8 @@ typedef struct { * In this case, return value cannot be NULL or a different pointer than * the i given as argument. */ -extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator); +extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** @@ -91,13 +92,14 @@ extern void lzma_index_end(lzma_index *i, lzma_allocator *allocator); * - LZMA_PROG_ERROR */ extern lzma_ret lzma_index_append(lzma_index *i, lzma_allocator *allocator, - lzma_vli total_size, lzma_vli uncompressed_size); + lzma_vli total_size, lzma_vli uncompressed_size) + lzma_attr_warn_unused_result; /** * \brief Get the number of Records */ -extern lzma_vli lzma_index_count(const lzma_index *i); +extern lzma_vli lzma_index_count(const lzma_index *i) lzma_attr_pure; /** @@ -105,7 +107,7 @@ extern lzma_vli lzma_index_count(const lzma_index *i); * * This is needed to verify the Index Size field from the Stream Footer. */ -extern lzma_vli lzma_index_size(const lzma_index *i); +extern lzma_vli lzma_index_size(const lzma_index *i) lzma_attr_pure; /** @@ -114,7 +116,7 @@ extern lzma_vli lzma_index_size(const lzma_index *i); * This doesn't include the Stream Header, Stream Footer, Stream Padding, * or Index fields. */ -extern lzma_vli lzma_index_total_size(const lzma_index *i); +extern lzma_vli lzma_index_total_size(const lzma_index *i) lzma_attr_pure; /** @@ -123,7 +125,7 @@ extern lzma_vli lzma_index_total_size(const lzma_index *i); * If multiple Indexes have been combined, this works as if the Blocks * were in a single Stream. */ -extern lzma_vli lzma_index_stream_size(const lzma_index *i); +extern lzma_vli lzma_index_stream_size(const lzma_index *i) lzma_attr_pure; /** @@ -133,19 +135,21 @@ extern lzma_vli lzma_index_stream_size(const lzma_index *i); * identical to lzma_index_stream_size(). If multiple Indexes have been * combined, this includes also the possible Stream Padding fields. */ -extern lzma_vli lzma_index_file_size(const lzma_index *i); +extern lzma_vli lzma_index_file_size(const lzma_index *i) lzma_attr_pure; /** * \brief Get the uncompressed size of the Stream */ -extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i); +extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i) + lzma_attr_pure; /** * \brief Get the next Record from the Index */ -extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record); +extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record) + lzma_attr_warn_unused_result; /** @@ -179,7 +183,8 @@ extern void lzma_index_rewind(lzma_index *i); * and the read position are not modified, and this function returns true. */ extern lzma_bool lzma_index_locate( - lzma_index *i, lzma_index_record *record, lzma_vli target); + lzma_index *i, lzma_index_record *record, lzma_vli target) + lzma_attr_warn_unused_result; /** @@ -202,7 +207,8 @@ extern lzma_bool lzma_index_locate( */ extern lzma_ret lzma_index_cat(lzma_index *lzma_restrict dest, lzma_index *lzma_restrict src, - lzma_allocator *allocator, lzma_vli padding); + lzma_allocator *allocator, lzma_vli padding) + lzma_attr_warn_unused_result; /** @@ -211,22 +217,26 @@ extern lzma_ret lzma_index_cat(lzma_index *lzma_restrict dest, * \return A copy of the Index, or NULL if memory allocation failed. */ extern lzma_index *lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator); + const lzma_index *i, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** * \brief Compares if two Index lists are identical */ -extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b); +extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b) + lzma_attr_pure; /** * \brief Initializes Index encoder */ -extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i); +extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i) + lzma_attr_warn_unused_result; /** * \brief Initializes Index decoder */ -extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i); +extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/index_hash.h b/src/liblzma/api/lzma/index_hash.h index 1edbbeaa..58fc8061 100644 --- a/src/liblzma/api/lzma/index_hash.h +++ b/src/liblzma/api/lzma/index_hash.h @@ -42,7 +42,8 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash given as argument. */ extern lzma_index_hash *lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator); + lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** @@ -66,7 +67,8 @@ extern void lzma_index_hash_end( * used when lzma_index_hash_decode() has already been used. */ extern lzma_ret lzma_index_hash_append(lzma_index_hash *index_hash, - lzma_vli total_size, lzma_vli uncompressed_size); + lzma_vli total_size, lzma_vli uncompressed_size) + lzma_attr_warn_unused_result; /** @@ -83,7 +85,8 @@ extern lzma_ret lzma_index_hash_append(lzma_index_hash *index_hash, * Records can be added using lzma_index_hash_append(). */ extern lzma_ret lzma_index_hash_decode(lzma_index_hash *index_hash, - const uint8_t *in, size_t *in_pos, size_t in_size); + const uint8_t *in, size_t *in_pos, size_t in_size) + lzma_attr_warn_unused_result; /** @@ -91,4 +94,5 @@ extern lzma_ret lzma_index_hash_decode(lzma_index_hash *index_hash, * * This is needed to verify the Index Size field from the Stream Footer. */ -extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash); +extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash) + lzma_attr_pure; diff --git a/src/liblzma/api/lzma/lzma.h b/src/liblzma/api/lzma/lzma.h index 9473f448..5a1cd912 100644 --- a/src/liblzma/api/lzma/lzma.h +++ b/src/liblzma/api/lzma/lzma.h @@ -24,43 +24,11 @@ /** * \brief Filter ID * - * Filter ID of the LZMA filter. This is used as lzma_options_filter.id. + * Filter ID of the LZMA filter. This is used as lzma_filter.id. */ #define LZMA_FILTER_LZMA LZMA_VLI_C(0x40) - -/** - * \brief LZMA compression modes - * - * Currently there are only two modes. Earlier LZMA SDKs had also third - * mode between fast and best. - */ -typedef enum { - LZMA_MODE_INVALID = -1, - /**< - * \brief Invalid mode - * - * Used as array terminator in lzma_available_modes. - */ - - - LZMA_MODE_FAST = 0, - /**< - * \brief Fast compression - * - * Fast mode is usually at its best when combined with - * a hash chain match finder. - */ - - LZMA_MODE_BEST = 2 - /**< - * \brief Best compression ratio - * - * This is usually notably slower than fast mode. Use this - * together with binary tree match finders to expose the - * full potential of the LZMA encoder. - */ -} lzma_mode; +#define LZMA_FILTER_LZMA2 LZMA_VLI_C(0x21) /** @@ -129,6 +97,72 @@ typedef enum { /** + * \brief Test if given match finder is supported + * + * Returns true if the given match finder is supported by this liblzma build. + * Otherwise false is returned. It is safe to call this with a value that + * isn't listed in lzma_match_finder enumeration; the return value will be + * false. + * + * There is no way to list which match finders are available in this + * particular liblzma version and build. It would be useless, because + * a new match finder, which the application developer wasn't aware, + * could require giving additional options to the encoder that the older + * match finders don't need. + */ +extern lzma_bool lzma_mf_is_supported(lzma_match_finder match_finder) + lzma_attr_const; + + +/** + * \brief LZMA compression modes + * + * This selects the function used to analyze the data produced by the match + * finder. + */ +typedef enum { + LZMA_MODE_INVALID = -1, + /**< + * \brief Invalid mode + * + * Used as array terminator in lzma_available_modes. + */ + + LZMA_MODE_FAST = 0, + /**< + * \brief Fast compression + * + * Fast mode is usually at its best when combined with + * a hash chain match finder. + */ + + LZMA_MODE_NORMAL = 1 + /**< + * \brief Normal compression + * + * This is usually notably slower than fast mode. Use this + * together with binary tree match finders to expose the + * full potential of the LZMA encoder. + */ +} lzma_mode; + + +/** + * \brief Test if given compression mode is supported + * + * Returns true if the given compression mode is supported by this liblzma + * build. Otherwise false is returned. It is safe to call this with a value + * that isn't listed in lzma_mode enumeration; the return value will be false. + * + * There is no way to list which modes are available in this particular + * liblzma version and build. It would be useless, because a new compression + * mode, which the application developer wasn't aware, could require giving + * additional options to the encoder that the older modes don't need. + */ +extern lzma_bool lzma_mode_is_available(lzma_mode mode) lzma_attr_const; + + +/** * \brief Options specific to the LZMA method handler */ typedef struct { @@ -157,6 +191,44 @@ typedef struct { # define LZMA_DICTIONARY_SIZE_DEFAULT (UINT32_C(1) << 23) /** + * \brief Pointer to an initial dictionary + * + * It is possible to initialize the LZ77 history window using + * a preset dictionary. Here is a good quote from zlib's + * documentation; this applies to LZMA as is: + * + * "The dictionary should consist of strings (byte sequences) that + * are likely to be encountered later in the data to be compressed, + * with the most commonly used strings preferably put towards the + * end of the dictionary. Using a dictionary is most useful when + * the data to be compressed is short and can be predicted with + * good accuracy; the data can then be compressed better than + * with the default empty dictionary." + * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3) + * + * This feature should be used only in special situations. + * It works correctly only with raw encoding and decoding. + * Currently none of the container formats supported by + * liblzma allow preset dictionary when decoding, thus if + * you create a .lzma file with preset dictionary, it cannot + * be decoded with the regular .lzma decoder functions. + * + * \todo This feature is not implemented yet. + */ + const uint8_t *preset_dictionary; + + /** + * \brief Size of the preset dictionary + * + * Specifies the size of the preset dictionary. If the size is + * bigger than dictionary_size, only the last dictionary_size + * bytes are processed. + * + * This variable is read only when preset_dictionary is not NULL. + */ + uint32_t preset_dictionary_size; + + /** * \brief Number of literal context bits * * How many of the highest bits of the previous uncompressed @@ -203,47 +275,22 @@ typedef struct { # define LZMA_POS_BITS_MAX 4 # define LZMA_POS_BITS_DEFAULT 2 - /** - * \brief Pointer to an initial dictionary - * - * It is possible to initialize the LZ77 history window using - * a preset dictionary. Here is a good quote from zlib's - * documentation; this applies to LZMA as is: - * - * "The dictionary should consist of strings (byte sequences) that - * are likely to be encountered later in the data to be compressed, - * with the most commonly used strings preferably put towards the - * end of the dictionary. Using a dictionary is most useful when - * the data to be compressed is short and can be predicted with - * good accuracy; the data can then be compressed better than - * with the default empty dictionary." - * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3) - * - * This feature should be used only in special situations. - * It works correctly only with raw encoding and decoding. - * Currently none of the container formats supported by - * liblzma allow preset dictionary when decoding, thus if - * you create a .lzma file with preset dictionary, it cannot - * be decoded with the regular .lzma decoder functions. - * - * \todo This feature is not implemented yet. - */ - const uint8_t *preset_dictionary; + /****************************************** + * LZMA options needed only when encoding * + ******************************************/ /** - * \brief Size of the preset dictionary + * \brief Indicate if the options structure is persistent * - * Specifies the size of the preset dictionary. If the size is - * bigger than dictionary_size, only the last dictionary_size - * bytes are processed. + * If this is true, the application must keep this options structure + * available after the LZMA2 encoder has been initialized. With + * persistent structure it is possible to change some encoder options + * in the middle of the encoding process without resetting the encoder. * - * This variable is read only when preset_dictionary is not NULL. + * This option is used only by LZMA2. LZMA1 ignores this and it is + * safeto not initialize this when encoding with LZMA1. */ - uint32_t preset_dictionary_size; - - /****************************************** - * LZMA options needed only when encoding * - ******************************************/ + lzma_bool persistent; /** LZMA compression mode */ lzma_mode mode; @@ -275,6 +322,20 @@ typedef struct { */ uint32_t match_finder_cycles; + /** + * \brief Reserved space for possible future extensions + * + * You should not touch these, because the names of these variables + * may change. These are and will never be used with the currently + * supported options, so it is safe to leave these uninitialized. + */ + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + void *reserved_ptr1; + void *reserved_ptr2; + } lzma_options_lzma; @@ -287,27 +348,6 @@ typedef struct { /** - * \brief Available LZMA encoding modes - * - * Pointer to an array containing the list of available encoding modes. - * - * This variable is available only if LZMA encoder has been enabled. - */ -extern const lzma_mode *const lzma_available_modes; - - -/** - * \brief Available match finders - * - * Pointer to an array containing the list of available match finders. - * The last element is LZMA_MF_INVALID. - * - * This variable is available only if LZMA encoder has been enabled. - */ -extern const lzma_match_finder *const lzma_available_match_finders; - - -/** * \brief Table of presets for the LZMA filter * * lzma_preset_lzma[0] is the fastest and lzma_preset_lzma[8] is the slowest. diff --git a/src/liblzma/api/lzma/memlimit.h b/src/liblzma/api/lzma/memlimit.h index 7a856a27..836b0854 100644 --- a/src/liblzma/api/lzma/memlimit.h +++ b/src/liblzma/api/lzma/memlimit.h @@ -58,7 +58,8 @@ typedef struct lzma_memlimit_s lzma_memlimit; * lzma_memlimit_ can be used even if lzma_init() hasn't been * called. */ -extern lzma_memlimit *lzma_memlimit_create(size_t limit); +extern lzma_memlimit *lzma_memlimit_create(size_t limit) + lzma_attr_warn_unused_result; /** @@ -79,7 +80,8 @@ extern void lzma_memlimit_set(lzma_memlimit *mem, size_t limit); /** * \brief Gets the current memory usage limit */ -extern size_t lzma_memlimit_get(const lzma_memlimit *mem); +extern size_t lzma_memlimit_get(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -89,7 +91,8 @@ extern size_t lzma_memlimit_get(const lzma_memlimit *mem); * thus it will always be larger than the total number of * bytes allocated via lzma_memlimit_alloc(). */ -extern size_t lzma_memlimit_used(const lzma_memlimit *mem); +extern size_t lzma_memlimit_used(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -134,7 +137,8 @@ extern lzma_bool lzma_memlimit_reached(lzma_memlimit *mem, lzma_bool clear); * been allocated with lzma_memlimit_alloc() or all memory allocated * has been freed or detached, this will return zero. */ -extern size_t lzma_memlimit_count(const lzma_memlimit *mem); +extern size_t lzma_memlimit_count(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -157,7 +161,8 @@ extern size_t lzma_memlimit_count(const lzma_memlimit *mem); * invalid amount of memory being allocated. */ extern void *lzma_memlimit_alloc( - lzma_memlimit *mem, size_t nmemb, size_t size); + lzma_memlimit *mem, size_t nmemb, size_t size) + lzma_attr_warn_unused_result; /** diff --git a/src/liblzma/api/lzma/raw.h b/src/liblzma/api/lzma/raw.h deleted file mode 100644 index db8cba15..00000000 --- a/src/liblzma/api/lzma/raw.h +++ /dev/null @@ -1,60 +0,0 @@ -/** - * \file lzma/raw.h - * \brief Raw encoder and decoder - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use <lzma.h> instead. -#endif - - -/** - * \brief Initializes raw encoder - * - * This function may be useful when implementing custom file formats. - * - * \param strm Pointer to properly prepared lzma_stream - * \param options Array of lzma_options_filter structures. - * The end of the array must be marked with - * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum - * number of filters is one and the maximum is four. - * - * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the - * filter chain supports it), or LZMA_FINISH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_raw_encoder( - lzma_stream *strm, const lzma_options_filter *options); - - -/** - * \brief Initializes raw decoder - * - * The initialization of raw decoder goes similarly to raw encoder. - * - * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_raw_decoder( - lzma_stream *strm, const lzma_options_filter *options); diff --git a/src/liblzma/api/lzma/simple.h b/src/liblzma/api/lzma/simple.h index 807a4c46..13417480 100644 --- a/src/liblzma/api/lzma/simple.h +++ b/src/liblzma/api/lzma/simple.h @@ -21,7 +21,7 @@ #endif -/* Filter IDs for lzma_options_filter.id */ +/* Filter IDs for lzma_filter.id */ #define LZMA_FILTER_X86 LZMA_VLI_C(0x04) /**< diff --git a/src/liblzma/api/lzma/stream.h b/src/liblzma/api/lzma/stream.h deleted file mode 100644 index 4bb17e7d..00000000 --- a/src/liblzma/api/lzma/stream.h +++ /dev/null @@ -1,53 +0,0 @@ -/** - * \file lzma/stream.h - * \brief .lzma Stream handling - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use <lzma.h> instead. -#endif - - -/** - * \brief Initializes .lzma Stream encoder - * - * \param strm Pointer to properly prepared lzma_stream - * \param filters Array of filters. This must be terminated with - * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must - * be 1-4 filters, but there are restrictions on how - * multiple filters can be combined. FIXME Tell where - * to find more information. - * \param check Type of the integrity check to calculate from - * uncompressed data. - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_stream_encoder(lzma_stream *strm, - const lzma_options_filter *filters, lzma_check_type check); - - -/** - * \brief Initializes decoder for .lzma Stream - * - * \param strm Pointer to propertily prepared lzma_stream - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR: Cannot allocate memory. - */ -extern lzma_ret lzma_stream_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/stream_flags.h b/src/liblzma/api/lzma/stream_flags.h index f4c5c335..80c5f00f 100644 --- a/src/liblzma/api/lzma/stream_flags.h +++ b/src/liblzma/api/lzma/stream_flags.h @@ -46,7 +46,7 @@ typedef struct { /** * Type of the Check calculated from uncompressed data */ - lzma_check_type check; + lzma_check check; } lzma_stream_flags; @@ -64,7 +64,8 @@ typedef struct { * - LZMA_PROG_ERROR: Invalid options. */ extern lzma_ret lzma_stream_header_encode( - const lzma_stream_flags *options, uint8_t *out); + const lzma_stream_flags *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -78,7 +79,8 @@ extern lzma_ret lzma_stream_header_encode( * - LZMA_PROG_ERROR: Invalid options. */ extern lzma_ret lzma_stream_footer_encode( - const lzma_stream_flags *options, uint8_t *out); + const lzma_stream_flags *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -101,7 +103,8 @@ extern lzma_ret lzma_stream_footer_encode( * in the header. */ extern lzma_ret lzma_stream_header_decode( - lzma_stream_flags *options, const uint8_t *in); + lzma_stream_flags *options, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -120,7 +123,8 @@ extern lzma_ret lzma_stream_header_decode( * in the footer. */ extern lzma_ret lzma_stream_footer_decode( - lzma_stream_flags *options, const uint8_t *in); + lzma_stream_flags *options, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -131,4 +135,5 @@ extern lzma_ret lzma_stream_footer_decode( * \return true if both structures are considered equal; false otherwise. */ extern lzma_bool lzma_stream_flags_equal( - const lzma_stream_flags *a, lzma_stream_flags *b); + const lzma_stream_flags *a, const lzma_stream_flags *b) + lzma_attr_pure; diff --git a/src/liblzma/api/lzma/subblock.h b/src/liblzma/api/lzma/subblock.h index 1db35b13..b9a3025b 100644 --- a/src/liblzma/api/lzma/subblock.h +++ b/src/liblzma/api/lzma/subblock.h @@ -24,7 +24,7 @@ /** * \brief Filter ID * - * Filter ID of the Subblock filter. This is used as lzma_options_filter.id. + * Filter ID of the Subblock filter. This is used as lzma_filter.id. */ #define LZMA_FILTER_SUBBLOCK LZMA_VLI_C(0x01) @@ -199,6 +199,6 @@ typedef struct { * * \note This variable is ignored if allow_subfilters is false. */ - lzma_options_filter subfilter_options; + lzma_filter subfilter_options; } lzma_options_subblock; diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h index 252458a3..811f93e0 100644 --- a/src/liblzma/api/lzma/version.h +++ b/src/liblzma/api/lzma/version.h @@ -41,17 +41,17 @@ /** * \brief liblzma version number as an integer * - * This is the value of LZMA_VERSION macro at the compile time of liblzma. + * Returns the value of LZMA_VERSION macro at the compile time of liblzma. * This allows the application to compare if it was built against the same, * older, or newer version of liblzma that is currently running. */ -extern const uint32_t lzma_version_number; +extern uint32_t lzma_version_number(void) lzma_attr_const; /** - * \brief Returns versions number of liblzma as a string + * \brief Version number of liblzma as a string * * This function may be useful if you want to display which version of - * libilzma your application is currently using. + * liblzma your application is currently using. */ -extern const char *const lzma_version_string; +extern const char *lzma_version_string(void) lzma_attr_const; diff --git a/src/liblzma/api/lzma/vli.h b/src/liblzma/api/lzma/vli.h index 15a9d0bf..294e5cdd 100644 --- a/src/liblzma/api/lzma/vli.h +++ b/src/liblzma/api/lzma/vli.h @@ -72,90 +72,24 @@ typedef uint64_t lzma_vli; /** - * \brief Sets VLI to given value with error checking - * - * \param dest Target variable which must have type of lzma_vli. - * \param src New value to be stored to dest. - * \param limit Maximum allowed value for src. - * - * \return False on success, true on error. If an error occurred, - * dest is left in undefined state (i.e. it's possible that - * it will be different in newer liblzma versions). - */ -#define lzma_vli_set_lim(dest, src, limit) \ - ((src) > (limit) || ((dest) = (src)) > (limit)) - -/** - * \brief - */ -#define lzma_vli_add_lim(dest, src, limit) \ - ((src) > (limit) || ((dest) += (src)) > (limit)) - -#define lzma_vli_add2_lim(dest, src1, src2, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit)) - -#define lzma_vli_add3_lim(dest, src1, src2, src3, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit)) - -#define lzma_vli_add4_lim(dest, src1, src2, src3, src4, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit) \ - || lzma_vli_add_lim(dest, src4, limit)) - -#define lzma_vli_sum_lim(dest, src1, src2, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit)) - -#define lzma_vli_sum3_lim(dest, src1, src2, src3, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit)) - -#define lzma_vli_sum4_lim(dest, src1, src2, src3, src4, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit) \ - || lzma_vli_add_lim(dest, src4, limit)) - -#define lzma_vli_set(dest, src) lzma_vli_set_lim(dest, src, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add(dest, src) lzma_vli_add_lim(dest, src, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add2(dest, src1, src2) \ - lzma_vli_add2_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add3(dest, src1, src2, src3) \ - lzma_vli_add3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add4(dest, src1, src2, src3, src4) \ - lzma_vli_add4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum(dest, src1, src2) \ - lzma_vli_sum_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum3(dest, src1, src2, src3) \ - lzma_vli_sum3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum4(dest, src1, src2, src3, src4) \ - lzma_vli_sum4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) - - -/** * \brief Encodes variable-length integer * - * In the new .lzma format, most integers are encoded in variable-length + * In the .lzma format, most integers are encoded in variable-length * representation. This saves space when smaller values are more likely * than bigger values. * * The encoding scheme encodes seven bits to every byte, using minimum - * number of bytes required to represent the given value. In other words, - * it puts 7-63 bits into 1-9 bytes. This implementation limits the number - * of bits used to 63, thus num must be at maximum of UINT64_MAX / 2. You - * may use LZMA_VLI_VALUE_MAX for clarity. + * number of bytes required to represent the given value. Encodings that use + * non-minimum number of bytes are invalid, thus every integer has exactly + * one encoded representation. The maximum number of bits in a VLI is 63, + * thus the vli argument must be at maximum of UINT64_MAX / 2. You should + * use LZMA_VLI_VALUE_MAX for clarity. + * + * This function has two modes: single-call and multi-call. Single-call mode + * encodes the whole integer at once; it is an error if the output buffer is + * too small. Multi-call mode saves the position in *vli_pos, and thus it is + * possible to continue encoding if the buffer becomes full before the whole + * integer has been encoded. * * \param vli Integer to be encoded * \param vli_pos How many VLI-encoded bytes have already been written @@ -170,19 +104,19 @@ typedef uint64_t lzma_vli; * \return Slightly different return values are used in multi-call and * single-call modes. * + * Single-call (vli_pos == NULL): + * - LZMA_OK: Integer successfully encoded. + * - LZMA_PROG_ERROR: Arguments are not sane. This can be due + * to too little output space; single-call mode doesn't use + * LZMA_BUF_ERROR, since the application should have checked + * the encoded size with lzma_vli_size(). + * * Multi-call (vli_pos != NULL): * - LZMA_OK: So far all OK, but the integer is not * completely written out yet. * - LZMA_STREAM_END: Integer successfully encoded. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due - * to no *out_pos == out_size; this function doesn't use - * LZMA_BUF_ERROR. - * - * Single-call (vli_pos == NULL): - * - LZMA_OK: Integer successfully encoded. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due - * to too little output space; this function doesn't use - * LZMA_BUF_ERROR. + * - LZMA_BUF_ERROR: No output space was provided. + * - LZMA_PROG_ERROR: Arguments are not sane. */ extern lzma_ret lzma_vli_encode( lzma_vli vli, size_t *lzma_restrict vli_pos, @@ -193,6 +127,8 @@ extern lzma_ret lzma_vli_encode( /** * \brief Decodes variable-length integer * + * Like lzma_vli_encode(), this function has single-call and multi-call modes. + * * \param vli Pointer to decoded integer. The decoder will * initialize it to zero when *vli_pos == 0, so * application isn't required to initialize *vli. @@ -208,20 +144,20 @@ extern lzma_ret lzma_vli_encode( * \return Slightly different return values are used in multi-call and * single-call modes. * + * Single-call (vli_pos == NULL): + * - LZMA_OK: Integer successfully decoded. + * - LZMA_DATA_ERROR: Integer is corrupt. This includes hitting + * the end of the input buffer before the whole integer was + * decoded; providing no input at all will use LZMA_DATA_ERROR. + * - LZMA_PROG_ERROR: Arguments are not sane. + * * Multi-call (vli_pos != NULL): * - LZMA_OK: So far all OK, but the integer is not * completely decoded yet. * - LZMA_STREAM_END: Integer successfully decoded. * - LZMA_DATA_ERROR: Integer is corrupt. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be - * due to *in_pos == in_size; this function doesn't use - * LZMA_BUF_ERROR. - * - * Single-call (vli_pos == NULL): - * - LZMA_OK: Integer successfully decoded. - * - LZMA_DATA_ERROR: Integer is corrupt. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due to - * too little input; this function doesn't use LZMA_BUF_ERROR. + * - LZMA_BUF_ERROR: No input was provided. + * - LZMA_PROG_ERROR: Arguments are not sane. */ extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli, size_t *lzma_restrict vli_pos, const uint8_t *lzma_restrict in, @@ -234,4 +170,5 @@ extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli, * \return Number of bytes on success (1-9). If vli isn't valid, * zero is returned. */ -extern uint32_t lzma_vli_size(lzma_vli vli); +extern uint32_t lzma_vli_size(lzma_vli vli) + lzma_attr_pure; diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c index 388b57e8..ed64fe5c 100644 --- a/src/liblzma/check/check.c +++ b/src/liblzma/check/check.c @@ -13,60 +13,77 @@ #include "check.h" -// See the .lzma header format specification section 2.1.1.2. -LZMA_API const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1] = { - 0, - 4, 4, 4, - 8, 8, 8, - 16, 16, 16, - 32, 32, 32, - 64, 64, 64 -}; +extern LZMA_API lzma_bool +lzma_check_is_supported(lzma_check type) +{ + if ((unsigned)(type) > LZMA_CHECK_ID_MAX) + return false; -LZMA_API const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1] = { - true, // LZMA_CHECK_NONE + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { + true, // LZMA_CHECK_NONE #ifdef HAVE_CHECK_CRC32 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved + false, // Reserved + false, // Reserved #ifdef HAVE_CHECK_CRC64 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved #ifdef HAVE_CHECK_SHA256 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved -}; + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + }; + + return available_checks[(unsigned)(type)]; +} -extern lzma_ret -lzma_check_init(lzma_check *check, lzma_check_type type) +extern LZMA_API uint32_t +lzma_check_size(lzma_check type) { - lzma_ret ret = LZMA_OK; + if ((unsigned)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + + // See file-format.txt section 2.1.1.2. + static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { + 0, + 4, 4, 4, + 8, 8, 8, + 16, 16, 16, + 32, 32, 32, + 64, 64, 64 + }; + + return check_sizes[(unsigned)(type)]; +} + +extern void +lzma_check_init(lzma_check_state *check, lzma_check type) +{ switch (type) { case LZMA_CHECK_NONE: break; @@ -90,19 +107,15 @@ lzma_check_init(lzma_check *check, lzma_check_type type) #endif default: - if ((unsigned)(type) <= LZMA_CHECK_ID_MAX) - ret = LZMA_UNSUPPORTED_CHECK; - else - ret = LZMA_PROG_ERROR; break; } - return ret; + return; } extern void -lzma_check_update(lzma_check *check, lzma_check_type type, +lzma_check_update(lzma_check_state *check, lzma_check type, const uint8_t *buf, size_t size) { switch (type) { @@ -133,18 +146,18 @@ lzma_check_update(lzma_check *check, lzma_check_type type, extern void -lzma_check_finish(lzma_check *check, lzma_check_type type) +lzma_check_finish(lzma_check_state *check, lzma_check type) { switch (type) { #ifdef HAVE_CHECK_CRC32 case LZMA_CHECK_CRC32: - *(uint32_t *)(check->buffer) = check->state.crc32; + check->buffer.u32[0] = integer_le_32(check->state.crc32); break; #endif #ifdef HAVE_CHECK_CRC64 case LZMA_CHECK_CRC64: - *(uint64_t *)(check->buffer) = check->state.crc64; + check->buffer.u64[0] = integer_le_64(check->state.crc64); break; #endif @@ -160,34 +173,3 @@ lzma_check_finish(lzma_check *check, lzma_check_type type) return; } - - -/* -extern bool -lzma_check_compare( - lzma_check *check1, lzma_check *check2, lzma_check_type type) -{ - bool ret; - - switch (type) { - case LZMA_CHECK_NONE: - break; - - case LZMA_CHECK_CRC32: - ret = check1->crc32 != check2->crc32; - break; - - case LZMA_CHECK_CRC64: - ret = check1->crc64 != check2->crc64; - break; - - default: - // Unsupported check - assert(type <= 7); - ret = false; - break; - } - - return ret; -} -*/ diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h index 45ca25e9..8f387799 100644 --- a/src/liblzma/check/check.h +++ b/src/liblzma/check/check.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file check.h -/// \brief Prototypes for different check functions +/// \brief Internal API to different integrity check functions // // This code has been put into the public domain. // @@ -17,8 +17,8 @@ #include "common.h" -// Index hashing used to verify the Index with O(1) memory usage needs -// a good hash function. +// Index hashing needs the best possible hash function (preferably +// a cryptographic hash) for maximum reliability. #if defined(HAVE_CHECK_SHA256) # define LZMA_CHECK_BEST LZMA_CHECK_SHA256 #elif defined(HAVE_CHECK_CRC64) @@ -28,24 +28,17 @@ #endif +/// \brief Structure to hold internal state of the check being calculated +/// +/// \note This is not in the public API because this structure may +/// change in future if new integrity check algorithms are added. typedef struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - -} lzma_sha256; - - -/// \note This is not in the public API because this structure will -/// change in future. -typedef struct { - // FIXME Guarantee 8-byte alignment - - /// Buffer to hold the final result; this is also used as a temporary - /// buffer in SHA256. Note that this buffer must be 8-byte aligned. - uint8_t buffer[64]; + /// Buffer to hold the final result and a temporary buffer for SHA256. + union { + uint8_t u8[64]; + uint32_t u32[16]; + uint64_t u64[8]; + } buffer; /// Check-specific data union { @@ -61,7 +54,7 @@ typedef struct { } sha256; } state; -} lzma_check; +} lzma_check_state; #ifdef HAVE_SMALL @@ -72,7 +65,6 @@ extern const uint32_t lzma_crc32_table[8][256]; extern const uint64_t lzma_crc64_table[4][256]; #endif -// Generic /// \brief Initializes *check depending on type /// @@ -80,46 +72,31 @@ extern const uint64_t lzma_crc64_table[4][256]; /// supported by the current version or build of liblzma. /// LZMA_PROG_ERROR if type > LZMA_CHECK_ID_MAX. /// -extern lzma_ret lzma_check_init(lzma_check *check, lzma_check_type type); +extern void lzma_check_init(lzma_check_state *check, lzma_check type); + /// \brief Updates *check /// -extern void lzma_check_update(lzma_check *check, lzma_check_type type, +extern void lzma_check_update(lzma_check_state *check, lzma_check type, const uint8_t *buf, size_t size); -/// \brief Finishes *check -/// -extern void lzma_check_finish(lzma_check *check, lzma_check_type type); - -/* -/// \brief Compare two checks -/// -/// \return false if the checks are identical; true if they differ. +/// \brief Finishes *check /// -extern bool lzma_check_compare( - lzma_check *check1, lzma_check *check2, lzma_check_type type); -*/ +extern void lzma_check_finish(lzma_check_state *check, lzma_check type); -// CRC32 - extern void lzma_crc32_init(void); -// CRC64 - extern void lzma_crc64_init(void); -// SHA256 - -extern void lzma_sha256_init(lzma_check *check); +extern void lzma_sha256_init(lzma_check_state *check); extern void lzma_sha256_update( - const uint8_t *buf, size_t size, lzma_check *check); - -extern void lzma_sha256_finish(lzma_check *check); + const uint8_t *buf, size_t size, lzma_check_state *check); +extern void lzma_sha256_finish(lzma_check_state *check); #endif diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c index ea51896e..9f90f7e7 100644 --- a/src/liblzma/check/sha256.c +++ b/src/liblzma/check/sha256.c @@ -104,16 +104,16 @@ transform(uint32_t state[static 8], const uint32_t data[static 16]) static void -process(lzma_check *check) +process(lzma_check_state *check) { #ifdef WORDS_BIGENDIAN - transform(check->state.sha256.state, (uint32_t *)(check->buffer)); + transform(check->state.sha256.state, check->buffer.u32); #else uint32_t data[16]; for (size_t i = 0; i < 16; ++i) - data[i] = bswap_32(*((uint32_t*)(check->buffer) + i)); + data[i] = bswap_32(check->buffer.u32[i]); transform(check->state.sha256.state, data); #endif @@ -123,7 +123,7 @@ process(lzma_check *check) extern void -lzma_sha256_init(lzma_check *check) +lzma_sha256_init(lzma_check_state *check) { static const uint32_t s[8] = { 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, @@ -138,7 +138,7 @@ lzma_sha256_init(lzma_check *check) extern void -lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) { // Copy the input data into a properly aligned temporary buffer. // This way we can be called with arbitrarily sized buffers @@ -150,7 +150,7 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) if (copy_size > size) copy_size = size; - memcpy(check->buffer + copy_start, buf, copy_size); + memcpy(check->buffer.u8 + copy_start, buf, copy_size); buf += copy_size; size -= copy_size; @@ -165,12 +165,12 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) extern void -lzma_sha256_finish(lzma_check *check) +lzma_sha256_finish(lzma_check_state *check) { // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; - check->buffer[pos++] = 0x80; + check->buffer.u8[pos++] = 0x80; while (pos != 64 - 8) { if (pos == 64) { @@ -178,28 +178,25 @@ lzma_sha256_finish(lzma_check *check) pos = 0; } - check->buffer[pos++] = 0x00; + check->buffer.u8[pos++] = 0x00; } // Convert the message size from bytes to bits. check->state.sha256.size *= 8; #ifdef WORDS_BIGENDIAN - *(uint64_t *)(check->buffer + 64 - 8) = check->state.sha256.size; + check->buffer.u64[(64 - 8) / 8] = check->state.sha256.size; #else - *(uint64_t *)(check->buffer + 64 - 8) - = bswap_64(check->state.sha256.size); + check->buffer.u64[(64 - 8) / 8] = bswap_64(check->state.sha256.size); #endif process(check); for (size_t i = 0; i < 8; ++i) #ifdef WORDS_BIGENDIAN - ((uint32_t *)(check->buffer))[i] - = check->state.sha256.state[i]; + check->buffer.u32[i] = check->state.sha256.state[i]; #else - ((uint32_t *)(check->buffer))[i] - = bswap_32(check->state.sha256.state[i]); + check->buffer.u32[i] = bswap_32(check->state.sha256.state[i]); #endif return; diff --git a/src/liblzma/common/Makefile.am b/src/liblzma/common/Makefile.am index 40b42250..3ec2e270 100644 --- a/src/liblzma/common/Makefile.am +++ b/src/liblzma/common/Makefile.am @@ -16,62 +16,43 @@ noinst_LTLIBRARIES = libcommon.la libcommon_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/check \ + -I@top_srcdir@/src/liblzma/rangecoder \ -I@top_srcdir@/src/liblzma/lz \ -I@top_srcdir@/src/liblzma/lzma \ - -I@top_srcdir@/src/liblzma/simple \ -I@top_srcdir@/src/liblzma/subblock \ - -I@top_srcdir@/src/liblzma/rangecoder + -I@top_srcdir@/src/liblzma/delta \ + -I@top_srcdir@/src/liblzma/simple + libcommon_la_SOURCES = \ + common.c \ common.h \ bsr.h \ - allocator.c \ block_util.c \ block_private.h \ - features.c \ + filter_common.c \ + filter_common.h \ index.c \ + index.h \ init.c \ memory_limiter.c \ - memory_usage.c \ - next_coder.c \ - raw_common.c \ - raw_common.h \ - stream_flags_equal.c \ - code.c \ - version.c - -if COND_FILTER_DELTA -libcommon_la_SOURCES += \ - delta_common.c \ - delta_common.h -if COND_MAIN_ENCODER -libcommon_la_SOURCES += \ - delta_encoder.c \ - delta_encoder.h -endif -if COND_MAIN_DECODER -libcommon_la_SOURCES += \ - delta_decoder.c \ - delta_decoder.h -endif -endif + stream_flags_common.c \ + stream_flags_common.h \ + vli_size.c if COND_MAIN_ENCODER libcommon_la_SOURCES += \ alignment.c \ - auto_decoder.c \ alone_encoder.c \ block_encoder.c \ block_encoder.h \ block_header_encoder.c \ easy.c \ + filter_encoder.c \ + filter_encoder.h \ filter_flags_encoder.c \ index_encoder.c \ index_encoder.h \ init_encoder.c \ - raw_encoder.c \ - raw_encoder.h \ - stream_common.c \ - stream_common.h \ stream_encoder.c \ stream_encoder.h \ stream_flags_encoder.c \ @@ -82,16 +63,18 @@ if COND_MAIN_DECODER libcommon_la_SOURCES += \ alone_decoder.c \ alone_decoder.h \ + auto_decoder.c \ block_decoder.c \ block_decoder.h \ block_header_decoder.c \ + filter_decoder.c \ + filter_decoder.h \ filter_flags_decoder.c \ index_decoder.c \ index_hash.c \ init_decoder.c \ - raw_decoder.c \ - raw_decoder.h \ stream_decoder.c \ + stream_decoder.h \ stream_flags_decoder.c \ stream_flags_decoder.h \ vli_decoder.c diff --git a/src/liblzma/common/alignment.c b/src/liblzma/common/alignment.c index c80e5fab..ba9ecb03 100644 --- a/src/liblzma/common/alignment.c +++ b/src/liblzma/common/alignment.c @@ -21,7 +21,7 @@ extern LZMA_API uint32_t -lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess) +lzma_alignment_input(const lzma_filter *filters, uint32_t guess) { for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { switch (filters[i].id) { @@ -66,7 +66,7 @@ lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess) extern LZMA_API uint32_t -lzma_alignment_output(const lzma_options_filter *filters, uint32_t guess) +lzma_alignment_output(const lzma_filter *filters, uint32_t guess) { if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN) return UINT32_MAX; diff --git a/src/liblzma/common/allocator.c b/src/liblzma/common/allocator.c deleted file mode 100644 index 5ced9d16..00000000 --- a/src/liblzma/common/allocator.c +++ /dev/null @@ -1,58 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file allocator.c -/// \brief Allocating and freeing memory -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - -#undef lzma_free - -extern void * lzma_attribute((malloc)) -lzma_alloc(size_t size, lzma_allocator *allocator) -{ - // Some malloc() variants return NULL if called with size == 0. - if (size == 0) - size = 1; - - void *ptr; - - if (allocator != NULL && allocator->alloc != NULL) - ptr = allocator->alloc(allocator->opaque, 1, size); - else - ptr = malloc(size); - -#ifndef NDEBUG - // This helps to catch some stupid mistakes, but also hides them from - // Valgrind. Uncomment when useful. -// if (ptr != NULL) -// memset(ptr, 0xFD, size); -#endif - - return ptr; -} - - -extern void -lzma_free(void *ptr, lzma_allocator *allocator) -{ - if (allocator != NULL && allocator->free != NULL) - allocator->free(allocator->opaque, ptr); - else - free(ptr); - - return; -} diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c index 062f6fab..006740f4 100644 --- a/src/liblzma/common/alone_decoder.c +++ b/src/liblzma/common/alone_decoder.c @@ -19,6 +19,7 @@ #include "alone_decoder.h" #include "lzma_decoder.h" +#include "lz_decoder.h" struct lzma_coder_s { @@ -38,6 +39,9 @@ struct lzma_coder_s { /// Uncompressed size decoded from the header lzma_vli uncompressed_size; + /// Memory usage limit + uint64_t memlimit; + /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; @@ -56,7 +60,7 @@ alone_decode(lzma_coder *coder, && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { case SEQ_PROPERTIES: - if (lzma_lzma_decode_properties(&coder->options, in[*in_pos])) + if (lzma_lzma_lclppb_decode(&coder->options, in[*in_pos])) return LZMA_FORMAT_ERROR; coder->sequence = SEQ_DICTIONARY_SIZE; @@ -69,8 +73,6 @@ alone_decode(lzma_coder *coder, if (++coder->pos == 4) { if (coder->options.dictionary_size - < LZMA_DICTIONARY_SIZE_MIN - || coder->options.dictionary_size > LZMA_DICTIONARY_SIZE_MAX) return LZMA_FORMAT_ERROR; @@ -119,7 +121,20 @@ alone_decode(lzma_coder *coder, break; case SEQ_CODER_INIT: { - // Two is enough because there won't be implicit filters. + // FIXME It is unfair that this doesn't add a fixed amount + // like lzma_memusage_common() does. + const uint64_t memusage + = lzma_lzma_decoder_memusage(&coder->options); + + // Use LZMA_PROG_ERROR since LZMA_Alone decoder cannot be + // built without LZMA support. + // FIXME TODO Make the above comment true. + if (memusage == UINT64_MAX) + return LZMA_PROG_ERROR; + + if (memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; + lzma_filter_info filters[2] = { { .init = &lzma_lzma_decoder_init, @@ -135,7 +150,7 @@ alone_decode(lzma_coder *coder, return ret; // Use a hack to set the uncompressed size. - lzma_lzma_decoder_uncompressed_size(&coder->next, + lzma_lz_decoder_uncompressed(coder->next.coder, coder->uncompressed_size); coder->sequence = SEQ_CODE; @@ -160,15 +175,18 @@ alone_decode(lzma_coder *coder, static void alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +extern lzma_ret +lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit) { + lzma_next_coder_init(lzma_alone_decoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -183,25 +201,20 @@ alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->coder->pos = 0; next->coder->options.dictionary_size = 0; next->coder->uncompressed_size = 0; + next->coder->memlimit = memlimit; return LZMA_OK; } -extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) -{ - lzma_next_coder_init0(alone_decoder_init, next, allocator); -} - - extern LZMA_API lzma_ret -lzma_alone_decoder(lzma_stream *strm) +lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init0(strm, alone_decoder_init); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/alone_decoder.h b/src/liblzma/common/alone_decoder.h index a9b7e84b..13284043 100644 --- a/src/liblzma/common/alone_decoder.h +++ b/src/liblzma/common/alone_decoder.h @@ -17,8 +17,13 @@ // /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_ALONE_DECODER_H +#define LZMA_ALONE_DECODER_H + #include "common.h" -extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator); +extern lzma_ret lzma_alone_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit); + +#endif diff --git a/src/liblzma/common/alone_encoder.c b/src/liblzma/common/alone_encoder.c index f94a21c1..7fb11570 100644 --- a/src/liblzma/common/alone_encoder.c +++ b/src/liblzma/common/alone_encoder.c @@ -48,7 +48,7 @@ alone_encode(lzma_coder *coder, while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: - bufcpy(coder->header, &coder->header_pos, + lzma_bufcpy(coder->header, &coder->header_pos, ALONE_HEADER_SIZE, out, out_pos, out_size); if (coder->header_pos < ALONE_HEADER_SIZE) @@ -74,7 +74,7 @@ alone_encode(lzma_coder *coder, static void alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } @@ -85,6 +85,8 @@ static lzma_ret alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_options_lzma *options) { + lzma_next_coder_init(alone_encoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -101,7 +103,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Encode the header: // - Properties (1 byte) - if (lzma_lzma_encode_properties(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, next->coder->header)) return LZMA_PROG_ERROR; // - Dictionary size (4 bytes) @@ -113,6 +115,9 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next. While the header would allow any 32-bit integer, // we do this to keep the decoder of liblzma accepting the resulting // files. + // + // FIXME Maybe LZMA_Alone needs some lower limit for maximum + // dictionary size? Must check decoders from old LZMA SDK version. uint32_t d = options->dictionary_size - 1; d |= d >> 2; d |= d >> 3; @@ -153,7 +158,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API lzma_ret lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init(strm, alone_encoder_init, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c index 765a27b1..5fcdf168 100644 --- a/src/liblzma/common/auto_decoder.c +++ b/src/liblzma/common/auto_decoder.c @@ -23,6 +23,8 @@ struct lzma_coder_s { lzma_next_coder next; + uint64_t memlimit; + uint32_t flags; bool initialized; }; @@ -41,9 +43,11 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, if (in[*in_pos] == 0xFF) ret = lzma_stream_decoder_init( - &coder->next, allocator); + &coder->next, allocator, + coder->memlimit, coder->flags); else - ret = lzma_alone_decoder_init(&coder->next, allocator); + ret = lzma_alone_decoder_init(&coder->next, + allocator, coder->memlimit); if (ret != LZMA_OK) return ret; @@ -59,15 +63,21 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) { + lzma_next_coder_init(auto_decoder_init, next, allocator); + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_HEADER_ERROR; + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -78,30 +88,22 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->coder->next = LZMA_NEXT_CODER_INIT; } + next->coder->memlimit = memlimit; + next->coder->flags = flags; next->coder->initialized = false; return LZMA_OK; } -/* -extern lzma_ret -lzma_auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_extra **header, lzma_extra **footer) -{ - lzma_next_coder_init( - auto_decoder_init, next, allocator, header, footer); -} -*/ - - extern LZMA_API lzma_ret -lzma_auto_decoder(lzma_stream *strm) +lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init0(strm, auto_decoder_init); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/block_decoder.c b/src/liblzma/common/block_decoder.c index f07c4e06..2c16a204 100644 --- a/src/liblzma/common/block_decoder.c +++ b/src/liblzma/common/block_decoder.c @@ -19,7 +19,7 @@ #include "block_decoder.h" #include "block_private.h" -#include "raw_decoder.h" +#include "filter_decoder.h" #include "check.h" @@ -35,7 +35,7 @@ struct lzma_coder_s { /// Decoding options; we also write Compressed Size and Uncompressed /// Size back to this structure when the encoding has been finished. - lzma_options_block *options; + lzma_block *options; /// Compressed Size calculated while encoding lzma_vli compressed_size; @@ -52,7 +52,7 @@ struct lzma_coder_s { size_t check_pos; /// Check of the uncompressed data - lzma_check check; + lzma_check_state check; }; @@ -64,9 +64,6 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, { switch (coder->sequence) { case SEQ_CODE: { - if (*out_pos >= out_size) - return LZMA_OK; - const size_t in_start = *in_pos; const size_t out_start = *out_pos; @@ -98,7 +95,7 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_PADDING: - // If Compressed Data is padded to a multiple of four bytes. + // Compressed Data is padded to a multiple of four bytes. while (coder->compressed_size & 3) { if (*in_pos >= in_size) return LZMA_OK; @@ -132,19 +129,29 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through - case SEQ_CHECK: + case SEQ_CHECK: { + const bool chksup = lzma_check_is_supported( + coder->options->check); + while (*in_pos < in_size) { - if (in[(*in_pos)++] != coder->check.buffer[ - coder->check_pos]) + // coder->check.buffer[] may be uninitialized when + // the Check ID is not supported. + if (chksup && coder->check.buffer.u8[coder->check_pos] + != in[*in_pos]) { + ++*in_pos; return LZMA_DATA_ERROR; + } - if (++coder->check_pos == lzma_check_sizes[ - coder->options->check]) + ++*in_pos; + + if (++coder->check_pos == lzma_check_size( + coder->options->check)) return LZMA_STREAM_END; } return LZMA_OK; } + } return LZMA_PROG_ERROR; } @@ -153,21 +160,28 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) +extern lzma_ret +lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *options) { + lzma_next_coder_init(lzma_block_decoder_init, next, allocator); + // While lzma_block_total_size_get() is meant to calculate the Total // Size, it also validates the options excluding the filters. if (lzma_block_total_size_get(options) == 0) return LZMA_PROG_ERROR; + // options->check is used for array indexing so we need to know that + // it is in the valid range. + if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + // Allocate and initialize *next->coder if needed. if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); @@ -192,30 +206,25 @@ block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, = options->compressed_size == LZMA_VLI_VALUE_UNKNOWN ? (LZMA_VLI_VALUE_MAX & ~LZMA_VLI_C(3)) - options->header_size - - lzma_check_sizes[options->check] + - lzma_check_size(options->check) : options->compressed_size; - // Initialize the check + // Initialize the check. It's caller's problem if the Check ID is not + // supported, and the Block decoder cannot verify the Check field. + // Caller can test lzma_checks[options->check]. next->coder->check_pos = 0; - return_if_error(lzma_check_init(&next->coder->check, options->check)); + lzma_check_init(&next->coder->check, options->check); + // Initialize the filter chain. return lzma_raw_decoder_init(&next->coder->next, allocator, options->filters); } -extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) -{ - lzma_next_coder_init(block_decoder_init, next, allocator, options); -} - - extern LZMA_API lzma_ret -lzma_block_decoder(lzma_stream *strm, lzma_options_block *options) +lzma_block_decoder(lzma_stream *strm, lzma_block *options) { - lzma_next_strm_init(strm, block_decoder_init, options); + lzma_next_strm_init(lzma_block_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; diff --git a/src/liblzma/common/block_decoder.h b/src/liblzma/common/block_decoder.h index af71128d..999aa748 100644 --- a/src/liblzma/common/block_decoder.h +++ b/src/liblzma/common/block_decoder.h @@ -24,6 +24,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_options_block *options); + lzma_allocator *allocator, lzma_block *options); #endif diff --git a/src/liblzma/common/block_encoder.c b/src/liblzma/common/block_encoder.c index 3add45a9..5aa3626b 100644 --- a/src/liblzma/common/block_encoder.c +++ b/src/liblzma/common/block_encoder.c @@ -19,7 +19,7 @@ #include "block_encoder.h" #include "block_private.h" -#include "raw_encoder.h" +#include "filter_encoder.h" #include "check.h" @@ -30,7 +30,7 @@ struct lzma_coder_s { /// Encoding options; we also write Total Size, Compressed Size, and /// Uncompressed Size back to this structure when the encoding has /// been finished. - lzma_options_block *options; + lzma_block *options; enum { SEQ_CODE, @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t check_pos; /// Check of the uncompressed data - lzma_check check; + lzma_check_state check; }; @@ -147,11 +147,11 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_CHECK: - out[*out_pos] = coder->check.buffer[coder->check_pos]; + out[*out_pos] = coder->check.buffer.u8[coder->check_pos]; ++*out_pos; if (++coder->check_pos - == lzma_check_sizes[coder->options->check]) + == lzma_check_size(coder->options->check)) return LZMA_STREAM_END; break; @@ -167,21 +167,31 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) +extern lzma_ret +lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *options) { + lzma_next_coder_init(lzma_block_encoder_init, next, allocator); + // While lzma_block_total_size_get() is meant to calculate the Total // Size, it also validates the options excluding the filters. if (lzma_block_total_size_get(options) == 0) return LZMA_PROG_ERROR; + // If the Check ID is not supported, we cannot calculate the check and + // thus not create a proper Block. + if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + // Allocate and initialize *next->coder if needed. if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); @@ -201,7 +211,7 @@ block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check next->coder->check_pos = 0; - return_if_error(lzma_check_init(&next->coder->check, options->check)); + lzma_check_init(&next->coder->check, options->check); // Initialize the requested filters. return lzma_raw_encoder_init(&next->coder->next, allocator, @@ -209,18 +219,10 @@ block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) -{ - lzma_next_coder_init(block_encoder_init, next, allocator, options); -} - - extern LZMA_API lzma_ret -lzma_block_encoder(lzma_stream *strm, lzma_options_block *options) +lzma_block_encoder(lzma_stream *strm, lzma_block *options) { - lzma_next_strm_init(strm, block_encoder_init, options); + lzma_next_strm_init(lzma_block_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/src/liblzma/common/block_encoder.h b/src/liblzma/common/block_encoder.h index eafcc618..7bc40139 100644 --- a/src/liblzma/common/block_encoder.h +++ b/src/liblzma/common/block_encoder.h @@ -24,6 +24,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_options_block *options); + lzma_allocator *allocator, lzma_block *options); #endif diff --git a/src/liblzma/common/block_header_decoder.c b/src/liblzma/common/block_header_decoder.c index b9e072e0..1d75f335 100644 --- a/src/liblzma/common/block_header_decoder.c +++ b/src/liblzma/common/block_header_decoder.c @@ -22,7 +22,7 @@ static void -free_properties(lzma_options_block *options, lzma_allocator *allocator) +free_properties(lzma_block *options, lzma_allocator *allocator) { // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of @@ -38,12 +38,12 @@ free_properties(lzma_options_block *options, lzma_allocator *allocator) extern LZMA_API lzma_ret -lzma_block_header_decode(lzma_options_block *options, +lzma_block_header_decode(lzma_block *options, lzma_allocator *allocator, const uint8_t *in) { // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers - // are invalid or not over 63 bits, or if the header is too small + // are invalid or over 63 bits, or if the header is too small // to contain the claimed information. // Initialize the filter options array. This way the caller can diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c index ed0c88ba..3a16e6c3 100644 --- a/src/liblzma/common/block_header_encoder.c +++ b/src/liblzma/common/block_header_encoder.c @@ -22,7 +22,7 @@ extern LZMA_API lzma_ret -lzma_block_header_size(lzma_options_block *options) +lzma_block_header_size(lzma_block *options) { // Block Header Size + Block Flags + CRC32. size_t size = 1 + 1 + 4; @@ -77,7 +77,7 @@ lzma_block_header_size(lzma_options_block *options) extern LZMA_API lzma_ret -lzma_block_header_encode(const lzma_options_block *options, uint8_t *out) +lzma_block_header_encode(const lzma_block *options, uint8_t *out) { if ((options->header_size & 3) || options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -127,8 +127,9 @@ lzma_block_header_encode(const lzma_options_block *options, uint8_t *out) if (filter_count == 4) return LZMA_PROG_ERROR; - return_if_error(lzma_filter_flags_encode(out, &out_pos, - out_size, options->filters + filter_count)); + return_if_error(lzma_filter_flags_encode( + options->filters + filter_count, + out, &out_pos, out_size)); } while (options->filters[++filter_count].id != LZMA_VLI_VALUE_UNKNOWN); diff --git a/src/liblzma/common/block_util.c b/src/liblzma/common/block_util.c index 6bffc2f1..798163bb 100644 --- a/src/liblzma/common/block_util.c +++ b/src/liblzma/common/block_util.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file block_header.c -/// \brief Utility functions to handle lzma_options_block +/// \brief Utility functions to handle lzma_block // // Copyright (C) 2008 Lasse Collin // @@ -21,7 +21,7 @@ extern LZMA_API lzma_ret -lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) +lzma_block_total_size_set(lzma_block *options, lzma_vli total_size) { // Validate. if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -32,7 +32,7 @@ lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) return LZMA_PROG_ERROR; const uint32_t container_size = options->header_size - + lzma_check_sizes[options->check]; + + lzma_check_size(options->check); // Validate that Compressed Size will be greater than zero. if (container_size <= total_size) @@ -45,7 +45,7 @@ lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) extern LZMA_API lzma_vli -lzma_block_total_size_get(const lzma_options_block *options) +lzma_block_total_size_get(const lzma_block *options) { // Validate the values that we are interested in. if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -61,7 +61,7 @@ lzma_block_total_size_get(const lzma_options_block *options) const lzma_vli total_size = options->compressed_size + options->header_size - + lzma_check_sizes[options->check]; + + lzma_check_size(options->check); // Validate the calculated Total Size. if (options->compressed_size > LZMA_VLI_VALUE_MAX diff --git a/src/liblzma/common/code.c b/src/liblzma/common/common.c index 0e3929b6..feac9cbf 100644 --- a/src/liblzma/common/code.c +++ b/src/liblzma/common/common.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file code.c -/// \brief zlib-like API wrapper for liblzma's internal API +/// \file common.h +/// \brief Common functions needed in many places in liblzma // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -20,19 +20,117 @@ #include "common.h" -LZMA_API const lzma_stream LZMA_STREAM_INIT_VAR = { - .next_in = NULL, - .avail_in = 0, - .total_in = 0, - .next_out = NULL, - .avail_out = 0, - .total_out = 0, - .allocator = NULL, - .internal = NULL, -}; +///////////// +// Version // +///////////// + +extern LZMA_API uint32_t +lzma_version_number(void) +{ + return LZMA_VERSION; +} + + +extern LZMA_API const char * +lzma_version_string(void) +{ + return PACKAGE_VERSION; +} + + +/////////////////////// +// Memory allocation // +/////////////////////// + +extern void * lzma_attribute((malloc)) +lzma_alloc(size_t size, lzma_allocator *allocator) +{ + // Some malloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) + ptr = allocator->alloc(allocator->opaque, 1, size); + else + ptr = malloc(size); + + return ptr; +} + + +extern void +lzma_free(void *ptr, lzma_allocator *allocator) +{ + if (allocator != NULL && allocator->free != NULL) + allocator->free(allocator->opaque, ptr); + else + free(ptr); + + return; +} + + +////////// +// Misc // +////////// + +extern size_t +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) +{ + const size_t in_avail = in_size - *in_pos; + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(in_avail, out_avail); + + memcpy(out + *out_pos, in + *in_pos, copy_size); + + *in_pos += copy_size; + *out_pos += copy_size; + + return copy_size; +} extern lzma_ret +lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_next_coder_init(filters[0].init, next, allocator); + + return filters[0].init == NULL + ? LZMA_OK : filters[0].init(next, allocator, filters); +} + + +extern void +lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +{ + if (next->init != (uintptr_t)(NULL)) { + // To avoid tiny end functions that simply call + // lzma_free(coder, allocator), we allow leaving next->end + // NULL and call lzma_free() here. + if (next->end != NULL) + next->end(next->coder, allocator); + else + lzma_free(next->coder, allocator); + + // Reset the variables so the we don't accidentally think + // that it is an already initialized coder. + *next = LZMA_NEXT_CODER_INIT; + } + + return; +} + + +////////////////////////////////////// +// External to internal API wrapper // +////////////////////////////////////// + +extern lzma_ret lzma_strm_init(lzma_stream *strm) { if (strm == NULL) @@ -191,10 +289,7 @@ extern LZMA_API void lzma_end(lzma_stream *strm) { if (strm != NULL && strm->internal != NULL) { - if (strm->internal->next.end != NULL) - strm->internal->next.end(strm->internal->next.coder, - strm->allocator); - + lzma_next_end(&strm->internal->next, strm->allocator); lzma_free(strm->internal, strm->allocator); strm->internal = NULL; } diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h index 4f30427d..81f2a9a4 100644 --- a/src/liblzma/common/common.h +++ b/src/liblzma/common/common.h @@ -3,7 +3,7 @@ /// \file common.h /// \brief Definitions common to the whole liblzma library // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -46,16 +46,32 @@ #define LZMA_BUFFER_SIZE 4096 +/// Start of internal Filter ID space. These IDs must never be used +/// in Streams. +#define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62) + + /// Internal helper filter used by Subblock decoder. It is mapped to an /// otherwise invalid Filter ID, which is impossible to get from any input /// file (even if malicious file). -#define LZMA_FILTER_SUBBLOCK_HELPER (UINT64_MAX - 2) +#define LZMA_FILTER_SUBBLOCK_HELPER (LZMA_FILTER_RESERVED_START + 1) + + +/// Supported flags that can be passed to lzma_stream_decoder() +/// or lzma_auto_decoder(). +#define LZMA_SUPPORTED_FLAGS \ + (LZMA_WARN_NO_CHECK \ + | LZMA_WARN_UNSUPPORTED_CHECK \ + | LZMA_TELL_CHECK \ + | LZMA_CONCATENATED) /////////// // Types // /////////// +/// Type of encoder/decoder specific data; the actual structure is defined +/// differently in different coders. typedef struct lzma_coder_s lzma_coder; typedef struct lzma_next_coder_s lzma_next_coder; @@ -63,10 +79,15 @@ typedef struct lzma_next_coder_s lzma_next_coder; typedef struct lzma_filter_info_s lzma_filter_info; +/// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +/// Type of a function to do some kind of coding work (filters, Stream, +/// Block encoders/decoders etc.). Some special coders use don't use both +/// input and output buffers, but for simplicity they still use this same +/// function prototype. typedef lzma_ret (*lzma_code_function)( lzma_coder *coder, lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, @@ -74,10 +95,24 @@ typedef lzma_ret (*lzma_code_function)( size_t *restrict out_pos, size_t out_size, lzma_action action); +/// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( lzma_coder *coder, lzma_allocator *allocator); +/// Raw coder validates and converts an array of lzma_filter structures to +/// an array of lzma_filter_info structures. This array is used with +/// lzma_next_filter_init to initialize the filter chain. +struct lzma_filter_info_s { + /// Pointer to function used to initialize the filter. + /// This is NULL to indicate end of array. + lzma_init_function init; + + /// Pointer to filter's options structure + void *options; +}; + + /// Hold data and function pointers of the next filter in the chain. struct lzma_next_coder_s { /// Pointer to coder-specific data @@ -85,28 +120,48 @@ struct lzma_next_coder_s { /// "Pointer" to init function. This is never called here. /// We need only to detect if we are initializing a coder - /// that was allocated earlier. See code.c and next_coder.c. + /// that was allocated earlier. See lzma_next_coder_init and + /// lzma_next_strm_init macros in this file. uintptr_t init; /// Pointer to function to do the actual coding lzma_code_function code; - /// Pointer to function to free lzma_next_coder.coder + /// Pointer to function to free lzma_next_coder.coder. This can + /// be NULL; in that case, lzma_free is called to free + /// lzma_next_coder.coder. lzma_end_function end; + + /// Pointer to function to return the type of the integrity check. + /// Most coders won't support this. + lzma_check (*see_check)(const lzma_coder *coder); + +// uint64_t (*memconfig)( +// lzma_coder *coder, uint64_t memlimit, bool change); }; + +/// Macro to initialize lzma_next_coder structure #define LZMA_NEXT_CODER_INIT \ (lzma_next_coder){ \ .coder = NULL, \ - .init = 0, \ + .init = (uintptr_t)(NULL), \ .code = NULL, \ .end = NULL, \ + .see_check = NULL, \ } +/// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to +/// this is stored in lzma_stream. struct lzma_internal_s { + /// The actual coder that should do something useful lzma_next_coder next; + /// Track the state of the coder. This is used to validate arguments + /// so that the actual coders can rely on e.g. that LZMA_SYNC_FLUSH + /// is used on every call to lzma_code until next.code has returned + /// LZMA_STREAM_END. enum { ISEQ_RUN, ISEQ_SYNC_FLUSH, @@ -116,33 +171,20 @@ struct lzma_internal_s { ISEQ_ERROR, } sequence; - bool supported_actions[4]; - bool allow_buf_error; + /// A copy of lzma_stream avail_in. This is used to verify that the + /// amount of input doesn't change once e.g. LZMA_FINISH has been + /// used. size_t avail_in; -}; - -struct lzma_filter_info_s { - /// Pointer to function used to initialize the filter. - /// This is NULL to indicate end of array. - lzma_init_function init; + /// Indicates which lzma_action values are allowed by next.code. + bool supported_actions[4]; - /// Pointer to filter's options structure - void *options; + /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was + /// made (no input consumed and no output produced by next.code). + bool allow_buf_error; }; -/* -typedef struct { - lzma_init_function init; - uint32_t (*input_alignment)(lzma_vli id, const void *options); - uint32_t (*output_alignment)(lzma_vli id, const void *options); - bool changes_uncompressed_size; - bool supports_eopm; -} lzma_filter_hook; -*/ - - /////////////// // Functions // /////////////// @@ -154,126 +196,69 @@ extern void *lzma_alloc(size_t size, lzma_allocator *allocator) /// Frees memory extern void lzma_free(void *ptr, lzma_allocator *allocator); -/// Initializes lzma_stream FIXME desc + +/// Allocates strm->internal if it is NULL, and initializes *strm and +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); -/// +/// Initializes the next filter in the chain, if any. This takes care of +/// freeing the memory of previously initialized filter if it is different +/// than the filter being initialized now. This way the actual filter +/// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -/// -extern void lzma_next_coder_end(lzma_next_coder *next, - lzma_allocator *allocator); +/// Frees the memory allocated for next->coder either using next->end or, +/// if next->end is NULL, using lzma_free. +extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); -/// \brief Wrapper for memcpy() -/// -/// This function copies as much data as possible from in[] to out[] and -/// updates *in_pos and *out_pos accordingly. -/// -static inline size_t -bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *restrict out, size_t *restrict out_pos, - size_t out_size) -{ - const size_t in_avail = in_size - *in_pos; - const size_t out_avail = out_size - *out_pos; - const size_t copy_size = MIN(in_avail, out_avail); - - memcpy(out + *out_pos, in + *in_pos, copy_size); - - *in_pos += copy_size; - *out_pos += copy_size; - - return copy_size; -} +/// Copy as much data as possible from in[] to out[] and update *in_pos +/// and *out_pos accordingly. Returns the number of bytes copied. +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); -/// \brief Initializing the next coder -/// -/// lzma_next_coder can point to different types of coders. The existing -/// coder may be different than what we are initializing now. In that case -/// we must git rid of the old coder first. Otherwise we reuse the existing -/// coder structure. +/// \brief Return if expression doesn't evaluate to LZMA_OK /// -#define lzma_next_coder_init2(next, allocator, cmpfunc, func, ...) \ +/// There are several situations where we want to return immediatelly +/// with the value of expr if it isn't LZMA_OK. This macro shortens +/// the code a little. +#define return_if_error(expr) \ do { \ - if ((uintptr_t)(&cmpfunc) != (next)->init) \ - lzma_next_coder_end(next, allocator); \ - const lzma_ret ret = func(next, __VA_ARGS__); \ - if (ret == LZMA_OK) { \ - (next)->init = (uintptr_t)(&cmpfunc); \ - assert((next)->code != NULL); \ - assert((next)->end != NULL); \ - } else { \ - lzma_next_coder_end(next, allocator); \ - } \ - return ret; \ + const lzma_ret ret_ = (expr); \ + if (ret_ != LZMA_OK) \ + return ret_; \ } while (0) -/// \brief Initializing lzma_next_coder -/// -/// Call the initialization function, which must take at least one -/// argument in addition to lzma_next_coder and lzma_allocator. -#define lzma_next_coder_init(func, next, allocator, ...) \ - lzma_next_coder_init2(next, allocator, \ - func, func, allocator, __VA_ARGS__) - -/// \brief Initializing lzma_next_coder -/// -/// Call the initialization function, which takes no other arguments than -/// lzma_next_coder and lzma_allocator. -#define lzma_next_coder_init0(func, next, allocator) \ - lzma_next_coder_init2(next, allocator, func, func, allocator) - -/// \brief Initializing lzma_stream -/// -/// lzma_strm initialization with more detailed options. -#define lzma_next_strm_init2(strm, cmpfunc, func, ...) \ +/// If next isn't already initialized, free the previous coder. Then mark +/// that next is _possibly_ initialized for the coder using this macro. +/// "Possibly" means that if e.g. allocation of next->coder fails, the +/// structure isn't actually initialized for this coder, but leaving +/// next->init to func is still OK. +#define lzma_next_coder_init(func, next, allocator) \ do { \ - lzma_ret ret = lzma_strm_init(strm); \ - if (ret != LZMA_OK) \ - return ret; \ - if ((uintptr_t)(&cmpfunc) != (strm)->internal->next.init) \ - lzma_next_coder_end(\ - &(strm)->internal->next, (strm)->allocator); \ - ret = func(&(strm)->internal->next, __VA_ARGS__); \ - if (ret != LZMA_OK) { \ - lzma_end(strm); \ - return ret; \ - } \ - (strm)->internal->next.init = (uintptr_t)(&cmpfunc); \ - assert((strm)->internal->next.code != NULL); \ - assert((strm)->internal->next.end != NULL); \ + if ((uintptr_t)(&func) != (next)->init) \ + lzma_next_end(next, allocator); \ + (next)->init = (uintptr_t)(&func); \ } while (0) -/// \brief Initializing lzma_stream -/// -/// Call the initialization function, which must take at least one -/// argument in addition to lzma_next_coder and lzma_allocator. -#define lzma_next_strm_init(strm, func, ...) \ - lzma_next_strm_init2(strm, func, func, (strm)->allocator, __VA_ARGS__) -/// \brief Initializing lzma_stream -/// -/// Call the initialization function, which takes no other arguments than -/// lzma_next_coder and lzma_allocator. -#define lzma_next_strm_init0(strm, func) \ - lzma_next_strm_init2(strm, func, func, (strm)->allocator) - - -/// \brief Return if expression doesn't evaluate to LZMA_OK -/// -/// There are several situations where we want to return immediatelly -/// with the value of expr if it isn't LZMA_OK. This macro shortens -/// the code a bit. -/// -#define return_if_error(expr) \ +/// Initializes lzma_strm and calls func() to initialize strm->internal->next. +/// (The function being called will use lzma_next_coder_init()). If +/// initialization fails, memory that wasn't freed by func() is freed +/// along strm->internal. +#define lzma_next_strm_init(func, strm, ...) \ do { \ - const lzma_ret ret_ = expr; \ - if (ret_ != LZMA_OK) \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ return ret_; \ + } \ } while (0) #endif diff --git a/src/liblzma/common/easy.c b/src/liblzma/common/easy.c index 6c258204..ae0e4f74 100644 --- a/src/liblzma/common/easy.c +++ b/src/liblzma/common/easy.c @@ -25,12 +25,12 @@ struct lzma_coder_s { /// We need to keep the filters array available in case /// LZMA_FULL_FLUSH is used. - lzma_options_filter filters[5]; + lzma_filter filters[5]; }; static bool -easy_set_filters(lzma_options_filter *filters, uint32_t level) +easy_set_filters(lzma_filter *filters, uint32_t level) { bool error = false; @@ -38,9 +38,9 @@ easy_set_filters(lzma_options_filter *filters, uint32_t level) // TODO FIXME Use Subblock or LZMA2 with no compression. error = true; -#ifdef HAVE_FILTER_LZMA +#ifdef HAVE_ENCODER_LZMA2 } else if (level <= 9) { - filters[0].id = LZMA_FILTER_LZMA; + filters[0].id = LZMA_FILTER_LZMA2; filters[0].options = (void *)(&lzma_preset_lzma[level - 1]); filters[1].id = LZMA_VLI_VALUE_UNKNOWN; #endif @@ -68,7 +68,7 @@ easy_encode(lzma_coder *coder, lzma_allocator *allocator, static void easy_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->stream_encoder, allocator); + lzma_next_end(&coder->stream_encoder, allocator); lzma_free(coder, allocator); return; } @@ -78,6 +78,8 @@ static lzma_ret easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_easy_level level) { + lzma_next_coder_init(easy_encoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -100,7 +102,7 @@ easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) { - lzma_next_strm_init(strm, easy_encoder_init, level); + lzma_next_strm_init(easy_encoder_init, strm, level); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; @@ -114,9 +116,9 @@ lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) extern LZMA_API uint32_t lzma_easy_memory_usage(lzma_easy_level level) { - lzma_options_filter filters[5]; + lzma_filter filters[5]; if (easy_set_filters(filters, level)) return UINT32_MAX; - return lzma_memory_usage(filters, true); + return lzma_memusage_encoder(filters); } diff --git a/src/liblzma/common/features.c b/src/liblzma/common/features.c deleted file mode 100644 index a02949d9..00000000 --- a/src/liblzma/common/features.c +++ /dev/null @@ -1,66 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file features.c -/// \brief Information about features enabled at compile time -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - - -static const lzma_vli filters[] = { -#ifdef HAVE_FILTER_SUBBLOCK - LZMA_FILTER_SUBBLOCK, -#endif - -#ifdef HAVE_FILTER_X86 - LZMA_FILTER_X86, -#endif - -#ifdef HAVE_FILTER_POWERPC - LZMA_FILTER_POWERPC, -#endif - -#ifdef HAVE_FILTER_IA64 - LZMA_FILTER_IA64, -#endif - -#ifdef HAVE_FILTER_ARM - LZMA_FILTER_ARM, -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - LZMA_FILTER_ARMTHUMB, -#endif - -#ifdef HAVE_FILTER_SPARC - LZMA_FILTER_SPARC, -#endif - -#ifdef HAVE_FILTER_DELTA - LZMA_FILTER_DELTA, -#endif - -#ifdef HAVE_FILTER_LZMA - LZMA_FILTER_LZMA, -#endif - - LZMA_VLI_VALUE_UNKNOWN -}; - - -LZMA_API const lzma_vli *const lzma_available_filter_encoders = filters; - -LZMA_API const lzma_vli *const lzma_available_filter_decoders = filters; diff --git a/src/liblzma/common/filter_common.c b/src/liblzma/common/filter_common.c new file mode 100644 index 00000000..886ddb53 --- /dev/null +++ b/src/liblzma/common/filter_common.c @@ -0,0 +1,262 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_common.h" + + +static const struct { + /// Filter ID + lzma_vli id; + + /// True if it is OK to use this filter as non-last filter in + /// the chain. + bool non_last_ok; + + /// True if it is OK to use this filter as the last filter in + /// the chain. + bool last_ok; + + /// True if the filter may change the size of the data (that is, the + /// amount of encoded output can be different than the amount of + /// uncompressed input). + bool changes_size; + +} features[] = { +#if defined (HAVE_ENCODER_LZMA) || defined(HAVE_DECODER_LZMA) + { + .id = LZMA_FILTER_LZMA, + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .id = LZMA_FILTER_LZMA2, + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK) + { + .id = LZMA_FILTER_SUBBLOCK, + .non_last_ok = true, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .id = LZMA_FILTER_X86, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) + { + .id = LZMA_FILTER_POWERPC, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .id = LZMA_FILTER_IA64, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) + { + .id = LZMA_FILTER_ARM, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) + { + .id = LZMA_FILTER_ARMTHUMB, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) + { + .id = LZMA_FILTER_SPARC, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) + { + .id = LZMA_FILTER_DELTA, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif + { + .id = LZMA_VLI_VALUE_UNKNOWN + } +}; + + +static lzma_ret +validate_chain(const lzma_filter *filters, size_t *count) +{ + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_PROG_ERROR; + + // Number of non-last filters that may change the size of the data + // significantly (that is, more than 1-2 % or so). + size_t changes_size_count = 0; + + // True if it is OK to add a new filter after the current filter. + bool non_last_ok = true; + + // True if the last filter in the given chain is actually usable as + // the last filter. Only filters that support embedding End of Payload + // Marker can be used as the last filter in the chain. + bool last_ok = false; + + size_t i = 0; + do { + size_t j; + for (j = 0; filters[i].id != features[j].id; ++j) + if (features[j].id == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_HEADER_ERROR; + + // If the previous filter in the chain cannot be a non-last + // filter, the chain is invalid. + if (!non_last_ok) + return LZMA_HEADER_ERROR; + + non_last_ok = features[j].non_last_ok; + last_ok = features[j].last_ok; + changes_size_count += features[j].changes_size; + + } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN); + + // There must be 1-4 filters. The last filter must be usable as + // the last filter in the chain. At maximum of three filters are + // allowed to change the size of the data. + if (i > LZMA_BLOCK_FILTERS_MAX || !last_ok || changes_size_count > 3) + return LZMA_HEADER_ERROR; + + *count = i; + return LZMA_OK; +} + + +extern lzma_ret +lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options, + lzma_filter_find coder_find, bool is_encoder) +{ + // Do some basic validation and get the number of filters. + size_t count; + return_if_error(validate_chain(options, &count)); + + // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_BLOCK_FILTERS_MAX + 1]; + if (is_encoder) { + for (size_t i = 0; i < count; ++i) { + // The order of the filters is reversed in the + // encoder. It allows more efficient handling + // of the uncompressed data. + const size_t j = count - i - 1; + + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_HEADER_ERROR; + + filters[j].init = fc->init; + filters[j].options = options[i].options; + } + } else { + for (size_t i = 0; i < count; ++i) { + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_HEADER_ERROR; + + filters[i].init = fc->init; + filters[i].options = options[i].options; + } + } + + // Terminate the array. + filters[count].init = NULL; + + // Initialize the filters. + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); + if (ret != LZMA_OK) + lzma_next_end(next, allocator); + + return ret; +} + + +extern uint64_t +lzma_memusage_coder(lzma_filter_find coder_find, + const lzma_filter *filters) +{ + // The chain has to have at least one filter. + if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN) + return UINT64_MAX; + + uint64_t total = 0; + size_t i = 0; + + do { + const lzma_filter_coder *const fc + = coder_find(filters[i].id); + if (fc == NULL) + return UINT64_MAX; // Unsupported Filter ID + + if (fc->memusage == NULL) { + // This filter doesn't have a function to calculate + // the memory usage. Such filters need only little + // memory, so we use 1 KiB as a good estimate. + total += 1024; + } else { + // Call the filter-specific memory usage calculation + // function. + const uint64_t usage + = fc->memusage(filters[i].options); + if (usage == UINT64_MAX) + return UINT64_MAX; // Invalid options + + total += usage; + } + } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN); + + // Add some fixed amount of extra. It's to compensate memory usage + // of Stream, Block etc. coders, malloc() overhead, stack etc. + return total + (1U << 15); +} diff --git a/src/liblzma/common/filter_common.h b/src/liblzma/common/filter_common.h new file mode 100644 index 00000000..9def50b9 --- /dev/null +++ b/src/liblzma/common/filter_common.h @@ -0,0 +1,52 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_COMMON_H +#define LZMA_FILTER_COMMON_H + +#include "common.h" + + +/// Both lzma_filter_encoder and lzma_filter_decoder begin with these members. +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + +} lzma_filter_coder; + + +typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); + + +extern lzma_ret lzma_raw_coder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, + lzma_filter_find coder_find, bool is_encoder); + + +extern uint64_t lzma_memusage_coder(lzma_filter_find coder_find, + const lzma_filter *filters); + + +#endif diff --git a/src/liblzma/common/filter_decoder.c b/src/liblzma/common/filter_decoder.c new file mode 100644 index 00000000..9fe94f7b --- /dev/null +++ b/src/liblzma/common/filter_decoder.c @@ -0,0 +1,236 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_decoder.h" +#include "filter_common.h" +#include "lzma_decoder.h" +#include "lzma2_decoder.h" +#include "subblock_decoder.h" +#include "subblock_decoder_helper.h" +#include "simple_decoder.h" +#include "delta_decoder.h" + + +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Decodes Filter Properties. + /// + /// \return - LZMA_OK: Properties decoded successfully. + /// - LZMA_HEADER_ERROR: Unsupported properties + /// - LZMA_MEM_ERROR: Memory allocation failed. + lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + +} lzma_filter_decoder; + + +static const lzma_vli ids[] = { +#ifdef HAVE_DECODER_LZMA + LZMA_FILTER_LZMA, +#endif + +#ifdef HAVE_DECODER_LZMA2 + LZMA_FILTER_LZMA2, +#endif + +#ifdef HAVE_DECODER_SUBBLOCK + LZMA_FILTER_SUBBLOCK, + LZMA_FILTER_SUBBLOCK_HELPER, +#endif + +#ifdef HAVE_DECODER_X86 + LZMA_FILTER_X86, +#endif + +#ifdef HAVE_DECODER_POWERPC + LZMA_FILTER_POWERPC, +#endif + +#ifdef HAVE_DECODER_IA64 + LZMA_FILTER_IA64, +#endif + +#ifdef HAVE_DECODER_ARM + LZMA_FILTER_ARM, +#endif + +#ifdef HAVE_DECODER_ARMTHUMB + LZMA_FILTER_ARMTHUMB, +#endif + +#ifdef HAVE_DECODER_SPARC + LZMA_FILTER_SPARC, +#endif + +#ifdef HAVE_DECODER_DELTA + LZMA_FILTER_DELTA, +#endif + + LZMA_VLI_VALUE_UNKNOWN +}; + + +// Using a pointer to avoid putting the size of the array to API/ABI. +LZMA_API const lzma_vli *const lzma_filter_decoders = ids; + + +// These must be in the same order as ids[]. +static const lzma_filter_decoder funcs[] = { +#ifdef HAVE_DECODER_LZMA + { + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SUBBLOCK + { + .init = &lzma_subblock_decoder_init, +// .memusage = &lzma_subblock_decoder_memusage, + .props_decode = NULL, + }, + { + .init = &lzma_subblock_decoder_helper_init, + .memusage = NULL, + .props_decode = NULL, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_POWERPC + { + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARM + { + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARMTHUMB + { + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SPARC + { + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_DELTA + { + .init = &lzma_delta_decoder_init, + .memusage = NULL, + .props_decode = &lzma_delta_props_decode, + }, +#endif +}; + + +static const lzma_filter_decoder * +decoder_find(lzma_vli id) +{ + for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i) + if (ids[i] == id) + return funcs + i; + + return NULL; +} + + +extern lzma_ret +lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&decoder_find), false); +} + + +extern LZMA_API lzma_ret +lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} + + +extern LZMA_API uint64_t +lzma_memusage_decoder(const lzma_filter *filters) +{ + return lzma_memusage_coder( + (lzma_filter_find)(&decoder_find), filters); +} + + +extern LZMA_API lzma_ret +lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + // Make it always NULL so that the caller can always safely free() it. + filter->options = NULL; + + const lzma_filter_decoder *const fd = decoder_find(filter->id); + if (fd == NULL) + return LZMA_HEADER_ERROR; + + if (fd->props_decode == NULL) + return props_size == 0 ? LZMA_OK : LZMA_HEADER_ERROR; + + return fd->props_decode( + &filter->options, allocator, props, props_size); +} diff --git a/src/liblzma/common/filter_decoder.h b/src/liblzma/common/filter_decoder.h new file mode 100644 index 00000000..33e491d1 --- /dev/null +++ b/src/liblzma/common/filter_decoder.h @@ -0,0 +1,35 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_DECODER_H +#define LZMA_FILTER_DECODER_H + +#include "common.h" + +// FIXME !!! Public API +extern lzma_ret lzma_properties_decode( + lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + + +extern lzma_ret lzma_raw_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options); + +#endif diff --git a/src/liblzma/common/filter_encoder.c b/src/liblzma/common/filter_encoder.c new file mode 100644 index 00000000..55862e18 --- /dev/null +++ b/src/liblzma/common/filter_encoder.c @@ -0,0 +1,308 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "filter_common.h" +#include "lzma_encoder.h" +#include "lzma2_encoder.h" +#include "subblock_encoder.h" +#include "simple_encoder.h" +#include "delta_encoder.h" + + +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Calculates the minimum sane size for Blocks (or other types of + /// chunks) to which the input data can be splitted to make + /// multithreaded encoding possible. If this is NULL, it is assumed + /// that the encoder is fast enough with single thread. + lzma_vli (*chunk_size)(const void *options); + + /// Tells the size of the Filter Properties field. If options are + /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed + /// is used. + lzma_ret (*props_size_get)(uint32_t *size, const void *options); + uint32_t props_size_fixed; + + /// Encodes Filter Properties. + /// + /// \return - LZMA_OK: Properties encoded sucessfully. + /// - LZMA_HEADER_ERROR: Unsupported options + /// - LZMA_PROG_ERROR: Invalid options or not enough + /// output space + lzma_ret (*props_encode)(const void *options, uint8_t *out); + +} lzma_filter_encoder; + + +static const lzma_vli ids[] = { +#ifdef HAVE_ENCODER_LZMA + LZMA_FILTER_LZMA, +#endif + +#ifdef HAVE_ENCODER_LZMA2 + LZMA_FILTER_LZMA2, +#endif + +#ifdef HAVE_ENCODER_SUBBLOCK + LZMA_FILTER_SUBBLOCK, +#endif + +#ifdef HAVE_ENCODER_X86 + LZMA_FILTER_X86, +#endif + +#ifdef HAVE_ENCODER_POWERPC + LZMA_FILTER_POWERPC, +#endif + +#ifdef HAVE_ENCODER_IA64 + LZMA_FILTER_IA64, +#endif + +#ifdef HAVE_ENCODER_ARM + LZMA_FILTER_ARM, +#endif + +#ifdef HAVE_ENCODER_ARMTHUMB + LZMA_FILTER_ARMTHUMB, +#endif + +#ifdef HAVE_ENCODER_SPARC + LZMA_FILTER_SPARC, +#endif + +#ifdef HAVE_ENCODER_DELTA + LZMA_FILTER_DELTA, +#endif + + LZMA_VLI_VALUE_UNKNOWN +}; + + +// Using a pointer to avoid putting the size of the array to API/ABI. +LZMA_API const lzma_vli *const lzma_filter_encoders = ids; + + +// These must be in the same order as ids[]. +static const lzma_filter_encoder funcs[] = { +#ifdef HAVE_ENCODER_LZMA + { + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_LZMA2 + { + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SUBBLOCK + { + .init = &lzma_subblock_encoder_init, +// .memusage = &lzma_subblock_encoder_memusage, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 0, + .props_encode = NULL, + }, +#endif +#ifdef HAVE_ENCODER_X86 + { + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_POWERPC + { + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_IA64 + { + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARM + { + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARMTHUMB + { + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SPARC + { + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_DELTA + { + .init = &lzma_delta_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, + }, +#endif +}; + + +static const lzma_filter_encoder * +encoder_find(lzma_vli id) +{ + for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i) + if (ids[i] == id) + return funcs + i; + + return NULL; +} + + +extern lzma_ret +lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&encoder_find), true); +} + + +extern LZMA_API lzma_ret +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_coder_init, strm, options, + (lzma_filter_find)(&encoder_find), true); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +extern LZMA_API uint64_t +lzma_memusage_encoder(const lzma_filter *filters) +{ + return lzma_memusage_coder( + (lzma_filter_find)(&encoder_find), filters); +} + + +extern LZMA_API lzma_vli +lzma_chunk_size(const lzma_filter *filters) +{ + uint64_t max = 0; + + for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { + const lzma_filter_encoder *const fe + = encoder_find(filters[i].id); + if (fe->chunk_size != NULL) { + const lzma_vli size + = fe->chunk_size(filters[i].options); + if (size == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_VLI_VALUE_UNKNOWN; + + if (size > max) + max = size; + } + } + + return max; +} + + +extern LZMA_API lzma_ret +lzma_properties_size(uint32_t *size, const lzma_filter *filter) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) { + // Unknown filter - if the Filter ID is a proper VLI, + // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR, + // because it's possible that we just don't have support + // compiled in for the requested filter. + return filter->id <= LZMA_VLI_VALUE_MAX + ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR; + } + + if (fe->props_size_get == NULL) { + // No props_size() function, use props_size_fixed. + *size = fe->props_size_fixed; + return LZMA_OK; + } + + return fe->props_size_get(size, filter->options); +} + + +extern LZMA_API lzma_ret +lzma_properties_encode(const lzma_filter *filter, uint8_t *props) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) + return LZMA_PROG_ERROR; + + if (fe->props_encode == NULL) + return LZMA_OK; + + return fe->props_encode(filter->options, props); +} diff --git a/src/liblzma/common/filter_encoder.h b/src/liblzma/common/filter_encoder.h new file mode 100644 index 00000000..b2bf851d --- /dev/null +++ b/src/liblzma/common/filter_encoder.h @@ -0,0 +1,38 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_encoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_ENCODER_H +#define LZMA_FILTER_ENCODER_H + +#include "common.h" + + +// FIXME !!! Public API +extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +extern lzma_ret lzma_properties_size( + uint32_t *size, const lzma_filter *filter); +extern lzma_ret lzma_properties_encode( + const lzma_filter *filter, uint8_t *props); + + +extern lzma_ret lzma_raw_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options); + +#endif diff --git a/src/liblzma/common/filter_flags_decoder.c b/src/liblzma/common/filter_flags_decoder.c index 498b2ad6..c2247312 100644 --- a/src/liblzma/common/filter_flags_decoder.c +++ b/src/liblzma/common/filter_flags_decoder.c @@ -17,192 +17,37 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" -#include "lzma_decoder.h" - - -#ifdef HAVE_FILTER_SUBBLOCK -static lzma_ret -properties_subblock(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props lzma_attribute((unused)), - size_t prop_size lzma_attribute((unused))) -{ - if (prop_size != 0) - return LZMA_HEADER_ERROR; - - options->options = lzma_alloc( - sizeof(lzma_options_subblock), allocator); - if (options->options == NULL) - return LZMA_MEM_ERROR; - - ((lzma_options_subblock *)(options->options))->allow_subfilters = true; - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_SIMPLE -static lzma_ret -properties_simple(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - if (prop_size == 0) - return LZMA_OK; - - if (prop_size != 4) - return LZMA_HEADER_ERROR; - - lzma_options_simple *simple = lzma_alloc( - sizeof(lzma_options_simple), allocator); - if (simple == NULL) - return LZMA_MEM_ERROR; - - simple->start_offset = integer_read_32(props); - - // Don't leave an options structure allocated if start_offset is zero. - if (simple->start_offset == 0) - lzma_free(simple, allocator); - else - options->options = simple; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_DELTA -static lzma_ret -properties_delta(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - if (prop_size != 1) - return LZMA_HEADER_ERROR; - - options->options = lzma_alloc(sizeof(lzma_options_delta), allocator); - if (options->options == NULL) - return LZMA_MEM_ERROR; - - ((lzma_options_delta *)(options->options))->distance - = (uint32_t)(props[0]) + 1; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_LZMA -static lzma_ret -properties_lzma(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - // LZMA properties are always two bytes (at least for now). - if (prop_size != 2) - return LZMA_HEADER_ERROR; - - lzma_options_lzma *lzma = lzma_alloc( - sizeof(lzma_options_lzma), allocator); - if (lzma == NULL) - return LZMA_MEM_ERROR; - - // Decode lc, lp, and pb. - if (lzma_lzma_decode_properties(lzma, props[0])) - goto error; - - // Check that reserved bits are unset. - if (props[1] & 0xC0) - goto error; - - // Decode the dictionary size. - // FIXME The specification says that maximum is 4 GiB. - if (props[1] > 36) - goto error; -#if LZMA_DICTIONARY_SIZE_MAX != UINT32_C(1) << 30 -# error Update the if()-condition a few lines -# error above to match LZMA_DICTIONARY_SIZE_MAX. -#endif - - lzma->dictionary_size = 2 | (props[1] & 1); - lzma->dictionary_size <<= props[1] / 2 + 11; - - options->options = lzma; - return LZMA_OK; - -error: - lzma_free(lzma, allocator); - return LZMA_HEADER_ERROR; -} -#endif +#include "filter_decoder.h" extern LZMA_API lzma_ret lzma_filter_flags_decode( - lzma_options_filter *options, lzma_allocator *allocator, + lzma_filter *filter, lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { // Set the pointer to NULL so the caller can always safely free it. - options->options = NULL; + filter->options = NULL; // Filter ID - return_if_error(lzma_vli_decode(&options->id, NULL, + return_if_error(lzma_vli_decode(&filter->id, NULL, in, in_pos, in_size)); + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_DATA_ERROR; + // Size of Properties - lzma_vli prop_size; - return_if_error(lzma_vli_decode(&prop_size, NULL, + lzma_vli props_size; + return_if_error(lzma_vli_decode(&props_size, NULL, in, in_pos, in_size)); - // Check that we have enough input. - if (prop_size > in_size - *in_pos) + // Filter Properties + if (in_size - *in_pos < props_size) return LZMA_DATA_ERROR; - // Determine the function to decode the properties. - lzma_ret (*get_properties)(lzma_options_filter *options, - lzma_allocator *allocator, const uint8_t *props, - size_t prop_size); + const lzma_ret ret = lzma_properties_decode( + filter, allocator, in + *in_pos, props_size); - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - get_properties = &properties_subblock; - break; -#endif -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - get_properties = &properties_simple; - break; -#endif -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - get_properties = &properties_delta; - break; -#endif -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - get_properties = &properties_lzma; - break; -#endif - default: - return LZMA_HEADER_ERROR; - } + *in_pos += props_size; - const uint8_t *props = in + *in_pos; - *in_pos += prop_size; - return get_properties(options, allocator, props, prop_size); + return ret; } diff --git a/src/liblzma/common/filter_flags_encoder.c b/src/liblzma/common/filter_flags_encoder.c index 45fbbb00..46464c0f 100644 --- a/src/liblzma/common/filter_flags_encoder.c +++ b/src/liblzma/common/filter_flags_encoder.c @@ -17,267 +17,46 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" -#include "lzma_encoder.h" -#include "fastpos.h" - - -/// Calculate the size of the Filter Properties field -static lzma_ret -get_properties_size(uint32_t *size, const lzma_options_filter *options) -{ - lzma_ret ret = LZMA_OK; - - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - *size = 0; - break; -#endif - -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - if (options->options == NULL || ((const lzma_options_simple *)( - options->options))->start_offset == 0) - *size = 0; - else - *size = 4; - break; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - *size = 1; - break; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - *size = 2; - break; -#endif - - default: - // Unknown filter - if the Filter ID is a proper VLI, - // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR, - // because it's possible that we just don't have support - // compiled in for the requested filter. - ret = options->id <= LZMA_VLI_VALUE_MAX - ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR; - break; - } - - return ret; -} +#include "filter_encoder.h" extern LZMA_API lzma_ret -lzma_filter_flags_size(uint32_t *size, const lzma_options_filter *options) -{ - // Get size of Filter Properties. This also validates the Filter ID. - uint32_t prop_size; - return_if_error(get_properties_size(&prop_size, options)); - - // Calculate the size of the Filter ID and Size of Properties fields. - // These cannot fail since get_properties_size() already succeeded. - *size = lzma_vli_size(options->id) + lzma_vli_size(prop_size) - + prop_size; - - return LZMA_OK; -} - - -#ifdef HAVE_FILTER_SIMPLE -/// Encodes Filter Properties of the so called simple filters -static lzma_ret -properties_simple(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_simple *options) -{ - if (options == NULL || options->start_offset == 0) - return LZMA_OK; - - if (out_size - *out_pos < 4) - return LZMA_PROG_ERROR; - - integer_write_32(out + *out_pos, options->start_offset); - *out_pos += 4; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_DELTA -/// Encodes Filter Properties of the Delta filter -static lzma_ret -properties_delta(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_delta *options) -{ - if (options == NULL) - return LZMA_PROG_ERROR; - - // It's possible that newer liblzma versions will support larger - // distance values. - if (options->distance < LZMA_DELTA_DISTANCE_MIN - || options->distance > LZMA_DELTA_DISTANCE_MAX) - return LZMA_HEADER_ERROR; - - if (out_size - *out_pos < 1) - return LZMA_PROG_ERROR; - - out[*out_pos] = options->distance - LZMA_DELTA_DISTANCE_MIN; - ++*out_pos; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_LZMA -/// Encodes LZMA Properties and Dictionary Flags (two bytes) -static lzma_ret -properties_lzma(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_lzma *options) +lzma_filter_flags_size(uint32_t *size, const lzma_filter *filter) { - if (options == NULL) - return LZMA_PROG_ERROR; - - if (out_size - *out_pos < 2) - return LZMA_PROG_ERROR; - - // LZMA Properties - if (lzma_lzma_encode_properties(options, out + *out_pos)) - return LZMA_HEADER_ERROR; - - ++*out_pos; + return_if_error(lzma_properties_size(size, filter)); - // Dictionary flags - // - // Dictionary size is encoded using similar encoding that is used - // internally by LZMA. - // - // This won't work if dictionary size can be zero: -# if LZMA_DICTIONARY_SIZE_MIN < 1 -# error LZMA_DICTIONARY_SIZE_MIN cannot be zero. -# endif - - uint32_t d = options->dictionary_size; - - // Validate it: - if (d < LZMA_DICTIONARY_SIZE_MIN || d > LZMA_DICTIONARY_SIZE_MAX) - return LZMA_HEADER_ERROR; - - // Round up to to the next 2^n or 2^n + 2^(n - 1) depending on which - // one is the next: - --d; - d |= d >> 2; - d |= d >> 3; - d |= d >> 4; - d |= d >> 8; - d |= d >> 16; - ++d; - - // Get the highest two bits using the proper encoding: - out[*out_pos] = get_pos_slot(d) - 24; - ++*out_pos; + // lzma_properties_size() validates the Filter ID as a side-effect, + // so we know that it is a valid VLI. + *size += lzma_vli_size(filter->id) + lzma_vli_size(*size); return LZMA_OK; } -#endif extern LZMA_API lzma_ret -lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_filter *options) +lzma_filter_flags_encode(const lzma_filter *filter, + uint8_t *out, size_t *out_pos, size_t out_size) { - // Minimum output is one byte (everything fits into Misc). - // The caller should have checked that there is enough output space, - // so we return LZMA_PROG_ERROR instead of LZMA_BUF_ERROR. - if (*out_pos >= out_size) - return LZMA_PROG_ERROR; - - // Get size of Filter Properties. - uint32_t prop_size; - return_if_error(get_properties_size(&prop_size, options)); - // Filter ID - return_if_error(lzma_vli_encode(options->id, NULL, + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_HEADER_ERROR; + + return_if_error(lzma_vli_encode(filter->id, NULL, out, out_pos, out_size)); // Size of Properties - return_if_error(lzma_vli_encode(prop_size, NULL, + uint32_t props_size; + return_if_error(lzma_properties_size(&props_size, filter)); + return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); // Filter Properties - lzma_ret ret; - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - assert(prop_size == 0); - ret = LZMA_OK; - break; -#endif - -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - ret = properties_simple(out, out_pos, out_size, - options->options); - break; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - ret = properties_delta(out, out_pos, out_size, - options->options); - break; -#endif + if (out_size - *out_pos < props_size) + return LZMA_PROG_ERROR; -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - ret = properties_lzma(out, out_pos, out_size, - options->options); - break; -#endif + return_if_error(lzma_properties_encode(filter, out + *out_pos)); - default: - assert(0); - ret = LZMA_PROG_ERROR; - break; - } + *out_pos += props_size; - return ret; + return LZMA_OK; } diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c index 1635948c..ae66595a 100644 --- a/src/liblzma/common/index_decoder.c +++ b/src/liblzma/common/index_decoder.c @@ -201,6 +201,8 @@ static lzma_ret index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index **i) { + lzma_next_coder_init(index_decoder_init, next, allocator); + if (i == NULL) return LZMA_PROG_ERROR; @@ -231,20 +233,10 @@ index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -/* -extern lzma_ret -lzma_index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_index **i) -{ - lzma_next_coder_init(index_decoder_init, next, allocator, i); -} -*/ - - extern LZMA_API lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i) { - lzma_next_strm_init(strm, index_decoder_init, i); + lzma_next_strm_init(index_decoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; diff --git a/src/liblzma/common/index_encoder.c b/src/liblzma/common/index_encoder.c index 5a7d8c8c..3005f835 100644 --- a/src/liblzma/common/index_encoder.c +++ b/src/liblzma/common/index_encoder.c @@ -176,10 +176,12 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) } -static lzma_ret -index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +extern lzma_ret +lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index *i) { + lzma_next_coder_init(lzma_index_encoder_init, next, allocator); + if (i == NULL) return LZMA_PROG_ERROR; @@ -203,18 +205,10 @@ index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_index *i) -{ - lzma_next_coder_init(index_encoder_init, next, allocator, i); -} - - extern LZMA_API lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i) { - lzma_next_strm_init(strm, index_encoder_init, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; diff --git a/src/liblzma/common/index_hash.c b/src/liblzma/common/index_hash.c index 35dea41f..dc533f9e 100644 --- a/src/liblzma/common/index_hash.c +++ b/src/liblzma/common/index_hash.c @@ -36,7 +36,7 @@ typedef struct { lzma_vli index_list_size; /// Check calculated from Total Sizes and Uncompressed Sizes. - lzma_check check; + lzma_check_state check; } lzma_index_hash_info; @@ -300,9 +300,9 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // Finish the hashes and compare them. lzma_check_finish(&index_hash->blocks.check, LZMA_CHECK_BEST); lzma_check_finish(&index_hash->records.check, LZMA_CHECK_BEST); - if (memcmp(index_hash->blocks.check.buffer, - index_hash->records.check.buffer, - lzma_check_sizes[LZMA_CHECK_BEST]) != 0) + if (memcmp(index_hash->blocks.check.buffer.u8, + index_hash->records.check.buffer.u8, + lzma_check_size(LZMA_CHECK_BEST)) != 0) return LZMA_DATA_ERROR; // Finish the CRC32 calculation. diff --git a/src/liblzma/common/init_encoder.c b/src/liblzma/common/init_encoder.c index 8a1644be..c5f12a91 100644 --- a/src/liblzma/common/init_encoder.c +++ b/src/liblzma/common/init_encoder.c @@ -31,7 +31,7 @@ lzma_init_encoder(void) lzma_init_check(); -#if defined(HAVE_SMALL) && defined(HAVE_ENCODER) && defined(HAVE_FILTER_LZMA) +#if defined(HAVE_SMALL) && defined(HAVE_ENCODER_LZMA) lzma_rc_init(); #endif diff --git a/src/liblzma/common/memory_usage.c b/src/liblzma/common/memory_usage.c deleted file mode 100644 index 8244c404..00000000 --- a/src/liblzma/common/memory_usage.c +++ /dev/null @@ -1,112 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file memory_usage.c -/// \brief Calculate rough amount of memory required by filters -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" -#include "lz_encoder.h" -#include "lzma_literal.h" - - -static uint64_t -get_usage(const lzma_options_filter *filter, bool is_encoder) -{ - uint64_t ret; - - switch (filter->id) { - case LZMA_FILTER_X86: - case LZMA_FILTER_POWERPC: - case LZMA_FILTER_IA64: - case LZMA_FILTER_ARM: - case LZMA_FILTER_ARMTHUMB: - case LZMA_FILTER_SPARC: - case LZMA_FILTER_DELTA: - // These don't require any significant amount of memory. - ret = 0; - break; - - case LZMA_FILTER_SUBBLOCK: - if (is_encoder) { - const lzma_options_subblock *options = filter->options; - ret = options->subblock_data_size; - } else { - ret = 0; - } - break; - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: { - const lzma_options_lzma *options = filter->options; - - // Literal coder - this can be signficant if both values are - // big, or if sizeof(probability) is big. - ret = literal_states(options->literal_context_bits, - options->literal_pos_bits) * LIT_SIZE - * sizeof(probability); - - // Dictionary base size - ret += options->dictionary_size; - - if (is_encoder) { -# ifdef HAVE_ENCODER - // This is rough, but should be accurate enough - // in practice. - ret += options->dictionary_size / 2; - - uint32_t dummy1; - uint32_t dummy2; - uint32_t num_items; - if (lzma_lz_encoder_hash_properties( - options->match_finder, - options->dictionary_size, - &dummy1, &dummy2, &num_items)) - return UINT64_MAX; - - ret += (uint64_t)(num_items) * sizeof(uint32_t); -# else - return UINT64_MAX; -# endif - } - - break; - } -#endif - - default: - return UINT64_MAX; - } - - return ret; -} - - -extern LZMA_API uint32_t -lzma_memory_usage(const lzma_options_filter *filters, lzma_bool is_encoder) -{ - uint64_t usage = 0; - - for (size_t i = 0; filters[i].id != UINT64_MAX; ++i) { - const uint64_t ret = get_usage(filters + i, is_encoder); - if (ret == UINT64_MAX) - return UINT32_MAX; - - usage += ret; - } - - // Convert to mebibytes with rounding. - return usage / (1024 * 1024) + (usage % (1024 * 1024) >= 512 ? 1 : 0); -} diff --git a/src/liblzma/common/next_coder.c b/src/liblzma/common/next_coder.c deleted file mode 100644 index c10fe24d..00000000 --- a/src/liblzma/common/next_coder.c +++ /dev/null @@ -1,65 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file next_coder.c -/// \brief Initializing and freeing the next coder in the chain -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - -extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter_info *filters) -{ - lzma_ret ret = LZMA_OK; - - // Free the existing coder if it is different than the current one. - if ((uintptr_t)(filters[0].init) != next->init) - lzma_next_coder_end(next, allocator); - - if (filters[0].init != NULL) { - // Initialize the new coder. - ret = filters[0].init(next, allocator, filters); - - // Set the init function pointer if initialization was - // successful. next->code and next->end are set by the - // initialization function itself. - if (ret == LZMA_OK) { - next->init = (uintptr_t)(filters[0].init); - assert(next->code != NULL); - assert(next->end != NULL); - } else { - lzma_next_coder_end(next, allocator); - } - } - - return ret; -} - - -extern void -lzma_next_coder_end(lzma_next_coder *next, lzma_allocator *allocator) -{ - if (next != NULL) { - if (next->end != NULL) - next->end(next->coder, allocator); - - // Reset the variables so the we don't accidentally think - // that it is an already initialized coder. - *next = LZMA_NEXT_CODER_INIT; - } - - return; -} diff --git a/src/liblzma/common/raw_common.c b/src/liblzma/common/raw_common.c deleted file mode 100644 index 35252fc2..00000000 --- a/src/liblzma/common/raw_common.c +++ /dev/null @@ -1,127 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_common.c -/// \brief Stuff shared between raw encoder and raw decoder -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_common.h" - - -static lzma_ret -validate_options(const lzma_options_filter *options, size_t *count) -{ - if (options == NULL) - return LZMA_PROG_ERROR; - - // Number of non-last filters that may change the size of the data - // significantly (that is, more than 1-2 % or so). - size_t change = 0; - - // True if the last filter in the given chain is actually usable as - // the last filter. Only filters that support embedding End of Payload - // Marker can be used as the last filter in the chain. - bool last_ok = false; - - size_t i; - for (i = 0; options[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { - switch (options[i].id) { - // Not #ifdeffing these for simplicity. - case LZMA_FILTER_X86: - case LZMA_FILTER_POWERPC: - case LZMA_FILTER_IA64: - case LZMA_FILTER_ARM: - case LZMA_FILTER_ARMTHUMB: - case LZMA_FILTER_SPARC: - case LZMA_FILTER_DELTA: - // These don't change the size of the data and cannot - // be used as the last filter in the chain. - last_ok = false; - break; - -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - last_ok = true; - ++change; - break; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - last_ok = true; - break; -#endif - - default: - return LZMA_HEADER_ERROR; - } - } - - // There must be 1-4 filters and the last filter must be usable as - // the last filter in the chain. - if (i == 0 || i > 4 || !last_ok) - return LZMA_HEADER_ERROR; - - // At maximum of two non-last filters are allowed to change the - // size of the data. - if (change > 2) - return LZMA_HEADER_ERROR; - - *count = i; - return LZMA_OK; -} - - -extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options, - lzma_init_function (*get_function)(lzma_vli id), - bool is_encoder) -{ - // Do some basic validation and get the number of filters. - size_t count; - return_if_error(validate_options(options, &count)); - - // Set the filter functions and copy the options pointer. - lzma_filter_info filters[count + 1]; - if (is_encoder) { - for (size_t i = 0; i < count; ++i) { - // The order of the filters is reversed in the - // encoder. It allows more efficient handling - // of the uncompressed data. - const size_t j = count - i - 1; - - filters[j].init = get_function(options[i].id); - if (filters[j].init == NULL) - return LZMA_HEADER_ERROR; - - filters[j].options = options[i].options; - } - } else { - for (size_t i = 0; i < count; ++i) { - filters[i].init = get_function(options[i].id); - if (filters[i].init == NULL) - return LZMA_HEADER_ERROR; - - filters[i].options = options[i].options; - } - } - - // Terminate the array. - filters[count].init = NULL; - - // Initialize the filters. - return lzma_next_filter_init(next, allocator, filters); -} diff --git a/src/liblzma/common/raw_decoder.c b/src/liblzma/common/raw_decoder.c deleted file mode 100644 index 4fb7111c..00000000 --- a/src/liblzma/common/raw_decoder.c +++ /dev/null @@ -1,116 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_decoder.c -/// \brief Raw decoder initialization API -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_decoder.h" -#include "simple_coder.h" -#include "subblock_decoder.h" -#include "subblock_decoder_helper.h" -#include "delta_decoder.h" -#include "lzma_decoder.h" - - -static lzma_init_function -get_function(lzma_vli id) -{ - switch (id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - return &lzma_subblock_decoder_init; -#endif - -#ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: - return &lzma_simple_x86_decoder_init; -#endif - -#ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: - return &lzma_simple_powerpc_decoder_init; -#endif - -#ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: - return &lzma_simple_ia64_decoder_init; -#endif - -#ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: - return &lzma_simple_arm_decoder_init; -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: - return &lzma_simple_armthumb_decoder_init; -#endif - -#ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: - return &lzma_simple_sparc_decoder_init; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - return &lzma_delta_decoder_init; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - return &lzma_lzma_decoder_init; -#endif - -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK_HELPER: - return &lzma_subblock_decoder_helper_init; -#endif - } - - return NULL; -} - - -extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options) -{ - const lzma_ret ret = lzma_raw_coder_init(next, allocator, - options, &get_function, false); - - if (ret != LZMA_OK) - lzma_next_coder_end(next, allocator); - - return ret; -} - - -extern LZMA_API lzma_ret -lzma_raw_decoder(lzma_stream *strm, const lzma_options_filter *options) -{ - return_if_error(lzma_strm_init(strm)); - - strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; - - const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, - strm->allocator, options, &get_function, false); - - if (ret != LZMA_OK) - lzma_end(strm); - - return ret; -} diff --git a/src/liblzma/common/raw_encoder.c b/src/liblzma/common/raw_encoder.c deleted file mode 100644 index 9b8cbfae..00000000 --- a/src/liblzma/common/raw_encoder.c +++ /dev/null @@ -1,111 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_encoder.c -/// \brief Raw encoder initialization API -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_encoder.h" -#include "simple_coder.h" -#include "subblock_encoder.h" -#include "delta_encoder.h" -#include "lzma_encoder.h" - - -static lzma_init_function -get_function(lzma_vli id) -{ - switch (id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - return &lzma_subblock_encoder_init; -#endif - -#ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: - return &lzma_simple_x86_encoder_init; -#endif - -#ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: - return &lzma_simple_powerpc_encoder_init; -#endif - -#ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: - return &lzma_simple_ia64_encoder_init; -#endif - -#ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: - return &lzma_simple_arm_encoder_init; -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: - return &lzma_simple_armthumb_encoder_init; -#endif - -#ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: - return &lzma_simple_sparc_encoder_init; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - return &lzma_delta_encoder_init; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - return &lzma_lzma_encoder_init; -#endif - } - - return NULL; -} - - -extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options) -{ - const lzma_ret ret = lzma_raw_coder_init(next, allocator, - options, &get_function, true); - - if (ret != LZMA_OK) - lzma_next_coder_end(next, allocator); - - return ret; -} - - -extern LZMA_API lzma_ret -lzma_raw_encoder(lzma_stream *strm, const lzma_options_filter *options) -{ - return_if_error(lzma_strm_init(strm)); - - strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; - strm->internal->supported_actions[LZMA_FINISH] = true; - - const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, - strm->allocator, options, &get_function, true); - - if (ret != LZMA_OK) - lzma_end(strm); - - return ret; -} diff --git a/src/liblzma/common/stream_common.c b/src/liblzma/common/stream_common.c deleted file mode 100644 index 121a6674..00000000 --- a/src/liblzma/common/stream_common.c +++ /dev/null @@ -1,23 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file stream_common.c -/// \brief Common stuff for Stream coders -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "stream_common.h" - -const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 }; -const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c index 1bf7f1f8..5b46819d 100644 --- a/src/liblzma/common/stream_decoder.c +++ b/src/liblzma/common/stream_decoder.c @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" #include "stream_decoder.h" +#include "stream_flags_common.h" #include "check.h" #include "stream_flags_decoder.h" #include "block_decoder.h" @@ -31,6 +31,7 @@ struct lzma_coder_s { SEQ_BLOCK, SEQ_INDEX, SEQ_STREAM_FOOTER, + SEQ_STREAM_PADDING, } sequence; /// Block or Metadata decoder. This takes little memory and the same @@ -40,7 +41,7 @@ struct lzma_coder_s { /// Block options decoded by the Block Header decoder and used by /// the Block decoder. - lzma_options_block block_options; + lzma_block block_options; /// Stream Flags from Stream Header lzma_stream_flags stream_flags; @@ -49,8 +50,35 @@ struct lzma_coder_s { /// with O(1) memory usage. lzma_index_hash *index_hash; - /// Write position in buffer[] - size_t buffer_pos; + /// Memory usage limit + uint64_t memlimit; + + /// If true, LZMA_NO_CHECK is returned if the Stream has + /// no integrity check. + bool warn_no_check; + + /// If true, LZMA_UNSUPPORTED_CHECK is returned if the Stream has + /// an integrity check that isn't supported by this liblzma build. + bool warn_unsupported_check; + + /// If true, LZMA_SEE_CHECK is returned after decoding Stream Header. + bool tell_check; + + /// If true, we will decode concatenated Streams that possibly have + /// Stream Padding between or after them. LZMA_STREAM_END is returned + /// once the application isn't giving us any new input and we aren't + /// in the middle of a Stream and possible Stream Padding is a + /// multiple of four bytes. FIXME + bool concatenated; + + /// When decoding concatenated Streams, this is true as long as we + /// are decoding the first Stream. This is needed to avoid misleading + /// LZMA_FORMAT_ERROR in case the later Streams don't have valid magic + /// bytes. + bool first_stream; + + /// Write position in buffer[] and position in Stream Padding + size_t pos; /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. @@ -59,6 +87,23 @@ struct lzma_coder_s { static lzma_ret +stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +{ + // Initialize the Index hash used to verify the Index. + coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); + if (coder->index_hash == NULL) + return LZMA_MEM_ERROR; + + // Reset the rest of the variables. + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.filters = NULL; + coder->pos = 0; + + return LZMA_OK; +} + + +static lzma_ret stream_decode(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, @@ -66,43 +111,56 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, { // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. - while (*out_pos < out_size && (*in_pos < in_size - || coder->sequence == SEQ_BLOCK)) + while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { // Copy the Stream Header to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); // Return if we didn't get the whole Stream Header yet. - if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE) + if (coder->pos < LZMA_STREAM_HEADER_SIZE) return LZMA_OK; - coder->buffer_pos = 0; + coder->pos = 0; // Decode the Stream Header. - return_if_error(lzma_stream_header_decode( - &coder->stream_flags, coder->buffer)); + const lzma_ret ret = lzma_stream_header_decode( + &coder->stream_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR && !coder->first_stream + ? LZMA_DATA_ERROR : ret; // Copy the type of the Check so that Block Header and Block // decoders see it. coder->block_options.check = coder->stream_flags.check; - // Even if we return LZMA_UNSUPPORTED_CHECK below, we want + // Even if we return LZMA_*_CHECK below, we want // to continue from Block Header decoding. coder->sequence = SEQ_BLOCK_HEADER; - // Detect if the Check type is supported and give appropriate - // warning if it isn't. We don't warn every time a new Block - // is started. - if (!lzma_available_checks[coder->block_options.check]) + // Detect if there's no integrity check or if it is + // unsupported if those were requested by the application. + if (coder->warn_no_check && coder->stream_flags.check + == LZMA_CHECK_NONE) + return LZMA_NO_CHECK; + + if (coder->warn_unsupported_check + && !lzma_check_is_supported( + coder->stream_flags.check)) return LZMA_UNSUPPORTED_CHECK; + if (coder->tell_check) + return LZMA_SEE_CHECK; + break; } case SEQ_BLOCK_HEADER: { - if (coder->buffer_pos == 0) { + if (*in_pos >= in_size) + return LZMA_OK; + + if (coder->pos == 0) { // Detect if it's Index. if (in[*in_pos] == 0x00) { coder->sequence = SEQ_INDEX; @@ -118,29 +176,41 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } // Copy the Block Header to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, coder->block_options.header_size); // Return if we didn't get the whole Block Header yet. - if (coder->buffer_pos < coder->block_options.header_size) + if (coder->pos < coder->block_options.header_size) return LZMA_OK; - coder->buffer_pos = 0; + coder->pos = 0; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); - // Initialize the Block decoder. - const lzma_ret ret = lzma_block_decoder_init( - &coder->block_decoder, - allocator, &coder->block_options); + // Check the memory usage limit. + const uint64_t memusage = lzma_memusage_decoder(filters); + lzma_ret ret; + + if (memusage == UINT64_MAX) { + // One or more unknown Filter IDs. + ret = LZMA_HEADER_ERROR; + } else if (memusage > coder->memlimit) { + // The chain would need too much memory. + ret = LZMA_MEMLIMIT_ERROR; + } else { + // Memory usage is OK. Initialize the Block decoder. + ret = lzma_block_decoder_init( + &coder->block_decoder, + allocator, &coder->block_options); + } // Free the allocated filter options since they are needed // only to initialize the Block decoder. @@ -149,10 +219,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block_options.filters = NULL; - // Check if Block enocoder initialization succeeded. Don't - // warn about unsupported check anymore since we did it - // earlier if it was needed. - if (ret != LZMA_OK && ret != LZMA_UNSUPPORTED_CHECK) + // Check if memory usage calculation and Block enocoder + // initialization succeeded. + if (ret != LZMA_OK) return ret; coder->sequence = SEQ_BLOCK; @@ -160,7 +229,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK: { - lzma_ret ret = coder->block_decoder.code( + const lzma_ret ret = coder->block_decoder.code( coder->block_decoder.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -180,6 +249,12 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { + // If we don't have any input, don't call + // lzma_index_hash_decode() since it would return + // LZMA_BUF_ERROR, which we must not do here. + if (*in_pos >= in_size) + return LZMA_OK; + // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, @@ -193,14 +268,17 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, case SEQ_STREAM_FOOTER: // Copy the Stream Footer to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); // Return if we didn't get the whole Stream Footer yet. - if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE) + if (coder->pos < LZMA_STREAM_HEADER_SIZE) return LZMA_OK; + coder->pos = 0; + // Decode the Stream Footer. + // FIXME LZMA_FORMAT_ERROR doesn't make sense here. lzma_stream_flags footer_flags; return_if_error(lzma_stream_footer_decode( &footer_flags, coder->buffer)); @@ -217,7 +295,48 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, &footer_flags)) return LZMA_DATA_ERROR; - return LZMA_STREAM_END; + if (!coder->concatenated) + return LZMA_STREAM_END; + + coder->sequence = SEQ_STREAM_PADDING; + break; + + case SEQ_STREAM_PADDING: + assert(coder->concatenated); + + while (true) { + if (*in_pos >= in_size) { + // Unless LZMA_FINISH was used, we cannot + // know if there's more input coming later. + if (action != LZMA_FINISH) + return LZMA_OK; + + // Stream Padding must be a multiple of + // four bytes. + return coder->pos == 0 + ? LZMA_STREAM_END + : LZMA_DATA_ERROR; + } + + if (in[*in_pos] != 0x00) { + if (coder->pos != 0) { + // Stream Padding is not a multiple of + // four bytes. + ++*in_pos; + return LZMA_DATA_ERROR; + } + + // Prepare to decode the next Stream. + return_if_error(stream_decoder_reset( + coder, allocator)); + break; + } + + ++*in_pos; + coder->pos = (coder->pos + 1) & 3; + } + + break; default: assert(0); @@ -231,16 +350,29 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->block_decoder, allocator); + lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +static lzma_check +stream_decoder_see_check(const lzma_coder *coder) { + return coder->stream_flags.check; +} + + +extern lzma_ret +lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) +{ + lzma_next_coder_init(lzma_stream_decoder_init, next, allocator); + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_HEADER_ERROR; + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -248,40 +380,32 @@ stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->code = &stream_decode; next->end = &stream_decoder_end; + next->see_check = &stream_decoder_see_check; next->coder->block_decoder = LZMA_NEXT_CODER_INIT; next->coder->index_hash = NULL; } - // Initialize the Index hash used to verify the Index. - next->coder->index_hash = lzma_index_hash_init( - next->coder->index_hash, allocator); - if (next->coder->index_hash == NULL) - return LZMA_MEM_ERROR; - - // Reset the rest of the variables. - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.filters = NULL; - next->coder->buffer_pos = 0; + next->coder->memlimit = memlimit; + next->coder->warn_no_check = (flags & LZMA_WARN_NO_CHECK) != 0; + next->coder->warn_unsupported_check + = (flags & LZMA_WARN_UNSUPPORTED_CHECK) != 0; + next->coder->tell_check = (flags & LZMA_TELL_CHECK) != 0; + next->coder->concatenated + = (flags & LZMA_CONCATENATED) != 0; - return LZMA_OK; -} - - -extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) -{ - lzma_next_coder_init0(stream_decoder_init, next, allocator); + return stream_decoder_reset(next->coder, allocator); } extern LZMA_API lzma_ret -lzma_stream_decoder(lzma_stream *strm) +lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init0(strm, stream_decoder_init); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; // FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/stream_decoder.h b/src/liblzma/common/stream_decoder.h index dcda387d..59d58c6f 100644 --- a/src/liblzma/common/stream_decoder.h +++ b/src/liblzma/common/stream_decoder.h @@ -22,7 +22,7 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator); +extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); #endif diff --git a/src/liblzma/common/stream_encoder.c b/src/liblzma/common/stream_encoder.c index 767b8014..9d56c899 100644 --- a/src/liblzma/common/stream_encoder.c +++ b/src/liblzma/common/stream_encoder.c @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" #include "stream_encoder.h" +#include "stream_flags_common.h" #include "block_encoder.h" #include "index_encoder.h" @@ -37,7 +37,7 @@ struct lzma_coder_s { lzma_next_coder block_encoder; /// Options for the Block encoder - lzma_options_block block_options; + lzma_block block_options; /// Index encoder. This is separate from Block encoder, because this /// doesn't take much memory, and when encoding multiple Streams @@ -86,8 +86,8 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, case SEQ_STREAM_HEADER: case SEQ_BLOCK_HEADER: case SEQ_STREAM_FOOTER: - bufcpy(coder->buffer, &coder->buffer_pos, coder->buffer_size, - out, out_pos, out_size); + lzma_bufcpy(coder->buffer, &coder->buffer_pos, + coder->buffer_size, out, out_pos, out_size); if (coder->buffer_pos < coder->buffer_size) return LZMA_OK; @@ -202,18 +202,20 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->block_encoder, allocator); - lzma_next_coder_end(&coder->index_encoder, allocator); + lzma_next_end(&coder->block_encoder, allocator); + lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check) +extern lzma_ret +lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, lzma_check check) { + lzma_next_coder_init(lzma_stream_encoder_init, next, allocator); + if (filters == NULL) return LZMA_PROG_ERROR; @@ -233,7 +235,7 @@ stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Basic initializations next->coder->sequence = SEQ_STREAM_HEADER; next->coder->block_options.check = check; - next->coder->block_options.filters = (lzma_options_filter *)(filters); + next->coder->block_options.filters = (lzma_filter *)(filters); // Initialize the Index next->coder->index = lzma_index_init(next->coder->index, allocator); @@ -258,20 +260,11 @@ stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check) -{ - lzma_next_coder_init(stream_encoder_init, next, allocator, - filters, check); -} - - extern LZMA_API lzma_ret lzma_stream_encoder(lzma_stream *strm, - const lzma_options_filter *filters, lzma_check_type check) + const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init(strm, stream_encoder_init, filters, check); + lzma_next_strm_init(lzma_stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; diff --git a/src/liblzma/common/stream_encoder.h b/src/liblzma/common/stream_encoder.h index 3ce29561..cec2e5b5 100644 --- a/src/liblzma/common/stream_encoder.h +++ b/src/liblzma/common/stream_encoder.h @@ -25,6 +25,6 @@ extern lzma_ret lzma_stream_encoder_init( lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check); + const lzma_filter *filters, lzma_check check); #endif diff --git a/src/liblzma/common/stream_flags_equal.c b/src/liblzma/common/stream_flags_common.c index db22567f..c44b3ff2 100644 --- a/src/liblzma/common/stream_flags_equal.c +++ b/src/liblzma/common/stream_flags_common.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_flags_equal.c -/// \brief Compare Stream Header and Stream Footer +/// \file stream_flags_common.c +/// \brief Common stuff for Stream flags coders // -// Copyright (C) 2008 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,11 +17,15 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" +#include "stream_flags_common.h" + + +const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 }; +const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; extern LZMA_API lzma_bool -lzma_stream_flags_equal(const lzma_stream_flags *a, lzma_stream_flags *b) +lzma_stream_flags_equal(const lzma_stream_flags *a, const lzma_stream_flags *b) { if (a->check != b->check) return false; diff --git a/src/liblzma/common/stream_common.h b/src/liblzma/common/stream_flags_common.h index 4f83fc58..6e57857b 100644 --- a/src/liblzma/common/stream_common.h +++ b/src/liblzma/common/stream_flags_common.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_common.h -/// \brief Common stuff for Stream coders +/// \file stream_flags_common.h +/// \brief Common stuff for Stream flags coders // // Copyright (C) 2007 Lasse Collin // @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_COMMON_H -#define LZMA_STREAM_COMMON_H +#ifndef LZMA_STREAM_FLAGS_COMMON_H +#define LZMA_STREAM_FLAGS_COMMON_H #include "common.h" diff --git a/src/liblzma/common/stream_flags_decoder.c b/src/liblzma/common/stream_flags_decoder.c index 0270875a..ccc1539d 100644 --- a/src/liblzma/common/stream_flags_decoder.c +++ b/src/liblzma/common/stream_flags_decoder.c @@ -17,7 +17,7 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" +#include "stream_flags_common.h" static bool diff --git a/src/liblzma/common/stream_flags_encoder.c b/src/liblzma/common/stream_flags_encoder.c index 4efbb6f4..1d736a8a 100644 --- a/src/liblzma/common/stream_flags_encoder.c +++ b/src/liblzma/common/stream_flags_encoder.c @@ -17,7 +17,7 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" +#include "stream_flags_common.h" static bool diff --git a/src/liblzma/common/vli_decoder.c b/src/liblzma/common/vli_decoder.c index faff6ccb..60874baa 100644 --- a/src/liblzma/common/vli_decoder.c +++ b/src/liblzma/common/vli_decoder.c @@ -27,17 +27,30 @@ lzma_vli_decode(lzma_vli *restrict vli, size_t *restrict vli_pos, { // If we haven't been given vli_pos, work in single-call mode. size_t vli_pos_internal = 0; - if (vli_pos == NULL) + if (vli_pos == NULL) { vli_pos = &vli_pos_internal; - - // Initialize *vli when starting to decode a new integer. - if (*vli_pos == 0) *vli = 0; - // Validate the arguments. - if (*vli_pos >= LZMA_VLI_BYTES_MAX || *in_pos >= in_size - || (*vli >> (*vli_pos * 7)) != 0) - return LZMA_PROG_ERROR;; + // If there's no input, use LZMA_DATA_ERROR. This way it is + // easy to decode VLIs from buffers that have known size, + // and get the correct error code in case the buffer is + // too short. + if (*in_pos >= in_size) + return LZMA_DATA_ERROR; + + } else { + // Initialize *vli when starting to decode a new integer. + if (*vli_pos == 0) + *vli = 0; + + // Validate the arguments. + if (*vli_pos >= LZMA_VLI_BYTES_MAX + || (*vli >> (*vli_pos * 7)) != 0) + return LZMA_PROG_ERROR;; + + if (*in_pos >= in_size) + return LZMA_BUF_ERROR; + } do { // Read the next byte. diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c index c48d6474..53022f16 100644 --- a/src/liblzma/common/vli_encoder.c +++ b/src/liblzma/common/vli_encoder.c @@ -31,10 +31,12 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos, vli_pos = &vli_pos_internal; // Validate the arguments. - if (*vli_pos >= LZMA_VLI_BYTES_MAX || *out_pos >= out_size - || vli > LZMA_VLI_VALUE_MAX) + if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_VALUE_MAX) return LZMA_PROG_ERROR; + if (*out_pos >= out_size) + return LZMA_BUF_ERROR; + // Write the non-last bytes in a loop. while ((vli >> (*vli_pos * 7)) >= 0x80) { out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7)) | 0x80; @@ -55,20 +57,3 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos, return vli_pos == &vli_pos_internal ? LZMA_OK : LZMA_STREAM_END; } - - -extern LZMA_API uint32_t -lzma_vli_size(lzma_vli vli) -{ - if (vli > LZMA_VLI_VALUE_MAX) - return 0; - - uint32_t i = 0; - do { - vli >>= 7; - ++i; - } while (vli != 0); - - assert(i <= LZMA_VLI_BYTES_MAX); - return i; -} diff --git a/src/liblzma/common/version.c b/src/liblzma/common/vli_size.c index dffec7ff..547bba0b 100644 --- a/src/liblzma/common/version.c +++ b/src/liblzma/common/vli_size.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file version.c -/// \brief liblzma version number +/// \file vli_size.c +/// \brief Calculates the encoded size of a variable-length integer // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -20,6 +20,18 @@ #include "common.h" -LZMA_API const uint32_t lzma_version_number = LZMA_VERSION; +extern LZMA_API uint32_t +lzma_vli_size(lzma_vli vli) +{ + if (vli > LZMA_VLI_VALUE_MAX) + return 0; -LZMA_API const char *const lzma_version_string = PACKAGE_VERSION; + uint32_t i = 0; + do { + vli >>= 7; + ++i; + } while (vli != 0); + + assert(i <= LZMA_VLI_BYTES_MAX); + return i; +} diff --git a/src/liblzma/delta/Makefile.am b/src/liblzma/delta/Makefile.am new file mode 100644 index 00000000..fc09f5b8 --- /dev/null +++ b/src/liblzma/delta/Makefile.am @@ -0,0 +1,34 @@ +## +## Copyright (C) 2008 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = libdelta.la +libdelta_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common + +libdelta_la_SOURCES = \ + delta_common.c \ + delta_common.h + +if COND_ENCODER_DELTA +libdelta_la_SOURCES += \ + delta_encoder.c \ + delta_encoder.h +endif + +if COND_DECODER_DELTA +libdelta_la_SOURCES += \ + delta_decoder.c \ + delta_decoder.h +endif diff --git a/src/liblzma/common/delta_common.c b/src/liblzma/delta/delta_common.c index acd31e14..d40e0c7f 100644 --- a/src/liblzma/common/delta_common.c +++ b/src/liblzma/delta/delta_common.c @@ -23,7 +23,7 @@ static void delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } diff --git a/src/liblzma/common/delta_common.h b/src/liblzma/delta/delta_common.h index 1d58899d..1d58899d 100644 --- a/src/liblzma/common/delta_common.h +++ b/src/liblzma/delta/delta_common.h diff --git a/src/liblzma/common/delta_decoder.c b/src/liblzma/delta/delta_decoder.c index 8f5a4cbf..ee22ba02 100644 --- a/src/liblzma/common/delta_decoder.c +++ b/src/liblzma/delta/delta_decoder.c @@ -59,3 +59,24 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, { return lzma_delta_coder_init(next, allocator, filters, &delta_decode); } + + +extern lzma_ret +lzma_delta_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 1) + return LZMA_HEADER_ERROR; + + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + opt->type = LZMA_DELTA_TYPE_BYTE; + opt->distance = props[0] + 1; + + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/common/delta_decoder.h b/src/liblzma/delta/delta_decoder.h index bef9f58a..84852bf3 100644 --- a/src/liblzma/common/delta_decoder.h +++ b/src/liblzma/delta/delta_decoder.h @@ -25,4 +25,8 @@ extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +extern lzma_ret lzma_delta_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + #endif diff --git a/src/liblzma/common/delta_encoder.c b/src/liblzma/delta/delta_encoder.c index a8bb9341..d8f40287 100644 --- a/src/liblzma/common/delta_encoder.c +++ b/src/liblzma/delta/delta_encoder.c @@ -96,3 +96,24 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, { return lzma_delta_coder_init(next, allocator, filters, &delta_encode); } + + +extern lzma_ret +lzma_delta_props_encode(const void *options, uint8_t *out) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + const lzma_options_delta *opt = options; + + // It's possible that newer liblzma versions will support larger + // distance values. + if (opt->type != LZMA_DELTA_TYPE_BYTE + || opt->distance < LZMA_DELTA_DISTANCE_MIN + || opt->distance > LZMA_DELTA_DISTANCE_MAX) + return LZMA_HEADER_ERROR; + + out[0] = opt->distance - LZMA_DELTA_DISTANCE_MIN; + + return LZMA_OK; +} diff --git a/src/liblzma/common/delta_encoder.h b/src/liblzma/delta/delta_encoder.h index c669458d..b8b29c61 100644 --- a/src/liblzma/common/delta_encoder.h +++ b/src/liblzma/delta/delta_encoder.h @@ -25,4 +25,6 @@ extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); + #endif diff --git a/src/liblzma/lz/Makefile.am b/src/liblzma/lz/Makefile.am index 5c27e2f2..bf41d8e6 100644 --- a/src/liblzma/lz/Makefile.am +++ b/src/liblzma/lz/Makefile.am @@ -20,43 +20,16 @@ liblz_la_CPPFLAGS = \ liblz_la_SOURCES = -if COND_MAIN_ENCODER +if COND_ENCODER_LZ liblz_la_SOURCES += \ lz_encoder.c \ lz_encoder.h \ - lz_encoder_private.h \ - match_c.h \ - match_h.h - -if COND_MF_HC3 -liblz_la_SOURCES += hc3.c hc3.h -liblz_la_CPPFLAGS += -DHAVE_HC3 -endif - -if COND_MF_HC4 -liblz_la_SOURCES += hc4.c hc4.h -liblz_la_CPPFLAGS += -DHAVE_HC4 -endif - -if COND_MF_BT2 -liblz_la_SOURCES += bt2.c bt2.h -liblz_la_CPPFLAGS += -DHAVE_BT2 -endif - -if COND_MF_BT3 -liblz_la_SOURCES += bt3.c bt3.h -liblz_la_CPPFLAGS += -DHAVE_BT3 -endif - -if COND_MF_BT4 -liblz_la_SOURCES += bt4.c bt4.h -liblz_la_CPPFLAGS += -DHAVE_BT4 -endif - + lz_encoder_hash.h \ + lz_encoder_mf.c endif -if COND_MAIN_DECODER +if COND_DECODER_LZ liblz_la_SOURCES += \ lz_decoder.c \ lz_decoder.h diff --git a/src/liblzma/lz/bt2.c b/src/liblzma/lz/bt2.c deleted file mode 100644 index 7dc4cb80..00000000 --- a/src/liblzma/lz/bt2.c +++ /dev/null @@ -1,27 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt2.c -/// \brief Binary Tree 2 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt2.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt2.h b/src/liblzma/lz/bt2.h deleted file mode 100644 index 33cb52cd..00000000 --- a/src/liblzma/lz/bt2.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt2.h -/// \brief Binary Tree 2 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT2_H -#define LZMA_BT2_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt2 -#define LZMA_MATCH_FINDER_NAME_UPPER BT2 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/bt3.c b/src/liblzma/lz/bt3.c deleted file mode 100644 index d44310f3..00000000 --- a/src/liblzma/lz/bt3.c +++ /dev/null @@ -1,29 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt3.c -/// \brief Binary Tree 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt3.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define HASH_ARRAY_2 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt3.h b/src/liblzma/lz/bt3.h deleted file mode 100644 index 247c7e5f..00000000 --- a/src/liblzma/lz/bt3.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt3.h -/// \brief Binary Tree 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT3_H -#define LZMA_BT3_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt3 -#define LZMA_MATCH_FINDER_NAME_UPPER BT3 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/bt4.c b/src/liblzma/lz/bt4.c deleted file mode 100644 index 6e1042c9..00000000 --- a/src/liblzma/lz/bt4.c +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt4.c -/// \brief Binary Tree 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt4.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt4.h b/src/liblzma/lz/bt4.h deleted file mode 100644 index e3fcf6ac..00000000 --- a/src/liblzma/lz/bt4.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt4.h -/// \brief Binary Tree 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT4_H -#define LZMA_BT4_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt4 -#define LZMA_MATCH_FINDER_NAME_UPPER BT4 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/hc3.c b/src/liblzma/lz/hc3.c deleted file mode 100644 index 22b5689b..00000000 --- a/src/liblzma/lz/hc3.c +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc3.c -/// \brief Hash Chain 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "hc3.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define IS_HASH_CHAIN -#define HASH_ARRAY_2 - -#include "match_c.h" diff --git a/src/liblzma/lz/hc3.h b/src/liblzma/lz/hc3.h deleted file mode 100644 index 97be0b1d..00000000 --- a/src/liblzma/lz/hc3.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc3.h -/// \brief Hash Chain 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_HC3_H -#define LZMA_HC3_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER hc3 -#define LZMA_MATCH_FINDER_NAME_UPPER HC3 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/hc4.c b/src/liblzma/lz/hc4.c deleted file mode 100644 index a55cfd09..00000000 --- a/src/liblzma/lz/hc4.c +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc4.c -/// \brief Hash Chain 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "hc4.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define IS_HASH_CHAIN -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/hc4.h b/src/liblzma/lz/hc4.h deleted file mode 100644 index dc072e2f..00000000 --- a/src/liblzma/lz/hc4.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc4.h -/// \brief Hash Chain 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_HC4_H -#define LZMA_HC4_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER hc4 -#define LZMA_MATCH_FINDER_NAME_UPPER HC4 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/lz_decoder.c b/src/liblzma/lz/lz_decoder.c index ae969d62..5c3f1d18 100644 --- a/src/liblzma/lz/lz_decoder.c +++ b/src/liblzma/lz/lz_decoder.c @@ -18,351 +18,142 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "lz_decoder.h" +// liblzma supports multiple LZ77-based filters. The LZ part is shared +// between these filters. The LZ code takes care of dictionary handling +// and passing the data between filters in the chain. The filter-specific +// part decodes from the input buffer to the dictionary. -/// Minimum size of allocated dictionary -#define DICT_SIZE_MIN 8192 +#include "lz_decoder.h" -/// When there is less than this amount of data available for decoding, -/// it is moved to the temporary buffer which -/// - protects from reads past the end of the buffer; and -/// - stored the incomplete data between lzma_code() calls. -/// -/// \note TEMP_LIMIT must be at least as much as -/// REQUIRED_IN_BUFFER_SIZE defined in lzma_decoder.c. -#define TEMP_LIMIT 32 -// lzma_lz_decoder.dict[] must be three times the size of TEMP_LIMIT. -// 2 * TEMP_LIMIT is used for the actual data, and the third TEMP_LIMIT -// bytes is needed for safety to allow decode_dummy() in lzma_decoder.c -// to read past end of the buffer. This way it should be both fast and simple. -#if LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT -# error LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT -#endif +struct lzma_coder_s { + /// Dictionary (history buffer) + lzma_dict dict; + /// The actual LZ-based decoder e.g. LZMA + lzma_lz_decoder lz; -struct lzma_coder_s { + /// Next filter in the chain, if any. Note that LZMA and LZMA2 are + /// only allowed as the last filter, but the long-range filter in + /// future can be in the middle of the chain. lzma_next_coder next; - lzma_lz_decoder lz; - // There are more members in this structure but they are not - // visible in LZ coder. + /// True if the next filter in the chain has returned LZMA_STREAM_END. + bool next_finished; + + /// True if the LZ decoder (e.g. LZMA) has detected end of payload + /// marker. This may become true before next_finished becomes true. + bool this_finished; + + /// Temporary buffer needed when the LZ-based filter is not the last + /// filter in the chain. The output of the next filter is first + /// decoded into buffer[], which is then used as input for the actual + /// LZ-based decoder. + struct { + size_t pos; + size_t size; + uint8_t buffer[LZMA_BUFFER_SIZE]; + } temp; }; -/// - Copy as much data as possible from lz->dict[] to out[]. -/// - Update *out_pos, lz->start, and lz->end accordingly. -/// - Wrap lz-pos to the beginning of lz->dict[] if there is a danger that -/// it may go past the end of the buffer (lz->pos >= lz->must_flush_pos). -static inline bool -flush(lzma_lz_decoder *restrict lz, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size) -{ - // Flush uncompressed data from the history buffer to - // the output buffer. This is done in two phases. - - assert(lz->start <= lz->end); - - // Flush if pos < start < end. - if (lz->pos < lz->start && lz->start < lz->end) { - bufcpy(lz->dict, &lz->start, lz->end, out, out_pos, out_size); - - // If we reached end of the data in history buffer, - // wrap to the beginning. - if (lz->start == lz->end) - lz->start = 0; - } - - // Flush if start start < pos <= end. This is not as `else' for - // previous `if' because the previous one may make this one true. - if (lz->start < lz->pos) { - bufcpy(lz->dict, &lz->start, - lz->pos, out, out_pos, out_size); - - if (lz->pos >= lz->must_flush_pos) { - // Wrap the flushing position if we have - // flushed the whole history buffer. - if (lz->pos == lz->start) - lz->start = 0; - - // Wrap the write position and store to lz.end - // how much there is new data available. - lz->end = lz->pos; - lz->pos = 0; - lz->is_full = true; - } - } - - assert(lz->pos < lz->must_flush_pos); - - return *out_pos == out_size; -} - - -/// Calculate safe value for lz->limit. If no safe value can be found, -/// set lz->limit to zero. When flushing, only as little data will be -/// decoded as is needed to fill the output buffer (lowers both latency -/// and throughput). -/// -/// \return true if there is no space for new uncompressed data. -/// -static inline bool -set_limit(lzma_lz_decoder *lz, size_t out_avail, bool flushing) -{ - // Set the limit so that writing to dict[limit + match_max_len - 1] - // doesn't overwrite any unflushed data and doesn't write past the - // end of the dict buffer. - if (lz->start <= lz->pos) { - // We can fill the buffer from pos till the end - // of the dict buffer. - lz->limit = lz->must_flush_pos; - } else if (lz->pos + lz->match_max_len < lz->start) { - // There's some unflushed data between pos and end of the - // buffer. Limit so that we don't overwrite the unflushed data. - lz->limit = lz->start - lz->match_max_len; - } else { - // Buffer is too full. - lz->limit = 0; - return true; - } - - // Finetune the limit a bit if it isn't zero. - - assert(lz->limit > lz->pos); - const size_t dict_avail = lz->limit - lz->pos; - - if (lz->uncompressed_size < dict_avail) { - // Finishing a stream that doesn't have - // an end of stream marker. - lz->limit = lz->pos + lz->uncompressed_size; - - } else if (flushing && out_avail < dict_avail) { - // Flushing enabled, decoding only as little as needed to - // fill the out buffer (if there's enough input, of course). - lz->limit = lz->pos + out_avail; - } - - return lz->limit == lz->pos; -} - - -/// Takes care of wrapping the data into temporary buffer when needed, -/// and calls the actual decoder. -/// -/// \return true if error occurred -/// -static inline bool -call_process(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size) -{ - // It would be nice and simple if we could just give in[] to the - // decoder, but the requirement of zlib-like API forces us to be - // able to make *in_pos == in_size whenever there is enough output - // space. If needed, we will append a few bytes from in[] to - // a temporary buffer and decode enough to reach the part that - // was copied from in[]. Then we can continue with the real in[]. - - bool error; - const size_t dict_old_pos = coder->lz.pos; - const size_t in_avail = in_size - *in_pos; - - if (coder->lz.temp_size + in_avail < 2 * TEMP_LIMIT) { - // Copy all the available input from in[] to temp[]. - memcpy(coder->lz.temp + coder->lz.temp_size, - in + *in_pos, in_avail); - coder->lz.temp_size += in_avail; - *in_pos += in_avail; - assert(*in_pos == in_size); - - // Decode as much as possible. - size_t temp_used = 0; - error = coder->lz.process(coder, coder->lz.temp, &temp_used, - coder->lz.temp_size, true); - assert(temp_used <= coder->lz.temp_size); - - // Move the remaining data to the beginning of temp[]. - coder->lz.temp_size -= temp_used; - memmove(coder->lz.temp, coder->lz.temp + temp_used, - coder->lz.temp_size); - - } else if (coder->lz.temp_size > 0) { - // Fill temp[] unless it is already full because we aren't - // the last filter in the chain. - size_t copy_size = 0; - if (coder->lz.temp_size < 2 * TEMP_LIMIT) { - assert(*in_pos < in_size); - copy_size = 2 * TEMP_LIMIT - coder->lz.temp_size; - memcpy(coder->lz.temp + coder->lz.temp_size, - in + *in_pos, copy_size); - // NOTE: We don't update lz.temp_size or *in_pos yet. - } - - size_t temp_used = 0; - error = coder->lz.process(coder, coder->lz.temp, &temp_used, - coder->lz.temp_size + copy_size, false); - - if (temp_used < coder->lz.temp_size) { - // Only very little input data was consumed. Move - // the unprocessed data to the beginning temp[]. - coder->lz.temp_size += copy_size - temp_used; - memmove(coder->lz.temp, coder->lz.temp + temp_used, - coder->lz.temp_size); - *in_pos += copy_size; - assert(*in_pos <= in_size); - - } else { - // We were able to decode so much data that next time - // we can decode directly from in[]. That is, we can - // consider temp[] to be empty now. - *in_pos += temp_used - coder->lz.temp_size; - coder->lz.temp_size = 0; - assert(*in_pos <= in_size); - } - - } else { - // Decode directly from in[]. - error = coder->lz.process(coder, in, in_pos, in_size, false); - assert(*in_pos <= in_size); - } - - assert(coder->lz.pos >= dict_old_pos); - if (coder->lz.uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { - // Update uncompressed size. - coder->lz.uncompressed_size -= coder->lz.pos - dict_old_pos; - - // Check that End of Payload Marker hasn't been detected - // since it must not be present because uncompressed size - // is known. - if (coder->lz.eopm_detected) - error = true; - } - - return error; -} - - static lzma_ret decode_buffer(lzma_coder *coder, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - bool flushing) + size_t *restrict out_pos, size_t out_size) { - bool stop = false; - while (true) { - // Flush from coder->lz.dict to out[]. - flush(&coder->lz, out, out_pos, out_size); - - // All done? - if (*out_pos == out_size - || stop - || coder->lz.eopm_detected - || coder->lz.uncompressed_size == 0) - break; - - // Set write limit in the dictionary. - if (set_limit(&coder->lz, out_size - *out_pos, flushing)) - break; - - // Decode more data. - if (call_process(coder, in, in_pos, in_size)) - return LZMA_DATA_ERROR; - - // Set stop to true if we must not call call_process() again - // during this function call. - // FIXME: Can this make the loop exist too early? It wouldn't - // cause data corruption so not a critical problem. It can - // happen if dictionary gets full and lz.temp still contains - // a few bytes data that we could decode right now. - if (*in_pos == in_size && coder->lz.temp_size <= TEMP_LIMIT - && coder->lz.pos < coder->lz.limit) - stop = true; + // Wrap the dictionary if needed. + if (coder->dict.pos == coder->dict.size) + coder->dict.pos = 0; + + // Store the current dictionary position. It is needed to know + // where to start copying to the out[] buffer. + const size_t dict_start = coder->dict.pos; + + // Calculate how much we allow the process() function to + // decode. It must not decode past the end of the dictionary + // buffer, and we don't want it to decode more than is + // actually needed to fill the out[] buffer. + coder->dict.limit = coder->dict.pos + MIN(out_size - *out_pos, + coder->dict.size - coder->dict.pos); + + // Call the process() function to do the actual decoding. + const lzma_ret ret = coder->lz.code( + coder->lz.coder, &coder->dict, + in, in_pos, in_size); + + // Copy the decoded data from the dictionary to the out[] + // buffer. + const size_t copy_size = coder->dict.pos - dict_start; + assert(copy_size <= out_size - *out_pos); + memcpy(out + *out_pos, coder->dict.buf + dict_start, + copy_size); + *out_pos += copy_size; + + // Return if everything got decoded or an error occurred, or + // if there's no more data to decode. + if (ret != LZMA_OK || *out_pos == out_size + || coder->dict.pos < coder->dict.size) + return ret; } - - // If we have decoded everything (EOPM detected or uncompressed_size - // bytes were processed) to the history buffer, and also flushed - // everything from the history buffer, our job is done. - if ((coder->lz.eopm_detected - || coder->lz.uncompressed_size == 0) - && coder->lz.start == coder->lz.pos) - return LZMA_STREAM_END; - - return LZMA_OK; } -extern lzma_ret -lzma_lz_decode(lzma_coder *coder, +static lzma_ret +lz_decode(lzma_coder *coder, lzma_allocator *allocator lzma_attribute((unused)), const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { - if (coder->next.code == NULL) { - const lzma_ret ret = decode_buffer(coder, in, in_pos, in_size, - out, out_pos, out_size, - action == LZMA_SYNC_FLUSH); - - if (*out_pos == out_size || ret == LZMA_STREAM_END) { - // Unread to make coder->temp[] empty. This is easy, - // because we know that all the data currently in - // coder->temp[] has been copied form in[] during this - // call to the decoder. - // - // If we didn't do this, we could have data left in - // coder->temp[] when end of stream is reached. That - // data could be left there from *previous* call to - // the decoder; in that case we wouldn't know where - // to put that data. - assert(*in_pos >= coder->lz.temp_size); - *in_pos -= coder->lz.temp_size; - coder->lz.temp_size = 0; - } - - return ret; - } + if (coder->next.code == NULL) + return decode_buffer(coder, in, in_pos, in_size, + out, out_pos, out_size); // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. - const bool flushing = action == LZMA_SYNC_FLUSH; while (*out_pos < out_size) { - if (!coder->lz.next_finished - && coder->lz.temp_size < LZMA_BUFFER_SIZE) { + // Fill the temporary buffer if it is empty. + if (!coder->next_finished + && coder->temp.pos == coder->temp.size) { + coder->temp.pos = 0; + coder->temp.size = 0; + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, - coder->lz.temp, &coder->lz.temp_size, + coder->temp.buffer, &coder->temp.size, LZMA_BUFFER_SIZE, action); if (ret == LZMA_STREAM_END) - coder->lz.next_finished = true; - else if (coder->lz.temp_size < LZMA_BUFFER_SIZE - || ret != LZMA_OK) + coder->next_finished = true; + else if (ret != LZMA_OK || coder->temp.size == 0) return ret; } - if (coder->lz.this_finished) { - if (coder->lz.temp_size != 0) + if (coder->this_finished) { + if (coder->temp.size != 0) return LZMA_DATA_ERROR; - if (coder->lz.next_finished) + if (coder->next_finished) return LZMA_STREAM_END; return LZMA_OK; } - size_t dummy = 0; - const lzma_ret ret = decode_buffer(coder, NULL, &dummy, 0, - out, out_pos, out_size, flushing); + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, + &coder->temp.pos, coder->temp.size, + out, out_pos, out_size); if (ret == LZMA_STREAM_END) - coder->lz.this_finished = true; + coder->this_finished = true; else if (ret != LZMA_OK) return ret; - else if (coder->lz.next_finished && *out_pos < out_size) + else if (coder->next_finished && *out_pos < out_size) return LZMA_DATA_ERROR; } @@ -370,94 +161,104 @@ lzma_lz_decode(lzma_coder *coder, } -/// \brief Initializes LZ part of the LZMA decoder or Inflate -/// -/// \param history_size Number of bytes the LZ out window is -/// supposed keep available from the output -/// history. -/// \param match_max_len Number of bytes a single decoding loop -/// can advance the write position (lz->pos) -/// in the history buffer (lz->dict). -/// -/// \note This function is called by LZMA decoder and Inflate init()s. -/// It's up to those functions allocate *lz and initialize it -/// with LZMA_LZ_DECODER_INIT. +static void +lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder->dict.buf, allocator); + + if (coder->lz.end != NULL) + coder->lz.end(coder->lz.coder, allocator); + else + lzma_free(coder->lz.coder, allocator); + + lzma_free(coder, allocator); + return; +} + + extern lzma_ret -lzma_lz_decoder_reset(lzma_lz_decoder *lz, lzma_allocator *allocator, - bool (*process)(lzma_coder *restrict coder, - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, bool has_safe_buffer), - size_t history_size, size_t match_max_len) +lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_decoder *lz, + lzma_allocator *allocator, const void *options, + size_t *dict_size)) { - // Known uncompressed size is used only with LZMA_Alone files so we - // set it always to unknown by default. - lz->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; - - // Limit the history size to roughly sane values. This is primarily - // to prevent integer overflows. - if (history_size > UINT32_MAX / 2) - return LZMA_HEADER_ERROR; - - // Store the value actually requested. We use it for sanity checks - // when repeating data from the history buffer. - lz->requested_size = history_size; - - // Avoid tiny history buffer sizes for performance reasons. - // TODO: Test if this actually helps... - if (history_size < DICT_SIZE_MIN) - history_size = DICT_SIZE_MIN; - - // The real size of the history buffer is a bit bigger than - // requested by our caller. This allows us to do some optimizations, - // which help not only speed but simplicity of the code; specifically, - // we can make sure that there is always at least match_max_len - // bytes immediatelly available for writing without a need to wrap - // the history buffer. - const size_t dict_real_size = history_size + 2 * match_max_len + 1; - - // Reallocate memory if needed. - if (history_size != lz->size || match_max_len != lz->match_max_len) { - // Destroy the old buffer. - lzma_lz_decoder_end(lz, allocator); - - lz->size = history_size; - lz->match_max_len = match_max_len; - lz->must_flush_pos = history_size + match_max_len + 1; - - lz->dict = lzma_alloc(dict_real_size, allocator); - if (lz->dict == NULL) + // Allocate the base structure if it isn't already allocated. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) return LZMA_MEM_ERROR; + + next->code = &lz_decode; + next->end = &lz_decoder_end; + + next->coder->dict.buf = NULL; + next->coder->dict.size = 0; + next->coder->lz = LZMA_LZ_DECODER_INIT; + next->coder->next = LZMA_NEXT_CODER_INIT; } - // Reset the variables so that lz_get_byte(lz, 0) will return '\0'. - lz->pos = 0; - lz->start = 0; - lz->end = dict_real_size; - lz->dict[dict_real_size - 1] = 0; - lz->is_full = false; - lz->eopm_detected = false; - lz->next_finished = false; - lz->this_finished = false; - lz->temp_size = 0; - - // Clean up the temporary buffer to make it very sure that there are - // no information leaks when multiple steams are decoded with the - // same decoder structures. - memzero(lz->temp, LZMA_BUFFER_SIZE); - - // Set the process function pointer. - lz->process = process; + // Allocate and initialize the LZ-based decoder. It will also give + // us the dictionary size. + size_t dict_size; + return_if_error(lz_init(&next->coder->lz, allocator, + filters[0].options, &dict_size)); + + // If the dictionary size is very small, increase it to 4096 bytes. + // This is to prevent constant wrapping of the dictionary, which + // would slow things down. The downside is that since we don't check + // separately for the real dictionary size, we may happily accept + // corrupt files. + if (dict_size < 4096) + dict_size = 4096; + + // Make dictionary size a multipe of 16. Some LZ-based decoders like + // LZMA use the lowest bits lzma_dict.pos to know the alignment of the + // data. Aligned buffer is also good when memcpying from the + // dictionary to the output buffer, since applications are + // recommended to give aligned buffers to liblzma. + // + // Avoid integer overflow. FIXME Should the return value be + // LZMA_HEADER_ERROR or LZMA_MEM_ERROR? + if (dict_size > SIZE_MAX - 15) + return LZMA_MEM_ERROR; + + dict_size = (dict_size + 15) & (SIZE_MAX - 15); + + // Allocate and initialize the dictionary. + if (next->coder->dict.size != dict_size) { + lzma_free(next->coder->dict.buf, allocator); + next->coder->dict.buf = lzma_alloc(dict_size, allocator); + if (next->coder->dict.buf == NULL) + return LZMA_MEM_ERROR; - return LZMA_OK; + next->coder->dict.size = dict_size; + } + + dict_reset(&next->coder->dict); + + // Miscellaneous initializations + next->coder->next_finished = false; + next->coder->this_finished = false; + next->coder->temp.pos = 0; + next->coder->temp.size = 0; + + // Initialize the next filter in the chain, if any. + return lzma_next_filter_init(&next->coder->next, allocator, + filters + 1); +} + + +extern uint64_t +lzma_lz_decoder_memusage(size_t dictionary_size) +{ + return sizeof(lzma_coder) + (uint64_t)(dictionary_size); } extern void -lzma_lz_decoder_end(lzma_lz_decoder *lz, lzma_allocator *allocator) +lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) { - lzma_free(lz->dict, allocator); - lz->dict = NULL; - lz->size = 0; - lz->match_max_len = 0; - return; + coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h index 1acf9831..d2a77ba4 100644 --- a/src/liblzma/lz/lz_decoder.h +++ b/src/liblzma/lz/lz_decoder.h @@ -18,201 +18,215 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_LZ_OUT_H -#define LZMA_LZ_OUT_H +#ifndef LZMA_LZ_DECODER_H +#define LZMA_LZ_DECODER_H #include "common.h" -/// Get a byte from the history buffer. -#define lz_get_byte(lz, distance) \ - ((distance) < (lz).pos \ - ? (lz).dict[(lz).pos - (distance) - 1] \ - : (lz).dict[(lz).pos - (distance) - 1 + (lz).end]) - - -/// Test if dictionary is empty. -#define lz_is_empty(lz) \ - ((lz).pos == 0 && !(lz).is_full) - - -#define LZMA_LZ_DECODER_INIT \ - (lzma_lz_decoder){ .dict = NULL, .size = 0, .match_max_len = 0 } - - typedef struct { - /// Function to do the actual decoding (LZMA or Inflate) - bool (*process)(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t size_in, - bool has_safe_buffer); + /// Pointer to the dictionary buffer. It can be an allocated buffer + /// internal to liblzma, or it can a be a buffer given by the + /// application when in single-call mode (not implemented yet). + uint8_t *buf; - /// Pointer to dictionary (history) buffer. - /// \note Not 'restrict' because can alias next_out. - uint8_t *dict; - - /// Next write goes to dict[pos]. + /// Write position in dictionary. The next byte will be written to + /// buf[pos]. size_t pos; - /// Next byte to flush is buffer[start]. - size_t start; - - /// First byte to not flush is buffer[end]. - size_t end; + /// Indicates how full the dictionary is. This is used by + /// dict_is_distance_valid() to detect corrupt files that would + /// read beyond the beginning of the dictionary. + size_t full; - /// First position to which data must not be written. + /// Write limit size_t limit; - /// True if dictionary has needed wrapping. - bool is_full; - - /// True if process() has detected End of Payload Marker. - bool eopm_detected; + /// Size of the dictionary + size_t size; - /// True if the next coder in the chain has returned LZMA_STREAM_END. - bool next_finished; +} lzma_dict; - /// True if the LZ decoder (e.g. LZMA) has detected End of Payload - /// Marker. This may become true before next_finished becomes true. - bool this_finished; - /// When pos >= must_flush_pos, we must not call process(). - size_t must_flush_pos; +typedef struct { + /// Data specific to the LZ-based decoder + lzma_coder *coder; - /// Maximum number of bytes that a single decoding loop inside - /// process() can produce data into dict. This amount is kept - /// always available at dict + pos i.e. it is safe to write a byte - /// to dict[pos + match_max_len - 1]. - size_t match_max_len; + /// Function to decode from in[] to *dict + lzma_ret (*code)(lzma_coder *restrict coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - /// Number of bytes allocated to dict. - size_t size; + void (*reset)(lzma_coder *coder, const void *options); - /// Requested size of the dictionary. This is needed because we avoid - /// using extremely tiny history buffers. - size_t requested_size; + /// Set the uncompressed size + void (*set_uncompressed)(lzma_coder *coder, + lzma_vli uncompressed_size); - /// Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN if unknown. - lzma_vli uncompressed_size; + /// Free allocated resources + void (*end)(lzma_coder *coder, lzma_allocator *allocator); - /// Number of bytes currently in temp[]. - size_t temp_size; +} lzma_lz_decoder; - /// Temporary buffer needed when - /// 1) we cannot make the input buffer completely empty; or - /// 2) we are not the last filter in the chain. - uint8_t temp[LZMA_BUFFER_SIZE]; -} lzma_lz_decoder; +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } -///////////////////////// -// Function prototypes // -///////////////////////// +extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_decoder *lz, + lzma_allocator *allocator, const void *options, + size_t *dict_size)); -extern lzma_ret lzma_lz_decoder_reset(lzma_lz_decoder *lz, - lzma_allocator *allocator, bool (*process)( - lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size, - bool has_safe_buffer), - size_t history_size, size_t match_max_len); +extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); -extern lzma_ret lzma_lz_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - lzma_action action); +extern void lzma_lz_decoder_uncompressed( + lzma_coder *coder, lzma_vli uncompressed_size); -/// Deallocates the history buffer if one exists. -extern void lzma_lz_decoder_end( - lzma_lz_decoder *lz, lzma_allocator *allocator); ////////////////////// // Inline functions // ////////////////////// -// Repeat a block of data from the history. Because memcpy() is faster -// than copying byte by byte in a loop, the copying process gets split -// into three cases: -// 1. distance < length -// Source and target areas overlap, thus we can't use memcpy() -// (nor memmove()) safely. -// TODO: If this is common enough, it might be worth optimizing this -// more e.g. by checking if distance > sizeof(uint8_t*) and using -// memcpy in small chunks. -// 2. distance < pos -// This is the easiest and the fastest case. The block being copied -// is a contiguous piece in the history buffer. The buffer offset -// doesn't need wrapping. -// 3. distance >= pos -// We need to wrap the position, because otherwise we would try copying -// behind the first byte of the allocated buffer. It is possible that -// the block is fragmeneted into two pieces, thus we might need to call -// memcpy() twice. -// NOTE: The function using this macro must ensure that length is positive -// and that distance is FIXME +/// Get a byte from the history buffer. +static inline uint8_t +dict_get(const lzma_dict *const dict, const uint32_t distance) +{ + return dict->buf[dict->pos - distance - 1 + + (distance < dict->pos ? 0 : dict->size)]; +} + + +/// Test if dictionary is empty. +static inline bool +dict_is_empty(const lzma_dict *const dict) +{ + return dict->full == 0; +} + + +/// Validate the match distance +static inline bool +dict_is_distance_valid(const lzma_dict *const dict, const size_t distance) +{ + return dict->full >= distance; +} + + +/// Repeat *len bytes at distance. static inline bool -lzma_lz_out_repeat(lzma_lz_decoder *lz, size_t distance, size_t length) +dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) { - // Validate offset of the block to be repeated. It doesn't - // make sense to copy data behind the beginning of the stream. - // Leaving this check away would lead to a security problem, - // in which e.g. the data of the previously decoded file(s) - // would be leaked (or whatever happens to be in unused - // part of the dictionary buffer). - if (unlikely(distance >= lz->pos && !lz->is_full)) - return false; - - // It also doesn't make sense to copy data farer than - // the dictionary size. - if (unlikely(distance >= lz->requested_size)) - return false; - - // The caller must have checked these! - assert(distance <= lz->size); - assert(length > 0); - assert(length <= lz->match_max_len); - - // Copy the amount of data requested by the decoder. - if (distance < length) { + // Don't write past the end of the dictionary. + const size_t dict_avail = dict->limit - dict->pos; + uint32_t left = MIN(dict_avail, *len); + *len -= left; + + // Repeat a block of data from the history. Because memcpy() is faster + // than copying byte by byte in a loop, the copying process gets split + // into three cases. + if (distance < left) { // Source and target areas overlap, thus we can't use - // memcpy() nor even memmove() safely. :-( - // TODO: Copying byte by byte is slow. It might be - // worth optimizing this more if this case is common. + // memcpy() nor even memmove() safely. do { - lz->dict[lz->pos] = lz_get_byte(*lz, distance); - ++lz->pos; - } while (--length > 0); + dict->buf[dict->pos] = dict_get(dict, distance); + ++dict->pos; + } while (--left > 0); - } else if (distance < lz->pos) { + } else if (distance < dict->pos) { // The easiest and fastest case - memcpy(lz->dict + lz->pos, - lz->dict + lz->pos - distance - 1, - length); - lz->pos += length; + memcpy(dict->buf + dict->pos, + dict->buf + dict->pos - distance - 1, + left); + dict->pos += left; } else { // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. - assert(lz->is_full); - const uint32_t copy_pos = lz->pos - distance - 1 + lz->end; - uint32_t copy_size = lz->end - copy_pos; + assert(dict->full == dict->size); + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; - if (copy_size < length) { - memcpy(lz->dict + lz->pos, lz->dict + copy_pos, + if (copy_size < left) { + memcpy(dict->buf + dict->pos, dict->buf + copy_pos, copy_size); - lz->pos += copy_size; - copy_size = length - copy_size; - memcpy(lz->dict + lz->pos, lz->dict, copy_size); - lz->pos += copy_size; + dict->pos += copy_size; + copy_size = left - copy_size; + memcpy(dict->buf + dict->pos, dict->buf, copy_size); + dict->pos += copy_size; } else { - memcpy(lz->dict + lz->pos, lz->dict + copy_pos, - length); - lz->pos += length; + memcpy(dict->buf + dict->pos, dict->buf + copy_pos, + left); + dict->pos += left; } } - return true; + // Update how full the dictionary is. + if (dict->full < dict->pos) + dict->full = dict->pos; + + return unlikely(*len != 0); +} + + +/// Puts one byte into the dictionary. Returns true if the dictionary was +/// already full and the byte couldn't be added. +static inline bool +dict_put(lzma_dict *dict, uint8_t byte) +{ + if (unlikely(dict->pos == dict->limit)) + return true; + + dict->buf[dict->pos++] = byte; + + if (dict->pos > dict->full) + dict->full = dict->pos; + + return false; +} + + +/// Copies arbitrary amount of data into the dictionary. +static inline void +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) +{ + // NOTE: If we are being given more data than the size of the + // dictionary, it could be possible to optimize the LZ decoder + // so that not everything needs to go through the dictionary. + // This shouldn't be very common thing in practice though, and + // the slowdown of one extra memcpy() isn't bad compared to how + // much time it would have taken if the data were compressed. + + if (in_size - *in_pos > *left) + in_size = *in_pos + *left; + + *left -= lzma_bufcpy(in, in_pos, in_size, + dict->buf, &dict->pos, dict->limit); + + if (dict->pos > dict->full) + dict->full = dict->pos; + + return; +} + + +static inline void +dict_reset(lzma_dict *dict) +{ + dict->pos = 0; + dict->full = 0; + dict->buf[dict->size - 1] = '\0'; } #endif diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c index 82b9103f..d5f84826 100644 --- a/src/liblzma/lz/lz_encoder.c +++ b/src/liblzma/lz/lz_encoder.c @@ -3,8 +3,8 @@ /// \file lz_encoder.c /// \brief LZ in window // -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -18,496 +18,492 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "lz_encoder_private.h" +#include "lz_encoder.h" +#include "lz_encoder_hash.h" -// Hash Chains -#ifdef HAVE_HC3 -# include "hc3.h" -#endif -#ifdef HAVE_HC4 -# include "hc4.h" -#endif -// Binary Trees -#ifdef HAVE_BT2 -# include "bt2.h" -#endif -#ifdef HAVE_BT3 -# include "bt3.h" -#endif -#ifdef HAVE_BT4 -# include "bt4.h" -#endif +struct lzma_coder_s { + /// LZ-based encoder e.g. LZMA + lzma_lz_encoder lz; + /// History buffer and match finder + lzma_mf mf; -/// This is needed in two places so provide a macro. -#define get_cyclic_buffer_size(history_size) ((history_size) + 1) + /// Next coder in the chain + lzma_next_coder next; +}; -/// Calculate certain match finder properties and validate the calculated -/// values. This is as its own function, because *num_items is needed to -/// calculate memory requirements in common/memory.c. -extern bool -lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, - uint32_t history_size, uint32_t *restrict hash_mask, - uint32_t *restrict hash_size_sum, uint32_t *restrict num_items) +/// \brief Moves the data in the input window to free space for new data +/// +/// mf->buffer is a sliding input window, which keeps mf->keep_size_before +/// bytes of input history available all the time. Now and then we need to +/// "slide" the buffer to make space for the new data to the end of the +/// buffer. At the same time, data older than keep_size_before is dropped. +/// +static void +move_window(lzma_mf *mf) { - uint32_t fix_hash_size; - uint32_t sons; + // Align the move to a multiple of 16 bytes. Some LZ-based encoders + // like LZMA use the lowest bits of mf->read_pos to know the + // alignment of the uncompressed data. We also get better speed + // for memmove() with aligned buffers. + assert(mf->read_pos > mf->keep_size_before); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); - switch (match_finder) { -#ifdef HAVE_HC3 - case LZMA_MF_HC3: - fix_hash_size = LZMA_HC3_FIX_HASH_SIZE; - sons = 1; - break; -#endif -#ifdef HAVE_HC4 - case LZMA_MF_HC4: - fix_hash_size = LZMA_HC4_FIX_HASH_SIZE; - sons = 1; - break; -#endif -#ifdef HAVE_BT2 - case LZMA_MF_BT2: - fix_hash_size = LZMA_BT2_FIX_HASH_SIZE; - sons = 2; - break; -#endif -#ifdef HAVE_BT3 - case LZMA_MF_BT3: - fix_hash_size = LZMA_BT3_FIX_HASH_SIZE; - sons = 2; - break; -#endif -#ifdef HAVE_BT4 - case LZMA_MF_BT4: - fix_hash_size = LZMA_BT4_FIX_HASH_SIZE; - sons = 2; - break; -#endif - default: - return true; - } + assert(mf->write_pos > move_offset); + const size_t move_size = mf->write_pos - move_offset; - uint32_t hs; + assert(move_offset + move_size <= mf->size); -#ifdef HAVE_LZMA_BT2 - if (match_finder == LZMA_BT2) { - // NOTE: hash_mask is not used by the BT2 match finder, - // but it is initialized just in case. - hs = LZMA_BT2_HASH_SIZE; - *hash_mask = 0; - } else -#endif - { - hs = history_size - 1; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - hs >>= 1; - hs |= 0xFFFF; + memmove(mf->buffer, mf->buffer + move_offset, move_size); - if (hs > (UINT32_C(1) << 24)) { - if (match_finder == LZMA_MF_HC4 - || match_finder == LZMA_MF_BT4) - hs >>= 1; - else - hs = (1 << 24) - 1; - } + mf->offset += move_offset; + mf->read_pos -= move_offset; + mf->read_limit -= move_offset; + mf->write_pos -= move_offset; + + return; +} - *hash_mask = hs; - ++hs; - } - *hash_size_sum = hs + fix_hash_size; +/// \brief Tries to fill the input window (mf->buffer) +/// +/// If we are the last encoder in the chain, our input data is in in[]. +/// Otherwise we call the next filter in the chain to process in[] and +/// write its output to mf->buffer. +/// +/// This function must not be called once it has returned LZMA_STREAM_END. +/// +static lzma_ret +fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, + size_t *in_pos, size_t in_size, lzma_action action) +{ + assert(coder->mf.read_pos <= coder->mf.write_pos); - *num_items = *hash_size_sum - + get_cyclic_buffer_size(history_size) * sons; + // Move the sliding window if needed. + if (coder->mf.read_pos >= coder->mf.size - coder->mf.keep_size_after) + move_window(&coder->mf); - return false; -} + size_t in_used; + lzma_ret ret; + if (coder->next.code == NULL) { + // Not using a filter, simply memcpy() as much as possible. + in_used = lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, + &coder->mf.write_pos, coder->mf.size); + ret = action != LZMA_RUN && *in_pos == in_size + ? LZMA_STREAM_END : LZMA_OK; -extern lzma_ret -lzma_lz_encoder_reset(lzma_lz_encoder *lz, lzma_allocator *allocator, - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size), - size_t history_size, size_t additional_buffer_before, - size_t match_max_len, size_t additional_buffer_after, - lzma_match_finder match_finder, uint32_t match_finder_cycles, - const uint8_t *preset_dictionary, - size_t preset_dictionary_size) -{ - lz->sequence = SEQ_RUN; + } else { + const size_t in_start = *in_pos; + ret = coder->next.code(coder->next.coder, allocator, + in, in_pos, in_size, + coder->mf.buffer, &coder->mf.write_pos, + coder->mf.size, action); + in_used = *in_pos - in_start; + } - /////////////// - // In Window // - /////////////// + // If end of stream has been reached or flushing completed, we allow + // the encoder to process all the input (that is, read_pos is allowed + // to reach write_pos). Otherwise we keep keep_size_after bytes + // available as prebuffer. + if (ret == LZMA_STREAM_END) { + assert(*in_pos == in_size); + ret = LZMA_OK; + coder->mf.action = action; + coder->mf.read_limit = coder->mf.write_pos; - // Validate history size. - if (history_size < LZMA_DICTIONARY_SIZE_MIN - || history_size > LZMA_DICTIONARY_SIZE_MAX) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; + } else if (coder->mf.write_pos > coder->mf.keep_size_after) { + // This needs to be done conditionally, because if we got + // only little new input, there may be too little input + // to do any encoding yet. + coder->mf.read_limit = coder->mf.write_pos + - coder->mf.keep_size_after; } - assert(history_size <= MAX_VAL_FOR_NORMALIZE - 256); - assert(LZMA_DICTIONARY_SIZE_MAX <= MAX_VAL_FOR_NORMALIZE - 256); + // Restart the match finder after finished LZMA_SYNC_FLUSH. + if (coder->mf.pending > 0 + && coder->mf.read_pos < coder->mf.read_limit) { + // Match finder may update coder->pending and expects it to + // start from zero, so use a temporary variable. + const size_t pending = coder->mf.pending; + coder->mf.pending = 0; - // Calculate the size of the history buffer to allocate. - // TODO: Get a reason for magic constant of 256. - const size_t size_reserv = (history_size + additional_buffer_before - + match_max_len + additional_buffer_after) / 2 + 256; + // Rewind read_pos so that the match finder can hash + // the pending bytes. + assert(coder->mf.read_pos >= pending); + coder->mf.read_pos -= pending; - lz->keep_size_before = history_size + additional_buffer_before; - lz->keep_size_after = match_max_len + additional_buffer_after; + // Call the skip function directly instead of using + // lz_dict_skip(), since we don't want to touch + // mf->read_ahead. + coder->mf.skip(&coder->mf, pending); + } - const size_t buffer_size = lz->keep_size_before + lz->keep_size_after - + size_reserv; + return ret; +} - // Allocate history buffer if its size has changed. - if (buffer_size != lz->size) { - lzma_free(lz->buffer, allocator); - lz->buffer = lzma_alloc(buffer_size, allocator); - if (lz->buffer == NULL) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; + +static lzma_ret +lz_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_action action) +{ + while (*out_pos < out_size + && (*in_pos < in_size || action != LZMA_RUN)) { + // Read more data to coder->mf.buffer if needed. + if (coder->mf.action == LZMA_RUN && coder->mf.read_pos + >= coder->mf.read_limit) + return_if_error(fill_window(coder, allocator, + in, in_pos, in_size, action)); + + // Encode + const lzma_ret ret = coder->lz.code(coder->lz.coder, + &coder->mf, out, out_pos, out_size); + if (ret != LZMA_OK) { + // Setting this to LZMA_RUN for cases when we are + // flushing. It doesn't matter when finishing or if + // an error occurred. + coder->mf.action = LZMA_RUN; + return ret; } } - // Allocation successful. Store the new size. - lz->size = buffer_size; + return LZMA_OK; +} + + +static bool +lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, + const lzma_lz_options *lz_options) +{ + if (lz_options->dictionary_size < LZMA_DICTIONARY_SIZE_MIN + || lz_options->dictionary_size + > LZMA_DICTIONARY_SIZE_MAX + || lz_options->find_len_max + > lz_options->match_len_max) + return true; + + mf->keep_size_before = lz_options->before_size + + lz_options->dictionary_size; - // Reset in window variables. - lz->offset = 0; - lz->read_pos = 0; - lz->read_limit = 0; - lz->write_pos = 0; - lz->pending = 0; + mf->keep_size_after = lz_options->after_size + + lz_options->match_len_max; + // To avoid constant memmove()s, allocate some extra space. Since + // memmove()s become more expensive when the size of the buffer + // increases, we reserve more space when a large dictionary is + // used to make the memmove() calls rarer. + uint32_t reserve = lz_options->dictionary_size / 2; + if (reserve > (UINT32_C(1) << 30)) + reserve /= 2; - ////////////////// - // Match Finder // - ////////////////// + reserve += (lz_options->before_size + lz_options->match_len_max + + lz_options->after_size) / 2 + (UINT32_C(1) << 19); - // Validate match_finder, set function pointers and a few match - // finder specific variables. - switch (match_finder) { -#ifdef HAVE_HC3 + const uint32_t old_size = mf->size; + mf->size = mf->keep_size_before + reserve + mf->keep_size_after; + + // FIXME Integer overflows + + // Deallocate the old history buffer if it exists but has different + // size than what is needed now. + if (mf->buffer != NULL && old_size != mf->size) { + lzma_free(mf->buffer, allocator); + mf->buffer = NULL; + } + + // Match finder options + mf->match_len_max = lz_options->match_len_max; + mf->find_len_max = lz_options->find_len_max; + mf->cyclic_buffer_size = lz_options->dictionary_size + 1; + + // Validate the match finder ID and setup the function pointers. + switch (lz_options->match_finder) { +#ifdef HAVE_MF_HC3 case LZMA_MF_HC3: - lz->get_matches = &lzma_hc3_get_matches; - lz->skip = &lzma_hc3_skip; - lz->cut_value = 8 + (match_max_len >> 2); + mf->find = &lzma_mf_hc3_find; + mf->skip = &lzma_mf_hc3_skip; break; #endif -#ifdef HAVE_HC4 +#ifdef HAVE_MF_HC4 case LZMA_MF_HC4: - lz->get_matches = &lzma_hc4_get_matches; - lz->skip = &lzma_hc4_skip; - lz->cut_value = 8 + (match_max_len >> 2); + mf->find = &lzma_mf_hc4_find; + mf->skip = &lzma_mf_hc4_skip; break; #endif -#ifdef HAVE_BT2 +#ifdef HAVE_MF_BT2 case LZMA_MF_BT2: - lz->get_matches = &lzma_bt2_get_matches; - lz->skip = &lzma_bt2_skip; - lz->cut_value = 16 + (match_max_len >> 1); + mf->find = &lzma_mf_bt2_find; + mf->skip = &lzma_mf_bt2_skip; break; #endif -#ifdef HAVE_BT3 +#ifdef HAVE_MF_BT3 case LZMA_MF_BT3: - lz->get_matches = &lzma_bt3_get_matches; - lz->skip = &lzma_bt3_skip; - lz->cut_value = 16 + (match_max_len >> 1); + mf->find = &lzma_mf_bt3_find; + mf->skip = &lzma_mf_bt3_skip; break; #endif -#ifdef HAVE_BT4 +#ifdef HAVE_MF_BT4 case LZMA_MF_BT4: - lz->get_matches = &lzma_bt4_get_matches; - lz->skip = &lzma_bt4_skip; - lz->cut_value = 16 + (match_max_len >> 1); + mf->find = &lzma_mf_bt4_find; + mf->skip = &lzma_mf_bt4_skip; break; #endif + default: - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; + return true; } - // Check if we have been requested to use a non-default cut_value. - if (match_finder_cycles > 0) - lz->cut_value = match_finder_cycles; - - lz->match_max_len = match_max_len; - lz->cyclic_buffer_size = get_cyclic_buffer_size(history_size); + // Calculate the sizes of mf->hash and mf->son. + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; - uint32_t hash_size_sum; - uint32_t num_items; - if (lzma_lz_encoder_hash_properties(match_finder, history_size, - &lz->hash_mask, &hash_size_sum, &num_items)) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; - } + if (hash_bytes == 2) { + hs = 0xFFFF; + } else { + // Round dictionary size up to the next 2^n - 1 so it can + // be used as a hash mask. + hs = lz_options->dictionary_size - 1; + hs |= hs >> 1; + hs |= hs >> 2; + hs |= hs >> 4; + hs |= hs >> 8; + hs >>= 1; + hs |= 0xFFFF; - if (num_items != lz->num_items) { -#if UINT32_MAX >= SIZE_MAX / 4 - // Check for integer overflow. (Huge dictionaries are not - // possible on 32-bit CPU.) - if (num_items > SIZE_MAX / sizeof(uint32_t)) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; + if (hs > (UINT32_C(1) << 24)) { + if (hash_bytes == 3) + hs = (UINT32_C(1) << 24) - 1; + else + hs >>= 1; } -#endif - - const size_t size_in_bytes - = (size_t)(num_items) * sizeof(uint32_t); + } - lzma_free(lz->hash, allocator); - lz->hash = lzma_alloc(size_in_bytes, allocator); - if (lz->hash == NULL) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; - } + mf->hash_mask = hs; + + ++hs; + if (hash_bytes > 2) + hs += HASH_2_SIZE; + if (hash_bytes > 3) + hs += HASH_3_SIZE; +/* + No match finder uses this at the moment. + if (mf->hash_bytes > 4) + hs += HASH_4_SIZE; +*/ + + const uint32_t old_count = mf->hash_size_sum + mf->sons_count; + mf->hash_size_sum = hs; + mf->sons_count = mf->cyclic_buffer_size; + if (is_bt) + mf->sons_count *= 2; + + const uint32_t new_count = mf->hash_size_sum + mf->sons_count; + + // Deallocate the old hash array if it exists and has different size + // than what is needed now. + if (mf->hash != NULL && old_count != new_count) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + } - lz->num_items = num_items; + // Maximum number of match finder cycles + mf->loops = lz_options->match_finder_cycles; + if (mf->loops == 0) { + mf->loops = 16 + (lz_options->find_len_max / 2); + if (!is_bt) + mf->loops /= 2; } - lz->son = lz->hash + hash_size_sum; + return false; +} - // Reset the hash table to empty hash values. - { - uint32_t *restrict items = lz->hash; - for (uint32_t i = 0; i < hash_size_sum; ++i) - items[i] = EMPTY_HASH_VALUE; +static bool +lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator) +{ + // Allocate the history buffer. + if (mf->buffer == NULL) { + mf->buffer = lzma_alloc(mf->size, allocator); + if (mf->buffer == NULL) + return true; } - lz->cyclic_buffer_pos = 0; + // Use cyclic_buffer_size as initial mf->offset. This allows + // avoiding a few branches in the match finders. The downside is + // that match finder needs to be normalized more often, which may + // hurt performance with huge dictionaries. + mf->offset = mf->cyclic_buffer_size; + mf->read_pos = 0; + mf->read_ahead = 0; + mf->read_limit = 0; + mf->write_pos = 0; + mf->pending = 0; - // Because zero is used as empty hash value, make the first byte - // appear at buffer[1 - offset]. - ++lz->offset; + // Allocate match finder's hash array. + const size_t alloc_count = mf->hash_size_sum + mf->sons_count; - // If we are using a preset dictionary, read it now. - // TODO: This isn't implemented yet so return LZMA_HEADER_ERROR. - if (preset_dictionary != NULL && preset_dictionary_size > 0) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; +#if UINT32_MAX >= SIZE_MAX / 4 + // Check for integer overflow. (Huge dictionaries are not + // possible on 32-bit CPU.) + if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + return true; +#endif + + if (mf->hash == NULL) { + mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + allocator); + if (mf->hash == NULL) + return true; } - // Set the process function pointer. - lz->process = process; + mf->son = mf->hash + mf->hash_size_sum; + mf->cyclic_buffer_pos = 0; + + // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we + // can use memset(). +/* + for (uint32_t i = 0; i < hash_size_sum; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; +*/ + memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + + // We don't need to initialize mf->son, but not doing that will + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). + // + // Skipping this initialization is *very* good when big dictionary is + // used but only small amount of data gets actually compressed: most + // of the mf->hash won't get actually allocated by the kernel, so + // we avoid wasting RAM and improve initialization speed a lot. + //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + + mf->action = LZMA_RUN; - return LZMA_OK; + return false; } -extern void -lzma_lz_encoder_end(lzma_lz_encoder *lz, lzma_allocator *allocator) +extern uint64_t +lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) { - lzma_free(lz->hash, allocator); - lz->hash = NULL; - lz->num_items = 0; - - lzma_free(lz->buffer, allocator); - lz->buffer = NULL; - lz->size = 0; - - return; + // Old buffers must not exist when calling lz_encoder_prepare(). + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + }; + + // Setup the size information into mf. + if (lz_encoder_prepare(&mf, NULL, lz_options)) + return UINT64_MAX; + + // Calculate the memory usage. + return (uint64_t)(mf.hash_size_sum + mf.sons_count) + * sizeof(uint32_t) + + (uint64_t)(mf.size) + sizeof(lzma_coder); } -/// \brief Moves the data in the input window to free space for new data -/// -/// lz->buffer is a sliding input window, which keeps lz->keep_size_before -/// bytes of input history available all the time. Now and then we need to -/// "slide" the buffer to make space for the new data to the end of the -/// buffer. At the same time, data older than keep_size_before is dropped. -/// static void -move_window(lzma_lz_encoder *lz) +lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - // buffer[move_offset] will become buffer[0]. - assert(lz->read_pos > lz->keep_size_after); - size_t move_offset = lz->read_pos - lz->keep_size_before; - - // We need one additional byte, since move_pos() moves on 1 byte. - // TODO: Clean up? At least document more. - if (move_offset > 0) - --move_offset; - - assert(lz->write_pos > move_offset); - const size_t move_size = lz->write_pos - move_offset; + lzma_next_end(&coder->next, allocator); - assert(move_offset + move_size <= lz->size); + lzma_free(coder->mf.hash, allocator); + lzma_free(coder->mf.buffer, allocator); - memmove(lz->buffer, lz->buffer + move_offset, move_size); - - lz->offset += move_offset; - lz->read_pos -= move_offset; - lz->read_limit -= move_offset; - lz->write_pos -= move_offset; + if (coder->lz.end != NULL) + coder->lz.end(coder->lz.coder, allocator); + else + lzma_free(coder->lz.coder, allocator); + lzma_free(coder, allocator); return; } -/// \brief Tries to fill the input window (lz->buffer) -/// -/// If we are the last encoder in the chain, our input data is in in[]. -/// Otherwise we call the next filter in the chain to process in[] and -/// write its output to lz->buffer. -/// -/// This function must not be called once it has returned LZMA_STREAM_END. -/// -static lzma_ret -fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, - size_t *in_pos, size_t in_size, lzma_action action) +extern lzma_ret +lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_encoder *lz, + lzma_allocator *allocator, const void *options, + lzma_lz_options *lz_options)) { - assert(coder->lz.read_pos <= coder->lz.write_pos); + // Allocate and initialize the base data structure. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; - // Move the sliding window if needed. - if (coder->lz.read_pos >= coder->lz.size - coder->lz.keep_size_after) - move_window(&coder->lz); + next->code = &lz_encode; + next->end = &lz_encoder_end; - size_t in_used; - lzma_ret ret; - if (coder->next.code == NULL) { - // Not using a filter, simply memcpy() as much as possible. - in_used = bufcpy(in, in_pos, in_size, coder->lz.buffer, - &coder->lz.write_pos, coder->lz.size); + next->coder->lz.coder = NULL; + next->coder->lz.code = NULL; + next->coder->lz.end = NULL; - if (action != LZMA_RUN && *in_pos == in_size) - ret = LZMA_STREAM_END; - else - ret = LZMA_OK; + next->coder->mf.buffer = NULL; + next->coder->mf.hash = NULL; - } else { - const size_t in_start = *in_pos; - ret = coder->next.code(coder->next.coder, allocator, - in, in_pos, in_size, - coder->lz.buffer, &coder->lz.write_pos, - coder->lz.size, action); - in_used = *in_pos - in_start; + next->coder->next = LZMA_NEXT_CODER_INIT; } - // If end of stream has been reached or flushing completed, we allow - // the encoder to process all the input (that is, read_pos is allowed - // to reach write_pos). Otherwise we keep keep_size_after bytes - // available as prebuffer. - if (ret == LZMA_STREAM_END) { - assert(*in_pos == in_size); - coder->lz.read_limit = coder->lz.write_pos; - ret = LZMA_OK; + // Initialize the LZ-based encoder. + lzma_lz_options lz_options; + return_if_error(lz_init(&next->coder->lz, allocator, + filters[0].options, &lz_options)); - switch (action) { - case LZMA_SYNC_FLUSH: - coder->lz.sequence = SEQ_FLUSH; - break; - - case LZMA_FINISH: - coder->lz.sequence = SEQ_FINISH; - break; - - default: - assert(0); - ret = LZMA_PROG_ERROR; - break; - } - - } else if (coder->lz.write_pos > coder->lz.keep_size_after) { - // This needs to be done conditionally, because if we got - // only little new input, there may be too little input - // to do any encoding yet. - coder->lz.read_limit = coder->lz.write_pos - - coder->lz.keep_size_after; - } - - // Restart the match finder after finished LZMA_SYNC_FLUSH. - if (coder->lz.pending > 0 - && coder->lz.read_pos < coder->lz.read_limit) { - // Match finder may update coder->pending and expects it to - // start from zero, so use a temporary variable. - const size_t pending = coder->lz.pending; - coder->lz.pending = 0; + // Setup the size information into next->coder->mf and deallocate + // old buffers if they have wrong size. + if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + return LZMA_HEADER_ERROR; - // Rewind read_pos so that the match finder can hash - // the pending bytes. - assert(coder->lz.read_pos >= pending); - coder->lz.read_pos -= pending; - coder->lz.skip(&coder->lz, pending); - } + // Allocate new buffers if needed, and do the rest of + // the initialization. + if (lz_encoder_init(&next->coder->mf, allocator)) + return LZMA_MEM_ERROR; - return ret; + // Initialize the next filter in the chain, if any. + return lzma_next_filter_init(&next->coder->next, allocator, + filters + 1); } -extern lzma_ret -lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, - uint8_t *restrict out, size_t *restrict out_pos, - size_t out_size, lzma_action action) +extern LZMA_API lzma_bool +lzma_mf_is_supported(lzma_match_finder mf) { - while (*out_pos < out_size - && (*in_pos < in_size || action != LZMA_RUN)) { - // Read more data to coder->lz.buffer if needed. - if (coder->lz.sequence == SEQ_RUN - && coder->lz.read_pos >= coder->lz.read_limit) - return_if_error(fill_window(coder, allocator, - in, in_pos, in_size, action)); + bool ret = false; - // Encode - if (coder->lz.process(coder, out, out_pos, out_size)) { - // Setting this to SEQ_RUN for cases when we are - // flushing. It doesn't matter when finishing. - coder->lz.sequence = SEQ_RUN; - return action != LZMA_RUN ? LZMA_STREAM_END : LZMA_OK; - } - } +#ifdef HAVE_MF_HC3 + if (mf == LZMA_MF_HC3) + ret = true; +#endif - return LZMA_OK; -} +#ifdef HAVE_MF_HC4 + if (mf == LZMA_MF_HC4) + ret = true; +#endif +#ifdef HAVE_MF_BT2 + if (mf == LZMA_MF_BT2) + ret = true; +#endif -/// \brief Normalizes hash values -/// -/// lzma_lz_normalize is called when lz->pos hits MAX_VAL_FOR_NORMALIZE, -/// which currently happens once every 2 GiB of input data (to be exact, -/// after the first 2 GiB it happens once every 2 GiB minus dictionary_size -/// bytes). lz->pos is incremented by lzma_lz_move_pos(). -/// -/// lz->hash contains big amount of offsets relative to lz->buffer. -/// The offsets are stored as uint32_t, which is the only reasonable -/// datatype for these offsets; uint64_t would waste far too much RAM -/// and uint16_t would limit the dictionary to 64 KiB (far too small). -/// -/// When compressing files over 2 GiB, lz->buffer needs to be moved forward -/// to avoid integer overflows. We scan the lz->hash array and fix every -/// value to match the updated lz->buffer. -extern void -lzma_lz_encoder_normalize(lzma_lz_encoder *lz) -{ - const uint32_t subvalue = lz->read_pos - lz->cyclic_buffer_size; - assert(subvalue <= INT32_MAX); - - { - const uint32_t num_items = lz->num_items; - uint32_t *restrict items = lz->hash; - - for (uint32_t i = 0; i < num_items; ++i) { - // If the distance is greater than the dictionary - // size, we can simply mark the item as empty. - if (items[i] <= subvalue) - items[i] = EMPTY_HASH_VALUE; - else - items[i] -= subvalue; - } - } +#ifdef HAVE_MF_BT3 + if (mf == LZMA_MF_BT3) + ret = true; +#endif - // Update offset to match the new locations. - lz->offset -= subvalue; +#ifdef HAVE_MF_BT4 + if (mf == LZMA_MF_BT4) + ret = true; +#endif - return; + return ret; } diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h index da0e0804..45bb8462 100644 --- a/src/liblzma/lz/lz_encoder.h +++ b/src/liblzma/lz/lz_encoder.h @@ -3,8 +3,8 @@ /// \file lz_encoder.h /// \brief LZ in window and match finder API // -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -24,19 +24,16 @@ #include "common.h" -typedef struct lzma_lz_encoder_s lzma_lz_encoder; -struct lzma_lz_encoder_s { - enum { - SEQ_RUN, - SEQ_FLUSH, - SEQ_FINISH, - } sequence; +/// A table of these is used by the LZ-based encoder to hold +/// the length-distance pairs found by the match finder. +typedef struct { + uint32_t len; + uint32_t dist; +} lzma_match; - /// Function to do the actual encoding from the sliding input window - /// to the output stream. - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size); +typedef struct lzma_mf_s lzma_mf; +struct lzma_mf_s { /////////////// // In Window // /////////////// @@ -46,17 +43,33 @@ struct lzma_lz_encoder_s { /// Total size of the allocated buffer (that is, including all /// the extra space) - size_t size; + uint32_t size; + + /// Number of bytes that must be kept available in our input history. + /// That is, once keep_size_before bytes have been processed, + /// buffer[read_pos - keep_size_before] is the oldest byte that + /// must be available for reading. + uint32_t keep_size_before; + + /// Number of bytes that must be kept in buffer after read_pos. + /// That is, read_pos <= write_pos - keep_size_after as long as + /// stream_end_was_reached is false (once it is true, read_pos + /// is allowed to reach write_pos). + uint32_t keep_size_after; /// Match finders store locations of matches using 32-bit integers. /// To avoid adjusting several megabytes of integers every time the /// input window is moved with move_window(), we only adjust the /// offset of the buffer. Thus, buffer[match_finder_pos - offset] /// is the byte pointed by match_finder_pos. - size_t offset; + uint32_t offset; /// buffer[read_pos] is the current byte. - size_t read_pos; + uint32_t read_pos; + + /// Number of bytes that have been ran through the match finder, but + /// which haven't been encoded by the LZ-based encoder yet. + uint32_t read_ahead; /// As long as read_pos is less than read_limit, there is enough /// input available in buffer for at least one encoding loop. @@ -64,92 +77,253 @@ struct lzma_lz_encoder_s { /// Because of the stateful API, read_limit may and will get greater /// than read_pos quite often. This is taken into account when /// calculating the value for keep_size_after. - size_t read_limit; + uint32_t read_limit; /// buffer[write_pos] is the first byte that doesn't contain valid /// uncompressed data; that is, the next input byte will be copied /// to buffer[write_pos]. - size_t write_pos; + uint32_t write_pos; /// Number of bytes not hashed before read_pos. This is needed to /// restart the match finder after LZMA_SYNC_FLUSH. - size_t pending; - - /// Number of bytes that must be kept available in our input history. - /// That is, once keep_size_before bytes have been processed, - /// buffer[read_pos - keep_size_before] is the oldest byte that - /// must be available for reading. - size_t keep_size_before; - - /// Number of bytes that must be kept in buffer after read_pos. - /// That is, read_pos <= write_pos - keep_size_after as long as - /// stream_end_was_reached is false (once it is true, read_pos - /// is allowed to reach write_pos). - size_t keep_size_after; + uint32_t pending; ////////////////// // Match Finder // ////////////////// - // Pointers to match finder functions - void (*get_matches)(lzma_lz_encoder *restrict lz, - uint32_t *restrict distances); - void (*skip)(lzma_lz_encoder *restrict lz, uint32_t num); + /// Find matches. Returns the number of distance-length pairs written + /// to the matches array. This is called only via lzma_mf_find. + uint32_t (*find)(lzma_mf *mf, lzma_match *matches); + + /// Skips num bytes. This is like find() but doesn't make the + /// distance-length pairs available, thus being a little faster. + /// This is called only via mf_skip function. + void (*skip)(lzma_mf *mf, uint32_t num); - // Match finder data - uint32_t *hash; // TODO: Check if hash aliases son - uint32_t *son; // and add 'restrict' if possible. + uint32_t *hash; + uint32_t *son; uint32_t cyclic_buffer_pos; uint32_t cyclic_buffer_size; // Must be dictionary_size + 1. uint32_t hash_mask; - uint32_t cut_value; + + /// Maximum number of loops in the match finder + uint32_t loops; + + /// Maximum length of a match that the match finder will try to find. + uint32_t find_len_max; + + /// Maximum length of a match supported by the LZ-based encoder. + /// If the longest match found by the match finder is find_len_max, + /// lz_dict_find() tries to expand it up to match_len_max bytes. + uint32_t match_len_max; + + /// When running out of input, binary tree match finders need to know + /// if it is due to flushing or finishing. The action is used also + /// by the LZ-based encoders themselves. + lzma_action action; + + /// Number of elements in hash[] uint32_t hash_size_sum; - uint32_t num_items; - uint32_t match_max_len; + + /// Number of elements in son[] + uint32_t sons_count; }; -#define LZMA_LZ_ENCODER_INIT \ - (lzma_lz_encoder){ \ - .buffer = NULL, \ - .size = 0, \ - .hash = NULL, \ - .num_items = 0, \ +typedef struct { + /// Extra amount of data to keep available before the "actual" + /// dictionary. + size_t before_size; + + /// Size of the history buffer + size_t dictionary_size; + + /// Extra amount of data to keep available after the "actual" + /// dictionary. + size_t after_size; + + /// Maximum length of a match that the LZ-based encoder can accept. + /// This is used to extend matches of length find_len_max to the + /// maximum possible length. + size_t match_len_max; + + /// Match finder will search matches of at maximum of this length. + /// This must be less than or equal to match_len_max. + size_t find_len_max; + + /// Type of the match finder to use + lzma_match_finder match_finder; + + /// TODO: Comment + uint32_t match_finder_cycles; + + /// TODO: Comment + const uint8_t *preset_dictionary; + + uint32_t preset_dictionary_size; + +} lzma_lz_options; + + +// The total usable buffer space at any moment outside the match finder: +// before_size + dictionary_size + after_size + match_len_max +// +// In reality, there's some extra space allocated to prevent the number of +// memmove() calls reasonable. The bigger the dictionary_size is, the bigger +// this extra buffer will be since with bigger dictionaries memmove() would +// also take longer. +// +// A single encoder loop in the LZ-based encoder may call the match finder +// (lz_dict_find() or lz_dict_skip()) at maximum of after_size times. +// In other words, a single encoder loop may advance lz_dict.read_pos at +// maximum of after_size times. Since matches are looked up to +// lz_dict.buffer[lz_dict.read_pos + match_len_max - 1], the total +// amount of extra buffer needed after dictionary_size becomes +// after_size + match_len_max. +// +// before_size has two uses. The first one is to keep literals available +// in cases when the LZ-based encoder has made some read ahead. +// TODO: Maybe this could be changed by making the LZ-based encoders to +// store the actual literals as they do with length-distance pairs. +// +// Alrogithms such as LZMA2 first try to compress a chunk, and then check +// if the encoded result is smaller than the uncompressed one. If the chunk +// was uncompressible, it is better to store it in uncompressed form in +// the output stream. To do this, the whole uncompressed chunk has to be +// still available in the history buffer. before_size achieves that. + + +typedef struct { + /// Data specific to the LZ-based encoder + lzma_coder *coder; + + /// Function to encode from *dict to out[] + lzma_ret (*code)(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); + + /// Free allocated resources + void (*end)(lzma_coder *coder, lzma_allocator *allocator); + +} lzma_lz_encoder; + + +// Basic steps: +// 1. Input gets copied into the dictionary. +// 2. Data in dictionary gets run through the match finder byte by byte. +// 3. The literals and matches are encoded using e.g. LZMA. +// +// The bytes that have been ran through the match finder, but not encoded yet, +// are called `read ahead'. + + +/// Get pointer to the first byte not ran through the match finder +static inline const uint8_t * +mf_ptr(const lzma_mf *mf) +{ + return mf->buffer + mf->read_pos; +} + + +/// Get the number of bytes that haven't been ran through the match finder yet. +static inline uint32_t +mf_avail(const lzma_mf *mf) +{ + return mf->write_pos - mf->read_pos; +} + + +/// Get the number of bytes that haven't been encoded yet (some of these +/// bytes may have been ran through the match finder though). +static inline uint32_t +mf_unencoded(const lzma_mf *mf) +{ + return mf->write_pos - mf->read_pos - mf->read_ahead; +} + + +/// Calculate the absolute offset from the beginning of the most recent +/// dictionary reset. Only the lowest four bits are important, so there's no +/// problem that we don't know the 64-bit size of the data encoded so far. +/// +/// NOTE: When moving the input window, we need to do it so that the lowest +/// bits of dict->read_pos are not modified to keep this macro working +/// as intended. +static inline uint32_t +mf_position(const lzma_mf *mf) +{ + return mf->read_pos - mf->read_ahead; +} + + +/// Since everything else begins with mf_, use it also for lzma_mf_find(). +#define mf_find lzma_mf_find + + +/// Skip the given number of bytes. This is used when a good match was found. +/// For example, if mf_find() finds a match of 200 bytes long, the first byte +/// of that match was already consumed by mf_find(), and the rest 199 bytes +/// have to be skipped with mf_skip(mf, 199). +static inline void +mf_skip(lzma_mf *mf, uint32_t amount) +{ + if (amount != 0) { + mf->skip(mf, amount); + mf->read_ahead += amount; } +} + + +/// Copies at maximum of *left amount of bytes from the history buffer +/// to out[]. This is needed by LZMA2 to encode uncompressed chunks. +static inline void +mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, + size_t *left) +{ + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(out_avail, *left); + + assert(mf->read_ahead == 0); + assert(mf->read_pos >= *left); + + memcpy(out + *out_pos, mf->buffer + mf->read_pos - *left, + copy_size); + + *out_pos += copy_size; + *left -= copy_size; + return; +} + + +extern lzma_ret lzma_lz_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_encoder *lz, + lzma_allocator *allocator, const void *options, + lzma_lz_options *lz_options)); + + +extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options); + + +// These are only for LZ encoder's internal use. +extern uint32_t lzma_mf_find( + lzma_mf *mf, uint32_t *count, lzma_match *matches); + +extern uint32_t lzma_mf_hc3_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_hc3_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_hc4_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_hc4_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_bt2_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt2_skip(lzma_mf *dict, uint32_t amount); +extern uint32_t lzma_mf_bt3_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt3_skip(lzma_mf *dict, uint32_t amount); -/// Calculates -extern bool lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, - uint32_t history_size, uint32_t *restrict hash_mask, - uint32_t *restrict hash_size_sum, - uint32_t *restrict num_items); - -// NOTE: liblzma doesn't use callback API like LZMA SDK does. The caller -// must make sure that keep_size_after is big enough for single encoding pass -// i.e. keep_size_after >= maximum number of bytes possibly needed after -// the current position between calls to lzma_lz_read(). -extern lzma_ret lzma_lz_encoder_reset(lzma_lz_encoder *lz, - lzma_allocator *allocator, - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size), - size_t history_size, size_t additional_buffer_before, - size_t match_max_len, size_t additional_buffer_after, - lzma_match_finder match_finder, uint32_t match_finder_cycles, - const uint8_t *preset_dictionary, - size_t preset_dictionary_size); - -/// Frees memory allocated for in window and match finder buffers. -extern void lzma_lz_encoder_end( - lzma_lz_encoder *lz, lzma_allocator *allocator); - -extern lzma_ret lzma_lz_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((unused)), - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - lzma_action action); - -/// This should not be called directly, but only via move_pos() macro. -extern void lzma_lz_encoder_normalize(lzma_lz_encoder *lz); +extern uint32_t lzma_mf_bt4_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt4_skip(lzma_mf *dict, uint32_t amount); #endif diff --git a/src/liblzma/lz/lz_encoder_hash.h b/src/liblzma/lz/lz_encoder_hash.h new file mode 100644 index 00000000..0841c38f --- /dev/null +++ b/src/liblzma/lz/lz_encoder_hash.h @@ -0,0 +1,104 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder_hash.h +/// \brief Hash macros for match finders +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZ_ENCODER_HASH_H +#define LZMA_LZ_ENCODER_HASH_H + +#define HASH_2_SIZE (UINT32_C(1) << 10) +#define HASH_3_SIZE (UINT32_C(1) << 16) +#define HASH_4_SIZE (UINT32_C(1) << 20) + +#define HASH_2_MASK (HASH_2_SIZE - 1) +#define HASH_3_MASK (HASH_3_SIZE - 1) +#define HASH_4_MASK (HASH_4_SIZE - 1) + +#define FIX_3_HASH_SIZE (HASH_2_SIZE) +#define FIX_4_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE) +#define FIX_5_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE + HASH_4_SIZE) + +// TODO Benchmark, and probably doesn't need to be endian dependent. +#if !defined(WORDS_BIGENDIAN) && defined(HAVE_FAST_UNALIGNED_ACCESS) +# define hash_2_calc() \ + const uint32_t hash_value = *(const uint16_t *)(cur); +#else +# define hash_2_calc() \ + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) +#endif + +#define hash_3_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + +#define hash_4_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + ^ (lzma_crc32_table[0][cur[3]] << 5)) & mf->hash_mask + + +// The following are not currently used. + +#define hash_5_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \ + ^ lzma_crc32_table[0][cur[3]] << 5); \ + const uint32_t hash_value \ + = (hash_4_value ^ (lzma_crc32_table[0][cur[4]] << 3)) \ + & mf->hash_mask; \ + hash_4_value &= HASH_4_MASK + +/* +#define hash_zip_calc() \ + const uint32_t hash_value \ + = (((uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8)) \ + ^ lzma_crc32_table[0][cur[2]]) & 0xFFFF +*/ + +#define hash_zip_calc() \ + const uint32_t hash_value \ + = (((uint32_t)(cur[2]) | ((uint32_t)(cur[0]) << 8)) \ + ^ lzma_crc32_table[0][cur[1]]) & 0xFFFF + +#define mt_hash_2_calc() \ + const uint32_t hash_2_value \ + = (lzma_crc32_table[0][cur[0]] ^ cur[1]) & HASH_2_MASK + +#define mt_hash_3_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK + +#define mt_hash_4_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \ + (lzma_crc32_table[0][cur[3]] << 5)) & HASH_4_MASK + +#endif diff --git a/src/liblzma/lz/lz_encoder_mf.c b/src/liblzma/lz/lz_encoder_mf.c new file mode 100644 index 00000000..b1c20f50 --- /dev/null +++ b/src/liblzma/lz/lz_encoder_mf.c @@ -0,0 +1,780 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder_mf.c +/// \brief Match finders +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder.h" +#include "lz_encoder_hash.h" +#include "check.h" + + +/// \brief Find matches starting from the current byte +/// +/// \return The length of the longest match found +extern uint32_t +lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) +{ + // Call the match finder. It returns the number of length-distance + // pairs found. + // FIXME: Minimum count is zero, what _exactly_ is the maximum? + const uint32_t count = mf->find(mf, matches); + + // Length of the longest match; assume that no matches were found + // and thus the maximum length is zero. + uint32_t len_best = 0; + + if (count > 0) { +#ifndef NDEBUG + // Validate the matches. + for (uint32_t i = 0; i < count; ++i) { + assert(matches[i].len <= mf->find_len_max); + assert(matches[i].dist < mf->read_pos); + assert(memcmp(mf_ptr(mf) - 1, + mf_ptr(mf) - matches[i].dist - 2, + matches[i].len) == 0); + } +#endif + + // The last used element in the array contains + // the longest match. + len_best = matches[count - 1].len; + + // If a match of maximum search length was found, try to + // extend the match to maximum possible length. + if (len_best == mf->find_len_max) { + // The limit for the match length is either the + // maximum match length supported by the LZ-based + // encoder or the number of bytes left in the + // dictionary, whichever is smaller. + uint32_t limit = mf_avail(mf) + 1; + if (limit > mf->match_len_max) + limit = mf->match_len_max; + + // Pointer to the byte we just ran through + // the match finder. + const uint8_t *p1 = mf_ptr(mf) - 1; + + // Pointer to the beginning of the match. We need -1 + // here because the match distances are zero based. + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; + + while (len_best < limit + && p1[len_best] == p2[len_best]) + ++len_best; + } + } + + *count_ptr = count; + + // Finally update the read position to indicate that match finder was + // run for this dictionary offset. + ++mf->read_ahead; + + return len_best; +} + + +/// Hash value to indicate unused element in the hash. Since we start the +/// positions from dictionary_size + 1, zero is always too far to qualify +/// as usable match position. +#define EMPTY_HASH_VALUE 0 + + +/// Normalization must be done when lzma_mf.offset + lzma_mf.read_pos +/// reaches MUST_NORMALIZE_POS. +#define MUST_NORMALIZE_POS UINT32_MAX + + +/// \brief Normalizes hash values +/// +/// The hash arrays store positions of match candidates. The positions are +/// relative to an arbitrary offset that is not the same as the absolute +/// offset in the input stream. The relative position of the current byte +/// is lzma_mf.offset + lzma_mf.read_pos. The distances of the matches are +/// the differences of the current read position and the position found from +/// the hash. +/// +/// To prevent integer overflows of the offsets stored in the hash arrays, +/// we need to "normalize" the stored values now and then. During the +/// normalization, we drop values that indicate distance greater than the +/// dictionary size, thus making space for new values. +static void +normalize(lzma_mf *mf) +{ + assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); + + // In future we may not want to touch the lowest bits, because there + // may be match finders that use larger resolution than one byte. + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_buffer_size); + // & (~(UINT32_C(1) << 10) - 1); + + const uint32_t count = mf->hash_size_sum + mf->sons_count; + uint32_t *hash = mf->hash; + + for (uint32_t i = 0; i < count; ++i) { + // If the distance is greater than the dictionary size, + // we can simply mark the hash element as empty. + // + // NOTE: Only the first mf->hash_size_sum elements are + // initialized for sure. There may be uninitialized elements + // in mf->son. Since we go through both mf->hash and + // mf->son here in normalization, Valgrind may complain + // that the "if" below depends on uninitialized value. In + // this case it is safe to ignore the warning. See also the + // comments in lz_encoder_init() in lz_encoder.c. + if (hash[i] <= subvalue) + hash[i] = EMPTY_HASH_VALUE; + else + hash[i] -= subvalue; + } + + // Update offset to match the new locations. + mf->offset -= subvalue; + + return; +} + + +/// Mark the current byte as processed from point of view of the match finder. +static void +move_pos(lzma_mf *mf) +{ + if (++mf->cyclic_buffer_pos == mf->cyclic_buffer_size) + mf->cyclic_buffer_pos = 0; + + ++mf->read_pos; + assert(mf->read_pos <= mf->write_pos); + + if (unlikely(mf->read_pos + mf->offset == UINT32_MAX)) + normalize(mf); +} + + +/// When flushing, we cannot run the match finder unless there is find_len_max +/// bytes available in the dictionary. Instead, we skip running the match +/// finder (indicating that no match was found), and count how many bytes we +/// have ignored this way. +/// +/// When new data is given after the flushing was completed, the match finder +/// is restarted by rewinding mf->read_pos backwards by mf->pending. Then +/// the missed bytes are added to the hash using the match finder's skip +/// function (with small amount of input, it may start using mf->pending +/// again if flushing). +/// +/// Due to this rewinding, we don't touch cyclic_buffer_pos or test for +/// normalization. It will be done when the match finder's skip function +/// catches up after a flush. +static void +move_pending(lzma_mf *mf) +{ + ++mf->read_pos; + assert(mf->read_pos <= mf->write_pos); + ++mf->pending; +} + + +/// Calculate len_limit and determine if there is enough input to run +/// the actual match finder code. Sets up "cur" and "pos". This macro +/// is used by all find functions and binary tree skip functions. Hash +/// chain skip function doesn't need len_limit so a simpler code is used +/// in them. +#define header(is_bt, len_min, ret_op) \ + uint32_t len_limit = mf_avail(mf); \ + if (mf->find_len_max <= len_limit) { \ + len_limit = mf->find_len_max; \ + } else if (len_limit < (len_min) \ + || (is_bt && mf->action == LZMA_SYNC_FLUSH)) { \ + assert(mf->action != LZMA_RUN); \ + move_pending(mf); \ + ret_op; \ + } \ + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset + + +/// Header for find functions. "return 0" indicates that zero matches +/// were found. +#define header_find(is_bt, len_min) \ + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 + + +/// Header for a loop in a skip function. "continue" tells to skip the rest +/// of the code in the loop. +#define header_skip(is_bt, len_min) \ + header(is_bt, len_min, continue) + + +/// Calls hc_find_func() or bt_find_func() and calculates the total number +/// of matches found. Updates the dictionary position and returns the number +/// of matches found. +#define call_find(func, len_best) \ +do { \ + matches_count = func(len_limit, pos, cur, cur_match, mf->loops, \ + mf->son, mf->cyclic_buffer_pos, \ + mf->cyclic_buffer_size, \ + matches + matches_count, len_best) \ + - matches; \ + move_pos(mf); \ + return matches_count; \ +} while (0) + + +//////////////// +// Hash Chain // +//////////////// + +#if defined(HAVE_MF_HC3) || defined(HAVE_MF_HC4) +/// +/// +/// \param len_limit Don't look for matches longer than len_limit. +/// \param pos lzma_mf.read_pos + lzma_mf.offset +/// \param cur Pointer to current byte (lzma_dict_ptr(mf)) +/// \param cur_match Start position of the current match candidate +/// \param loops Maximum length of the hash chain +/// \param son lzma_mf.son (contains the hash chain) +/// \param cyclic_buffer_pos +/// \param cyclic_buffer_size +/// \param matches Array to hold the matches. +/// \param len_best The length of the longest match found so far. +static lzma_match * +hc_find_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size, + lzma_match *matches, + uint32_t len_best) +{ + son[cyclic_buffer_pos] = cur_match; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) + return matches; + + const uint8_t *const pb = cur - delta; + cur_match = son[cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)]; + + if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { + uint32_t len = 0; + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len_best < len) { + len_best = len; + matches->len = len; + matches->dist = delta - 1; + ++matches; + + if (len == len_limit) + return matches; + } + } + } +} + +/* +#define hc_header_find(len_min, ret_op) \ + uint32_t len_limit = mf_avail(mf); \ + if (mf->find_len_max <= len_limit) { \ + len_limit = mf->find_len_max; \ + } else if (len_limit < (len_min)) { \ + move_pending(mf); \ + ret_op; \ + } \ +#define header_hc(len_min, ret_op) \ +do { \ + if (mf_avail(mf) < (len_min)) { \ + move_pending(mf); \ + ret_op; \ + } \ +} while (0) +*/ + +#define hc_find(len_best) \ + call_find(hc_find_func, len_best) + + +#define hc_skip() \ +do { \ + mf->son[mf->cyclic_buffer_pos] = cur_match; \ + move_pos(mf); \ +} while (0) + +#endif + + +#ifdef HAVE_MF_HC3 +extern uint32_t +lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(false, 3); + + hash_3_calc(); + + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 2; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[0].len = len_best; + matches[0].dist = delta2 - 1; + matches_count = 1; + + if (len_best == len_limit) { + hc_skip(); + return 1; // matches_count + } + } + + hc_find(len_best); +} + + +extern void +lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) +{ + do { + if (mf_avail(mf) < 3) { + move_pending(mf); + continue; + } + + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; + + hash_3_calc(); + + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + hc_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_HC4 +extern uint32_t +lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(false, 4); + + hash_4_calc(); + + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value ] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 1; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + len_best = 2; + matches[0].len = 2; + matches[0].dist = delta2 - 1; + matches_count = 1; + } + + if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size + && *(cur - delta3) == *cur) { + len_best = 3; + matches[matches_count++].dist = delta3 - 1; + delta2 = delta3; + } + + if (matches_count != 0) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[matches_count - 1].len = len_best; + + if (len_best == len_limit) { + hc_skip(); + return matches_count; + } + } + + if (len_best < 3) + len_best = 3; + + hc_find(len_best); +} + + +extern void +lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) +{ + do { + if (mf_avail(mf) < 4) { + move_pending(mf); + continue; + } + + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; + + hash_4_calc(); + + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + hc_skip(); + + } while (--amount != 0); +} +#endif + + +///////////////// +// Binary Tree // +///////////////// + +#if defined(HAVE_MF_BT2) || defined(HAVE_MF_BT3) || defined(HAVE_MF_BT4) +static lzma_match * +bt_find_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size, + lzma_match *matches, + uint32_t len_best) +{ + uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = son + (cyclic_buffer_pos << 1); + + uint32_t len0 = 0; + uint32_t len1 = 0; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) { + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; + return matches; + } + + uint32_t *const pair = son + ((cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)) << 1); + + const uint8_t *const pb = cur - delta; + uint32_t len = MIN(len0, len1); + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len_best < len) { + len_best = len; + matches->len = len; + matches->dist = delta - 1; + ++matches; + + if (len == len_limit) { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + return matches; + } + } + } + + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } + } +} + + +static void +bt_skip_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size) +{ + uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = son + (cyclic_buffer_pos << 1); + + uint32_t len0 = 0; + uint32_t len1 = 0; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) { + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; + return; + } + + uint32_t *pair = son + ((cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)) << 1); + const uint8_t *pb = cur - delta; + uint32_t len = MIN(len0, len1); + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len == len_limit) { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + return; + } + } + + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } + } +} + + +#define bt_find(len_best) \ + call_find(bt_find_func, len_best) + +#define bt_skip() \ +do { \ + bt_skip_func(len_limit, pos, cur, cur_match, mf->loops, \ + mf->son, mf->cyclic_buffer_pos, \ + mf->cyclic_buffer_size); \ + move_pos(mf); \ +} while (0) + +#endif + + +#ifdef HAVE_MF_BT2 +extern uint32_t +lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 2); + + hash_2_calc(); + + const uint32_t cur_match = mf->hash[hash_value]; + mf->hash[hash_value] = pos; + + bt_find(1); +} + + +extern void +lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 2); + + hash_2_calc(); + + const uint32_t cur_match = mf->hash[hash_value]; + mf->hash[hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_BT3 +extern uint32_t +lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 3); + + hash_3_calc(); + + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 2; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[0].len = len_best; + matches[0].dist = delta2 - 1; + matches_count = 1; + + if (len_best == len_limit) { + bt_skip(); + return 1; // matches_count + } + } + + bt_find(len_best); +} + + +extern void +lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 3); + + hash_3_calc(); + + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_BT4 +extern uint32_t +lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 4); + + hash_4_calc(); + + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 1; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + len_best = 2; + matches[0].len = 2; + matches[0].dist = delta2 - 1; + matches_count = 1; + } + + if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size + && *(cur - delta3) == *cur) { + len_best = 3; + matches[matches_count++].dist = delta3 - 1; + delta2 = delta3; + } + + if (matches_count != 0) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[matches_count - 1].len = len_best; + + if (len_best == len_limit) { + bt_skip(); + return matches_count; + } + } + + if (len_best < 3) + len_best = 3; + + bt_find(len_best); +} + + +extern void +lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 4); + + hash_4_calc(); + + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif diff --git a/src/liblzma/lz/lz_encoder_private.h b/src/liblzma/lz/lz_encoder_private.h deleted file mode 100644 index 638fcb2d..00000000 --- a/src/liblzma/lz/lz_encoder_private.h +++ /dev/null @@ -1,40 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lz_encoder_private.h -/// \brief Private definitions for LZ encoder -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_LZ_ENCODER_PRIVATE_H -#define LZMA_LZ_ENCODER_PRIVATE_H - -#include "lz_encoder.h" - -/// Value used to indicate unused slot -#define EMPTY_HASH_VALUE 0 - -/// When the dictionary and hash variables need to be adjusted to prevent -/// integer overflows. Since we use uint32_t to store the offsets, half -/// of it is the biggest safe limit. -#define MAX_VAL_FOR_NORMALIZE (UINT32_MAX / 2) - - -struct lzma_coder_s { - lzma_next_coder next; - lzma_lz_encoder lz; -}; - -#endif diff --git a/src/liblzma/lz/match_c.h b/src/liblzma/lz/match_c.h deleted file mode 100644 index 664db290..00000000 --- a/src/liblzma/lz/match_c.h +++ /dev/null @@ -1,412 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file match_c.h -/// \brief Template for different match finders -/// -/// This file is included by hc3.c, hc4, bt2.c, bt3.c and bt4.c. Each file -/// sets slighly different #defines, resulting the different match finders. -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -////////////// -// Includes // -////////////// - -#include "check.h" - - -/////////////// -// Constants // -/////////////// - -#define START_MAX_LEN 1 - -#ifdef HASH_ARRAY_2 -# define NUM_HASH_DIRECT_BYTES 0 -# define HASH_2_SIZE (1 << 10) -# ifdef HASH_ARRAY_3 -# define NUM_HASH_BYTES 4 -# define HASH_3_SIZE (1 << 16) -# define HASH_3_OFFSET HASH_2_SIZE -# define FIX_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE) -# else -# define NUM_HASH_BYTES 3 -# define FIX_HASH_SIZE HASH_2_SIZE -# endif -# define HASH_SIZE 0 -# define MIN_MATCH_CHECK NUM_HASH_BYTES -#else -# define NUM_HASH_DIRECT_BYTES 2 -# define NUM_HASH_BYTES 2 -# define HASH_SIZE (1 << (8 * NUM_HASH_BYTES)) -# define MIN_MATCH_CHECK (NUM_HASH_BYTES + 1) -# define FIX_HASH_SIZE 0 -#endif - - -//////////// -// Macros // -//////////// - -#ifdef HASH_ARRAY_2 -# ifdef HASH_ARRAY_3 -# define HASH_CALC() \ - do { \ - const uint32_t temp = lzma_crc32_table[0][ \ - cur[0]] ^ cur[1]; \ - hash_2_value = temp & (HASH_2_SIZE - 1); \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ - & (HASH_3_SIZE - 1); \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ - ^ (lzma_crc32_table[0][cur[3]] << 5)) \ - & lz->hash_mask; \ - } while (0) -# else -# define HASH_CALC() \ - do { \ - const uint32_t temp = lzma_crc32_table[0][ \ - cur[0]] ^ cur[1]; \ - hash_2_value = temp & (HASH_2_SIZE - 1); \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ - & lz->hash_mask; \ - } while (0) -# endif -#else -# define HASH_CALC() hash_value = cur[0] ^ ((uint32_t)(cur[1]) << 8) -#endif - - -// Moves the current read position forward by one byte. In LZMA SDK, -// CLZInWindow::MovePos() can read more input data if needed, because of -// the callback style API. In liblzma we must have ensured earlier, that -// there is enough data available in lz->buffer. -#define move_pos() \ -do { \ - if (++lz->cyclic_buffer_pos == lz->cyclic_buffer_size) \ - lz->cyclic_buffer_pos = 0; \ - ++lz->read_pos; \ - assert(lz->read_pos <= lz->write_pos); \ - if (lz->read_pos == MAX_VAL_FOR_NORMALIZE) \ - lzma_lz_encoder_normalize(lz); \ -} while (0) - - -#define move_pending() \ -do { \ - ++lz->read_pos; \ - assert(lz->read_pos <= lz->write_pos); \ - ++lz->pending; \ -} while (0) - - -////////////////////// -// Global constants // -////////////////////// - -LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = HASH_SIZE; -LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = FIX_HASH_SIZE; - - -/////////////////// -// API functions // -/////////////////// - -LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER) -{ - uint32_t len_limit; - if (lz->read_pos + lz->match_max_len <= lz->write_pos) { - len_limit = lz->match_max_len; - } else { - len_limit = lz->write_pos - lz->read_pos; - if (len_limit < MIN_MATCH_CHECK || lz->sequence == SEQ_FLUSH) { - distances[0] = 0; - move_pending(); - return; - } - } - - assert(lz->pending == 0); - - int32_t offset = 1; - const uint32_t match_min_pos - = lz->read_pos + lz->offset > lz->cyclic_buffer_size - ? lz->read_pos + lz->offset - lz->cyclic_buffer_size - : 0; - const uint8_t *cur = lz->buffer + lz->read_pos; - uint32_t max_len = START_MAX_LEN; // to avoid items for len < hash_size - -#ifdef HASH_ARRAY_2 - uint32_t hash_2_value; -# ifdef HASH_ARRAY_3 - uint32_t hash_3_value; -# endif -#endif - uint32_t hash_value; - HASH_CALC(); - - uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; -#ifdef HASH_ARRAY_2 - uint32_t cur_match2 = lz->hash[hash_2_value]; -# ifdef HASH_ARRAY_3 - uint32_t cur_match3 = lz->hash[HASH_3_OFFSET + hash_3_value]; -# endif - lz->hash[hash_2_value] = lz->read_pos + lz->offset; - - if (cur_match2 > match_min_pos) { - if (lz->buffer[cur_match2 - lz->offset] == cur[0]) { - max_len = 2; - distances[offset++] = 2; - distances[offset++] = lz->read_pos + lz->offset - - cur_match2 - 1; - } - } - -# ifdef HASH_ARRAY_3 - lz->hash[HASH_3_OFFSET + hash_3_value] = lz->read_pos + lz->offset; - if (cur_match3 > match_min_pos) { - if (lz->buffer[cur_match3 - lz->offset] == cur[0]) { - if (cur_match3 == cur_match2) - offset -= 2; - - max_len = 3; - distances[offset++] = 3; - distances[offset++] = lz->read_pos + lz->offset - - cur_match3 - 1; - cur_match2 = cur_match3; - } - } -# endif - - if (offset != 1 && cur_match2 == cur_match) { - offset -= 2; - max_len = START_MAX_LEN; - } -#endif - - lz->hash[FIX_HASH_SIZE + hash_value] = lz->read_pos + lz->offset; - -#ifdef IS_HASH_CHAIN - lz->son[lz->cyclic_buffer_pos] = cur_match; -#else - uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; - uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); - - uint32_t len0 = NUM_HASH_DIRECT_BYTES; - uint32_t len1 = NUM_HASH_DIRECT_BYTES; -#endif - -#if NUM_HASH_DIRECT_BYTES != 0 - if (cur_match > match_min_pos) { - if (lz->buffer[cur_match + NUM_HASH_DIRECT_BYTES - lz->offset] - != cur[NUM_HASH_DIRECT_BYTES]) { - max_len = NUM_HASH_DIRECT_BYTES; - distances[offset++] = NUM_HASH_DIRECT_BYTES; - distances[offset++] = lz->read_pos + lz->offset - - cur_match - 1; - } - } -#endif - - uint32_t count = lz->cut_value; - - while (true) { - if (cur_match <= match_min_pos || count-- == 0) { -#ifndef IS_HASH_CHAIN - *ptr0 = EMPTY_HASH_VALUE; - *ptr1 = EMPTY_HASH_VALUE; -#endif - break; - } - - const uint32_t delta = lz->read_pos + lz->offset - cur_match; - const uint32_t cyclic_pos = delta <= lz->cyclic_buffer_pos - ? lz->cyclic_buffer_pos - delta - : lz->cyclic_buffer_pos - delta - + lz->cyclic_buffer_size; - uint32_t *pair = lz->son + -#ifdef IS_HASH_CHAIN - cyclic_pos; -#else - (cyclic_pos << 1); -#endif - - const uint8_t *pb = lz->buffer + cur_match - lz->offset; - uint32_t len = -#ifdef IS_HASH_CHAIN - NUM_HASH_DIRECT_BYTES; - if (pb[max_len] == cur[max_len]) -#else - MIN(len0, len1); -#endif - - if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; - - if (max_len < len) { - max_len = len; - distances[offset++] = len; - distances[offset++] = delta - 1; - if (len == len_limit) { -#ifndef IS_HASH_CHAIN - *ptr1 = pair[0]; - *ptr0 = pair[1]; -#endif - break; - } - } - } - -#ifdef IS_HASH_CHAIN - cur_match = *pair; -#else - if (pb[len] < cur[len]) { - *ptr1 = cur_match; - ptr1 = pair + 1; - cur_match = *ptr1; - len1 = len; - } else { - *ptr0 = cur_match; - ptr0 = pair; - cur_match = *ptr0; - len0 = len; - } -#endif - } - - distances[0] = offset - 1; - - move_pos(); - - return; -} - - -LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER) -{ - do { -#ifdef IS_HASH_CHAIN - if (lz->write_pos - lz->read_pos < NUM_HASH_BYTES) { - move_pending(); - continue; - } -#else - uint32_t len_limit; - if (lz->read_pos + lz->match_max_len <= lz->write_pos) { - len_limit = lz->match_max_len; - } else { - len_limit = lz->write_pos - lz->read_pos; - if (len_limit < MIN_MATCH_CHECK - || lz->sequence == SEQ_FLUSH) { - move_pending(); - continue; - } - } - const uint32_t match_min_pos - = lz->read_pos + lz->offset > lz->cyclic_buffer_size - ? lz->read_pos + lz->offset - lz->cyclic_buffer_size - : 0; -#endif - - assert(lz->pending == 0); - - const uint8_t *cur = lz->buffer + lz->read_pos; - -#ifdef HASH_ARRAY_2 - uint32_t hash_2_value; -# ifdef HASH_ARRAY_3 - uint32_t hash_3_value; - uint32_t hash_value; - HASH_CALC(); - lz->hash[HASH_3_OFFSET + hash_3_value] - = lz->read_pos + lz->offset; -# else - uint32_t hash_value; - HASH_CALC(); -# endif - lz->hash[hash_2_value] = lz->read_pos + lz->offset; -#else - uint32_t hash_value; - HASH_CALC(); -#endif - - uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; - lz->hash[FIX_HASH_SIZE + hash_value] - = lz->read_pos + lz->offset; - -#ifdef IS_HASH_CHAIN - lz->son[lz->cyclic_buffer_pos] = cur_match; -#else - uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; - uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); - - uint32_t len0 = NUM_HASH_DIRECT_BYTES; - uint32_t len1 = NUM_HASH_DIRECT_BYTES; - uint32_t count = lz->cut_value; - - while (true) { - if (cur_match <= match_min_pos || count-- == 0) { - *ptr0 = EMPTY_HASH_VALUE; - *ptr1 = EMPTY_HASH_VALUE; - break; - } - - const uint32_t delta = lz->read_pos - + lz->offset - cur_match; - const uint32_t cyclic_pos - = delta <= lz->cyclic_buffer_pos - ? lz->cyclic_buffer_pos - delta - : lz->cyclic_buffer_pos - delta - + lz->cyclic_buffer_size; - uint32_t *pair = lz->son + (cyclic_pos << 1); - - const uint8_t *pb = lz->buffer + cur_match - - lz->offset; - uint32_t len = MIN(len0, len1); - - if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; - - if (len == len_limit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - break; - } - } - - if (pb[len] < cur[len]) { - *ptr1 = cur_match; - ptr1 = pair + 1; - cur_match = *ptr1; - len1 = len; - } else { - *ptr0 = cur_match; - ptr0 = pair; - cur_match = *ptr0; - len0 = len; - } - } -#endif - - move_pos(); - - } while (--num != 0); - - return; -} diff --git a/src/liblzma/lz/match_h.h b/src/liblzma/lz/match_h.h deleted file mode 100644 index 2eae90ba..00000000 --- a/src/liblzma/lz/match_h.h +++ /dev/null @@ -1,69 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file match_h.h -/// \brief Header template for different match finders -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lz_encoder_private.h" - - -////////////////////// -// Global constants // -////////////////////// - -#undef LZMA_HASH_SIZE -#undef LZMA_FIX_HASH_SIZE -#undef LZMA_HASH_SIZE_C -#undef LZMA_FIX_HASH_SIZE_C - -#define LZMA_HASH_SIZE(mf_name) LZMA_HASH_SIZE_C(mf_name) -#define LZMA_FIX_HASH_SIZE(mf_name) LZMA_FIX_HASH_SIZE_C(mf_name) - -#define LZMA_HASH_SIZE_C(mf_name) \ - const uint32_t LZMA_ ## mf_name ## _HASH_SIZE - -#define LZMA_FIX_HASH_SIZE_C(mf_name) \ - const uint32_t LZMA_ ## mf_name ## _FIX_HASH_SIZE - -extern LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); -extern LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); - - -/////////////// -// Functions // -/////////////// - -#undef LZMA_GET_MATCHES -#undef LZMA_SKIP -#undef LZMA_GET_MATCHES_C -#undef LZMA_SKIP_C - -#define LZMA_GET_MATCHES(mf_name) LZMA_GET_MATCHES_C(mf_name) -#define LZMA_SKIP(mf_name) LZMA_SKIP_C(mf_name) - -#define LZMA_GET_MATCHES_C(mf_name) \ - extern void lzma_ ## mf_name ## _get_matches( \ - lzma_lz_encoder *restrict lz, \ - uint32_t *restrict distances) - -#define LZMA_SKIP_C(mf_name) \ - extern void lzma_ ## mf_name ## _skip( \ - lzma_lz_encoder *lz, uint32_t num) - -LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER); - -LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER); diff --git a/src/liblzma/lzma/Makefile.am b/src/liblzma/lzma/Makefile.am index 59ded214..7aeceb63 100644 --- a/src/liblzma/lzma/Makefile.am +++ b/src/liblzma/lzma/Makefile.am @@ -14,37 +14,46 @@ EXTRA_DIST = fastpos_tablegen.c -noinst_LTLIBRARIES = liblzma4.la -liblzma4_la_CPPFLAGS = \ +## Using liblzma2 since liblzma is already used for the final library. +noinst_LTLIBRARIES = liblzma2.la +liblzma2_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common \ -I@top_srcdir@/src/liblzma/lz \ -I@top_srcdir@/src/liblzma/rangecoder -liblzma4_la_SOURCES = \ - lzma_common.h \ - lzma_literal.c \ - lzma_literal.h +liblzma2_la_SOURCES = lzma_common.h -if COND_MAIN_ENCODER -liblzma4_la_SOURCES += \ +if COND_ENCODER_LZMA +liblzma2_la_SOURCES += \ fastpos.h \ lzma_encoder.h \ lzma_encoder.c \ lzma_encoder_presets.c \ lzma_encoder_private.h \ - lzma_encoder_init.c \ lzma_encoder_features.c \ - lzma_encoder_getoptimum.c \ - lzma_encoder_getoptimumfast.c + lzma_encoder_optimum_fast.c \ + lzma_encoder_optimum_normal.c if !COND_SMALL -liblzma4_la_SOURCES += fastpos_table.c +liblzma2_la_SOURCES += fastpos_table.c endif endif -if COND_MAIN_DECODER -liblzma4_la_SOURCES += \ +if COND_DECODER_LZMA +liblzma2_la_SOURCES += \ lzma_decoder.c \ lzma_decoder.h endif + +if COND_ENCODER_LZMA2 +liblzma2_la_SOURCES += \ + lzma2_encoder.c \ + lzma2_encoder.h +endif + +if COND_DECODER_LZMA2 +liblzma2_la_SOURCES += \ + lzma2_decoder.c \ + lzma2_decoder.h +endif diff --git a/src/liblzma/lzma/fastpos.h b/src/liblzma/lzma/fastpos.h index 57a94556..503be275 100644 --- a/src/liblzma/lzma/fastpos.h +++ b/src/liblzma/lzma/fastpos.h @@ -81,8 +81,6 @@ // I'm making the table version the default, because that has good speed // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -// -// Finally, this code isn't a major bottle neck in LZMA encoding anyway. #ifdef HAVE_SMALL # include "bsr.h" @@ -135,11 +133,7 @@ get_pos_slot(uint32_t pos) static inline uint32_t get_pos_slot_2(uint32_t pos) { - // FIXME: This assert() cannot be enabled at the moment, because - // lzma_getoptimum.c calls this function so that this assertion - // fails; however, it ignores the result of this function when - // this assert() would have failed. - // assert(pos >= FULL_DISTANCES); + assert(pos >= FULL_DISTANCES); if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); diff --git a/src/liblzma/lzma/lzma2_decoder.c b/src/liblzma/lzma/lzma2_decoder.c new file mode 100644 index 00000000..b16c40ce --- /dev/null +++ b/src/liblzma/lzma/lzma2_decoder.c @@ -0,0 +1,318 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_decoder.c +/// \brief LZMA2 decoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma2_decoder.h" +#include "lz_decoder.h" +#include "lzma_decoder.h" + + +struct lzma_coder_s { + enum sequence { + SEQ_CONTROL, + SEQ_UNCOMPRESSED_1, + SEQ_UNCOMPRESSED_2, + SEQ_COMPRESSED_0, + SEQ_COMPRESSED_1, + SEQ_PROPERTIES, + SEQ_LZMA, + SEQ_COPY, + } sequence; + + /// Sequence after the size fields have been decoded. + enum sequence next_sequence; + + /// LZMA decoder + lzma_lz_decoder lzma; + + /// Uncompressed size of LZMA chunk + size_t uncompressed_size; + + /// Compressed size of the chunk (naturally equals to uncompressed + /// size of uncompressed chunk) + size_t compressed_size; + + /// True if properties are needed. This is false before the + /// first LZMA chunk. + bool need_properties; + + /// True if dictionary reset is needed. This is false before the + /// first chunk (LZMA or uncompressed). + bool need_dictionary_reset; + + lzma_options_lzma options; +}; + + +static lzma_ret +lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size) +{ + // With SEQ_LZMA it is possible that no new input is needed to do + // some progress. The rest of the sequences assume that there is + // at least one byte of input. + while (*in_pos < in_size || coder->sequence == SEQ_LZMA) + switch (coder->sequence) { + case SEQ_CONTROL: + if (in[*in_pos] & 0x80) { + // Get the highest five bits of uncompressed size. + coder->uncompressed_size + = (uint32_t)(in[*in_pos] & 0x1F) << 16; + coder->sequence = SEQ_UNCOMPRESSED_1; + + // See if we need to reset dictionary or state. + switch ((in[(*in_pos)++] >> 5) & 3) { + case 3: + dict_reset(dict); + coder->need_dictionary_reset = false; + + // Fall through + + case 2: + if (coder->need_dictionary_reset) + return LZMA_DATA_ERROR; + + coder->need_properties = false; + coder->next_sequence = SEQ_PROPERTIES; + break; + + case 1: + if (coder->need_properties) + return LZMA_DATA_ERROR; + + coder->lzma.reset(coder->lzma.coder, + &coder->options); + + coder->next_sequence = SEQ_LZMA; + break; + + case 0: + if (coder->need_properties) + return LZMA_DATA_ERROR; + + coder->next_sequence = SEQ_LZMA; + break; + } + + } else { + switch (in[(*in_pos)++]) { + case 0: + // End of payload marker + return LZMA_STREAM_END; + + case 1: + // Dictionary reset + dict_reset(dict); + coder->need_dictionary_reset = false; + + // Fall through + + case 2: + if (coder->need_dictionary_reset) + return LZMA_DATA_ERROR; + + // Uncompressed chunk; we need to read total + // size first. + coder->sequence = SEQ_COMPRESSED_0; + coder->next_sequence = SEQ_COPY; + break; + + default: + return LZMA_DATA_ERROR; + } + } + + break; + + case SEQ_UNCOMPRESSED_1: + coder->uncompressed_size += (uint32_t)(in[(*in_pos)++]) << 8; + coder->sequence = SEQ_UNCOMPRESSED_2; + break; + + case SEQ_UNCOMPRESSED_2: + coder->uncompressed_size += in[(*in_pos)++] + 1; + coder->sequence = SEQ_COMPRESSED_0; + coder->lzma.set_uncompressed(coder->lzma.coder, + coder->uncompressed_size); + break; + + case SEQ_COMPRESSED_0: + coder->compressed_size = (uint32_t)(in[(*in_pos)++]) << 8; + coder->sequence = SEQ_COMPRESSED_1; + break; + + case SEQ_COMPRESSED_1: + coder->compressed_size += in[(*in_pos)++] + 1; + coder->sequence = coder->next_sequence; + break; + + case SEQ_PROPERTIES: + if (lzma_lzma_lclppb_decode(&coder->options, in[(*in_pos)++])) + return LZMA_DATA_ERROR; + + coder->lzma.reset(coder->lzma.coder, &coder->options); + + coder->sequence = SEQ_LZMA; + break; + + case SEQ_LZMA: { + // Store the start offset so that we can update + // coder->compressed_size later. + const size_t in_start = *in_pos; + + // Decode from in[] to *dict. + const lzma_ret ret = coder->lzma.code(coder->lzma.coder, + dict, in, in_pos, in_size); + + // Validate and update coder->compressed_size. + const size_t in_used = *in_pos - in_start; + if (in_used > coder->compressed_size) + return LZMA_DATA_ERROR; + + coder->compressed_size -= in_used; + + // Return if we didn't finish the chunk, or an error occurred. + if (ret != LZMA_STREAM_END) + return ret; + + // The LZMA decoder must have consumed the whole chunk now. + // We don't need to worry about uncompressed size since it + // is checked by the LZMA decoder. + if (coder->compressed_size != 0) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_CONTROL; + break; + } + + case SEQ_COPY: { + // Copy from input to the dictionary as is. + // FIXME Can copy too much? + dict_write(dict, in, in_pos, in_size, &coder->compressed_size); + if (coder->compressed_size != 0) + return LZMA_OK; + + coder->sequence = SEQ_CONTROL; + break; + } + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + assert(coder->lzma.end == NULL); + lzma_free(coder->lzma.coder, allocator); + + lzma_free(coder, allocator); + + return; +} + + +static lzma_ret +lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *options, size_t *dict_size) +{ + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma2_decode; + lz->end = &lzma2_decoder_end; + + lz->coder->lzma = LZMA_LZ_DECODER_INIT; + } + + lz->coder->sequence = SEQ_CONTROL; + lz->coder->need_properties = true; + lz->coder->need_dictionary_reset = true; + + return lzma_lzma_decoder_create(&lz->coder->lzma, + allocator, options, dict_size); +} + + +extern lzma_ret +lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // LZMA2 can only be the last filter in the chain. This is enforced + // by the raw_decoder initialization. + assert(filters[1].init == NULL); + + return lzma_lz_decoder_init(next, allocator, filters, + &lzma2_decoder_init); +} + + +extern uint64_t +lzma_lzma2_decoder_memusage(const void *options) +{ + const uint64_t lzma_memusage = lzma_lzma_decoder_memusage(options); + if (lzma_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lzma_memusage; +} + + +extern lzma_ret +lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 1) + return LZMA_HEADER_ERROR; + + // Check that reserved bits are unset. + if (props[0] & 0xC0) + return LZMA_HEADER_ERROR; + + // Decode the dictionary size. + if (props[0] > 40) + return LZMA_HEADER_ERROR; + + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + if (props[0] == 40) { + opt->dictionary_size = UINT32_MAX; + } else { + opt->dictionary_size = 2 | (props[0] & 1); + opt->dictionary_size <<= props[0] / 2 + 11; + } + + opt->preset_dictionary = NULL; + opt->preset_dictionary_size = 0; + + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/lzma/lzma2_decoder.h b/src/liblzma/lzma/lzma2_decoder.h new file mode 100644 index 00000000..a7504863 --- /dev/null +++ b/src/liblzma/lzma/lzma2_decoder.h @@ -0,0 +1,35 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_decoder.h +/// \brief LZMA2 decoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA2_DECODER_H +#define LZMA_LZMA2_DECODER_H + +#include "common.h" + +extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern uint64_t lzma_lzma2_decoder_memusage(const void *options); + +extern lzma_ret lzma_lzma2_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + +#endif diff --git a/src/liblzma/lzma/lzma2_encoder.c b/src/liblzma/lzma/lzma2_encoder.c new file mode 100644 index 00000000..b2cd176b --- /dev/null +++ b/src/liblzma/lzma/lzma2_encoder.c @@ -0,0 +1,406 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_encoder.c +/// \brief LZMA2 encoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder.h" +#include "lzma_encoder.h" +#include "fastpos.h" +#include "lzma2_encoder.h" + + +/// Maximum number of bytes of actual data per chunk (no headers) +#define LZMA2_CHUNK_MAX (UINT32_C(1) << 16) + +/// Maximum uncompressed size of LZMA chunk (no headers) +#define LZMA2_UNCOMPRESSED_MAX (UINT32_C(1) << 21) + +/// Maximum size of LZMA2 headers +#define LZMA2_HEADER_MAX 6 + +/// Size of a header for uncompressed chunk +#define LZMA2_HEADER_UNCOMPRESSED 3 + + +struct lzma_coder_s { + enum { + SEQ_INIT, + SEQ_LZMA_ENCODE, + SEQ_LZMA_COPY, + SEQ_UNCOMPRESSED_HEADER, + SEQ_UNCOMPRESSED_COPY, + } sequence; + + /// LZMA encoder + lzma_coder *lzma; + + /// If this is not NULL, we will check new options from this + /// structure when starting a new chunk. + const lzma_options_lzma *opt_new; + + /// LZMA options currently in use. + lzma_options_lzma opt_cur; + + bool need_properties; + bool need_state_reset; + bool need_dictionary_reset; + + /// Uncompressed size of a chunk + size_t uncompressed_size; + + /// Compressed size of a chunk (excluding headers); this is also used + /// to indicate the end of buf[] in SEQ_LZMA_COPY. + size_t compressed_size; + + /// Read position in buf[] + size_t buf_pos; + + /// Buffer to hold the chunk header and LZMA compressed data + uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; +}; + + +static void +lzma2_header_lzma(lzma_coder *coder) +{ + assert(coder->uncompressed_size > 0); + assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); + assert(coder->compressed_size > 0); + assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + + size_t pos; + + if (coder->need_properties) { + pos = 0; + + if (coder->need_dictionary_reset) + coder->buf[pos] = 0x80 + (3 << 5); + else + coder->buf[pos] = 0x80 + (2 << 5); + } else { + pos = 1; + + if (coder->need_state_reset) + coder->buf[pos] = 0x80 + (1 << 5); + else + coder->buf[pos] = 0x80; + } + + // Set the start position for copying. + coder->buf_pos = pos; + + // Uncompressed size + size_t size = coder->uncompressed_size - 1; + coder->buf[pos++] += size >> 16; + coder->buf[pos++] = (size >> 8) & 0xFF; + coder->buf[pos++] = size & 0xFF; + + // Compressed size + size = coder->compressed_size - 1; + coder->buf[pos++] = size >> 8; + coder->buf[pos++] = size & 0xFF; + + // Properties, if needed + if (coder->need_properties) + lzma_lzma_lclppb_encode(&coder->opt_cur, coder->buf + pos); + + coder->need_properties = false; + coder->need_state_reset = false; + coder->need_dictionary_reset = false; + + // The copying code uses coder->compressed_size to indicate the end + // of coder->buf[], so we need add the maximum size of the header here. + coder->compressed_size += LZMA2_HEADER_MAX; + + return; +} + + +static void +lzma2_header_uncompressed(lzma_coder *coder) +{ + assert(coder->uncompressed_size > 0); + assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); + + // If this is the first chunk, we need to include dictionary + // reset indicator. + if (coder->need_dictionary_reset) + coder->buf[0] = 1; + else + coder->buf[0] = 2; + + coder->need_dictionary_reset = false; + + // "Compressed" size + coder->buf[1] = (coder->uncompressed_size - 1) >> 8; + coder->buf[2] = (coder->uncompressed_size - 1) & 0xFF; + + // Set the start position for copying. + coder->buf_pos = 0; + return; +} + + +static lzma_ret +lzma2_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_INIT: + // If there's no input left and we are flushing or finishing, + // don't start a new chunk. + if (mf_unencoded(mf) == 0) { + // Write end of payload marker if finishing. + if (mf->action == LZMA_FINISH) + out[(*out_pos)++] = 0; + + return mf->action == LZMA_RUN + ? LZMA_OK : LZMA_STREAM_END; + } + + // Look if there are new options. At least for now, + // only lc/lp/pb can be changed. + if (coder->opt_new != NULL + && (coder->opt_cur.literal_context_bits + != coder->opt_new->literal_context_bits + || coder->opt_cur.literal_pos_bits + != coder->opt_new->literal_pos_bits + || coder->opt_cur.pos_bits + != coder->opt_new->pos_bits)) { + // Options have been changed, copy them to opt_cur. + coder->opt_cur.literal_context_bits + = coder->opt_new->literal_context_bits; + coder->opt_cur.literal_pos_bits + = coder->opt_new->literal_pos_bits; + coder->opt_cur.pos_bits + = coder->opt_new->pos_bits; + + // We need to write the new options and reset + // the encoder state. + coder->need_properties = true; + coder->need_state_reset = true; + } + + if (coder->need_state_reset) + lzma_lzma_encoder_reset(coder->lzma, &coder->opt_cur); + + coder->uncompressed_size = 0; + coder->compressed_size = 0; + coder->sequence = SEQ_LZMA_ENCODE; + + // Fall through + + case SEQ_LZMA_ENCODE: { + // Calculate how much more uncompressed data this chunk + // could accept. + const uint32_t left = LZMA2_UNCOMPRESSED_MAX + - coder->uncompressed_size; + uint32_t limit; + + if (left < mf->match_len_max) { + // Must flush immediatelly since the next LZMA symbol + // could make the uncompressed size of the chunk too + // big. + limit = 0; + } else { + // Calculate maximum read_limit that is OK from point + // of view of LZMA2 chunk size. + limit = mf->read_pos - mf->read_ahead + + left - mf->match_len_max; + } + + // Save the start position so that we can update + // coder->uncompressed_size. + const uint32_t read_start = mf->read_pos - mf->read_ahead; + + // Call the LZMA encoder until the chunk is finished. + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, + coder->buf + LZMA2_HEADER_MAX, + &coder->compressed_size, + LZMA2_CHUNK_MAX, limit); + + coder->uncompressed_size += mf->read_pos - mf->read_ahead + - read_start; + + assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); + + if (ret != LZMA_STREAM_END) + return LZMA_OK; + + // See if the chunk compressed. If it didn't, we encode it + // as uncompressed chunk. This saves a few bytes of space + // and makes decoding faster. + if (coder->compressed_size >= coder->uncompressed_size) { + coder->uncompressed_size += mf->read_ahead; + assert(coder->uncompressed_size + <= LZMA2_UNCOMPRESSED_MAX); + mf->read_ahead = 0; + lzma2_header_uncompressed(coder); + coder->need_state_reset = true; + coder->sequence = SEQ_UNCOMPRESSED_HEADER; + break; + } + + // The chunk did compress at least by one byte, so we store + // the chunk as LZMA. + lzma2_header_lzma(coder); + + coder->sequence = SEQ_LZMA_COPY; + } + + // Fall through + + case SEQ_LZMA_COPY: + // Copy the compressed chunk along its headers to the + // output buffer. + lzma_bufcpy(coder->buf, &coder->buf_pos, + coder->compressed_size, + out, out_pos, out_size); + if (coder->buf_pos != coder->compressed_size) + return LZMA_OK; + + coder->sequence = SEQ_INIT; + break; + + case SEQ_UNCOMPRESSED_HEADER: + // Copy the three-byte header to indicate uncompressed chunk. + lzma_bufcpy(coder->buf, &coder->buf_pos, + LZMA2_HEADER_UNCOMPRESSED, + out, out_pos, out_size); + if (coder->buf_pos != LZMA2_HEADER_UNCOMPRESSED) + return LZMA_OK; + + coder->sequence = SEQ_UNCOMPRESSED_COPY; + + // Fall through + + case SEQ_UNCOMPRESSED_COPY: + // Copy the uncompressed data as is from the dictionary + // to the output buffer. + mf_read(mf, out, out_pos, out_size, &coder->uncompressed_size); + if (coder->uncompressed_size != 0) + return LZMA_OK; + + coder->sequence = SEQ_INIT; + break; + } + + return LZMA_OK; +} + + +static void +lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder->lzma, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, + const void *options, lzma_lz_options *lz_options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma2_encode; + lz->end = &lzma2_encoder_end; + + lz->coder->lzma = NULL; + } + + lz->coder->sequence = SEQ_INIT; + lz->coder->need_properties = true; + lz->coder->need_state_reset = false; + lz->coder->need_dictionary_reset = true; + + lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + lz->coder->opt_new = lz->coder->opt_cur.persistent + ? options : NULL; + + // Initialize LZMA encoder + return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, + &lz->coder->opt_cur, lz_options)); + + // Make sure that we will always have enough history available in + // case we need to use uncompressed chunks. They are used when the + // compressed size of a chunk is not smaller than the uncompressed + // size, so we need to have at least LZMA2_COMPRESSED_MAX bytes + // history available. + if (lz_options->before_size + lz_options->dictionary_size + < LZMA2_CHUNK_MAX) + lz_options->before_size = LZMA2_CHUNK_MAX + - lz_options->dictionary_size; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_lz_encoder_init( + next, allocator, filters, &lzma2_encoder_init); +} + + +extern uint64_t +lzma_lzma2_encoder_memusage(const void *options) +{ + const uint64_t lzma_memusage = lzma_lzma_encoder_memusage(options); + if (lzma_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lzma_memusage; +} + + +extern lzma_ret +lzma_lzma2_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_lzma *const opt = options; + uint32_t d = MAX(opt->dictionary_size, LZMA_DICTIONARY_SIZE_MIN); + + // Round up to to the next 2^n - 1 or 2^n + 2^(n - 1) - 1 depending + // on which one is the next: + --d; + d |= d >> 2; + d |= d >> 3; + d |= d >> 4; + d |= d >> 8; + d |= d >> 16; + + // Get the highest two bits using the proper encoding: + if (d == UINT32_MAX) + out[0] = 40; + else + out[0] = get_pos_slot(d + 1) - 24; + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_common.h b/src/liblzma/lzma/lzma2_encoder.h index 0a27f3dc..3e27f680 100644 --- a/src/liblzma/common/raw_common.h +++ b/src/liblzma/lzma/lzma2_encoder.h @@ -1,9 +1,10 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_common.h -/// \brief Stuff shared between raw encoder and raw decoder +/// \file lzma2_encoder.h +/// \brief LZMA2 encoder // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,14 +18,17 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_COMMON_H -#define LZMA_RAW_COMMON_H +#ifndef LZMA_LZMA2_ENCODER_H +#define LZMA_LZMA2_ENCODER_H #include "common.h" -extern lzma_ret lzma_raw_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options, - lzma_init_function (*get_function)(lzma_vli id), - bool is_encoder); +extern lzma_ret lzma_lzma2_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern uint64_t lzma_lzma2_encoder_memusage(const void *options); + +extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); #endif diff --git a/src/liblzma/lzma/lzma_common.h b/src/liblzma/lzma/lzma_common.h index f677fcce..6909969b 100644 --- a/src/liblzma/lzma/lzma_common.h +++ b/src/liblzma/lzma/lzma_common.h @@ -22,81 +22,31 @@ #define LZMA_LZMA_COMMON_H #include "common.h" -#include "lzma_literal.h" #include "range_common.h" -/////////////// -// Constants // -/////////////// - -#define REP_DISTANCES 4 - -#define POS_SLOT_BITS 6 -#define DICT_LOG_SIZE_MAX 30 -#define DIST_TABLE_SIZE_MAX (DICT_LOG_SIZE_MAX * 2) -#if (UINT32_C(1) << DICT_LOG_SIZE_MAX) != LZMA_DICTIONARY_SIZE_MAX -# error DICT_LOG_SIZE_MAX is inconsistent with LZMA_DICTIONARY_SIZE_MAX -#endif - -// 2 is for speed optimization -#define LEN_TO_POS_STATES_BITS 2 -#define LEN_TO_POS_STATES (1 << LEN_TO_POS_STATES_BITS) - -#define MATCH_MIN_LEN 2 - -#define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) - -#define START_POS_MODEL_INDEX 4 -#define END_POS_MODEL_INDEX 14 -#define POS_MODELS (END_POS_MODEL_INDEX - START_POS_MODEL_INDEX) - -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) -#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) +/////////////////// +// Miscellaneous // +/////////////////// +/// Maximum number of position states. A position state is the lowest pos bits +/// number of bits of the current uncompressed offset. In some places there +/// are different sets of probabilities for different pos states. #define POS_STATES_MAX (1 << LZMA_POS_BITS_MAX) -// Length coder & Length price table encoder -#define LEN_LOW_BITS 3 -#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS) -#define LEN_MID_BITS 3 -#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS) -#define LEN_HIGH_BITS 8 -#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS) -#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS) -#define LEN_SPEC_SYMBOLS (LOW_LOW_SYMBOLS + LEN_MID_LEN_SYMBOLS) -#define MATCH_MAX_LEN (MATCH_MIN_LEN + LEN_SYMBOLS - 1) - -// Total number of probs in a Len Encoder -#define LEN_CODER_TOTAL_PROBS (LEN_HIGH_CODER + LEN_HIGH_SYMBOLS) - -// Price table size of Len Encoder -#define LEN_PRICES (LEN_SYMBOLS << LZMA_POS_BITS_MAX) - -// Special lengths used together with distance == UINT32_MAX -#define LEN_SPECIAL_EOPM MATCH_MIN_LEN -#define LEN_SPECIAL_FLUSH (LEN_SPECIAL_EOPM + 1) - - -// Optimal - Number of entries in the optimum array. -#define OPTS (1 << 12) - - -// Miscellaneous -#define INFINITY_PRICE 0x0FFFFFFF - - -//////////// -// Macros // -//////////// - -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_MIN_LEN \ - ? (len) - MATCH_MIN_LEN \ - : LEN_TO_POS_STATES - 1) +/// Validates literal_context_bits, literal_pos_bits, and pos_bits. +static inline bool +is_lclppb_valid(const lzma_options_lzma *options) +{ + return options->literal_context_bits <= LZMA_LITERAL_CONTEXT_BITS_MAX + && options->literal_pos_bits + <= LZMA_LITERAL_POS_BITS_MAX + && options->literal_context_bits + + options->literal_pos_bits + <= LZMA_LITERAL_BITS_MAX + && options->pos_bits <= LZMA_POS_BITS_MAX; +} /////////// @@ -161,4 +111,126 @@ typedef enum { #define is_literal_state(state) \ ((state) < LIT_STATES) + +///////////// +// Literal // +///////////// + +/// Each literal coder is divided in three sections: +/// - 0x001-0x0FF: Without match byte +/// - 0x101-0x1FF: With match byte; match bit is 0 +/// - 0x201-0x2FF: With match byte; match bit is 1 +/// +/// Match byte is used when the previous LZMA symbol was something else than +/// a literal (that is, it was some kind of match). +#define LITERAL_CODER_SIZE 0x300 + +/// Maximum number of literal coders +#define LITERAL_CODERS_MAX (1 << LZMA_LITERAL_BITS_MAX) + +/// Locate the literal coder for the next literal byte. The choice depends on +/// - the lowest literal_pos_bits bits of the position of the current +/// byte; and +/// - the highest literal_context_bits bits of the previous byte. +#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \ + ((probs)[(((pos) & lp_mask) << lc) + ((prev_byte) >> (8 - lc))]) + + +static inline void +literal_init(probability (*probs)[LITERAL_CODER_SIZE], + uint32_t literal_context_bits, uint32_t literal_pos_bits) +{ + assert(literal_context_bits + literal_pos_bits + <= LZMA_LITERAL_BITS_MAX); + + const uint32_t coders + = 1U << (literal_context_bits + literal_pos_bits); + + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) + bit_reset(probs[i][j]); + + return; +} + + +////////////////// +// Match length // +////////////////// + +// Minimum length of a match is two bytes. +#define MATCH_LEN_MIN 2 + +// Match length is encoded with 4, 5, or 10 bits. +// +// Length Bits +// 2-9 4 = Choice=0 + 3 bits +// 10-17 5 = Choice=1 + Choice2=0 + 3 bits +// 18-273 10 = Choice=1 + Choice2=1 + 8 bits +#define LEN_LOW_BITS 3 +#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS) +#define LEN_MID_BITS 3 +#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS) +#define LEN_HIGH_BITS 8 +#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS) +#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS) + +// Maximum length of a match is 273 which is a result of the encoding +// described above. +#define MATCH_LEN_MAX (MATCH_LEN_MIN + LEN_SYMBOLS - 1) + + +//////////////////// +// Match distance // +//////////////////// + +// Different set of probabilities is used for match distances that have very +// short match length: Lengths of 2, 3, and 4 bytes have a separate set of +// probabilities for each length. The matches with longer length use a shared +// set of probabilities. +#define LEN_TO_POS_STATES 4 + +// Macro to get the index of the appropriate probability array. +#define get_len_to_pos_state(len) \ + ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ + ? (len) - MATCH_LEN_MIN \ + : LEN_TO_POS_STATES - 1) + +// The highest two bits of a match distance (pos slot) are encoded using six +// bits. See fastpos.h for more explanation. +#define POS_SLOT_BITS 6 +#define POS_SLOTS (1 << POS_SLOT_BITS) + +// Match distances up to 127 are fully encoded using probabilities. Since +// the highest two bits (pos slot) are always encoded using six bits, the +// distances 0-3 don't need any additional bits to encode, since the pos +// slot itself is the same as the actual distance. START_POS_MODEL_INDEX +// indicates the first pos slot where at least one additional bit is needed. +#define START_POS_MODEL_INDEX 4 + +// Match distances greater than 127 are encoded in three pieces: +// - pos slot: the highest two bits +// - direct bits: 2-26 bits below the highest two bits +// - alignment bits: four lowest bits +// +// Direct bits don't use any probabilities. +// +// The pos slot value of 14 is for distances 128-191 (see the table in +// fastpos.h to understand why). +#define END_POS_MODEL_INDEX 14 + +// Seven-bit distances use the full FIXME +#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) + +// For match distances greater than 127, only the highest two bits and the +// lowest four bits (alignment) is encoded using probabilities. +#define ALIGN_BITS 4 +#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) + +// LZMA remembers the four most recent match distances. Reusing these distances +// tends to take less space than re-encoding the actual distance value. +#define REP_DISTANCES 4 + #endif diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c index 68941021..e9d047d3 100644 --- a/src/liblzma/lzma/lzma_decoder.c +++ b/src/liblzma/lzma/lzma_decoder.c @@ -4,7 +4,7 @@ /// \brief LZMA decoder // // Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -18,74 +18,147 @@ // /////////////////////////////////////////////////////////////////////////////// -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - +#include "lz_decoder.h" #include "lzma_common.h" #include "lzma_decoder.h" -#include "lz_decoder.h" #include "range_decoder.h" -/// REQUIRED_IN_BUFFER_SIZE is the number of required input bytes -/// for the worst case: longest match with longest distance. -/// LZMA_IN_BUFFER_SIZE must be larger than REQUIRED_IN_BUFFER_SIZE. -/// 23 bits = 2 (match select) + 10 (len) + 6 (distance) + 4 (align) -/// + 1 (rc_normalize) -/// -/// \todo Is this correct for sure? -/// -#define REQUIRED_IN_BUFFER_SIZE \ - ((23 * (BIT_MODEL_TOTAL_BITS - MOVE_BITS + 1) + 26 + 9) / 8) +#ifdef HAVE_SMALL +// Macros for (somewhat) size-optimized code. +#define seq_4(seq) seq -// Length decoders are easiest to have as macros, because they use range -// decoder macros, which use local variables rc_range and rc_code. +#define seq_6(seq) seq -#define length_decode(target, len_decoder, pos_state) \ +#define seq_8(seq) seq + +#define seq_len(seq) \ + seq ## _CHOICE, \ + seq ## _CHOICE2, \ + seq ## _BITTREE + +#define len_decode(target, ld, pos_state, seq) \ do { \ - if_bit_0(len_decoder.choice) { \ - update_bit_0(len_decoder.choice); \ - target = MATCH_MIN_LEN; \ - bittree_decode(target, len_decoder.low[pos_state], LEN_LOW_BITS); \ +case seq ## _CHOICE: \ + rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_update_0(ld.choice); \ + probs = ld.low[pos_state];\ + limit = LEN_LOW_SYMBOLS; \ + target = MATCH_LEN_MIN; \ } else { \ - update_bit_1(len_decoder.choice); \ - if_bit_0(len_decoder.choice2) { \ - update_bit_0(len_decoder.choice2); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ - bittree_decode(target, len_decoder.mid[pos_state], LEN_MID_BITS); \ + rc_update_1(ld.choice); \ +case seq ## _CHOICE2: \ + rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_update_0(ld.choice2); \ + probs = ld.mid[pos_state]; \ + limit = LEN_MID_SYMBOLS; \ + target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ - update_bit_1(len_decoder.choice2); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ - bittree_decode(target, len_decoder.high, LEN_HIGH_BITS); \ + rc_update_1(ld.choice2); \ + probs = ld.high; \ + limit = LEN_HIGH_SYMBOLS; \ + target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS \ + + LEN_MID_SYMBOLS; \ } \ } \ + symbol = 1; \ +case seq ## _BITTREE: \ + do { \ + rc_bit(probs[symbol], , , seq ## _BITTREE); \ + } while (symbol < limit); \ + target += symbol - limit; \ } while (0) - -#define length_decode_dummy(target, len_decoder, pos_state) \ +#else // HAVE_SMALL + +// Unrolled versions +#define seq_4(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3 + +#define seq_6(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3, \ + seq ## 4, \ + seq ## 5 + +#define seq_8(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3, \ + seq ## 4, \ + seq ## 5, \ + seq ## 6, \ + seq ## 7 + +#define seq_len(seq) \ + seq ## _CHOICE, \ + seq ## _LOW0, \ + seq ## _LOW1, \ + seq ## _LOW2, \ + seq ## _CHOICE2, \ + seq ## _MID0, \ + seq ## _MID1, \ + seq ## _MID2, \ + seq ## _HIGH0, \ + seq ## _HIGH1, \ + seq ## _HIGH2, \ + seq ## _HIGH3, \ + seq ## _HIGH4, \ + seq ## _HIGH5, \ + seq ## _HIGH6, \ + seq ## _HIGH7 + +#define len_decode(target, ld, pos_state, seq) \ do { \ - if_bit_0(len_decoder.choice) { \ - update_bit_0_dummy(); \ - target = MATCH_MIN_LEN; \ - bittree_decode_dummy(target, \ - len_decoder.low[pos_state], LEN_LOW_BITS); \ + symbol = 1; \ +case seq ## _CHOICE: \ + rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_update_0(ld.choice); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ + target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ } else { \ - update_bit_1_dummy(); \ - if_bit_0(len_decoder.choice2) { \ - update_bit_0_dummy(); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ - bittree_decode_dummy(target, len_decoder.mid[pos_state], \ - LEN_MID_BITS); \ + rc_update_1(ld.choice); \ +case seq ## _CHOICE2: \ + rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_update_0(ld.choice2); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID0); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID1); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID2); \ + target = symbol - LEN_MID_SYMBOLS \ + + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ - update_bit_1_dummy(); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ - bittree_decode_dummy(target, len_decoder.high, LEN_HIGH_BITS); \ + rc_update_1(ld.choice2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ + target = symbol - LEN_HIGH_SYMBOLS \ + + MATCH_LEN_MIN \ + + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ } \ } \ } while (0) +#endif // HAVE_SMALL + +/// Length decoder probabilities; see comments in lzma_common.h. typedef struct { probability choice; probability choice2; @@ -96,26 +169,12 @@ typedef struct { struct lzma_coder_s { - /// Data of the next coder, if any. - lzma_next_coder next; - - /// Sliding output window a.k.a. dictionary a.k.a. history buffer. - lzma_lz_decoder lz; - - // Range coder - lzma_range_decoder rc; - - // State - lzma_lzma_state state; - uint32_t rep0; ///< Distance of the latest match - uint32_t rep1; ///< Distance of second latest match - uint32_t rep2; ///< Distance of third latest match - uint32_t rep3; ///< Distance of fourth latest match - uint32_t pos_bits; - uint32_t pos_mask; - uint32_t now_pos; // Lowest 32-bits are enough here. + /////////////////// + // Probabilities // + /////////////////// - lzma_literal_coder literal_coder; + /// Literals; see comments in lzma_common.h. + probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; /// If 1, it's a match. Otherwise it's a single 8-bit literal. probability is_match[STATES][POS_STATES_MAX]; @@ -138,178 +197,107 @@ struct lzma_coder_s { /// the length is decoded from rep_len_decoder. probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot_decoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; - probability pos_decoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align_decoder[1 << ALIGN_BITS]; - - /// Length of a match - lzma_length_decoder match_len_decoder; - - /// Length of a repeated match. - lzma_length_decoder rep_len_decoder; - - /// True when we have produced at least one byte of output since the - /// beginning of the stream or the latest flush marker. - bool has_produced_output; -}; - - -/// \brief Check if the next iteration of the decoder loop can be run. -/// -/// \note There must always be REQUIRED_IN_BUFFER_SIZE bytes -/// readable space! -/// -static bool lzma_attribute((pure)) -decode_dummy(const lzma_coder *restrict coder, - const uint8_t *restrict in, size_t in_pos_local, - const size_t in_size, lzma_range_decoder rc, - uint32_t state, uint32_t rep0, const uint32_t now_pos) -{ - uint32_t rc_bound; - - do { - const uint32_t pos_state = now_pos & coder->pos_mask; - - if_bit_0(coder->is_match[state][pos_state]) { - // It's a literal i.e. a single 8-bit byte. - - update_bit_0_dummy(); - - const probability *subcoder = literal_get_subcoder( - coder->literal_coder, now_pos, lz_get_byte(coder->lz, 0)); - uint32_t symbol = 1; - - if (is_literal_state(state)) { - // Decode literal without match byte. - do { - if_bit_0(subcoder[symbol]) { - update_bit_0_dummy(); - symbol <<= 1; - } else { - update_bit_1_dummy(); - symbol = (symbol << 1) | 1; - } - } while (symbol < 0x100); - - } else { - // Decode literal with match byte. - uint32_t match_byte = lz_get_byte(coder->lz, rep0); - uint32_t subcoder_offset = 0x100; - - do { - match_byte <<= 1; - const uint32_t match_bit = match_byte & subcoder_offset; - const uint32_t subcoder_index - = subcoder_offset + match_bit + symbol; - - if_bit_0(subcoder[subcoder_index]) { - update_bit_0_dummy(); - symbol <<= 1; - subcoder_offset &= ~match_bit; - } else { - update_bit_1_dummy(); - symbol = (symbol << 1) | 1; - subcoder_offset &= match_bit; - } - } while (symbol < 0x100); - } - - break; - } - - update_bit_1_dummy(); - uint32_t len; - - if_bit_0(coder->is_rep[state]) { - update_bit_0_dummy(); - length_decode_dummy(len, coder->match_len_decoder, pos_state); - - const uint32_t len_to_pos_state = get_len_to_pos_state(len); - uint32_t pos_slot = 0; - bittree_decode_dummy(pos_slot, - coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS); - assert(pos_slot <= 63); - - if (pos_slot >= START_POS_MODEL_INDEX) { - uint32_t direct_bits = (pos_slot >> 1) - 1; - assert(direct_bits >= 1 && direct_bits <= 31); - rep0 = 2 | (pos_slot & 1); - - if (pos_slot < END_POS_MODEL_INDEX) { - assert(direct_bits <= 5); - rep0 <<= direct_bits; - assert(rep0 <= 96); - // -1 is fine, because bittree_reverse_decode() - // starts from table index [1] (not [0]). - assert((int32_t)(rep0 - pos_slot - 1) >= -1); - assert((int32_t)(rep0 - pos_slot - 1) <= 82); - // We add the result to rep0, so rep0 - // must not be part of second argument - // of the macro. - const int32_t offset = rep0 - pos_slot - 1; - bittree_reverse_decode_dummy(coder->pos_decoders + offset, - direct_bits); - } else { - assert(pos_slot >= 14); - assert(direct_bits >= 6); - direct_bits -= ALIGN_BITS; - assert(direct_bits >= 2); - rc_decode_direct_dummy(direct_bits); - - bittree_reverse_decode_dummy(coder->pos_align_decoder, - ALIGN_BITS); - } - } + /// Probability tree for the highest two bits of the match distance. + /// There is a separate probability tree for match lengths of + /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. + probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - } else { - update_bit_1_dummy(); + /// Probility trees for additional bits for match distance when the + /// distance is in the range [4, 127]. + probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; - if_bit_0(coder->is_rep0[state]) { - update_bit_0_dummy(); + /// Probability tree for the lowest four bits of a match distance + /// that is equal to or greater than 128. + probability pos_align[ALIGN_TABLE_SIZE]; - if_bit_0(coder->is_rep0_long[state][pos_state]) { - update_bit_0_dummy(); - break; - } else { - update_bit_1_dummy(); - } + /// Length of a normal match + lzma_length_decoder match_len_decoder; - } else { - update_bit_1_dummy(); + /// Length of a repeated match + lzma_length_decoder rep_len_decoder; - if_bit_0(coder->is_rep1[state]) { - update_bit_0_dummy(); - } else { - update_bit_1_dummy(); + /////////////////// + // Decoder state // + /////////////////// - if_bit_0(coder->is_rep2[state]) { - update_bit_0_dummy(); - } else { - update_bit_1_dummy(); - } - } - } + // Range coder + lzma_range_decoder rc; - length_decode_dummy(len, coder->rep_len_decoder, pos_state); - } - } while (0); + // Types of the most recently seen LZMA symbols + lzma_lzma_state state; - rc_normalize(); + uint32_t rep0; ///< Distance of the latest match + uint32_t rep1; ///< Distance of second latest match + uint32_t rep2; ///< Distance of third latest match + uint32_t rep3; ///< Distance of fourth latest match - return in_pos_local <= in_size; -} + uint32_t pos_mask; // (1U << pos_bits) - 1 + uint32_t literal_context_bits; + uint32_t literal_pos_mask; + + /// Uncompressed size as bytes, or LZMA_VLI_VALUE_UNKNOWN if end of + /// payload marker is expected. + lzma_vli uncompressed_size; + + //////////////////////////////// + // State of incomplete symbol // + //////////////////////////////// + + /// Position where to continue the decoder loop + enum { + SEQ_NORMALIZE, + SEQ_IS_MATCH, + seq_8(SEQ_LITERAL), + seq_8(SEQ_LITERAL_MATCHED), + SEQ_LITERAL_WRITE, + SEQ_IS_REP, + seq_len(SEQ_MATCH_LEN), + seq_6(SEQ_POS_SLOT), + SEQ_POS_MODEL, + SEQ_DIRECT, + seq_4(SEQ_ALIGN), + SEQ_EOPM, + SEQ_IS_REP0, + SEQ_SHORTREP, + SEQ_IS_REP0_LONG, + SEQ_IS_REP1, + SEQ_IS_REP2, + seq_len(SEQ_REP_LEN), + SEQ_COPY, + } sequence; + + /// Base of the current probability tree + probability *probs; + + /// Symbol being decoded. This is also used as an index variable in + /// bittree decoders: probs[symbol] + uint32_t symbol; + + /// Used as a loop termination condition on bittree decoders and + /// direct bits decoder. + uint32_t limit; + + /// Matched literal decoder: 0x100 or 0 to help avoiding branches. + /// Bittree reverse decoders: Offset of the next bit: 1 << offset + uint32_t offset; + + /// If decoding a literal: match byte. + /// If decoding a match: length of the match. + uint32_t len; +}; -static bool -decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size, bool has_safe_buffer) +static lzma_ret +lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { //////////////////// // Initialization // //////////////////// if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return false; + return LZMA_OK; /////////////// // Variables // @@ -318,8 +306,12 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // Making local copies of often-used variables improves both // speed and readability. + lzma_dict dict = *dictptr; + + const size_t dict_start = dict.pos; + // Range decoder - rc_to_local(coder->rc); + rc_to_local(coder->rc, *in_pos); // State uint32_t state = coder->state; @@ -328,87 +320,168 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, uint32_t rep2 = coder->rep2; uint32_t rep3 = coder->rep3; - // Misc - uint32_t now_pos = coder->now_pos; - bool has_produced_output = coder->has_produced_output; - - // Variables derived from decoder settings const uint32_t pos_mask = coder->pos_mask; - size_t in_pos_local = *in_pos; // Local copy - size_t in_limit; - if (in_size <= REQUIRED_IN_BUFFER_SIZE) - in_limit = 0; - else - in_limit = in_size - REQUIRED_IN_BUFFER_SIZE; - - - while (coder->lz.pos < coder->lz.limit - && (in_pos_local < in_limit || (has_safe_buffer - && decode_dummy(coder, in, in_pos_local, in_size, - rc, state, rep0, now_pos)))) { - - ///////////////////// - // Actual decoding // - ///////////////////// - - const uint32_t pos_state = now_pos & pos_mask; + // These variables are actually needed only if we last time ran + // out of input in the middle of the decoder loop. + probability *probs = coder->probs; + uint32_t symbol = coder->symbol; + uint32_t limit = coder->limit; + uint32_t offset = coder->offset; + uint32_t len = coder->len; + + const uint32_t literal_pos_mask = coder->literal_pos_mask; + const uint32_t literal_context_bits = coder->literal_context_bits; + + // Temporary variables + uint32_t pos_state = dict.pos & pos_mask; + + lzma_ret ret = LZMA_OK; + + // If uncompressed size is known, there must be no end of payload + // marker. + const bool no_eopm = coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN; + if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) + dict.limit = dict.pos + (size_t)(coder->uncompressed_size); + + // The main decoder loop. The "switch" is used to restart the decoder at + // correct location. Once restarted, the "switch" is no longer used. + switch (coder->sequence) + while (true) { + // Calculate new pos_state. This is skipped on the first loop + // since we already calculated it when setting up the local + // variables. + pos_state = dict.pos & pos_mask; + + case SEQ_NORMALIZE: + case SEQ_IS_MATCH: + if (unlikely(no_eopm && dict.pos == dict.limit)) + break; - if_bit_0(coder->is_match[state][pos_state]) { - update_bit_0(coder->is_match[state][pos_state]); + rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. - probability *subcoder = literal_get_subcoder(coder->literal_coder, - now_pos, lz_get_byte(coder->lz, 0)); - uint32_t symbol = 1; + probs = literal_subcoder(coder->literal, + literal_context_bits, literal_pos_mask, + dict.pos, dict_get(&dict, 0)); + symbol = 1; if (is_literal_state(state)) { // Decode literal without match byte. +#ifdef HAVE_SMALL + case SEQ_LITERAL: do { - if_bit_0(subcoder[symbol]) { - update_bit_0(subcoder[symbol]); - symbol <<= 1; - } else { - update_bit_1(subcoder[symbol]); - symbol = (symbol << 1) | 1; - } - } while (symbol < 0x100); - + rc_bit(probs[symbol], , , SEQ_LITERAL); + } while (symbol < (1 << 8)); +#else + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); +#endif } else { // Decode literal with match byte. // - // The usage of subcoder_offset allows omitting some - // branches, which should give tiny speed improvement on - // some CPUs. subcoder_offset gets set to zero if match_bit - // didn't match. - uint32_t match_byte = lz_get_byte(coder->lz, rep0); - uint32_t subcoder_offset = 0x100; - + // We store the byte we compare against + // ("match byte") to "len" to minimize the + // number of variables we need to store + // between decoder calls. + len = dict_get(&dict, rep0) << 1; + + // The usage of "offset" allows omitting some + // branches, which should give tiny speed + // improvement on some CPUs. "offset" gets + // set to zero if match_bit didn't match. + offset = 0x100; + +#ifdef HAVE_SMALL + case SEQ_LITERAL_MATCHED: do { - match_byte <<= 1; - const uint32_t match_bit = match_byte & subcoder_offset; + const uint32_t match_bit + = len & offset; const uint32_t subcoder_index - = subcoder_offset + match_bit + symbol; + = offset + match_bit + + symbol; + + rc_bit(probs[subcoder_index], + offset &= ~match_bit, + offset &= match_bit, + SEQ_LITERAL_MATCHED); + + // It seems to be faster to do this + // here instead of putting it to the + // beginning of the loop and then + // putting the "case" in the middle + // of the loop. + len <<= 1; + + } while (symbol < (1 << 8)); +#else + // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; + +# define d(seq) \ + case seq: \ + match_bit = len & offset; \ + subcoder_index = offset + match_bit + symbol; \ + rc_bit(probs[subcoder_index], \ + offset &= ~match_bit, \ + offset &= match_bit, \ + seq) + + d(SEQ_LITERAL_MATCHED0); + len <<= 1; + d(SEQ_LITERAL_MATCHED1); + len <<= 1; + d(SEQ_LITERAL_MATCHED2); + len <<= 1; + d(SEQ_LITERAL_MATCHED3); + len <<= 1; + d(SEQ_LITERAL_MATCHED4); + len <<= 1; + d(SEQ_LITERAL_MATCHED5); + len <<= 1; + d(SEQ_LITERAL_MATCHED6); + len <<= 1; + d(SEQ_LITERAL_MATCHED7); +# undef d +#endif + } - if_bit_0(subcoder[subcoder_index]) { - update_bit_0(subcoder[subcoder_index]); - symbol <<= 1; - subcoder_offset &= ~match_bit; - } else { - update_bit_1(subcoder[subcoder_index]); - symbol = (symbol << 1) | 1; - subcoder_offset &= match_bit; - } - } while (symbol < 0x100); + //update_literal(state); + // Use a lookup table to update to literal state, + // since compared to other state updates, this would + // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; + state = next_state[state]; + + case SEQ_LITERAL_WRITE: + if (unlikely(dict_put(&dict, symbol))) { + coder->sequence = SEQ_LITERAL_WRITE; + goto out; } - // Put the decoded byte to the dictionary, update the - // decoder state, and start a new decoding loop. - coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol); - ++now_pos; - update_literal(state); - has_produced_output = true; continue; } @@ -416,115 +489,196 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // (distance and length) which will be repeated from our // output history. - update_bit_1(coder->is_match[state][pos_state]); - uint32_t len; - - if_bit_0(coder->is_rep[state]) { - update_bit_0(coder->is_rep[state]); + rc_update_1(coder->is_match[state][pos_state]); + case SEQ_IS_REP: + rc_if_0(coder->is_rep[state], SEQ_IS_REP) { // Not a repeated match - // - // We will decode a new distance and store - // the value to distance. - - // Decode the length of the match. - length_decode(len, coder->match_len_decoder, pos_state); - + rc_update_0(coder->is_rep[state]); update_match(state); - const uint32_t len_to_pos_state = get_len_to_pos_state(len); - uint32_t pos_slot = 0; - bittree_decode(pos_slot, - coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS); - assert(pos_slot <= 63); - - if (pos_slot >= START_POS_MODEL_INDEX) { - uint32_t direct_bits = (pos_slot >> 1) - 1; - assert(direct_bits >= 1 && direct_bits <= 30); - uint32_t distance = 2 | (pos_slot & 1); - - if (pos_slot < END_POS_MODEL_INDEX) { - assert(direct_bits <= 5); - distance <<= direct_bits; - assert(distance <= 96); - // -1 is fine, because - // bittree_reverse_decode() - // starts from table index [1] - // (not [0]). - assert((int32_t)(distance - pos_slot - 1) >= -1); - assert((int32_t)(distance - pos_slot - 1) <= 82); - // We add the result to distance, so distance - // must not be part of second argument - // of the macro. - const int32_t offset = distance - pos_slot - 1; - bittree_reverse_decode(distance, coder->pos_decoders + offset, - direct_bits); + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + + // Decode the length of the match. + len_decode(len, coder->match_len_decoder, + pos_state, SEQ_MATCH_LEN); + + // Prepare to decode the highest two bits of the + // match distance. + probs = coder->pos_slot[get_len_to_pos_state(len)]; + symbol = 1; + +#ifdef HAVE_SMALL + case SEQ_POS_SLOT: + do { + rc_bit(probs[symbol], , , SEQ_POS_SLOT); + } while (symbol < POS_SLOTS); +#else + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT5); +#endif + // Get rid of the highest bit that was needed for + // indexing of the probability array. + symbol -= POS_SLOTS; + assert(symbol <= 63); + + if (symbol < START_POS_MODEL_INDEX) { + // Match distances [0, 3] have only two bits. + rep0 = symbol; + } else { + // Decode the lowest [1, 29] bits of + // the match distance. + limit = (symbol >> 1) - 1; + assert(limit >= 1 && limit <= 30); + rep0 = 2 + (symbol & 1); + + if (symbol < END_POS_MODEL_INDEX) { + // Prepare to decode the low bits for + // a distance of [4, 127]. + assert(limit <= 5); + rep0 <<= limit; + assert(rep0 <= 96); + // -1 is fine, because we start + // decoding at probs[1], not probs[0]. + // NOTE: This violates the C standard, + // since we are doing pointer + // arithmetic past the beginning of + // the array. + assert((int32_t)(rep0 - symbol - 1) + >= -1); + assert((int32_t)(rep0 - symbol - 1) + <= 82); + probs = coder->pos_special + rep0 + - symbol - 1; + symbol = 1; + offset = 0; + case SEQ_POS_MODEL: +#ifdef HAVE_SMALL + do { + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + } while (++offset < limit); +#else + switch (limit) { + case 5: + assert(offset == 0); + rc_bit(probs[symbol], , + rep0 += 1, + SEQ_POS_MODEL); + ++offset; + --limit; + case 4: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 3: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 2: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 1: + // We need "symbol" only for + // indexing the probability + // array, thus we can use + // rc_bit_last() here to omit + // the unneeded updating of + // "symbol". + rc_bit_last(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + } +#endif } else { - assert(pos_slot >= 14); - assert(direct_bits >= 6); - direct_bits -= ALIGN_BITS; - assert(direct_bits >= 2); - rc_decode_direct(distance, direct_bits); - distance <<= ALIGN_BITS; - - bittree_reverse_decode(distance, coder->pos_align_decoder, - ALIGN_BITS); - - if (distance == UINT32_MAX) { - if (len == LEN_SPECIAL_EOPM) { - // End of Payload Marker found. - coder->lz.eopm_detected = true; - break; - - } else if (len == LEN_SPECIAL_FLUSH) { - // Flush marker detected. We must have produced - // at least one byte of output since the previous - // flush marker or the beginning of the stream. - // This is to prevent hanging the decoder with - // malicious input files. - if (!has_produced_output) - return true; - - has_produced_output = false; - - // We know that we have enough input to call - // this macro, because it is tested at the - // end of decode_dummy(). - rc_normalize(); - - rc_reset(rc); - - // If we don't have enough input here, we jump - // out of the loop. Note that while there is a - // useless call to rc_normalize(), it does nothing - // since we have just reset the range decoder. - if (!rc_read_init(&rc, in, &in_pos_local, in_size)) - break; - - continue; - - } else { - return true; + // The distace is >= 128. Decode the + // lower bits without probabilities + // except the lowest four bits. + assert(symbol >= 14); + assert(limit >= 6); + limit -= ALIGN_BITS; + assert(limit >= 2); + case SEQ_DIRECT: + // Not worth manual unrolling + do { + rc_direct(rep0, SEQ_DIRECT); + } while (--limit > 0); + + // Decode the lowest four bits using + // probabilities. + rep0 <<= ALIGN_BITS; + symbol = 1; +#ifdef HAVE_SMALL + offset = 0; + case SEQ_ALIGN: + do { + rc_bit(coder->pos_align[ + symbol], , + rep0 += 1 << offset, + SEQ_ALIGN); + } while (++offset < ALIGN_BITS); +#else + case SEQ_ALIGN0: + rc_bit(coder->pos_align[symbol], , + rep0 += 1, SEQ_ALIGN0); + case SEQ_ALIGN1: + rc_bit(coder->pos_align[symbol], , + rep0 += 2, SEQ_ALIGN1); + case SEQ_ALIGN2: + rc_bit(coder->pos_align[symbol], , + rep0 += 4, SEQ_ALIGN2); + case SEQ_ALIGN3: + // Like in SEQ_POS_MODEL, we don't + // need "symbol" for anything else + // than indexing the probability array. + rc_bit_last(coder->pos_align[symbol], , + rep0 += 8, SEQ_ALIGN3); +#endif + + if (rep0 == UINT32_MAX) { + // End of payload marker was + // found. It must not be + // present if uncompressed + // size is known. + if (coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + ret = LZMA_DATA_ERROR; + goto out; } + + case SEQ_EOPM: + // TODO Comment + rc_normalize(SEQ_EOPM); + ret = LZMA_STREAM_END; + goto out; } } + } - // The latest three match distances are kept in - // memory in case there are repeated matches. - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = distance; - - } else { - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = pos_slot; + // Validate the distance we just decoded. + if (unlikely(!dict_is_distance_valid(&dict, rep0))) { + ret = LZMA_DATA_ERROR; + goto out; } } else { - update_bit_1(coder->is_rep[state]); + rc_update_1(coder->is_rep[state]); // Repeated match // @@ -532,242 +686,318 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // earlier. The latest four match distances are // available as rep0, rep1, rep2 and rep3. We will // now decode which of them is the new distance. + // + // There cannot be a match if we haven't produced + // any output, so check that first. + if (unlikely(!dict_is_distance_valid(&dict, 0))) { + ret = LZMA_DATA_ERROR; + goto out; + } - if_bit_0(coder->is_rep0[state]) { - update_bit_0(coder->is_rep0[state]); - + case SEQ_IS_REP0: + rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) { + rc_update_0(coder->is_rep0[state]); // The distance is rep0. - if_bit_0(coder->is_rep0_long[state][pos_state]) { - update_bit_0(coder->is_rep0_long[state][pos_state]); + case SEQ_IS_REP0_LONG: + rc_if_0(coder->is_rep0_long[state][pos_state], + SEQ_IS_REP0_LONG) { + rc_update_0(coder->is_rep0_long[ + state][pos_state]); update_short_rep(state); - // Repeat exactly one byte and start a new decoding loop. - // Note that rep0 is known to have a safe value, thus we - // don't need to check if we are wrapping the dictionary - // when it isn't full yet. - if (unlikely(lz_is_empty(coder->lz))) - return true; - - coder->lz.dict[coder->lz.pos] - = lz_get_byte(coder->lz, rep0); - ++coder->lz.pos; - ++now_pos; - has_produced_output = true; - continue; - - } else { - update_bit_1(coder->is_rep0_long[state][pos_state]); + case SEQ_SHORTREP: + if (unlikely(dict_put(&dict, dict_get( + &dict, rep0)))) { + coder->sequence = SEQ_SHORTREP; + goto out; + } - // Repeating more than one byte at - // distance of rep0. + continue; } + // Repeating more than one byte at + // distance of rep0. + rc_update_1(coder->is_rep0_long[ + state][pos_state]); + } else { - update_bit_1(coder->is_rep0[state]); + rc_update_1(coder->is_rep0[state]); + case SEQ_IS_REP1: // The distance is rep1, rep2 or rep3. Once // we find out which one of these three, it // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. + rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { + rc_update_0(coder->is_rep1[state]); - uint32_t distance; + const uint32_t distance = rep1; + rep1 = rep0; + rep0 = distance; - if_bit_0(coder->is_rep1[state]) { - update_bit_0(coder->is_rep1[state]); - distance = rep1; } else { - update_bit_1(coder->is_rep1[state]); + rc_update_1(coder->is_rep1[state]); + case SEQ_IS_REP2: + rc_if_0(coder->is_rep2[state], + SEQ_IS_REP2) { + rc_update_0(coder->is_rep2[ + state]); + + const uint32_t distance = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; - if_bit_0(coder->is_rep2[state]) { - update_bit_0(coder->is_rep2[state]); - distance = rep2; } else { - update_bit_1(coder->is_rep2[state]); - distance = rep3; + rc_update_1(coder->is_rep2[ + state]); + + const uint32_t distance = rep3; rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; } - - rep2 = rep1; } - - rep1 = rep0; - rep0 = distance; } update_long_rep(state); // Decode the length of the repeated match. - length_decode(len, coder->rep_len_decoder, pos_state); + len_decode(len, coder->rep_len_decoder, + pos_state, SEQ_REP_LEN); } - ///////////////////////////////// // Repeat from history buffer. // ///////////////////////////////// // The length is always between these limits. There is no way // to trigger the algorithm to set len outside this range. - assert(len >= MATCH_MIN_LEN); - assert(len <= MATCH_MAX_LEN); - - now_pos += len; - has_produced_output = true; + assert(len >= MATCH_LEN_MIN); + assert(len <= MATCH_LEN_MAX); + case SEQ_COPY: // Repeat len bytes from distance of rep0. - if (!lzma_lz_out_repeat(&coder->lz, rep0, len)) - return true; + if (unlikely(dict_repeat(&dict, rep0, &len))) { + coder->sequence = SEQ_COPY; + goto out; + } } - rc_normalize(); + rc_normalize(SEQ_NORMALIZE); + coder->sequence = SEQ_IS_MATCH; +out: + // Save state - ///////////////////////////////// - // Update the *data structure. // - ///////////////////////////////// + // NOTE: Must not copy dict.limit. + dictptr->pos = dict.pos; + dictptr->full = dict.full; - // Range decoder - rc_from_local(coder->rc); + rc_from_local(coder->rc, *in_pos); - // State coder->state = state; coder->rep0 = rep0; coder->rep1 = rep1; coder->rep2 = rep2; coder->rep3 = rep3; - // Misc - coder->now_pos = now_pos; - coder->has_produced_output = has_produced_output; - *in_pos = in_pos_local; + coder->probs = probs; + coder->symbol = symbol; + coder->limit = limit; + coder->offset = offset; + coder->len = len; + + // Update the remaining amount of uncompressed data if uncompressed + // size was known. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + coder->uncompressed_size -= dict.pos - dict_start; + + // Since there cannot be end of payload marker if the + // uncompressed size was known, we check here if we + // finished decoding. + if (coder->uncompressed_size == 0 && ret == LZMA_OK + && coder->sequence != SEQ_NORMALIZE) + ret = coder->sequence == SEQ_IS_MATCH + ? LZMA_STREAM_END : LZMA_DATA_ERROR; + } + + // We can do an additional check in the range decoder to catch some + // corrupted files. + if (ret == LZMA_STREAM_END) { + if (!rc_is_finished(coder->rc)) + ret = LZMA_DATA_ERROR; - return false; + // Reset the range decoder so that it is ready to reinitialize + // for a new LZMA2 chunk. + rc_reset(coder->rc); + } + + return ret; } + static void -lzma_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) { - lzma_next_coder_end(&coder->next, allocator); - lzma_lz_decoder_end(&coder->lz, allocator); - lzma_free(coder, allocator); - return; + coder->uncompressed_size = uncompressed_size; } - -extern lzma_ret -lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter_info *filters) +/* +extern void +lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { - // LZMA can only be the last filter in the chain. - assert(filters[1].init == NULL); - - // Validate pos_bits. Other options are validated by the - // respective initialization functions. - const lzma_options_lzma *options = filters[0].options; - if (options->pos_bits > LZMA_POS_BITS_MAX) - return LZMA_HEADER_ERROR; + // This is hack. + (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; +} +*/ - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) - return LZMA_MEM_ERROR; +static void +lzma_decoder_reset(lzma_coder *coder, const void *opt) +{ + const lzma_options_lzma *options = opt; - next->code = &lzma_lz_decode; - next->end = &lzma_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->lz = LZMA_LZ_DECODER_INIT; - } + // NOTE: We assume that lc/lp/pb are valid since they were + // successfully decoded with lzma_lzma_decode_properties(). + // FIXME? - // Store the pos_bits and calculate pos_mask. - next->coder->pos_bits = options->pos_bits; - next->coder->pos_mask = (1U << next->coder->pos_bits) - 1; + // Calculate pos_mask. We don't need pos_bits as is for anything. + coder->pos_mask = (1U << options->pos_bits) - 1; // Initialize the literal decoder. - return_if_error(lzma_literal_init(&next->coder->literal_coder, - options->literal_context_bits, - options->literal_pos_bits)); + literal_init(coder->literal, options->literal_context_bits, + options->literal_pos_bits); - // Allocate and initialize the LZ decoder. - return_if_error(lzma_lz_decoder_reset(&next->coder->lz, allocator, - &decode_real, options->dictionary_size, - MATCH_MAX_LEN)); + coder->literal_context_bits = options->literal_context_bits; + coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1; // State - next->coder->state = 0; - next->coder->rep0 = 0; - next->coder->rep1 = 0; - next->coder->rep2 = 0; - next->coder->rep3 = 0; - next->coder->pos_bits = options->pos_bits; - next->coder->pos_mask = (1 << next->coder->pos_bits) - 1; - next->coder->now_pos = 0; + coder->state = STATE_LIT_LIT; + coder->rep0 = 0; + coder->rep1 = 0; + coder->rep2 = 0; + coder->rep3 = 0; + coder->pos_mask = (1 << options->pos_bits) - 1; // Range decoder - rc_reset(next->coder->rc); + rc_reset(coder->rc); // Bit and bittree decoders for (uint32_t i = 0; i < STATES; ++i) { - for (uint32_t j = 0; j <= next->coder->pos_mask; ++j) { - bit_reset(next->coder->is_match[i][j]); - bit_reset(next->coder->is_rep0_long[i][j]); + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { + bit_reset(coder->is_match[i][j]); + bit_reset(coder->is_rep0_long[i][j]); } - bit_reset(next->coder->is_rep[i]); - bit_reset(next->coder->is_rep0[i]); - bit_reset(next->coder->is_rep1[i]); - bit_reset(next->coder->is_rep2[i]); + bit_reset(coder->is_rep[i]); + bit_reset(coder->is_rep0[i]); + bit_reset(coder->is_rep1[i]); + bit_reset(coder->is_rep2[i]); } for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(next->coder->pos_slot_decoder[i], POS_SLOT_BITS); + bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(next->coder->pos_decoders[i]); + bit_reset(coder->pos_special[i]); - bittree_reset(next->coder->pos_align_decoder, ALIGN_BITS); + bittree_reset(coder->pos_align, ALIGN_BITS); // Len decoders (also bit/bittree) - const uint32_t num_pos_states = 1 << next->coder->pos_bits; - bit_reset(next->coder->match_len_decoder.choice); - bit_reset(next->coder->match_len_decoder.choice2); - bit_reset(next->coder->rep_len_decoder.choice); - bit_reset(next->coder->rep_len_decoder.choice2); + const uint32_t num_pos_states = 1 << options->pos_bits; + bit_reset(coder->match_len_decoder.choice); + bit_reset(coder->match_len_decoder.choice2); + bit_reset(coder->rep_len_decoder.choice); + bit_reset(coder->rep_len_decoder.choice2); for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { - bittree_reset(next->coder->match_len_decoder.low[pos_state], + bittree_reset(coder->match_len_decoder.low[pos_state], LEN_LOW_BITS); - bittree_reset(next->coder->match_len_decoder.mid[pos_state], + bittree_reset(coder->match_len_decoder.mid[pos_state], LEN_MID_BITS); - bittree_reset(next->coder->rep_len_decoder.low[pos_state], + bittree_reset(coder->rep_len_decoder.low[pos_state], LEN_LOW_BITS); - bittree_reset(next->coder->rep_len_decoder.mid[pos_state], + bittree_reset(coder->rep_len_decoder.mid[pos_state], LEN_MID_BITS); } - bittree_reset(next->coder->match_len_decoder.high, LEN_HIGH_BITS); - bittree_reset(next->coder->rep_len_decoder.high, LEN_HIGH_BITS); + bittree_reset(coder->match_len_decoder.high, LEN_HIGH_BITS); + bittree_reset(coder->rep_len_decoder.high, LEN_HIGH_BITS); + + coder->sequence = SEQ_IS_MATCH; + coder->probs = NULL; + coder->symbol = 0; + coder->limit = 0; + coder->offset = 0; + coder->len = 0; + + return; +} + + +extern lzma_ret +lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *opt, size_t *dict_size) +{ + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma_decode; + lz->reset = &lzma_decoder_reset; + lz->set_uncompressed = &lzma_decoder_uncompressed; + } - next->coder->has_produced_output = false; + // All dictionary sizes are OK here. LZ decoder will take care of + // the special cases. + const lzma_options_lzma *options = opt; + *dict_size = options->dictionary_size; return LZMA_OK; } -extern void -lzma_lzma_decoder_uncompressed_size( - lzma_next_coder *next, lzma_vli uncompressed_size) +/// Allocate and initialize LZMA decoder. This is used only via LZ +/// initialization (lzma_lzma_decoder_init() passes function pointer to +/// the LZ initialization). +static lzma_ret +lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *options, size_t *dict_size) { - next->coder->lz.uncompressed_size = uncompressed_size; - return; + if (!is_lclppb_valid(options)) + return LZMA_PROG_ERROR; + + return_if_error(lzma_lzma_decoder_create( + lz, allocator, options, dict_size)); + + lzma_decoder_reset(lz->coder, options); + lzma_decoder_uncompressed(lz->coder, LZMA_VLI_VALUE_UNKNOWN); + + return LZMA_OK; +} + + +extern lzma_ret +lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // LZMA can only be the last filter in the chain. This is enforced + // by the raw_decoder initialization. + assert(filters[1].init == NULL); + + return lzma_lz_decoder_init(next, allocator, filters, + &lzma_decoder_init); } extern bool -lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte) +lzma_lzma_lclppb_decode(lzma_options_lzma *options, uint8_t byte) { if (byte > (4 * 5 + 4) * 9 + 8) return true; @@ -781,3 +1011,49 @@ lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte) return options->literal_context_bits + options->literal_pos_bits > LZMA_LITERAL_BITS_MAX; } + + +extern uint64_t +lzma_lzma_decoder_memusage(const void *options) +{ + const lzma_options_lzma *const opt = options; + const uint64_t lz_memusage + = lzma_lz_decoder_memusage(opt->dictionary_size); + if (lz_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lz_memusage; +} + + +extern lzma_ret +lzma_lzma_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 5) + return LZMA_HEADER_ERROR; + + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + if (lzma_lzma_lclppb_decode(opt, props[0])) + goto error; + + // All dictionary sizes are accepted, including zero. LZ decoder + // will automatically use a dictionary at least a few KiB even if + // a smaller dictionary is requested. + opt->dictionary_size = integer_read_32(props + 1); + + opt->preset_dictionary = NULL; + opt->preset_dictionary_size = 0; + + *options = opt; + + return LZMA_OK; + +error: + lzma_free(opt, allocator); + return LZMA_HEADER_ERROR; +} diff --git a/src/liblzma/lzma/lzma_decoder.h b/src/liblzma/lzma/lzma_decoder.h index 9d57c7e5..3792f452 100644 --- a/src/liblzma/lzma/lzma_decoder.h +++ b/src/liblzma/lzma/lzma_decoder.h @@ -28,16 +28,27 @@ extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -/// Set known uncompressed size. This is a hack needed to support -/// LZMA_Alone files that don't have EOPM. -extern void lzma_lzma_decoder_uncompressed_size( - lzma_next_coder *next, lzma_vli uncompressed_size); +extern uint64_t lzma_lzma_decoder_memusage(const void *options); + +extern lzma_ret lzma_lzma_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + /// \brief Decodes the LZMA Properties byte (lc/lp/pb) /// /// \return true if error occorred, false on success /// -extern bool lzma_lzma_decode_properties( +extern bool lzma_lzma_lclppb_decode( lzma_options_lzma *options, uint8_t byte); + +#ifdef LZMA_LZ_DECODER_H +/// Allocate and setup function pointers only. This is used by LZMA1 and +/// LZMA2 decoders. +extern lzma_ret lzma_lzma_decoder_create( + lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *opt, size_t *dict_size); +#endif + #endif diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c index afb1d5ed..a84801e7 100644 --- a/src/liblzma/lzma/lzma_encoder.c +++ b/src/liblzma/lzma/lzma_encoder.c @@ -30,40 +30,33 @@ static inline void literal_matched(lzma_range_encoder *rc, probability *subcoder, uint32_t match_byte, uint32_t symbol) { - uint32_t context = 1; - uint32_t bit_count = 8; + uint32_t offset = 0x100; + symbol += UINT32_C(1) << 8; do { - uint32_t bit = (symbol >> --bit_count) & 1; - const uint32_t match_bit = (match_byte >> bit_count) & 1; - rc_bit(rc, &subcoder[(0x100 << match_bit) + context], bit); - context = (context << 1) | bit; - - if (match_bit != bit) { - // The bit from the literal being encoded and the bit - // from the previous match differ. Finish encoding - // as a normal literal. - while (bit_count != 0) { - bit = (symbol >> --bit_count) & 1; - rc_bit(rc, &subcoder[context], bit); - context = (context << 1) | bit; - } + match_byte <<= 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; + rc_bit(rc, &subcoder[subcoder_index], bit); - break; - } + symbol <<= 1; + offset &= ~(match_byte ^ symbol); - } while (bit_count != 0); + } while (symbol < (UINT32_C(1) << 16)); } static inline void -literal(lzma_coder *coder) +literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. - const uint8_t cur_byte = coder->lz.buffer[ - coder->lz.read_pos - coder->additional_offset]; - probability *subcoder = literal_get_subcoder(coder->literal_coder, - coder->now_pos, coder->previous_byte); + const uint8_t cur_byte = mf->buffer[ + mf->read_pos - mf->read_ahead]; + probability *subcoder = literal_subcoder(coder->literal, + coder->literal_context_bits, coder->literal_pos_mask, + position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); if (is_literal_state(coder->state)) { // Previous LZMA-symbol was a literal. Encode a normal @@ -73,14 +66,13 @@ literal(lzma_coder *coder) // Previous LZMA-symbol was a match. Use the last byte of // the match as a "match byte". That is, compare the bits // of the current literal and the match byte. - const uint8_t match_byte = coder->lz.buffer[ - coder->lz.read_pos - coder->reps[0] - 1 - - coder->additional_offset]; + const uint8_t match_byte = mf->buffer[ + mf->read_pos - coder->reps[0] - 1 + - mf->read_ahead]; literal_matched(&coder->rc, subcoder, match_byte, cur_byte); } update_literal(coder->state); - coder->previous_byte = cur_byte; } @@ -88,12 +80,41 @@ literal(lzma_coder *coder) // Match length // ////////////////// +static void +length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) +{ + const uint32_t table_size = lc->table_size; + lc->counters[pos_state] = table_size; + + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + + uint32_t i; + for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) + prices[i] = a0 + rc_bittree_price(lc->low[pos_state], + LEN_LOW_BITS, i); + + for (; i < table_size && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) + prices[i] = b0 + rc_bittree_price(lc->mid[pos_state], + LEN_MID_BITS, i - LEN_LOW_SYMBOLS); + + for (; i < table_size; ++i) + prices[i] = b1 + rc_bittree_price(lc->high, LEN_HIGH_BITS, + i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); + + return; +} + + static inline void length(lzma_range_encoder *rc, lzma_length_encoder *lc, - const uint32_t pos_state, uint32_t len) + const uint32_t pos_state, uint32_t len, const bool fast_mode) { - assert(len <= MATCH_MAX_LEN); - len -= MATCH_MIN_LEN; + assert(len <= MATCH_LEN_MAX); + len -= MATCH_LEN_MIN; if (len < LEN_LOW_SYMBOLS) { rc_bit(rc, &lc->choice, 0); @@ -111,6 +132,12 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, rc_bittree(rc, lc->high, LEN_HIGH_BITS, len); } } + + // Only getoptimum uses the prices so don't update the table when + // in fast mode. + if (!fast_mode) + if (--lc->counters[pos_state] == 0) + length_update_prices(lc, pos_state); } @@ -124,12 +151,12 @@ match(lzma_coder *coder, const uint32_t pos_state, { update_match(coder->state); - length(&coder->rc, &coder->match_len_encoder, pos_state, len); - coder->prev_len_encoder = &coder->match_len_encoder; + length(&coder->rc, &coder->match_len_encoder, pos_state, len, + coder->fast_mode); const uint32_t pos_slot = get_pos_slot(distance); const uint32_t len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot_encoder[len_to_pos_state], + rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], POS_SLOT_BITS, pos_slot); if (pos_slot >= START_POS_MODEL_INDEX) { @@ -139,13 +166,13 @@ match(lzma_coder *coder, const uint32_t pos_state, if (pos_slot < END_POS_MODEL_INDEX) { rc_bittree_reverse(&coder->rc, - &coder->pos_encoders[base - pos_slot - 1], + &coder->pos_special[base - pos_slot - 1], footer_bits, pos_reduced); } else { rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align_encoder, + &coder->rc, coder->pos_align, ALIGN_BITS, pos_reduced & ALIGN_MASK); ++coder->align_price_count; } @@ -196,8 +223,8 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, if (len == 1) { update_short_rep(coder->state); } else { - length(&coder->rc, &coder->rep_len_encoder, pos_state, len); - coder->prev_len_encoder = &coder->rep_len_encoder; + length(&coder->rc, &coder->rep_len_encoder, pos_state, len, + coder->fast_mode); update_long_rep(coder->state); } } @@ -208,117 +235,123 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, uint32_t pos, uint32_t len) +encode_symbol(lzma_coder *coder, lzma_mf *mf, + uint32_t back, uint32_t len, uint32_t position) { - const uint32_t pos_state = coder->now_pos & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - if (len == 1 && pos == UINT32_MAX) { + if (back == UINT32_MAX) { // Literal i.e. eight-bit byte + assert(len == 1); rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 0); - literal(coder); + literal(coder, mf, position); } else { // Some type of match rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (pos < REP_DISTANCES) { + if (back < REP_DISTANCES) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); - rep_match(coder, pos_state, pos, len); + rep_match(coder, pos_state, back, len); } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, pos - REP_DISTANCES, len); + match(coder, pos_state, back - REP_DISTANCES, len); } + } + + assert(mf->read_ahead >= len); + mf->read_ahead -= len; +} + + +static bool +encode_init(lzma_coder *coder, lzma_mf *mf) +{ + if (mf->read_pos == mf->read_limit) { + if (mf->action == LZMA_RUN) + return false; // We cannot do anything. - coder->previous_byte = coder->lz.buffer[ - coder->lz.read_pos + len - 1 - - coder->additional_offset]; + // We are finishing (we cannot get here when flushing). + assert(mf->write_pos == mf->read_pos); + assert(mf->action == LZMA_FINISH); + } else { + // Do the actual initialization. The first LZMA symbol must + // always be a literal. + mf_skip(mf, 1); + mf->read_ahead = 0; + rc_bit(&coder->rc, &coder->is_match[0][0], 0); + rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]); } - assert(coder->additional_offset >= len); - coder->additional_offset -= len; - coder->now_pos += len; + // Initialization is done (except if empty file). + coder->is_initialized = true; + + return true; } static void -encode_eopm(lzma_coder *coder) +encode_eopm(lzma_coder *coder, uint32_t position) { - const uint32_t pos_state = coder->now_pos & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, UINT32_MAX, MATCH_MIN_LEN); + match(coder, pos_state, UINT32_MAX, MATCH_LEN_MIN); } -/** - * \brief LZMA encoder - * - * \return true if end of stream was reached, false otherwise. - */ -extern bool -lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size) +/// Number of bytes that a single encoding loop in lzma_lzma_encode() can +/// consume from the dictionary. This limit comes from lzma_lzma_optimum() +/// and may need to be updated if that function is significantly modified. +#define LOOP_INPUT_MAX (OPTS + 1) + + +extern lzma_ret +lzma_lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, uint32_t limit) { // Initialize the stream if no data has been encoded yet. - if (!coder->is_initialized) { - if (coder->lz.read_pos == coder->lz.read_limit) { - if (coder->lz.sequence == SEQ_RUN) - return false; // We cannot do anything. - - // We are finishing (we cannot get here when flushing). - assert(coder->lz.write_pos == coder->lz.read_pos); - assert(coder->lz.sequence == SEQ_FINISH); - } else { - // Do the actual initialization. - uint32_t len; - uint32_t num_distance_pairs; - lzma_read_match_distances(coder, - &len, &num_distance_pairs); + if (!coder->is_initialized && !encode_init(coder, mf)) + return LZMA_OK; - encode_symbol(coder, UINT32_MAX, 1); + // Get the lowest bits of the uncompressed offset from the LZ layer. + uint32_t position = mf_position(mf); - assert(coder->additional_offset == 0); + while (true) { + // Encode pending bits, if any. Calling this before encoding + // the next symbol is needed only with plain LZMA, since + // LZMA2 always provides big enough buffer to flush + // everything out from the range encoder. For the same reason, + // rc_encode() never returns true when this function is used + // as part of LZMA2 encoder. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + assert(limit == UINT32_MAX); + return LZMA_OK; } - // Initialization is done (except if empty file). - coder->is_initialized = true; - } - - // Encoding loop - while (true) { - // Encode pending bits, if any. - if (rc_encode(&coder->rc, out, out_pos, out_size)) - return false; + // With LZMA2 we need to take care that compressed size of + // a chunk doesn't get too big. + // TODO + if (limit != UINT32_MAX + && (mf->read_pos - mf->read_ahead >= limit + || *out_pos + rc_pending(&coder->rc) + >= (UINT32_C(1) << 16) + - LOOP_INPUT_MAX)) + break; // Check that there is some input to process. - if (coder->lz.read_pos >= coder->lz.read_limit) { - // If flushing or finishing, we must keep encoding - // until additional_offset becomes zero to make - // all the input available at output. - if (coder->lz.sequence == SEQ_RUN) - return false; - - if (coder->additional_offset == 0) - break; - } - - assert(coder->lz.read_pos <= coder->lz.write_pos); + if (mf->read_pos >= mf->read_limit) { + if (mf->action == LZMA_RUN) + return LZMA_OK; -#ifndef NDEBUG - if (coder->lz.sequence != SEQ_RUN) { - assert(coder->lz.read_limit == coder->lz.write_pos); - } else { - assert(coder->lz.read_limit + coder->lz.keep_size_after - == coder->lz.write_pos); + if (mf->read_ahead == 0) + break; } -#endif - - uint32_t pos; - uint32_t len; // Get optimal match (repeat position and length). // Value ranges for pos: @@ -327,33 +360,324 @@ lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, // match at (pos - REP_DISTANCES) // - UINT32_MAX: not a match but a literal // Value ranges for len: - // - [MATCH_MIN_LEN, MATCH_MAX_LEN] - if (coder->best_compression) - lzma_get_optimum(coder, &pos, &len); + // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; + + if (coder->fast_mode) + lzma_lzma_optimum_fast(coder, mf, &back, &len); else - lzma_get_optimum_fast(coder, &pos, &len); + lzma_lzma_optimum_normal( + coder, mf, &back, &len, position); + + encode_symbol(coder, mf, back, len, position); + + position += len; + } + + if (!coder->is_flushed) { + coder->is_flushed = true; - encode_symbol(coder, pos, len); + // We don't support encoding plain LZMA streams without EOPM, + // and LZMA2 doesn't use EOPM at LZMA level. + if (limit == UINT32_MAX) + encode_eopm(coder, position); + + // Flush the remaining bytes from the range encoder. + rc_flush(&coder->rc); + + // Copy the remaining bytes to the output buffer. If there + // isn't enough output space, we will copy out the remaining + // bytes on the next call to this function by using + // the rc_encode() call in the encoding loop above. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + assert(limit == UINT32_MAX); + return LZMA_OK; + } } - assert(!coder->longest_match_was_found); + // Make it ready for the next LZMA2 chunk. + coder->is_flushed = false; + + return LZMA_STREAM_END; +} + + +static lzma_ret +lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + // Plain LZMA has no support for sync-flushing. + if (unlikely(mf->action == LZMA_SYNC_FLUSH)) + return LZMA_HEADER_ERROR; + + return lzma_lzma_encode(coder, mf, out, out_pos, out_size, UINT32_MAX); +} + - if (coder->is_flushed) { - coder->is_flushed = false; +//////////////////// +// Initialization // +//////////////////// + +static bool +set_lz_options(lzma_lz_options *lz_options, const lzma_options_lzma *options) +{ + if (!is_lclppb_valid(options) + || options->fast_bytes < LZMA_FAST_BYTES_MIN + || options->fast_bytes > LZMA_FAST_BYTES_MAX) return true; + + // FIXME validation + + lz_options->before_size = OPTS; + lz_options->dictionary_size = options->dictionary_size; + lz_options->after_size = LOOP_INPUT_MAX; + lz_options->match_len_max = MATCH_LEN_MAX; + lz_options->find_len_max = options->fast_bytes; + lz_options->match_finder = options->match_finder; + lz_options->match_finder_cycles = options->match_finder_cycles; + lz_options->preset_dictionary = options->preset_dictionary; + lz_options->preset_dictionary_size = options->preset_dictionary_size; + + return false; +} + + +static void +length_encoder_reset(lzma_length_encoder *lencoder, + const uint32_t num_pos_states, const bool fast_mode) +{ + bit_reset(lencoder->choice); + bit_reset(lencoder->choice2); + + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { + bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); + bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); } - // We don't support encoding old LZMA streams without EOPM, and LZMA2 - // doesn't use EOPM at LZMA level. - if (coder->write_eopm) - encode_eopm(coder); + bittree_reset(lencoder->high, LEN_HIGH_BITS); - rc_flush(&coder->rc); + if (!fast_mode) + for (size_t pos_state = 0; pos_state < num_pos_states; + ++pos_state) + length_update_prices(lencoder, pos_state); - if (rc_encode(&coder->rc, out, out_pos, out_size)) { - coder->is_flushed = true; - return false; + return; +} + + +extern void +lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +{ + assert(!coder->is_flushed); + + coder->pos_mask = (1U << options->pos_bits) - 1; + coder->literal_context_bits = options->literal_context_bits; + coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1; + + + // Range coder + rc_reset(&coder->rc); + + // State + coder->state = 0; + for (size_t i = 0; i < REP_DISTANCES; ++i) + coder->reps[i] = 0; + + literal_init(coder->literal, options->literal_context_bits, + options->literal_pos_bits); + + // Bit encoders + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { + bit_reset(coder->is_match[i][j]); + bit_reset(coder->is_rep0_long[i][j]); + } + + bit_reset(coder->is_rep[i]); + bit_reset(coder->is_rep0[i]); + bit_reset(coder->is_rep1[i]); + bit_reset(coder->is_rep2[i]); } - return true; + for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + bit_reset(coder->pos_special[i]); + + // Bit tree encoders + for (size_t i = 0; i < LEN_TO_POS_STATES; ++i) + bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + + bittree_reset(coder->pos_align, ALIGN_BITS); + + // Length encoders + length_encoder_reset(&coder->match_len_encoder, + 1U << options->pos_bits, coder->fast_mode); + + length_encoder_reset(&coder->rep_len_encoder, + 1U << options->pos_bits, coder->fast_mode); + + // FIXME: Too big or too small won't work when resetting in the middle of LZMA2. + coder->match_price_count = UINT32_MAX / 2; + coder->align_price_count = UINT32_MAX / 2; + + coder->opts_end_index = 0; + coder->opts_current_index = 0; +} + + +extern lzma_ret +lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, + const lzma_options_lzma *options, lzma_lz_options *lz_options) +{ + if (*coder_ptr == NULL) { + *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + if (*coder_ptr == NULL) + return LZMA_MEM_ERROR; + } + + lzma_coder *coder = *coder_ptr; + + // Validate options that aren't validated elsewhere. + if (!is_lclppb_valid(options) + || options->fast_bytes < LZMA_FAST_BYTES_MIN + || options->fast_bytes > LZMA_FAST_BYTES_MAX) + return LZMA_HEADER_ERROR; + + // Set compression mode. + switch (options->mode) { + case LZMA_MODE_FAST: + coder->fast_mode = true; + break; + + case LZMA_MODE_NORMAL: { + coder->fast_mode = false; + + // Set dist_table_size. + // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; + while ((UINT32_C(1) << log_size) + < options->dictionary_size) + ++log_size; + + coder->dist_table_size = log_size * 2; + + // Length encoders' price table size + coder->match_len_encoder.table_size + = options->fast_bytes + 1 - MATCH_LEN_MIN; + coder->rep_len_encoder.table_size + = options->fast_bytes + 1 - MATCH_LEN_MIN; + break; + } + + default: + return LZMA_HEADER_ERROR; + } + + coder->is_initialized = false; + coder->is_flushed = false; + + lzma_lzma_encoder_reset(coder, options); + + // LZ encoder options FIXME validation + if (set_lz_options(lz_options, options)) + return LZMA_HEADER_ERROR; + + return LZMA_OK; +} + + +static lzma_ret +lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, + const void *options, lzma_lz_options *lz_options) +{ + lz->code = &lzma_encode; + return lzma_lzma_encoder_create( + &lz->coder, allocator, options, lz_options); +} + + +extern lzma_ret +lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // Initialization call chain: + // + // lzma_lzma_encoder_init() + // `-- lzma_lz_encoder_init() + // `-- lzma_encoder_init() + // `-- lzma_encoder_init2() + // + // The above complexity is to let LZ encoder store the pointer to + // the LZMA encoder structure. Encoding call tree: + // + // lz_encode() + // |-- fill_window() + // | `-- Next coder in the chain, if any + // `-- lzma_encode() + // |-- lzma_dict_find() + // `-- lzma_dict_skip() + // + // FIXME ^ + // + return lzma_lz_encoder_init( + next, allocator, filters, &lzma_encoder_init); +} + + +extern uint64_t +lzma_lzma_encoder_memusage(const void *options) +{ + lzma_lz_options lz_options; + if (set_lz_options(&lz_options, options)) + return UINT64_MAX; + + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); + if (lz_memusage == UINT64_MAX) + return UINT64_MAX; + + return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; +} + + +extern bool +lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte) +{ + if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX + || options->literal_pos_bits + > LZMA_LITERAL_POS_BITS_MAX + || options->pos_bits > LZMA_POS_BITS_MAX + || options->literal_context_bits + + options->literal_pos_bits + > LZMA_LITERAL_BITS_MAX) + return true; + + *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9 + + options->literal_context_bits; + assert(*byte <= (4 * 5 + 4) * 9 + 8); + + return false; +} + + +#ifdef HAVE_ENCODER_LZMA +extern lzma_ret +lzma_lzma_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_lzma *const opt = options; + + if (lzma_lzma_lclppb_encode(opt, out)) + return LZMA_PROG_ERROR; + + integer_write_32(out + 1, opt->dictionary_size); + + return LZMA_OK; +} +#endif + + +extern LZMA_API lzma_bool +lzma_mode_is_available(lzma_mode mode) +{ + return mode == LZMA_MODE_FAST || mode == LZMA_MODE_NORMAL; } diff --git a/src/liblzma/lzma/lzma_encoder.h b/src/liblzma/lzma/lzma_encoder.h index 1c57f80a..e270cc27 100644 --- a/src/liblzma/lzma/lzma_encoder.h +++ b/src/liblzma/lzma/lzma_encoder.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.h -/// \brief LZMA method handler API +/// \brief LZMA encoder API // // Copyright (C) 1999-2006 Igor Pavlov // Copyright (C) 2007 Lasse Collin @@ -23,13 +23,47 @@ #include "common.h" + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -extern bool lzma_lzma_encode_properties( + +extern uint64_t lzma_lzma_encoder_memusage(const void *options); + +extern lzma_ret lzma_lzma_props_encode(const void *options, uint8_t *out); + + +/// Encodes lc/lp/pb into one byte. Returns false on success and true on error. +extern bool lzma_lzma_lclppb_encode( const lzma_options_lzma *options, uint8_t *byte); + +#ifdef HAVE_SMALL + /// Initializes the lzma_fastpos[] array. extern void lzma_fastpos_init(void); #endif + + +#ifdef LZMA_LZ_ENCODER_H + +/// Initializes raw LZMA encoder; this is used by LZMA2. +extern lzma_ret lzma_lzma_encoder_create( + lzma_coder **coder_ptr, lzma_allocator *allocator, + const lzma_options_lzma *options, lzma_lz_options *lz_options); + + +/// Resets an already initialized LZMA encoder; this is used by LZMA2. +extern void lzma_lzma_encoder_reset( + lzma_coder *coder, const lzma_options_lzma *options); + + +extern lzma_ret lzma_lzma_encode(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + uint32_t read_limit); + +#endif + +#endif diff --git a/src/liblzma/lzma/lzma_encoder_features.c b/src/liblzma/lzma/lzma_encoder_features.c index 56e59c6a..9fecee48 100644 --- a/src/liblzma/lzma/lzma_encoder_features.c +++ b/src/liblzma/lzma/lzma_encoder_features.c @@ -22,7 +22,7 @@ static lzma_mode modes[] = { LZMA_MODE_FAST, - LZMA_MODE_BEST, + LZMA_MODE_NORMAL, LZMA_MODE_INVALID }; diff --git a/src/liblzma/lzma/lzma_encoder_getoptimum.c b/src/liblzma/lzma/lzma_encoder_getoptimum.c deleted file mode 100644 index b175e4cb..00000000 --- a/src/liblzma/lzma/lzma_encoder_getoptimum.c +++ /dev/null @@ -1,925 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_getoptimum.c -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - - -// "Would you love the monster code? -// Could you understand beauty of the beast?" -// --Adapted from Lordi's "Would you love a monster man". - - -#include "lzma_encoder_private.h" -#include "fastpos.h" - - -#define length_get_price(length_encoder, symbol, pos_state) \ - (length_encoder).prices[pos_state][symbol] - - -#define get_rep_len_1_price(state, pos_state) \ - bit_get_price_0(coder->is_rep0[state]) \ - + bit_get_price_0(coder->is_rep0_long[state][pos_state]) - - -// Adds to price_target. -#define get_pure_rep_price(price_target, rep_index, state, pos_state) \ -do { \ - if ((rep_index) == 0) { \ - price_target += bit_get_price_0(coder->is_rep0[state]); \ - price_target += bit_get_price_1( \ - coder->is_rep0_long[state][pos_state]); \ - } else { \ - price_target += bit_get_price_1(coder->is_rep0[state]); \ - if ((rep_index) == 1) { \ - price_target += bit_get_price_0(coder->is_rep1[state]); \ - } else { \ - price_target += bit_get_price_1(coder->is_rep1[state]); \ - price_target += bit_get_price( \ - coder->is_rep2[state], (rep_index) - 2); \ - } \ - } \ -} while (0) - - -// Adds to price_target. -#define get_rep_price(price_target, rep_index, len, state, pos_state) \ -do { \ - get_pure_rep_price(price_target, rep_index, state, pos_state); \ - price_target += length_get_price(coder->rep_len_encoder, \ - (len) - MATCH_MIN_LEN, pos_state); \ -} while (0) - - -// Adds to price_target. -#define get_pos_len_price(price_target, pos, len, pos_state) \ -do { \ - const uint32_t len_to_pos_state_tmp = get_len_to_pos_state(len); \ - if ((pos) < FULL_DISTANCES) { \ - price_target += distances_prices[len_to_pos_state_tmp][pos]; \ - } else { \ - price_target \ - += pos_slot_prices[len_to_pos_state_tmp][get_pos_slot_2(pos)] \ - + align_prices[(pos) & ALIGN_MASK]; \ - } \ - price_target += length_get_price( \ - coder->match_len_encoder, (len) - MATCH_MIN_LEN, pos_state); \ -} while (0) - - -// Three macros to manipulate lzma_optimal structures: -#define make_as_char(opt) \ -do { \ - (opt).back_prev = UINT32_MAX; \ - (opt).prev_1_is_char = false; \ -} while (0) - - -#define make_as_short_rep(opt) \ -do { \ - (opt).back_prev = 0; \ - (opt).prev_1_is_char = false; \ -} while (0) - - -#define is_short_rep(opt) \ - ((opt).back_prev == 0) - - -static void -fill_length_prices(lzma_length_encoder *lc, uint32_t pos_state) -{ - const uint32_t num_symbols = lc->table_size; - const uint32_t a0 = bit_get_price_0(lc->choice); - const uint32_t a1 = bit_get_price_1(lc->choice); - const uint32_t b0 = a1 + bit_get_price_0(lc->choice2); - const uint32_t b1 = a1 + bit_get_price_1(lc->choice2); - - uint32_t *prices = lc->prices[pos_state]; - uint32_t i = 0; - - for (i = 0; i < num_symbols && i < LEN_LOW_SYMBOLS; ++i) - prices[i] = a0 + bittree_get_price(lc->low[pos_state], - LEN_LOW_BITS, i); - - for (; i < num_symbols && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) - prices[i] = b0 + bittree_get_price(lc->mid[pos_state], - LEN_MID_BITS, i - LEN_LOW_SYMBOLS); - - for (; i < num_symbols; ++i) - prices[i] = b1 + bittree_get_price(lc->high, LEN_HIGH_BITS, - i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); - - lc->counters[pos_state] = num_symbols; - - return; -} - - -static void -fill_distances_prices(lzma_coder *coder) -{ - uint32_t temp_prices[FULL_DISTANCES]; - - for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - temp_prices[i] = bittree_reverse_get_price( - coder->pos_encoders + base - pos_slot - 1, - footer_bits, i - base); - } - - const uint32_t dist_table_size = coder->dist_table_size; - - for (uint32_t len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { - - const probability *encoder = coder->pos_slot_encoder[len_to_pos_state]; - uint32_t *pos_slot_prices = coder->pos_slot_prices[len_to_pos_state]; - - for (uint32_t pos_slot = 0; - pos_slot < dist_table_size; - ++pos_slot) { - pos_slot_prices[pos_slot] = bittree_get_price(encoder, - POS_SLOT_BITS, pos_slot); - } - - for (uint32_t pos_slot = END_POS_MODEL_INDEX; - pos_slot < dist_table_size; - ++pos_slot) - pos_slot_prices[pos_slot] += (((pos_slot >> 1) - 1) - - ALIGN_BITS) << BIT_PRICE_SHIFT_BITS; - - - uint32_t *distances_prices - = coder->distances_prices[len_to_pos_state]; - - uint32_t i; - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - distances_prices[i] = pos_slot_prices[i]; - - for (; i < FULL_DISTANCES; ++i) - distances_prices[i] = pos_slot_prices[get_pos_slot(i)] - + temp_prices[i]; - } - - coder->match_price_count = 0; - - return; -} - - -static void -fill_align_prices(lzma_coder *coder) -{ - for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i) - coder->align_prices[i] = bittree_reverse_get_price( - coder->pos_align_encoder, ALIGN_BITS, i); - - coder->align_price_count = 0; - return; -} - - -// The first argument is a pointer returned by literal_get_subcoder(). -static uint32_t -literal_get_price(const probability *encoders, const bool match_mode, - const uint8_t match_byte, const uint8_t symbol) -{ - uint32_t price = 0; - uint32_t context = 1; - int i = 8; - - if (match_mode) { - do { - --i; - const uint32_t match_bit = (match_byte >> i) & 1; - const uint32_t bit = (symbol >> i) & 1; - const uint32_t subcoder_index - = 0x100 + (match_bit << 8) + context; - - price += bit_get_price(encoders[subcoder_index], bit); - context = (context << 1) | bit; - - if (match_bit != bit) - break; - - } while (i != 0); - } - - while (i != 0) { - --i; - const uint32_t bit = (symbol >> i) & 1; - price += bit_get_price(encoders[context], bit); - context = (context << 1) | bit; - } - - return price; -} - - -static void -backward(lzma_coder *restrict coder, uint32_t *restrict len_res, - uint32_t *restrict back_res, uint32_t cur) -{ - coder->optimum_end_index = cur; - - uint32_t pos_mem = coder->optimum[cur].pos_prev; - uint32_t back_mem = coder->optimum[cur].back_prev; - - do { - if (coder->optimum[cur].prev_1_is_char) { - make_as_char(coder->optimum[pos_mem]); - coder->optimum[pos_mem].pos_prev = pos_mem - 1; - - if (coder->optimum[cur].prev_2) { - coder->optimum[pos_mem - 1].prev_1_is_char = false; - coder->optimum[pos_mem - 1].pos_prev - = coder->optimum[cur].pos_prev_2; - coder->optimum[pos_mem - 1].back_prev - = coder->optimum[cur].back_prev_2; - } - } - - uint32_t pos_prev = pos_mem; - uint32_t back_cur = back_mem; - - back_mem = coder->optimum[pos_prev].back_prev; - pos_mem = coder->optimum[pos_prev].pos_prev; - - coder->optimum[pos_prev].back_prev = back_cur; - coder->optimum[pos_prev].pos_prev = cur; - cur = pos_prev; - - } while (cur != 0); - - coder->optimum_current_index = coder->optimum[0].pos_prev; - *len_res = coder->optimum[0].pos_prev; - *back_res = coder->optimum[0].back_prev; - - return; -} - - -extern void -lzma_get_optimum(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res) -{ - uint32_t position = coder->now_pos; - uint32_t pos_state = position & coder->pos_mask; - - // Update the price tables. In the C++ LZMA SDK 4.42 this was done in both - // initialization function and in the main loop. In liblzma they were - // moved into this single place. - if (coder->additional_offset == 0) { - if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); - - if (coder->align_price_count >= ALIGN_TABLE_SIZE) - fill_align_prices(coder); - } - - if (coder->prev_len_encoder != NULL) { - if (--coder->prev_len_encoder->counters[pos_state] == 0) - fill_length_prices(coder->prev_len_encoder, pos_state); - - coder->prev_len_encoder = NULL; - } - - - if (coder->optimum_end_index != coder->optimum_current_index) { - *len_res = coder->optimum[coder->optimum_current_index].pos_prev - - coder->optimum_current_index; - *back_res = coder->optimum[coder->optimum_current_index].back_prev; - coder->optimum_current_index = coder->optimum[ - coder->optimum_current_index].pos_prev; - return; - } - - coder->optimum_current_index = 0; - coder->optimum_end_index = 0; - - - const uint32_t fast_bytes = coder->fast_bytes; - uint32_t *match_distances = coder->match_distances; - - uint32_t len_main; - uint32_t num_distance_pairs; - - if (!coder->longest_match_was_found) { - lzma_read_match_distances(coder, &len_main, &num_distance_pairs); - } else { - len_main = coder->longest_match_length; - num_distance_pairs = coder->num_distance_pairs; - coder->longest_match_was_found = false; - } - - - const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; - uint32_t num_available_bytes - = coder->lz.write_pos - coder->lz.read_pos + 1; - if (num_available_bytes < 2) { - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - if (num_available_bytes > MATCH_MAX_LEN) - num_available_bytes = MATCH_MAX_LEN; - - - uint32_t reps[REP_DISTANCES]; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - reps[i] = coder->reps[i]; - const uint32_t back_offset = reps[i] + 1; - - if (buf[0] != *(buf - back_offset) - || buf[1] != *(buf + 1 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - uint32_t len_test; - for (len_test = 2; len_test < num_available_bytes - && buf[len_test] == *(buf + len_test - back_offset); - ++len_test) ; - - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) - rep_max_index = i; - } - - if (rep_lens[rep_max_index] >= fast_bytes) { - *back_res = rep_max_index; - *len_res = rep_lens[rep_max_index]; - move_pos(*len_res - 1); - return; - } - - - if (len_main >= fast_bytes) { - *back_res = match_distances[num_distance_pairs] + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 1); - return; - } - - uint8_t current_byte = *buf; - uint8_t match_byte = *(buf - reps[0] - 1); - - if (len_main < 2 && current_byte != match_byte - && rep_lens[rep_max_index] < 2) { - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - coder->optimum[0].state = coder->state; - - coder->optimum[1].price = bit_get_price_0( - coder->is_match[coder->state][pos_state]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position, coder->previous_byte), - !is_literal_state(coder->state), match_byte, current_byte); - - make_as_char(coder->optimum[1]); - - uint32_t match_price - = bit_get_price_1(coder->is_match[coder->state][pos_state]); - uint32_t rep_match_price - = match_price + bit_get_price_1(coder->is_rep[coder->state]); - - - if (match_byte == current_byte) { - const uint32_t short_rep_price = rep_match_price - + get_rep_len_1_price(coder->state, pos_state); - - if (short_rep_price < coder->optimum[1].price) { - coder->optimum[1].price = short_rep_price; - make_as_short_rep(coder->optimum[1]); - } - } - - uint32_t len_end = (len_main >= rep_lens[rep_max_index]) - ? len_main - : rep_lens[rep_max_index]; - - if (len_end < 2) { - *back_res = coder->optimum[1].back_prev; - *len_res = 1; - return; - } - - coder->optimum[1].pos_prev = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) - coder->optimum[0].backs[i] = reps[i]; - - uint32_t len = len_end; - do { - coder->optimum[len].price = INFINITY_PRICE; - } while (--len >= 2); - - - uint32_t (*distances_prices)[FULL_DISTANCES] = coder->distances_prices; - uint32_t (*pos_slot_prices)[DIST_TABLE_SIZE_MAX] = coder->pos_slot_prices; - uint32_t *align_prices = coder->align_prices; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - uint32_t rep_len = rep_lens[i]; - if (rep_len < 2) - continue; - - uint32_t price = rep_match_price; - get_pure_rep_price(price, i, coder->state, pos_state); - - do { - const uint32_t cur_and_len_price = price - + length_get_price( - coder->rep_len_encoder, - rep_len - 2, pos_state); - - if (cur_and_len_price < coder->optimum[rep_len].price) { - coder->optimum[rep_len].price = cur_and_len_price; - coder->optimum[rep_len].pos_prev = 0; - coder->optimum[rep_len].back_prev = i; - coder->optimum[rep_len].prev_1_is_char = false; - } - } while (--rep_len >= 2); - } - - - uint32_t normal_match_price = match_price - + bit_get_price_0(coder->is_rep[coder->state]); - - len = (rep_lens[0] >= 2) ? rep_lens[0] + 1 : 2; - - if (len <= len_main) { - uint32_t offs = 0; - - while (len > match_distances[offs + 1]) - offs += 2; - - for(; ; ++len) { - const uint32_t distance = match_distances[offs + 2]; - uint32_t cur_and_len_price = normal_match_price; - get_pos_len_price(cur_and_len_price, distance, len, pos_state); - - if (cur_and_len_price < coder->optimum[len].price) { - coder->optimum[len].price = cur_and_len_price; - coder->optimum[len].pos_prev = 0; - coder->optimum[len].back_prev = distance + REP_DISTANCES; - coder->optimum[len].prev_1_is_char = false; - } - - if (len == match_distances[offs + 1]) { - offs += 2; - if (offs == num_distance_pairs) - break; - } - } - } - - - ////////////////// - // Big loop ;-) // - ////////////////// - - uint32_t cur = 0; - - // The rest of this function is a huge while-loop. To avoid extreme - // indentation, the indentation level is not increased here. - while (true) { - - ++cur; - - assert(cur < OPTS); - - if (cur == len_end) { - backward(coder, len_res, back_res, cur); - return; - } - - uint32_t new_len; - - lzma_read_match_distances(coder, &new_len, &num_distance_pairs); - - if (new_len >= fast_bytes) { - coder->num_distance_pairs = num_distance_pairs; - coder->longest_match_length = new_len; - coder->longest_match_was_found = true; - backward(coder, len_res, back_res, cur); - return; - } - - - ++position; - - uint32_t pos_prev = coder->optimum[cur].pos_prev; - uint32_t state; - - if (coder->optimum[cur].prev_1_is_char) { - --pos_prev; - - if (coder->optimum[cur].prev_2) { - state = coder->optimum[coder->optimum[cur].pos_prev_2].state; - - if (coder->optimum[cur].back_prev_2 < REP_DISTANCES) - update_long_rep(state); - else - update_match(state); - - } else { - state = coder->optimum[pos_prev].state; - } - - update_literal(state); - - } else { - state = coder->optimum[pos_prev].state; - } - - if (pos_prev == cur - 1) { - if (is_short_rep(coder->optimum[cur])) - update_short_rep(state); - else - update_literal(state); - } else { - uint32_t pos; - if (coder->optimum[cur].prev_1_is_char && coder->optimum[cur].prev_2) { - pos_prev = coder->optimum[cur].pos_prev_2; - pos = coder->optimum[cur].back_prev_2; - update_long_rep(state); - } else { - pos = coder->optimum[cur].back_prev; - if (pos < REP_DISTANCES) - update_long_rep(state); - else - update_match(state); - } - - if (pos < REP_DISTANCES) { - reps[0] = coder->optimum[pos_prev].backs[pos]; - - uint32_t i; - for (i = 1; i <= pos; ++i) - reps[i] = coder->optimum[pos_prev].backs[i - 1]; - - for (; i < REP_DISTANCES; ++i) - reps[i] = coder->optimum[pos_prev].backs[i]; - - } else { - reps[0] = pos - REP_DISTANCES; - - for (uint32_t i = 1; i < REP_DISTANCES; ++i) - reps[i] = coder->optimum[pos_prev].backs[i - 1]; - } - } - - coder->optimum[cur].state = state; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) - coder->optimum[cur].backs[i] = reps[i]; - - const uint32_t cur_price = coder->optimum[cur].price; - - buf = coder->lz.buffer + coder->lz.read_pos - 1; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); - - pos_state = position & coder->pos_mask; - - const uint32_t cur_and_1_price = cur_price - + bit_get_price_0(coder->is_match[state][pos_state]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position, buf[-1]), - !is_literal_state(state), match_byte, current_byte); - - bool next_is_char = false; - - if (cur_and_1_price < coder->optimum[cur + 1].price) { - coder->optimum[cur + 1].price = cur_and_1_price; - coder->optimum[cur + 1].pos_prev = cur; - make_as_char(coder->optimum[cur + 1]); - next_is_char = true; - } - - match_price = cur_price - + bit_get_price_1(coder->is_match[state][pos_state]); - rep_match_price = match_price - + bit_get_price_1(coder->is_rep[state]); - - if (match_byte == current_byte - && !(coder->optimum[cur + 1].pos_prev < cur - && coder->optimum[cur + 1].back_prev == 0)) { - - const uint32_t short_rep_price = rep_match_price - + get_rep_len_1_price(state, pos_state); - - if (short_rep_price <= coder->optimum[cur + 1].price) { - coder->optimum[cur + 1].price = short_rep_price; - coder->optimum[cur + 1].pos_prev = cur; - make_as_short_rep(coder->optimum[cur + 1]); - next_is_char = true; - } - } - - uint32_t num_available_bytes_full - = coder->lz.write_pos - coder->lz.read_pos + 1; - num_available_bytes_full = MIN(OPTS - 1 - cur, num_available_bytes_full); - num_available_bytes = num_available_bytes_full; - - if (num_available_bytes < 2) - continue; - - if (num_available_bytes > fast_bytes) - num_available_bytes = fast_bytes; - - if (!next_is_char && match_byte != current_byte) { // speed optimization - // try literal + rep0 - const uint32_t back_offset = reps[0] + 1; - const uint32_t limit = MIN(num_available_bytes_full, fast_bytes + 1); - - uint32_t temp; - for (temp = 1; temp < limit - && buf[temp] == *(buf + temp - back_offset); - ++temp) ; - - const uint32_t len_test_2 = temp - 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_literal(state_2); - - const uint32_t pos_state_next = (position + 1) & coder->pos_mask; - const uint32_t next_rep_match_price = cur_and_1_price - + bit_get_price_1(coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - - // for (; len_test_2 >= 2; --len_test_2) { - const uint32_t offset = cur + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = false; - } -// } - } - } - - - uint32_t start_len = 2; // speed optimization - - for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - const uint32_t back_offset = reps[rep_index] + 1; - - if (buf[0] != *(buf - back_offset) || buf[1] != *(buf + 1 - back_offset)) - continue; - - uint32_t len_test; - for (len_test = 2; len_test < num_available_bytes - && buf[len_test] == *(buf + len_test - back_offset); - ++len_test) ; - - while (len_end < cur + len_test) - coder->optimum[++len_end].price = INFINITY_PRICE; - - const uint32_t len_test_temp = len_test; - uint32_t price = rep_match_price; - get_pure_rep_price(price, rep_index, state, pos_state); - - do { - const uint32_t cur_and_len_price = price - + length_get_price(coder->rep_len_encoder, - len_test - 2, pos_state); - - if (cur_and_len_price < coder->optimum[cur + len_test].price) { - coder->optimum[cur + len_test].price = cur_and_len_price; - coder->optimum[cur + len_test].pos_prev = cur; - coder->optimum[cur + len_test].back_prev = rep_index; - coder->optimum[cur + len_test].prev_1_is_char = false; - } - } while (--len_test >= 2); - - len_test = len_test_temp; - - if (rep_index == 0) - start_len = len_test + 1; - - - uint32_t len_test_2 = len_test + 1; - const uint32_t limit = MIN(num_available_bytes_full, - len_test_2 + fast_bytes); - for (; len_test_2 < limit - && buf[len_test_2] == *(buf + len_test_2 - back_offset); - ++len_test_2) ; - - len_test_2 -= len_test + 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_long_rep(state_2); - - uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - - const uint32_t cur_and_len_char_price = price - + length_get_price(coder->rep_len_encoder, - len_test - 2, pos_state) - + bit_get_price_0(coder->is_match[state_2][pos_state_next]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position + len_test, buf[len_test - 1]), - true, *(buf + len_test - back_offset), buf[len_test]); - - update_literal(state_2); - - pos_state_next = (position + len_test + 1) & coder->pos_mask; - - const uint32_t next_rep_match_price = cur_and_len_char_price - + bit_get_price_1(coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - -// for(; len_test_2 >= 2; len_test_2--) { - const uint32_t offset = cur + len_test + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + len_test + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = true; - coder->optimum[offset].pos_prev_2 = cur; - coder->optimum[offset].back_prev_2 = rep_index; - } -// } - } - } - - -// for (uint32_t len_test = 2; len_test <= new_len; ++len_test) - if (new_len > num_available_bytes) { - new_len = num_available_bytes; - - for (num_distance_pairs = 0; - new_len > match_distances[num_distance_pairs + 1]; - num_distance_pairs += 2) ; - - match_distances[num_distance_pairs + 1] = new_len; - num_distance_pairs += 2; - } - - - if (new_len >= start_len) { - normal_match_price = match_price - + bit_get_price_0(coder->is_rep[state]); - - while (len_end < cur + new_len) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t offs = 0; - while (start_len > match_distances[offs + 1]) - offs += 2; - - uint32_t cur_back = match_distances[offs + 2]; - uint32_t pos_slot = get_pos_slot_2(cur_back); - - for (uint32_t len_test = start_len; ; ++len_test) { - uint32_t cur_and_len_price = normal_match_price; - const uint32_t len_to_pos_state = get_len_to_pos_state(len_test); - - if (cur_back < FULL_DISTANCES) - cur_and_len_price += distances_prices[ - len_to_pos_state][cur_back]; - else - cur_and_len_price += pos_slot_prices[ - len_to_pos_state][pos_slot] - + align_prices[cur_back & ALIGN_MASK]; - - cur_and_len_price += length_get_price(coder->match_len_encoder, - len_test - MATCH_MIN_LEN, pos_state); - - if (cur_and_len_price < coder->optimum[cur + len_test].price) { - coder->optimum[cur + len_test].price = cur_and_len_price; - coder->optimum[cur + len_test].pos_prev = cur; - coder->optimum[cur + len_test].back_prev - = cur_back + REP_DISTANCES; - coder->optimum[cur + len_test].prev_1_is_char = false; - } - - if (len_test == match_distances[offs + 1]) { - // Try Match + Literal + Rep0 - const uint32_t back_offset = cur_back + 1; - uint32_t len_test_2 = len_test + 1; - const uint32_t limit = MIN(num_available_bytes_full, - len_test_2 + fast_bytes); - - for (; len_test_2 < limit && - buf[len_test_2] == *(buf + len_test_2 - back_offset); - ++len_test_2) ; - - len_test_2 -= len_test + 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_match(state_2); - uint32_t pos_state_next - = (position + len_test) & coder->pos_mask; - - const uint32_t cur_and_len_char_price = cur_and_len_price - + bit_get_price_0( - coder->is_match[state_2][pos_state_next]) - + literal_get_price( - literal_get_subcoder( - coder->literal_coder, - position + len_test, - buf[len_test - 1]), - true, - *(buf + len_test - back_offset), - buf[len_test]); - - update_literal(state_2); - pos_state_next = (pos_state_next + 1) & coder->pos_mask; - - const uint32_t next_rep_match_price - = cur_and_len_char_price - + bit_get_price_1( - coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - - // for(; len_test_2 >= 2; --len_test_2) { - const uint32_t offset = cur + len_test + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + len_test + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = true; - coder->optimum[offset].pos_prev_2 = cur; - coder->optimum[offset].back_prev_2 - = cur_back + REP_DISTANCES; - } -// } - } - - offs += 2; - if (offs == num_distance_pairs) - break; - - cur_back = match_distances[offs + 2]; - if (cur_back >= FULL_DISTANCES) - pos_slot = get_pos_slot_2(cur_back); - } - } - } - - } // Closes: while (true) -} diff --git a/src/liblzma/lzma/lzma_encoder_getoptimumfast.c b/src/liblzma/lzma/lzma_encoder_getoptimumfast.c deleted file mode 100644 index fa06be21..00000000 --- a/src/liblzma/lzma/lzma_encoder_getoptimumfast.c +++ /dev/null @@ -1,201 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_getoptimumfast.c -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - - -#include "lzma_encoder_private.h" - - -#define change_pair(small_dist, big_dist) \ - (((big_dist) >> 7) > (small_dist)) - - -extern void -lzma_get_optimum_fast(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res) -{ - // Local copies - const uint32_t fast_bytes = coder->fast_bytes; - - uint32_t len_main; - uint32_t num_distance_pairs; - if (!coder->longest_match_was_found) { - lzma_read_match_distances(coder, &len_main, &num_distance_pairs); - } else { - len_main = coder->longest_match_length; - num_distance_pairs = coder->num_distance_pairs; - coder->longest_match_was_found = false; - } - - const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; - uint32_t num_available_bytes - = coder->lz.write_pos - coder->lz.read_pos + 1; - - if (num_available_bytes < 2) { - // There's not enough input left to encode a match. - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - if (num_available_bytes > MATCH_MAX_LEN) - num_available_bytes = MATCH_MAX_LEN; - - - // Look for repetitive matches; scan the previous four match distances - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - const uint32_t back_offset = coder->reps[i] + 1; - - // If the first two bytes (2 == MATCH_MIN_LEN) do not match, - // this rep_distance[i] is not useful. This is indicated - // using zero as the length of the repetitive match. - if (buf[0] != *(buf - back_offset) - || buf[1] != *(buf + 1 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - // The first two bytes matched. - // Calculate the length of the match. - uint32_t len; - for (len = 2; len < num_available_bytes - && buf[len] == *(buf + len - back_offset); - ++len) ; - - // If we have found a repetitive match that is at least - // as long as fast_bytes, return it immediatelly. - if (len >= fast_bytes) { - *back_res = i; - *len_res = len; - move_pos(len - 1); - return; - } - - rep_lens[i] = len; - - // After this loop, rep_lens[rep_max_index] is the biggest - // value of all values in rep_lens[]. - if (len > rep_lens[rep_max_index]) - rep_max_index = i; - } - - - if (len_main >= fast_bytes) { - *back_res = coder->match_distances[num_distance_pairs] - + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 1); - return; - } - - uint32_t back_main = 0; - if (len_main >= 2) { - back_main = coder->match_distances[num_distance_pairs]; - - while (num_distance_pairs > 2 && len_main == - coder->match_distances[num_distance_pairs - 3] + 1) { - if (!change_pair(coder->match_distances[ - num_distance_pairs - 2], back_main)) - break; - - num_distance_pairs -= 2; - len_main = coder->match_distances[num_distance_pairs - 1]; - back_main = coder->match_distances[num_distance_pairs]; - } - - if (len_main == 2 && back_main >= 0x80) - len_main = 1; - } - - if (rep_lens[rep_max_index] >= 2) { - if (rep_lens[rep_max_index] + 1 >= len_main - || (rep_lens[rep_max_index] + 2 >= len_main - && (back_main > (1 << 9))) - || (rep_lens[rep_max_index] + 3 >= len_main - && (back_main > (1 << 15)))) { - *back_res = rep_max_index; - *len_res = rep_lens[rep_max_index]; - move_pos(*len_res - 1); - return; - } - } - - if (len_main >= 2 && num_available_bytes > 2) { - lzma_read_match_distances(coder, &coder->longest_match_length, - &coder->num_distance_pairs); - - if (coder->longest_match_length >= 2) { - const uint32_t new_distance = coder->match_distances[ - coder->num_distance_pairs]; - - if ((coder->longest_match_length >= len_main - && new_distance < back_main) - || (coder->longest_match_length == len_main + 1 - && !change_pair(back_main, new_distance)) - || (coder->longest_match_length > len_main + 1) - || (coder->longest_match_length + 1 >= len_main - && len_main >= 3 - && change_pair(new_distance, back_main))) { - coder->longest_match_was_found = true; - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - } - - ++buf; - --num_available_bytes; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - const uint32_t back_offset = coder->reps[i] + 1; - - if (buf[1] != *(buf + 1 - back_offset) - || buf[2] != *(buf + 2 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - uint32_t len; - for (len = 2; len < num_available_bytes - && buf[len] == *(buf + len - back_offset); - ++len) ; - - if (len + 1 >= len_main) { - coder->longest_match_was_found = true; - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - } - - *back_res = back_main + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 2); - return; - } - - *back_res = UINT32_MAX; - *len_res = 1; - return; -} diff --git a/src/liblzma/lzma/lzma_encoder_init.c b/src/liblzma/lzma/lzma_encoder_init.c deleted file mode 100644 index 21335f95..00000000 --- a/src/liblzma/lzma/lzma_encoder_init.c +++ /dev/null @@ -1,228 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_init.c -/// \brief Creating, resetting and destroying the LZMA encoder -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lzma_encoder_private.h" - - -/// \brief Initializes the length encoder -static void -length_encoder_reset(lzma_length_encoder *lencoder, - const uint32_t num_pos_states, const uint32_t table_size) -{ - // NLength::CPriceTableEncoder::SetTableSize() - lencoder->table_size = table_size; - - // NLength::CEncoder::Init() - bit_reset(lencoder->choice); - bit_reset(lencoder->choice2); - - for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { - bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); - bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); - } - - bittree_reset(lencoder->high, LEN_HIGH_BITS); - - // NLength::CPriceTableEncoder::UpdateTables() - for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) - lencoder->counters[pos_state] = 1; - - return; -} - - -static void -lzma_lzma_encoder_end(lzma_coder *coder, lzma_allocator *allocator) -{ - lzma_lz_encoder_end(&coder->lz, allocator); - lzma_free(coder, allocator); - return; -} - - -extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter_info *filters) -{ - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) - return LZMA_MEM_ERROR; - - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->lz = LZMA_LZ_ENCODER_INIT; - } - - // Validate options that aren't validated elsewhere. - const lzma_options_lzma *options = filters[0].options; - if (options->pos_bits > LZMA_POS_BITS_MAX - || options->fast_bytes < LZMA_FAST_BYTES_MIN - || options->fast_bytes > LZMA_FAST_BYTES_MAX) { - lzma_lzma_encoder_end(next->coder, allocator); - return LZMA_HEADER_ERROR; - } - - // Set compression mode. - switch (options->mode) { - case LZMA_MODE_FAST: - next->coder->best_compression = false; - break; - - case LZMA_MODE_BEST: - next->coder->best_compression = true; - break; - - default: - lzma_lzma_encoder_end(next->coder, allocator); - return LZMA_HEADER_ERROR; - } - - // Initialize literal coder. - { - const lzma_ret ret = lzma_literal_init( - &next->coder->literal_coder, - options->literal_context_bits, - options->literal_pos_bits); - if (ret != LZMA_OK) - return ret; - } - - // Initialize LZ encoder. - { - const lzma_ret ret = lzma_lz_encoder_reset( - &next->coder->lz, allocator, &lzma_lzma_encode, - options->dictionary_size, OPTS, - options->fast_bytes, MATCH_MAX_LEN + 1 + OPTS, - options->match_finder, - options->match_finder_cycles, - options->preset_dictionary, - options->preset_dictionary_size); - if (ret != LZMA_OK) { - lzma_lzma_encoder_end(next->coder, allocator); - return ret; - } - } - - // Set dist_table_size. - { - // Round the dictionary size up to next 2^n. - uint32_t log_size; - for (log_size = 0; (UINT32_C(1) << log_size) - < options->dictionary_size; ++log_size) ; - - next->coder->dist_table_size = log_size * 2; - } - - // Misc FIXME desc - next->coder->align_price_count = UINT32_MAX; - next->coder->match_price_count = UINT32_MAX; - next->coder->dictionary_size = options->dictionary_size; - next->coder->pos_mask = (1U << options->pos_bits) - 1; - next->coder->fast_bytes = options->fast_bytes; - - // Range coder - rc_reset(&next->coder->rc); - - // State - next->coder->state = 0; - next->coder->previous_byte = 0; - for (size_t i = 0; i < REP_DISTANCES; ++i) - next->coder->reps[i] = 0; - - // Bit encoders - for (size_t i = 0; i < STATES; ++i) { - for (size_t j = 0; j <= next->coder->pos_mask; ++j) { - bit_reset(next->coder->is_match[i][j]); - bit_reset(next->coder->is_rep0_long[i][j]); - } - - bit_reset(next->coder->is_rep[i]); - bit_reset(next->coder->is_rep0[i]); - bit_reset(next->coder->is_rep1[i]); - bit_reset(next->coder->is_rep2[i]); - } - - for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(next->coder->pos_encoders[i]); - - // Bit tree encoders - for (size_t i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(next->coder->pos_slot_encoder[i], POS_SLOT_BITS); - - bittree_reset(next->coder->pos_align_encoder, ALIGN_BITS); - - // Length encoders - length_encoder_reset(&next->coder->match_len_encoder, - 1U << options->pos_bits, - options->fast_bytes + 1 - MATCH_MIN_LEN); - - length_encoder_reset(&next->coder->rep_len_encoder, - 1U << options->pos_bits, - next->coder->fast_bytes + 1 - MATCH_MIN_LEN); - - next->coder->prev_len_encoder = NULL; - - // Misc - next->coder->longest_match_was_found = false; - next->coder->optimum_end_index = 0; - next->coder->optimum_current_index = 0; - next->coder->additional_offset = 0; - - next->coder->now_pos = 0; - next->coder->is_initialized = false; - next->coder->is_flushed = false, - next->coder->write_eopm = true; - - // Initialize the next decoder in the chain, if any. - { - const lzma_ret ret = lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); - if (ret != LZMA_OK) { - lzma_lzma_encoder_end(next->coder, allocator); - return ret; - } - } - - // Initialization successful. Set the function pointers. - next->code = &lzma_lz_encode; - next->end = &lzma_lzma_encoder_end; - - return LZMA_OK; -} - - -extern bool -lzma_lzma_encode_properties(const lzma_options_lzma *options, uint8_t *byte) -{ - if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX - || options->literal_pos_bits - > LZMA_LITERAL_POS_BITS_MAX - || options->pos_bits > LZMA_POS_BITS_MAX - || options->literal_context_bits - + options->literal_pos_bits - > LZMA_LITERAL_BITS_MAX) - return true; - - *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9 - + options->literal_context_bits; - assert(*byte <= (4 * 5 + 4) * 9 + 8); - - return false; -} diff --git a/src/liblzma/lzma/lzma_encoder_optimum_fast.c b/src/liblzma/lzma/lzma_encoder_optimum_fast.c new file mode 100644 index 00000000..9da7e79e --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -0,0 +1,193 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_optimum_fast.c +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder_private.h" + + +#define change_pair(small_dist, big_dist) \ + (((big_dist) >> 7) > (small_dist)) + + +static inline void +literal(const lzma_coder *restrict coder, const uint8_t *restrict buf, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + // Try short rep0 instead of always coding it as a literal. + *back_res = *buf == *(buf - coder->reps[0] - 1) ? 0 : UINT32_MAX; + *len_res = 1; + return; +} + + +extern void +lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + const uint32_t fast_bytes = mf->find_len_max; + + uint32_t len_main; + uint32_t matches_count; + if (mf->read_ahead == 0) { + len_main = mf_find(mf, &matches_count, coder->matches); + } else { + assert(mf->read_ahead == 1); + len_main = coder->longest_match_length; + matches_count = coder->matches_count; + } + + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX); + + if (buf_avail < 2) { + // There's not enough input left to encode a match. + literal(coder, buf, back_res, len_res); + return; + } + + // Look for repeated matches; scan the previous four match distances + uint32_t rep_len = 0; + uint32_t rep_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + // Pointer to the beginning of the match candidate + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + // If the first two bytes (2 == MATCH_LEN_MIN) do not match, + // this rep is not useful. + if (not_equal_16(buf, buf_back)) + continue; + + // The first two bytes matched. + // Calculate the length of the match. + uint32_t len; + for (len = 2; len < buf_avail + && buf[len] == buf_back[len]; ++len) ; + + // If we have found a repeated match that is at least + // fast_bytes long, return it immediatelly. + if (len >= fast_bytes) { + *back_res = i; + *len_res = len; + mf_skip(mf, len - 1); + return; + } + + if (len > rep_len) { + rep_index = i; + rep_len = len; + } + } + + // We didn't find a long enough repeated match. Encode it as a normal + // match if the match length is at least fast_bytes. + if (len_main >= fast_bytes) { + *back_res = coder->matches[matches_count - 1].dist + + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 1); + return; + } + + uint32_t back_main = 0; + if (len_main >= 2) { + back_main = coder->matches[matches_count - 1].dist; + + while (matches_count > 1 && len_main == + coder->matches[matches_count - 2].len + 1) { + if (!change_pair(coder->matches[ + matches_count - 2].dist, + back_main)) + break; + + --matches_count; + len_main = coder->matches[matches_count - 1].len; + back_main = coder->matches[matches_count - 1].dist; + } + + if (len_main == 2 && back_main >= 0x80) + len_main = 1; + } + + if (rep_len >= 2) { + if (rep_len + 1 >= len_main + || (rep_len + 2 >= len_main + && back_main > (UINT32_C(1) << 9)) + || (rep_len + 3 >= len_main + && back_main > (UINT32_C(1) << 15))) { + *back_res = rep_index; + *len_res = rep_len; + mf_skip(mf, rep_len - 1); + return; + } + } + + if (len_main < 2 || buf_avail <= 2) { + literal(coder, buf, back_res, len_res); + return; + } + + // Get the matches for the next byte. If we find a better match, + // the current byte is encoded as a literal. + coder->longest_match_length = mf_find(mf, + &coder->matches_count, coder->matches); + + if (coder->longest_match_length >= 2) { + const uint32_t new_dist = coder->matches[ + coder->matches_count - 1].dist; + + if ((coder->longest_match_length >= len_main + && new_dist < back_main) + || (coder->longest_match_length == len_main + 1 + && !change_pair(back_main, new_dist)) + || (coder->longest_match_length > len_main + 1) + || (coder->longest_match_length + 1 >= len_main + && len_main >= 3 + && change_pair(new_dist, back_main))) { + literal(coder, buf, back_res, len_res); + return; + } + } + + // In contrast to LZMA SDK, dictionary could not have been moved + // between mf_find() calls, thus it is safe to just increment + // the old buf pointer instead of recalculating it with mf_ptr(). + ++buf; + + const uint32_t limit = len_main - 1; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + if (not_equal_16(buf, buf_back)) + continue; + + uint32_t len; + for (len = 2; len < limit + && buf[len] == buf_back[len]; ++len) ; + + if (len >= limit) { + literal(coder, buf - 1, back_res, len_res); + return; + } + } + + *back_res = back_main + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 2); + return; +} diff --git a/src/liblzma/lzma/lzma_encoder_optimum_normal.c b/src/liblzma/lzma/lzma_encoder_optimum_normal.c new file mode 100644 index 00000000..f0dd92c9 --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -0,0 +1,875 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_optimum_normal.c +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder_private.h" +#include "fastpos.h" + + +//////////// +// Prices // +//////////// + +static uint32_t +get_literal_price(const lzma_coder *const coder, const uint32_t pos, + const uint32_t prev_byte, const bool match_mode, + uint32_t match_byte, uint32_t symbol) +{ + const probability *const subcoder = literal_subcoder(coder->literal, + coder->literal_context_bits, coder->literal_pos_mask, + pos, prev_byte); + + uint32_t price = 0; + + if (!match_mode) { + price = rc_bittree_price(subcoder, 8, symbol); + } else { + uint32_t offset = 0x100; + symbol += UINT32_C(1) << 8; + + do { + match_byte <<= 1; + + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; + price += rc_bit_price(subcoder[subcoder_index], bit); + + symbol <<= 1; + offset &= ~(match_byte ^ symbol); + + } while (symbol < (UINT32_C(1) << 16)); + } + + return price; +} + + +static inline uint32_t +get_len_price(const lzma_length_encoder *const lencoder, + const uint32_t len, const uint32_t pos_state) +{ + // NOTE: Unlike the other price tables, length prices are updated + // in lzma_encoder.c + return lencoder->prices[pos_state][len - MATCH_LEN_MIN]; +} + + +static inline uint32_t +get_short_rep_price(const lzma_coder *const coder, + const lzma_lzma_state state, const uint32_t pos_state) +{ + return rc_bit_0_price(coder->is_rep0[state]) + + rc_bit_0_price(coder->is_rep0_long[state][pos_state]); +} + + +static inline uint32_t +get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, + const lzma_lzma_state state, uint32_t pos_state) +{ + uint32_t price; + + if (rep_index == 0) { + price = rc_bit_0_price(coder->is_rep0[state]); + price += rc_bit_1_price(coder->is_rep0_long[state][pos_state]); + } else { + price = rc_bit_1_price(coder->is_rep0[state]); + + if (rep_index == 1) { + price += rc_bit_0_price(coder->is_rep1[state]); + } else { + price += rc_bit_1_price(coder->is_rep1[state]); + price += rc_bit_price(coder->is_rep2[state], + rep_index - 2); + } + } + + return price; +} + + +static inline uint32_t +get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, + const uint32_t len, const lzma_lzma_state state, + const uint32_t pos_state) +{ + return get_len_price(&coder->rep_len_encoder, len, pos_state) + + get_pure_rep_price(coder, rep_index, state, pos_state); +} + + +static inline uint32_t +get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, + const uint32_t len, const uint32_t pos_state) +{ + const uint32_t len_to_pos_state = get_len_to_pos_state(len); + uint32_t price; + + if (pos < FULL_DISTANCES) { + price = coder->distances_prices[len_to_pos_state][pos]; + } else { + const uint32_t pos_slot = get_pos_slot_2(pos); + price = coder->pos_slot_prices[len_to_pos_state][pos_slot] + + coder->align_prices[pos & ALIGN_MASK]; + } + + price += get_len_price(&coder->match_len_encoder, len, pos_state); + + return price; +} + + +static void +fill_distances_prices(lzma_coder *coder) +{ + for (uint32_t len_to_pos_state = 0; + len_to_pos_state < LEN_TO_POS_STATES; + ++len_to_pos_state) { + + uint32_t *const pos_slot_prices + = coder->pos_slot_prices[len_to_pos_state]; + + // Price to encode the pos_slot. + for (uint32_t pos_slot = 0; + pos_slot < coder->dist_table_size; ++pos_slot) + pos_slot_prices[pos_slot] = rc_bittree_price( + coder->pos_slot[len_to_pos_state], + POS_SLOT_BITS, pos_slot); + + // For matches with distance >= FULL_DISTANCES, add the price + // of the direct bits part of the match distance. (Align bits + // are handled by fill_align_prices()). + for (uint32_t pos_slot = END_POS_MODEL_INDEX; + pos_slot < coder->dist_table_size; ++pos_slot) + pos_slot_prices[pos_slot] += rc_direct_price( + ((pos_slot >> 1) - 1) - ALIGN_BITS); + + // Distances in the range [0, 3] are fully encoded with + // pos_slot, so they are used for coder->distances_prices + // as is. + for (uint32_t i = 0; i < START_POS_MODEL_INDEX; ++i) + coder->distances_prices[len_to_pos_state][i] + = pos_slot_prices[i]; + } + + // Distances in the range [4, 127] depend on pos_slot and pos_special. + // We do this in a loop separate from the above loop to avoid + // redundant calls to get_pos_slot(). + for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { + const uint32_t pos_slot = get_pos_slot(i); + const uint32_t footer_bits = ((pos_slot >> 1) - 1); + const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + const uint32_t price = rc_bittree_reverse_price( + coder->pos_special + base - pos_slot - 1, + footer_bits, i - base); + + for (uint32_t len_to_pos_state = 0; + len_to_pos_state < LEN_TO_POS_STATES; + ++len_to_pos_state) + coder->distances_prices[len_to_pos_state][i] + = price + coder->pos_slot_prices[ + len_to_pos_state][pos_slot]; + } + + coder->match_price_count = 0; + return; +} + + +static void +fill_align_prices(lzma_coder *coder) +{ + for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i) + coder->align_prices[i] = rc_bittree_reverse_price( + coder->pos_align, ALIGN_BITS, i); + + coder->align_price_count = 0; + return; +} + + +///////////// +// Optimal // +///////////// + +static inline void +make_literal(lzma_optimal *optimal) +{ + optimal->back_prev = UINT32_MAX; + optimal->prev_1_is_literal = false; +} + + +static inline void +make_short_rep(lzma_optimal *optimal) +{ + optimal->back_prev = 0; + optimal->prev_1_is_literal = false; +} + + +#define is_short_rep(optimal) \ + ((optimal).back_prev == 0) + + +static void +backward(lzma_coder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) +{ + coder->opts_end_index = cur; + + uint32_t pos_mem = coder->opts[cur].pos_prev; + uint32_t back_mem = coder->opts[cur].back_prev; + + do { + if (coder->opts[cur].prev_1_is_literal) { + make_literal(&coder->opts[pos_mem]); + coder->opts[pos_mem].pos_prev = pos_mem - 1; + + if (coder->opts[cur].prev_2) { + coder->opts[pos_mem - 1].prev_1_is_literal + = false; + coder->opts[pos_mem - 1].pos_prev + = coder->opts[cur].pos_prev_2; + coder->opts[pos_mem - 1].back_prev + = coder->opts[cur].back_prev_2; + } + } + + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + + back_mem = coder->opts[pos_prev].back_prev; + pos_mem = coder->opts[pos_prev].pos_prev; + + coder->opts[pos_prev].back_prev = back_cur; + coder->opts[pos_prev].pos_prev = cur; + cur = pos_prev; + + } while (cur != 0); + + coder->opts_current_index = coder->opts[0].pos_prev; + *len_res = coder->opts[0].pos_prev; + *back_res = coder->opts[0].back_prev; + + return; +} + + +////////// +// Main // +////////// + +static inline uint32_t +helper1(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, + uint32_t position) +{ + const uint32_t fast_bytes = mf->find_len_max; + + uint32_t len_main; + uint32_t matches_count; + + if (mf->read_ahead == 0) { + len_main = mf_find(mf, &matches_count, coder->matches); + } else { + assert(mf->read_ahead == 1); + len_main = coder->longest_match_length; + matches_count = coder->matches_count; + } + + const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX); + if (buf_avail < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return UINT32_MAX; + } + + const uint8_t *const buf = mf_ptr(mf) - 1; + + uint32_t rep_lens[REP_DISTANCES]; + uint32_t rep_max_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + if (not_equal_16(buf, buf_back)) { + rep_lens[i] = 0; + continue; + } + + uint32_t len_test; + for (len_test = 2; len_test < buf_avail + && buf[len_test] == buf_back[len_test]; + ++len_test) ; + + rep_lens[i] = len_test; + if (len_test > rep_lens[rep_max_index]) + rep_max_index = i; + } + + if (rep_lens[rep_max_index] >= fast_bytes) { + *back_res = rep_max_index; + *len_res = rep_lens[rep_max_index]; + mf_skip(mf, *len_res - 1); + return UINT32_MAX; + } + + + if (len_main >= fast_bytes) { + *back_res = coder->matches[matches_count - 1].dist + + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 1); + return UINT32_MAX; + } + + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); + + if (len_main < 2 && current_byte != match_byte + && rep_lens[rep_max_index] < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return UINT32_MAX; + } + + coder->opts[0].state = coder->state; + + const uint32_t pos_state = position & coder->pos_mask; + + coder->opts[1].price = rc_bit_0_price( + coder->is_match[coder->state][pos_state]) + + get_literal_price(coder, position, buf[-1], + !is_literal_state(coder->state), + match_byte, current_byte); + + make_literal(&coder->opts[1]); + + const uint32_t match_price = rc_bit_1_price( + coder->is_match[coder->state][pos_state]); + const uint32_t rep_match_price = match_price + + rc_bit_1_price(coder->is_rep[coder->state]); + + if (match_byte == current_byte) { + const uint32_t short_rep_price = rep_match_price + + get_short_rep_price( + coder, coder->state, pos_state); + + if (short_rep_price < coder->opts[1].price) { + coder->opts[1].price = short_rep_price; + make_short_rep(&coder->opts[1]); + } + } + + const uint32_t len_end = MAX(len_main, rep_lens[rep_max_index]); + + if (len_end < 2) { + *back_res = coder->opts[1].back_prev; + *len_res = 1; + return UINT32_MAX; + } + + coder->opts[1].pos_prev = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->opts[0].backs[i] = coder->reps[i]; + + uint32_t len = len_end; + do { + coder->opts[len].price = RC_INFINITY_PRICE; + } while (--len >= 2); + + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + uint32_t rep_len = rep_lens[i]; + if (rep_len < 2) + continue; + + const uint32_t price = rep_match_price + get_pure_rep_price( + coder, i, coder->state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + get_len_price( + &coder->rep_len_encoder, + rep_len, pos_state); + + if (cur_and_len_price < coder->opts[rep_len].price) { + coder->opts[rep_len].price = cur_and_len_price; + coder->opts[rep_len].pos_prev = 0; + coder->opts[rep_len].back_prev = i; + coder->opts[rep_len].prev_1_is_literal = false; + } + } while (--rep_len >= 2); + } + + + const uint32_t normal_match_price = match_price + + rc_bit_0_price(coder->is_rep[coder->state]); + + len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; + if (len <= len_main) { + uint32_t i = 0; + while (len > coder->matches[i].len) + ++i; + + for(; ; ++len) { + const uint32_t dist = coder->matches[i].dist; + const uint32_t cur_and_len_price = normal_match_price + + get_pos_len_price(coder, + dist, len, pos_state); + + if (cur_and_len_price < coder->opts[len].price) { + coder->opts[len].price = cur_and_len_price; + coder->opts[len].pos_prev = 0; + coder->opts[len].back_prev + = dist + REP_DISTANCES; + coder->opts[len].prev_1_is_literal = false; + } + + if (len == coder->matches[i].len) + if (++i == matches_count) + break; + } + } + + return len_end; +} + + +static inline uint32_t +helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, + uint32_t len_end, uint32_t position, const uint32_t cur, + const uint32_t fast_bytes, const uint32_t buf_avail_full) +{ + uint32_t matches_count = coder->matches_count; + uint32_t new_len = coder->longest_match_length; + uint32_t pos_prev = coder->opts[cur].pos_prev; + uint32_t state; + + if (coder->opts[cur].prev_1_is_literal) { + --pos_prev; + + if (coder->opts[cur].prev_2) { + state = coder->opts[coder->opts[cur].pos_prev_2].state; + + if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + update_long_rep(state); + else + update_match(state); + + } else { + state = coder->opts[pos_prev].state; + } + + update_literal(state); + + } else { + state = coder->opts[pos_prev].state; + } + + if (pos_prev == cur - 1) { + if (is_short_rep(coder->opts[cur])) + update_short_rep(state); + else + update_literal(state); + } else { + uint32_t pos; + if (coder->opts[cur].prev_1_is_literal + && coder->opts[cur].prev_2) { + pos_prev = coder->opts[cur].pos_prev_2; + pos = coder->opts[cur].back_prev_2; + update_long_rep(state); + } else { + pos = coder->opts[cur].back_prev; + if (pos < REP_DISTANCES) + update_long_rep(state); + else + update_match(state); + } + + if (pos < REP_DISTANCES) { + reps[0] = coder->opts[pos_prev].backs[pos]; + + uint32_t i; + for (i = 1; i <= pos; ++i) + reps[i] = coder->opts[pos_prev].backs[i - 1]; + + for (; i < REP_DISTANCES; ++i) + reps[i] = coder->opts[pos_prev].backs[i]; + + } else { + reps[0] = pos - REP_DISTANCES; + + for (uint32_t i = 1; i < REP_DISTANCES; ++i) + reps[i] = coder->opts[pos_prev].backs[i - 1]; + } + } + + coder->opts[cur].state = state; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->opts[cur].backs[i] = reps[i]; + + const uint32_t cur_price = coder->opts[cur].price; + + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); + + const uint32_t pos_state = position & coder->pos_mask; + + const uint32_t cur_and_1_price = cur_price + + rc_bit_0_price(coder->is_match[state][pos_state]) + + get_literal_price(coder, position, buf[-1], + !is_literal_state(state), match_byte, current_byte); + + bool next_is_literal = false; + + if (cur_and_1_price < coder->opts[cur + 1].price) { + coder->opts[cur + 1].price = cur_and_1_price; + coder->opts[cur + 1].pos_prev = cur; + make_literal(&coder->opts[cur + 1]); + next_is_literal = true; + } + + const uint32_t match_price = cur_price + + rc_bit_1_price(coder->is_match[state][pos_state]); + const uint32_t rep_match_price = match_price + + rc_bit_1_price(coder->is_rep[state]); + + if (match_byte == current_byte + && !(coder->opts[cur + 1].pos_prev < cur + && coder->opts[cur + 1].back_prev == 0)) { + + const uint32_t short_rep_price = rep_match_price + + get_short_rep_price(coder, state, pos_state); + + if (short_rep_price <= coder->opts[cur + 1].price) { + coder->opts[cur + 1].price = short_rep_price; + coder->opts[cur + 1].pos_prev = cur; + make_short_rep(&coder->opts[cur + 1]); + next_is_literal = true; + } + } + + if (buf_avail_full < 2) + return len_end; + + const uint32_t buf_avail = MIN(buf_avail_full, fast_bytes); + + if (!next_is_literal && match_byte != current_byte) { // speed optimization + // try literal + rep0 + const uint8_t *const buf_back = buf - reps[0] - 1; + const uint32_t limit = MIN(buf_avail_full, fast_bytes + 1); + + uint32_t len_test = 1; + while (len_test < limit && buf[len_test] == buf_back[len_test]) + ++len_test; + + --len_test; + + if (len_test >= 2) { + uint32_t state_2 = state; + update_literal(state_2); + + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + //for (; len_test >= 2; --len_test) { + const uint32_t offset = cur + 1 + len_test; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = false; + } + //} + } + } + + + uint32_t start_len = 2; // speed optimization + + for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { + const uint8_t *const buf_back = buf - reps[rep_index] - 1; + if (not_equal_16(buf, buf_back)) + continue; + + uint32_t len_test; + for (len_test = 2; len_test < buf_avail + && buf[len_test] == buf_back[len_test]; + ++len_test) ; + + while (len_end < cur + len_test) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( + coder, rep_index, state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + get_len_price(&coder->rep_len_encoder, + len_test, pos_state); + + if (cur_and_len_price < coder->opts[cur + len_test].price) { + coder->opts[cur + len_test].price = cur_and_len_price; + coder->opts[cur + len_test].pos_prev = cur; + coder->opts[cur + len_test].back_prev = rep_index; + coder->opts[cur + len_test].prev_1_is_literal = false; + } + } while (--len_test >= 2); + + len_test = len_test_temp; + + if (rep_index == 0) + start_len = len_test + 1; + + + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(buf_avail_full, + len_test_2 + fast_bytes); + for (; len_test_2 < limit + && buf[len_test_2] == buf_back[len_test_2]; + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_long_rep(state_2); + + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; + + const uint32_t cur_and_len_literal_price = price + + get_len_price(&coder->rep_len_encoder, + len_test, pos_state) + + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) + + get_literal_price(coder, position + len_test, + buf[len_test - 1], true, + buf_back[len_test], buf[len_test]); + + update_literal(state_2); + + pos_state_next = (position + len_test + 1) & coder->pos_mask; + + const uint32_t next_rep_match_price = cur_and_len_literal_price + + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + //for(; len_test_2 >= 2; len_test_2--) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test_2, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + len_test + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = true; + coder->opts[offset].pos_prev_2 = cur; + coder->opts[offset].back_prev_2 = rep_index; + } + //} + } + } + + + //for (uint32_t len_test = 2; len_test <= new_len; ++len_test) + if (new_len > buf_avail) { + new_len = buf_avail; + + matches_count = 0; + while (new_len > coder->matches[matches_count].len) + ++matches_count; + + coder->matches[matches_count++].len = new_len; + } + + + if (new_len >= start_len) { + const uint32_t normal_match_price = match_price + + rc_bit_0_price(coder->is_rep[state]); + + while (len_end < cur + new_len) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + uint32_t i = 0; + while (start_len > coder->matches[i].len) + ++i; + + for (uint32_t len_test = start_len; ; ++len_test) { + const uint32_t cur_back = coder->matches[i].dist; + uint32_t cur_and_len_price = normal_match_price + + get_pos_len_price(coder, + cur_back, len_test, pos_state); + + if (cur_and_len_price < coder->opts[cur + len_test].price) { + coder->opts[cur + len_test].price = cur_and_len_price; + coder->opts[cur + len_test].pos_prev = cur; + coder->opts[cur + len_test].back_prev + = cur_back + REP_DISTANCES; + coder->opts[cur + len_test].prev_1_is_literal = false; + } + + if (len_test == coder->matches[i].len) { + // Try Match + Literal + Rep0 + const uint8_t *const buf_back = buf - cur_back - 1; + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(buf_avail_full, + len_test_2 + fast_bytes); + + for (; len_test_2 < limit && + buf[len_test_2] == buf_back[len_test_2]; + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_match(state_2); + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; + + const uint32_t cur_and_len_literal_price = cur_and_len_price + + rc_bit_0_price( + coder->is_match[state_2][pos_state_next]) + + get_literal_price(coder, + position + len_test, + buf[len_test - 1], + true, + buf_back[len_test], + buf[len_test]); + + update_literal(state_2); + pos_state_next = (pos_state_next + 1) & coder->pos_mask; + + const uint32_t next_rep_match_price + = cur_and_len_literal_price + + rc_bit_1_price( + coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + // for(; len_test_2 >= 2; --len_test_2) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test_2, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + len_test + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = true; + coder->opts[offset].pos_prev_2 = cur; + coder->opts[offset].back_prev_2 + = cur_back + REP_DISTANCES; + } + //} + } + + if (++i == matches_count) + break; + } + } + } + + return len_end; +} + + +extern void +lzma_lzma_optimum_normal(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, + uint32_t position) +{ + // If we have symbols pending, return the next pending symbol. + if (coder->opts_end_index != coder->opts_current_index) { + assert(mf->read_ahead > 0); + *len_res = coder->opts[coder->opts_current_index].pos_prev + - coder->opts_current_index; + *back_res = coder->opts[coder->opts_current_index].back_prev; + coder->opts_current_index = coder->opts[ + coder->opts_current_index].pos_prev; + return; + } + + // Update the price tables. In LZMA SDK <= 4.60 (and possibly later) + // this was done in both initialization function and in the main loop. + // In liblzma they were moved into this single place. + if (mf->read_ahead == 0) { + if (coder->match_price_count >= (1 << 7)) + fill_distances_prices(coder); + + if (coder->align_price_count >= ALIGN_TABLE_SIZE) + fill_align_prices(coder); + } + + // TODO: This needs quite a bit of cleaning still. But splitting + // the oroginal function to two pieces makes it at least a little + // more readable, since those two parts don't share many variables. + + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); + if (len_end == UINT32_MAX) + return; + + uint32_t reps[REP_DISTANCES]; + memcpy(reps, coder->reps, sizeof(reps)); + + uint32_t cur; + for (cur = 1; cur < len_end; ++cur) { + assert(cur < OPTS); + + coder->longest_match_length = mf_find( + mf, &coder->matches_count, coder->matches); + + if (coder->longest_match_length >= mf->find_len_max) + break; + + len_end = helper2(coder, reps, mf_ptr(mf) - 1, len_end, + position + cur, cur, mf->find_len_max, + MIN(mf_avail(mf) + 1, OPTS - 1 - cur)); + } + + backward(coder, len_res, back_res, cur); + return; +} diff --git a/src/liblzma/lzma/lzma_encoder_presets.c b/src/liblzma/lzma/lzma_encoder_presets.c index 966c7c86..08f339e9 100644 --- a/src/liblzma/lzma/lzma_encoder_presets.c +++ b/src/liblzma/lzma/lzma_encoder_presets.c @@ -20,15 +20,47 @@ #include "common.h" +#define pow2(e) (UINT32_C(1) << (e)) + + LZMA_API const lzma_options_lzma lzma_preset_lzma[9] = { -// dictionary_size lc lp pb mode fb mf mfc -{ UINT32_C(1) << 16, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0 }, -{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0 }, -{ UINT32_C(1) << 19, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 21, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 22, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 23, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 24, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 25, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, +// dict lc lp pb mode fb mf mfc +{ pow2(16), NULL, 0, 3, 0, 2, false, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(19), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(21), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(22), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(23), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(24), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(25), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, }; + + +/* +extern LZMA_API lzma_bool +lzma_preset_lzma(lzma_options_lzma *options, uint32_t level) +{ + *options = (lzma_options_lzma){ + + }; + + options->literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT + options->literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT; + options->pos_bits = LZMA_POS_BITS_DEFAULT; + options->preset_dictionary = NULL; + options->preset_dictionary_size = 0; + options->persistent = false; + + options->mode = level <= 2 ? LZMA_MODE_FAST : LZMA_MODE_NORMAL; + options->fast_bytes = level <= + + options->match_finder = level == 1 ? LZMA_MF_HC3 + : (level == 2 ? LZMA_MF_HC4 : LZMA_MF_BT4); + options->match_finder_cycles = 0; + + + + options->dictionary_size = +} +*/ diff --git a/src/liblzma/lzma/lzma_encoder_private.h b/src/liblzma/lzma/lzma_encoder_private.h index a16051f8..7533bc79 100644 --- a/src/liblzma/lzma/lzma_encoder_private.h +++ b/src/liblzma/lzma/lzma_encoder_private.h @@ -21,20 +21,27 @@ #ifndef LZMA_LZMA_ENCODER_PRIVATE_H #define LZMA_LZMA_ENCODER_PRIVATE_H -#include "lzma_encoder.h" -#include "lzma_common.h" #include "lz_encoder.h" #include "range_encoder.h" +#include "lzma_common.h" +#include "lzma_encoder.h" + +// Macro to compare if the first two bytes in two buffers differ. This is +// needed in lzma_lzma_optimum_*() to test if the match is at least +// MATCH_LEN_MIN bytes. Unaligned access gives tiny gain so there's no +// reason to not use it when it is supported. +#ifdef HAVE_FAST_UNALIGNED_ACCESS +# define not_equal_16(a, b) \ + (*(const uint16_t *)(a) != *(const uint16_t *)(b)) +#else +# define not_equal_16(a, b) \ + ((a)[0] != (b)[0] || (a)[1] != (b)[1]) +#endif -#define move_pos(num) \ -do { \ - assert((int32_t)(num) >= 0); \ - if ((num) != 0) { \ - coder->additional_offset += num; \ - coder->lz.skip(&coder->lz, num); \ - } \ -} while (0) + +// Optimal - Number of entries in the optimum array. +#define OPTS (1 << 12) typedef struct { @@ -54,7 +61,7 @@ typedef struct { typedef struct { lzma_lzma_state state; - bool prev_1_is_char; + bool prev_1_is_literal; bool prev_2; uint32_t pos_prev_2; @@ -70,132 +77,79 @@ typedef struct { struct lzma_coder_s { - // Next coder in the chain - lzma_next_coder next; - - // In window and match finder - lzma_lz_encoder lz; - - // Range encoder + /// Range encoder lzma_range_encoder rc; - // State + /// State lzma_lzma_state state; - uint8_t previous_byte; + + /// The four most recent match distances uint32_t reps[REP_DISTANCES]; - // Misc - uint32_t match_distances[MATCH_MAX_LEN * 2 + 2 + 1]; - uint32_t num_distance_pairs; - uint32_t additional_offset; - uint32_t now_pos; // Lowest 32 bits are enough here. - bool best_compression; ///< True when LZMA_MODE_BEST is used + /// Array of match candidates + lzma_match matches[MATCH_LEN_MAX + 1]; + + /// Number of match candidates in matches[] + uint32_t matches_count; + + /// Varibale to hold the length of the longest match between calls + /// to lzma_lzma_optimum_*(). + uint32_t longest_match_length; + + /// True if using getoptimumfast + bool fast_mode; + + /// True if the encoder has been initialized by encoding the first + /// byte as a literal. bool is_initialized; + + /// True if the range encoder has been flushed, but not all bytes + /// have been written to the output buffer yet. bool is_flushed; - bool write_eopm; - // Literal encoder - lzma_literal_coder literal_coder; + uint32_t pos_mask; ///< (1 << pos_bits) - 1 + uint32_t literal_context_bits; + uint32_t literal_pos_mask; - // Bit encoders + // These are the same as in lzma_decoder.c. See comments there. + probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; probability is_match[STATES][POS_STATES_MAX]; probability is_rep[STATES]; probability is_rep0[STATES]; probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_encoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_align[ALIGN_TABLE_SIZE]; - // Bit Tree Encoders - probability pos_slot_encoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; - probability pos_align_encoder[1 << ALIGN_BITS]; - - // Length encoders + // These are the same as in lzma_decoder.c except that the encoders + // include also price tables. lzma_length_encoder match_len_encoder; lzma_length_encoder rep_len_encoder; - lzma_length_encoder *prev_len_encoder; - // Optimal - lzma_optimal optimum[OPTS]; - uint32_t optimum_end_index; - uint32_t optimum_current_index; - uint32_t longest_match_length; - bool longest_match_was_found; - - // Prices - uint32_t pos_slot_prices[LEN_TO_POS_STATES][DIST_TABLE_SIZE_MAX]; + // Price tables + uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; - uint32_t align_prices[ALIGN_TABLE_SIZE]; - uint32_t align_price_count; uint32_t dist_table_size; uint32_t match_price_count; - // LZMA specific settings - uint32_t dictionary_size; ///< Size in bytes - uint32_t fast_bytes; - uint32_t pos_state_bits; - uint32_t pos_mask; ///< (1 << pos_state_bits) - 1 -}; - - -extern void lzma_length_encoder_update_table(lzma_length_encoder *lencoder, - const uint32_t pos_state); + uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_price_count; -extern bool lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size); + // Optimal + uint32_t opts_end_index; + uint32_t opts_current_index; + lzma_optimal opts[OPTS]; +}; -extern void lzma_get_optimum(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_get_optimum_fast(lzma_coder *restrict coder, +extern void lzma_lzma_optimum_fast( + lzma_coder *restrict coder, lzma_mf *restrict mf, uint32_t *restrict back_res, uint32_t *restrict len_res); - -// NOTE: Don't add 'restrict'. -static inline void -lzma_read_match_distances(lzma_coder *coder, - uint32_t *len_res, uint32_t *num_distance_pairs) -{ - *len_res = 0; - - coder->lz.get_matches(&coder->lz, coder->match_distances); - - *num_distance_pairs = coder->match_distances[0]; - - if (*num_distance_pairs > 0) { - *len_res = coder->match_distances[*num_distance_pairs - 1]; - assert(*len_res <= MATCH_MAX_LEN); - - if (*len_res == coder->fast_bytes) { - uint32_t offset = *len_res - 1; - const uint32_t distance = coder->match_distances[ - *num_distance_pairs] + 1; - uint32_t limit = MATCH_MAX_LEN - *len_res; - - assert(offset + limit < coder->lz.keep_size_after); - assert(coder->lz.read_pos <= coder->lz.write_pos); - - // If we are close to end of the stream, we may need - // to limit the length of the match. - if (coder->lz.write_pos - coder->lz.read_pos - < offset + limit) - limit = coder->lz.write_pos - - (coder->lz.read_pos + offset); - - offset += coder->lz.read_pos; - uint32_t i = 0; - while (i < limit && coder->lz.buffer[offset + i] - == coder->lz.buffer[ - offset + i - distance]) - ++i; - - *len_res += i; - } - } - - ++coder->additional_offset; - - return; -} +extern void lzma_lzma_optimum_normal(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/src/liblzma/lzma/lzma_literal.c b/src/liblzma/lzma/lzma_literal.c deleted file mode 100644 index 3611a1f7..00000000 --- a/src/liblzma/lzma/lzma_literal.c +++ /dev/null @@ -1,51 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_literal.c -/// \brief Literal Coder -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lzma_literal.h" - - -extern lzma_ret -lzma_literal_init(lzma_literal_coder *coder, - uint32_t literal_context_bits, uint32_t literal_pos_bits) -{ - // Verify that arguments are sane. - if (literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX - || literal_pos_bits > LZMA_LITERAL_POS_BITS_MAX) - return LZMA_HEADER_ERROR; - - // Calculate the number of states the literal coder must store. - const uint32_t states = literal_states( - literal_pos_bits, literal_context_bits); - - // Store the new settings. - coder->literal_context_bits = literal_context_bits; - coder->literal_pos_bits = literal_pos_bits; - - // Calculate also the literal_pos_mask. It's not changed - // anywhere else than here. - coder->literal_pos_mask = (1 << literal_pos_bits) - 1; - - // Reset the literal coder. - for (uint32_t i = 0; i < states; ++i) - for (uint32_t j = 0; j < LIT_SIZE; ++j) - bit_reset(coder->coders[i][j]); - - return LZMA_OK; -} diff --git a/src/liblzma/lzma/lzma_literal.h b/src/liblzma/lzma/lzma_literal.h deleted file mode 100644 index 208abd99..00000000 --- a/src/liblzma/lzma/lzma_literal.h +++ /dev/null @@ -1,71 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_literal.h -/// \brief Literal Coder -/// -/// This is used as is by both LZMA encoder and decoder. -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_LITERAL_H -#define LZMA_LITERAL_H - -#include "common.h" - -// We need typedef of `probability'. -#include "range_common.h" - - -/// Each literal coder is divided in three sections: -/// - 0x001-0x0FF: Without match byte -/// - 0x101-0x1FF: With match byte; match bit is 0 -/// - 0x201-0x2FF: With match byte; match bit is 1 -#define LIT_SIZE 0x300 - -/// Calculate how many states are needed. Each state has -/// LIT_SIZE `probability' variables. -#define literal_states(literal_context_bits, literal_pos_bits) \ - (1U << ((literal_context_bits) + (literal_pos_bits))) - -/// Locate the literal coder for the next literal byte. The choice depends on -/// - the lowest literal_pos_bits bits of the position of the current -/// byte; and -/// - the highest literal_context_bits bits of the previous byte. -#define literal_get_subcoder(literal_coder, pos, prev_byte) \ - (literal_coder).coders[(((pos) & (literal_coder).literal_pos_mask) \ - << (literal_coder).literal_context_bits) \ - + ((prev_byte) >> (8 - (literal_coder).literal_context_bits))] - - -typedef struct { - uint32_t literal_context_bits; - uint32_t literal_pos_bits; - - /// literal_pos_mask is always (1 << literal_pos_bits) - 1. - uint32_t literal_pos_mask; - - /// There are (1 << (literal_pos_bits + literal_context_bits)) - /// literal coders. - probability coders[1 << LZMA_LITERAL_BITS_MAX][LIT_SIZE]; - -} lzma_literal_coder; - - -extern lzma_ret lzma_literal_init( - lzma_literal_coder *coder, - uint32_t literal_context_bits, uint32_t literal_pos_bits); - -#endif diff --git a/src/liblzma/rangecoder/Makefile.am b/src/liblzma/rangecoder/Makefile.am index 6e80f8d7..f6824292 100644 --- a/src/liblzma/rangecoder/Makefile.am +++ b/src/liblzma/rangecoder/Makefile.am @@ -12,7 +12,7 @@ ## Lesser General Public License for more details. ## -EXTRA_DIST = price_table_gen.c +EXTRA_DIST = price_tablegen.c noinst_LTLIBRARIES = librangecoder.la @@ -21,8 +21,10 @@ librangecoder_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common -if COND_MAIN_ENCODER -librangecoder_la_SOURCES += range_encoder.h +if COND_ENCODER_LZMA +librangecoder_la_SOURCES += \ + range_encoder.h \ + price.h if COND_SMALL librangecoder_la_SOURCES += price_table_init.c else @@ -30,6 +32,6 @@ librangecoder_la_SOURCES += price_table.c endif endif -if COND_MAIN_DECODER +if COND_DECODER_LZMA librangecoder_la_SOURCES += range_decoder.h endif diff --git a/src/liblzma/rangecoder/price.h b/src/liblzma/rangecoder/price.h new file mode 100644 index 00000000..001f753d --- /dev/null +++ b/src/liblzma/rangecoder/price.h @@ -0,0 +1,111 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file price.h +/// \brief Probability price calculation +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_PRICE_H +#define LZMA_PRICE_H + + +#define RC_MOVE_REDUCING_BITS 4 +#define RC_BIT_PRICE_SHIFT_BITS 4 +#define RC_PRICE_TABLE_SIZE (RC_BIT_MODEL_TOTAL >> RC_MOVE_REDUCING_BITS) + +#define RC_INFINITY_PRICE (UINT32_C(1) << 30) + + +#if !defined(LZMA_RANGE_ENCODER_H) || defined(HAVE_SMALL) +/// Probability prices used by *_get_price() macros. This is initialized +/// by lzma_rc_init() and is not modified later. +extern uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; + +/// Initializes lzma_rc_prices[]. This needs to be called only once. +extern void lzma_rc_init(void); + +#else +// Not building a size optimized version, so we use a precomputed +// constant table. +extern const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; + +#endif + + +static inline uint32_t +rc_bit_price(const probability prob, const uint32_t bit) +{ + return lzma_rc_prices[(prob ^ ((UINT32_C(0) - bit) + & (RC_BIT_MODEL_TOTAL - 1))) >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bit_0_price(const probability prob) +{ + return lzma_rc_prices[prob >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bit_1_price(const probability prob) +{ + return lzma_rc_prices[(prob ^ (RC_BIT_MODEL_TOTAL - 1)) + >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bittree_price(const probability *const probs, + const uint32_t bit_levels, uint32_t symbol) +{ + uint32_t price = 0; + symbol += UINT32_C(1) << bit_levels; + + do { + const uint32_t bit = symbol & 1; + symbol >>= 1; + price += rc_bit_price(probs[symbol], bit); + } while (symbol != 1); + + return price; +} + + +static inline uint32_t +rc_bittree_reverse_price(const probability *const probs, + uint32_t bit_levels, uint32_t symbol) +{ + uint32_t price = 0; + uint32_t model_index = 1; + + do { + const uint32_t bit = symbol & 1; + symbol >>= 1; + price += rc_bit_price(probs[model_index], bit); + model_index = (model_index << 1) + bit; + } while (--bit_levels != 0); + + return price; +} + + +static inline uint32_t +rc_direct_price(const uint32_t bits) +{ + return bits << RC_BIT_PRICE_SHIFT_BITS; +} + +#endif diff --git a/src/liblzma/rangecoder/price_table.c b/src/liblzma/rangecoder/price_table.c index d0b50fa6..539206b1 100644 --- a/src/liblzma/rangecoder/price_table.c +++ b/src/liblzma/rangecoder/price_table.c @@ -1,70 +1,22 @@ -/* This file has been automatically generated by price_table_gen.c. */ +/* This file has been automatically generated by price_tablegen.c. */ #include "range_encoder.h" -const uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = { - 0, 576, 512, 480, 448, 432, 416, 400, - 384, 376, 368, 360, 352, 344, 336, 328, - 320, 316, 312, 308, 304, 300, 296, 292, - 288, 284, 280, 276, 272, 268, 264, 260, - 256, 254, 252, 250, 248, 246, 244, 242, - 240, 238, 236, 234, 232, 230, 228, 226, - 224, 222, 220, 218, 216, 214, 212, 210, - 208, 206, 204, 202, 200, 198, 196, 194, - 192, 191, 190, 189, 188, 187, 186, 185, - 184, 183, 182, 181, 180, 179, 178, 177, - 176, 175, 174, 173, 172, 171, 170, 169, - 168, 167, 166, 165, 164, 163, 162, 161, - 160, 159, 158, 157, 156, 155, 154, 153, - 152, 151, 150, 149, 148, 147, 146, 145, - 144, 143, 142, 141, 140, 139, 138, 137, - 136, 135, 134, 133, 132, 131, 130, 129, - 128, 127, 127, 126, 126, 125, 125, 124, - 124, 123, 123, 122, 122, 121, 121, 120, - 120, 119, 119, 118, 118, 117, 117, 116, - 116, 115, 115, 114, 114, 113, 113, 112, - 112, 111, 111, 110, 110, 109, 109, 108, - 108, 107, 107, 106, 106, 105, 105, 104, - 104, 103, 103, 102, 102, 101, 101, 100, - 100, 99, 99, 98, 98, 97, 97, 96, - 96, 95, 95, 94, 94, 93, 93, 92, - 92, 91, 91, 90, 90, 89, 89, 88, - 88, 87, 87, 86, 86, 85, 85, 84, - 84, 83, 83, 82, 82, 81, 81, 80, - 80, 79, 79, 78, 78, 77, 77, 76, - 76, 75, 75, 74, 74, 73, 73, 72, - 72, 71, 71, 70, 70, 69, 69, 68, - 68, 67, 67, 66, 66, 65, 65, 64, - 64, 63, 63, 63, 63, 62, 62, 62, - 62, 61, 61, 61, 61, 60, 60, 60, - 60, 59, 59, 59, 59, 58, 58, 58, - 58, 57, 57, 57, 57, 56, 56, 56, - 56, 55, 55, 55, 55, 54, 54, 54, - 54, 53, 53, 53, 53, 52, 52, 52, - 52, 51, 51, 51, 51, 50, 50, 50, - 50, 49, 49, 49, 49, 48, 48, 48, - 48, 47, 47, 47, 47, 46, 46, 46, - 46, 45, 45, 45, 45, 44, 44, 44, - 44, 43, 43, 43, 43, 42, 42, 42, - 42, 41, 41, 41, 41, 40, 40, 40, - 40, 39, 39, 39, 39, 38, 38, 38, - 38, 37, 37, 37, 37, 36, 36, 36, - 36, 35, 35, 35, 35, 34, 34, 34, - 34, 33, 33, 33, 33, 32, 32, 32, - 32, 31, 31, 31, 31, 30, 30, 30, - 30, 29, 29, 29, 29, 28, 28, 28, - 28, 27, 27, 27, 27, 26, 26, 26, - 26, 25, 25, 25, 25, 24, 24, 24, - 24, 23, 23, 23, 23, 22, 22, 22, - 22, 21, 21, 21, 21, 20, 20, 20, - 20, 19, 19, 19, 19, 18, 18, 18, - 18, 17, 17, 17, 17, 16, 16, 16, - 16, 15, 15, 15, 15, 14, 14, 14, - 14, 13, 13, 13, 13, 12, 12, 12, - 12, 11, 11, 11, 11, 10, 10, 10, - 10, 9, 9, 9, 9, 8, 8, 8, - 8, 7, 7, 7, 7, 6, 6, 6, - 6, 5, 5, 5, 5, 4, 4, 4, - 4, 3, 3, 3, 3, 2, 2, 2, - 2, 1, 1, 1, 1, 0, 0, 0 +const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE] = { + 128, 103, 91, 84, 78, 73, 69, 66, + 63, 61, 58, 56, 54, 52, 51, 49, + 48, 46, 45, 44, 43, 42, 41, 40, + 39, 38, 37, 36, 35, 34, 34, 33, + 32, 31, 31, 30, 29, 29, 28, 28, + 27, 26, 26, 25, 25, 24, 24, 23, + 23, 22, 22, 22, 21, 21, 20, 20, + 19, 19, 19, 18, 18, 17, 17, 17, + 16, 16, 16, 15, 15, 15, 14, 14, + 14, 13, 13, 13, 12, 12, 12, 11, + 11, 11, 11, 10, 10, 10, 10, 9, + 9, 9, 9, 8, 8, 8, 8, 7, + 7, 7, 7, 6, 6, 6, 6, 5, + 5, 5, 5, 5, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 2, 2, 2, + 2, 2, 2, 1, 1, 1, 1, 1 }; diff --git a/src/liblzma/rangecoder/price_table_init.c b/src/liblzma/rangecoder/price_table_init.c index 4714dfd6..9c7d799b 100644 --- a/src/liblzma/rangecoder/price_table_init.c +++ b/src/liblzma/rangecoder/price_table_init.c @@ -23,25 +23,32 @@ #endif -#define NUM_BITS (BIT_MODEL_TOTAL_BITS - MOVE_REDUCING_BITS) - - -uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; +uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; extern void lzma_rc_init(void) { - // Initialize lzma_rc_prob_prices[]. - for (int i = NUM_BITS - 1; i >= 0; --i) { - const uint32_t start = 1 << (NUM_BITS - i - 1); - const uint32_t end = 1 << (NUM_BITS - i); - - for (uint32_t j = start; j < end; ++j) { - lzma_rc_prob_prices[j] = (i << BIT_PRICE_SHIFT_BITS) - + (((end - j) << BIT_PRICE_SHIFT_BITS) - >> (NUM_BITS - i - 1)); + for (uint32_t i = (UINT32_C(1) << RC_MOVE_REDUCING_BITS) / 2; + i < RC_BIT_MODEL_TOTAL; + i += (UINT32_C(1) << RC_MOVE_REDUCING_BITS)) { + const uint32_t cycles_bits = RC_BIT_PRICE_SHIFT_BITS; + uint32_t w = i; + uint32_t bit_count = 0; + + for (uint32_t j = 0; j < cycles_bits; ++j) { + w *= w; + bit_count <<= 1; + + while (w >= (UINT32_C(1) << 16)) { + w >>= 1; + ++bit_count; + } } + + lzma_rc_prices[i >> RC_MOVE_REDUCING_BITS] + = (RC_BIT_MODEL_TOTAL_BITS << cycles_bits) + - 15 - bit_count; } return; diff --git a/src/liblzma/rangecoder/price_table_gen.c b/src/liblzma/rangecoder/price_tablegen.c index 946d8215..68513635 100644 --- a/src/liblzma/rangecoder/price_table_gen.c +++ b/src/liblzma/rangecoder/price_tablegen.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file price_table_gen.c +/// \file price_tablegen.c /// \brief Probability price table generator /// -/// Compiling: gcc -std=c99 -o price_table_gen price_table_gen.c +/// Compiling: gcc -std=c99 -o price_tablegen price_tablegen.c // // Copyright (C) 2007 Lasse Collin // @@ -19,10 +19,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include <sys/types.h> +#include <stddef.h> #include <inttypes.h> #include <stdio.h> #include "range_common.h" +#include "price.h" #include "price_table_init.c" @@ -32,18 +33,18 @@ main(void) lzma_rc_init(); printf("/* This file has been automatically generated by " - "price_table_gen.c. */\n\n" + "price_tablegen.c. */\n\n" "#include \"range_encoder.h\"\n\n" - "const uint32_t lzma_rc_prob_prices[" - "BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = {"); + "const uint32_t lzma_rc_prices[" + "RC_PRICE_TABLE_SIZE] = {"); - const size_t array_size = sizeof(lzma_rc_prob_prices) - / sizeof(lzma_rc_prob_prices[0]); + const size_t array_size = sizeof(lzma_rc_prices) + / sizeof(lzma_rc_prices[0]); for (size_t i = 0; i < array_size; ++i) { if (i % 8 == 0) printf("\n\t"); - printf("%4" PRIu32, lzma_rc_prob_prices[i]); + printf("%4" PRIu32, lzma_rc_prices[i]); if (i != array_size - 1) printf(","); diff --git a/src/liblzma/rangecoder/range_common.h b/src/liblzma/rangecoder/range_common.h index 7613621a..6e5b0994 100644 --- a/src/liblzma/rangecoder/range_common.h +++ b/src/liblzma/rangecoder/range_common.h @@ -30,15 +30,12 @@ // Constants // /////////////// -#define SHIFT_BITS 8 -#define TOP_BITS 24 -#define TOP_VALUE (UINT32_C(1) << TOP_BITS) -#define BIT_MODEL_TOTAL_BITS 11 -#define BIT_MODEL_TOTAL (UINT32_C(1) << BIT_MODEL_TOTAL_BITS) -#define MOVE_BITS 5 - -#define MOVE_REDUCING_BITS 2 -#define BIT_PRICE_SHIFT_BITS 6 +#define RC_SHIFT_BITS 8 +#define RC_TOP_BITS 24 +#define RC_TOP_VALUE (UINT32_C(1) << RC_TOP_BITS) +#define RC_BIT_MODEL_TOTAL_BITS 11 +#define RC_BIT_MODEL_TOTAL (UINT32_C(1) << RC_BIT_MODEL_TOTAL_BITS) +#define RC_MOVE_BITS 5 //////////// @@ -47,7 +44,7 @@ // Resets the probability so that both 0 and 1 have probability of 50 % #define bit_reset(prob) \ - prob = BIT_MODEL_TOTAL >> 1 + prob = RC_BIT_MODEL_TOTAL >> 1 // This does the same for a complete bit tree. // (A tree represented as an array.) diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h index 62162448..ca2d392e 100644 --- a/src/liblzma/rangecoder/range_decoder.h +++ b/src/liblzma/rangecoder/range_decoder.h @@ -31,6 +31,7 @@ typedef struct { } lzma_range_decoder; +/// Reads the first five bytes to initialize the range decoder. static inline bool rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) @@ -48,14 +49,22 @@ rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, } -/// Makes local copies of range decoder variables. -#define rc_to_local(range_decoder) \ +/// Makes local copies of range decoder and *in_pos variables. Doing this +/// improves speed significantly. The range decoder macros expect also +/// variables `in' and `in_size' to be defined. +#define rc_to_local(range_decoder, in_pos) \ lzma_range_decoder rc = range_decoder; \ + size_t rc_in_pos = (in_pos); \ uint32_t rc_bound + /// Stores the local copes back to the range decoder structure. -#define rc_from_local(range_decoder) \ - range_decoder = rc +#define rc_from_local(range_decoder, in_pos) \ +do { \ + range_decoder = rc; \ + in_pos = rc_in_pos; \ +} while (0) + /// Resets the range decoder structure. #define rc_reset(range_decoder) \ @@ -66,158 +75,112 @@ do { \ } while (0) -// All of the macros in this file expect the following variables being defined: -// - lzma_range_decoder range_decoder; -// - uint32_t rc_bound; // Temporary variable -// - uint8_t *in; -// - size_t in_pos_local; // Local alias for *in_pos - +/// When decoding has been properly finished, rc.code is always zero unless +/// the input stream is corrupt. So checking this can catch some corrupt +/// files especially if they don't have any other integrity check. +#define rc_is_finished(range_decoder) \ + ((range_decoder).code == 0) -////////////////// -// Buffer "I/O" // -////////////////// -// Read the next byte of compressed data from buffer_in, if needed. -#define rc_normalize() \ +/// Read the next input byte if needed. If more input is needed but there is +/// no more input available, "goto out" is used to jump out of the main +/// decoder loop. +#define rc_normalize(seq) \ do { \ - if (rc.range < TOP_VALUE) { \ - rc.range <<= SHIFT_BITS; \ - rc.code = (rc.code << SHIFT_BITS) | in[in_pos_local++]; \ + if (rc.range < RC_TOP_VALUE) { \ + if (unlikely(rc_in_pos == in_size)) { \ + coder->sequence = seq; \ + goto out; \ + } \ + rc.range <<= RC_SHIFT_BITS; \ + rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \ } \ } while (0) -////////////////// -// Bit decoding // -////////////////// - -// Range decoder's DecodeBit() is splitted into three macros: -// if_bit_0(prob) { -// update_bit_0(prob) -// ... -// } else { -// update_bit_1(prob) -// ... -// } - -#define if_bit_0(prob) \ - rc_normalize(); \ - rc_bound = (rc.range >> BIT_MODEL_TOTAL_BITS) * (prob); \ +/// Start decoding a bit. This must be used together with rc_update_0() +/// and rc_update_1(): +/// +/// rc_if_0(prob, seq) { +/// rc_update_0(prob); +/// // Do something +/// } else { +/// rc_update_1(prob); +/// // Do something else +/// } +/// +#define rc_if_0(prob, seq) \ + rc_normalize(seq); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) -#define update_bit_0(prob) \ +/// Update the range decoder state and the used probability variable to +/// match a decoded bit of 0. +#define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ - prob += (BIT_MODEL_TOTAL - (prob)) >> MOVE_BITS; \ + prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ } while (0) -#define update_bit_1(prob) \ +/// Update the range decoder state and the used probability variable to +/// match a decoded bit of 1. +#define rc_update_1(prob) \ do { \ rc.range -= rc_bound; \ rc.code -= rc_bound; \ - prob -= (prob) >> MOVE_BITS; \ + prob -= (prob) >> RC_MOVE_BITS; \ } while (0) -#define rc_decode_direct(dest, count) \ +/// Decodes one bit and runs action0 or action1 depending on the decoded bit. +/// This macro is used as the last step in bittree reverse decoders since +/// those don't use "symbol" for anything else than indexing the probability +/// arrays. +#define rc_bit_last(prob, action0, action1, seq) \ do { \ - rc_normalize(); \ - rc.range >>= 1; \ - rc.code -= rc.range; \ - rc_bound = UINT32_C(0) - (rc.code >> 31); \ - rc.code += rc.range & rc_bound; \ - dest = (dest << 1) + (rc_bound + 1); \ -} while (--count > 0) + rc_if_0(prob, seq) { \ + rc_update_0(prob); \ + action0; \ + } else { \ + rc_update_1(prob); \ + action1; \ + } \ +} while (0) -// Dummy versions don't update prob or dest. -#define update_bit_0_dummy() \ - rc.range = rc_bound +/// Decodes one bit, updates "symbol", and runs action0 or action1 depending +/// on the decoded bit. +#define rc_bit(prob, action0, action1, seq) \ + rc_bit_last(prob, \ + symbol <<= 1; action0, \ + symbol = (symbol << 1) + 1; action1, \ + seq); -#define update_bit_1_dummy() \ -do { \ - rc.range -= rc_bound; \ - rc.code -= rc_bound; \ -} while (0) +/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled +/// loops more readable because the code isn't littered with "case" +/// statements. On the other hand this also makes it less readable, since +/// spotting the places where the decoder loop may be restarted is less +/// obvious. +#define rc_bit_case(prob, action0, action1, seq) \ + case seq: rc_bit(prob, action0, action1, seq) -#define rc_decode_direct_dummy(count) \ +/// Decode a bit without using a probability. +#define rc_direct(dest, seq) \ do { \ - rc_normalize(); \ + rc_normalize(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ - rc.code += rc.range & (UINT32_C(0) - (rc.code >> 31)); \ -} while (--count > 0) - - -/////////////////////// -// Bit tree decoding // -/////////////////////// - -#define bittree_decode(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0((probs)[model_index]); \ - model_index <<= 1; \ - } else { \ - update_bit_1((probs)[model_index]); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ - target += model_index - (1 << bit_levels); \ -} while (0) - - -#define bittree_reverse_decode(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0((probs)[model_index]); \ - model_index <<= 1; \ - } else { \ - update_bit_1((probs)[model_index]); \ - model_index = (model_index << 1) | 1; \ - target += 1 << bit_index; \ - } \ - } \ -} while (0) - - -// Dummy versions don't update prob. -#define bittree_decode_dummy(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0_dummy(); \ - model_index <<= 1; \ - } else { \ - update_bit_1_dummy(); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ - target += model_index - (1 << bit_levels); \ + rc_bound = UINT32_C(0) - (rc.code >> 31); \ + rc.code += rc.range & rc_bound; \ + dest = (dest << 1) + (rc_bound + 1); \ } while (0) -#define bittree_reverse_decode_dummy(probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0_dummy(); \ - model_index <<= 1; \ - } else { \ - update_bit_1_dummy(); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ -} while (0) +// NOTE: No macros are provided for bittree decoding. It seems to be simpler +// to just write them open in the code. #endif diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h index b156ee7f..f66e955c 100644 --- a/src/liblzma/rangecoder/range_encoder.h +++ b/src/liblzma/rangecoder/range_encoder.h @@ -22,6 +22,7 @@ #define LZMA_RANGE_ENCODER_H #include "range_common.h" +#include "price.h" /// Maximum number of symbols that can be put pending into lzma_range_encoder @@ -87,7 +88,7 @@ rc_bittree(lzma_range_encoder *rc, probability *probs, do { const uint32_t bit = (symbol >> --bit_count) & 1; rc_bit(rc, &probs[model_index], bit); - model_index = (model_index << 1) | bit; + model_index = (model_index << 1) + bit; } while (bit_count != 0); } @@ -102,7 +103,7 @@ rc_bittree_reverse(lzma_range_encoder *rc, probability *probs, const uint32_t bit = symbol & 1; symbol >>= 1; rc_bit(rc, &probs[model_index], bit); - model_index = (model_index << 1) | bit; + model_index = (model_index << 1) + bit; } while (--bit_count != 0); } @@ -146,7 +147,7 @@ rc_shift_low(lzma_range_encoder *rc, } ++rc->cache_size; - rc->low = (rc->low & 0x00FFFFFF) << SHIFT_BITS; + rc->low = (rc->low & 0x00FFFFFF) << RC_SHIFT_BITS; return false; } @@ -156,32 +157,35 @@ static inline bool rc_encode(lzma_range_encoder *rc, uint8_t *out, size_t *out_pos, size_t out_size) { + assert(rc->count <= RC_SYMBOLS_MAX); + while (rc->pos < rc->count) { // Normalize - if (rc->range < TOP_VALUE) { + if (rc->range < RC_TOP_VALUE) { if (rc_shift_low(rc, out, out_pos, out_size)) return true; - rc->range <<= SHIFT_BITS; + rc->range <<= RC_SHIFT_BITS; } // Encode a bit switch (rc->symbols[rc->pos]) { case RC_BIT_0: { probability prob = *rc->probs[rc->pos]; - rc->range = (rc->range >> BIT_MODEL_TOTAL_BITS) * prob; - prob += (BIT_MODEL_TOTAL - prob) >> MOVE_BITS; + rc->range = (rc->range >> RC_BIT_MODEL_TOTAL_BITS) + * prob; + prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS; *rc->probs[rc->pos] = prob; break; } case RC_BIT_1: { probability prob = *rc->probs[rc->pos]; - const uint32_t bound = prob - * (rc->range >> BIT_MODEL_TOTAL_BITS); + const uint32_t bound = prob * (rc->range + >> RC_BIT_MODEL_TOTAL_BITS); rc->low += bound; rc->range -= bound; - prob -= prob >> MOVE_BITS; + prob -= prob >> RC_MOVE_BITS; *rc->probs[rc->pos] = prob; break; } @@ -231,72 +235,4 @@ rc_pending(const lzma_range_encoder *rc) return rc->cache_size + 5 - 1; } - -//////////// -// Prices // -//////////// - -#ifdef HAVE_SMALL -/// Probability prices used by *_get_price() macros. This is initialized -/// by lzma_rc_init() and is not modified later. -extern uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; - -/// Initializes lzma_rc_prob_prices[]. This needs to be called only once. -extern void lzma_rc_init(void); - -#else -// Not building a size optimized version, so we use a precomputed -// constant table. -extern const uint32_t -lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; - -#endif - - -#define bit_get_price(prob, symbol) \ - lzma_rc_prob_prices[((((prob) - (symbol)) ^ (-(symbol))) \ - & (BIT_MODEL_TOTAL - 1)) >> MOVE_REDUCING_BITS] - - -#define bit_get_price_0(prob) \ - lzma_rc_prob_prices[(prob) >> MOVE_REDUCING_BITS] - - -#define bit_get_price_1(prob) \ - lzma_rc_prob_prices[(BIT_MODEL_TOTAL - (prob)) >> MOVE_REDUCING_BITS] - - -static inline uint32_t -bittree_get_price(const probability *probs, - uint32_t bit_levels, uint32_t symbol) -{ - uint32_t price = 0; - symbol |= UINT32_C(1) << bit_levels; - - do { - price += bit_get_price(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } while (symbol != 1); - - return price; -} - - -static inline uint32_t -bittree_reverse_get_price(const probability *probs, - uint32_t bit_levels, uint32_t symbol) -{ - uint32_t price = 0; - uint32_t model_index = 1; - - do { - const uint32_t bit = symbol & 1; - symbol >>= 1; - price += bit_get_price(probs[model_index], bit); - model_index = (model_index << 1) | bit; - } while (--bit_levels != 0); - - return price; -} - #endif diff --git a/src/liblzma/simple/Makefile.am b/src/liblzma/simple/Makefile.am index a37f1eb5..f8cd4888 100644 --- a/src/liblzma/simple/Makefile.am +++ b/src/liblzma/simple/Makefile.am @@ -21,6 +21,18 @@ libsimple_la_SOURCES = \ simple_coder.h \ simple_private.h +if COND_ENCODER_SIMPLE +libsimple_la_SOURCES += \ + simple_encoder.c \ + simple_encoder.h +endif + +if COND_DECODER_SIMPLE +libsimple_la_SOURCES += \ + simple_decoder.c \ + simple_decoder.h +endif + if COND_FILTER_X86 libsimple_la_SOURCES += x86.c endif diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c index 078f1b95..3ab56582 100644 --- a/src/liblzma/simple/simple_coder.c +++ b/src/liblzma/simple/simple_coder.c @@ -33,7 +33,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, assert(!coder->end_was_reached); if (coder->next.code == NULL) { - bufcpy(in, in_pos, in_size, out, out_pos, out_size); + lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size); // Check if end of stream was reached. if (coder->is_encoder && action == LZMA_FINISH @@ -91,7 +91,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // Flush already filtered data from coder->buffer[] to out[]. if (coder->pos < coder->filtered) { - bufcpy(coder->buffer, &coder->pos, coder->filtered, + lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered, out, out_pos, out_size); // If we couldn't flush all the filtered data, return to @@ -195,7 +195,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, coder->filtered = coder->size; // Flush as much as possible. - bufcpy(coder->buffer, &coder->pos, coder->filtered, + lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered, out, out_pos, out_size); } @@ -210,7 +210,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); return; diff --git a/src/liblzma/simple/simple_decoder.c b/src/liblzma/simple/simple_decoder.c new file mode 100644 index 00000000..72f8ee16 --- /dev/null +++ b/src/liblzma/simple/simple_decoder.c @@ -0,0 +1,47 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_decoder.c +/// \brief Properties decoder for simple filters +// +// Copyright (C) 2007-2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_decoder.h" + + +extern lzma_ret +lzma_simple_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size == 0) + return LZMA_OK; + + if (props_size != 4) + return LZMA_HEADER_ERROR; + + lzma_options_simple *opt = lzma_alloc( + sizeof(lzma_options_simple), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + opt->start_offset = integer_read_32(props); + + // Don't leave an options structure allocated if start_offset is zero. + if (opt->start_offset == 0) + lzma_free(opt, allocator); + else + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_decoder.h b/src/liblzma/simple/simple_decoder.h index c0e626a8..7d1f3d35 100644 --- a/src/liblzma/common/raw_decoder.h +++ b/src/liblzma/simple/simple_decoder.h @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_decoder.h -/// \brief Raw decoder initialization API +/// \file simple_decoder.h +/// \brief Properties decoder for simple filters // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,13 +17,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_DECODER_H -#define LZMA_RAW_DECODER_H +#ifndef LZMA_SIMPLE_DECODER_H +#define LZMA_SIMPLE_DECODER_H -#include "raw_common.h" +#include "simple_coder.h" - -extern lzma_ret lzma_raw_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options); +extern lzma_ret lzma_simple_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); #endif diff --git a/src/liblzma/simple/simple_encoder.c b/src/liblzma/simple/simple_encoder.c new file mode 100644 index 00000000..15d888d9 --- /dev/null +++ b/src/liblzma/simple/simple_encoder.c @@ -0,0 +1,45 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_encoder.c +/// \brief Properties encoder for simple filters +// +// Copyright (C) 2007-2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_encoder.h" + + +extern lzma_ret +lzma_simple_props_size(uint32_t *size, const void *options) +{ + const lzma_options_simple *const opt = options; + *size = (opt == NULL || opt->start_offset == 0) ? 0 : 4; + return LZMA_OK; +} + + +extern lzma_ret +lzma_simple_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_simple *const opt = options; + + // The default start offset is zero, so we don't need to store any + // options unless the start offset is non-zero. + if (opt == NULL || opt->start_offset == 0) + return LZMA_OK; + + integer_write_32(out, opt->start_offset); + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_encoder.h b/src/liblzma/simple/simple_encoder.h index 4e148489..be4ca9fc 100644 --- a/src/liblzma/common/raw_encoder.h +++ b/src/liblzma/simple/simple_encoder.h @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_encoder.h -/// \brief Raw encoder initialization API +/// \file simple_encoder.c +/// \brief Properties encoder for simple filters // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,13 +17,14 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_ENCODER_H -#define LZMA_RAW_ENCODER_H +#ifndef LZMA_SIMPLE_ENCODER_H +#define LZMA_SIMPLE_ENCODER_H -#include "raw_common.h" +#include "simple_coder.h" -extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options); +extern lzma_ret lzma_simple_props_size(uint32_t *size, const void *options); + +extern lzma_ret lzma_simple_props_encode(const void *options, uint8_t *out); #endif diff --git a/src/liblzma/subblock/Makefile.am b/src/liblzma/subblock/Makefile.am index 8f2daf59..2577e0b4 100644 --- a/src/liblzma/subblock/Makefile.am +++ b/src/liblzma/subblock/Makefile.am @@ -18,13 +18,13 @@ libsubblock_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common -if COND_MAIN_ENCODER +if COND_ENCODER_SUBBLOCK libsubblock_la_SOURCES += \ subblock_encoder.c \ subblock_encoder.h endif -if COND_MAIN_DECODER +if COND_DECODER_SUBBLOCK libsubblock_la_SOURCES += \ subblock_decoder.c \ subblock_decoder.h \ diff --git a/src/liblzma/subblock/subblock_decoder.c b/src/liblzma/subblock/subblock_decoder.c index 39ec35c1..faf198c6 100644 --- a/src/liblzma/subblock/subblock_decoder.c +++ b/src/liblzma/subblock/subblock_decoder.c @@ -19,7 +19,7 @@ #include "subblock_decoder.h" #include "subblock_decoder_helper.h" -#include "raw_decoder.h" +#include "filter_decoder.h" /// Maximum number of consecutive Subblocks with Subblock Type Padding @@ -78,7 +78,7 @@ struct lzma_coder_s { lzma_next_coder filter_flags_decoder; /// The filter_flags_decoder stores its results here. - lzma_options_filter filter_flags; + lzma_filter filter_flags; /// Options for the Subblock decoder helper. This is used to tell /// the helper when it should return LZMA_STREAM_END to the subfilter. @@ -239,7 +239,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, // if Subfilter isn't used again, we could leave // a memory-hogging filter dangling until someone // frees Subblock filter itself. - lzma_next_coder_end(&coder->subfilter, allocator); + lzma_next_end(&coder->subfilter, allocator); // Free memory used for subfilter options. This is // safe, because we don't support any Subfilter that @@ -276,7 +276,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, coder->helper.end_was_reached = false; - lzma_options_filter filters[3] = { + lzma_filter filters[3] = { { .id = coder->filter_flags.id, .options = coder->filter_flags.options, @@ -406,7 +406,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, in_limit = in_size; if (coder->subfilter.code == NULL) { - const size_t copy_size = bufcpy( + const size_t copy_size = lzma_bufcpy( in, in_pos, in_limit, out, out_pos, out_size); @@ -480,7 +480,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, } if (coder->subfilter.code == NULL) { - bufcpy(coder->repeat.buffer, + lzma_bufcpy(coder->repeat.buffer, &coder->repeat.pos, coder->repeat.size, out, out_pos, out_size); @@ -586,9 +586,9 @@ subblock_decode(lzma_coder *coder, lzma_allocator *allocator, static void subblock_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); - lzma_next_coder_end(&coder->subfilter, allocator); - lzma_next_coder_end(&coder->filter_flags_decoder, allocator); + lzma_next_end(&coder->next, allocator); + lzma_next_end(&coder->subfilter, allocator); + lzma_next_end(&coder->filter_flags_decoder, allocator); lzma_free(coder->filter_flags.options, allocator); lzma_free(coder, allocator); return; @@ -612,7 +612,7 @@ lzma_subblock_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->filter_flags_decoder = LZMA_NEXT_CODER_INIT; } else { - lzma_next_coder_end(&next->coder->subfilter, allocator); + lzma_next_end(&next->coder->subfilter, allocator); lzma_free(next->coder->filter_flags.options, allocator); } diff --git a/src/liblzma/subblock/subblock_decoder_helper.c b/src/liblzma/subblock/subblock_decoder_helper.c index e8063e1e..ca8fed93 100644 --- a/src/liblzma/subblock/subblock_decoder_helper.c +++ b/src/liblzma/subblock/subblock_decoder_helper.c @@ -40,7 +40,7 @@ helper_decode(lzma_coder *coder, // We can safely copy as much as possible, because we are never // given more data than a single Subblock Data field. - bufcpy(in, in_pos, in_size, out, out_pos, out_size); + lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size); // Return LZMA_STREAM_END when instructed so by the Subblock decoder. return coder->options->end_was_reached ? LZMA_STREAM_END : LZMA_OK; diff --git a/src/liblzma/subblock/subblock_encoder.c b/src/liblzma/subblock/subblock_encoder.c index 01e8007a..e78ffca6 100644 --- a/src/liblzma/subblock/subblock_encoder.c +++ b/src/liblzma/subblock/subblock_encoder.c @@ -18,7 +18,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "subblock_encoder.h" -#include "raw_encoder.h" +#include "filter_encoder.h" /// Maximum number of repeats that a single Repeating Data can indicate. @@ -398,7 +398,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, assert(coder->subfilter.subcoder.code == NULL); // No Subfilter is enabled, just copy the data as is. - coder->subblock.in_pending += bufcpy( + coder->subblock.in_pending += lzma_bufcpy( in, in_pos, in_size, coder->subblock.data, &coder->subblock.size, @@ -480,7 +480,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, // otherwise the Subfilter's memory could be // left allocated for long time, and would // just waste memory. - lzma_next_coder_end(&coder->subfilter.subcoder, + lzma_next_end(&coder->subfilter.subcoder, allocator); // We need to flush the currently buffered @@ -728,7 +728,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, break; case SEQ_RLE_DATA: - bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size, + lzma_bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size, out, out_pos, out_size); if (coder->pos < coder->rle.size) return LZMA_OK; @@ -767,7 +767,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, break; case SEQ_DATA: - bufcpy(coder->subblock.data, &coder->pos, + lzma_bufcpy(coder->subblock.data, &coder->pos, coder->subblock.size, out, out_pos, out_size); if (coder->pos < coder->subblock.size) return LZMA_OK; @@ -791,7 +791,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, return LZMA_HEADER_ERROR; // Initialize a raw encoder to work as a Subfilter. - lzma_options_filter options[2]; + lzma_filter options[2]; options[0] = coder->options->subfilter_options; options[1].id = LZMA_VLI_VALUE_UNKNOWN; @@ -817,8 +817,8 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, // Now we have a big-enough buffer. Encode the Filter Flags. // Like above, this should never fail. size_t dummy = 0; - ret = lzma_filter_flags_encode(coder->subfilter.flags, - &dummy, coder->subfilter.flags_size, options); + ret = lzma_filter_flags_encode(options, coder->subfilter.flags, + &dummy, coder->subfilter.flags_size); assert(ret == LZMA_OK); assert(dummy == coder->subfilter.flags_size); if (ret != LZMA_OK || dummy != coder->subfilter.flags_size) @@ -833,15 +833,15 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, coder->sequence = SEQ_SUBFILTER_FLAGS; // It is safe to fall through because SEQ_SUBFILTER_FLAGS - // uses bufcpy() which doesn't write unless there is output - // space. + // uses lzma_bufcpy() which doesn't write unless there is + // output space. } // Fall through case SEQ_SUBFILTER_FLAGS: // Copy the Filter Flags to the output stream. - bufcpy(coder->subfilter.flags, &coder->pos, + lzma_bufcpy(coder->subfilter.flags, &coder->pos, coder->subfilter.flags_size, out, out_pos, out_size); if (coder->pos < coder->subfilter.flags_size) @@ -912,8 +912,8 @@ subblock_encode(lzma_coder *coder, lzma_allocator *allocator, static void subblock_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); - lzma_next_coder_end(&coder->subfilter.subcoder, allocator); + lzma_next_end(&coder->next, allocator); + lzma_next_end(&coder->subfilter.subcoder, allocator); lzma_free(coder->subblock.data, allocator); lzma_free(coder->subfilter.flags, allocator); lzma_free(coder, allocator); @@ -938,7 +938,7 @@ lzma_subblock_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->subblock.limit = 0; next->coder->subfilter.subcoder = LZMA_NEXT_CODER_INIT; } else { - lzma_next_coder_end(&next->coder->subfilter.subcoder, + lzma_next_end(&next->coder->subfilter.subcoder, allocator); lzma_free(next->coder->subfilter.flags, allocator); } diff --git a/src/lzma/args.c b/src/lzma/args.c index a4764032..30df4522 100644 --- a/src/lzma/args.c +++ b/src/lzma/args.c @@ -39,8 +39,8 @@ bool opt_force = false; bool opt_keep_original = false; bool opt_preserve_name = false; -lzma_check_type opt_check = LZMA_CHECK_CRC64; -lzma_options_filter opt_filters[8]; +lzma_check opt_check = LZMA_CHECK_CRC64; +lzma_filter opt_filters[8]; // We don't modify or free() this, but we need to assign it in some // non-const pointers. @@ -61,6 +61,7 @@ enum { OPT_SPARC, OPT_DELTA, OPT_LZMA, + OPT_LZMA2, OPT_FILES, OPT_FILES0, @@ -108,6 +109,7 @@ static const struct option long_opts[] = { { "sparc", no_argument, NULL, OPT_SPARC }, { "delta", optional_argument, NULL, OPT_DELTA }, { "lzma", optional_argument, NULL, OPT_LZMA }, + { "lzma2", optional_argument, NULL, OPT_LZMA2 }, // Other { "format", required_argument, NULL, 'F' }, @@ -141,6 +143,7 @@ add_filter(lzma_vli id, const char *opt_str) break; case LZMA_FILTER_LZMA: + case LZMA_FILTER_LZMA2: opt_filters[filter_count].options = parse_options_lzma(opt_str); break; @@ -301,6 +304,10 @@ parse_real(int argc, char **argv) add_filter(LZMA_FILTER_LZMA, optarg); break; + case OPT_LZMA2: + add_filter(LZMA_FILTER_LZMA2, optarg); + break; + // Other // --format @@ -445,7 +452,8 @@ static void set_compression_settings(void) { if (filter_count == 0) { - opt_filters[0].id = LZMA_FILTER_LZMA; + opt_filters[0].id = opt_header == HEADER_ALONE + ? LZMA_FILTER_LZMA : LZMA_FILTER_LZMA2; opt_filters[0].options = (lzma_options_lzma *)( lzma_preset_lzma + preset_number); filter_count = 1; @@ -463,13 +471,15 @@ set_compression_settings(void) my_exit(ERROR); } - const uint32_t memory_limit = opt_memory / (1024 * 1024) + 1; - uint32_t memory_usage = lzma_memory_usage(opt_filters, true); + uint64_t memory_usage = lzma_memusage_encoder(opt_filters); + /* opt_mode == MODE_COMPRESS + ? lzma_memusage_encoder(opt_filters) + : lzma_memusage_decoder(opt_filters); */ // Don't go over the memory limits when the default // setting is used. if (preset_default) { - while (memory_usage > memory_limit) { + while (memory_usage > opt_memory) { if (preset_number == 0) { errmsg(V_ERROR, _("Memory usage limit is too " "small for any internal " @@ -481,11 +491,10 @@ set_compression_settings(void) opt_filters[0].options = (lzma_options_lzma *)( lzma_preset_lzma + preset_number); - memory_usage = lzma_memory_usage(opt_filters, - true); + memory_usage = lzma_memusage_encoder(opt_filters); } } else { - if (memory_usage > memory_limit) { + if (memory_usage > opt_memory) { errmsg(V_ERROR, _("Memory usage limit is too small " "for the given filter setup")); my_exit(ERROR); @@ -494,12 +503,8 @@ set_compression_settings(void) // Limit the number of worked threads so that memory usage // limit isn't exceeded. - // FIXME: Probably should use bytes instead of mebibytes for - // memory_usage and memory_limit. - if (memory_usage == 0) - memory_usage = 1; - - size_t thread_limit = memory_limit / memory_usage; + assert(memory_usage > 0); + size_t thread_limit = opt_memory / memory_usage; if (thread_limit == 0) thread_limit = 1; diff --git a/src/lzma/args.h b/src/lzma/args.h index c6098558..b60e7545 100644 --- a/src/lzma/args.h +++ b/src/lzma/args.h @@ -52,8 +52,8 @@ extern bool opt_preserve_name; extern enum tool_mode opt_mode; extern enum header_type opt_header; -extern lzma_check_type opt_check; -extern lzma_options_filter opt_filters[8]; +extern lzma_check opt_check; +extern lzma_filter opt_filters[8]; extern const char *stdin_filename; diff --git a/src/lzma/options.c b/src/lzma/options.c index c82cb1a0..b2ec200e 100644 --- a/src/lzma/options.c +++ b/src/lzma/options.c @@ -299,9 +299,9 @@ extern lzma_options_lzma * parse_options_lzma(const char *str) { static const name_id_map modes[] = { - { "fast", LZMA_MODE_FAST }, - { "best", LZMA_MODE_BEST }, - { NULL, 0 } + { "fast", LZMA_MODE_FAST }, + { "normal", LZMA_MODE_NORMAL }, + { NULL, 0 } }; static const name_id_map mfs[] = { @@ -317,9 +317,9 @@ parse_options_lzma(const char *str) { "dict", NULL, LZMA_DICTIONARY_SIZE_MIN, LZMA_DICTIONARY_SIZE_MAX }, { "lc", NULL, LZMA_LITERAL_CONTEXT_BITS_MIN, - LZMA_LITERAL_CONTEXT_BITS_MAX }, + LZMA_LITERAL_CONTEXT_BITS_MAX }, { "lp", NULL, LZMA_LITERAL_POS_BITS_MIN, - LZMA_LITERAL_POS_BITS_MAX }, + LZMA_LITERAL_POS_BITS_MAX }, { "pb", NULL, LZMA_POS_BITS_MIN, LZMA_POS_BITS_MAX }, { "mode", modes, 0, 0 }, { "fb", NULL, LZMA_FAST_BYTES_MIN, LZMA_FAST_BYTES_MAX }, @@ -334,7 +334,9 @@ parse_options_lzma(const char *str) .literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT, .literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT, .pos_bits = LZMA_POS_BITS_DEFAULT, - .mode = LZMA_MODE_BEST, + .preset_dictionary = NULL, + .persistent = false, + .mode = LZMA_MODE_NORMAL, .fast_bytes = LZMA_FAST_BYTES_DEFAULT, .match_finder = LZMA_MF_BT4, .match_finder_cycles = 0, diff --git a/src/lzma/process.c b/src/lzma/process.c index c180caf7..b4387709 100644 --- a/src/lzma/process.c +++ b/src/lzma/process.c @@ -63,12 +63,7 @@ process_init(void) } for (size_t i = 0; i < opt_threads; ++i) - threads[i] = (thread_data){ - .strm = LZMA_STREAM_INIT_VAR, - .options = NULL, - .pair = NULL, - .in_use = false, - }; + memzero(&threads[i], sizeof(threads[0])); if (pthread_attr_init(&thread_attr) || pthread_attr_setdetachstate( @@ -169,7 +164,9 @@ single_init(thread_data *t) } } else { // TODO Restrict file format if requested on the command line. - ret = lzma_auto_decoder(&t->strm); + ret = lzma_auto_decoder(&t->strm, opt_memory, + LZMA_WARN_UNSUPPORTED_CHECK + | LZMA_CONCATENATED); } if (ret != LZMA_OK) { @@ -185,36 +182,6 @@ single_init(thread_data *t) } -static lzma_ret -single_skip_padding(thread_data *t, uint8_t *in_buf) -{ - // Handle decoding of concatenated Streams. There can be arbitrary - // number of nul-byte padding between the Streams, which must be - // ignored. - // - // NOTE: Concatenating LZMA_Alone files works only if at least - // one of lc, lp, and pb is non-zero. Using the concatenation - // on LZMA_Alone files is strongly discouraged. - while (true) { - while (t->strm.avail_in > 0) { - if (*t->strm.next_in != '\0') - return LZMA_OK; - - ++t->strm.next_in; - --t->strm.avail_in; - } - - if (t->pair->src_eof) - return LZMA_STREAM_END; - - t->strm.next_in = in_buf; - t->strm.avail_in = io_read(t->pair, in_buf, BUFSIZ); - if (t->strm.avail_in == SIZE_MAX) - return LZMA_DATA_ERROR; - } -} - - static void * single(thread_data *t) { @@ -227,10 +194,11 @@ single(thread_data *t) uint8_t in_buf[BUFSIZ]; uint8_t out_buf[BUFSIZ]; lzma_action action = LZMA_RUN; - lzma_ret ret; bool success = false; t->strm.avail_in = 0; + t->strm.next_out = out_buf; + t->strm.avail_out = BUFSIZ; while (!user_abort) { if (t->strm.avail_in == 0 && !t->pair->src_eof) { @@ -239,48 +207,36 @@ single(thread_data *t) if (t->strm.avail_in == SIZE_MAX) break; - else if (t->pair->src_eof - && opt_mode == MODE_COMPRESS) + + if (t->pair->src_eof) action = LZMA_FINISH; } - t->strm.next_out = out_buf; - t->strm.avail_out = BUFSIZ; - - ret = lzma_code(&t->strm, action); + const lzma_ret ret = lzma_code(&t->strm, action); - if (opt_mode != MODE_TEST) + if ((t->strm.avail_out == 0 || ret != LZMA_OK) + && opt_mode != MODE_TEST) { if (io_write(t->pair, out_buf, BUFSIZ - t->strm.avail_out)) break; + t->strm.next_out = out_buf; + t->strm.avail_out = BUFSIZ; + } + if (ret != LZMA_OK) { if (ret == LZMA_STREAM_END) { - if (opt_mode == MODE_COMPRESS) { - assert(t->pair->src_eof); - success = true; - break; - } - - // Support decoding concatenated .lzma files. - ret = single_skip_padding(t, in_buf); - - if (ret == LZMA_STREAM_END) { - assert(t->pair->src_eof); - success = true; - break; - } - - if (ret == LZMA_OK && !single_init(t)) - continue; - - break; - + // FIXME !!! This doesn't work when decoding + // LZMA_Alone files, because LZMA_Alone decoder + // doesn't wait for LZMA_FINISH. + assert(t->pair->src_eof); + success = true; } else { errmsg(V_ERROR, "%s: %s", t->pair->src_name, str_strm_error(ret)); - break; } + + break; } } diff --git a/src/lzmadec/lzmadec.c b/src/lzmadec/lzmadec.c index 1fc561b7..ed5947ad 100644 --- a/src/lzmadec/lzmadec.c +++ b/src/lzmadec/lzmadec.c @@ -65,14 +65,7 @@ static uint8_t out_buf[BUFSIZ]; static lzma_stream strm = LZMA_STREAM_INIT; /// Number of bytes to use memory at maximum -static size_t mem_limit; - -/// Memory allocation hooks -static lzma_allocator allocator = { - .alloc = (void *(*)(void *, size_t, size_t))(&lzma_memlimit_alloc), - .free = (void (*)(void *, void *))(&lzma_memlimit_free), - .opaque = NULL, -}; +static size_t memlimit; /// Program name to be shown in error messages static const char *argv0; @@ -116,7 +109,7 @@ help(void) " MiB of memory at maximum.\n" "\n" "Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n", - argv0, ((uint64_t)(mem_limit) + 512 * 1024) / (1024 * 1024)); + argv0, ((uint64_t)(memlimit) + 512 * 1024) / (1024 * 1024)); // Using PRIu64 above instead of %zu to support pre-C99 libc. exit(0); } @@ -148,7 +141,7 @@ version(void) /// Finds out the amount of physical memory in the system, and sets /// a default memory usage limit. static void -set_default_mem_limit(void) +set_default_memlimit(void) { uint64_t mem = physmem(); if (mem != 0) { @@ -159,10 +152,10 @@ set_default_mem_limit(void) mem = SIZE_MAX; #endif - mem_limit = mem / 3; + memlimit = mem / 3; } else { // Cannot autodetect, use 10 MiB as the default limit. - mem_limit = (1U << 23) + (1U << 21); + memlimit = (1U << 23) + (1U << 21); } return; @@ -272,7 +265,7 @@ parse_options(int argc, char **argv) break; case 'M': - mem_limit = str_to_size(optarg); + memlimit = str_to_size(optarg); break; case 'h': @@ -309,19 +302,20 @@ parse_options(int argc, char **argv) static void init(void) { + const uint32_t flags = LZMA_WARN_UNSUPPORTED_CHECK | LZMA_CONCATENATED; lzma_ret ret; switch (format_type) { case FORMAT_AUTO: - ret = lzma_auto_decoder(&strm); + ret = lzma_auto_decoder(&strm, memlimit, flags); break; case FORMAT_NATIVE: - ret = lzma_stream_decoder(&strm); + ret = lzma_stream_decoder(&strm, memlimit, flags); break; case FORMAT_ALONE: - ret = lzma_alone_decoder(&strm); + ret = lzma_alone_decoder(&strm, memlimit); break; default: @@ -345,50 +339,6 @@ init(void) static void -read_input(void) -{ - strm.next_in = in_buf; - strm.avail_in = fread(in_buf, 1, BUFSIZ, file); - - if (ferror(file)) { - // POSIX says that fread() sets errno if an error occurred. - // ferror() doesn't touch errno. - fprintf(stderr, "%s: %s: Error reading input file: %s\n", - argv0, filename, strerror(errno)); - exit(ERROR); - } - - return; -} - - -static bool -skip_padding(void) -{ - // Handle concatenated Streams. There can be arbitrary number of - // nul-byte padding between the Streams, which must be ignored. - // - // NOTE: Concatenating LZMA_Alone files works only if at least - // one of lc, lp, and pb is non-zero. Using the concatenation - // on LZMA_Alone files is strongly discouraged. - while (true) { - while (strm.avail_in > 0) { - if (*strm.next_in != '\0') - return true; - - ++strm.next_in; - --strm.avail_in; - } - - if (feof(file)) - return false; - - read_input(); - } -} - - -static void uncompress(void) { if (file == stdin && !force && isatty(STDIN_FILENO)) { @@ -400,44 +350,65 @@ uncompress(void) } init(); + strm.avail_in = 0; + strm.next_out = out_buf; + strm.avail_out = BUFSIZ; + + lzma_action action = LZMA_RUN; while (true) { - if (strm.avail_in == 0) - read_input(); + if (strm.avail_in == 0) { + strm.next_in = in_buf; + strm.avail_in = fread(in_buf, 1, BUFSIZ, file); + + if (ferror(file)) { + // POSIX says that fread() sets errno if + // an error occurred. ferror() doesn't + // touch errno. + fprintf(stderr, "%s: %s: Error reading " + "input file: %s\n", + argv0, filename, + strerror(errno)); + exit(ERROR); + } - strm.next_out = out_buf; - strm.avail_out = BUFSIZ; + if (feof(file)) + action = LZMA_FINISH; + } - const lzma_ret ret = lzma_code(&strm, LZMA_RUN); + const lzma_ret ret = lzma_code(&strm, action); // Write and check write error before checking decoder error. // This way as much data as possible gets written to output - // even if decoder detected an error. Checking write error - // needs to be done before checking decoder error due to - // how concatenated Streams are handled a few lines later. - const size_t write_size = BUFSIZ - strm.avail_out; - if (fwrite(out_buf, 1, write_size, stdout) != write_size) { - // Wouldn't be a surprise if writing to stderr would - // fail too but at least try to show an error message. - fprintf(stderr, "%s: Cannot write to " - "standard output: %s\n", argv0, - strerror(errno)); - exit(ERROR); + // even if decoder detected an error. + if (strm.avail_out == 0 || ret != LZMA_OK) { + const size_t write_size = BUFSIZ - strm.avail_out; + + if (fwrite(out_buf, 1, write_size, stdout) + != write_size) { + // Wouldn't be a surprise if writing to stderr + // would fail too but at least try to show an + // error message. + fprintf(stderr, "%s: Cannot write to " + "standard output: %s\n", argv0, + strerror(errno)); + exit(ERROR); + } + + strm.next_out = out_buf; + strm.avail_out = BUFSIZ; } if (ret != LZMA_OK) { - if (ret == LZMA_STREAM_END) { - if (skip_padding()) { - init(); - continue; - } - + // FIXME !!! Doesn't work with LZMA_Alone for the + // same reason as in process.c. + if (ret == LZMA_STREAM_END) return; - } fprintf(stderr, "%s: %s: ", argv0, filename); + // FIXME Add LZMA_*_CHECK and LZMA_FORMAT_ERROR. switch (ret) { case LZMA_DATA_ERROR: fprintf(stderr, "File is corrupt\n"); @@ -452,6 +423,11 @@ uncompress(void) fprintf(stderr, "%s\n", strerror(ENOMEM)); exit(ERROR); + case LZMA_MEMLIMIT_ERROR: + fprintf(stderr, "Memory usage limit " + "reached\n"); + exit(ERROR); + case LZMA_BUF_ERROR: fprintf(stderr, "Unexpected end of input\n"); exit(ERROR); @@ -479,23 +455,12 @@ main(int argc, char **argv) { argv0 = argv[0]; - set_default_mem_limit(); + set_default_memlimit(); parse_options(argc, argv); lzma_init_decoder(); - lzma_memlimit *mem_limiter = lzma_memlimit_create(mem_limit); - if (mem_limiter == NULL) { - fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM)); - exit(ERROR); - } - - assert(lzma_memlimit_count(mem_limiter) == 0); - - allocator.opaque = mem_limiter; - strm.allocator = &allocator; - #ifdef WIN32 setmode(fileno(stdin), O_BINARY); setmode(fileno(stdout), O_BINARY); @@ -531,8 +496,6 @@ main(int argc, char **argv) // Free the memory only when debugging. Freeing wastes some time, // but allows detecting possible memory leaks with Valgrind. lzma_end(&strm); - assert(lzma_memlimit_count(mem_limiter) == 0); - lzma_memlimit_end(mem_limiter, false); #endif return exit_status; |