diff options
Diffstat (limited to 'src/liblzma')
158 files changed, 25171 insertions, 0 deletions
diff --git a/src/liblzma/Makefile.am b/src/liblzma/Makefile.am new file mode 100644 index 00000000..78a072f4 --- /dev/null +++ b/src/liblzma/Makefile.am @@ -0,0 +1,47 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +SUBDIRS = api common check + +lib_LTLIBRARIES = liblzma.la +liblzma_la_SOURCES = +liblzma_la_LDFLAGS = -version-info 0:0:0 + +liblzma_la_LIBADD = \ + common/libcommon.la \ + check/libcheck.la + +if COND_FILTER_LZMA +SUBDIRS += lz lzma rangecoder +liblzma_la_LIBADD += \ + lz/liblz.la \ + lzma/liblzma4.la \ + rangecoder/librangecoder.la +endif + +if COND_FILTER_SUBBLOCK +SUBDIRS += subblock +liblzma_la_LIBADD += subblock/libsubblock.la +endif + +if COND_MAIN_SIMPLE +SUBDIRS += simple +liblzma_la_LIBADD += simple/libsimple.la +endif + + +## pkg-config +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = lzma.pc +EXTRA_DIST = lzma.pc.in diff --git a/src/liblzma/api/Makefile.am b/src/liblzma/api/Makefile.am new file mode 100644 index 00000000..7f5e6de4 --- /dev/null +++ b/src/liblzma/api/Makefile.am @@ -0,0 +1,39 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +nobase_include_HEADERS = \ + lzma.h \ + lzma/alignment.h \ + lzma/alone.h \ + lzma/auto.h \ + lzma/base.h \ + lzma/block.h \ + lzma/check.h \ + lzma/copy.h \ + lzma/delta.h \ + lzma/extra.h \ + lzma/filter.h \ + lzma/index.h \ + lzma/info.h \ + lzma/init.h \ + lzma/lzma.h \ + lzma/memlimit.h \ + lzma/metadata.h \ + lzma/raw.h \ + lzma/simple.h \ + lzma/stream.h \ + lzma/stream_flags.h \ + lzma/subblock.h \ + lzma/version.h \ + lzma/vli.h diff --git a/src/liblzma/api/lzma.h b/src/liblzma/api/lzma.h new file mode 100644 index 00000000..186ae12c --- /dev/null +++ b/src/liblzma/api/lzma.h @@ -0,0 +1,122 @@ +/** + * \file lzma.h + * \brief The public API of liblzma + * + * liblzma is a LZMA compression library with a zlib-like API. + * liblzma is based on LZMA SDK found from http://7-zip.org/sdk.html. + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H +#define LZMA_H + +/******************** + * External headers * + ********************/ + +/* size_t */ +#include <sys/types.h> + +/* NULL */ +#include <stddef.h> + +/* uint8_t, uint32_t, uint64_t, UINT32_C, UINT64_C, UINT64_MAX. */ +#include <inttypes.h> + + +/****************** + * GCC extensions * + ******************/ + +/* + * GCC extensions are used conditionally in the public API. It doesn't + * break anything if these are sometimes enabled and sometimes not, only + * affects warnings and optimizations. + */ +#if defined(__GNUC__) && __GNUC__ >= 3 +# ifndef lzma_attribute +# define lzma_attribute(attr) __attribute__(attr) +# endif +# ifndef lzma_restrict +# define lzma_restrict __restrict__ +# endif +#else +# ifndef lzma_attribute +# define lzma_attribute(attr) +# endif +# ifndef lzma_restrict +# define lzma_restrict +# endif +#endif + + +/************** + * Subheaders * + **************/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Subheaders check that this is defined. It is to prevent including + * them directly from applications. + */ +#define LZMA_H_INTERNAL 1 + +/* Basic features */ +#include "lzma/init.h" +#include "lzma/base.h" +#include "lzma/vli.h" +#include "lzma/filter.h" +#include "lzma/check.h" + +/* Filters */ +#include "lzma/copy.h" +#include "lzma/subblock.h" +#include "lzma/simple.h" +#include "lzma/delta.h" +#include "lzma/lzma.h" + +/* Container formats and Metadata */ +#include "lzma/block.h" +#include "lzma/index.h" +#include "lzma/extra.h" +#include "lzma/metadata.h" +#include "lzma/stream.h" +#include "lzma/alone.h" +#include "lzma/raw.h" +#include "lzma/auto.h" + +/* Advanced features */ +#include "lzma/info.h" +#include "lzma/alignment.h" +#include "lzma/stream_flags.h" +#include "lzma/memlimit.h" + +/* Version number */ +#include "lzma/version.h" + +/* + * All subheaders included. Undefine LZMA_H_INTERNAL to prevent applications + * re-including the subheaders. + */ +#undef LZMA_H_INTERNAL + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef LZMA_H */ diff --git a/src/liblzma/api/lzma/alignment.h b/src/liblzma/api/lzma/alignment.h new file mode 100644 index 00000000..6672656c --- /dev/null +++ b/src/liblzma/api/lzma/alignment.h @@ -0,0 +1,60 @@ +/** + * \file lzma/alignment.h + * \brief Calculating input and output alignment of filter chains + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Calculates the preferred alignment of the input data + * + * FIXME desc + */ +extern uint32_t lzma_alignment_input( + const lzma_options_filter *filters, uint32_t guess); + + +/** + * \brief Calculates the alignment of the encoded output + * + * Knowing the alignment of the output data is useful e.g. in the Block + * encoder which tries to align the Compressed Data field optimally. + * + * \param filters Pointer to lzma_options_filter array, whose last + * member must have .id = LZMA_VLI_VALUE_UNKNOWN. + * \param guess The value to return if the alignment of the output + * is the same as the alignment of the input data. + * If you want to always detect this special case, + * this guess to zero; this function never returns + * zero unless guess is zero. + * + * \return In most cases, a small positive integer is returned; + * for optimal use, the encoded output of this filter + * chain should start at on offset that is a multiple of + * the returned integer value. + * + * If the alignment of the output is the same as the input + * data (which this function cannot know), \a guess is + * returned. + * + * If an error occurs (that is, unknown Filter IDs or filter + * options), UINT32_MAX is returned. + */ +extern uint32_t lzma_alignment_output( + const lzma_options_filter *filters, uint32_t guess); diff --git a/src/liblzma/api/lzma/alone.h b/src/liblzma/api/lzma/alone.h new file mode 100644 index 00000000..1a6b8e27 --- /dev/null +++ b/src/liblzma/api/lzma/alone.h @@ -0,0 +1,82 @@ +/** + * \file lzma/alone.h + * \brief Handling of the legacy LZMA_Alone format + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Options for files in the LZMA_Alone format + */ +typedef struct { + /** + * \brief Uncompressed Size and usage of End of Payload Marker + * + * In contrast to .lzma Blocks, LZMA_Alone format cannot have both + * uncompressed size field in the header and end of payload marker. + * If you don't know the uncompressed size beforehand, set it to + * LZMA_VLI_VALUE_UNKNOWN and liblzma will embed end of payload + * marker. + */ + lzma_vli uncompressed_size; + + /** + * \brief LZMA options + * + * The LZMA_Alone format supports only one filter: the LZMA filter. + * + * \note There exists also an undocumented variant of the + * LZMA_Alone format, which uses the x86 filter in + * addition to LZMA. This format was never supported + * by LZMA Utils and is not supported by liblzma either. + */ + lzma_options_lzma lzma; + +} lzma_options_alone; + + +/** + * \brief Initializes LZMA_Alone encoder + * + * LZMA_Alone files have the suffix .lzma like the .lzma Stream files. + * LZMA_Alone format supports only one filter, the LZMA filter. There is + * no support for integrity checks like CRC32. + * + * Use this format if and only if you need to create files readable by + * legacy LZMA tools. + * + * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_alone_encoder( + lzma_stream *strm, const lzma_options_alone *options); + + +/** + * \brief Initializes decoder for LZMA_Alone file + * + * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + */ +extern lzma_ret lzma_alone_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/auto.h b/src/liblzma/api/lzma/auto.h new file mode 100644 index 00000000..327e726f --- /dev/null +++ b/src/liblzma/api/lzma/auto.h @@ -0,0 +1,41 @@ +/** + * \file lzma/auto.h + * \brief Decoder with automatic file format detection + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Decode .lzma Streams and LZMA_Alone files with autodetection + * + * Autodetects between the .lzma Stream and LZMA_Alone formats, and + * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once + * the type of the file has been detected. + * + * \param strm Pointer to propertily prepared lzma_stream + * \param header Pointer to hold a pointer to Extra Records read + * from the Header Metadata Block. Use NULL if + * you don't care about Extra Records. + * \param footer Same as header, but for Footer Metadata Block. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_auto_decoder(lzma_stream *strm, + lzma_extra **header, lzma_extra **footer); diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h new file mode 100644 index 00000000..53cf89f5 --- /dev/null +++ b/src/liblzma/api/lzma/base.h @@ -0,0 +1,410 @@ +/** + * \file lzma/base.h + * \brief Data types and functions used in many places of the public API + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Boolean + * + * This is here because C89 doesn't have stdbool.h. To set a value for + * variables having type lzma_bool, you can use + * - C99's `true' and `false' from stdbool.h; + * - C++'s internal `true' and `false'; or + * - integers one (true) and zero (false). + */ +typedef unsigned char lzma_bool; + + +/** + * \brief Return values used by several functions in liblzma + * + * Check the descriptions of specific functions to find out which return + * values they can return and the exact meanings of the values in every + * situation. The descriptions given here are only suggestive. + */ +typedef enum { + LZMA_OK = 0, + /**< + * \brief Operation completed successfully + */ + + LZMA_STREAM_END = 1, + /**< + * \brief End of stream was reached + * + * The application should pick the last remaining output + * bytes from strm->next_out. + */ + + LZMA_PROG_ERROR = -2, + /**< + * \brief Programming error + * + * This indicates that the arguments given to the function are + * invalid or the internal state of the decoder is corrupt. + * - Function arguments are invalid or the structures + * pointed by the argument pointers are invalid + * e.g. if strm->next_out has been set to NULL and + * strm->avail_out > 0 when calling lzma_code(). + * - lzma_* functions have been called in wrong order + * e.g. lzma_code() was called right after lzma_end(). + * - If errors occur randomly, the reason might be flaky + * hardware. + * + * If you think that your code is correct, this error code + * can be a sign of a bug in liblzma. See the documentation + * how to report bugs. + */ + + LZMA_DATA_ERROR = -3, + /**< + * \brief Data is corrupt + * + * - Encoder: The input size doesn't match the uncompressed + * size given to lzma_*_encoder_init(). + * - Decoder: The input is corrupt. This includes corrupted + * header, corrupted compressed data, and unmatching + * integrity Check. + * + * \todo What can be done if encoder returns this? + * Probably can continue by fixing the input + * amount, but make sure. + */ + + LZMA_MEM_ERROR = -4, + /**< + * \brief Cannot allocate memory + * + * Memory allocation failed. + */ + + LZMA_BUF_ERROR = -5, + /**< + * \brief No progress is possible + * + * This may happen when avail_in or avail_out is zero. + * + * \note This error is not fatal. Coding can continue + * normally once the reason for this error has + * been fixed. + */ + + LZMA_HEADER_ERROR = -6, + /**< + * \brief Invalid or unsupported header + * + * Invalid or unsupported options, for example + * - unsupported filter(s) or filter options; or + * - reserved bits set in headers (decoder only). + * + * Rebuilding liblzma with more features enabled, or + * upgrading to a newer version of liblzma may help. + */ + + LZMA_UNSUPPORTED_CHECK = -7, + /**< + * \brief Check type is unknown + * + * The type of Check is not supported, and thus the Check + * cannot be calculated. In the encoder, this is an error. + * In the decoder, this is only a warning and decoding can + * still proceed normally (but the Check is ignored). + */ +} lzma_ret; + + +/** + * \brief The `action' argument for lzma_code() + */ +typedef enum { + LZMA_RUN = 0, + /**< + * Encoder: Encode as much input as possible. Some internal + * buffering will probably be done (depends on the filter + * chain in use), which causes latency: the input used won't + * usually be decodeable from the output of the same + * lzma_code() call. + * + * Decoder: Decode as much input as possible and produce as + * much output as possible. This action provides best + * throughput, but may introduce latency, because the + * decoder may decode more data into its internal buffers + * than that fits into next_out. + */ + + LZMA_SYNC_FLUSH = 1, + /**< + * Encoder: Makes all the data given to liblzma via next_in + * available in next_out without resetting the filters. Call + * lzma_code() with LZMA_SYNC_FLUSH until it returns + * LZMA_STREAM_END. Then continue encoding normally. + * + * \note Synchronous flushing is supported only by + * some filters. Some filters support it only + * partially. + * + * Decoder: Asks the decoder to decode only as much as is + * needed to fill next_out. This decreases latency with some + * filters, but is likely to decrease also throughput. It is + * a good idea to use this flag only when it is likely that + * you don't need more output soon. + * + * \note With decoder, this is not comparable to + * zlib's Z_SYNC_FLUSH. + */ + + LZMA_FULL_FLUSH = 2, + /**< + * Finishes encoding of the current Data Block. All the input + * data going to the current Data Block must have been given + * to the encoder (the last bytes can still be pending in + * next_in). Call lzma_code() with LZMA_FULL_FLUSH until + * it returns LZMA_STREAM_END. Then continue normally with + * LZMA_RUN or finish the Stream with LZMA_FINISH. + * + * This action is supported only by Multi-Block Stream + * encoder. If there is no unfinished Data Block, no empty + * Data Block is created. + */ + + LZMA_FINISH = 3 + /**< + * Finishes the encoding operation. All the input data must + * have been given to the encoder (the last bytes can still + * be pending in next_in). Call lzma_code() with LZMA_FINISH + * until it returns LZMA_STREAM_END. + * + * This action is not supported by decoders. + */ +} lzma_action; + + +/** + * \brief Custom functions for memory handling + * + * A pointer to lzma_allocator may be passed via lzma_stream structure + * to liblzma. The library will use these functions for memory handling + * instead of the default malloc() and free(). + * + * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is + * OK to change these function pointers in the middle of the coding + * process, but obviously it must be done carefully to make sure that the + * replacement `free' can deallocate memory allocated by the earlier + * `alloc' function(s). + */ +typedef struct { + /** + * \brief Pointer to custom memory allocation function + * + * Set this to point to your custom memory allocation function. + * It can be useful for example if you want to limit how much + * memory liblzma is allowed to use: for this, you may use + * a pointer to lzma_memory_alloc(). + * + * If you don't want a custom allocator, but still want + * custom free(), set this to NULL and liblzma will use + * the standard malloc(). + * + * \param opaque lzma_allocator.opaque (see below) + * \param nmemb Number of elements like in calloc(). + * liblzma will always set nmemb to 1. + * This argument exists only for + * compatibility with zlib and libbzip2. + * \param size Size of an element in bytes. + * liblzma never sets this to zero. + * + * \return Pointer to the beginning of a memory block of + * size nmemb * size, or NULL if allocation fails + * for some reason. When allocation fails, functions + * of liblzma return LZMA_MEM_ERROR. + */ + void *(*alloc)(void *opaque, size_t nmemb, size_t size); + + /** + * \brief Pointer to custom memory freeing function + * + * Set this to point to your custom memory freeing function. + * If lzma_memory_alloc() is used as allocator, this should + * be set to lzma_memory_free(). + * + * If you don't want a custom freeing function, but still + * want a custom allocator, set this to NULL and liblzma + * will use the standard free(). + * + * \param opaque lzma_allocator.opaque (see below) + * \param ptr Pointer returned by + * lzma_allocator.alloc(), or when it + * is set to NULL, a pointer returned + * by the standard malloc(). + */ + void (*free)(void *opaque, void *ptr); + + /** + * \brief Pointer passed to .alloc() and .free() + * + * opaque is passed as the first argument to lzma_allocator.alloc() + * and lzma_allocator.free(). This intended to ease implementing + * custom memory allocation functions for use with liblzma. + * + * When using lzma_memory_alloc() and lzma_memory_free(), opaque + * must point to lzma_memory_limitter structure allocated and + * initialized with lzma_memory_limitter_create(). + * + * If you don't need this, you should set it to NULL. + */ + void *opaque; + +} lzma_allocator; + + +/** + * \brief Internal data structure + * + * The contents of this structure is not visible outside the library. + */ +typedef struct lzma_internal_s lzma_internal; + + +/** + * \brief Passing data to and from liblzma + * + * The lzma_stream structure is used for + * - passing pointers to input and output buffers to liblzma; + * - defining custom memory hander functions; and + * - holding a pointer to coder-specific internal data structures. + * + * Before calling any of the lzma_*_init() functions the first time, + * the application must reset lzma_stream to LZMA_STREAM_INIT. The + * lzma_*_init() function will verify the options, allocate internal + * data structures and store pointer to them into `internal'. Finally + * total_in and total_out are reset to zero. In contrast to zlib, + * next_in and avail_in are ignored by the initialization functions. + * + * The actual coding is done with the lzma_code() function. Application + * must update next_in, avail_in, next_out, and avail_out between + * calls to lzma_decode() just like with zlib. + * + * In contrast to zlib, even the decoder requires that there always + * is at least one byte space in next_out; if avail_out == 0, + * LZMA_BUF_ERROR is returned immediatelly. This shouldn't be a problem + * for most applications that already use zlib, but it's still worth + * checking your application. + * + * Application may modify values of total_in and total_out as it wants. + * They are updated by liblzma to match the amount of data read and + * written, but liblzma doesn't use the values internally. + * + * Application must not touch the `internal' pointer. + */ +typedef struct { + uint8_t *next_in; /**< Pointer to the next input byte. */ + size_t avail_in; /**< Number of available input bytes in next_in. */ + uint64_t total_in; /**< Total number of bytes read by liblzma. */ + + uint8_t *next_out; /**< Pointer to the next output position. */ + size_t avail_out; /**< Amount of free space in next_out. */ + uint64_t total_out; /**< Total number of bytes written by liblzma. */ + + /** + * Custom memory allocation functions. Set to NULL to use + * the standard malloc() and free(). + */ + lzma_allocator *allocator; + + /** Internal state is not visible to outsiders. */ + lzma_internal *internal; + +} lzma_stream; + + +/** + * \brief Initialization for lzma_stream + * + * When you declare an instance of lzma_stream, you can immediatelly + * initialize it so that initialization functions know that no memory + * has been allocated yet: + * + * lzma_stream strm = LZMA_STREAM_INIT; + */ +#define LZMA_STREAM_INIT { NULL, 0, 0, NULL, 0, 0, NULL, NULL } + + +/** + * \brief Initialization for lzma_stream + * + * This is like LZMA_STREAM_INIT, but this can be used when the lzma_stream + * has already been allocated: + * + * lzma_stream *strm = malloc(sizeof(lzma_stream)); + * if (strm == NULL) + * return LZMA_MEM_ERROR; + * *strm = LZMA_STREAM_INIT_VAR; + */ +extern const lzma_stream LZMA_STREAM_INIT_VAR; + + +/** + * \brief Encodes or decodes data + * + * Once the lzma_stream has been successfully initialized (e.g. with + * lzma_stream_encoder_single()), the actual encoding or decoding is + * done using this function. + * + * \return Some coders may have more exact meaning for different return + * values, which are mentioned separately in the description of + * the initialization functions. Here are the typical meanings: + * - LZMA_OK: So far all good. + * - LZMA_STREAM_END: + * - Encoder: LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or + * LZMA_FINISH completed. + * - Decoder: End of uncompressed data was reached. + * - LZMA_BUF_ERROR: Unable to progress. Provide more input or + * output space, and call this function again. This cannot + * occur if both avail_in and avail_out were non-zero (or + * there's a bug in liblzma). + * - LZMA_MEM_ERROR: Unable to allocate memory. Due to lazy + * programming, the coding cannot continue even if the + * application could free more memory. The next call must + * be lzma_end() or some initialization function. + * - LZMA_DATA_ERROR: + * - Encoder: Filter(s) cannot process the given data. + * - Decoder: Compressed data is corrupt. + * - LZMA_HEADER_ERROR: Unsupported options. Rebuilding liblzma + * with more features enabled or upgrading to a newer version + * may help, although usually this is a sign of invalid options + * (encoder) or corrupted input data (decoder). + * - LZMA_PROG_ERROR: Invalid arguments or the internal state + * of the coder is corrupt. + */ +extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action); + + +/** + * \brief Frees memory allocated for the coder data structures + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * + * \note zlib indicates an error if application end()s unfinished + * stream. liblzma doesn't do this, and assumes that + * application knows what it is doing. + */ +extern void lzma_end(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/block.h b/src/liblzma/api/lzma/block.h new file mode 100644 index 00000000..210c1d87 --- /dev/null +++ b/src/liblzma/api/lzma/block.h @@ -0,0 +1,409 @@ +/** + * \file lzma/block.h + * \brief .lzma Block handling + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Options for the Block Header encoder and decoder + * + * Different things use different parts of this structure. Some read + * some members, other functions write, and some do both. Only the + * members listed for reading need to be initialized when the specified + * functions are called. The members marked for writing will be assigned + * new values at some point either by calling the given function or by + * later calls to lzma_code(). + */ +typedef struct { + /** + * \brief Type of integrity Check + * + * The type of the integrity Check is not stored into the Block + * Header, thus its value must be provided also when decoding. + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_check_type check; + + /** + * \brief Precense of CRC32 of the Block Header + * + * Set this to true if CRC32 of the Block Header should be + * calculated and stored in the Block Header. + * + * There is no way to autodetect if CRC32 is present in the Block + * Header, thus this information must be provided also when decoding. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * - lzma_block_header_decoder() + */ + lzma_bool has_crc32; + + /** + * \brief Usage of End of Payload Marker + * + * If this is true, End of Payload Marker is used even if + * Uncompressed Size is known. + * + * Read by: + * - lzma_block_header_encoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_header_decoder() + */ + lzma_bool has_eopm; + + /** + * \brief True if the Block is a Metadata Block + * + * If this is true, the Metadata bit will be set in the Block Header. + * It is up to the application to store correctly formatted data + * into Metadata Block. + * + * Read by: + * - lzma_block_header_encoder() + * + * Written by: + * - lzma_block_header_decoder() + */ + lzma_bool is_metadata; + + /** + * \brief True if Uncompressed Size is in Block Footer + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_bool has_uncompressed_size_in_footer; + + /** + * \brief True if Backward Size is in Block Footer + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_bool has_backward_size; + + /** + * \brief True if Block coder should take care of Padding + * + * In liblzma, Stream decoder sets this to true when decoding + * Header Metadata Block or Data Blocks from Multi-Block Stream, + * and to false when decoding Single-Block Stream or Footer + * Metadata Block from a Multi-Block Stream. + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_bool handle_padding; + + /** + * \brief Size of the Compressed Data in bytes + * + * Usually you don't know this value when encoding in streamed mode. + * In non-streamed mode you can reserve space for this field when + * encoding the Block Header the first time, and then re-encode the + * Block Header and copy it over the original one after the encoding + * of the Block has been finished. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_header_decoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_vli compressed_size; + + /** + * \brief Uncompressed Size in bytes + * + * Encoder: If this value is not LZMA_VLI_VALUE_UNKNOWN, it is stored + * to the Uncompressed Size field in the Block Header. The real + * uncompressed size of the data being compressed must match + * the Uncompressed Size or LZMA_HEADER_ERROR is returned. + * + * If Uncompressed Size is unknown, End of Payload Marker must + * be used. If uncompressed_size == LZMA_VLI_VALUE_UNKNOWN and + * has_eopm == 0, LZMA_HEADER_ERROR will be returned. + * + * Decoder: If this value is not LZMA_VLI_VALUE_UNKNOWN, it is + * compared to the real Uncompressed Size. If they do not match, + * LZMA_HEADER_ERROR is returned. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_header_decoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_vli uncompressed_size; + + /** + * \brief Number of bytes to reserve for Compressed Size + * + * This is useful if you want to be able to store the Compressed Size + * to the Block Header, but you don't know it when starting to encode. + * Setting this to non-zero value at maximum of LZMA_VLI_BYTES_MAX, + * the Block Header encoder will force the Compressed Size field to + * occupy specified number of bytes. You can later rewrite the Block + * Header to contain correct information by using otherwise identical + * lzma_options_block structure except the correct compressed_size. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * + * Written by: + * - lzma_block_header_decoder() + */ + uint32_t compressed_reserve; + + /** + * \brief Number of bytes to reserve for Uncompressed Size + * + * See the description of compressed_size above. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * + * Written by: + * - lzma_block_header_decoder() + */ + uint32_t uncompressed_reserve; + + /** + * \brief Total Size of the Block in bytes + * + * This is useful in the decoder, which can verify the Total Size + * if it is known from Index. + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_vli total_size; + + /** + * \brief Upper limit of Total Size + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_vli total_limit; + + /** + * \brief Upper limit of Uncompressed Size + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + */ + lzma_vli uncompressed_limit; + + /** + * \brief Array of filters + * + * There can be at maximum of seven filters. The end of the array + * is marked with .id = LZMA_VLI_VALUE_UNKNOWN. Minimum number of + * filters is zero; in that case, an implicit Copy filter is used. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_header_decoder(): Note that this does NOT free() + * the old filter options structures. If decoding fails, the + * caller must take care of freeing the options structures + * that may have been allocated and decoded before the error + * occurred. + */ + lzma_options_filter filters[8]; + + /** + * \brief Size of the Padding field + * + * The Padding field exist to allow aligning the Compressed Data field + * optimally in the Block. See lzma_options_stream.alignment in + * stream.h for more information. + * + * If you want the Block Header encoder to automatically calculate + * optimal size for the Padding field by looking at the information + * in filters[], set this to LZMA_BLOCK_HEADER_PADDING_AUTO. In that + * case, you must also set the aligmnet variable to tell the the + * encoder the aligmnet of the beginning of the Block Header. + * + * The decoder never sets this to LZMA_BLOCK_HEADER_PADDING_AUTO. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder(): Note that this doesn't + * accept LZMA_BLOCK_HEADER_PADDING_AUTO. + * + * Written by (these never set padding to + * LZMA_BLOCK_HEADER_PADDING_AUTO): + * - lzma_block_header_size() + * - lzma_block_header_decoder() + */ + int32_t padding; +# define LZMA_BLOCK_HEADER_PADDING_AUTO (-1) +# define LZMA_BLOCK_HEADER_PADDING_MIN 0 +# define LZMA_BLOCK_HEADER_PADDING_MAX 31 + + /** + * \brief Alignment of the beginning of the Block Header + * + * This variable is read only if padding has been set to + * LZMA_BLOCK_HEADER_PADDING_AUTO. + * + * Read by: + * - lzma_block_header_size() + * - lzma_block_header_encoder() + */ + uint32_t alignment; + + /** + * \brief Size of the Block Header + * + * Read by: + * - lzma_block_encoder() + * - lzma_block_decoder() + * + * Written by: + * - lzma_block_header_size() + * - lzma_block_header_decoder() + */ + uint32_t header_size; + +} lzma_options_block; + + +/** + * \brief Calculates the size of Header Padding and Block Header + * + * \return - LZMA_OK: Size calculated successfully and stored to + * options->header_size. + * - LZMA_HEADER_ERROR: Unsupported filters or filter options. + * - LZMA_PROG_ERROR: Invalid options + * + * \note This doesn't check that all the options are valid i.e. this + * may return LZMA_OK even if lzma_block_header_encode() or + * lzma_block_encoder() would fail. + */ +extern lzma_ret lzma_block_header_size(lzma_options_block *options); + + +/** + * \brief Encodes Block Header + * + * Encoding of the Block options is done with a single call instead of + * first initializing and then doing the actual work with lzma_code(). + * + * \param out Beginning of the output buffer. This must be + * at least options->header_size bytes. + * \param options Block options to be encoded. + * + * \return - LZMA_OK: Encoding was successful. options->header_size + * bytes were written to output buffer. + * - LZMA_HEADER_ERROR: Invalid or unsupported options. + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_block_header_encode( + uint8_t *out, const lzma_options_block *options); + + +/** + * \brief Initializes Block Header decoder + * + * Because the results of this decoder are placed into *options, + * strm->next_in, strm->avail_in, and strm->total_in are not used. + * + * The only valid `action' with lzma_code() is LZMA_RUN. + * + * \return - LZMA_OK: Encoding was successful. options->header_size + * bytes were written to output buffer. + * - LZMA_HEADER_ERROR: Invalid or unsupported options. + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_block_header_decoder( + lzma_stream *strm, lzma_options_block *options); + + +/** + * \brief Initializes .lzma Block encoder + * + * This function is required for multi-thread encoding. It may also be + * useful when implementing custom file formats. + * + * \return - LZMA_OK: All good, continue with lzma_code(). + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_DATA_ERROR: Limits (total_limit and uncompressed_limit) + * have been reached already. + * - LZMA_UNSUPPORTED_CHECK: options->check specfies a Check + * that is not supported by this buid of liblzma. Initializing + * the encoder failed. + * - LZMA_PROG_ERROR + * + * lzma_code() can return FIXME + */ +extern lzma_ret lzma_block_encoder( + lzma_stream *strm, lzma_options_block *options); + + +/** + * \brief Initializes decoder for .lzma Block + * + * \return - LZMA_OK: All good, continue with lzma_code(). + * - LZMA_UNSUPPORTED_CHECK: Initialization was successful, but + * the given Check type is not supported, thus Check will be + * ignored. + * - LZMA_PROG_ERROR + * - LZMA_MEM_ERROR + */ +extern lzma_ret lzma_block_decoder( + lzma_stream *strm, lzma_options_block *options); diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h new file mode 100644 index 00000000..4a2a453b --- /dev/null +++ b/src/liblzma/api/lzma/check.h @@ -0,0 +1,128 @@ +/** + * \file lzma/check.h + * \brief Integrity checks + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Type of the Check + * + * The .lzma format supports multiple types of Checks that are calculated + * from the uncompressed data (unless it is empty; then it's calculated + * from Block Header). + */ +typedef enum { + LZMA_CHECK_NONE = 0, + /**< + * No Check is calculated. + * + * Size of the Check field: 0 bytes + */ + + LZMA_CHECK_CRC32 = 1, + /**< + * CRC32 using the polynomial from the IEEE 802.3 standard + * + * Size of the Check field: 4 bytes + */ + + LZMA_CHECK_CRC64 = 3, + /**< + * CRC64 using the polynomial from the ECMA-182 standard + * + * Size of the Check field: 8 bytes + */ + + LZMA_CHECK_SHA256 = 5 + /**< + * SHA-256 + * + * Size of the Check field: 32 bytes + */ +} lzma_check_type; + + +/** + * \brief Maximum valid Check ID + * + * The .lzma file format specification specifies eight Check IDs (0-7). Some + * of them are only reserved i.e. no actual Check algorithm has been assigned. + * Still liblzma accepts any of these eight IDs for future compatibility + * when decoding files. If a valid but unsupported Check ID is detected, + * liblzma indicates a warning with LZMA_UNSUPPORTED_CHECK. + * + * FIXME bad desc + */ +#define LZMA_CHECK_ID_MAX 7 + + +/** + * \brief Check IDs supported by this liblzma build + * + * If lzma_available_checks[n] is true, the Check ID n is supported by this + * liblzma build. You can assume that LZMA_CHECK_NONE and LZMA_CHECK_CRC32 + * are always available. + */ +extern const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1]; + + +/** + * \brief Size of the Check field with different Check IDs + * + * Although not all Check IDs have a check algorithm associated, the size of + * every Check is already frozen. This array contains the size (in bytes) of + * the Check field with specified Check ID. The values are taken from the + * section 2.2.2 of the .lzma file format specification: + * { 0, 4, 4, 8, 16, 32, 32, 64 } + */ +extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1]; + + +/** + * \brief Calculate CRC32 + * + * Calculates CRC32 using the polynomial from the IEEE 802.3 standard. + * + * \param buf Pointer to the input buffer + * \param size Size of the input buffer + * \param crc Previously returned CRC value. This is used to + * calculate the CRC of a big buffer in smaller chunks. + * Set to zero when there is no previous value. + * + * \return Updated CRC value, which can be passed to this function + * again to continue CRC calculation. + */ +extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); + + +/** + * \brief Calculate CRC64 + * + * Calculates CRC64 using the polynomial from the ECMA-182 standard. + * + * This function is used similarly to lzma_crc32(). See its documentation. + */ +extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); + + +/* + * SHA256 functions are currently not exported to public API. + * Contact the author if you think it should be. + */ diff --git a/src/liblzma/api/lzma/copy.h b/src/liblzma/api/lzma/copy.h new file mode 100644 index 00000000..f5617462 --- /dev/null +++ b/src/liblzma/api/lzma/copy.h @@ -0,0 +1,29 @@ +/** + * \file lzma/copy.h + * \brief Copy filter + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Filter ID + * + * Filter ID of the Copy filter. This is used as lzma_options_filter.id. + */ +#define LZMA_FILTER_COPY LZMA_VLI_C(0x00) diff --git a/src/liblzma/api/lzma/delta.h b/src/liblzma/api/lzma/delta.h new file mode 100644 index 00000000..58afec18 --- /dev/null +++ b/src/liblzma/api/lzma/delta.h @@ -0,0 +1,49 @@ +/** + * \file lzma/delta.h + * \brief Delta filter + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Filter ID + * + * Filter ID of the Delta filter. This is used as lzma_options_filter.id. + */ +#define LZMA_FILTER_DELTA LZMA_VLI_C(0x20) + + +/** + * \brief Options for the Delta filter + * + * These options are needed by both encoder and decoder. + */ +typedef struct { + /** + * \brief Delta distance as bytes + * + * Examples: + * - 16-bit stereo audio: distance = 4 bytes + * - 24-bit RGB image data: distance = 3 bytes + */ + uint32_t distance; +# define LZMA_DELTA_DISTANCE_MIN 1 +# define LZMA_DELTA_DISTANCE_MAX 256 + +} lzma_options_delta; diff --git a/src/liblzma/api/lzma/extra.h b/src/liblzma/api/lzma/extra.h new file mode 100644 index 00000000..29426a74 --- /dev/null +++ b/src/liblzma/api/lzma/extra.h @@ -0,0 +1,114 @@ +/** + * \file lzma/extra.h + * \brief Handling of Extra Records in Metadata + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/* + * Extra Record IDs + * + * See the .lzma file format specification for description what each + * Extra Record type exactly means. + * + * If you ever need to update .lzma files with Extra Records, note that + * the Record IDs are divided in two categories: + * - Safe-to-Copy Records may be preserved as is when the + * Stream is modified in ways that don't change the actual + * uncompressed data. Examples of such operatings include + * recompressing and adding, modifying, or deleting unrelated + * Extra Records. + * - Unsafe-to-Copy Records should be removed (and possibly + * recreated) when any kind of changes are made to the Stream. + */ + +#define LZMA_EXTRA_PADDING 0x00 +#define LZMA_EXTRA_OPENPGP 0x01 +#define LZMA_EXTRA_FILTERS 0x02 +#define LZMA_EXTRA_COMMENT 0x03 +#define LZMA_EXTRA_CHECKS 0x04 +#define LZMA_EXTRA_FILENAME 0x05 +#define LZMA_EXTRA_MTIME 0x07 +#define LZMA_EXTRA_MTIME_HR 0x09 +#define LZMA_EXTRA_MIME_TYPE 0x0B +#define LZMA_EXTRA_HOMEPAGE 0x0D + + +/** + * \brief Extra Records + * + * The .lzma format provides a way to store custom information along + * the actual compressed content. Information about these Records + * are passed to and from liblzma via this linked list. + */ +typedef struct lzma_extra_s lzma_extra; +struct lzma_extra_s { + /** + * \brief Pointer to the next Extra Record + * + * This is NULL on the last Extra Record. + */ + lzma_extra *next; + + /** + * \brief Record ID + * + * Extra Record IDs are divided in three categories: + * - Zero is a special case used for padding. It doesn't have + * Size of Data fields. + * - Odd IDs (1, 3, 5, ...) are Safe-to-Copy IDs. + * These can be preserved as is if the Stream is + * modified in a way that doesn't alter the actual + * uncompressed content. + * - Even IDs (2, 4, 6, ...) are Unsafe-to-Copy IDs. + * If the .lzma Stream is modified in any way, + * the Extra Records having a sensitive ID should + * be removed or updated accordingly. + * + * Refer to the .lzma file format specification for + * the up to date list of Extra Record IDs. + */ + lzma_vli id; + + /** + * \brief Size of the Record data + * + * In case of strings, this should not include the + * trailing '\0'. + */ + size_t size; + + /** + * \brief Record data + * + * Record data is often a string in UTF-8 encoding, + * but it can be arbitrary binary data. In case of + * strings, the trailing '\0' is usually not stored + * in the .lzma file. + * + * To ease working with Extra Records containing strings, + * liblzma always adds '\0' to the end of data even when + * it wasn't present in the .lzma file. This '\0' is not + * counted in the size of the data. + */ + uint8_t *data; +}; + + +extern void lzma_extra_free(lzma_extra *extra, lzma_allocator *allocator); diff --git a/src/liblzma/api/lzma/filter.h b/src/liblzma/api/lzma/filter.h new file mode 100644 index 00000000..a8bdd4bd --- /dev/null +++ b/src/liblzma/api/lzma/filter.h @@ -0,0 +1,166 @@ +/** + * \file lzma/filter.h + * \brief Common filter related types + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Filter options + * + * This structure is used to pass Filter ID and a pointer filter's options + * to liblzma. + */ +typedef struct { + /** + * \brief Filter ID + * + * Use constants whose name begin with `LZMA_FILTER_' to specify + * different filters. In an array of lzma_option_filter structures, + * use LZMA_VLI_VALUE_UNKNOWN to indicate end of filters. + */ + lzma_vli id; + + /** + * \brief Pointer to filter-specific options structure + * + * If the filter doesn't need options, set this to NULL. If id is + * set to LZMA_VLI_VALUE_UNKNOWN, options is ignored, and thus + * doesn't need be initialized. + * + * Some filters support changing the options in the middle of + * the encoding process. These filters store the pointer of the + * options structure and communicate with the application via + * modifications of the options structure. + */ + void *options; + +} lzma_options_filter; + + +/** + * \brief Filters available for encoding + * + * Pointer to an array containing the list of available Filter IDs that + * can be used for encoding. The last element is LZMA_VLI_VALUE_UNKNOWN. + * + * If lzma_available_filter_encoders[0] == LZMA_VLI_VALUE_UNKNOWN, the + * encoder components haven't been built at all. This means that the + * encoding-specific functions are probably missing from the library + * API/ABI completely. + */ +extern const lzma_vli *const lzma_available_filter_encoders; + + +/** + * \brief Filters available for decoding + * + * Pointer to an array containing the list of available Filter IDs that + * can be used for decoding. The last element is LZMA_VLI_VALUE_UNKNOWN. + * + * If lzma_available_filter_decoders[0] == LZMA_VLI_VALUE_UNKNOWN, the + * decoder components haven't been built at all. This means that the + * decoding-specific functions are probably missing from the library + * API/ABI completely. + */ +extern const lzma_vli *const lzma_available_filter_decoders; + + +/** + * \brief Calculate rough memory requirements for given filter chain + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_VALUE_UNKNOWN. + * \param is_encoder Set to true when calculating memory requirements + * of an encoder; false for decoder. + * + * \return Number of mebibytes (MiB i.e. 2^20) required for the given + * encoder or decoder filter chain. + * + * \note If calculating memory requirements of encoder, lzma_init() or + * lzma_init_encoder() must have been called earlier. Similarly, + * if calculating memory requirements of decoder, lzma_init() or + * lzma_init_decoder() must have been called earlier. + */ +extern uint32_t lzma_memory_usage( + const lzma_options_filter *filters, lzma_bool is_encoder); + + +/** + * \brief Calculates encoded size of a Filter Flags field + * + * Knowing the size of Filter Flags is useful to know when allocating + * memory to hold the encoded Filter Flags. + * + * \param size Pointer to integer to hold the calculated size + * \param options Filter ID and associated options whose encoded + * size is to be calculted + * + * \return - LZMA_OK: *size set successfully. Note that this doesn't + * guarantee that options->options is valid, thus + * lzma_filter_flags_encode() may still fail. + * - LZMA_HEADER_ERROR: Unknown Filter ID or unsupported options. + * - LZMA_PROG_ERROR: Invalid options + * + * \note If you need to calculate size of List of Filter Flags, + * you need to loop over every lzma_options_filter entry. + */ +extern lzma_ret lzma_filter_flags_size( + uint32_t *size, const lzma_options_filter *options); + + +/** + * \brief Encodes Filter Flags into given buffer + * + * In contrast to some functions, this doesn't allocate the needed buffer. + * This is due to how this function is used internally by liblzma. + * + * \param out Beginning of the output buffer + * \param out_pos out[*out_pos] is the next write position. This + * is updated by the encoder. + * \param out_size out[out_size] is the first byte to not write. + * \param options Filter options to be encoded + * + * \return - LZMA_OK: Encoding was successful. + * - LZMA_HEADER_ERROR: Invalid or unsupported options. + * - LZMA_PROG_ERROR: Invalid options or not enough output + * buffer space (you should have checked it with + * lzma_filter_flags_size()). + */ +extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, + size_t out_size, const lzma_options_filter *options); + + +/** + * \brief Initializes Filter Flags decoder + * + * The decoded result is stored into *options. options->options is + * initialized but the old value is NOT free()d. + * + * Because the results of this decoder are placed into *options, + * strm->next_in, strm->avail_in, and strm->total_in are not used + * when calling lzma_code(). The only valid action for lzma_code() + * is LZMA_RUN + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_filter_flags_decoder( + lzma_stream *strm, lzma_options_filter *options); diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h new file mode 100644 index 00000000..7e59c4b3 --- /dev/null +++ b/src/liblzma/api/lzma/index.h @@ -0,0 +1,84 @@ +/** + * \file lzma/index.h + * \brief Handling of Index lists + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief + * + * FIXME desc + */ +typedef struct lzma_index_s lzma_index; +struct lzma_index_s { + /** + * \brief Total Size of the Block + * + * This includes Block Header, Compressed Data, and Block Footer. + */ + lzma_vli total_size; + + /** + * \brief Uncompressed Size of the Block + */ + lzma_vli uncompressed_size; + + /** + * \brief Pointer to the next Index Record + * + * This is NULL on the last Index Record. + */ + lzma_index *next; +}; + + +/** + * \brief Duplicates an Index list + * + * \return A copy of the Index list, or NULL if memory allocation + * failed or the original Index was empty. + */ +extern lzma_index *lzma_index_dup( + const lzma_index *index, lzma_allocator *allocator); + + +/** + * \brief Frees an Index list + * + * All Index Recors in the list are freed. This function is convenient when + * getting rid of lzma_metadata structures containing an Index. + */ +extern void lzma_index_free(lzma_index *index, lzma_allocator *allocator); + + +/** + * \brief Calculates information about the Index + * + * \return LZMA_OK on success, LZMA_PROG_ERROR on error. FIXME + */ +extern lzma_ret lzma_index_count(const lzma_index *index, size_t *count, + lzma_vli *lzma_restrict total_size, + lzma_vli *lzma_restrict uncompressed_size); + + +/** + * \brief Compares if two Index lists are identical + */ +extern lzma_bool lzma_index_is_equal(const lzma_index *a, const lzma_index *b); diff --git a/src/liblzma/api/lzma/info.h b/src/liblzma/api/lzma/info.h new file mode 100644 index 00000000..3a91850f --- /dev/null +++ b/src/liblzma/api/lzma/info.h @@ -0,0 +1,315 @@ +/** + * \file lzma/info.h + * \brief Handling of Stream size information + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/********** + * Basics * + **********/ + +/** + * \brief Opaque data type to hold the size information + */ +typedef struct lzma_info_s lzma_info; + + +typedef struct { + /** + * \brief Total Size of this Block + * + * This can be LZMA_VLI_VALUE_UNKNOWN. + */ + lzma_vli total_size; + + /** + * \brief Uncompressed Size of this Block + * + * This can be LZMA_VLI_VALUE_UNKNOWN. + */ + lzma_vli uncompressed_size; + + /** + * \brief Offset of the first byte of the Block + * + * In encoder, this is useful to find out the alignment of the Block. + * + * In decoder, this is useful when doing random-access reading + * with help from lzma_info_data_locate(). + */ + lzma_vli stream_offset; + + /** + * \brief Uncompressed offset of the Block + * + * Offset of the first uncompressed byte of the Block relative to + * all uncompressed data in the Block. + * FIXME desc + */ + lzma_vli uncompressed_offset; + + /** + * \brief Pointers to internal data structures + * + * Applications must not touch these. + */ + void *internal[4]; + +} lzma_info_iter; + + +typedef enum { + LZMA_INFO_STREAM_START, + LZMA_INFO_HEADER_METADATA, + LZMA_INFO_TOTAL, + LZMA_INFO_UNCOMPRESSED, + LZMA_INFO_FOOTER_METADATA +} lzma_info_size; + + +/** + * \brief Allocates and initializes a new lzma_info structure + * + * If info is NULL, a new lzma_info structure is allocated, initialized, and + * a pointer to it returned. If allocation fails, NULL is returned. + * + * If info is non-NULL, it is reinitialized and the same pointer returned. + * (In this case, return value cannot be NULL or a different pointer than + * the info given as argument.) + */ +extern lzma_info *lzma_info_init(lzma_info *info, lzma_allocator *allocator); + + +/** + * \brief Resets lzma_info + * + * This is like calling lzma_info_end() and lzma_info_create(), but + * re-uses the existing base structure. + */ +extern void lzma_info_reset( + lzma_info *info, lzma_allocator *allocator); + + +/** + * \brief Frees memory allocated for a lzma_info structure + */ +extern void lzma_info_free(lzma_info *info, lzma_allocator *allocator); + + +/************************ + * Setting known values * + ************************/ + +/** + * \brief Set a known size value + * + * \param info Pointer returned by lzma_info_create() + * \param type Any value from lzma_info_size + * \param size Value to set or verify + * + * \return LZMA_OK on success, LZMA_DATA_ERROR if the size doesn't + * match the existing information, or LZMA_PROG_ERROR + * if type is invalid or size is not a valid VLI. + */ +extern lzma_ret lzma_info_size_set( + lzma_info *info, lzma_info_size type, lzma_vli size); + + +/** + * \brief Sets the Index + * + * The given lzma_index list is "absorbed" by this function. The application + * must not access it after this function call, even if this function returns + * an error. + * + * \note The given lzma_index will at some point get freed by the + * lzma_info_* functions. If you use a custom lzma_allocator, + * make sure that it can free the lzma_index. + */ +extern lzma_ret lzma_info_index_set( + lzma_info *info, lzma_allocator *allocator, + lzma_index *index, lzma_bool eat_index); + + +/** + * \brief Sets information from a known Metadata Block + * + * This is a shortcut for calling lzma_info_size_set() with different type + * arguments, lzma_info_index_set() with metadata->index. + */ +extern lzma_ret lzma_info_metadata_set(lzma_info *info, + lzma_allocator *allocator, lzma_metadata *metadata, + lzma_bool is_header_metadata, lzma_bool eat_index); + + +/*************** + * Incremental * + ***************/ + +/** + * \brief Prepares an iterator to be used with given lzma_info structure + * + * + */ +extern void lzma_info_iter_begin(lzma_info *info, lzma_info_iter *iter); + + +/** + * \brief Moves to the next Index Record + * + * + */ +extern lzma_ret lzma_info_iter_next( + lzma_info_iter *iter, lzma_allocator *allocator); + + +/** + * \brief Sets or verifies the sizes in the Index Record + * + * \param iter Pointer to iterator to be set or verified + * \param total_size + * Total Size in bytes or LZMA_VLI_VALUE_UNKNOWN + * \param uncompressed_size + * Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN + * + * \return - LZMA_OK: All OK. + * - LZMA_DATA_ERROR: Given sizes don't match with the already + * known sizes. + * - LZMA_PROG_ERROR: Internal error, possibly integer + * overflow (e.g. the sum of all the known sizes is too big) + */ +extern lzma_ret lzma_info_iter_set(lzma_info_iter *iter, + lzma_vli total_size, lzma_vli uncompressed_size); + + +/** + * \brief Locates a Data Block + * + * \param iter Properly initialized iterator + * \param allocator Pointer to lzma_allocator or NULL + * \param uncompressed_offset + * Target offset to locate. The final offset + * will be equal or smaller than this. + * \param allow_alloc True if this function is allowed to call + * lzma_info_iter_next() to allocate a new Record + * if the requested offset reached end of Index + * Record list. Note that if Index has been marked + * final, lzma_info_iter_next() is never called. + * + * \return - LZMA_OK: All OK, *iter updated accordingly. + * - LZMA_DATA_ERROR: Trying to search past the end of the Index + * Record list, and allocating a new Record was not allowed + * either because allow_alloc was false or Index was final. + * - LZMA_PROG_ERROR: Internal error (probably integer + * overflow causing some lzma_vli getting too big). + */ +extern lzma_ret lzma_info_iter_locate(lzma_info_iter *iter, + lzma_allocator *allocator, lzma_vli uncompressed_offset, + lzma_bool allow_alloc); + + +/** + * \brief Finishes incrementally constructed Index + * + * This sets the known Total Size and Uncompressed of the Data Blocks + * based on the information collected from the Index Records, and marks + * the Index as final. + */ +extern lzma_ret lzma_info_index_finish(lzma_info *info); + + +/*************************** + * Reading the information * + ***************************/ + +/** + * \brief Gets a known size + * + * + */ +extern lzma_vli lzma_info_size_get( + const lzma_info *info, lzma_info_size type); + +extern lzma_vli lzma_info_metadata_locate( + const lzma_info *info, lzma_bool is_header_metadata); + +/** + * \brief Gets a pointer to the beginning of the Index list + * + * If detach is true, the Index will be detached from the lzma_info + * structure, and thus not be modified or freed by lzma_info_end(). + * + * If detach is false, the application must not modify the Index in any way. + * Also, the Index list is guaranteed to be valid only till the next call + * to any lzma_info_* function. + */ +extern lzma_index *lzma_info_index_get(lzma_info *info, lzma_bool detach); + + +extern size_t lzma_info_index_count_get(const lzma_info *info); + + +extern uint32_t lzma_info_metadata_alignment_get( + const lzma_info *info, lzma_bool is_header_metadata); + + + +/** + * \brief Locate a Block containing the given uncompressed offset + * + * This function is useful when you need to do random-access reading in + * a Multi-Block Stream. + * + * \param info Pointer to lzma_info that has at least one + * Index Record. The Index doesn't need to be finished. + * \param uncompressed_target + * Uncompressed target offset which the caller would + * like to locate from the Stream. + * \param stream_offset + * Starting offset (relative to the beginning the Stream) + * of the Block containing the requested location. + * \param uncompressed_offset + * The actual uncompressed offset of the beginning of + * the Block. uncompressed_offset <= uncompressed_target + * is always true; the application needs to uncompress + * uncompressed_target - uncompressed_offset bytes to + * reach the requested target offset. + * \param total_size + * Total Size of the Block. If the Index is incomplete, + * this may be LZMA_VLI_VALUE_UNKNOWN indicating unknown + * size. + * \param uncompressed_size + * Uncompressed Size of the Block. If the Index is + * incomplete, this may be LZMA_VLI_VALUE_UNKNOWN + * indicating unknown size. The application must pass + * this value to the Block decoder to verify FIXME + * + * \return + * + * \note This function is currently implemented as a linear search. + * If there are many Index Records, this can be really slow. + * This can be improved in newer liblzma versions if needed. + */ +extern lzma_bool lzma_info_data_locate(const lzma_info *info, + lzma_vli uncompressed_target, + lzma_vli *lzma_restrict stream_offset, + lzma_vli *lzma_restrict uncompressed_offset, + lzma_vli *lzma_restrict total_size, + lzma_vli *lzma_restrict uncompressed_size); diff --git a/src/liblzma/api/lzma/init.h b/src/liblzma/api/lzma/init.h new file mode 100644 index 00000000..f7b79246 --- /dev/null +++ b/src/liblzma/api/lzma/init.h @@ -0,0 +1,85 @@ +/** + * \file lzma/init.h + * \brief Initializations + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Initialize all internal static variables + * + * Depending on the build options, liblzma may have some internal static + * variables, that must be initialized before using any other part of + * the library (*). It is recommended to do these initializations in the very + * beginning of the application by calling appropriate initialization function. + * + * (*) There are some exceptions to this rule. FIXME + * + * The initialization functions are not necessarily thread-safe, thus the + * required initializations must be done before creating any threads. (The + * rest of the functions of liblzma are thread-safe.) Calling the + * initialization functions multiple times does no harm, although it + * still shouldn't be done when there are multiple threads running. + * + * lzma_init() initializes all internal static variables by calling + * lzma_lzma_init_encoder() and lzma_init_decoder(). + * + * If you need only encoder, decoder, or neither-encoder-nor-decoder + * functions, you may use other initialization functions, which initialize + * only a subset of liblzma's internal static variables. Using those + * functions have the following advantages: + * - When linking statically against liblzma, less useless functions will + * get linked into the binary. E.g. if you need only the decoder functions, + * using lzma_init_decoder() avoids linking bunch of encoder related code. + * - There is less things to initialize, making the initialization + * process slightly faster. + */ +extern void lzma_init(void); + + +/** + * \brief Initialize internal static variables needed by encoders + * + * If you need only the encoder functions, you may use this function to + * initialize only the things required by encoders. + * + * This function also calls lzma_init_check(). + */ +extern void lzma_init_encoder(void); + + +/** + * \brief Initialize internal static variables needed by decoders + * + * If you need only the decoder functions, you may use this function to + * initialize only the things required by decoders. + * + * This function also calls lzma_init_check(). + */ +extern void lzma_init_decoder(void); + + +/** + * \brief Initialize internal static variables needed by integrity checks + * + * Currently this initializes CRC32 and CRC64 lookup tables if precalculated + * tables haven't been built into the library. This function can be useful + * if the only thing you need from liblzma is the integrity check functions. + */ +extern void lzma_init_check(void); diff --git a/src/liblzma/api/lzma/lzma.h b/src/liblzma/api/lzma/lzma.h new file mode 100644 index 00000000..0fe74854 --- /dev/null +++ b/src/liblzma/api/lzma/lzma.h @@ -0,0 +1,312 @@ +/** + * \file lzma/lzma.h + * \brief LZMA filter + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Filter ID + * + * Filter ID of the LZMA filter. This is used as lzma_options_filter.id. + */ +#define LZMA_FILTER_LZMA LZMA_VLI_C(0x40) + + +/** + * \brief LZMA compression modes + * + * Currently there are only two modes. Earlier LZMA SDKs had also third + * mode between fast and best. + */ +typedef enum { + LZMA_MODE_INVALID = -1, + /**< + * \brief Invalid mode + * + * Used as array terminator in lzma_available_modes. + */ + + + LZMA_MODE_FAST = 0, + /**< + * \brief Fast compression + * + * Fast mode is usually at its best when combined with + * a hash chain match finder. + */ + + LZMA_MODE_BEST = 2 + /**< + * \brief Best compression ratio + * + * This is usually notably slower than fast mode. Use this + * together with binary tree match finders to expose the + * full potential of the LZMA encoder. + */ +} lzma_mode; + + +/** + * \brief Match finders + * + * Match finder has major effect on both speed and compression ratio. + * Usually hash chains are faster than binary trees. + */ +typedef enum { + LZMA_MF_INVALID = -1, + /**< + * \brief Invalid match finder ID + * + * Used as array terminator in lzma_available_match_finders. + */ + + LZMA_MF_HC3 = 0x03, + /**< + * \brief Hash Chain with 3 bytes hashing + * + * \todo Memory requirements + * + * \note It's possible that this match finder gets + * removed in future. The definition will stay + * in this header, but liblzma may return + * LZMA_HEADER_ERROR if it is specified (just + * like it would if the match finder had been + * disabled at compile time). + */ + + LZMA_MF_HC4 = 0x04, + /**< + * \brief Hash Chain with 4 bytes hashing + * + * Memory requirements: 7.5 * dictionary_size + 4 MiB + * + * \note It's possible that this match finder gets + * removed in future. The definition will stay + * in this header, but liblzma may return + * LZMA_HEADER_ERROR if it is specified (just + * like it would if the match finder had been + * disabled at compile time). + */ + + LZMA_MF_BT2 = 0x12, + /**< + * \brief Binary Tree with 2 bytes hashing + * + * Memory requirements: 9.5 * dictionary_size + 4 MiB + */ + + LZMA_MF_BT3 = 0x13, + /**< + * \brief Binary Tree with 3 bytes hashing + * + * Memory requirements: 11.5 * dictionary_size + 4 MiB + */ + + LZMA_MF_BT4 = 0x14 + /**< + * \brief Binary Tree with 4 bytes hashing + * + * Memory requirements: 11.5 * dictionary_size + 4 MiB + */ +} lzma_match_finder; + + +/** + * \brief Options specific to the LZMA method handler + */ +typedef struct { + /********************************** + * LZMA encoding/decoding options * + **********************************/ + + /* These options are required in encoder and also with raw decoding. */ + + /** + * \brief Dictionary size in bytes + * + * Dictionary size indicates how many bytes of the recently processed + * uncompressed data is kept in memory. One method to reduce size of + * the uncompressed data is to store distance-length pairs, which + * indicate what data to repeat from the dictionary buffer. Thus, + * the bigger the dictionary, the better compression ratio usually is. + * + * Raw decoding: Too big dictionary does no other harm than + * wasting memory. This value is ignored by lzma_raw_decode_buffer(), + * because it uses the target buffer as the dictionary. + */ + uint32_t dictionary_size; +# define LZMA_DICTIONARY_SIZE_MIN 1 +# define LZMA_DICTIONARY_SIZE_MAX (UINT32_C(1) << 30) +# define LZMA_DICTIONARY_SIZE_DEFAULT (UINT32_C(1) << 23) + + /** + * \brief Number of literal context bits + * + * How many of the highest bits of the previous uncompressed + * eight-bit byte (also known as `literal') are taken into + * account when predicting the bits of the next literal. + * + * \todo Example + */ + uint32_t literal_context_bits; +# define LZMA_LITERAL_CONTEXT_BITS_MIN 0 +# define LZMA_LITERAL_CONTEXT_BITS_MAX 8 +# define LZMA_LITERAL_CONTEXT_BITS_DEFAULT 3 + + /** + * \brief Number of literal position bits + * + * How many of the lowest bits of the current position (number + * of bytes from the beginning of the uncompressed data) in the + * uncompressed data is taken into account when predicting the + * bits of the next literal (a single eight-bit byte). + * + * \todo Example + */ + uint32_t literal_pos_bits; +# define LZMA_LITERAL_POS_BITS_MIN 0 +# define LZMA_LITERAL_POS_BITS_MAX 4 +# define LZMA_LITERAL_POS_BITS_DEFAULT 0 + + /** + * \brief Number of position bits + * + * How many of the lowest bits of the current position in the + * uncompressed data is taken into account when estimating + * probabilities of matches. A match is a sequence of bytes for + * which a matching sequence is found from the dictionary and + * thus can be stored as distance-length pair. + * + * Example: If most of the matches occur at byte positions + * of 8 * n + 3, that is, 3, 11, 19, ... set pos_bits to 3, + * because 2**3 == 8. + */ + uint32_t pos_bits; +# define LZMA_POS_BITS_MIN 0 +# define LZMA_POS_BITS_MAX 4 +# define LZMA_POS_BITS_DEFAULT 2 + + /** + * \brief Pointer to an initial dictionary + * + * It is possible to initialize the LZ77 history window using + * a preset dictionary. Here is a good quote from zlib's + * documentation; this applies to LZMA as is: + * + * "The dictionary should consist of strings (byte sequences) that + * are likely to be encountered later in the data to be compressed, + * with the most commonly used strings preferably put towards the + * end of the dictionary. Using a dictionary is most useful when + * the data to be compressed is short and can be predicted with + * good accuracy; the data can then be compressed better than + * with the default empty dictionary." + * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3) + * + * This feature should be used only in special situations. + * It works correctly only with raw encoding and decoding. + * Currently none of the container formats supported by + * liblzma allow preset dictionary when decoding, thus if + * you create a .lzma file with preset dictionary, it cannot + * be decoded with the regular .lzma decoder functions. + * + * \todo This feature is not implemented yet. + */ + const uint8_t *preset_dictionary; + + /** + * \brief Size of the preset dictionary + * + * Specifies the size of the preset dictionary. If the size is + * bigger than dictionary_size, only the last dictionary_size + * bytes are processed. + * + * This variable is read only when preset_dictionary is not NULL. + */ + uint32_t preset_dictionary_size; + + /****************************************** + * LZMA options needed only when encoding * + ******************************************/ + + /** LZMA compression mode */ + lzma_mode mode; + + /** + * \brief Number of fast bytes + * + * Number of fast bytes determines how many bytes the encoder + * compares from the match candidates when looking for the best + * match. Bigger fast bytes value usually increase both compression + * ratio and time. + */ + uint32_t fast_bytes; +# define LZMA_FAST_BYTES_MIN 5 +# define LZMA_FAST_BYTES_MAX 273 +# define LZMA_FAST_BYTES_DEFAULT 128 + + /** Match finder ID */ + lzma_match_finder match_finder; + + /** + * \brief Match finder cycles + * + * Higher values give slightly better compression ratio but + * decrease speed. Use special value 0 to let liblzma use + * match-finder-dependent default value. + * + * \todo Write much better description. + */ + uint32_t match_finder_cycles; + +} lzma_options_lzma; + + +/** + * \brief Available LZMA encoding modes + * + * Pointer to an array containing the list of available encoding modes. + * + * This variable is available only if LZMA encoder has been enabled. + */ +extern const lzma_mode *const lzma_available_modes; + + +/** + * \brief Available match finders + * + * Pointer to an array containing the list of available match finders. + * The last element is LZMA_MF_INVALID. + * + * This variable is available only if LZMA encoder has been enabled. + */ +extern const lzma_match_finder *const lzma_available_match_finders; + + +/** + * \brief Table of presets for the LZMA filter + * + * lzma_presets[0] is the fastest and lzma_preset_lzma[8] is the slowest. + * These presets match the switches -1 .. -9 of the lzma command line tool + * + * The preset values are subject to changes between liblzma versions. + * + * This variable is available only if LZMA encoder has been enabled. + */ +extern const lzma_options_lzma lzma_preset_lzma[9]; diff --git a/src/liblzma/api/lzma/memlimit.h b/src/liblzma/api/lzma/memlimit.h new file mode 100644 index 00000000..26ec50fe --- /dev/null +++ b/src/liblzma/api/lzma/memlimit.h @@ -0,0 +1,157 @@ +/** + * \file lzma/memlimit.h + * \brief Memory usage limitter + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Opaque data type used with the memory usage limitting functions + */ +typedef struct lzma_memlimit_s lzma_memlimit; + + +/** + * \brief Allocates and initializes a new lzma_memlimit structure + * + * It is easy to make liblzma to use huge amounts of memory. This can + * be a problem especially with the decoder, since it a file requiring + * huge amounts of memory to uncompress could allow even a denial of + * service attack if the memory usage wasn't limited. + * + * liblzma provides a set of functions to control memory usage. Pointers + * to these functions can be used in lzma_allocator structure, which makes + * it easy to limit memory usage with liblzma. + * + * The memory limitter functions are not tied to limitting memory usage + * with liblzma itself. You can use them with anything you like. + * + * In multi-threaded applications, only one thread at once may use the same + * lzma_memlimit structure. If there is a need, this limitation may + * be removed in future versions without breaking the libary API/ABI. + * + * \param limit Initial memory usage limit in bytes + * + * \return Pointer to allocated and initialized lzma_memlimit + * structure. On error, NULL is returned. The reason behind + * an error is either that malloc() failed or that the given + * limit was so small that it didn't allow allocating even + * the lzma_memlimit structure itself. + * + * \note Excluding lzma_memlimit_usage(), the functions whose name begin + * lzma_memlimit_ can be used even if lzma_init() hasn't been + * called. + */ +extern lzma_memlimit *lzma_memlimit_create(size_t limit); + + +/** + * \brief Sets a new memory usage limit + * + * \param mem Pointer to a lzma_memlimit structure returned + * earlier by lzma_memry_limit_create(). + * \param limit New memory usage limit + * + * The new usage limit may be smaller than the amount of memory currently + * allocated via *mem: New allocations will fail until enough memory has + * been freed or a new limit is set, but the existing allocatations will + * stay untouched. + */ +extern void lzma_memlimit_set(lzma_memlimit *mem, size_t limit); + + +/** + * \brief Gets the current memory usage limit + */ +extern size_t lzma_memlimit_get(const lzma_memlimit *mem); + + +/** + * \brief Gets the amount of currently allocated memory + * + * \note This value includes the sizes of some helper structures, + * thus it will always be larger than the total number of + * bytes allocated via lzma_memlimit_alloc(). + */ +extern size_t lzma_memlimit_used(const lzma_memlimit *mem); + + +/** + * \brief Allocates memory with malloc() if memory limit allows + * + * \param mem Pointer to a lzma_memlimit structure returned + * earlier by lzma_memry_limit_create(). + * \param nmemb Number of elements to allocate. While liblzma always + * sets this to one, this function still takes the + * value of nmemb into account to keep the function + * usable with zlib and libbzip2. + * \param size Size of an element. + * + * \return Pointer to memory allocated with malloc(nmemb * size), + * except if nmemb * size == 0 which returns malloc(1). + * On error, NULL is returned. + * + * \note This function assumes that nmemb * size is at maximum of + * SIZE_MAX. If it isn't, an overflow will occur resulting + * invalid amount of memory being allocated. + */ +extern void *lzma_memlimit_alloc( + lzma_memlimit *mem, size_t nmemb, size_t size); + + +/** + * \brief Removes the pointer from memory limitting list + * + * \param mem Pointer to a lzma_memlimit structure returned + * earlier by lzma_memry_limit_create(). + * \param ptr Pointer returned earlier by lzma_memlimit_alloc(). + * + * This function removes ptr from the internal list and decreases the + * counter of used memory accordingly. The ptr itself isn't freed. This is + * useful when Extra Records allocated by liblzma using lzma_memlimit + * are needed by the application and must not be freed when the + * lzma_memlimit structure is destroyed. + * + * It is OK to call this function with ptr that hasn't been allocated with + * lzma_memlimit_alloc(). In that case, this has no effect other than wasting + * a few CPU cycles. + */ +extern void lzma_memlimit_detach(lzma_memlimit *mem, void *ptr); + + +/** + * \brief Frees memory and updates the memory limit list + * + * This is like lzma_memlimit_detach() but also frees the given pointer. + */ +extern void lzma_memlimit_free(lzma_memlimit *mem, void *ptr); + + +/** + * \brief Frees the memory allocated for and by the memory usage limitter + * + * \param mem Pointer to memory limitter + * \param free_allocated If this is non-zero, all the memory allocated + * by lzma_memlimit_alloc() using *mem is also + * freed if it hasn't already been freed with + * lzma_memlimit_free(). Usually this should be + * set to true. + */ +extern void lzma_memlimit_end( + lzma_memlimit *mem, lzma_bool free_allocated); diff --git a/src/liblzma/api/lzma/metadata.h b/src/liblzma/api/lzma/metadata.h new file mode 100644 index 00000000..69592a3a --- /dev/null +++ b/src/liblzma/api/lzma/metadata.h @@ -0,0 +1,100 @@ +/** + * \file lzma/metadata.h + * \brief Metadata handling + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Information stored into a Metadata Block + * + * This structure holds all the information that can be stored to + * a Metadata Block. + */ +typedef struct { + /** + * \brief Size of Header Metadata Block + */ + lzma_vli header_metadata_size; + + /** + * \brief Total Size of the Stream + */ + lzma_vli total_size; + + /** + * \brief Uncompressed Size of the Stream + */ + lzma_vli uncompressed_size; + + /** + * \brief Index of the Blocks stored in the Stream + */ + lzma_index *index; + + /** + * \brief Extra information + */ + lzma_extra *extra; + +} lzma_metadata; + + +/** + * \brief Calculate the encoded size of Metadata + * + * \return Uncompressed size of the Metadata in encoded form. This value + * may be passed to Block encoder as Uncompressed Size when using + * Metadata filter. On error, zero is returned. + */ +extern lzma_vli lzma_metadata_size(const lzma_metadata *metadata); + + +/** + * \brief Initializes Metadata encoder + * + * \param coder Pointer to a pointer to hold Metadata encoder's + * internal state. Original value is ignored, thus + * you don't need to initialize the pointer. + * \param allocator Custom memory allocator; usually NULL. + * \param metadata Pointer to Metadata to encoded + * + * \return - LZMA_OK: Initialization succeeded. + * - LZMA_MEM_ERROR: Cannot allocate memory for *coder. + * + * The initialization function makes internal copy of the *metadata structure. + * However, the linked lists metadata->index and metadata->extra are NOT + * copied. Thus, the application may destroy *metadata after initialization + * if it likes, but not Index or Extra. + */ +extern lzma_ret lzma_metadata_encoder(lzma_stream *strm, + lzma_options_block *options, const lzma_metadata *metadata); + + +/** + * \brief Initializes Metadata decoder + * + * \param want_extra If this is true, Extra Records will be stored + * to metadata->extra. If this is false, Extra + * Records will be parsed but not stored anywhere, + * metadata->extra will be set to NULL. + */ +extern lzma_ret lzma_metadata_decoder( + lzma_stream *strm, lzma_options_block *options, + lzma_metadata *metadata, lzma_bool want_extra); diff --git a/src/liblzma/api/lzma/raw.h b/src/liblzma/api/lzma/raw.h new file mode 100644 index 00000000..c1ee41d8 --- /dev/null +++ b/src/liblzma/api/lzma/raw.h @@ -0,0 +1,72 @@ +/** + * \file lzma/raw.h + * \brief Raw encoder and decoder + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Initializes raw encoder + * + * This function may be useful when implementing custom file formats. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Array of lzma_options_filter structures. + * The end of the array must be marked with + * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum + * number of filters is zero; the maximum is + * determined by available memory. + * \param uncompressed_size + * Size of the uncompressed data. If it is unknown, + * use LZMA_VLI_VALUE_UNKNOWN. You need to give the + * same value to the raw decoder to decode the data. + * \param allow_implicit + * If true, an implicit Copy or Subblock filter should be + * automatically added when needed. If this is false and + * an implicit filter would be needed, LZMA_PROG_ERROR is + * returned. + * + * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * filter chain support it), or LZMA_FINISH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_encoder( + lzma_stream *strm, const lzma_options_filter *options, + lzma_vli uncompressed_size, lzma_bool allow_implicit); + + +/** + * \brief Initializes raw decoder + * + * The initialization of raw decoder goes similarly to raw encoder. + * + * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_decoder( + lzma_stream *strm, const lzma_options_filter *options, + lzma_vli uncompressed_size, lzma_bool allow_implicit); diff --git a/src/liblzma/api/lzma/simple.h b/src/liblzma/api/lzma/simple.h new file mode 100644 index 00000000..fb78d01f --- /dev/null +++ b/src/liblzma/api/lzma/simple.h @@ -0,0 +1,85 @@ +/** + * \file lzma/simple.h + * \brief So called "simple" filters + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/* Filter IDs for lzma_options_filter.id */ + +#define LZMA_FILTER_X86 LZMA_VLI_C(0x04) + /**< + * BCJ (Branch, Call, Jump) filter for x86 binaries + */ + +#define LZMA_FILTER_POWERPC LZMA_VLI_C(0x05) + /**< + * Filter for Big endian PowerPC binaries + */ + +#define LZMA_FILTER_IA64 LZMA_VLI_C(0x06) + /**< + * Filter for IA64 (Itanium) binaries. + */ + +#define LZMA_FILTER_ARM LZMA_VLI_C(0x07) + /**< + * Filter for ARM binaries. + */ + +#define LZMA_FILTER_ARMTHUMB LZMA_VLI_C(0x08) + /**< + * Filter for ARMThumb binaries. + */ + +#define LZMA_FILTER_SPARC LZMA_VLI_C(0x09) + /**< + * Filter for SPARC binaries. + */ + + +/** + * \brief Options for so called "simple" filters + * + * The simple filters never change the size of the data. Specifying options + * for them is optional: if pointer to options is NULL, default values are + * used. You probably never need to specify these options, so just set the + * options pointer to NULL and be happy. + * + * If options with non-default values have been specified when encoding, + * the same options must also be specified when decoding. + */ +typedef struct { + /** + * \brief Start offset for branch conversions + * + * This setting is useful only when the same filter is used + * _separately_ for multiple sections of the same executable file, + * and the sections contain cross-section branch/call/jump + * instructions. In that case it is benefical to set the start + * offset of the non-first sections so that the relative addresses + * of the cross-section branch/call/jump instructions will use the + * same absolute addresses as in the first section. + * + * When the pointer to options is NULL, the default value is used. + * The default value is zero. + */ + uint32_t start_offset; + +} lzma_options_simple; diff --git a/src/liblzma/api/lzma/stream.h b/src/liblzma/api/lzma/stream.h new file mode 100644 index 00000000..be86075f --- /dev/null +++ b/src/liblzma/api/lzma/stream.h @@ -0,0 +1,178 @@ +/** + * \file lzma/stream.h + * \brief .lzma Stream handling + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Options for .lzma Stream encoder + */ +typedef struct { + /** + * \brief Type of integrity Check + * + * The type of the integrity Check is stored into Stream Header + * and Stream Footer. The same Check is used for all Blocks in + * the Stream. + */ + lzma_check_type check; + + /** + * \brief Precense of CRC32 of the Block Header + * + * Set this to true if CRC32 of every Block Header should be + * calculated and stored in the Block Header. This is recommended. + * + * This setting is stored into Stream Header and Stream Footer. + */ + lzma_bool has_crc32; + + /** + * \brief Uncompressed Size in bytes + * + * This is somewhat advanced feature. Most users want to set this to + * LZMA_VLI_VALUE_UNKNOWN to indicate unknown Uncompressed Size. + * + * If the Uncompressed Size of the Stream being encoded is known, + * it can be stored to the beginning of the Stream. The details + * differ for Single-Block and Multi-Block Streams: + * - With Single-Block Streams, the Uncompressed Size is stored to + * the Block Header and End of Payload Marker is omitted. + * - With Multi-Block Streams, the Uncompressed Size is stored to + * the Header Metadata Block. The Uncompressed Size of the Blocks + * will be unknown, because liblzma cannot predict how the + * application is going to split the data in Blocks. + */ + lzma_vli uncompressed_size; + + /** + * \brief Alignment of the beginning of the Stream + * + * Certain filters handle data in bigger chunks than single bytes. + * This affects two things: + * - Performance: aligned memory access is usually faster. + * - Further compression ratio in custom file formats: if you + * encode multiple Blocks with some non-compression filter + * such as LZMA_FILTER_POWERPC, it is a good idea to keep + * the inter-Block alignment correct to maximize compression + * ratio when all these Blocks are finally compressed as a + * single step. + * + * Usually the Stream is stored into its own file, thus + * the alignment is usually zero. + */ + uint32_t alignment; + + /** + * \brief Array of filters used to encode Data Blocks + * + * There can be at maximum of seven filters. The end of the array is + * marked with .id = LZMA_VLI_VALUE_UNKNOWN. (That's why the array + * has eight members.) Minimum number of filters is zero; in that + * case, an implicit Copy filter is used. + */ + lzma_options_filter filters[8]; + + /** + * \brief Array of filters used to encode Metadata Blocks + * + * This is like filters[] but for Metadata Blocks. If Metadata + * Blocks are compressed, they usually are compressed with + * settings that require only little memory to uncompress e.g. + * LZMA with 64 KiB dictionary. + * + * \todo Recommend a preset. + * + * When liblzma sees that the Metadata Block would be very small + * even in uncompressed form, it is not compressed no matter + * what filter have been set here. This is to avoid possibly + * increasing the size of the Metadata Block with bad compression, + * and to avoid useless overhead of filters in uncompression phase. + */ + lzma_options_filter metadata_filters[8]; + + /** + * \brief Extra information in the Header Metadata Block + */ + lzma_extra *header; + + /** + * \brief Extra information in the Footer Metadata Block + * + * It is enough to set this pointer any time before calling + * lzma_code() with LZMA_FINISH as the second argument. + */ + lzma_extra *footer; + +} lzma_options_stream; + + +/** + * \brief Initializes Single-Block .lzma Stream encoder + * + * This is the function that most developers are looking for. :-) It + * compresses using the specified options without storing any extra + * information. + * + * \todo Describe that is_metadata is ignored, maybe something else. + */ +extern lzma_ret lzma_stream_encoder_single( + lzma_stream *strm, const lzma_options_stream *options); + + +/** + * \brief Initializes Multi-Block .lzma Stream encoder + * + */ +extern lzma_ret lzma_stream_encoder_multi( + lzma_stream *strm, const lzma_options_stream *options); + + +/** + * \brief Initializes decoder for .lzma Stream + * + * \param strm Pointer to propertily prepared lzma_stream + * \param header Pointer to hold a pointer to Extra Records read + * from the Header Metadata Block. Use NULL if + * you don't care about Extra Records. + * \param footer Same as header, but for Footer Metadata Block. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + * + * If header and/or footer are not NULL, *header and/or *footer will be + * initially set to NULL. + * + * The application can detect that Header Metadata Block has been completely + * parsed when the decoder procudes some output the first time. If *header + * is still NULL, there was no Extra field in the Header Metadata Block (or + * the whole Header Metadata Block wasn't present at all). + * + * The application can detect that Footer Metadata Block has been parsed + * completely when lzma_code() returns LZMA_STREAM_END. If *footer is still + * NULL, there was no Extra field in the Footer Metadata Block. + * + * \note If you use lzma_memory_limitter, the Extra Records will be + * allocated with it, and thus remain in the lzma_memory_limitter + * even after they get exported to the application via *header + * and *footer pointers. + */ +extern lzma_ret lzma_stream_decoder(lzma_stream *strm, + lzma_extra **header, lzma_extra **footer); diff --git a/src/liblzma/api/lzma/stream_flags.h b/src/liblzma/api/lzma/stream_flags.h new file mode 100644 index 00000000..070c91c9 --- /dev/null +++ b/src/liblzma/api/lzma/stream_flags.h @@ -0,0 +1,142 @@ +/** + * \file lzma/stream_flags.h + * \brief .lzma Stream Header and Stream tail encoder and decoder + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Size of Stream Header + * + * Magic Bytes (6) + Stream Flags (1) + CRC32 (4) + */ +#define LZMA_STREAM_HEADER_SIZE (6 + 1 + 4) + + +/** + * \brief Size of Stream tail + * + * Because Stream Footer already has a defined meaning in the file format + * specification, we use Stream tail to denote these two fields: + * Stream Flags (1) + Magic Bytes (2) + */ +#define LZMA_STREAM_TAIL_SIZE (1 + 2) + + +/** + * Options for encoding and decoding Stream Header and Stream tail + */ +typedef struct { + /** + * Type of the Check calculated from uncompressed data + */ + lzma_check_type check; + + /** + * True if Block Headers have the CRC32 field. Note that the CRC32 + * field is always present in the Stream Header. + */ + lzma_bool has_crc32; + + /** + * True if the Stream is a Multi-Block Stream. + */ + lzma_bool is_multi; + +} lzma_stream_flags; + + +#define lzma_stream_flags_is_equal(a, b) \ + ((a).check == (b).check \ + && (a).has_crc32 == (b).has_crc32 \ + && (a).is_multi == (b).is_multi) + + +/** + * \brief Encodes Stream Header + * + * Encoding of the Stream Header is done with a single call instead of + * first initializing and then doing the actual work with lzma_code(). + * + * \param out Beginning of the output buffer + * \param out_pos out[*out_pos] is the next write position. This + * is updated by the encoder. + * \param out_size out[out_size] is the first byte to not write. + * \param options Stream Header options to be encoded. + * + * \return - LZMA_OK: Encoding was successful. + * - LZMA_PROG_ERROR: Invalid options. + * - LZMA_BUF_ERROR: Not enough output buffer space. + */ +extern lzma_ret lzma_stream_header_encode( + uint8_t *out, const lzma_stream_flags *options); + + +/** + * \brief Encodes Stream tail + * + * \param footer Pointer to a pointer that will hold the + * allocated buffer. Caller must free it once + * it isn't needed anymore. + * \param footer_size Pointer to a variable that will the final size + * of the footer buffer. + * \param allocator lzma_allocator for custom allocator functions. + * Set to NULL to use malloc(). + * \param options Stream Header options to be encoded. + * + * \return - LZMA_OK: Success; *header and *header_size set. + * - LZMA_PROG_ERROR: *options is invalid. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_stream_tail_encode( + uint8_t *out, const lzma_stream_flags *options); + + +/** + * \brief Initializes Stream Header decoder + * + * \param strm Pointer to lzma_stream used to pass input data + * \param options Target structure for parsed results + * + * \return - LZMA_OK: Successfully initialized + * - LZMA_MEM_ERROR: Cannot allocate memory + * + * The actual decoding is done with lzma_code() and freed with lzma_end(). + */ +extern lzma_ret lzma_stream_header_decoder( + lzma_stream *strm, lzma_stream_flags *options); + + +/** + * \brief Initializes Stream tail decoder + * + * \param strm Pointer to lzma_stream used to pass input data + * \param options Target structure for parsed results. + * \param decode_uncompressed_size + * Set to true if the first field to decode is + * Uncompressed Size. Set to false if the first + * field to decode is Backward Size. + * + * \return - LZMA_OK: Successfully initialized + * - LZMA_MEM_ERROR: Cannot allocate memory + * + * The actual decoding is done with lzma_code() and freed with lzma_end(). + */ +extern lzma_ret lzma_stream_tail_decoder( + lzma_stream *strm, lzma_stream_flags *options); diff --git a/src/liblzma/api/lzma/subblock.h b/src/liblzma/api/lzma/subblock.h new file mode 100644 index 00000000..0474b6af --- /dev/null +++ b/src/liblzma/api/lzma/subblock.h @@ -0,0 +1,197 @@ +/** + * \file lzma/subblock.h + * \brief Subblock filter + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Filter ID + * + * Filter ID of the Subblock filter. This is used as lzma_options_filter.id. + */ +#define LZMA_FILTER_SUBBLOCK LZMA_VLI_C(0x01) + + +/** + * \brief Subfilter mode + * + * See lzma_options_subblock.subfilter_mode for details. + */ +typedef enum { + LZMA_SUBFILTER_NONE, + /**< + * No Subfilter is in use. + */ + + LZMA_SUBFILTER_SET, + /**< + * New Subfilter has been requested to be initialized. + */ + + LZMA_SUBFILTER_RUN, + /**< + * Subfilter is active. + */ + + LZMA_SUBFILTER_FINISH + /**< + * Subfilter has been requested to be finished. + */ +} lzma_subfilter_mode; + + +/** + * \brief Options for the Subblock filter + * + * Specifying options for the Subblock filter is optional: if the pointer + * options is NULL, no subfilters are allowed and the default value is used + * for subblock_data_size. + */ +typedef struct { + /* Options for encoder and decoder */ + + /** + * \brief Allowing subfilters + * + * If this true, subfilters are allowed. + * + * In the encoder, if this is set to false, subfilter_mode and + * subfilter_options are completely ignored. + */ + lzma_bool allow_subfilters; + + /* Options for encoder only */ + + /** + * \brief Alignment + * + * The Subblock filter encapsulates the input data into Subblocks. + * Each Subblock has a header which takes a few bytes of space. + * When the output of the Subblock encoder is fed to another filter + * that takes advantage of the alignment of the input data (e.g. LZMA), + * the Subblock filter can add padding to keep the actual data parts + * in the Subblocks aligned correctly. + * + * The alignment should be a positive integer. Subblock filter will + * add enough padding between Subblocks so that this is true for + * every payload byte: + * input_offset % alignment == output_offset % alignment + * + * The Subblock filter assumes that the first output byte will be + * written to a position in the output stream that is properly aligned. + * + * FIXME desc + */ + uint32_t alignment; +# define LZMA_SUBBLOCK_ALIGNMENT_MIN 1 +# define LZMA_SUBBLOCK_ALIGNMENT_MAX 32 +# define LZMA_SUBBLOCK_ALIGNMENT_DEFAULT 4 + + /** + * \brief Size of the Subblock Data part of each Subblock + * + * This value is re-read every time a new Subblock is started. + * + * Bigger values + * - save a few bytes of space; + * - increase latency in the encoder (but no effect for decoding); + * - decrease memory locality (increased cache pollution) in the + * encoder (no effect in decoding). + */ + uint32_t subblock_data_size; +# define LZMA_SUBBLOCK_DATA_SIZE_MIN 1 +# define LZMA_SUBBLOCK_DATA_SIZE_MAX (UINT32_C(1) << 28) +# define LZMA_SUBBLOCK_DATA_SIZE_DEFAULT 4096 + + /** + * \brief Run-length encoder remote control + * + * The Subblock filter has an internal run-length encoder (RLE). It + * can be useful when the data includes byte sequences that repeat + * very many times. The RLE can be used also when a Subfilter is + * in use; the RLE will be applied to the output of the Subfilter. + * + * Note that in contrast to traditional RLE, this RLE is intended to + * be used only when there's a lot of data to be repeated. If the + * input data has e.g. 500 bytes of NULs now and then, this RLE + * is probably useless, because plain LZMA should provide better + * results. + * + * Due to above reasons, it was decided to keep the implementation + * of the RLE very simple. When the rle variable is non-zero, it + * subblock_data_size must be a multiple of rle. Once the Subblock + * encoder has got subblock_data_size bytes of input, it will check + * if the whole buffer of the last subblock_data_size can be + * represented with repeats of chunks having size of rle bytes. + * + * If there are consecutive identical buffers of subblock_data_size + * bytes, they will be encoded using a single repeat entry if + * possible. + * + * If need arises, more advanced RLE can be implemented later + * without breaking API or ABI. + */ + uint32_t rle; +# define LZMA_SUBBLOCK_RLE_OFF 0 +# define LZMA_SUBBLOCK_RLE_MIN 1 +# define LZMA_SUBBLOCK_RLE_MAX 256 + + /** + * \brief Subfilter remote control + * + * When the Subblock filter is initialized, this variable must be + * LZMA_SUBFILTER_NONE or LZMA_SUBFILTER_SET. + * + * When subfilter_mode is LZMA_SUBFILTER_NONE, the application may + * put Subfilter options to subfilter_options structure, and then + * set subfilter_mode to LZMA_SUBFILTER_SET. This implies setting + * flush to true. No new input data will be read until the Subfilter + * has been enabled. Once the Subfilter has been enabled, liblzma + * will set subfilter_mode to LZMA_SUBFILTER_RUN. + * + * When subfilter_mode is LZMA_SUBFILTER_RUN, the application may + * set subfilter_mode to LZMA_SUBFILTER_FINISH. No new input data + * will be read until the Subfilter has been finished. Once the + * Subfilter has been finished, liblzma will set subfilter_mode + * to LZMA_SUBFILTER_NONE. + * + * If the intent is to have Subfilter enabled to the very end of + * the data, it is not needed to separately disable Subfilter with + * LZMA_SUBFILTER_FINISH. Using LZMA_FINISH as the second argument + * of lzma_code() will make the Subblock encoder to disable the + * Subfilter once all the data has been ran through the Subfilter. + * + * \note This variable is ignored if allow_subfilters is false. + */ + lzma_subfilter_mode subfilter_mode; + + /** + * \brief Subfilter and its options + * + * When no Subfilter is used, the data is copied as is into Subblocks. + * Setting a Subfilter allows encoding some parts of the data with + * an additional filter. It is possible to many different Subfilters + * in the same Block, although only one can be used at once. + * + * \note This variable is ignored if allow_subfilters is false. + */ + lzma_options_filter subfilter_options; + +} lzma_options_subblock; diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h new file mode 100644 index 00000000..ffbf8a81 --- /dev/null +++ b/src/liblzma/api/lzma/version.h @@ -0,0 +1,59 @@ +/** + * \file lzma/version.h + * \brief Version number + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Compile-time version number + * + * The version number is of format xyyyuuus where + * - x is the major LZMA SDK version + * - yyy is the minor LZMA SDK version + * - uuu is LZMA Utils version (reset to zero every time SDK version + * is incremented) + * - s indicates stability: 0 = alpha, 1 = beta, 2 = stable + */ +#define LZMA_VERSION UINT32_C(40420010) + + +/** + * \brief liblzma version number as an integer + * + * This is the value of LZMA_VERSION macro at the compile time of liblzma. + * This allows the application to compare if it was built against the same, + * older, or newer version of liblzma that is currently running. + */ +extern const uint32_t lzma_version_number; + + +/** + * \brief Returns versions number of liblzma as a string + * + * This function may be useful if you want to display which version of + * libilzma your application is currently using. + * + * \return Returns a pointer to a statically allocated string constant, + * which contains the version number of liblzma. The format of + * the version string is usually (but not necessarily) x.y.z + * e.g. "4.42.1". Alpha and beta versions contain a suffix + * ("4.42.0alpha"). + */ +extern const char *const lzma_version_string; diff --git a/src/liblzma/api/lzma/vli.h b/src/liblzma/api/lzma/vli.h new file mode 100644 index 00000000..322014e1 --- /dev/null +++ b/src/liblzma/api/lzma/vli.h @@ -0,0 +1,244 @@ +/** + * \file lzma/vli.h + * \brief Variable-length integer handling + * + * \author Copyright (C) 1999-2006 Igor Pavlov + * \author Copyright (C) 2007 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use <lzma.h> instead. +#endif + + +/** + * \brief Maximum supported value of variable-length integer + */ +#define LZMA_VLI_VALUE_MAX (UINT64_MAX / 2) + +/** + * \brief VLI value to denote that the value is unknown + */ +#define LZMA_VLI_VALUE_UNKNOWN UINT64_MAX + +/** + * \brief Maximum supported length of variable length integers + */ +#define LZMA_VLI_BYTES_MAX 9 + + +/** + * \brief VLI constant suffix + */ +#define LZMA_VLI_C(n) UINT64_C(n) + + +/** + * \brief Variable-length integer type + * + * This will always be unsigned integer. Valid VLI values are in the range + * [0, LZMA_VLI_VALUE_MAX]. Unknown value is indicated with + * LZMA_VLI_VALUE_UNKNOWN, which is the maximum value of the underlaying + * integer type (this feature is useful in several situations). + * + * In future, even if lzma_vli is typdefined to something else than uint64_t, + * it is guaranteed that 2 * LZMA_VLI_VALUE_MAX will not overflow lzma_vli. + * This simplifies integer overflow detection. + */ +typedef uint64_t lzma_vli; + + +/** + * \brief Simple macro to validate variable-length integer + * + * This is useful to test that application has given acceptable values + * for example in the uncompressed_size and compressed_size variables. + * + * \return True if the integer is representable as VLI or if it + * indicates unknown value. + */ +#define lzma_vli_is_valid(vli) \ + ((vli) <= LZMA_VLI_VALUE_MAX || (vli) == LZMA_VLI_VALUE_UNKNOWN) + + +/** + * \brief Sets VLI to given value with error checking + * + * \param dest Target variable which must have type of lzma_vli. + * \param src New value to be stored to dest. + * \param limit Maximum allowed value for src. + * + * \return False on success, true on error. If an error occurred, + * dest is left in undefined state (i.e. it's possible that + * it will be different in newer liblzma versions). + */ +#define lzma_vli_set_lim(dest, src, limit) \ + ((src) > (limit) || ((dest) = (src)) > (limit)) + +/** + * \brief + */ +#define lzma_vli_add_lim(dest, src, limit) \ + ((src) > (limit) || ((dest) += (src)) > (limit)) + +#define lzma_vli_add2_lim(dest, src1, src2, limit) \ + (lzma_vli_add_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit)) + +#define lzma_vli_add3_lim(dest, src1, src2, src3, limit) \ + (lzma_vli_add_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit) \ + || lzma_vli_add_lim(dest, src3, limit)) + +#define lzma_vli_add4_lim(dest, src1, src2, src3, src4, limit) \ + (lzma_vli_add_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit) \ + || lzma_vli_add_lim(dest, src3, limit) \ + || lzma_vli_add_lim(dest, src4, limit)) + +#define lzma_vli_sum_lim(dest, src1, src2, limit) \ + (lzma_vli_set_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit)) + +#define lzma_vli_sum3_lim(dest, src1, src2, src3, limit) \ + (lzma_vli_set_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit) \ + || lzma_vli_add_lim(dest, src3, limit)) + +#define lzma_vli_sum4_lim(dest, src1, src2, src3, src4, limit) \ + (lzma_vli_set_lim(dest, src1, limit) \ + || lzma_vli_add_lim(dest, src2, limit) \ + || lzma_vli_add_lim(dest, src3, limit) \ + || lzma_vli_add_lim(dest, src4, limit)) + +#define lzma_vli_set(dest, src) lzma_vli_set_lim(dest, src, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_add(dest, src) lzma_vli_add_lim(dest, src, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_add2(dest, src1, src2) \ + lzma_vli_add2_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_add3(dest, src1, src2, src3) \ + lzma_vli_add3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_add4(dest, src1, src2, src3, src4) \ + lzma_vli_add4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_sum(dest, src1, src2) \ + lzma_vli_sum_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_sum3(dest, src1, src2, src3) \ + lzma_vli_sum3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) + +#define lzma_vli_sum4(dest, src1, src2, src3, src4) \ + lzma_vli_sum4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) + + +/** + * \brief Encodes variable-length integer + * + * In the new .lzma format, most integers are encoded in variable-length + * representation. This saves space when smaller values are more likely + * than bigger values. + * + * The encoding scheme encodes seven bits to every byte, using minimum + * number of bytes required to represent the given value. In other words, + * it puts 7-63 bits into 1-9 bytes. This implementation limits the number + * of bits used to 63, thus num must be at maximum of INT64_MAX / 2. You + * may use LZMA_VLI_VALUE_MAX for clarity. + * + * \param vli Integer to be encoded + * \param vli_pos How many bytes have already been written out. This + * must be less than 9 before calling this function. + * \param vli_size Minimum size that the variable-length representation + * must take. This is useful if you want to use + * variable-length integers as padding. Usually you want + * to set this to zero. The maximum allowed value is 9. + * \param out Beginning of the output buffer + * \param out_pos The next byte will be written to out[*out_pos]. + * \param out_size Size of the out buffer; the first byte into + * which no data is written to is out[out_size]. + * + * \return - LZMA_OK: So far all OK, but the integer is not + * completely written out yet. + * - LZMA_STREAM_END: Integer successfully encoded. + * - LZMA_BUF_ERROR: No output space (*out_pos == out_size) + * - LZMA_PROG_ERROR: Arguments are not sane. + */ +extern lzma_ret lzma_vli_encode( + lzma_vli vli, size_t *lzma_restrict vli_pos, size_t vli_size, + uint8_t *lzma_restrict out, size_t *lzma_restrict out_pos, + size_t out_size); + + +/** + * \brief Decodes variable-length integer + * + * \param vli Pointer to decoded integer. The decoder will + * initialize it to zero when *vli_pos == 0, so + * application isn't required to initialize *vli. + * \param vli_pos How many bytes have already been decoded. When + * starting to decode a new integer, *vli_pos must + * be initialized to zero. + * \param in Beginning of the input buffer + * \param in_pos The next byte will be read from in[*in_pos]. + * \param in_size Size of the input buffer; the first byte that + * won't be read is in[in_size]. + * + * \return - LZMA_OK: So far all OK, but the integer is not + * completely decoded yet. + * - LZMA_STREAM_END: Integer successfully decoded. + * - LZMA_BUF_ERROR: No input data (*in_pos == in_size) + * - LZMA_DATA_ERROR: Integer is longer than nine bytes. + * - LZMA_PROG_ERROR: Arguments are not sane. + */ +extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli, + size_t *lzma_restrict vli_pos, const uint8_t *lzma_restrict in, + size_t *lzma_restrict in_pos, size_t in_size); + + +/** + * \brief Decodes variable-length integer reading buffer backwards + * + * The variable-length integer encoding is designed so that it can be read + * either from the beginning to the end, or from the end to the beginning. + * This feature is needed to make the Stream parseable backwards; + * specifically, to read the Backward Size field in Stream Footer. + * + * \param vli Pointer to variable to hold the decoded integer. + * \param in Beginning of the input buffer + * \param in_size Number of bytes available in the in[] buffer. + * On successful decoding, this is updated to match + * the number of bytes used. (in[*in_size - 1] is the + * first byte to process. After successful decoding, + * in[*in_size] will point to the first byte of the + * variable-length integer.) + * + * \return - LZMA_OK: Decoding successful + * - LZMA_DATA_ERROR: No valid variable-length integer was found. + * - LZMA_BUF_ERROR: Not enough input. Note that in practice, + * this tends to be a sign of broken input, because the + * applications usually do give as much input to this function + * as the applications have available. + */ +extern lzma_ret lzma_vli_reverse_decode( + lzma_vli *vli, const uint8_t *in, size_t *in_size); + + +/** + * \brief Gets the minimum number of bytes required to encode vli + * + * \return Number of bytes on success (1-9). If vli isn't valid, + * zero is returned. + */ +extern size_t lzma_vli_size(lzma_vli vli); diff --git a/src/liblzma/check/Makefile.am b/src/liblzma/check/Makefile.am new file mode 100644 index 00000000..f00d71a0 --- /dev/null +++ b/src/liblzma/check/Makefile.am @@ -0,0 +1,64 @@ +## +## This code has been put into the public domain. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +EXTRA_DIST = crc32_tablegen.c crc64_tablegen.c + +noinst_LTLIBRARIES = libcheck.la +libcheck_la_SOURCES = \ + check.c \ + check.h \ + check_init.c \ + check_byteswap.h \ + crc_macros.h +libcheck_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common + +if COND_CHECK_CRC32 + +if COND_ASM_X86 +libcheck_la_SOURCES += crc32_x86.s +else +libcheck_la_SOURCES += crc32.c +endif + +if COND_SMALL +libcheck_la_SOURCES += crc32_init.c +else +libcheck_la_SOURCES += crc32_table.c crc32_table_le.h crc32_table_be.h +endif + +endif + + +if COND_CHECK_CRC64 + +if COND_ASM_X86 +libcheck_la_SOURCES += crc64_x86.s +else +libcheck_la_SOURCES += crc64.c +endif + +if COND_SMALL +libcheck_la_SOURCES += crc64_init.c +else +libcheck_la_SOURCES += crc64_table.c crc64_table_le.h crc64_table_be.h +endif + +endif + + +if COND_CHECK_SHA256 +libcheck_la_SOURCES += sha256.c +# Hide bogus warning to allow usage of -Werror. If more issues like this +# pop up, we'll drop -Werror. +if COND_WNO_UNINITIALIZED +CFLAGS += -Wno-uninitialized +endif +endif diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c new file mode 100644 index 00000000..ba59af2e --- /dev/null +++ b/src/liblzma/check/check.c @@ -0,0 +1,160 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file check.c +/// \brief Check sizes +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "check.h" + +// See the .lzma header format specification section 2.2.2. +LZMA_API const uint32_t lzma_check_sizes[8] = { 0, 4, 4, 8, 16, 32, 32, 64 }; + + +LZMA_API const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1] = { + true, // LZMA_CHECK_NONE + +#ifdef HAVE_CHECK_CRC32 + true, +#else + false, +#endif + + false, // Reserved + +#ifdef HAVE_CHECK_CRC64 + true, +#else + false, +#endif + + false, // Reserved + +#ifdef HAVE_CHECK_SHA256 + true, +#else + false, +#endif + + false, // Reserved + false, // Reserved +}; + + +extern lzma_ret +lzma_check_init(lzma_check *check, lzma_check_type type) +{ + lzma_ret ret = LZMA_OK; + + switch (type) { + case LZMA_CHECK_NONE: + break; + +#ifdef HAVE_CHECK_CRC32 + case LZMA_CHECK_CRC32: + check->crc32 = 0; + break; +#endif + +#ifdef HAVE_CHECK_CRC64 + case LZMA_CHECK_CRC64: + check->crc64 = 0; + break; +#endif + +#ifdef HAVE_CHECK_SHA256 + case LZMA_CHECK_SHA256: + lzma_sha256_init(&check->sha256); + break; +#endif + + default: + if (type <= LZMA_CHECK_ID_MAX) + ret = LZMA_UNSUPPORTED_CHECK; + else + ret = LZMA_PROG_ERROR; + break; + } + + return ret; +} + + +extern void +lzma_check_update(lzma_check *check, lzma_check_type type, + const uint8_t *buf, size_t size) +{ + switch (type) { +#ifdef HAVE_CHECK_CRC32 + case LZMA_CHECK_CRC32: + check->crc32 = lzma_crc32(buf, size, check->crc32); + break; +#endif + +#ifdef HAVE_CHECK_CRC64 + case LZMA_CHECK_CRC64: + check->crc64 = lzma_crc64(buf, size, check->crc64); + break; +#endif + +#ifdef HAVE_CHECK_SHA256 + case LZMA_CHECK_SHA256: + lzma_sha256_update(buf, size, &check->sha256); + break; +#endif + + default: + break; + } + + return; +} + + +extern void +lzma_check_finish(lzma_check *check, lzma_check_type type) +{ +#ifdef HAVE_CHECK_SHA256 + if (type == LZMA_CHECK_SHA256) + lzma_sha256_finish(&check->sha256); +#endif + + return; +} + + +/* +extern bool +lzma_check_compare( + lzma_check *check1, lzma_check *check2, lzma_check_type type) +{ + bool ret; + + switch (type) { + case LZMA_CHECK_NONE: + break; + + case LZMA_CHECK_CRC32: + ret = check1->crc32 != check2->crc32; + break; + + case LZMA_CHECK_CRC64: + ret = check1->crc64 != check2->crc64; + break; + + default: + // Unsupported check + assert(type <= 7); + ret = false; + break; + } + + return ret; +} +*/ diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h new file mode 100644 index 00000000..74279ceb --- /dev/null +++ b/src/liblzma/check/check.h @@ -0,0 +1,102 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file check.h +/// \brief Prototypes for different check functions +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CHECK_H +#define LZMA_CHECK_H + +#include "common.h" + + +typedef struct { + /// Internal state + uint32_t state[8]; + + /// Temporary 8-byte aligned buffer to hold incomplete chunk. + /// After lzma_check_finish(), the first 32 bytes will contain + /// the final digest in big endian byte order. + uint8_t buffer[64]; + + /// Size of the message excluding padding + uint64_t size; + +} lzma_sha256; + + +/// \note This is not in the public API because this structure will +/// change in future. +typedef union { + uint32_t crc32; + uint64_t crc64; + lzma_sha256 sha256; +} lzma_check; + + +#ifdef HAVE_SMALL +extern uint32_t lzma_crc32_table[8][256]; +extern uint64_t lzma_crc64_table[4][256]; +#else +extern const uint32_t lzma_crc32_table[8][256]; +extern const uint64_t lzma_crc64_table[4][256]; +#endif + +// Generic + +/// \brief Initializes *check depending on type +/// +/// \return LZMA_OK on success. LZMA_UNSUPPORTED_CHECK if the type is not +/// supported by the current version or build of liblzma. +/// LZMA_PROG_ERROR if type > LZMA_CHECK_ID_MAX. +/// +extern lzma_ret lzma_check_init(lzma_check *check, lzma_check_type type); + +/// \brief Updates *check +/// +extern void lzma_check_update(lzma_check *check, lzma_check_type type, + const uint8_t *buf, size_t size); + +/// \brief Finishes *check +/// +extern void lzma_check_finish(lzma_check *check, lzma_check_type type); + + +/* +/// \brief Compare two checks +/// +/// \return false if the checks are identical; true if they differ. +/// +extern bool lzma_check_compare( + lzma_check *check1, lzma_check *check2, lzma_check_type type); +*/ + + +// CRC32 + +extern void lzma_crc32_init(void); + + +// CRC64 + +extern void lzma_crc64_init(void); + + +// SHA256 + +extern void lzma_sha256_init(lzma_sha256 *sha256); + +extern void lzma_sha256_update( + const uint8_t *buf, size_t size, lzma_sha256 *sha256); + +extern void lzma_sha256_finish(lzma_sha256 *sha256); + + +#endif diff --git a/src/liblzma/check/check_byteswap.h b/src/liblzma/check/check_byteswap.h new file mode 100644 index 00000000..264def26 --- /dev/null +++ b/src/liblzma/check/check_byteswap.h @@ -0,0 +1,43 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file check_byteswap.h +/// \brief Byteswapping needed by the checks +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CHECK_BYTESWAP_H +#define LZMA_CHECK_BYTESWAP_H + +#ifdef HAVE_CONFIG_H +# include <config.h> +#endif + +// byteswap.h is a GNU extension. It contains inline assembly versions +// for byteswapping. When byteswap.h is not available, we use generic code. +#ifdef HAVE_BYTESWAP_H +# include <byteswap.h> +#else +# define bswap_32(num) \ + ( (((num) << 24) ) \ + | (((num) << 8) & UINT32_C(0x00FF0000)) \ + | (((num) >> 8) & UINT32_C(0x0000FF00)) \ + | (((num) >> 24) ) ) + +# define bswap_64(num) \ + ( (((num) << 56) ) \ + | (((num) << 40) & UINT64_C(0x00FF000000000000)) \ + | (((num) << 24) & UINT64_C(0x0000FF0000000000)) \ + | (((num) << 8) & UINT64_C(0x000000FF00000000)) \ + | (((num) >> 8) & UINT64_C(0x00000000FF000000)) \ + | (((num) >> 24) & UINT64_C(0x0000000000FF0000)) \ + | (((num) >> 40) & UINT64_C(0x000000000000FF00)) \ + | (((num) >> 56) ) ) +#endif + +#endif diff --git a/src/liblzma/check/check_init.c b/src/liblzma/check/check_init.c new file mode 100644 index 00000000..1b2cfe02 --- /dev/null +++ b/src/liblzma/check/check_init.c @@ -0,0 +1,37 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file check_init.c +/// \brief Static initializations for integrity checks +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "check.h" + + +extern LZMA_API void +lzma_init_check(void) +{ +#ifdef HAVE_SMALL + static bool already_initialized = false; + if (already_initialized) + return; + +# ifdef HAVE_CHECK_CRC32 + lzma_crc32_init(); +# endif + +# ifdef HAVE_CHECK_CRC64 + lzma_crc64_init(); +# endif + + already_initialized = true; +#endif + + return; +} diff --git a/src/liblzma/check/crc32.c b/src/liblzma/check/crc32.c new file mode 100644 index 00000000..091e1422 --- /dev/null +++ b/src/liblzma/check/crc32.c @@ -0,0 +1,88 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32.c +/// \brief CRC32 calculation +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "check.h" +#include "crc_macros.h" + + +// If you make any changes, do some bench marking! Seemingly unrelated +// changes can very easily ruin the performance (and very probably is +// very compiler dependent). +extern uint32_t +lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +{ + crc = ~crc; + +#ifdef WORDS_BIGENDIAN + crc = bswap_32(crc); +#endif + + if (size > 8) { + // Fix the alignment, if needed. The if statement above + // ensures that this won't read past the end of buf[]. + while ((uintptr_t)(buf) & 7) { + crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc); + --size; + } + + // Calculate the position where to stop. + const uint8_t *const limit = buf + (size & ~(size_t)(7)); + + // Calculate how many bytes must be calculated separately + // before returning the result. + size &= (size_t)(7); + + // Calculate the CRC32 using the slice-by-eight algorithm. + // It is explained in this document: + // http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf + // + // The code below is different than the code in Intel's + // paper, but the principle is identical. This should be + // faster with GCC than Intel's code. This is tested only + // with GCC 3.4.6 and 4.1.2 on x86, so your results may vary. + // + // Using -Os and -fomit-frame-pointer seem to give the best + // results at least with GCC 4.1.2 on x86. It's sill far + // from the speed of hand-optimized assembler. + while (buf < limit) { + crc ^= *(uint32_t *)(buf); + buf += 4; + + crc = lzma_crc32_table[7][A(crc)] + ^ lzma_crc32_table[6][B(crc)] + ^ lzma_crc32_table[5][C(crc)] + ^ lzma_crc32_table[4][D(crc)]; + + const uint32_t tmp = *(uint32_t *)(buf); + buf += 4; + + // It is critical for performance, that + // the crc variable is XORed between the + // two table-lookup pairs. + crc = lzma_crc32_table[3][A(tmp)] + ^ lzma_crc32_table[2][B(tmp)] + ^ crc + ^ lzma_crc32_table[1][C(tmp)] + ^ lzma_crc32_table[0][D(tmp)]; + } + } + + while (size-- != 0) + crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc); + +#ifdef WORDS_BIGENDIAN + crc = bswap_32(crc); +#endif + + return ~crc; +} diff --git a/src/liblzma/check/crc32_init.c b/src/liblzma/check/crc32_init.c new file mode 100644 index 00000000..eee90400 --- /dev/null +++ b/src/liblzma/check/crc32_init.c @@ -0,0 +1,58 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_init.c +/// \brief CRC32 table initialization +// +// This code is based on various public domain sources. +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include <config.h> +#endif + +#include <sys/types.h> +#include <inttypes.h> + +#ifdef WORDS_BIGENDIAN +# include "check_byteswap.h" +#endif + + +uint32_t lzma_crc32_table[8][256]; + + +extern void +lzma_crc32_init(void) +{ + static const uint32_t poly32 = UINT32_C(0xEDB88320); + + for (size_t s = 0; s < 8; ++s) { + for (size_t b = 0; b < 256; ++b) { + uint32_t r = s == 0 ? b : lzma_crc32_table[s - 1][b]; + + for (size_t i = 0; i < 8; ++i) { + if (r & 1) + r = (r >> 1) ^ poly32; + else + r >>= 1; + } + + lzma_crc32_table[s][b] = r; + } + } + +#ifdef WORDS_BIGENDIAN + for (size_t s = 0; s < 8; ++s) + for (size_t b = 0; b < 256; ++b) + lzma_crc32_table[s][b] + = bswap_32(lzma_crc32_table[s][b]); +#endif + + return; +} diff --git a/src/liblzma/check/crc32_table.c b/src/liblzma/check/crc32_table.c new file mode 100644 index 00000000..b59642d4 --- /dev/null +++ b/src/liblzma/check/crc32_table.c @@ -0,0 +1,22 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_table.c +/// \brief Precalculated CRC32 table with correct endianness +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include <config.h> +#endif + +#ifdef WORDS_BIGENDIAN +# include "crc32_table_be.h" +#else +# include "crc32_table_le.h" +#endif diff --git a/src/liblzma/check/crc32_table_be.h b/src/liblzma/check/crc32_table_be.h new file mode 100644 index 00000000..bc5a5fbd --- /dev/null +++ b/src/liblzma/check/crc32_table_be.h @@ -0,0 +1,527 @@ +/* This file has been automatically generated by crc32_tablegen.c. */ + +#include <inttypes.h> + +const uint32_t lzma_crc32_table[8][256] = { + { + 0x00000000, 0x96300777, 0x2C610EEE, 0xBA510999, + 0x19C46D07, 0x8FF46A70, 0x35A563E9, 0xA395649E, + 0x3288DB0E, 0xA4B8DC79, 0x1EE9D5E0, 0x88D9D297, + 0x2B4CB609, 0xBD7CB17E, 0x072DB8E7, 0x911DBF90, + 0x6410B71D, 0xF220B06A, 0x4871B9F3, 0xDE41BE84, + 0x7DD4DA1A, 0xEBE4DD6D, 0x51B5D4F4, 0xC785D383, + 0x56986C13, 0xC0A86B64, 0x7AF962FD, 0xECC9658A, + 0x4F5C0114, 0xD96C0663, 0x633D0FFA, 0xF50D088D, + 0xC8206E3B, 0x5E10694C, 0xE44160D5, 0x727167A2, + 0xD1E4033C, 0x47D4044B, 0xFD850DD2, 0x6BB50AA5, + 0xFAA8B535, 0x6C98B242, 0xD6C9BBDB, 0x40F9BCAC, + 0xE36CD832, 0x755CDF45, 0xCF0DD6DC, 0x593DD1AB, + 0xAC30D926, 0x3A00DE51, 0x8051D7C8, 0x1661D0BF, + 0xB5F4B421, 0x23C4B356, 0x9995BACF, 0x0FA5BDB8, + 0x9EB80228, 0x0888055F, 0xB2D90CC6, 0x24E90BB1, + 0x877C6F2F, 0x114C6858, 0xAB1D61C1, 0x3D2D66B6, + 0x9041DC76, 0x0671DB01, 0xBC20D298, 0x2A10D5EF, + 0x8985B171, 0x1FB5B606, 0xA5E4BF9F, 0x33D4B8E8, + 0xA2C90778, 0x34F9000F, 0x8EA80996, 0x18980EE1, + 0xBB0D6A7F, 0x2D3D6D08, 0x976C6491, 0x015C63E6, + 0xF4516B6B, 0x62616C1C, 0xD8306585, 0x4E0062F2, + 0xED95066C, 0x7BA5011B, 0xC1F40882, 0x57C40FF5, + 0xC6D9B065, 0x50E9B712, 0xEAB8BE8B, 0x7C88B9FC, + 0xDF1DDD62, 0x492DDA15, 0xF37CD38C, 0x654CD4FB, + 0x5861B24D, 0xCE51B53A, 0x7400BCA3, 0xE230BBD4, + 0x41A5DF4A, 0xD795D83D, 0x6DC4D1A4, 0xFBF4D6D3, + 0x6AE96943, 0xFCD96E34, 0x468867AD, 0xD0B860DA, + 0x732D0444, 0xE51D0333, 0x5F4C0AAA, 0xC97C0DDD, + 0x3C710550, 0xAA410227, 0x10100BBE, 0x86200CC9, + 0x25B56857, 0xB3856F20, 0x09D466B9, 0x9FE461CE, + 0x0EF9DE5E, 0x98C9D929, 0x2298D0B0, 0xB4A8D7C7, + 0x173DB359, 0x810DB42E, 0x3B5CBDB7, 0xAD6CBAC0, + 0x2083B8ED, 0xB6B3BF9A, 0x0CE2B603, 0x9AD2B174, + 0x3947D5EA, 0xAF77D29D, 0x1526DB04, 0x8316DC73, + 0x120B63E3, 0x843B6494, 0x3E6A6D0D, 0xA85A6A7A, + 0x0BCF0EE4, 0x9DFF0993, 0x27AE000A, 0xB19E077D, + 0x44930FF0, 0xD2A30887, 0x68F2011E, 0xFEC20669, + 0x5D5762F7, 0xCB676580, 0x71366C19, 0xE7066B6E, + 0x761BD4FE, 0xE02BD389, 0x5A7ADA10, 0xCC4ADD67, + 0x6FDFB9F9, 0xF9EFBE8E, 0x43BEB717, 0xD58EB060, + 0xE8A3D6D6, 0x7E93D1A1, 0xC4C2D838, 0x52F2DF4F, + 0xF167BBD1, 0x6757BCA6, 0xDD06B53F, 0x4B36B248, + 0xDA2B0DD8, 0x4C1B0AAF, 0xF64A0336, 0x607A0441, + 0xC3EF60DF, 0x55DF67A8, 0xEF8E6E31, 0x79BE6946, + 0x8CB361CB, 0x1A8366BC, 0xA0D26F25, 0x36E26852, + 0x95770CCC, 0x03470BBB, 0xB9160222, 0x2F260555, + 0xBE3BBAC5, 0x280BBDB2, 0x925AB42B, 0x046AB35C, + 0xA7FFD7C2, 0x31CFD0B5, 0x8B9ED92C, 0x1DAEDE5B, + 0xB0C2649B, 0x26F263EC, 0x9CA36A75, 0x0A936D02, + 0xA906099C, 0x3F360EEB, 0x85670772, 0x13570005, + 0x824ABF95, 0x147AB8E2, 0xAE2BB17B, 0x381BB60C, + 0x9B8ED292, 0x0DBED5E5, 0xB7EFDC7C, 0x21DFDB0B, + 0xD4D2D386, 0x42E2D4F1, 0xF8B3DD68, 0x6E83DA1F, + 0xCD16BE81, 0x5B26B9F6, 0xE177B06F, 0x7747B718, + 0xE65A0888, 0x706A0FFF, 0xCA3B0666, 0x5C0B0111, + 0xFF9E658F, 0x69AE62F8, 0xD3FF6B61, 0x45CF6C16, + 0x78E20AA0, 0xEED20DD7, 0x5483044E, 0xC2B30339, + 0x612667A7, 0xF71660D0, 0x4D476949, 0xDB776E3E, + 0x4A6AD1AE, 0xDC5AD6D9, 0x660BDF40, 0xF03BD837, + 0x53AEBCA9, 0xC59EBBDE, 0x7FCFB247, 0xE9FFB530, + 0x1CF2BDBD, 0x8AC2BACA, 0x3093B353, 0xA6A3B424, + 0x0536D0BA, 0x9306D7CD, 0x2957DE54, 0xBF67D923, + 0x2E7A66B3, 0xB84A61C4, 0x021B685D, 0x942B6F2A, + 0x37BE0BB4, 0xA18E0CC3, 0x1BDF055A, 0x8DEF022D + }, { + 0x00000000, 0x41311B19, 0x82623632, 0xC3532D2B, + 0x04C56C64, 0x45F4777D, 0x86A75A56, 0xC796414F, + 0x088AD9C8, 0x49BBC2D1, 0x8AE8EFFA, 0xCBD9F4E3, + 0x0C4FB5AC, 0x4D7EAEB5, 0x8E2D839E, 0xCF1C9887, + 0x5112C24A, 0x1023D953, 0xD370F478, 0x9241EF61, + 0x55D7AE2E, 0x14E6B537, 0xD7B5981C, 0x96848305, + 0x59981B82, 0x18A9009B, 0xDBFA2DB0, 0x9ACB36A9, + 0x5D5D77E6, 0x1C6C6CFF, 0xDF3F41D4, 0x9E0E5ACD, + 0xA2248495, 0xE3159F8C, 0x2046B2A7, 0x6177A9BE, + 0xA6E1E8F1, 0xE7D0F3E8, 0x2483DEC3, 0x65B2C5DA, + 0xAAAE5D5D, 0xEB9F4644, 0x28CC6B6F, 0x69FD7076, + 0xAE6B3139, 0xEF5A2A20, 0x2C09070B, 0x6D381C12, + 0xF33646DF, 0xB2075DC6, 0x715470ED, 0x30656BF4, + 0xF7F32ABB, 0xB6C231A2, 0x75911C89, 0x34A00790, + 0xFBBC9F17, 0xBA8D840E, 0x79DEA925, 0x38EFB23C, + 0xFF79F373, 0xBE48E86A, 0x7D1BC541, 0x3C2ADE58, + 0x054F79F0, 0x447E62E9, 0x872D4FC2, 0xC61C54DB, + 0x018A1594, 0x40BB0E8D, 0x83E823A6, 0xC2D938BF, + 0x0DC5A038, 0x4CF4BB21, 0x8FA7960A, 0xCE968D13, + 0x0900CC5C, 0x4831D745, 0x8B62FA6E, 0xCA53E177, + 0x545DBBBA, 0x156CA0A3, 0xD63F8D88, 0x970E9691, + 0x5098D7DE, 0x11A9CCC7, 0xD2FAE1EC, 0x93CBFAF5, + 0x5CD76272, 0x1DE6796B, 0xDEB55440, 0x9F844F59, + 0x58120E16, 0x1923150F, 0xDA703824, 0x9B41233D, + 0xA76BFD65, 0xE65AE67C, 0x2509CB57, 0x6438D04E, + 0xA3AE9101, 0xE29F8A18, 0x21CCA733, 0x60FDBC2A, + 0xAFE124AD, 0xEED03FB4, 0x2D83129F, 0x6CB20986, + 0xAB2448C9, 0xEA1553D0, 0x29467EFB, 0x687765E2, + 0xF6793F2F, 0xB7482436, 0x741B091D, 0x352A1204, + 0xF2BC534B, 0xB38D4852, 0x70DE6579, 0x31EF7E60, + 0xFEF3E6E7, 0xBFC2FDFE, 0x7C91D0D5, 0x3DA0CBCC, + 0xFA368A83, 0xBB07919A, 0x7854BCB1, 0x3965A7A8, + 0x4B98833B, 0x0AA99822, 0xC9FAB509, 0x88CBAE10, + 0x4F5DEF5F, 0x0E6CF446, 0xCD3FD96D, 0x8C0EC274, + 0x43125AF3, 0x022341EA, 0xC1706CC1, 0x804177D8, + 0x47D73697, 0x06E62D8E, 0xC5B500A5, 0x84841BBC, + 0x1A8A4171, 0x5BBB5A68, 0x98E87743, 0xD9D96C5A, + 0x1E4F2D15, 0x5F7E360C, 0x9C2D1B27, 0xDD1C003E, + 0x120098B9, 0x533183A0, 0x9062AE8B, 0xD153B592, + 0x16C5F4DD, 0x57F4EFC4, 0x94A7C2EF, 0xD596D9F6, + 0xE9BC07AE, 0xA88D1CB7, 0x6BDE319C, 0x2AEF2A85, + 0xED796BCA, 0xAC4870D3, 0x6F1B5DF8, 0x2E2A46E1, + 0xE136DE66, 0xA007C57F, 0x6354E854, 0x2265F34D, + 0xE5F3B202, 0xA4C2A91B, 0x67918430, 0x26A09F29, + 0xB8AEC5E4, 0xF99FDEFD, 0x3ACCF3D6, 0x7BFDE8CF, + 0xBC6BA980, 0xFD5AB299, 0x3E099FB2, 0x7F3884AB, + 0xB0241C2C, 0xF1150735, 0x32462A1E, 0x73773107, + 0xB4E17048, 0xF5D06B51, 0x3683467A, 0x77B25D63, + 0x4ED7FACB, 0x0FE6E1D2, 0xCCB5CCF9, 0x8D84D7E0, + 0x4A1296AF, 0x0B238DB6, 0xC870A09D, 0x8941BB84, + 0x465D2303, 0x076C381A, 0xC43F1531, 0x850E0E28, + 0x42984F67, 0x03A9547E, 0xC0FA7955, 0x81CB624C, + 0x1FC53881, 0x5EF42398, 0x9DA70EB3, 0xDC9615AA, + 0x1B0054E5, 0x5A314FFC, 0x996262D7, 0xD85379CE, + 0x174FE149, 0x567EFA50, 0x952DD77B, 0xD41CCC62, + 0x138A8D2D, 0x52BB9634, 0x91E8BB1F, 0xD0D9A006, + 0xECF37E5E, 0xADC26547, 0x6E91486C, 0x2FA05375, + 0xE836123A, 0xA9070923, 0x6A542408, 0x2B653F11, + 0xE479A796, 0xA548BC8F, 0x661B91A4, 0x272A8ABD, + 0xE0BCCBF2, 0xA18DD0EB, 0x62DEFDC0, 0x23EFE6D9, + 0xBDE1BC14, 0xFCD0A70D, 0x3F838A26, 0x7EB2913F, + 0xB924D070, 0xF815CB69, 0x3B46E642, 0x7A77FD5B, + 0xB56B65DC, 0xF45A7EC5, 0x370953EE, 0x763848F7, + 0xB1AE09B8, 0xF09F12A1, 0x33CC3F8A, 0x72FD2493 + }, { + 0x00000000, 0x376AC201, 0x6ED48403, 0x59BE4602, + 0xDCA80907, 0xEBC2CB06, 0xB27C8D04, 0x85164F05, + 0xB851130E, 0x8F3BD10F, 0xD685970D, 0xE1EF550C, + 0x64F91A09, 0x5393D808, 0x0A2D9E0A, 0x3D475C0B, + 0x70A3261C, 0x47C9E41D, 0x1E77A21F, 0x291D601E, + 0xAC0B2F1B, 0x9B61ED1A, 0xC2DFAB18, 0xF5B56919, + 0xC8F23512, 0xFF98F713, 0xA626B111, 0x914C7310, + 0x145A3C15, 0x2330FE14, 0x7A8EB816, 0x4DE47A17, + 0xE0464D38, 0xD72C8F39, 0x8E92C93B, 0xB9F80B3A, + 0x3CEE443F, 0x0B84863E, 0x523AC03C, 0x6550023D, + 0x58175E36, 0x6F7D9C37, 0x36C3DA35, 0x01A91834, + 0x84BF5731, 0xB3D59530, 0xEA6BD332, 0xDD011133, + 0x90E56B24, 0xA78FA925, 0xFE31EF27, 0xC95B2D26, + 0x4C4D6223, 0x7B27A022, 0x2299E620, 0x15F32421, + 0x28B4782A, 0x1FDEBA2B, 0x4660FC29, 0x710A3E28, + 0xF41C712D, 0xC376B32C, 0x9AC8F52E, 0xADA2372F, + 0xC08D9A70, 0xF7E75871, 0xAE591E73, 0x9933DC72, + 0x1C259377, 0x2B4F5176, 0x72F11774, 0x459BD575, + 0x78DC897E, 0x4FB64B7F, 0x16080D7D, 0x2162CF7C, + 0xA4748079, 0x931E4278, 0xCAA0047A, 0xFDCAC67B, + 0xB02EBC6C, 0x87447E6D, 0xDEFA386F, 0xE990FA6E, + 0x6C86B56B, 0x5BEC776A, 0x02523168, 0x3538F369, + 0x087FAF62, 0x3F156D63, 0x66AB2B61, 0x51C1E960, + 0xD4D7A665, 0xE3BD6464, 0xBA032266, 0x8D69E067, + 0x20CBD748, 0x17A11549, 0x4E1F534B, 0x7975914A, + 0xFC63DE4F, 0xCB091C4E, 0x92B75A4C, 0xA5DD984D, + 0x989AC446, 0xAFF00647, 0xF64E4045, 0xC1248244, + 0x4432CD41, 0x73580F40, 0x2AE64942, 0x1D8C8B43, + 0x5068F154, 0x67023355, 0x3EBC7557, 0x09D6B756, + 0x8CC0F853, 0xBBAA3A52, 0xE2147C50, 0xD57EBE51, + 0xE839E25A, 0xDF53205B, 0x86ED6659, 0xB187A458, + 0x3491EB5D, 0x03FB295C, 0x5A456F5E, 0x6D2FAD5F, + 0x801B35E1, 0xB771F7E0, 0xEECFB1E2, 0xD9A573E3, + 0x5CB33CE6, 0x6BD9FEE7, 0x3267B8E5, 0x050D7AE4, + 0x384A26EF, 0x0F20E4EE, 0x569EA2EC, 0x61F460ED, + 0xE4E22FE8, 0xD388EDE9, 0x8A36ABEB, 0xBD5C69EA, + 0xF0B813FD, 0xC7D2D1FC, 0x9E6C97FE, 0xA90655FF, + 0x2C101AFA, 0x1B7AD8FB, 0x42C49EF9, 0x75AE5CF8, + 0x48E900F3, 0x7F83C2F2, 0x263D84F0, 0x115746F1, + 0x944109F4, 0xA32BCBF5, 0xFA958DF7, 0xCDFF4FF6, + 0x605D78D9, 0x5737BAD8, 0x0E89FCDA, 0x39E33EDB, + 0xBCF571DE, 0x8B9FB3DF, 0xD221F5DD, 0xE54B37DC, + 0xD80C6BD7, 0xEF66A9D6, 0xB6D8EFD4, 0x81B22DD5, + 0x04A462D0, 0x33CEA0D1, 0x6A70E6D3, 0x5D1A24D2, + 0x10FE5EC5, 0x27949CC4, 0x7E2ADAC6, 0x494018C7, + 0xCC5657C2, 0xFB3C95C3, 0xA282D3C1, 0x95E811C0, + 0xA8AF4DCB, 0x9FC58FCA, 0xC67BC9C8, 0xF1110BC9, + 0x740744CC, 0x436D86CD, 0x1AD3C0CF, 0x2DB902CE, + 0x4096AF91, 0x77FC6D90, 0x2E422B92, 0x1928E993, + 0x9C3EA696, 0xAB546497, 0xF2EA2295, 0xC580E094, + 0xF8C7BC9F, 0xCFAD7E9E, 0x9613389C, 0xA179FA9D, + 0x246FB598, 0x13057799, 0x4ABB319B, 0x7DD1F39A, + 0x3035898D, 0x075F4B8C, 0x5EE10D8E, 0x698BCF8F, + 0xEC9D808A, 0xDBF7428B, 0x82490489, 0xB523C688, + 0x88649A83, 0xBF0E5882, 0xE6B01E80, 0xD1DADC81, + 0x54CC9384, 0x63A65185, 0x3A181787, 0x0D72D586, + 0xA0D0E2A9, 0x97BA20A8, 0xCE0466AA, 0xF96EA4AB, + 0x7C78EBAE, 0x4B1229AF, 0x12AC6FAD, 0x25C6ADAC, + 0x1881F1A7, 0x2FEB33A6, 0x765575A4, 0x413FB7A5, + 0xC429F8A0, 0xF3433AA1, 0xAAFD7CA3, 0x9D97BEA2, + 0xD073C4B5, 0xE71906B4, 0xBEA740B6, 0x89CD82B7, + 0x0CDBCDB2, 0x3BB10FB3, 0x620F49B1, 0x55658BB0, + 0x6822D7BB, 0x5F4815BA, 0x06F653B8, 0x319C91B9, + 0xB48ADEBC, 0x83E01CBD, 0xDA5E5ABF, 0xED3498BE + }, { + 0x00000000, 0x6567BCB8, 0x8BC809AA, 0xEEAFB512, + 0x5797628F, 0x32F0DE37, 0xDC5F6B25, 0xB938D79D, + 0xEF28B4C5, 0x8A4F087D, 0x64E0BD6F, 0x018701D7, + 0xB8BFD64A, 0xDDD86AF2, 0x3377DFE0, 0x56106358, + 0x9F571950, 0xFA30A5E8, 0x149F10FA, 0x71F8AC42, + 0xC8C07BDF, 0xADA7C767, 0x43087275, 0x266FCECD, + 0x707FAD95, 0x1518112D, 0xFBB7A43F, 0x9ED01887, + 0x27E8CF1A, 0x428F73A2, 0xAC20C6B0, 0xC9477A08, + 0x3EAF32A0, 0x5BC88E18, 0xB5673B0A, 0xD00087B2, + 0x6938502F, 0x0C5FEC97, 0xE2F05985, 0x8797E53D, + 0xD1878665, 0xB4E03ADD, 0x5A4F8FCF, 0x3F283377, + 0x8610E4EA, 0xE3775852, 0x0DD8ED40, 0x68BF51F8, + 0xA1F82BF0, 0xC49F9748, 0x2A30225A, 0x4F579EE2, + 0xF66F497F, 0x9308F5C7, 0x7DA740D5, 0x18C0FC6D, + 0x4ED09F35, 0x2BB7238D, 0xC518969F, 0xA07F2A27, + 0x1947FDBA, 0x7C204102, 0x928FF410, 0xF7E848A8, + 0x3D58149B, 0x583FA823, 0xB6901D31, 0xD3F7A189, + 0x6ACF7614, 0x0FA8CAAC, 0xE1077FBE, 0x8460C306, + 0xD270A05E, 0xB7171CE6, 0x59B8A9F4, 0x3CDF154C, + 0x85E7C2D1, 0xE0807E69, 0x0E2FCB7B, 0x6B4877C3, + 0xA20F0DCB, 0xC768B173, 0x29C70461, 0x4CA0B8D9, + 0xF5986F44, 0x90FFD3FC, 0x7E5066EE, 0x1B37DA56, + 0x4D27B90E, 0x284005B6, 0xC6EFB0A4, 0xA3880C1C, + 0x1AB0DB81, 0x7FD76739, 0x9178D22B, 0xF41F6E93, + 0x03F7263B, 0x66909A83, 0x883F2F91, 0xED589329, + 0x546044B4, 0x3107F80C, 0xDFA84D1E, 0xBACFF1A6, + 0xECDF92FE, 0x89B82E46, 0x67179B54, 0x027027EC, + 0xBB48F071, 0xDE2F4CC9, 0x3080F9DB, 0x55E74563, + 0x9CA03F6B, 0xF9C783D3, 0x176836C1, 0x720F8A79, + 0xCB375DE4, 0xAE50E15C, 0x40FF544E, 0x2598E8F6, + 0x73888BAE, 0x16EF3716, 0xF8408204, 0x9D273EBC, + 0x241FE921, 0x41785599, 0xAFD7E08B, 0xCAB05C33, + 0x3BB659ED, 0x5ED1E555, 0xB07E5047, 0xD519ECFF, + 0x6C213B62, 0x094687DA, 0xE7E932C8, 0x828E8E70, + 0xD49EED28, 0xB1F95190, 0x5F56E482, 0x3A31583A, + 0x83098FA7, 0xE66E331F, 0x08C1860D, 0x6DA63AB5, + 0xA4E140BD, 0xC186FC05, 0x2F294917, 0x4A4EF5AF, + 0xF3762232, 0x96119E8A, 0x78BE2B98, 0x1DD99720, + 0x4BC9F478, 0x2EAE48C0, 0xC001FDD2, 0xA566416A, + 0x1C5E96F7, 0x79392A4F, 0x97969F5D, 0xF2F123E5, + 0x05196B4D, 0x607ED7F5, 0x8ED162E7, 0xEBB6DE5F, + 0x528E09C2, 0x37E9B57A, 0xD9460068, 0xBC21BCD0, + 0xEA31DF88, 0x8F566330, 0x61F9D622, 0x049E6A9A, + 0xBDA6BD07, 0xD8C101BF, 0x366EB4AD, 0x53090815, + 0x9A4E721D, 0xFF29CEA5, 0x11867BB7, 0x74E1C70F, + 0xCDD91092, 0xA8BEAC2A, 0x46111938, 0x2376A580, + 0x7566C6D8, 0x10017A60, 0xFEAECF72, 0x9BC973CA, + 0x22F1A457, 0x479618EF, 0xA939ADFD, 0xCC5E1145, + 0x06EE4D76, 0x6389F1CE, 0x8D2644DC, 0xE841F864, + 0x51792FF9, 0x341E9341, 0xDAB12653, 0xBFD69AEB, + 0xE9C6F9B3, 0x8CA1450B, 0x620EF019, 0x07694CA1, + 0xBE519B3C, 0xDB362784, 0x35999296, 0x50FE2E2E, + 0x99B95426, 0xFCDEE89E, 0x12715D8C, 0x7716E134, + 0xCE2E36A9, 0xAB498A11, 0x45E63F03, 0x208183BB, + 0x7691E0E3, 0x13F65C5B, 0xFD59E949, 0x983E55F1, + 0x2106826C, 0x44613ED4, 0xAACE8BC6, 0xCFA9377E, + 0x38417FD6, 0x5D26C36E, 0xB389767C, 0xD6EECAC4, + 0x6FD61D59, 0x0AB1A1E1, 0xE41E14F3, 0x8179A84B, + 0xD769CB13, 0xB20E77AB, 0x5CA1C2B9, 0x39C67E01, + 0x80FEA99C, 0xE5991524, 0x0B36A036, 0x6E511C8E, + 0xA7166686, 0xC271DA3E, 0x2CDE6F2C, 0x49B9D394, + 0xF0810409, 0x95E6B8B1, 0x7B490DA3, 0x1E2EB11B, + 0x483ED243, 0x2D596EFB, 0xC3F6DBE9, 0xA6916751, + 0x1FA9B0CC, 0x7ACE0C74, 0x9461B966, 0xF10605DE + }, { + 0x00000000, 0xB029603D, 0x6053C07A, 0xD07AA047, + 0xC0A680F5, 0x708FE0C8, 0xA0F5408F, 0x10DC20B2, + 0xC14B7030, 0x7162100D, 0xA118B04A, 0x1131D077, + 0x01EDF0C5, 0xB1C490F8, 0x61BE30BF, 0xD1975082, + 0x8297E060, 0x32BE805D, 0xE2C4201A, 0x52ED4027, + 0x42316095, 0xF21800A8, 0x2262A0EF, 0x924BC0D2, + 0x43DC9050, 0xF3F5F06D, 0x238F502A, 0x93A63017, + 0x837A10A5, 0x33537098, 0xE329D0DF, 0x5300B0E2, + 0x042FC1C1, 0xB406A1FC, 0x647C01BB, 0xD4556186, + 0xC4894134, 0x74A02109, 0xA4DA814E, 0x14F3E173, + 0xC564B1F1, 0x754DD1CC, 0xA537718B, 0x151E11B6, + 0x05C23104, 0xB5EB5139, 0x6591F17E, 0xD5B89143, + 0x86B821A1, 0x3691419C, 0xE6EBE1DB, 0x56C281E6, + 0x461EA154, 0xF637C169, 0x264D612E, 0x96640113, + 0x47F35191, 0xF7DA31AC, 0x27A091EB, 0x9789F1D6, + 0x8755D164, 0x377CB159, 0xE706111E, 0x572F7123, + 0x4958F358, 0xF9719365, 0x290B3322, 0x9922531F, + 0x89FE73AD, 0x39D71390, 0xE9ADB3D7, 0x5984D3EA, + 0x88138368, 0x383AE355, 0xE8404312, 0x5869232F, + 0x48B5039D, 0xF89C63A0, 0x28E6C3E7, 0x98CFA3DA, + 0xCBCF1338, 0x7BE67305, 0xAB9CD342, 0x1BB5B37F, + 0x0B6993CD, 0xBB40F3F0, 0x6B3A53B7, 0xDB13338A, + 0x0A846308, 0xBAAD0335, 0x6AD7A372, 0xDAFEC34F, + 0xCA22E3FD, 0x7A0B83C0, 0xAA712387, 0x1A5843BA, + 0x4D773299, 0xFD5E52A4, 0x2D24F2E3, 0x9D0D92DE, + 0x8DD1B26C, 0x3DF8D251, 0xED827216, 0x5DAB122B, + 0x8C3C42A9, 0x3C152294, 0xEC6F82D3, 0x5C46E2EE, + 0x4C9AC25C, 0xFCB3A261, 0x2CC90226, 0x9CE0621B, + 0xCFE0D2F9, 0x7FC9B2C4, 0xAFB31283, 0x1F9A72BE, + 0x0F46520C, 0xBF6F3231, 0x6F159276, 0xDF3CF24B, + 0x0EABA2C9, 0xBE82C2F4, 0x6EF862B3, 0xDED1028E, + 0xCE0D223C, 0x7E244201, 0xAE5EE246, 0x1E77827B, + 0x92B0E6B1, 0x2299868C, 0xF2E326CB, 0x42CA46F6, + 0x52166644, 0xE23F0679, 0x3245A63E, 0x826CC603, + 0x53FB9681, 0xE3D2F6BC, 0x33A856FB, 0x838136C6, + 0x935D1674, 0x23747649, 0xF30ED60E, 0x4327B633, + 0x102706D1, 0xA00E66EC, 0x7074C6AB, 0xC05DA696, + 0xD0818624, 0x60A8E619, 0xB0D2465E, 0x00FB2663, + 0xD16C76E1, 0x614516DC, 0xB13FB69B, 0x0116D6A6, + 0x11CAF614, 0xA1E39629, 0x7199366E, 0xC1B05653, + 0x969F2770, 0x26B6474D, 0xF6CCE70A, 0x46E58737, + 0x5639A785, 0xE610C7B8, 0x366A67FF, 0x864307C2, + 0x57D45740, 0xE7FD377D, 0x3787973A, 0x87AEF707, + 0x9772D7B5, 0x275BB788, 0xF72117CF, 0x470877F2, + 0x1408C710, 0xA421A72D, 0x745B076A, 0xC4726757, + 0xD4AE47E5, 0x648727D8, 0xB4FD879F, 0x04D4E7A2, + 0xD543B720, 0x656AD71D, 0xB510775A, 0x05391767, + 0x15E537D5, 0xA5CC57E8, 0x75B6F7AF, 0xC59F9792, + 0xDBE815E9, 0x6BC175D4, 0xBBBBD593, 0x0B92B5AE, + 0x1B4E951C, 0xAB67F521, 0x7B1D5566, 0xCB34355B, + 0x1AA365D9, 0xAA8A05E4, 0x7AF0A5A3, 0xCAD9C59E, + 0xDA05E52C, 0x6A2C8511, 0xBA562556, 0x0A7F456B, + 0x597FF589, 0xE95695B4, 0x392C35F3, 0x890555CE, + 0x99D9757C, 0x29F01541, 0xF98AB506, 0x49A3D53B, + 0x983485B9, 0x281DE584, 0xF86745C3, 0x484E25FE, + 0x5892054C, 0xE8BB6571, 0x38C1C536, 0x88E8A50B, + 0xDFC7D428, 0x6FEEB415, 0xBF941452, 0x0FBD746F, + 0x1F6154DD, 0xAF4834E0, 0x7F3294A7, 0xCF1BF49A, + 0x1E8CA418, 0xAEA5C425, 0x7EDF6462, 0xCEF6045F, + 0xDE2A24ED, 0x6E0344D0, 0xBE79E497, 0x0E5084AA, + 0x5D503448, 0xED795475, 0x3D03F432, 0x8D2A940F, + 0x9DF6B4BD, 0x2DDFD480, 0xFDA574C7, 0x4D8C14FA, + 0x9C1B4478, 0x2C322445, 0xFC488402, 0x4C61E43F, + 0x5CBDC48D, 0xEC94A4B0, 0x3CEE04F7, 0x8CC764CA + }, { + 0x00000000, 0xA5D35CCB, 0x0BA1C84D, 0xAE729486, + 0x1642919B, 0xB391CD50, 0x1DE359D6, 0xB830051D, + 0x6D8253EC, 0xC8510F27, 0x66239BA1, 0xC3F0C76A, + 0x7BC0C277, 0xDE139EBC, 0x70610A3A, 0xD5B256F1, + 0x9B02D603, 0x3ED18AC8, 0x90A31E4E, 0x35704285, + 0x8D404798, 0x28931B53, 0x86E18FD5, 0x2332D31E, + 0xF68085EF, 0x5353D924, 0xFD214DA2, 0x58F21169, + 0xE0C21474, 0x451148BF, 0xEB63DC39, 0x4EB080F2, + 0x3605AC07, 0x93D6F0CC, 0x3DA4644A, 0x98773881, + 0x20473D9C, 0x85946157, 0x2BE6F5D1, 0x8E35A91A, + 0x5B87FFEB, 0xFE54A320, 0x502637A6, 0xF5F56B6D, + 0x4DC56E70, 0xE81632BB, 0x4664A63D, 0xE3B7FAF6, + 0xAD077A04, 0x08D426CF, 0xA6A6B249, 0x0375EE82, + 0xBB45EB9F, 0x1E96B754, 0xB0E423D2, 0x15377F19, + 0xC08529E8, 0x65567523, 0xCB24E1A5, 0x6EF7BD6E, + 0xD6C7B873, 0x7314E4B8, 0xDD66703E, 0x78B52CF5, + 0x6C0A580F, 0xC9D904C4, 0x67AB9042, 0xC278CC89, + 0x7A48C994, 0xDF9B955F, 0x71E901D9, 0xD43A5D12, + 0x01880BE3, 0xA45B5728, 0x0A29C3AE, 0xAFFA9F65, + 0x17CA9A78, 0xB219C6B3, 0x1C6B5235, 0xB9B80EFE, + 0xF7088E0C, 0x52DBD2C7, 0xFCA94641, 0x597A1A8A, + 0xE14A1F97, 0x4499435C, 0xEAEBD7DA, 0x4F388B11, + 0x9A8ADDE0, 0x3F59812B, 0x912B15AD, 0x34F84966, + 0x8CC84C7B, 0x291B10B0, 0x87698436, 0x22BAD8FD, + 0x5A0FF408, 0xFFDCA8C3, 0x51AE3C45, 0xF47D608E, + 0x4C4D6593, 0xE99E3958, 0x47ECADDE, 0xE23FF115, + 0x378DA7E4, 0x925EFB2F, 0x3C2C6FA9, 0x99FF3362, + 0x21CF367F, 0x841C6AB4, 0x2A6EFE32, 0x8FBDA2F9, + 0xC10D220B, 0x64DE7EC0, 0xCAACEA46, 0x6F7FB68D, + 0xD74FB390, 0x729CEF5B, 0xDCEE7BDD, 0x793D2716, + 0xAC8F71E7, 0x095C2D2C, 0xA72EB9AA, 0x02FDE561, + 0xBACDE07C, 0x1F1EBCB7, 0xB16C2831, 0x14BF74FA, + 0xD814B01E, 0x7DC7ECD5, 0xD3B57853, 0x76662498, + 0xCE562185, 0x6B857D4E, 0xC5F7E9C8, 0x6024B503, + 0xB596E3F2, 0x1045BF39, 0xBE372BBF, 0x1BE47774, + 0xA3D47269, 0x06072EA2, 0xA875BA24, 0x0DA6E6EF, + 0x4316661D, 0xE6C53AD6, 0x48B7AE50, 0xED64F29B, + 0x5554F786, 0xF087AB4D, 0x5EF53FCB, 0xFB266300, + 0x2E9435F1, 0x8B47693A, 0x2535FDBC, 0x80E6A177, + 0x38D6A46A, 0x9D05F8A1, 0x33776C27, 0x96A430EC, + 0xEE111C19, 0x4BC240D2, 0xE5B0D454, 0x4063889F, + 0xF8538D82, 0x5D80D149, 0xF3F245CF, 0x56211904, + 0x83934FF5, 0x2640133E, 0x883287B8, 0x2DE1DB73, + 0x95D1DE6E, 0x300282A5, 0x9E701623, 0x3BA34AE8, + 0x7513CA1A, 0xD0C096D1, 0x7EB20257, 0xDB615E9C, + 0x63515B81, 0xC682074A, 0x68F093CC, 0xCD23CF07, + 0x189199F6, 0xBD42C53D, 0x133051BB, 0xB6E30D70, + 0x0ED3086D, 0xAB0054A6, 0x0572C020, 0xA0A19CEB, + 0xB41EE811, 0x11CDB4DA, 0xBFBF205C, 0x1A6C7C97, + 0xA25C798A, 0x078F2541, 0xA9FDB1C7, 0x0C2EED0C, + 0xD99CBBFD, 0x7C4FE736, 0xD23D73B0, 0x77EE2F7B, + 0xCFDE2A66, 0x6A0D76AD, 0xC47FE22B, 0x61ACBEE0, + 0x2F1C3E12, 0x8ACF62D9, 0x24BDF65F, 0x816EAA94, + 0x395EAF89, 0x9C8DF342, 0x32FF67C4, 0x972C3B0F, + 0x429E6DFE, 0xE74D3135, 0x493FA5B3, 0xECECF978, + 0x54DCFC65, 0xF10FA0AE, 0x5F7D3428, 0xFAAE68E3, + 0x821B4416, 0x27C818DD, 0x89BA8C5B, 0x2C69D090, + 0x9459D58D, 0x318A8946, 0x9FF81DC0, 0x3A2B410B, + 0xEF9917FA, 0x4A4A4B31, 0xE438DFB7, 0x41EB837C, + 0xF9DB8661, 0x5C08DAAA, 0xF27A4E2C, 0x57A912E7, + 0x19199215, 0xBCCACEDE, 0x12B85A58, 0xB76B0693, + 0x0F5B038E, 0xAA885F45, 0x04FACBC3, 0xA1299708, + 0x749BC1F9, 0xD1489D32, 0x7F3A09B4, 0xDAE9557F, + 0x62D95062, 0xC70A0CA9, 0x6978982F, 0xCCABC4E4 + }, { + 0x00000000, 0xB40B77A6, 0x29119F97, 0x9D1AE831, + 0x13244FF4, 0xA72F3852, 0x3A35D063, 0x8E3EA7C5, + 0x674EEF33, 0xD3459895, 0x4E5F70A4, 0xFA540702, + 0x746AA0C7, 0xC061D761, 0x5D7B3F50, 0xE97048F6, + 0xCE9CDE67, 0x7A97A9C1, 0xE78D41F0, 0x53863656, + 0xDDB89193, 0x69B3E635, 0xF4A90E04, 0x40A279A2, + 0xA9D23154, 0x1DD946F2, 0x80C3AEC3, 0x34C8D965, + 0xBAF67EA0, 0x0EFD0906, 0x93E7E137, 0x27EC9691, + 0x9C39BDCF, 0x2832CA69, 0xB5282258, 0x012355FE, + 0x8F1DF23B, 0x3B16859D, 0xA60C6DAC, 0x12071A0A, + 0xFB7752FC, 0x4F7C255A, 0xD266CD6B, 0x666DBACD, + 0xE8531D08, 0x5C586AAE, 0xC142829F, 0x7549F539, + 0x52A563A8, 0xE6AE140E, 0x7BB4FC3F, 0xCFBF8B99, + 0x41812C5C, 0xF58A5BFA, 0x6890B3CB, 0xDC9BC46D, + 0x35EB8C9B, 0x81E0FB3D, 0x1CFA130C, 0xA8F164AA, + 0x26CFC36F, 0x92C4B4C9, 0x0FDE5CF8, 0xBBD52B5E, + 0x79750B44, 0xCD7E7CE2, 0x506494D3, 0xE46FE375, + 0x6A5144B0, 0xDE5A3316, 0x4340DB27, 0xF74BAC81, + 0x1E3BE477, 0xAA3093D1, 0x372A7BE0, 0x83210C46, + 0x0D1FAB83, 0xB914DC25, 0x240E3414, 0x900543B2, + 0xB7E9D523, 0x03E2A285, 0x9EF84AB4, 0x2AF33D12, + 0xA4CD9AD7, 0x10C6ED71, 0x8DDC0540, 0x39D772E6, + 0xD0A73A10, 0x64AC4DB6, 0xF9B6A587, 0x4DBDD221, + 0xC38375E4, 0x77880242, 0xEA92EA73, 0x5E999DD5, + 0xE54CB68B, 0x5147C12D, 0xCC5D291C, 0x78565EBA, + 0xF668F97F, 0x42638ED9, 0xDF7966E8, 0x6B72114E, + 0x820259B8, 0x36092E1E, 0xAB13C62F, 0x1F18B189, + 0x9126164C, 0x252D61EA, 0xB83789DB, 0x0C3CFE7D, + 0x2BD068EC, 0x9FDB1F4A, 0x02C1F77B, 0xB6CA80DD, + 0x38F42718, 0x8CFF50BE, 0x11E5B88F, 0xA5EECF29, + 0x4C9E87DF, 0xF895F079, 0x658F1848, 0xD1846FEE, + 0x5FBAC82B, 0xEBB1BF8D, 0x76AB57BC, 0xC2A0201A, + 0xF2EA1688, 0x46E1612E, 0xDBFB891F, 0x6FF0FEB9, + 0xE1CE597C, 0x55C52EDA, 0xC8DFC6EB, 0x7CD4B14D, + 0x95A4F9BB, 0x21AF8E1D, 0xBCB5662C, 0x08BE118A, + 0x8680B64F, 0x328BC1E9, 0xAF9129D8, 0x1B9A5E7E, + 0x3C76C8EF, 0x887DBF49, 0x15675778, 0xA16C20DE, + 0x2F52871B, 0x9B59F0BD, 0x0643188C, 0xB2486F2A, + 0x5B3827DC, 0xEF33507A, 0x7229B84B, 0xC622CFED, + 0x481C6828, 0xFC171F8E, 0x610DF7BF, 0xD5068019, + 0x6ED3AB47, 0xDAD8DCE1, 0x47C234D0, 0xF3C94376, + 0x7DF7E4B3, 0xC9FC9315, 0x54E67B24, 0xE0ED0C82, + 0x099D4474, 0xBD9633D2, 0x208CDBE3, 0x9487AC45, + 0x1AB90B80, 0xAEB27C26, 0x33A89417, 0x87A3E3B1, + 0xA04F7520, 0x14440286, 0x895EEAB7, 0x3D559D11, + 0xB36B3AD4, 0x07604D72, 0x9A7AA543, 0x2E71D2E5, + 0xC7019A13, 0x730AEDB5, 0xEE100584, 0x5A1B7222, + 0xD425D5E7, 0x602EA241, 0xFD344A70, 0x493F3DD6, + 0x8B9F1DCC, 0x3F946A6A, 0xA28E825B, 0x1685F5FD, + 0x98BB5238, 0x2CB0259E, 0xB1AACDAF, 0x05A1BA09, + 0xECD1F2FF, 0x58DA8559, 0xC5C06D68, 0x71CB1ACE, + 0xFFF5BD0B, 0x4BFECAAD, 0xD6E4229C, 0x62EF553A, + 0x4503C3AB, 0xF108B40D, 0x6C125C3C, 0xD8192B9A, + 0x56278C5F, 0xE22CFBF9, 0x7F3613C8, 0xCB3D646E, + 0x224D2C98, 0x96465B3E, 0x0B5CB30F, 0xBF57C4A9, + 0x3169636C, 0x856214CA, 0x1878FCFB, 0xAC738B5D, + 0x17A6A003, 0xA3ADD7A5, 0x3EB73F94, 0x8ABC4832, + 0x0482EFF7, 0xB0899851, 0x2D937060, 0x999807C6, + 0x70E84F30, 0xC4E33896, 0x59F9D0A7, 0xEDF2A701, + 0x63CC00C4, 0xD7C77762, 0x4ADD9F53, 0xFED6E8F5, + 0xD93A7E64, 0x6D3109C2, 0xF02BE1F3, 0x44209655, + 0xCA1E3190, 0x7E154636, 0xE30FAE07, 0x5704D9A1, + 0xBE749157, 0x0A7FE6F1, 0x97650EC0, 0x236E7966, + 0xAD50DEA3, 0x195BA905, 0x84414134, 0x304A3692 + }, { + 0x00000000, 0x9E00AACC, 0x7D072542, 0xE3078F8E, + 0xFA0E4A84, 0x640EE048, 0x87096FC6, 0x1909C50A, + 0xB51BE5D3, 0x2B1B4F1F, 0xC81CC091, 0x561C6A5D, + 0x4F15AF57, 0xD115059B, 0x32128A15, 0xAC1220D9, + 0x2B31BB7C, 0xB53111B0, 0x56369E3E, 0xC83634F2, + 0xD13FF1F8, 0x4F3F5B34, 0xAC38D4BA, 0x32387E76, + 0x9E2A5EAF, 0x002AF463, 0xE32D7BED, 0x7D2DD121, + 0x6424142B, 0xFA24BEE7, 0x19233169, 0x87239BA5, + 0x566276F9, 0xC862DC35, 0x2B6553BB, 0xB565F977, + 0xAC6C3C7D, 0x326C96B1, 0xD16B193F, 0x4F6BB3F3, + 0xE379932A, 0x7D7939E6, 0x9E7EB668, 0x007E1CA4, + 0x1977D9AE, 0x87777362, 0x6470FCEC, 0xFA705620, + 0x7D53CD85, 0xE3536749, 0x0054E8C7, 0x9E54420B, + 0x875D8701, 0x195D2DCD, 0xFA5AA243, 0x645A088F, + 0xC8482856, 0x5648829A, 0xB54F0D14, 0x2B4FA7D8, + 0x324662D2, 0xAC46C81E, 0x4F414790, 0xD141ED5C, + 0xEDC29D29, 0x73C237E5, 0x90C5B86B, 0x0EC512A7, + 0x17CCD7AD, 0x89CC7D61, 0x6ACBF2EF, 0xF4CB5823, + 0x58D978FA, 0xC6D9D236, 0x25DE5DB8, 0xBBDEF774, + 0xA2D7327E, 0x3CD798B2, 0xDFD0173C, 0x41D0BDF0, + 0xC6F32655, 0x58F38C99, 0xBBF40317, 0x25F4A9DB, + 0x3CFD6CD1, 0xA2FDC61D, 0x41FA4993, 0xDFFAE35F, + 0x73E8C386, 0xEDE8694A, 0x0EEFE6C4, 0x90EF4C08, + 0x89E68902, 0x17E623CE, 0xF4E1AC40, 0x6AE1068C, + 0xBBA0EBD0, 0x25A0411C, 0xC6A7CE92, 0x58A7645E, + 0x41AEA154, 0xDFAE0B98, 0x3CA98416, 0xA2A92EDA, + 0x0EBB0E03, 0x90BBA4CF, 0x73BC2B41, 0xEDBC818D, + 0xF4B54487, 0x6AB5EE4B, 0x89B261C5, 0x17B2CB09, + 0x909150AC, 0x0E91FA60, 0xED9675EE, 0x7396DF22, + 0x6A9F1A28, 0xF49FB0E4, 0x17983F6A, 0x899895A6, + 0x258AB57F, 0xBB8A1FB3, 0x588D903D, 0xC68D3AF1, + 0xDF84FFFB, 0x41845537, 0xA283DAB9, 0x3C837075, + 0xDA853B53, 0x4485919F, 0xA7821E11, 0x3982B4DD, + 0x208B71D7, 0xBE8BDB1B, 0x5D8C5495, 0xC38CFE59, + 0x6F9EDE80, 0xF19E744C, 0x1299FBC2, 0x8C99510E, + 0x95909404, 0x0B903EC8, 0xE897B146, 0x76971B8A, + 0xF1B4802F, 0x6FB42AE3, 0x8CB3A56D, 0x12B30FA1, + 0x0BBACAAB, 0x95BA6067, 0x76BDEFE9, 0xE8BD4525, + 0x44AF65FC, 0xDAAFCF30, 0x39A840BE, 0xA7A8EA72, + 0xBEA12F78, 0x20A185B4, 0xC3A60A3A, 0x5DA6A0F6, + 0x8CE74DAA, 0x12E7E766, 0xF1E068E8, 0x6FE0C224, + 0x76E9072E, 0xE8E9ADE2, 0x0BEE226C, 0x95EE88A0, + 0x39FCA879, 0xA7FC02B5, 0x44FB8D3B, 0xDAFB27F7, + 0xC3F2E2FD, 0x5DF24831, 0xBEF5C7BF, 0x20F56D73, + 0xA7D6F6D6, 0x39D65C1A, 0xDAD1D394, 0x44D17958, + 0x5DD8BC52, 0xC3D8169E, 0x20DF9910, 0xBEDF33DC, + 0x12CD1305, 0x8CCDB9C9, 0x6FCA3647, 0xF1CA9C8B, + 0xE8C35981, 0x76C3F34D, 0x95C47CC3, 0x0BC4D60F, + 0x3747A67A, 0xA9470CB6, 0x4A408338, 0xD44029F4, + 0xCD49ECFE, 0x53494632, 0xB04EC9BC, 0x2E4E6370, + 0x825C43A9, 0x1C5CE965, 0xFF5B66EB, 0x615BCC27, + 0x7852092D, 0xE652A3E1, 0x05552C6F, 0x9B5586A3, + 0x1C761D06, 0x8276B7CA, 0x61713844, 0xFF719288, + 0xE6785782, 0x7878FD4E, 0x9B7F72C0, 0x057FD80C, + 0xA96DF8D5, 0x376D5219, 0xD46ADD97, 0x4A6A775B, + 0x5363B251, 0xCD63189D, 0x2E649713, 0xB0643DDF, + 0x6125D083, 0xFF257A4F, 0x1C22F5C1, 0x82225F0D, + 0x9B2B9A07, 0x052B30CB, 0xE62CBF45, 0x782C1589, + 0xD43E3550, 0x4A3E9F9C, 0xA9391012, 0x3739BADE, + 0x2E307FD4, 0xB030D518, 0x53375A96, 0xCD37F05A, + 0x4A146BFF, 0xD414C133, 0x37134EBD, 0xA913E471, + 0xB01A217B, 0x2E1A8BB7, 0xCD1D0439, 0x531DAEF5, + 0xFF0F8E2C, 0x610F24E0, 0x8208AB6E, 0x1C0801A2, + 0x0501C4A8, 0x9B016E64, 0x7806E1EA, 0xE6064B26 + } +}; diff --git a/src/liblzma/check/crc32_table_le.h b/src/liblzma/check/crc32_table_le.h new file mode 100644 index 00000000..a96d772a --- /dev/null +++ b/src/liblzma/check/crc32_table_le.h @@ -0,0 +1,527 @@ +/* This file has been automatically generated by crc32_tablegen.c. */ + +#include <inttypes.h> + +const uint32_t lzma_crc32_table[8][256] = { + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }, { + 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, + 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, + 0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, + 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF, + 0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192, + 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496, + 0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A, + 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E, + 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761, + 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265, + 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69, + 0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D, + 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530, + 0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034, + 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38, + 0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C, + 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6, + 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2, + 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE, + 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA, + 0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97, + 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93, + 0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F, + 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B, + 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864, + 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60, + 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C, + 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768, + 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35, + 0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31, + 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D, + 0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539, + 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88, + 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C, + 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180, + 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484, + 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9, + 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD, + 0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1, + 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5, + 0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A, + 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E, + 0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522, + 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026, + 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B, + 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F, + 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773, + 0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277, + 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D, + 0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189, + 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85, + 0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81, + 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC, + 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8, + 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4, + 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0, + 0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F, + 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B, + 0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27, + 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23, + 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, + 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, + 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, + 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72 + }, { + 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, + 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, + 0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, + 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D, + 0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29, + 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5, + 0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91, + 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D, + 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9, + 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065, + 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901, + 0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD, + 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9, + 0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315, + 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71, + 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD, + 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399, + 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45, + 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221, + 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD, + 0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9, + 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835, + 0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151, + 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D, + 0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579, + 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5, + 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1, + 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D, + 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609, + 0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5, + 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1, + 0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D, + 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9, + 0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05, + 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461, + 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD, + 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9, + 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75, + 0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711, + 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD, + 0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339, + 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5, + 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281, + 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D, + 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049, + 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895, + 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1, + 0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D, + 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819, + 0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5, + 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1, + 0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D, + 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69, + 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5, + 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1, + 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D, + 0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9, + 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625, + 0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41, + 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D, + 0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, + 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, + 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, + 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED + }, { + 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, + 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, + 0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, + 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056, + 0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871, + 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26, + 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E, + 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9, + 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0, + 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787, + 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F, + 0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68, + 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F, + 0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018, + 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0, + 0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7, + 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3, + 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084, + 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C, + 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B, + 0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C, + 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B, + 0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3, + 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4, + 0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED, + 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA, + 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002, + 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755, + 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72, + 0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825, + 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D, + 0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA, + 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5, + 0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82, + 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A, + 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D, + 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A, + 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D, + 0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5, + 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2, + 0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB, + 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC, + 0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04, + 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953, + 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174, + 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623, + 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B, + 0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC, + 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8, + 0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF, + 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907, + 0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50, + 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677, + 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120, + 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98, + 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF, + 0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6, + 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981, + 0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639, + 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E, + 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, + 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, + 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, + 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1 + }, { + 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, + 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, + 0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, + 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1, + 0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52, + 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92, + 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693, + 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053, + 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4, + 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314, + 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15, + 0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5, + 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256, + 0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496, + 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997, + 0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57, + 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299, + 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459, + 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958, + 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98, + 0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B, + 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB, + 0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA, + 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A, + 0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D, + 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D, + 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C, + 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C, + 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F, + 0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF, + 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE, + 0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E, + 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42, + 0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82, + 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183, + 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743, + 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0, + 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00, + 0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601, + 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1, + 0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546, + 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386, + 0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87, + 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847, + 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4, + 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404, + 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905, + 0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5, + 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B, + 0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB, + 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA, + 0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A, + 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589, + 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349, + 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48, + 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888, + 0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F, + 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF, + 0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE, + 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E, + 0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, + 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, + 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, + 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C + }, { + 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, + 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, + 0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, + 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5, + 0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035, + 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223, + 0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258, + 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E, + 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798, + 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E, + 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5, + 0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3, + 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503, + 0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715, + 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E, + 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578, + 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2, + 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4, + 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF, + 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9, + 0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59, + 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F, + 0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834, + 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22, + 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4, + 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2, + 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99, + 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F, + 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F, + 0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79, + 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02, + 0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14, + 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676, + 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460, + 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B, + 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D, + 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED, + 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB, + 0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680, + 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496, + 0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340, + 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156, + 0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D, + 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B, + 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB, + 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD, + 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6, + 0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0, + 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A, + 0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C, + 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77, + 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61, + 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81, + 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97, + 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC, + 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA, + 0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C, + 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A, + 0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41, + 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957, + 0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, + 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, + 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, + 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC + }, { + 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, + 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, + 0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, + 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9, + 0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653, + 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240, + 0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834, + 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27, + 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301, + 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712, + 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66, + 0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975, + 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF, + 0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC, + 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8, + 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB, + 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4, + 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7, + 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183, + 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590, + 0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A, + 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739, + 0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D, + 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E, + 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678, + 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B, + 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F, + 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C, + 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6, + 0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5, + 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1, + 0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2, + 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F, + 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C, + 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08, + 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B, + 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1, + 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2, + 0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6, + 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5, + 0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3, + 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0, + 0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794, + 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387, + 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D, + 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E, + 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A, + 0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49, + 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516, + 0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105, + 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71, + 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62, + 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8, + 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB, + 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF, + 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC, + 0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A, + 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899, + 0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED, + 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE, + 0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, + 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, + 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, + 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30 + }, { + 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, + 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, + 0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, + 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC, + 0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8, + 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832, + 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D, + 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387, + 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5, + 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F, + 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00, + 0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA, + 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E, + 0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64, + 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B, + 0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1, + 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E, + 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4, + 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB, + 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041, + 0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425, + 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF, + 0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90, + 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A, + 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758, + 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2, + 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED, + 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217, + 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673, + 0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889, + 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6, + 0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C, + 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239, + 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3, + 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C, + 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776, + 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312, + 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8, + 0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7, + 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D, + 0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F, + 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95, + 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA, + 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520, + 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144, + 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE, + 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1, + 0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B, + 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4, + 0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E, + 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61, + 0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B, + 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF, + 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05, + 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A, + 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0, + 0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282, + 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78, + 0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937, + 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD, + 0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9, + 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53, + 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, + 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 + } +}; diff --git a/src/liblzma/check/crc32_tablegen.c b/src/liblzma/check/crc32_tablegen.c new file mode 100644 index 00000000..280d3b12 --- /dev/null +++ b/src/liblzma/check/crc32_tablegen.c @@ -0,0 +1,55 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_tablegen.c +/// \brief Generates CRC32 crc32_table.c +/// +/// Compiling: gcc -std=c99 -o crc32_tablegen crc32_tablegen.c crc32_init.c +/// Add -DWORDS_BIGENDIAN to generate big endian table. +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include <sys/types.h> +#include <inttypes.h> +#include <stdio.h> + + +extern void lzma_crc32_init(void); + +extern uint32_t lzma_crc32_table[8][256]; + + +int +main() +{ + lzma_crc32_init(); + + printf("/* This file has been automatically generated by " + "crc32_tablegen.c. */\n\n" + "#include <inttypes.h>\n\n" + "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); + + for (size_t s = 0; s < 8; ++s) { + for (size_t b = 0; b < 256; ++b) { + if ((b % 4) == 0) + printf("\n\t\t"); + + printf("0x%08" PRIX32, lzma_crc32_table[s][b]); + + if (b != 255) + printf(", "); + } + + if (s == 7) + printf("\n\t}\n};\n"); + else + printf("\n\t}, {"); + } + + return 0; +} diff --git a/src/liblzma/check/crc32_x86.s b/src/liblzma/check/crc32_x86.s new file mode 100644 index 00000000..ad4ef097 --- /dev/null +++ b/src/liblzma/check/crc32_x86.s @@ -0,0 +1,217 @@ +/* + * Speed-optimized CRC32 using slicing-by-eight algorithm + * Instruction set: i386 + * Optimized for: i686 + * + * This code has been put into the public domain by its authors: + * Original code by Igor Pavlov <http://7-zip.org/> + * Position-independent version by Lasse Collin <lasse.collin@tukaani.org> + * + * This code needs lzma_crc32_table, which can be created using the + * following C code: + +uint32_t lzma_crc32_table[8][256]; + +void +init_table(void) +{ + // IEEE-802.3 (CRC32) + static const uint32_t poly32 = UINT32_C(0xEDB88320); + + // Castagnoli (CRC32C) + // static const uint32_t poly32 = UINT32_C(0x82F63B78); + + // Koopman + // static const uint32_t poly32 = UINT32_C(0xEB31D82E); + + for (size_t s = 0; s < 8; ++s) { + for (size_t b = 0; b < 256; ++b) { + uint32_t r = s == 0 ? b : lzma_crc32_table[s - 1][b]; + + for (size_t i = 0; i < 8; ++i) { + if (r & 1) + r = (r >> 1) ^ poly32; + else + r >>= 1; + } + + lzma_crc32_table[s][b] = r; + } + } +} + + * The prototype of the CRC32 function: + * extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); + */ + + .text + .global lzma_crc32 + .type lzma_crc32, @function + + .align 16 +lzma_crc32: + /* + * Register usage: + * %eax crc + * %esi buf + * %edi size or buf + size + * %ebx lzma_crc32_table + * %ebp Table index + * %ecx Temporary + * %edx Temporary + */ + pushl %ebx + pushl %esi + pushl %edi + pushl %ebp + movl 0x14(%esp), %esi /* buf */ + movl 0x18(%esp), %edi /* size */ + movl 0x1C(%esp), %eax /* crc */ + + /* + * Store the address of lzma_crc32_table to %ebx. This is needed to + * get position-independent code (PIC). + */ + call .L_PIC +.L_PIC: + popl %ebx + addl $_GLOBAL_OFFSET_TABLE_+[.-.L_PIC], %ebx + movl lzma_crc32_table@GOT(%ebx), %ebx + + /* Complement the initial value. */ + notl %eax + + .align 16 +.L_align: + /* + * Check if there is enough input to use slicing-by-eight. + * We need 16 bytes, because the loop pre-reads eight bytes. + */ + cmpl $16, %edi + jl .L_rest + + /* Check if we have reached alignment of eight bytes. */ + testl $7, %esi + jz .L_slice + + /* Calculate CRC of the next input byte. */ + movzbl (%esi), %ebp + incl %esi + movzbl %al, %ecx + xorl %ecx, %ebp + shrl $8, %eax + xorl (%ebx, %ebp, 4), %eax + decl %edi + jmp .L_align + + .align 4 +.L_slice: + /* + * If we get here, there's at least 16 bytes of aligned input + * available. Make %edi multiple of eight bytes. Store the possible + * remainder over the "size" variable in the argument stack. + */ + movl %edi, 0x18(%esp) + andl $-8, %edi + subl %edi, 0x18(%esp) + + /* + * Let %edi be buf + size - 8 while running the main loop. This way + * we can compare for equality to determine when exit the loop. + */ + addl %esi, %edi + subl $8, %edi + + /* Read in the first eight aligned bytes. */ + xorl (%esi), %eax + movl 4(%esi), %ecx + movzbl %cl, %ebp + +.L_loop: + movl 0x0C00(%ebx, %ebp, 4), %edx + movzbl %ch, %ebp + xorl 0x0800(%ebx, %ebp, 4), %edx + shrl $16, %ecx + xorl 8(%esi), %edx + movzbl %cl, %ebp + xorl 0x0400(%ebx, %ebp, 4), %edx + movzbl %ch, %ebp + xorl (%ebx, %ebp, 4), %edx + movzbl %al, %ebp + + /* + * Read the next four bytes, for which the CRC is calculated + * on the next interation of the loop. + */ + movl 12(%esi), %ecx + + xorl 0x1C00(%ebx, %ebp, 4), %edx + movzbl %ah, %ebp + shrl $16, %eax + xorl 0x1800(%ebx, %ebp, 4), %edx + movzbl %ah, %ebp + movzbl %al, %eax + movl 0x1400(%ebx, %eax, 4), %eax + addl $8, %esi + xorl %edx, %eax + xorl 0x1000(%ebx, %ebp, 4), %eax + + /* Check for end of aligned input. */ + cmpl %edi, %esi + movzbl %cl, %ebp + jne .L_loop + + /* + * Process the remaining eight bytes, which we have already + * copied to %ecx and %edx. + */ + movl 0x0C00(%ebx, %ebp, 4), %edx + movzbl %ch, %ebp + xorl 0x0800(%ebx, %ebp, 4), %edx + shrl $16, %ecx + movzbl %cl, %ebp + xorl 0x0400(%ebx, %ebp, 4), %edx + movzbl %ch, %ebp + xorl (%ebx, %ebp, 4), %edx + movzbl %al, %ebp + + xorl 0x1C00(%ebx, %ebp, 4), %edx + movzbl %ah, %ebp + shrl $16, %eax + xorl 0x1800(%ebx, %ebp, 4), %edx + movzbl %ah, %ebp + movzbl %al, %eax + movl 0x1400(%ebx, %eax, 4), %eax + addl $8, %esi + xorl %edx, %eax + xorl 0x1000(%ebx, %ebp, 4), %eax + + /* Copy the number of remaining bytes to %edi. */ + movl 0x18(%esp), %edi + +.L_rest: + /* Check for end of input. */ + testl %edi, %edi + jz .L_return + + /* Calculate CRC of the next input byte. */ + movzbl (%esi), %ebp + incl %esi + movzbl %al, %ecx + xorl %ecx, %ebp + shrl $8, %eax + xorl (%ebx, %ebp, 4), %eax + decl %edi + jmp .L_rest + +.L_return: + /* Complement the final value. */ + notl %eax + + popl %ebp + popl %edi + popl %esi + popl %ebx + ret + + .size lzma_crc32, .-lzma_crc32 diff --git a/src/liblzma/check/crc64.c b/src/liblzma/check/crc64.c new file mode 100644 index 00000000..e31bc7ff --- /dev/null +++ b/src/liblzma/check/crc64.c @@ -0,0 +1,75 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc64.c +/// \brief CRC64 calculation +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "check.h" +#include "crc_macros.h" + + +#ifdef WORDS_BIGENDIAN +# define A1(x) ((x) >> 56) +#else +# define A1 A +#endif + + +// See comments in crc32.c. +extern uint64_t +lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) +{ + crc = ~crc; + +#ifdef WORDS_BIGENDIAN + crc = bswap_64(crc); +#endif + + if (size > 4) { + while ((uintptr_t)(buf) & 3) { + crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); + --size; + } + + const uint8_t *const limit = buf + (size & ~(size_t)(3)); + size &= (size_t)(3); + + // Calculate the CRC64 using the slice-by-four algorithm. + // + // In contrast to CRC32 code, this one seems to be fastest + // with -O3 -fomit-frame-pointer. + while (buf < limit) { +#ifdef WORDS_BIGENDIAN + const uint32_t tmp = (crc >> 32) ^ *(uint32_t *)(buf); +#else + const uint32_t tmp = crc ^ *(uint32_t *)(buf); +#endif + buf += 4; + + // It is critical for performance, that + // the crc variable is XORed between the + // two table-lookup pairs. + crc = lzma_crc64_table[3][A(tmp)] + ^ lzma_crc64_table[2][B(tmp)] + ^ S32(crc) + ^ lzma_crc64_table[1][C(tmp)] + ^ lzma_crc64_table[0][D(tmp)]; + } + } + + while (size-- != 0) + crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); + +#ifdef WORDS_BIGENDIAN + crc = bswap_64(crc); +#endif + + return ~crc; +} diff --git a/src/liblzma/check/crc64_init.c b/src/liblzma/check/crc64_init.c new file mode 100644 index 00000000..49c59e93 --- /dev/null +++ b/src/liblzma/check/crc64_init.c @@ -0,0 +1,58 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc64_init.c +/// \brief CRC64 table initialization +// +// This code is based on various public domain sources. +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include <config.h> +#endif + +#include <sys/types.h> +#include <inttypes.h> + +#ifdef WORDS_BIGENDIAN +# include "check_byteswap.h" +#endif + + +uint64_t lzma_crc64_table[4][256]; + + +extern void +lzma_crc64_init(void) +{ + static const uint64_t poly64 = UINT64_C(0xC96C5795D7870F42); + + for (size_t s = 0; s < 4; ++s) { + for (size_t b = 0; b < 256; ++b) { + uint64_t r = s == 0 ? b : lzma_crc64_table[s - 1][b]; + + for (size_t i = 0; i < 8; ++i) { + if (r & 1) + r = (r >> 1) ^ poly64; + else + r >>= 1; + } + + lzma_crc64_table[s][b] = r; + } + } + +#ifdef WORDS_BIGENDIAN + for (size_t s = 0; s < 4; ++s) + for (size_t b = 0; b < 256; ++b) + lzma_crc64_table[s][b] + = bswap_64(lzma_crc64_table[s][b]); +#endif + + return; +} diff --git a/src/liblzma/check/crc64_table.c b/src/liblzma/check/crc64_table.c new file mode 100644 index 00000000..0f2d1fb1 --- /dev/null +++ b/src/liblzma/check/crc64_table.c @@ -0,0 +1,22 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc64_table.c +/// \brief Precalculated CRC64 table with correct endianness +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include <config.h> +#endif + +#ifdef WORDS_BIGENDIAN +# include "crc64_table_be.h" +#else +# include "crc64_table_le.h" +#endif diff --git a/src/liblzma/check/crc64_table_be.h b/src/liblzma/check/crc64_table_be.h new file mode 100644 index 00000000..99518400 --- /dev/null +++ b/src/liblzma/check/crc64_table_be.h @@ -0,0 +1,523 @@ +/* This file has been automatically generated by crc64_tablegen.c. */ + +#include <inttypes.h> + +const uint64_t lzma_crc64_table[4][256] = { + { + UINT64_C(0x0000000000000000), UINT64_C(0x6F5FA703BE4C2EB3), + UINT64_C(0x5BA040A8573684F4), UINT64_C(0x34FFE7ABE97AAA47), + UINT64_C(0x335E8FFF84C3D07B), UINT64_C(0x5C0128FC3A8FFEC8), + UINT64_C(0x68FECF57D3F5548F), UINT64_C(0x07A168546DB97A3C), + UINT64_C(0x66BC1EFF0987A1F7), UINT64_C(0x09E3B9FCB7CB8F44), + UINT64_C(0x3D1C5E575EB12503), UINT64_C(0x5243F954E0FD0BB0), + UINT64_C(0x55E291008D44718C), UINT64_C(0x3ABD360333085F3F), + UINT64_C(0x0E42D1A8DA72F578), UINT64_C(0x611D76AB643EDBCB), + UINT64_C(0x4966335138A19B7D), UINT64_C(0x2639945286EDB5CE), + UINT64_C(0x12C673F96F971F89), UINT64_C(0x7D99D4FAD1DB313A), + UINT64_C(0x7A38BCAEBC624B06), UINT64_C(0x15671BAD022E65B5), + UINT64_C(0x2198FC06EB54CFF2), UINT64_C(0x4EC75B055518E141), + UINT64_C(0x2FDA2DAE31263A8A), UINT64_C(0x40858AAD8F6A1439), + UINT64_C(0x747A6D066610BE7E), UINT64_C(0x1B25CA05D85C90CD), + UINT64_C(0x1C84A251B5E5EAF1), UINT64_C(0x73DB05520BA9C442), + UINT64_C(0x4724E2F9E2D36E05), UINT64_C(0x287B45FA5C9F40B6), + UINT64_C(0x92CC66A2704237FB), UINT64_C(0xFD93C1A1CE0E1948), + UINT64_C(0xC96C260A2774B30F), UINT64_C(0xA633810999389DBC), + UINT64_C(0xA192E95DF481E780), UINT64_C(0xCECD4E5E4ACDC933), + UINT64_C(0xFA32A9F5A3B76374), UINT64_C(0x956D0EF61DFB4DC7), + UINT64_C(0xF470785D79C5960C), UINT64_C(0x9B2FDF5EC789B8BF), + UINT64_C(0xAFD038F52EF312F8), UINT64_C(0xC08F9FF690BF3C4B), + UINT64_C(0xC72EF7A2FD064677), UINT64_C(0xA87150A1434A68C4), + UINT64_C(0x9C8EB70AAA30C283), UINT64_C(0xF3D11009147CEC30), + UINT64_C(0xDBAA55F348E3AC86), UINT64_C(0xB4F5F2F0F6AF8235), + UINT64_C(0x800A155B1FD52872), UINT64_C(0xEF55B258A19906C1), + UINT64_C(0xE8F4DA0CCC207CFD), UINT64_C(0x87AB7D0F726C524E), + UINT64_C(0xB3549AA49B16F809), UINT64_C(0xDC0B3DA7255AD6BA), + UINT64_C(0xBD164B0C41640D71), UINT64_C(0xD249EC0FFF2823C2), + UINT64_C(0xE6B60BA416528985), UINT64_C(0x89E9ACA7A81EA736), + UINT64_C(0x8E48C4F3C5A7DD0A), UINT64_C(0xE11763F07BEBF3B9), + UINT64_C(0xD5E8845B929159FE), UINT64_C(0xBAB723582CDD774D), + UINT64_C(0xA187C3EBCA2BB664), UINT64_C(0xCED864E8746798D7), + UINT64_C(0xFA2783439D1D3290), UINT64_C(0x9578244023511C23), + UINT64_C(0x92D94C144EE8661F), UINT64_C(0xFD86EB17F0A448AC), + UINT64_C(0xC9790CBC19DEE2EB), UINT64_C(0xA626ABBFA792CC58), + UINT64_C(0xC73BDD14C3AC1793), UINT64_C(0xA8647A177DE03920), + UINT64_C(0x9C9B9DBC949A9367), UINT64_C(0xF3C43ABF2AD6BDD4), + UINT64_C(0xF46552EB476FC7E8), UINT64_C(0x9B3AF5E8F923E95B), + UINT64_C(0xAFC512431059431C), UINT64_C(0xC09AB540AE156DAF), + UINT64_C(0xE8E1F0BAF28A2D19), UINT64_C(0x87BE57B94CC603AA), + UINT64_C(0xB341B012A5BCA9ED), UINT64_C(0xDC1E17111BF0875E), + UINT64_C(0xDBBF7F457649FD62), UINT64_C(0xB4E0D846C805D3D1), + UINT64_C(0x801F3FED217F7996), UINT64_C(0xEF4098EE9F335725), + UINT64_C(0x8E5DEE45FB0D8CEE), UINT64_C(0xE10249464541A25D), + UINT64_C(0xD5FDAEEDAC3B081A), UINT64_C(0xBAA209EE127726A9), + UINT64_C(0xBD0361BA7FCE5C95), UINT64_C(0xD25CC6B9C1827226), + UINT64_C(0xE6A3211228F8D861), UINT64_C(0x89FC861196B4F6D2), + UINT64_C(0x334BA549BA69819F), UINT64_C(0x5C14024A0425AF2C), + UINT64_C(0x68EBE5E1ED5F056B), UINT64_C(0x07B442E253132BD8), + UINT64_C(0x00152AB63EAA51E4), UINT64_C(0x6F4A8DB580E67F57), + UINT64_C(0x5BB56A1E699CD510), UINT64_C(0x34EACD1DD7D0FBA3), + UINT64_C(0x55F7BBB6B3EE2068), UINT64_C(0x3AA81CB50DA20EDB), + UINT64_C(0x0E57FB1EE4D8A49C), UINT64_C(0x61085C1D5A948A2F), + UINT64_C(0x66A93449372DF013), UINT64_C(0x09F6934A8961DEA0), + UINT64_C(0x3D0974E1601B74E7), UINT64_C(0x5256D3E2DE575A54), + UINT64_C(0x7A2D961882C81AE2), UINT64_C(0x1572311B3C843451), + UINT64_C(0x218DD6B0D5FE9E16), UINT64_C(0x4ED271B36BB2B0A5), + UINT64_C(0x497319E7060BCA99), UINT64_C(0x262CBEE4B847E42A), + UINT64_C(0x12D3594F513D4E6D), UINT64_C(0x7D8CFE4CEF7160DE), + UINT64_C(0x1C9188E78B4FBB15), UINT64_C(0x73CE2FE4350395A6), + UINT64_C(0x4731C84FDC793FE1), UINT64_C(0x286E6F4C62351152), + UINT64_C(0x2FCF07180F8C6B6E), UINT64_C(0x4090A01BB1C045DD), + UINT64_C(0x746F47B058BAEF9A), UINT64_C(0x1B30E0B3E6F6C129), + UINT64_C(0x420F87D795576CC9), UINT64_C(0x2D5020D42B1B427A), + UINT64_C(0x19AFC77FC261E83D), UINT64_C(0x76F0607C7C2DC68E), + UINT64_C(0x715108281194BCB2), UINT64_C(0x1E0EAF2BAFD89201), + UINT64_C(0x2AF1488046A23846), UINT64_C(0x45AEEF83F8EE16F5), + UINT64_C(0x24B399289CD0CD3E), UINT64_C(0x4BEC3E2B229CE38D), + UINT64_C(0x7F13D980CBE649CA), UINT64_C(0x104C7E8375AA6779), + UINT64_C(0x17ED16D718131D45), UINT64_C(0x78B2B1D4A65F33F6), + UINT64_C(0x4C4D567F4F2599B1), UINT64_C(0x2312F17CF169B702), + UINT64_C(0x0B69B486ADF6F7B4), UINT64_C(0x6436138513BAD907), + UINT64_C(0x50C9F42EFAC07340), UINT64_C(0x3F96532D448C5DF3), + UINT64_C(0x38373B79293527CF), UINT64_C(0x57689C7A9779097C), + UINT64_C(0x63977BD17E03A33B), UINT64_C(0x0CC8DCD2C04F8D88), + UINT64_C(0x6DD5AA79A4715643), UINT64_C(0x028A0D7A1A3D78F0), + UINT64_C(0x3675EAD1F347D2B7), UINT64_C(0x592A4DD24D0BFC04), + UINT64_C(0x5E8B258620B28638), UINT64_C(0x31D482859EFEA88B), + UINT64_C(0x052B652E778402CC), UINT64_C(0x6A74C22DC9C82C7F), + UINT64_C(0xD0C3E175E5155B32), UINT64_C(0xBF9C46765B597581), + UINT64_C(0x8B63A1DDB223DFC6), UINT64_C(0xE43C06DE0C6FF175), + UINT64_C(0xE39D6E8A61D68B49), UINT64_C(0x8CC2C989DF9AA5FA), + UINT64_C(0xB83D2E2236E00FBD), UINT64_C(0xD762892188AC210E), + UINT64_C(0xB67FFF8AEC92FAC5), UINT64_C(0xD920588952DED476), + UINT64_C(0xEDDFBF22BBA47E31), UINT64_C(0x8280182105E85082), + UINT64_C(0x8521707568512ABE), UINT64_C(0xEA7ED776D61D040D), + UINT64_C(0xDE8130DD3F67AE4A), UINT64_C(0xB1DE97DE812B80F9), + UINT64_C(0x99A5D224DDB4C04F), UINT64_C(0xF6FA752763F8EEFC), + UINT64_C(0xC205928C8A8244BB), UINT64_C(0xAD5A358F34CE6A08), + UINT64_C(0xAAFB5DDB59771034), UINT64_C(0xC5A4FAD8E73B3E87), + UINT64_C(0xF15B1D730E4194C0), UINT64_C(0x9E04BA70B00DBA73), + UINT64_C(0xFF19CCDBD43361B8), UINT64_C(0x90466BD86A7F4F0B), + UINT64_C(0xA4B98C738305E54C), UINT64_C(0xCBE62B703D49CBFF), + UINT64_C(0xCC47432450F0B1C3), UINT64_C(0xA318E427EEBC9F70), + UINT64_C(0x97E7038C07C63537), UINT64_C(0xF8B8A48FB98A1B84), + UINT64_C(0xE388443C5F7CDAAD), UINT64_C(0x8CD7E33FE130F41E), + UINT64_C(0xB8280494084A5E59), UINT64_C(0xD777A397B60670EA), + UINT64_C(0xD0D6CBC3DBBF0AD6), UINT64_C(0xBF896CC065F32465), + UINT64_C(0x8B768B6B8C898E22), UINT64_C(0xE4292C6832C5A091), + UINT64_C(0x85345AC356FB7B5A), UINT64_C(0xEA6BFDC0E8B755E9), + UINT64_C(0xDE941A6B01CDFFAE), UINT64_C(0xB1CBBD68BF81D11D), + UINT64_C(0xB66AD53CD238AB21), UINT64_C(0xD935723F6C748592), + UINT64_C(0xEDCA9594850E2FD5), UINT64_C(0x829532973B420166), + UINT64_C(0xAAEE776D67DD41D0), UINT64_C(0xC5B1D06ED9916F63), + UINT64_C(0xF14E37C530EBC524), UINT64_C(0x9E1190C68EA7EB97), + UINT64_C(0x99B0F892E31E91AB), UINT64_C(0xF6EF5F915D52BF18), + UINT64_C(0xC210B83AB428155F), UINT64_C(0xAD4F1F390A643BEC), + UINT64_C(0xCC5269926E5AE027), UINT64_C(0xA30DCE91D016CE94), + UINT64_C(0x97F2293A396C64D3), UINT64_C(0xF8AD8E3987204A60), + UINT64_C(0xFF0CE66DEA99305C), UINT64_C(0x9053416E54D51EEF), + UINT64_C(0xA4ACA6C5BDAFB4A8), UINT64_C(0xCBF301C603E39A1B), + UINT64_C(0x7144229E2F3EED56), UINT64_C(0x1E1B859D9172C3E5), + UINT64_C(0x2AE46236780869A2), UINT64_C(0x45BBC535C6444711), + UINT64_C(0x421AAD61ABFD3D2D), UINT64_C(0x2D450A6215B1139E), + UINT64_C(0x19BAEDC9FCCBB9D9), UINT64_C(0x76E54ACA4287976A), + UINT64_C(0x17F83C6126B94CA1), UINT64_C(0x78A79B6298F56212), + UINT64_C(0x4C587CC9718FC855), UINT64_C(0x2307DBCACFC3E6E6), + UINT64_C(0x24A6B39EA27A9CDA), UINT64_C(0x4BF9149D1C36B269), + UINT64_C(0x7F06F336F54C182E), UINT64_C(0x105954354B00369D), + UINT64_C(0x382211CF179F762B), UINT64_C(0x577DB6CCA9D35898), + UINT64_C(0x6382516740A9F2DF), UINT64_C(0x0CDDF664FEE5DC6C), + UINT64_C(0x0B7C9E30935CA650), UINT64_C(0x642339332D1088E3), + UINT64_C(0x50DCDE98C46A22A4), UINT64_C(0x3F83799B7A260C17), + UINT64_C(0x5E9E0F301E18D7DC), UINT64_C(0x31C1A833A054F96F), + UINT64_C(0x053E4F98492E5328), UINT64_C(0x6A61E89BF7627D9B), + UINT64_C(0x6DC080CF9ADB07A7), UINT64_C(0x029F27CC24972914), + UINT64_C(0x3660C067CDED8353), UINT64_C(0x593F676473A1ADE0) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0x0DF1D05C9279E954), + UINT64_C(0x1AE2A1B924F3D2A9), UINT64_C(0x171371E5B68A3BFD), + UINT64_C(0xB1DA4DDC62497DC1), UINT64_C(0xBC2B9D80F0309495), + UINT64_C(0xAB38EC6546BAAF68), UINT64_C(0xA6C93C39D4C3463C), + UINT64_C(0xE7AB9517EE3D2210), UINT64_C(0xEA5A454B7C44CB44), + UINT64_C(0xFD4934AECACEF0B9), UINT64_C(0xF0B8E4F258B719ED), + UINT64_C(0x5671D8CB8C745FD1), UINT64_C(0x5B8008971E0DB685), + UINT64_C(0x4C937972A8878D78), UINT64_C(0x4162A92E3AFE642C), + UINT64_C(0xCE572B2FDC7B4420), UINT64_C(0xC3A6FB734E02AD74), + UINT64_C(0xD4B58A96F8889689), UINT64_C(0xD9445ACA6AF17FDD), + UINT64_C(0x7F8D66F3BE3239E1), UINT64_C(0x727CB6AF2C4BD0B5), + UINT64_C(0x656FC74A9AC1EB48), UINT64_C(0x689E171608B8021C), + UINT64_C(0x29FCBE3832466630), UINT64_C(0x240D6E64A03F8F64), + UINT64_C(0x331E1F8116B5B499), UINT64_C(0x3EEFCFDD84CC5DCD), + UINT64_C(0x9826F3E4500F1BF1), UINT64_C(0x95D723B8C276F2A5), + UINT64_C(0x82C4525D74FCC958), UINT64_C(0x8F358201E685200C), + UINT64_C(0x9CAF565EB8F78840), UINT64_C(0x915E86022A8E6114), + UINT64_C(0x864DF7E79C045AE9), UINT64_C(0x8BBC27BB0E7DB3BD), + UINT64_C(0x2D751B82DABEF581), UINT64_C(0x2084CBDE48C71CD5), + UINT64_C(0x3797BA3BFE4D2728), UINT64_C(0x3A666A676C34CE7C), + UINT64_C(0x7B04C34956CAAA50), UINT64_C(0x76F51315C4B34304), + UINT64_C(0x61E662F0723978F9), UINT64_C(0x6C17B2ACE04091AD), + UINT64_C(0xCADE8E953483D791), UINT64_C(0xC72F5EC9A6FA3EC5), + UINT64_C(0xD03C2F2C10700538), UINT64_C(0xDDCDFF708209EC6C), + UINT64_C(0x52F87D71648CCC60), UINT64_C(0x5F09AD2DF6F52534), + UINT64_C(0x481ADCC8407F1EC9), UINT64_C(0x45EB0C94D206F79D), + UINT64_C(0xE32230AD06C5B1A1), UINT64_C(0xEED3E0F194BC58F5), + UINT64_C(0xF9C0911422366308), UINT64_C(0xF4314148B04F8A5C), + UINT64_C(0xB553E8668AB1EE70), UINT64_C(0xB8A2383A18C80724), + UINT64_C(0xAFB149DFAE423CD9), UINT64_C(0xA24099833C3BD58D), + UINT64_C(0x0489A5BAE8F893B1), UINT64_C(0x097875E67A817AE5), + UINT64_C(0x1E6B0403CC0B4118), UINT64_C(0x139AD45F5E72A84C), + UINT64_C(0x385FADBC70EF1181), UINT64_C(0x35AE7DE0E296F8D5), + UINT64_C(0x22BD0C05541CC328), UINT64_C(0x2F4CDC59C6652A7C), + UINT64_C(0x8985E06012A66C40), UINT64_C(0x8474303C80DF8514), + UINT64_C(0x936741D93655BEE9), UINT64_C(0x9E969185A42C57BD), + UINT64_C(0xDFF438AB9ED23391), UINT64_C(0xD205E8F70CABDAC5), + UINT64_C(0xC5169912BA21E138), UINT64_C(0xC8E7494E2858086C), + UINT64_C(0x6E2E7577FC9B4E50), UINT64_C(0x63DFA52B6EE2A704), + UINT64_C(0x74CCD4CED8689CF9), UINT64_C(0x793D04924A1175AD), + UINT64_C(0xF6088693AC9455A1), UINT64_C(0xFBF956CF3EEDBCF5), + UINT64_C(0xECEA272A88678708), UINT64_C(0xE11BF7761A1E6E5C), + UINT64_C(0x47D2CB4FCEDD2860), UINT64_C(0x4A231B135CA4C134), + UINT64_C(0x5D306AF6EA2EFAC9), UINT64_C(0x50C1BAAA7857139D), + UINT64_C(0x11A3138442A977B1), UINT64_C(0x1C52C3D8D0D09EE5), + UINT64_C(0x0B41B23D665AA518), UINT64_C(0x06B06261F4234C4C), + UINT64_C(0xA0795E5820E00A70), UINT64_C(0xAD888E04B299E324), + UINT64_C(0xBA9BFFE10413D8D9), UINT64_C(0xB76A2FBD966A318D), + UINT64_C(0xA4F0FBE2C81899C1), UINT64_C(0xA9012BBE5A617095), + UINT64_C(0xBE125A5BECEB4B68), UINT64_C(0xB3E38A077E92A23C), + UINT64_C(0x152AB63EAA51E400), UINT64_C(0x18DB666238280D54), + UINT64_C(0x0FC817878EA236A9), UINT64_C(0x0239C7DB1CDBDFFD), + UINT64_C(0x435B6EF52625BBD1), UINT64_C(0x4EAABEA9B45C5285), + UINT64_C(0x59B9CF4C02D66978), UINT64_C(0x54481F1090AF802C), + UINT64_C(0xF2812329446CC610), UINT64_C(0xFF70F375D6152F44), + UINT64_C(0xE8638290609F14B9), UINT64_C(0xE59252CCF2E6FDED), + UINT64_C(0x6AA7D0CD1463DDE1), UINT64_C(0x67560091861A34B5), + UINT64_C(0x7045717430900F48), UINT64_C(0x7DB4A128A2E9E61C), + UINT64_C(0xDB7D9D11762AA020), UINT64_C(0xD68C4D4DE4534974), + UINT64_C(0xC19F3CA852D97289), UINT64_C(0xCC6EECF4C0A09BDD), + UINT64_C(0x8D0C45DAFA5EFFF1), UINT64_C(0x80FD9586682716A5), + UINT64_C(0x97EEE463DEAD2D58), UINT64_C(0x9A1F343F4CD4C40C), + UINT64_C(0x3CD6080698178230), UINT64_C(0x3127D85A0A6E6B64), + UINT64_C(0x2634A9BFBCE45099), UINT64_C(0x2BC579E32E9DB9CD), + UINT64_C(0xF5A054D6CA71FB90), UINT64_C(0xF851848A580812C4), + UINT64_C(0xEF42F56FEE822939), UINT64_C(0xE2B325337CFBC06D), + UINT64_C(0x447A190AA8388651), UINT64_C(0x498BC9563A416F05), + UINT64_C(0x5E98B8B38CCB54F8), UINT64_C(0x536968EF1EB2BDAC), + UINT64_C(0x120BC1C1244CD980), UINT64_C(0x1FFA119DB63530D4), + UINT64_C(0x08E9607800BF0B29), UINT64_C(0x0518B02492C6E27D), + UINT64_C(0xA3D18C1D4605A441), UINT64_C(0xAE205C41D47C4D15), + UINT64_C(0xB9332DA462F676E8), UINT64_C(0xB4C2FDF8F08F9FBC), + UINT64_C(0x3BF77FF9160ABFB0), UINT64_C(0x3606AFA5847356E4), + UINT64_C(0x2115DE4032F96D19), UINT64_C(0x2CE40E1CA080844D), + UINT64_C(0x8A2D32257443C271), UINT64_C(0x87DCE279E63A2B25), + UINT64_C(0x90CF939C50B010D8), UINT64_C(0x9D3E43C0C2C9F98C), + UINT64_C(0xDC5CEAEEF8379DA0), UINT64_C(0xD1AD3AB26A4E74F4), + UINT64_C(0xC6BE4B57DCC44F09), UINT64_C(0xCB4F9B0B4EBDA65D), + UINT64_C(0x6D86A7329A7EE061), UINT64_C(0x6077776E08070935), + UINT64_C(0x7764068BBE8D32C8), UINT64_C(0x7A95D6D72CF4DB9C), + UINT64_C(0x690F0288728673D0), UINT64_C(0x64FED2D4E0FF9A84), + UINT64_C(0x73EDA3315675A179), UINT64_C(0x7E1C736DC40C482D), + UINT64_C(0xD8D54F5410CF0E11), UINT64_C(0xD5249F0882B6E745), + UINT64_C(0xC237EEED343CDCB8), UINT64_C(0xCFC63EB1A64535EC), + UINT64_C(0x8EA4979F9CBB51C0), UINT64_C(0x835547C30EC2B894), + UINT64_C(0x94463626B8488369), UINT64_C(0x99B7E67A2A316A3D), + UINT64_C(0x3F7EDA43FEF22C01), UINT64_C(0x328F0A1F6C8BC555), + UINT64_C(0x259C7BFADA01FEA8), UINT64_C(0x286DABA6487817FC), + UINT64_C(0xA75829A7AEFD37F0), UINT64_C(0xAAA9F9FB3C84DEA4), + UINT64_C(0xBDBA881E8A0EE559), UINT64_C(0xB04B584218770C0D), + UINT64_C(0x1682647BCCB44A31), UINT64_C(0x1B73B4275ECDA365), + UINT64_C(0x0C60C5C2E8479898), UINT64_C(0x0191159E7A3E71CC), + UINT64_C(0x40F3BCB040C015E0), UINT64_C(0x4D026CECD2B9FCB4), + UINT64_C(0x5A111D096433C749), UINT64_C(0x57E0CD55F64A2E1D), + UINT64_C(0xF129F16C22896821), UINT64_C(0xFCD82130B0F08175), + UINT64_C(0xEBCB50D5067ABA88), UINT64_C(0xE63A8089940353DC), + UINT64_C(0xCDFFF96ABA9EEA11), UINT64_C(0xC00E293628E70345), + UINT64_C(0xD71D58D39E6D38B8), UINT64_C(0xDAEC888F0C14D1EC), + UINT64_C(0x7C25B4B6D8D797D0), UINT64_C(0x71D464EA4AAE7E84), + UINT64_C(0x66C7150FFC244579), UINT64_C(0x6B36C5536E5DAC2D), + UINT64_C(0x2A546C7D54A3C801), UINT64_C(0x27A5BC21C6DA2155), + UINT64_C(0x30B6CDC470501AA8), UINT64_C(0x3D471D98E229F3FC), + UINT64_C(0x9B8E21A136EAB5C0), UINT64_C(0x967FF1FDA4935C94), + UINT64_C(0x816C801812196769), UINT64_C(0x8C9D504480608E3D), + UINT64_C(0x03A8D24566E5AE31), UINT64_C(0x0E590219F49C4765), + UINT64_C(0x194A73FC42167C98), UINT64_C(0x14BBA3A0D06F95CC), + UINT64_C(0xB2729F9904ACD3F0), UINT64_C(0xBF834FC596D53AA4), + UINT64_C(0xA8903E20205F0159), UINT64_C(0xA561EE7CB226E80D), + UINT64_C(0xE403475288D88C21), UINT64_C(0xE9F2970E1AA16575), + UINT64_C(0xFEE1E6EBAC2B5E88), UINT64_C(0xF31036B73E52B7DC), + UINT64_C(0x55D90A8EEA91F1E0), UINT64_C(0x5828DAD278E818B4), + UINT64_C(0x4F3BAB37CE622349), UINT64_C(0x42CA7B6B5C1BCA1D), + UINT64_C(0x5150AF3402696251), UINT64_C(0x5CA17F6890108B05), + UINT64_C(0x4BB20E8D269AB0F8), UINT64_C(0x4643DED1B4E359AC), + UINT64_C(0xE08AE2E860201F90), UINT64_C(0xED7B32B4F259F6C4), + UINT64_C(0xFA68435144D3CD39), UINT64_C(0xF799930DD6AA246D), + UINT64_C(0xB6FB3A23EC544041), UINT64_C(0xBB0AEA7F7E2DA915), + UINT64_C(0xAC199B9AC8A792E8), UINT64_C(0xA1E84BC65ADE7BBC), + UINT64_C(0x072177FF8E1D3D80), UINT64_C(0x0AD0A7A31C64D4D4), + UINT64_C(0x1DC3D646AAEEEF29), UINT64_C(0x1032061A3897067D), + UINT64_C(0x9F07841BDE122671), UINT64_C(0x92F654474C6BCF25), + UINT64_C(0x85E525A2FAE1F4D8), UINT64_C(0x8814F5FE68981D8C), + UINT64_C(0x2EDDC9C7BC5B5BB0), UINT64_C(0x232C199B2E22B2E4), + UINT64_C(0x343F687E98A88919), UINT64_C(0x39CEB8220AD1604D), + UINT64_C(0x78AC110C302F0461), UINT64_C(0x755DC150A256ED35), + UINT64_C(0x624EB0B514DCD6C8), UINT64_C(0x6FBF60E986A53F9C), + UINT64_C(0xC9765CD0526679A0), UINT64_C(0xC4878C8CC01F90F4), + UINT64_C(0xD394FD697695AB09), UINT64_C(0xDE652D35E4EC425D) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0xCB6D6A914AE10B3F), + UINT64_C(0x96DBD42295C2177E), UINT64_C(0x5DB6BEB3DF231C41), + UINT64_C(0x2CB7A9452A852FFC), UINT64_C(0xE7DAC3D4606424C3), + UINT64_C(0xBA6C7D67BF473882), UINT64_C(0x710117F6F5A633BD), + UINT64_C(0xDD705D247FA5876A), UINT64_C(0x161D37B535448C55), + UINT64_C(0x4BAB8906EA679014), UINT64_C(0x80C6E397A0869B2B), + UINT64_C(0xF1C7F4615520A896), UINT64_C(0x3AAA9EF01FC1A3A9), + UINT64_C(0x671C2043C0E2BFE8), UINT64_C(0xAC714AD28A03B4D7), + UINT64_C(0xBAE1BA48FE4A0FD5), UINT64_C(0x718CD0D9B4AB04EA), + UINT64_C(0x2C3A6E6A6B8818AB), UINT64_C(0xE75704FB21691394), + UINT64_C(0x9656130DD4CF2029), UINT64_C(0x5D3B799C9E2E2B16), + UINT64_C(0x008DC72F410D3757), UINT64_C(0xCBE0ADBE0BEC3C68), + UINT64_C(0x6791E76C81EF88BF), UINT64_C(0xACFC8DFDCB0E8380), + UINT64_C(0xF14A334E142D9FC1), UINT64_C(0x3A2759DF5ECC94FE), + UINT64_C(0x4B264E29AB6AA743), UINT64_C(0x804B24B8E18BAC7C), + UINT64_C(0xDDFD9A0B3EA8B03D), UINT64_C(0x1690F09A7449BB02), + UINT64_C(0xF1DD7B3ED73AC638), UINT64_C(0x3AB011AF9DDBCD07), + UINT64_C(0x6706AF1C42F8D146), UINT64_C(0xAC6BC58D0819DA79), + UINT64_C(0xDD6AD27BFDBFE9C4), UINT64_C(0x1607B8EAB75EE2FB), + UINT64_C(0x4BB10659687DFEBA), UINT64_C(0x80DC6CC8229CF585), + UINT64_C(0x2CAD261AA89F4152), UINT64_C(0xE7C04C8BE27E4A6D), + UINT64_C(0xBA76F2383D5D562C), UINT64_C(0x711B98A977BC5D13), + UINT64_C(0x001A8F5F821A6EAE), UINT64_C(0xCB77E5CEC8FB6591), + UINT64_C(0x96C15B7D17D879D0), UINT64_C(0x5DAC31EC5D3972EF), + UINT64_C(0x4B3CC1762970C9ED), UINT64_C(0x8051ABE76391C2D2), + UINT64_C(0xDDE71554BCB2DE93), UINT64_C(0x168A7FC5F653D5AC), + UINT64_C(0x678B683303F5E611), UINT64_C(0xACE602A24914ED2E), + UINT64_C(0xF150BC119637F16F), UINT64_C(0x3A3DD680DCD6FA50), + UINT64_C(0x964C9C5256D54E87), UINT64_C(0x5D21F6C31C3445B8), + UINT64_C(0x00974870C31759F9), UINT64_C(0xCBFA22E189F652C6), + UINT64_C(0xBAFB35177C50617B), UINT64_C(0x71965F8636B16A44), + UINT64_C(0x2C20E135E9927605), UINT64_C(0xE74D8BA4A3737D3A), + UINT64_C(0xE2BBF77CAE758C71), UINT64_C(0x29D69DEDE494874E), + UINT64_C(0x7460235E3BB79B0F), UINT64_C(0xBF0D49CF71569030), + UINT64_C(0xCE0C5E3984F0A38D), UINT64_C(0x056134A8CE11A8B2), + UINT64_C(0x58D78A1B1132B4F3), UINT64_C(0x93BAE08A5BD3BFCC), + UINT64_C(0x3FCBAA58D1D00B1B), UINT64_C(0xF4A6C0C99B310024), + UINT64_C(0xA9107E7A44121C65), UINT64_C(0x627D14EB0EF3175A), + UINT64_C(0x137C031DFB5524E7), UINT64_C(0xD811698CB1B42FD8), + UINT64_C(0x85A7D73F6E973399), UINT64_C(0x4ECABDAE247638A6), + UINT64_C(0x585A4D34503F83A4), UINT64_C(0x933727A51ADE889B), + UINT64_C(0xCE819916C5FD94DA), UINT64_C(0x05ECF3878F1C9FE5), + UINT64_C(0x74EDE4717ABAAC58), UINT64_C(0xBF808EE0305BA767), + UINT64_C(0xE2363053EF78BB26), UINT64_C(0x295B5AC2A599B019), + UINT64_C(0x852A10102F9A04CE), UINT64_C(0x4E477A81657B0FF1), + UINT64_C(0x13F1C432BA5813B0), UINT64_C(0xD89CAEA3F0B9188F), + UINT64_C(0xA99DB955051F2B32), UINT64_C(0x62F0D3C44FFE200D), + UINT64_C(0x3F466D7790DD3C4C), UINT64_C(0xF42B07E6DA3C3773), + UINT64_C(0x13668C42794F4A49), UINT64_C(0xD80BE6D333AE4176), + UINT64_C(0x85BD5860EC8D5D37), UINT64_C(0x4ED032F1A66C5608), + UINT64_C(0x3FD1250753CA65B5), UINT64_C(0xF4BC4F96192B6E8A), + UINT64_C(0xA90AF125C60872CB), UINT64_C(0x62679BB48CE979F4), + UINT64_C(0xCE16D16606EACD23), UINT64_C(0x057BBBF74C0BC61C), + UINT64_C(0x58CD05449328DA5D), UINT64_C(0x93A06FD5D9C9D162), + UINT64_C(0xE2A178232C6FE2DF), UINT64_C(0x29CC12B2668EE9E0), + UINT64_C(0x747AAC01B9ADF5A1), UINT64_C(0xBF17C690F34CFE9E), + UINT64_C(0xA987360A8705459C), UINT64_C(0x62EA5C9BCDE44EA3), + UINT64_C(0x3F5CE22812C752E2), UINT64_C(0xF43188B9582659DD), + UINT64_C(0x85309F4FAD806A60), UINT64_C(0x4E5DF5DEE761615F), + UINT64_C(0x13EB4B6D38427D1E), UINT64_C(0xD88621FC72A37621), + UINT64_C(0x74F76B2EF8A0C2F6), UINT64_C(0xBF9A01BFB241C9C9), + UINT64_C(0xE22CBF0C6D62D588), UINT64_C(0x2941D59D2783DEB7), + UINT64_C(0x5840C26BD225ED0A), UINT64_C(0x932DA8FA98C4E635), + UINT64_C(0xCE9B164947E7FA74), UINT64_C(0x05F67CD80D06F14B), + UINT64_C(0xC477EFF95CEB18E3), UINT64_C(0x0F1A8568160A13DC), + UINT64_C(0x52AC3BDBC9290F9D), UINT64_C(0x99C1514A83C804A2), + UINT64_C(0xE8C046BC766E371F), UINT64_C(0x23AD2C2D3C8F3C20), + UINT64_C(0x7E1B929EE3AC2061), UINT64_C(0xB576F80FA94D2B5E), + UINT64_C(0x1907B2DD234E9F89), UINT64_C(0xD26AD84C69AF94B6), + UINT64_C(0x8FDC66FFB68C88F7), UINT64_C(0x44B10C6EFC6D83C8), + UINT64_C(0x35B01B9809CBB075), UINT64_C(0xFEDD7109432ABB4A), + UINT64_C(0xA36BCFBA9C09A70B), UINT64_C(0x6806A52BD6E8AC34), + UINT64_C(0x7E9655B1A2A11736), UINT64_C(0xB5FB3F20E8401C09), + UINT64_C(0xE84D819337630048), UINT64_C(0x2320EB027D820B77), + UINT64_C(0x5221FCF4882438CA), UINT64_C(0x994C9665C2C533F5), + UINT64_C(0xC4FA28D61DE62FB4), UINT64_C(0x0F9742475707248B), + UINT64_C(0xA3E60895DD04905C), UINT64_C(0x688B620497E59B63), + UINT64_C(0x353DDCB748C68722), UINT64_C(0xFE50B62602278C1D), + UINT64_C(0x8F51A1D0F781BFA0), UINT64_C(0x443CCB41BD60B49F), + UINT64_C(0x198A75F26243A8DE), UINT64_C(0xD2E71F6328A2A3E1), + UINT64_C(0x35AA94C78BD1DEDB), UINT64_C(0xFEC7FE56C130D5E4), + UINT64_C(0xA37140E51E13C9A5), UINT64_C(0x681C2A7454F2C29A), + UINT64_C(0x191D3D82A154F127), UINT64_C(0xD2705713EBB5FA18), + UINT64_C(0x8FC6E9A03496E659), UINT64_C(0x44AB83317E77ED66), + UINT64_C(0xE8DAC9E3F47459B1), UINT64_C(0x23B7A372BE95528E), + UINT64_C(0x7E011DC161B64ECF), UINT64_C(0xB56C77502B5745F0), + UINT64_C(0xC46D60A6DEF1764D), UINT64_C(0x0F000A3794107D72), + UINT64_C(0x52B6B4844B336133), UINT64_C(0x99DBDE1501D26A0C), + UINT64_C(0x8F4B2E8F759BD10E), UINT64_C(0x4426441E3F7ADA31), + UINT64_C(0x1990FAADE059C670), UINT64_C(0xD2FD903CAAB8CD4F), + UINT64_C(0xA3FC87CA5F1EFEF2), UINT64_C(0x6891ED5B15FFF5CD), + UINT64_C(0x352753E8CADCE98C), UINT64_C(0xFE4A3979803DE2B3), + UINT64_C(0x523B73AB0A3E5664), UINT64_C(0x9956193A40DF5D5B), + UINT64_C(0xC4E0A7899FFC411A), UINT64_C(0x0F8DCD18D51D4A25), + UINT64_C(0x7E8CDAEE20BB7998), UINT64_C(0xB5E1B07F6A5A72A7), + UINT64_C(0xE8570ECCB5796EE6), UINT64_C(0x233A645DFF9865D9), + UINT64_C(0x26CC1885F29E9492), UINT64_C(0xEDA17214B87F9FAD), + UINT64_C(0xB017CCA7675C83EC), UINT64_C(0x7B7AA6362DBD88D3), + UINT64_C(0x0A7BB1C0D81BBB6E), UINT64_C(0xC116DB5192FAB051), + UINT64_C(0x9CA065E24DD9AC10), UINT64_C(0x57CD0F730738A72F), + UINT64_C(0xFBBC45A18D3B13F8), UINT64_C(0x30D12F30C7DA18C7), + UINT64_C(0x6D67918318F90486), UINT64_C(0xA60AFB1252180FB9), + UINT64_C(0xD70BECE4A7BE3C04), UINT64_C(0x1C668675ED5F373B), + UINT64_C(0x41D038C6327C2B7A), UINT64_C(0x8ABD5257789D2045), + UINT64_C(0x9C2DA2CD0CD49B47), UINT64_C(0x5740C85C46359078), + UINT64_C(0x0AF676EF99168C39), UINT64_C(0xC19B1C7ED3F78706), + UINT64_C(0xB09A0B882651B4BB), UINT64_C(0x7BF761196CB0BF84), + UINT64_C(0x2641DFAAB393A3C5), UINT64_C(0xED2CB53BF972A8FA), + UINT64_C(0x415DFFE973711C2D), UINT64_C(0x8A30957839901712), + UINT64_C(0xD7862BCBE6B30B53), UINT64_C(0x1CEB415AAC52006C), + UINT64_C(0x6DEA56AC59F433D1), UINT64_C(0xA6873C3D131538EE), + UINT64_C(0xFB31828ECC3624AF), UINT64_C(0x305CE81F86D72F90), + UINT64_C(0xD71163BB25A452AA), UINT64_C(0x1C7C092A6F455995), + UINT64_C(0x41CAB799B06645D4), UINT64_C(0x8AA7DD08FA874EEB), + UINT64_C(0xFBA6CAFE0F217D56), UINT64_C(0x30CBA06F45C07669), + UINT64_C(0x6D7D1EDC9AE36A28), UINT64_C(0xA610744DD0026117), + UINT64_C(0x0A613E9F5A01D5C0), UINT64_C(0xC10C540E10E0DEFF), + UINT64_C(0x9CBAEABDCFC3C2BE), UINT64_C(0x57D7802C8522C981), + UINT64_C(0x26D697DA7084FA3C), UINT64_C(0xEDBBFD4B3A65F103), + UINT64_C(0xB00D43F8E546ED42), UINT64_C(0x7B602969AFA7E67D), + UINT64_C(0x6DF0D9F3DBEE5D7F), UINT64_C(0xA69DB362910F5640), + UINT64_C(0xFB2B0DD14E2C4A01), UINT64_C(0x3046674004CD413E), + UINT64_C(0x414770B6F16B7283), UINT64_C(0x8A2A1A27BB8A79BC), + UINT64_C(0xD79CA49464A965FD), UINT64_C(0x1CF1CE052E486EC2), + UINT64_C(0xB08084D7A44BDA15), UINT64_C(0x7BEDEE46EEAAD12A), + UINT64_C(0x265B50F53189CD6B), UINT64_C(0xED363A647B68C654), + UINT64_C(0x9C372D928ECEF5E9), UINT64_C(0x575A4703C42FFED6), + UINT64_C(0x0AECF9B01B0CE297), UINT64_C(0xC181932151EDE9A8) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0xDCA12C225E8AEE1D), + UINT64_C(0xB8435944BC14DD3B), UINT64_C(0x64E27566E29E3326), + UINT64_C(0x7087B2887829BA77), UINT64_C(0xAC269EAA26A3546A), + UINT64_C(0xC8C4EBCCC43D674C), UINT64_C(0x1465C7EE9AB78951), + UINT64_C(0xE00E6511F15274EF), UINT64_C(0x3CAF4933AFD89AF2), + UINT64_C(0x584D3C554D46A9D4), UINT64_C(0x84EC107713CC47C9), + UINT64_C(0x9089D799897BCE98), UINT64_C(0x4C28FBBBD7F12085), + UINT64_C(0x28CA8EDD356F13A3), UINT64_C(0xF46BA2FF6BE5FDBE), + UINT64_C(0x4503C48DC90A304C), UINT64_C(0x99A2E8AF9780DE51), + UINT64_C(0xFD409DC9751EED77), UINT64_C(0x21E1B1EB2B94036A), + UINT64_C(0x35847605B1238A3B), UINT64_C(0xE9255A27EFA96426), + UINT64_C(0x8DC72F410D375700), UINT64_C(0x5166036353BDB91D), + UINT64_C(0xA50DA19C385844A3), UINT64_C(0x79AC8DBE66D2AABE), + UINT64_C(0x1D4EF8D8844C9998), UINT64_C(0xC1EFD4FADAC67785), + UINT64_C(0xD58A13144071FED4), UINT64_C(0x092B3F361EFB10C9), + UINT64_C(0x6DC94A50FC6523EF), UINT64_C(0xB1686672A2EFCDF2), + UINT64_C(0x8A06881B93156098), UINT64_C(0x56A7A439CD9F8E85), + UINT64_C(0x3245D15F2F01BDA3), UINT64_C(0xEEE4FD7D718B53BE), + UINT64_C(0xFA813A93EB3CDAEF), UINT64_C(0x262016B1B5B634F2), + UINT64_C(0x42C263D7572807D4), UINT64_C(0x9E634FF509A2E9C9), + UINT64_C(0x6A08ED0A62471477), UINT64_C(0xB6A9C1283CCDFA6A), + UINT64_C(0xD24BB44EDE53C94C), UINT64_C(0x0EEA986C80D92751), + UINT64_C(0x1A8F5F821A6EAE00), UINT64_C(0xC62E73A044E4401D), + UINT64_C(0xA2CC06C6A67A733B), UINT64_C(0x7E6D2AE4F8F09D26), + UINT64_C(0xCF054C965A1F50D4), UINT64_C(0x13A460B40495BEC9), + UINT64_C(0x774615D2E60B8DEF), UINT64_C(0xABE739F0B88163F2), + UINT64_C(0xBF82FE1E2236EAA3), UINT64_C(0x6323D23C7CBC04BE), + UINT64_C(0x07C1A75A9E223798), UINT64_C(0xDB608B78C0A8D985), + UINT64_C(0x2F0B2987AB4D243B), UINT64_C(0xF3AA05A5F5C7CA26), + UINT64_C(0x974870C31759F900), UINT64_C(0x4BE95CE149D3171D), + UINT64_C(0x5F8C9B0FD3649E4C), UINT64_C(0x832DB72D8DEE7051), + UINT64_C(0xE7CFC24B6F704377), UINT64_C(0x3B6EEE6931FAAD6A), + UINT64_C(0x91131E980D8418A2), UINT64_C(0x4DB232BA530EF6BF), + UINT64_C(0x295047DCB190C599), UINT64_C(0xF5F16BFEEF1A2B84), + UINT64_C(0xE194AC1075ADA2D5), UINT64_C(0x3D3580322B274CC8), + UINT64_C(0x59D7F554C9B97FEE), UINT64_C(0x8576D976973391F3), + UINT64_C(0x711D7B89FCD66C4D), UINT64_C(0xADBC57ABA25C8250), + UINT64_C(0xC95E22CD40C2B176), UINT64_C(0x15FF0EEF1E485F6B), + UINT64_C(0x019AC90184FFD63A), UINT64_C(0xDD3BE523DA753827), + UINT64_C(0xB9D9904538EB0B01), UINT64_C(0x6578BC676661E51C), + UINT64_C(0xD410DA15C48E28EE), UINT64_C(0x08B1F6379A04C6F3), + UINT64_C(0x6C538351789AF5D5), UINT64_C(0xB0F2AF7326101BC8), + UINT64_C(0xA497689DBCA79299), UINT64_C(0x783644BFE22D7C84), + UINT64_C(0x1CD431D900B34FA2), UINT64_C(0xC0751DFB5E39A1BF), + UINT64_C(0x341EBF0435DC5C01), UINT64_C(0xE8BF93266B56B21C), + UINT64_C(0x8C5DE64089C8813A), UINT64_C(0x50FCCA62D7426F27), + UINT64_C(0x44990D8C4DF5E676), UINT64_C(0x983821AE137F086B), + UINT64_C(0xFCDA54C8F1E13B4D), UINT64_C(0x207B78EAAF6BD550), + UINT64_C(0x1B1596839E91783A), UINT64_C(0xC7B4BAA1C01B9627), + UINT64_C(0xA356CFC72285A501), UINT64_C(0x7FF7E3E57C0F4B1C), + UINT64_C(0x6B92240BE6B8C24D), UINT64_C(0xB7330829B8322C50), + UINT64_C(0xD3D17D4F5AAC1F76), UINT64_C(0x0F70516D0426F16B), + UINT64_C(0xFB1BF3926FC30CD5), UINT64_C(0x27BADFB03149E2C8), + UINT64_C(0x4358AAD6D3D7D1EE), UINT64_C(0x9FF986F48D5D3FF3), + UINT64_C(0x8B9C411A17EAB6A2), UINT64_C(0x573D6D38496058BF), + UINT64_C(0x33DF185EABFE6B99), UINT64_C(0xEF7E347CF5748584), + UINT64_C(0x5E16520E579B4876), UINT64_C(0x82B77E2C0911A66B), + UINT64_C(0xE6550B4AEB8F954D), UINT64_C(0x3AF42768B5057B50), + UINT64_C(0x2E91E0862FB2F201), UINT64_C(0xF230CCA471381C1C), + UINT64_C(0x96D2B9C293A62F3A), UINT64_C(0x4A7395E0CD2CC127), + UINT64_C(0xBE18371FA6C93C99), UINT64_C(0x62B91B3DF843D284), + UINT64_C(0x065B6E5B1ADDE1A2), UINT64_C(0xDAFA427944570FBF), + UINT64_C(0xCE9F8597DEE086EE), UINT64_C(0x123EA9B5806A68F3), + UINT64_C(0x76DCDCD362F45BD5), UINT64_C(0xAA7DF0F13C7EB5C8), + UINT64_C(0xA739329F30A7E9D6), UINT64_C(0x7B981EBD6E2D07CB), + UINT64_C(0x1F7A6BDB8CB334ED), UINT64_C(0xC3DB47F9D239DAF0), + UINT64_C(0xD7BE8017488E53A1), UINT64_C(0x0B1FAC351604BDBC), + UINT64_C(0x6FFDD953F49A8E9A), UINT64_C(0xB35CF571AA106087), + UINT64_C(0x4737578EC1F59D39), UINT64_C(0x9B967BAC9F7F7324), + UINT64_C(0xFF740ECA7DE14002), UINT64_C(0x23D522E8236BAE1F), + UINT64_C(0x37B0E506B9DC274E), UINT64_C(0xEB11C924E756C953), + UINT64_C(0x8FF3BC4205C8FA75), UINT64_C(0x535290605B421468), + UINT64_C(0xE23AF612F9ADD99A), UINT64_C(0x3E9BDA30A7273787), + UINT64_C(0x5A79AF5645B904A1), UINT64_C(0x86D883741B33EABC), + UINT64_C(0x92BD449A818463ED), UINT64_C(0x4E1C68B8DF0E8DF0), + UINT64_C(0x2AFE1DDE3D90BED6), UINT64_C(0xF65F31FC631A50CB), + UINT64_C(0x0234930308FFAD75), UINT64_C(0xDE95BF2156754368), + UINT64_C(0xBA77CA47B4EB704E), UINT64_C(0x66D6E665EA619E53), + UINT64_C(0x72B3218B70D61702), UINT64_C(0xAE120DA92E5CF91F), + UINT64_C(0xCAF078CFCCC2CA39), UINT64_C(0x165154ED92482424), + UINT64_C(0x2D3FBA84A3B2894E), UINT64_C(0xF19E96A6FD386753), + UINT64_C(0x957CE3C01FA65475), UINT64_C(0x49DDCFE2412CBA68), + UINT64_C(0x5DB8080CDB9B3339), UINT64_C(0x8119242E8511DD24), + UINT64_C(0xE5FB5148678FEE02), UINT64_C(0x395A7D6A3905001F), + UINT64_C(0xCD31DF9552E0FDA1), UINT64_C(0x1190F3B70C6A13BC), + UINT64_C(0x757286D1EEF4209A), UINT64_C(0xA9D3AAF3B07ECE87), + UINT64_C(0xBDB66D1D2AC947D6), UINT64_C(0x6117413F7443A9CB), + UINT64_C(0x05F5345996DD9AED), UINT64_C(0xD954187BC85774F0), + UINT64_C(0x683C7E096AB8B902), UINT64_C(0xB49D522B3432571F), + UINT64_C(0xD07F274DD6AC6439), UINT64_C(0x0CDE0B6F88268A24), + UINT64_C(0x18BBCC8112910375), UINT64_C(0xC41AE0A34C1BED68), + UINT64_C(0xA0F895C5AE85DE4E), UINT64_C(0x7C59B9E7F00F3053), + UINT64_C(0x88321B189BEACDED), UINT64_C(0x5493373AC56023F0), + UINT64_C(0x3071425C27FE10D6), UINT64_C(0xECD06E7E7974FECB), + UINT64_C(0xF8B5A990E3C3779A), UINT64_C(0x241485B2BD499987), + UINT64_C(0x40F6F0D45FD7AAA1), UINT64_C(0x9C57DCF6015D44BC), + UINT64_C(0x362A2C073D23F174), UINT64_C(0xEA8B002563A91F69), + UINT64_C(0x8E69754381372C4F), UINT64_C(0x52C85961DFBDC252), + UINT64_C(0x46AD9E8F450A4B03), UINT64_C(0x9A0CB2AD1B80A51E), + UINT64_C(0xFEEEC7CBF91E9638), UINT64_C(0x224FEBE9A7947825), + UINT64_C(0xD6244916CC71859B), UINT64_C(0x0A85653492FB6B86), + UINT64_C(0x6E671052706558A0), UINT64_C(0xB2C63C702EEFB6BD), + UINT64_C(0xA6A3FB9EB4583FEC), UINT64_C(0x7A02D7BCEAD2D1F1), + UINT64_C(0x1EE0A2DA084CE2D7), UINT64_C(0xC2418EF856C60CCA), + UINT64_C(0x7329E88AF429C138), UINT64_C(0xAF88C4A8AAA32F25), + UINT64_C(0xCB6AB1CE483D1C03), UINT64_C(0x17CB9DEC16B7F21E), + UINT64_C(0x03AE5A028C007B4F), UINT64_C(0xDF0F7620D28A9552), + UINT64_C(0xBBED03463014A674), UINT64_C(0x674C2F646E9E4869), + UINT64_C(0x93278D9B057BB5D7), UINT64_C(0x4F86A1B95BF15BCA), + UINT64_C(0x2B64D4DFB96F68EC), UINT64_C(0xF7C5F8FDE7E586F1), + UINT64_C(0xE3A03F137D520FA0), UINT64_C(0x3F01133123D8E1BD), + UINT64_C(0x5BE36657C146D29B), UINT64_C(0x87424A759FCC3C86), + UINT64_C(0xBC2CA41CAE3691EC), UINT64_C(0x608D883EF0BC7FF1), + UINT64_C(0x046FFD5812224CD7), UINT64_C(0xD8CED17A4CA8A2CA), + UINT64_C(0xCCAB1694D61F2B9B), UINT64_C(0x100A3AB68895C586), + UINT64_C(0x74E84FD06A0BF6A0), UINT64_C(0xA84963F2348118BD), + UINT64_C(0x5C22C10D5F64E503), UINT64_C(0x8083ED2F01EE0B1E), + UINT64_C(0xE4619849E3703838), UINT64_C(0x38C0B46BBDFAD625), + UINT64_C(0x2CA57385274D5F74), UINT64_C(0xF0045FA779C7B169), + UINT64_C(0x94E62AC19B59824F), UINT64_C(0x484706E3C5D36C52), + UINT64_C(0xF92F6091673CA1A0), UINT64_C(0x258E4CB339B64FBD), + UINT64_C(0x416C39D5DB287C9B), UINT64_C(0x9DCD15F785A29286), + UINT64_C(0x89A8D2191F151BD7), UINT64_C(0x5509FE3B419FF5CA), + UINT64_C(0x31EB8B5DA301C6EC), UINT64_C(0xED4AA77FFD8B28F1), + UINT64_C(0x19210580966ED54F), UINT64_C(0xC58029A2C8E43B52), + UINT64_C(0xA1625CC42A7A0874), UINT64_C(0x7DC370E674F0E669), + UINT64_C(0x69A6B708EE476F38), UINT64_C(0xB5079B2AB0CD8125), + UINT64_C(0xD1E5EE4C5253B203), UINT64_C(0x0D44C26E0CD95C1E) + } +}; diff --git a/src/liblzma/check/crc64_table_le.h b/src/liblzma/check/crc64_table_le.h new file mode 100644 index 00000000..3047ea16 --- /dev/null +++ b/src/liblzma/check/crc64_table_le.h @@ -0,0 +1,523 @@ +/* This file has been automatically generated by crc64_tablegen.c. */ + +#include <inttypes.h> + +const uint64_t lzma_crc64_table[4][256] = { + { + UINT64_C(0x0000000000000000), UINT64_C(0xB32E4CBE03A75F6F), + UINT64_C(0xF4843657A840A05B), UINT64_C(0x47AA7AE9ABE7FF34), + UINT64_C(0x7BD0C384FF8F5E33), UINT64_C(0xC8FE8F3AFC28015C), + UINT64_C(0x8F54F5D357CFFE68), UINT64_C(0x3C7AB96D5468A107), + UINT64_C(0xF7A18709FF1EBC66), UINT64_C(0x448FCBB7FCB9E309), + UINT64_C(0x0325B15E575E1C3D), UINT64_C(0xB00BFDE054F94352), + UINT64_C(0x8C71448D0091E255), UINT64_C(0x3F5F08330336BD3A), + UINT64_C(0x78F572DAA8D1420E), UINT64_C(0xCBDB3E64AB761D61), + UINT64_C(0x7D9BA13851336649), UINT64_C(0xCEB5ED8652943926), + UINT64_C(0x891F976FF973C612), UINT64_C(0x3A31DBD1FAD4997D), + UINT64_C(0x064B62BCAEBC387A), UINT64_C(0xB5652E02AD1B6715), + UINT64_C(0xF2CF54EB06FC9821), UINT64_C(0x41E11855055BC74E), + UINT64_C(0x8A3A2631AE2DDA2F), UINT64_C(0x39146A8FAD8A8540), + UINT64_C(0x7EBE1066066D7A74), UINT64_C(0xCD905CD805CA251B), + UINT64_C(0xF1EAE5B551A2841C), UINT64_C(0x42C4A90B5205DB73), + UINT64_C(0x056ED3E2F9E22447), UINT64_C(0xB6409F5CFA457B28), + UINT64_C(0xFB374270A266CC92), UINT64_C(0x48190ECEA1C193FD), + UINT64_C(0x0FB374270A266CC9), UINT64_C(0xBC9D3899098133A6), + UINT64_C(0x80E781F45DE992A1), UINT64_C(0x33C9CD4A5E4ECDCE), + UINT64_C(0x7463B7A3F5A932FA), UINT64_C(0xC74DFB1DF60E6D95), + UINT64_C(0x0C96C5795D7870F4), UINT64_C(0xBFB889C75EDF2F9B), + UINT64_C(0xF812F32EF538D0AF), UINT64_C(0x4B3CBF90F69F8FC0), + UINT64_C(0x774606FDA2F72EC7), UINT64_C(0xC4684A43A15071A8), + UINT64_C(0x83C230AA0AB78E9C), UINT64_C(0x30EC7C140910D1F3), + UINT64_C(0x86ACE348F355AADB), UINT64_C(0x3582AFF6F0F2F5B4), + UINT64_C(0x7228D51F5B150A80), UINT64_C(0xC10699A158B255EF), + UINT64_C(0xFD7C20CC0CDAF4E8), UINT64_C(0x4E526C720F7DAB87), + UINT64_C(0x09F8169BA49A54B3), UINT64_C(0xBAD65A25A73D0BDC), + UINT64_C(0x710D64410C4B16BD), UINT64_C(0xC22328FF0FEC49D2), + UINT64_C(0x85895216A40BB6E6), UINT64_C(0x36A71EA8A7ACE989), + UINT64_C(0x0ADDA7C5F3C4488E), UINT64_C(0xB9F3EB7BF06317E1), + UINT64_C(0xFE5991925B84E8D5), UINT64_C(0x4D77DD2C5823B7BA), + UINT64_C(0x64B62BCAEBC387A1), UINT64_C(0xD7986774E864D8CE), + UINT64_C(0x90321D9D438327FA), UINT64_C(0x231C512340247895), + UINT64_C(0x1F66E84E144CD992), UINT64_C(0xAC48A4F017EB86FD), + UINT64_C(0xEBE2DE19BC0C79C9), UINT64_C(0x58CC92A7BFAB26A6), + UINT64_C(0x9317ACC314DD3BC7), UINT64_C(0x2039E07D177A64A8), + UINT64_C(0x67939A94BC9D9B9C), UINT64_C(0xD4BDD62ABF3AC4F3), + UINT64_C(0xE8C76F47EB5265F4), UINT64_C(0x5BE923F9E8F53A9B), + UINT64_C(0x1C4359104312C5AF), UINT64_C(0xAF6D15AE40B59AC0), + UINT64_C(0x192D8AF2BAF0E1E8), UINT64_C(0xAA03C64CB957BE87), + UINT64_C(0xEDA9BCA512B041B3), UINT64_C(0x5E87F01B11171EDC), + UINT64_C(0x62FD4976457FBFDB), UINT64_C(0xD1D305C846D8E0B4), + UINT64_C(0x96797F21ED3F1F80), UINT64_C(0x2557339FEE9840EF), + UINT64_C(0xEE8C0DFB45EE5D8E), UINT64_C(0x5DA24145464902E1), + UINT64_C(0x1A083BACEDAEFDD5), UINT64_C(0xA9267712EE09A2BA), + UINT64_C(0x955CCE7FBA6103BD), UINT64_C(0x267282C1B9C65CD2), + UINT64_C(0x61D8F8281221A3E6), UINT64_C(0xD2F6B4961186FC89), + UINT64_C(0x9F8169BA49A54B33), UINT64_C(0x2CAF25044A02145C), + UINT64_C(0x6B055FEDE1E5EB68), UINT64_C(0xD82B1353E242B407), + UINT64_C(0xE451AA3EB62A1500), UINT64_C(0x577FE680B58D4A6F), + UINT64_C(0x10D59C691E6AB55B), UINT64_C(0xA3FBD0D71DCDEA34), + UINT64_C(0x6820EEB3B6BBF755), UINT64_C(0xDB0EA20DB51CA83A), + UINT64_C(0x9CA4D8E41EFB570E), UINT64_C(0x2F8A945A1D5C0861), + UINT64_C(0x13F02D374934A966), UINT64_C(0xA0DE61894A93F609), + UINT64_C(0xE7741B60E174093D), UINT64_C(0x545A57DEE2D35652), + UINT64_C(0xE21AC88218962D7A), UINT64_C(0x5134843C1B317215), + UINT64_C(0x169EFED5B0D68D21), UINT64_C(0xA5B0B26BB371D24E), + UINT64_C(0x99CA0B06E7197349), UINT64_C(0x2AE447B8E4BE2C26), + UINT64_C(0x6D4E3D514F59D312), UINT64_C(0xDE6071EF4CFE8C7D), + UINT64_C(0x15BB4F8BE788911C), UINT64_C(0xA6950335E42FCE73), + UINT64_C(0xE13F79DC4FC83147), UINT64_C(0x521135624C6F6E28), + UINT64_C(0x6E6B8C0F1807CF2F), UINT64_C(0xDD45C0B11BA09040), + UINT64_C(0x9AEFBA58B0476F74), UINT64_C(0x29C1F6E6B3E0301B), + UINT64_C(0xC96C5795D7870F42), UINT64_C(0x7A421B2BD420502D), + UINT64_C(0x3DE861C27FC7AF19), UINT64_C(0x8EC62D7C7C60F076), + UINT64_C(0xB2BC941128085171), UINT64_C(0x0192D8AF2BAF0E1E), + UINT64_C(0x4638A2468048F12A), UINT64_C(0xF516EEF883EFAE45), + UINT64_C(0x3ECDD09C2899B324), UINT64_C(0x8DE39C222B3EEC4B), + UINT64_C(0xCA49E6CB80D9137F), UINT64_C(0x7967AA75837E4C10), + UINT64_C(0x451D1318D716ED17), UINT64_C(0xF6335FA6D4B1B278), + UINT64_C(0xB199254F7F564D4C), UINT64_C(0x02B769F17CF11223), + UINT64_C(0xB4F7F6AD86B4690B), UINT64_C(0x07D9BA1385133664), + UINT64_C(0x4073C0FA2EF4C950), UINT64_C(0xF35D8C442D53963F), + UINT64_C(0xCF273529793B3738), UINT64_C(0x7C0979977A9C6857), + UINT64_C(0x3BA3037ED17B9763), UINT64_C(0x888D4FC0D2DCC80C), + UINT64_C(0x435671A479AAD56D), UINT64_C(0xF0783D1A7A0D8A02), + UINT64_C(0xB7D247F3D1EA7536), UINT64_C(0x04FC0B4DD24D2A59), + UINT64_C(0x3886B22086258B5E), UINT64_C(0x8BA8FE9E8582D431), + UINT64_C(0xCC0284772E652B05), UINT64_C(0x7F2CC8C92DC2746A), + UINT64_C(0x325B15E575E1C3D0), UINT64_C(0x8175595B76469CBF), + UINT64_C(0xC6DF23B2DDA1638B), UINT64_C(0x75F16F0CDE063CE4), + UINT64_C(0x498BD6618A6E9DE3), UINT64_C(0xFAA59ADF89C9C28C), + UINT64_C(0xBD0FE036222E3DB8), UINT64_C(0x0E21AC88218962D7), + UINT64_C(0xC5FA92EC8AFF7FB6), UINT64_C(0x76D4DE52895820D9), + UINT64_C(0x317EA4BB22BFDFED), UINT64_C(0x8250E80521188082), + UINT64_C(0xBE2A516875702185), UINT64_C(0x0D041DD676D77EEA), + UINT64_C(0x4AAE673FDD3081DE), UINT64_C(0xF9802B81DE97DEB1), + UINT64_C(0x4FC0B4DD24D2A599), UINT64_C(0xFCEEF8632775FAF6), + UINT64_C(0xBB44828A8C9205C2), UINT64_C(0x086ACE348F355AAD), + UINT64_C(0x34107759DB5DFBAA), UINT64_C(0x873E3BE7D8FAA4C5), + UINT64_C(0xC094410E731D5BF1), UINT64_C(0x73BA0DB070BA049E), + UINT64_C(0xB86133D4DBCC19FF), UINT64_C(0x0B4F7F6AD86B4690), + UINT64_C(0x4CE50583738CB9A4), UINT64_C(0xFFCB493D702BE6CB), + UINT64_C(0xC3B1F050244347CC), UINT64_C(0x709FBCEE27E418A3), + UINT64_C(0x3735C6078C03E797), UINT64_C(0x841B8AB98FA4B8F8), + UINT64_C(0xADDA7C5F3C4488E3), UINT64_C(0x1EF430E13FE3D78C), + UINT64_C(0x595E4A08940428B8), UINT64_C(0xEA7006B697A377D7), + UINT64_C(0xD60ABFDBC3CBD6D0), UINT64_C(0x6524F365C06C89BF), + UINT64_C(0x228E898C6B8B768B), UINT64_C(0x91A0C532682C29E4), + UINT64_C(0x5A7BFB56C35A3485), UINT64_C(0xE955B7E8C0FD6BEA), + UINT64_C(0xAEFFCD016B1A94DE), UINT64_C(0x1DD181BF68BDCBB1), + UINT64_C(0x21AB38D23CD56AB6), UINT64_C(0x9285746C3F7235D9), + UINT64_C(0xD52F0E859495CAED), UINT64_C(0x6601423B97329582), + UINT64_C(0xD041DD676D77EEAA), UINT64_C(0x636F91D96ED0B1C5), + UINT64_C(0x24C5EB30C5374EF1), UINT64_C(0x97EBA78EC690119E), + UINT64_C(0xAB911EE392F8B099), UINT64_C(0x18BF525D915FEFF6), + UINT64_C(0x5F1528B43AB810C2), UINT64_C(0xEC3B640A391F4FAD), + UINT64_C(0x27E05A6E926952CC), UINT64_C(0x94CE16D091CE0DA3), + UINT64_C(0xD3646C393A29F297), UINT64_C(0x604A2087398EADF8), + UINT64_C(0x5C3099EA6DE60CFF), UINT64_C(0xEF1ED5546E415390), + UINT64_C(0xA8B4AFBDC5A6ACA4), UINT64_C(0x1B9AE303C601F3CB), + UINT64_C(0x56ED3E2F9E224471), UINT64_C(0xE5C372919D851B1E), + UINT64_C(0xA26908783662E42A), UINT64_C(0x114744C635C5BB45), + UINT64_C(0x2D3DFDAB61AD1A42), UINT64_C(0x9E13B115620A452D), + UINT64_C(0xD9B9CBFCC9EDBA19), UINT64_C(0x6A978742CA4AE576), + UINT64_C(0xA14CB926613CF817), UINT64_C(0x1262F598629BA778), + UINT64_C(0x55C88F71C97C584C), UINT64_C(0xE6E6C3CFCADB0723), + UINT64_C(0xDA9C7AA29EB3A624), UINT64_C(0x69B2361C9D14F94B), + UINT64_C(0x2E184CF536F3067F), UINT64_C(0x9D36004B35545910), + UINT64_C(0x2B769F17CF112238), UINT64_C(0x9858D3A9CCB67D57), + UINT64_C(0xDFF2A94067518263), UINT64_C(0x6CDCE5FE64F6DD0C), + UINT64_C(0x50A65C93309E7C0B), UINT64_C(0xE388102D33392364), + UINT64_C(0xA4226AC498DEDC50), UINT64_C(0x170C267A9B79833F), + UINT64_C(0xDCD7181E300F9E5E), UINT64_C(0x6FF954A033A8C131), + UINT64_C(0x28532E49984F3E05), UINT64_C(0x9B7D62F79BE8616A), + UINT64_C(0xA707DB9ACF80C06D), UINT64_C(0x14299724CC279F02), + UINT64_C(0x5383EDCD67C06036), UINT64_C(0xE0ADA17364673F59) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0x54E979925CD0F10D), + UINT64_C(0xA9D2F324B9A1E21A), UINT64_C(0xFD3B8AB6E5711317), + UINT64_C(0xC17D4962DC4DDAB1), UINT64_C(0x959430F0809D2BBC), + UINT64_C(0x68AFBA4665EC38AB), UINT64_C(0x3C46C3D4393CC9A6), + UINT64_C(0x10223DEE1795ABE7), UINT64_C(0x44CB447C4B455AEA), + UINT64_C(0xB9F0CECAAE3449FD), UINT64_C(0xED19B758F2E4B8F0), + UINT64_C(0xD15F748CCBD87156), UINT64_C(0x85B60D1E9708805B), + UINT64_C(0x788D87A87279934C), UINT64_C(0x2C64FE3A2EA96241), + UINT64_C(0x20447BDC2F2B57CE), UINT64_C(0x74AD024E73FBA6C3), + UINT64_C(0x899688F8968AB5D4), UINT64_C(0xDD7FF16ACA5A44D9), + UINT64_C(0xE13932BEF3668D7F), UINT64_C(0xB5D04B2CAFB67C72), + UINT64_C(0x48EBC19A4AC76F65), UINT64_C(0x1C02B80816179E68), + UINT64_C(0x3066463238BEFC29), UINT64_C(0x648F3FA0646E0D24), + UINT64_C(0x99B4B516811F1E33), UINT64_C(0xCD5DCC84DDCFEF3E), + UINT64_C(0xF11B0F50E4F32698), UINT64_C(0xA5F276C2B823D795), + UINT64_C(0x58C9FC745D52C482), UINT64_C(0x0C2085E60182358F), + UINT64_C(0x4088F7B85E56AF9C), UINT64_C(0x14618E2A02865E91), + UINT64_C(0xE95A049CE7F74D86), UINT64_C(0xBDB37D0EBB27BC8B), + UINT64_C(0x81F5BEDA821B752D), UINT64_C(0xD51CC748DECB8420), + UINT64_C(0x28274DFE3BBA9737), UINT64_C(0x7CCE346C676A663A), + UINT64_C(0x50AACA5649C3047B), UINT64_C(0x0443B3C41513F576), + UINT64_C(0xF9783972F062E661), UINT64_C(0xAD9140E0ACB2176C), + UINT64_C(0x91D78334958EDECA), UINT64_C(0xC53EFAA6C95E2FC7), + UINT64_C(0x380570102C2F3CD0), UINT64_C(0x6CEC098270FFCDDD), + UINT64_C(0x60CC8C64717DF852), UINT64_C(0x3425F5F62DAD095F), + UINT64_C(0xC91E7F40C8DC1A48), UINT64_C(0x9DF706D2940CEB45), + UINT64_C(0xA1B1C506AD3022E3), UINT64_C(0xF558BC94F1E0D3EE), + UINT64_C(0x086336221491C0F9), UINT64_C(0x5C8A4FB0484131F4), + UINT64_C(0x70EEB18A66E853B5), UINT64_C(0x2407C8183A38A2B8), + UINT64_C(0xD93C42AEDF49B1AF), UINT64_C(0x8DD53B3C839940A2), + UINT64_C(0xB193F8E8BAA58904), UINT64_C(0xE57A817AE6757809), + UINT64_C(0x18410BCC03046B1E), UINT64_C(0x4CA8725E5FD49A13), + UINT64_C(0x8111EF70BCAD5F38), UINT64_C(0xD5F896E2E07DAE35), + UINT64_C(0x28C31C54050CBD22), UINT64_C(0x7C2A65C659DC4C2F), + UINT64_C(0x406CA61260E08589), UINT64_C(0x1485DF803C307484), + UINT64_C(0xE9BE5536D9416793), UINT64_C(0xBD572CA48591969E), + UINT64_C(0x9133D29EAB38F4DF), UINT64_C(0xC5DAAB0CF7E805D2), + UINT64_C(0x38E121BA129916C5), UINT64_C(0x6C0858284E49E7C8), + UINT64_C(0x504E9BFC77752E6E), UINT64_C(0x04A7E26E2BA5DF63), + UINT64_C(0xF99C68D8CED4CC74), UINT64_C(0xAD75114A92043D79), + UINT64_C(0xA15594AC938608F6), UINT64_C(0xF5BCED3ECF56F9FB), + UINT64_C(0x088767882A27EAEC), UINT64_C(0x5C6E1E1A76F71BE1), + UINT64_C(0x6028DDCE4FCBD247), UINT64_C(0x34C1A45C131B234A), + UINT64_C(0xC9FA2EEAF66A305D), UINT64_C(0x9D135778AABAC150), + UINT64_C(0xB177A9428413A311), UINT64_C(0xE59ED0D0D8C3521C), + UINT64_C(0x18A55A663DB2410B), UINT64_C(0x4C4C23F46162B006), + UINT64_C(0x700AE020585E79A0), UINT64_C(0x24E399B2048E88AD), + UINT64_C(0xD9D81304E1FF9BBA), UINT64_C(0x8D316A96BD2F6AB7), + UINT64_C(0xC19918C8E2FBF0A4), UINT64_C(0x9570615ABE2B01A9), + UINT64_C(0x684BEBEC5B5A12BE), UINT64_C(0x3CA2927E078AE3B3), + UINT64_C(0x00E451AA3EB62A15), UINT64_C(0x540D28386266DB18), + UINT64_C(0xA936A28E8717C80F), UINT64_C(0xFDDFDB1CDBC73902), + UINT64_C(0xD1BB2526F56E5B43), UINT64_C(0x85525CB4A9BEAA4E), + UINT64_C(0x7869D6024CCFB959), UINT64_C(0x2C80AF90101F4854), + UINT64_C(0x10C66C44292381F2), UINT64_C(0x442F15D675F370FF), + UINT64_C(0xB9149F60908263E8), UINT64_C(0xEDFDE6F2CC5292E5), + UINT64_C(0xE1DD6314CDD0A76A), UINT64_C(0xB5341A8691005667), + UINT64_C(0x480F903074714570), UINT64_C(0x1CE6E9A228A1B47D), + UINT64_C(0x20A02A76119D7DDB), UINT64_C(0x744953E44D4D8CD6), + UINT64_C(0x8972D952A83C9FC1), UINT64_C(0xDD9BA0C0F4EC6ECC), + UINT64_C(0xF1FF5EFADA450C8D), UINT64_C(0xA51627688695FD80), + UINT64_C(0x582DADDE63E4EE97), UINT64_C(0x0CC4D44C3F341F9A), + UINT64_C(0x308217980608D63C), UINT64_C(0x646B6E0A5AD82731), + UINT64_C(0x9950E4BCBFA93426), UINT64_C(0xCDB99D2EE379C52B), + UINT64_C(0x90FB71CAD654A0F5), UINT64_C(0xC41208588A8451F8), + UINT64_C(0x392982EE6FF542EF), UINT64_C(0x6DC0FB7C3325B3E2), + UINT64_C(0x518638A80A197A44), UINT64_C(0x056F413A56C98B49), + UINT64_C(0xF854CB8CB3B8985E), UINT64_C(0xACBDB21EEF686953), + UINT64_C(0x80D94C24C1C10B12), UINT64_C(0xD43035B69D11FA1F), + UINT64_C(0x290BBF007860E908), UINT64_C(0x7DE2C69224B01805), + UINT64_C(0x41A405461D8CD1A3), UINT64_C(0x154D7CD4415C20AE), + UINT64_C(0xE876F662A42D33B9), UINT64_C(0xBC9F8FF0F8FDC2B4), + UINT64_C(0xB0BF0A16F97FF73B), UINT64_C(0xE4567384A5AF0636), + UINT64_C(0x196DF93240DE1521), UINT64_C(0x4D8480A01C0EE42C), + UINT64_C(0x71C2437425322D8A), UINT64_C(0x252B3AE679E2DC87), + UINT64_C(0xD810B0509C93CF90), UINT64_C(0x8CF9C9C2C0433E9D), + UINT64_C(0xA09D37F8EEEA5CDC), UINT64_C(0xF4744E6AB23AADD1), + UINT64_C(0x094FC4DC574BBEC6), UINT64_C(0x5DA6BD4E0B9B4FCB), + UINT64_C(0x61E07E9A32A7866D), UINT64_C(0x350907086E777760), + UINT64_C(0xC8328DBE8B066477), UINT64_C(0x9CDBF42CD7D6957A), + UINT64_C(0xD073867288020F69), UINT64_C(0x849AFFE0D4D2FE64), + UINT64_C(0x79A1755631A3ED73), UINT64_C(0x2D480CC46D731C7E), + UINT64_C(0x110ECF10544FD5D8), UINT64_C(0x45E7B682089F24D5), + UINT64_C(0xB8DC3C34EDEE37C2), UINT64_C(0xEC3545A6B13EC6CF), + UINT64_C(0xC051BB9C9F97A48E), UINT64_C(0x94B8C20EC3475583), + UINT64_C(0x698348B826364694), UINT64_C(0x3D6A312A7AE6B799), + UINT64_C(0x012CF2FE43DA7E3F), UINT64_C(0x55C58B6C1F0A8F32), + UINT64_C(0xA8FE01DAFA7B9C25), UINT64_C(0xFC177848A6AB6D28), + UINT64_C(0xF037FDAEA72958A7), UINT64_C(0xA4DE843CFBF9A9AA), + UINT64_C(0x59E50E8A1E88BABD), UINT64_C(0x0D0C771842584BB0), + UINT64_C(0x314AB4CC7B648216), UINT64_C(0x65A3CD5E27B4731B), + UINT64_C(0x989847E8C2C5600C), UINT64_C(0xCC713E7A9E159101), + UINT64_C(0xE015C040B0BCF340), UINT64_C(0xB4FCB9D2EC6C024D), + UINT64_C(0x49C73364091D115A), UINT64_C(0x1D2E4AF655CDE057), + UINT64_C(0x216889226CF129F1), UINT64_C(0x7581F0B03021D8FC), + UINT64_C(0x88BA7A06D550CBEB), UINT64_C(0xDC53039489803AE6), + UINT64_C(0x11EA9EBA6AF9FFCD), UINT64_C(0x4503E72836290EC0), + UINT64_C(0xB8386D9ED3581DD7), UINT64_C(0xECD1140C8F88ECDA), + UINT64_C(0xD097D7D8B6B4257C), UINT64_C(0x847EAE4AEA64D471), + UINT64_C(0x794524FC0F15C766), UINT64_C(0x2DAC5D6E53C5366B), + UINT64_C(0x01C8A3547D6C542A), UINT64_C(0x5521DAC621BCA527), + UINT64_C(0xA81A5070C4CDB630), UINT64_C(0xFCF329E2981D473D), + UINT64_C(0xC0B5EA36A1218E9B), UINT64_C(0x945C93A4FDF17F96), + UINT64_C(0x6967191218806C81), UINT64_C(0x3D8E608044509D8C), + UINT64_C(0x31AEE56645D2A803), UINT64_C(0x65479CF41902590E), + UINT64_C(0x987C1642FC734A19), UINT64_C(0xCC956FD0A0A3BB14), + UINT64_C(0xF0D3AC04999F72B2), UINT64_C(0xA43AD596C54F83BF), + UINT64_C(0x59015F20203E90A8), UINT64_C(0x0DE826B27CEE61A5), + UINT64_C(0x218CD888524703E4), UINT64_C(0x7565A11A0E97F2E9), + UINT64_C(0x885E2BACEBE6E1FE), UINT64_C(0xDCB7523EB73610F3), + UINT64_C(0xE0F191EA8E0AD955), UINT64_C(0xB418E878D2DA2858), + UINT64_C(0x492362CE37AB3B4F), UINT64_C(0x1DCA1B5C6B7BCA42), + UINT64_C(0x5162690234AF5051), UINT64_C(0x058B1090687FA15C), + UINT64_C(0xF8B09A268D0EB24B), UINT64_C(0xAC59E3B4D1DE4346), + UINT64_C(0x901F2060E8E28AE0), UINT64_C(0xC4F659F2B4327BED), + UINT64_C(0x39CDD344514368FA), UINT64_C(0x6D24AAD60D9399F7), + UINT64_C(0x414054EC233AFBB6), UINT64_C(0x15A92D7E7FEA0ABB), + UINT64_C(0xE892A7C89A9B19AC), UINT64_C(0xBC7BDE5AC64BE8A1), + UINT64_C(0x803D1D8EFF772107), UINT64_C(0xD4D4641CA3A7D00A), + UINT64_C(0x29EFEEAA46D6C31D), UINT64_C(0x7D0697381A063210), + UINT64_C(0x712612DE1B84079F), UINT64_C(0x25CF6B4C4754F692), + UINT64_C(0xD8F4E1FAA225E585), UINT64_C(0x8C1D9868FEF51488), + UINT64_C(0xB05B5BBCC7C9DD2E), UINT64_C(0xE4B2222E9B192C23), + UINT64_C(0x1989A8987E683F34), UINT64_C(0x4D60D10A22B8CE39), + UINT64_C(0x61042F300C11AC78), UINT64_C(0x35ED56A250C15D75), + UINT64_C(0xC8D6DC14B5B04E62), UINT64_C(0x9C3FA586E960BF6F), + UINT64_C(0xA0796652D05C76C9), UINT64_C(0xF4901FC08C8C87C4), + UINT64_C(0x09AB957669FD94D3), UINT64_C(0x5D42ECE4352D65DE) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0x3F0BE14A916A6DCB), + UINT64_C(0x7E17C29522D4DB96), UINT64_C(0x411C23DFB3BEB65D), + UINT64_C(0xFC2F852A45A9B72C), UINT64_C(0xC3246460D4C3DAE7), + UINT64_C(0x823847BF677D6CBA), UINT64_C(0xBD33A6F5F6170171), + UINT64_C(0x6A87A57F245D70DD), UINT64_C(0x558C4435B5371D16), + UINT64_C(0x149067EA0689AB4B), UINT64_C(0x2B9B86A097E3C680), + UINT64_C(0x96A8205561F4C7F1), UINT64_C(0xA9A3C11FF09EAA3A), + UINT64_C(0xE8BFE2C043201C67), UINT64_C(0xD7B4038AD24A71AC), + UINT64_C(0xD50F4AFE48BAE1BA), UINT64_C(0xEA04ABB4D9D08C71), + UINT64_C(0xAB18886B6A6E3A2C), UINT64_C(0x94136921FB0457E7), + UINT64_C(0x2920CFD40D135696), UINT64_C(0x162B2E9E9C793B5D), + UINT64_C(0x57370D412FC78D00), UINT64_C(0x683CEC0BBEADE0CB), + UINT64_C(0xBF88EF816CE79167), UINT64_C(0x80830ECBFD8DFCAC), + UINT64_C(0xC19F2D144E334AF1), UINT64_C(0xFE94CC5EDF59273A), + UINT64_C(0x43A76AAB294E264B), UINT64_C(0x7CAC8BE1B8244B80), + UINT64_C(0x3DB0A83E0B9AFDDD), UINT64_C(0x02BB49749AF09016), + UINT64_C(0x38C63AD73E7BDDF1), UINT64_C(0x07CDDB9DAF11B03A), + UINT64_C(0x46D1F8421CAF0667), UINT64_C(0x79DA19088DC56BAC), + UINT64_C(0xC4E9BFFD7BD26ADD), UINT64_C(0xFBE25EB7EAB80716), + UINT64_C(0xBAFE7D685906B14B), UINT64_C(0x85F59C22C86CDC80), + UINT64_C(0x52419FA81A26AD2C), UINT64_C(0x6D4A7EE28B4CC0E7), + UINT64_C(0x2C565D3D38F276BA), UINT64_C(0x135DBC77A9981B71), + UINT64_C(0xAE6E1A825F8F1A00), UINT64_C(0x9165FBC8CEE577CB), + UINT64_C(0xD079D8177D5BC196), UINT64_C(0xEF72395DEC31AC5D), + UINT64_C(0xEDC9702976C13C4B), UINT64_C(0xD2C29163E7AB5180), + UINT64_C(0x93DEB2BC5415E7DD), UINT64_C(0xACD553F6C57F8A16), + UINT64_C(0x11E6F50333688B67), UINT64_C(0x2EED1449A202E6AC), + UINT64_C(0x6FF1379611BC50F1), UINT64_C(0x50FAD6DC80D63D3A), + UINT64_C(0x874ED556529C4C96), UINT64_C(0xB845341CC3F6215D), + UINT64_C(0xF95917C370489700), UINT64_C(0xC652F689E122FACB), + UINT64_C(0x7B61507C1735FBBA), UINT64_C(0x446AB136865F9671), + UINT64_C(0x057692E935E1202C), UINT64_C(0x3A7D73A3A48B4DE7), + UINT64_C(0x718C75AE7CF7BBE2), UINT64_C(0x4E8794E4ED9DD629), + UINT64_C(0x0F9BB73B5E236074), UINT64_C(0x30905671CF490DBF), + UINT64_C(0x8DA3F084395E0CCE), UINT64_C(0xB2A811CEA8346105), + UINT64_C(0xF3B432111B8AD758), UINT64_C(0xCCBFD35B8AE0BA93), + UINT64_C(0x1B0BD0D158AACB3F), UINT64_C(0x2400319BC9C0A6F4), + UINT64_C(0x651C12447A7E10A9), UINT64_C(0x5A17F30EEB147D62), + UINT64_C(0xE72455FB1D037C13), UINT64_C(0xD82FB4B18C6911D8), + UINT64_C(0x9933976E3FD7A785), UINT64_C(0xA6387624AEBDCA4E), + UINT64_C(0xA4833F50344D5A58), UINT64_C(0x9B88DE1AA5273793), + UINT64_C(0xDA94FDC5169981CE), UINT64_C(0xE59F1C8F87F3EC05), + UINT64_C(0x58ACBA7A71E4ED74), UINT64_C(0x67A75B30E08E80BF), + UINT64_C(0x26BB78EF533036E2), UINT64_C(0x19B099A5C25A5B29), + UINT64_C(0xCE049A2F10102A85), UINT64_C(0xF10F7B65817A474E), + UINT64_C(0xB01358BA32C4F113), UINT64_C(0x8F18B9F0A3AE9CD8), + UINT64_C(0x322B1F0555B99DA9), UINT64_C(0x0D20FE4FC4D3F062), + UINT64_C(0x4C3CDD90776D463F), UINT64_C(0x73373CDAE6072BF4), + UINT64_C(0x494A4F79428C6613), UINT64_C(0x7641AE33D3E60BD8), + UINT64_C(0x375D8DEC6058BD85), UINT64_C(0x08566CA6F132D04E), + UINT64_C(0xB565CA530725D13F), UINT64_C(0x8A6E2B19964FBCF4), + UINT64_C(0xCB7208C625F10AA9), UINT64_C(0xF479E98CB49B6762), + UINT64_C(0x23CDEA0666D116CE), UINT64_C(0x1CC60B4CF7BB7B05), + UINT64_C(0x5DDA28934405CD58), UINT64_C(0x62D1C9D9D56FA093), + UINT64_C(0xDFE26F2C2378A1E2), UINT64_C(0xE0E98E66B212CC29), + UINT64_C(0xA1F5ADB901AC7A74), UINT64_C(0x9EFE4CF390C617BF), + UINT64_C(0x9C4505870A3687A9), UINT64_C(0xA34EE4CD9B5CEA62), + UINT64_C(0xE252C71228E25C3F), UINT64_C(0xDD592658B98831F4), + UINT64_C(0x606A80AD4F9F3085), UINT64_C(0x5F6161E7DEF55D4E), + UINT64_C(0x1E7D42386D4BEB13), UINT64_C(0x2176A372FC2186D8), + UINT64_C(0xF6C2A0F82E6BF774), UINT64_C(0xC9C941B2BF019ABF), + UINT64_C(0x88D5626D0CBF2CE2), UINT64_C(0xB7DE83279DD54129), + UINT64_C(0x0AED25D26BC24058), UINT64_C(0x35E6C498FAA82D93), + UINT64_C(0x74FAE74749169BCE), UINT64_C(0x4BF1060DD87CF605), + UINT64_C(0xE318EB5CF9EF77C4), UINT64_C(0xDC130A1668851A0F), + UINT64_C(0x9D0F29C9DB3BAC52), UINT64_C(0xA204C8834A51C199), + UINT64_C(0x1F376E76BC46C0E8), UINT64_C(0x203C8F3C2D2CAD23), + UINT64_C(0x6120ACE39E921B7E), UINT64_C(0x5E2B4DA90FF876B5), + UINT64_C(0x899F4E23DDB20719), UINT64_C(0xB694AF694CD86AD2), + UINT64_C(0xF7888CB6FF66DC8F), UINT64_C(0xC8836DFC6E0CB144), + UINT64_C(0x75B0CB09981BB035), UINT64_C(0x4ABB2A430971DDFE), + UINT64_C(0x0BA7099CBACF6BA3), UINT64_C(0x34ACE8D62BA50668), + UINT64_C(0x3617A1A2B155967E), UINT64_C(0x091C40E8203FFBB5), + UINT64_C(0x4800633793814DE8), UINT64_C(0x770B827D02EB2023), + UINT64_C(0xCA382488F4FC2152), UINT64_C(0xF533C5C265964C99), + UINT64_C(0xB42FE61DD628FAC4), UINT64_C(0x8B2407574742970F), + UINT64_C(0x5C9004DD9508E6A3), UINT64_C(0x639BE59704628B68), + UINT64_C(0x2287C648B7DC3D35), UINT64_C(0x1D8C270226B650FE), + UINT64_C(0xA0BF81F7D0A1518F), UINT64_C(0x9FB460BD41CB3C44), + UINT64_C(0xDEA84362F2758A19), UINT64_C(0xE1A3A228631FE7D2), + UINT64_C(0xDBDED18BC794AA35), UINT64_C(0xE4D530C156FEC7FE), + UINT64_C(0xA5C9131EE54071A3), UINT64_C(0x9AC2F254742A1C68), + UINT64_C(0x27F154A1823D1D19), UINT64_C(0x18FAB5EB135770D2), + UINT64_C(0x59E69634A0E9C68F), UINT64_C(0x66ED777E3183AB44), + UINT64_C(0xB15974F4E3C9DAE8), UINT64_C(0x8E5295BE72A3B723), + UINT64_C(0xCF4EB661C11D017E), UINT64_C(0xF045572B50776CB5), + UINT64_C(0x4D76F1DEA6606DC4), UINT64_C(0x727D1094370A000F), + UINT64_C(0x3361334B84B4B652), UINT64_C(0x0C6AD20115DEDB99), + UINT64_C(0x0ED19B758F2E4B8F), UINT64_C(0x31DA7A3F1E442644), + UINT64_C(0x70C659E0ADFA9019), UINT64_C(0x4FCDB8AA3C90FDD2), + UINT64_C(0xF2FE1E5FCA87FCA3), UINT64_C(0xCDF5FF155BED9168), + UINT64_C(0x8CE9DCCAE8532735), UINT64_C(0xB3E23D8079394AFE), + UINT64_C(0x64563E0AAB733B52), UINT64_C(0x5B5DDF403A195699), + UINT64_C(0x1A41FC9F89A7E0C4), UINT64_C(0x254A1DD518CD8D0F), + UINT64_C(0x9879BB20EEDA8C7E), UINT64_C(0xA7725A6A7FB0E1B5), + UINT64_C(0xE66E79B5CC0E57E8), UINT64_C(0xD96598FF5D643A23), + UINT64_C(0x92949EF28518CC26), UINT64_C(0xAD9F7FB81472A1ED), + UINT64_C(0xEC835C67A7CC17B0), UINT64_C(0xD388BD2D36A67A7B), + UINT64_C(0x6EBB1BD8C0B17B0A), UINT64_C(0x51B0FA9251DB16C1), + UINT64_C(0x10ACD94DE265A09C), UINT64_C(0x2FA73807730FCD57), + UINT64_C(0xF8133B8DA145BCFB), UINT64_C(0xC718DAC7302FD130), + UINT64_C(0x8604F9188391676D), UINT64_C(0xB90F185212FB0AA6), + UINT64_C(0x043CBEA7E4EC0BD7), UINT64_C(0x3B375FED7586661C), + UINT64_C(0x7A2B7C32C638D041), UINT64_C(0x45209D785752BD8A), + UINT64_C(0x479BD40CCDA22D9C), UINT64_C(0x789035465CC84057), + UINT64_C(0x398C1699EF76F60A), UINT64_C(0x0687F7D37E1C9BC1), + UINT64_C(0xBBB45126880B9AB0), UINT64_C(0x84BFB06C1961F77B), + UINT64_C(0xC5A393B3AADF4126), UINT64_C(0xFAA872F93BB52CED), + UINT64_C(0x2D1C7173E9FF5D41), UINT64_C(0x121790397895308A), + UINT64_C(0x530BB3E6CB2B86D7), UINT64_C(0x6C0052AC5A41EB1C), + UINT64_C(0xD133F459AC56EA6D), UINT64_C(0xEE3815133D3C87A6), + UINT64_C(0xAF2436CC8E8231FB), UINT64_C(0x902FD7861FE85C30), + UINT64_C(0xAA52A425BB6311D7), UINT64_C(0x9559456F2A097C1C), + UINT64_C(0xD44566B099B7CA41), UINT64_C(0xEB4E87FA08DDA78A), + UINT64_C(0x567D210FFECAA6FB), UINT64_C(0x6976C0456FA0CB30), + UINT64_C(0x286AE39ADC1E7D6D), UINT64_C(0x176102D04D7410A6), + UINT64_C(0xC0D5015A9F3E610A), UINT64_C(0xFFDEE0100E540CC1), + UINT64_C(0xBEC2C3CFBDEABA9C), UINT64_C(0x81C922852C80D757), + UINT64_C(0x3CFA8470DA97D626), UINT64_C(0x03F1653A4BFDBBED), + UINT64_C(0x42ED46E5F8430DB0), UINT64_C(0x7DE6A7AF6929607B), + UINT64_C(0x7F5DEEDBF3D9F06D), UINT64_C(0x40560F9162B39DA6), + UINT64_C(0x014A2C4ED10D2BFB), UINT64_C(0x3E41CD0440674630), + UINT64_C(0x83726BF1B6704741), UINT64_C(0xBC798ABB271A2A8A), + UINT64_C(0xFD65A96494A49CD7), UINT64_C(0xC26E482E05CEF11C), + UINT64_C(0x15DA4BA4D78480B0), UINT64_C(0x2AD1AAEE46EEED7B), + UINT64_C(0x6BCD8931F5505B26), UINT64_C(0x54C6687B643A36ED), + UINT64_C(0xE9F5CE8E922D379C), UINT64_C(0xD6FE2FC403475A57), + UINT64_C(0x97E20C1BB0F9EC0A), UINT64_C(0xA8E9ED51219381C1) + }, { + UINT64_C(0x0000000000000000), UINT64_C(0x1DEE8A5E222CA1DC), + UINT64_C(0x3BDD14BC445943B8), UINT64_C(0x26339EE26675E264), + UINT64_C(0x77BA297888B28770), UINT64_C(0x6A54A326AA9E26AC), + UINT64_C(0x4C673DC4CCEBC4C8), UINT64_C(0x5189B79AEEC76514), + UINT64_C(0xEF7452F111650EE0), UINT64_C(0xF29AD8AF3349AF3C), + UINT64_C(0xD4A9464D553C4D58), UINT64_C(0xC947CC137710EC84), + UINT64_C(0x98CE7B8999D78990), UINT64_C(0x8520F1D7BBFB284C), + UINT64_C(0xA3136F35DD8ECA28), UINT64_C(0xBEFDE56BFFA26BF4), + UINT64_C(0x4C300AC98DC40345), UINT64_C(0x51DE8097AFE8A299), + UINT64_C(0x77ED1E75C99D40FD), UINT64_C(0x6A03942BEBB1E121), + UINT64_C(0x3B8A23B105768435), UINT64_C(0x2664A9EF275A25E9), + UINT64_C(0x0057370D412FC78D), UINT64_C(0x1DB9BD5363036651), + UINT64_C(0xA34458389CA10DA5), UINT64_C(0xBEAAD266BE8DAC79), + UINT64_C(0x98994C84D8F84E1D), UINT64_C(0x8577C6DAFAD4EFC1), + UINT64_C(0xD4FE714014138AD5), UINT64_C(0xC910FB1E363F2B09), + UINT64_C(0xEF2365FC504AC96D), UINT64_C(0xF2CDEFA2726668B1), + UINT64_C(0x986015931B88068A), UINT64_C(0x858E9FCD39A4A756), + UINT64_C(0xA3BD012F5FD14532), UINT64_C(0xBE538B717DFDE4EE), + UINT64_C(0xEFDA3CEB933A81FA), UINT64_C(0xF234B6B5B1162026), + UINT64_C(0xD4072857D763C242), UINT64_C(0xC9E9A209F54F639E), + UINT64_C(0x771447620AED086A), UINT64_C(0x6AFACD3C28C1A9B6), + UINT64_C(0x4CC953DE4EB44BD2), UINT64_C(0x5127D9806C98EA0E), + UINT64_C(0x00AE6E1A825F8F1A), UINT64_C(0x1D40E444A0732EC6), + UINT64_C(0x3B737AA6C606CCA2), UINT64_C(0x269DF0F8E42A6D7E), + UINT64_C(0xD4501F5A964C05CF), UINT64_C(0xC9BE9504B460A413), + UINT64_C(0xEF8D0BE6D2154677), UINT64_C(0xF26381B8F039E7AB), + UINT64_C(0xA3EA36221EFE82BF), UINT64_C(0xBE04BC7C3CD22363), + UINT64_C(0x9837229E5AA7C107), UINT64_C(0x85D9A8C0788B60DB), + UINT64_C(0x3B244DAB87290B2F), UINT64_C(0x26CAC7F5A505AAF3), + UINT64_C(0x00F95917C3704897), UINT64_C(0x1D17D349E15CE94B), + UINT64_C(0x4C9E64D30F9B8C5F), UINT64_C(0x5170EE8D2DB72D83), + UINT64_C(0x7743706F4BC2CFE7), UINT64_C(0x6AADFA3169EE6E3B), + UINT64_C(0xA218840D981E1391), UINT64_C(0xBFF60E53BA32B24D), + UINT64_C(0x99C590B1DC475029), UINT64_C(0x842B1AEFFE6BF1F5), + UINT64_C(0xD5A2AD7510AC94E1), UINT64_C(0xC84C272B3280353D), + UINT64_C(0xEE7FB9C954F5D759), UINT64_C(0xF391339776D97685), + UINT64_C(0x4D6CD6FC897B1D71), UINT64_C(0x50825CA2AB57BCAD), + UINT64_C(0x76B1C240CD225EC9), UINT64_C(0x6B5F481EEF0EFF15), + UINT64_C(0x3AD6FF8401C99A01), UINT64_C(0x273875DA23E53BDD), + UINT64_C(0x010BEB384590D9B9), UINT64_C(0x1CE5616667BC7865), + UINT64_C(0xEE288EC415DA10D4), UINT64_C(0xF3C6049A37F6B108), + UINT64_C(0xD5F59A785183536C), UINT64_C(0xC81B102673AFF2B0), + UINT64_C(0x9992A7BC9D6897A4), UINT64_C(0x847C2DE2BF443678), + UINT64_C(0xA24FB300D931D41C), UINT64_C(0xBFA1395EFB1D75C0), + UINT64_C(0x015CDC3504BF1E34), UINT64_C(0x1CB2566B2693BFE8), + UINT64_C(0x3A81C88940E65D8C), UINT64_C(0x276F42D762CAFC50), + UINT64_C(0x76E6F54D8C0D9944), UINT64_C(0x6B087F13AE213898), + UINT64_C(0x4D3BE1F1C854DAFC), UINT64_C(0x50D56BAFEA787B20), + UINT64_C(0x3A78919E8396151B), UINT64_C(0x27961BC0A1BAB4C7), + UINT64_C(0x01A58522C7CF56A3), UINT64_C(0x1C4B0F7CE5E3F77F), + UINT64_C(0x4DC2B8E60B24926B), UINT64_C(0x502C32B8290833B7), + UINT64_C(0x761FAC5A4F7DD1D3), UINT64_C(0x6BF126046D51700F), + UINT64_C(0xD50CC36F92F31BFB), UINT64_C(0xC8E24931B0DFBA27), + UINT64_C(0xEED1D7D3D6AA5843), UINT64_C(0xF33F5D8DF486F99F), + UINT64_C(0xA2B6EA171A419C8B), UINT64_C(0xBF586049386D3D57), + UINT64_C(0x996BFEAB5E18DF33), UINT64_C(0x848574F57C347EEF), + UINT64_C(0x76489B570E52165E), UINT64_C(0x6BA611092C7EB782), + UINT64_C(0x4D958FEB4A0B55E6), UINT64_C(0x507B05B56827F43A), + UINT64_C(0x01F2B22F86E0912E), UINT64_C(0x1C1C3871A4CC30F2), + UINT64_C(0x3A2FA693C2B9D296), UINT64_C(0x27C12CCDE095734A), + UINT64_C(0x993CC9A61F3718BE), UINT64_C(0x84D243F83D1BB962), + UINT64_C(0xA2E1DD1A5B6E5B06), UINT64_C(0xBF0F57447942FADA), + UINT64_C(0xEE86E0DE97859FCE), UINT64_C(0xF3686A80B5A93E12), + UINT64_C(0xD55BF462D3DCDC76), UINT64_C(0xC8B57E3CF1F07DAA), + UINT64_C(0xD6E9A7309F3239A7), UINT64_C(0xCB072D6EBD1E987B), + UINT64_C(0xED34B38CDB6B7A1F), UINT64_C(0xF0DA39D2F947DBC3), + UINT64_C(0xA1538E481780BED7), UINT64_C(0xBCBD041635AC1F0B), + UINT64_C(0x9A8E9AF453D9FD6F), UINT64_C(0x876010AA71F55CB3), + UINT64_C(0x399DF5C18E573747), UINT64_C(0x24737F9FAC7B969B), + UINT64_C(0x0240E17DCA0E74FF), UINT64_C(0x1FAE6B23E822D523), + UINT64_C(0x4E27DCB906E5B037), UINT64_C(0x53C956E724C911EB), + UINT64_C(0x75FAC80542BCF38F), UINT64_C(0x6814425B60905253), + UINT64_C(0x9AD9ADF912F63AE2), UINT64_C(0x873727A730DA9B3E), + UINT64_C(0xA104B94556AF795A), UINT64_C(0xBCEA331B7483D886), + UINT64_C(0xED6384819A44BD92), UINT64_C(0xF08D0EDFB8681C4E), + UINT64_C(0xD6BE903DDE1DFE2A), UINT64_C(0xCB501A63FC315FF6), + UINT64_C(0x75ADFF0803933402), UINT64_C(0x6843755621BF95DE), + UINT64_C(0x4E70EBB447CA77BA), UINT64_C(0x539E61EA65E6D666), + UINT64_C(0x0217D6708B21B372), UINT64_C(0x1FF95C2EA90D12AE), + UINT64_C(0x39CAC2CCCF78F0CA), UINT64_C(0x24244892ED545116), + UINT64_C(0x4E89B2A384BA3F2D), UINT64_C(0x536738FDA6969EF1), + UINT64_C(0x7554A61FC0E37C95), UINT64_C(0x68BA2C41E2CFDD49), + UINT64_C(0x39339BDB0C08B85D), UINT64_C(0x24DD11852E241981), + UINT64_C(0x02EE8F674851FBE5), UINT64_C(0x1F0005396A7D5A39), + UINT64_C(0xA1FDE05295DF31CD), UINT64_C(0xBC136A0CB7F39011), + UINT64_C(0x9A20F4EED1867275), UINT64_C(0x87CE7EB0F3AAD3A9), + UINT64_C(0xD647C92A1D6DB6BD), UINT64_C(0xCBA943743F411761), + UINT64_C(0xED9ADD965934F505), UINT64_C(0xF07457C87B1854D9), + UINT64_C(0x02B9B86A097E3C68), UINT64_C(0x1F5732342B529DB4), + UINT64_C(0x3964ACD64D277FD0), UINT64_C(0x248A26886F0BDE0C), + UINT64_C(0x7503911281CCBB18), UINT64_C(0x68ED1B4CA3E01AC4), + UINT64_C(0x4EDE85AEC595F8A0), UINT64_C(0x53300FF0E7B9597C), + UINT64_C(0xEDCDEA9B181B3288), UINT64_C(0xF02360C53A379354), + UINT64_C(0xD610FE275C427130), UINT64_C(0xCBFE74797E6ED0EC), + UINT64_C(0x9A77C3E390A9B5F8), UINT64_C(0x879949BDB2851424), + UINT64_C(0xA1AAD75FD4F0F640), UINT64_C(0xBC445D01F6DC579C), + UINT64_C(0x74F1233D072C2A36), UINT64_C(0x691FA96325008BEA), + UINT64_C(0x4F2C37814375698E), UINT64_C(0x52C2BDDF6159C852), + UINT64_C(0x034B0A458F9EAD46), UINT64_C(0x1EA5801BADB20C9A), + UINT64_C(0x38961EF9CBC7EEFE), UINT64_C(0x257894A7E9EB4F22), + UINT64_C(0x9B8571CC164924D6), UINT64_C(0x866BFB923465850A), + UINT64_C(0xA05865705210676E), UINT64_C(0xBDB6EF2E703CC6B2), + UINT64_C(0xEC3F58B49EFBA3A6), UINT64_C(0xF1D1D2EABCD7027A), + UINT64_C(0xD7E24C08DAA2E01E), UINT64_C(0xCA0CC656F88E41C2), + UINT64_C(0x38C129F48AE82973), UINT64_C(0x252FA3AAA8C488AF), + UINT64_C(0x031C3D48CEB16ACB), UINT64_C(0x1EF2B716EC9DCB17), + UINT64_C(0x4F7B008C025AAE03), UINT64_C(0x52958AD220760FDF), + UINT64_C(0x74A614304603EDBB), UINT64_C(0x69489E6E642F4C67), + UINT64_C(0xD7B57B059B8D2793), UINT64_C(0xCA5BF15BB9A1864F), + UINT64_C(0xEC686FB9DFD4642B), UINT64_C(0xF186E5E7FDF8C5F7), + UINT64_C(0xA00F527D133FA0E3), UINT64_C(0xBDE1D8233113013F), + UINT64_C(0x9BD246C15766E35B), UINT64_C(0x863CCC9F754A4287), + UINT64_C(0xEC9136AE1CA42CBC), UINT64_C(0xF17FBCF03E888D60), + UINT64_C(0xD74C221258FD6F04), UINT64_C(0xCAA2A84C7AD1CED8), + UINT64_C(0x9B2B1FD69416ABCC), UINT64_C(0x86C59588B63A0A10), + UINT64_C(0xA0F60B6AD04FE874), UINT64_C(0xBD188134F26349A8), + UINT64_C(0x03E5645F0DC1225C), UINT64_C(0x1E0BEE012FED8380), + UINT64_C(0x383870E3499861E4), UINT64_C(0x25D6FABD6BB4C038), + UINT64_C(0x745F4D278573A52C), UINT64_C(0x69B1C779A75F04F0), + UINT64_C(0x4F82599BC12AE694), UINT64_C(0x526CD3C5E3064748), + UINT64_C(0xA0A13C6791602FF9), UINT64_C(0xBD4FB639B34C8E25), + UINT64_C(0x9B7C28DBD5396C41), UINT64_C(0x8692A285F715CD9D), + UINT64_C(0xD71B151F19D2A889), UINT64_C(0xCAF59F413BFE0955), + UINT64_C(0xECC601A35D8BEB31), UINT64_C(0xF1288BFD7FA74AED), + UINT64_C(0x4FD56E9680052119), UINT64_C(0x523BE4C8A22980C5), + UINT64_C(0x74087A2AC45C62A1), UINT64_C(0x69E6F074E670C37D), + UINT64_C(0x386F47EE08B7A669), UINT64_C(0x2581CDB02A9B07B5), + UINT64_C(0x03B253524CEEE5D1), UINT64_C(0x1E5CD90C6EC2440D) + } +}; diff --git a/src/liblzma/check/crc64_tablegen.c b/src/liblzma/check/crc64_tablegen.c new file mode 100644 index 00000000..0f1005f4 --- /dev/null +++ b/src/liblzma/check/crc64_tablegen.c @@ -0,0 +1,56 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc64_tablegen.c +/// \brief Generates CRC64 crc64_table.c +/// +/// Compiling: gcc -std=c99 -o crc64_tablegen crc64_tablegen.c crc64_init.c +/// Add -DWORDS_BIGENDIAN to generate big endian table. +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include <sys/types.h> +#include <inttypes.h> +#include <stdio.h> + + +extern void lzma_crc64_init(void); + +extern uint64_t lzma_crc64_table[4][256]; + + +int +main() +{ + lzma_crc64_init(); + + printf("/* This file has been automatically generated by " + "crc64_tablegen.c. */\n\n" + "#include <inttypes.h>\n\n" + "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); + + for (size_t s = 0; s < 4; ++s) { + for (size_t b = 0; b < 256; ++b) { + if ((b % 2) == 0) + printf("\n\t\t"); + + printf("UINT64_C(0x%016" PRIX64 ")", + lzma_crc64_table[s][b]); + + if (b != 255) + printf(", "); + } + + if (s == 3) + printf("\n\t}\n};\n"); + else + printf("\n\t}, {"); + } + + return 0; +} diff --git a/src/liblzma/check/crc64_x86.s b/src/liblzma/check/crc64_x86.s new file mode 100644 index 00000000..2b8d349e --- /dev/null +++ b/src/liblzma/check/crc64_x86.s @@ -0,0 +1,203 @@ +/* + * Speed-optimized CRC64 using slicing-by-four algorithm + * Instruction set: i386 + * Optimized for: i686 + * + * This code has been put into the public domain by its authors: + * Igor Pavlov <http://7-zip.org/> + * Lasse Collin <lasse.collin@tukaani.org> + * + * This code needs lzma_crc64_table, which can be created using the + * following C code: + +uint64_t lzma_crc64_table[4][256]; + +void +init_table(void) +{ + static const uint64_t poly64 = UINT64_C(0xC96C5795D7870F42); + + for (size_t s = 0; s < 4; ++s) { + for (size_t b = 0; b < 256; ++b) { + uint64_t r = s == 0 ? b : lzma_crc64_table[s - 1][b]; + + for (size_t i = 0; i < 8; ++i) { + if (r & 1) + r = (r >> 1) ^ poly64; + else + r >>= 1; + } + + lzma_crc64_table[s][b] = r; + } + } +} + + * The prototype of the CRC64 function: + * extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); + */ + + .text + .global lzma_crc64 + .type lzma_crc64, @function + + .align 16 +lzma_crc64: + /* + * Register usage: + * %eax crc LSB + * %edx crc MSB + * %esi buf + * %edi size or buf + size + * %ebx lzma_crc64_table + * %ebp Table index + * %ecx Temporary + */ + pushl %ebx + pushl %esi + pushl %edi + pushl %ebp + movl 0x14(%esp), %esi /* buf */ + movl 0x18(%esp), %edi /* size */ + movl 0x1C(%esp), %eax /* crc LSB */ + movl 0x20(%esp), %edx /* crc MSB */ + + /* + * Store the address of lzma_crc64_table to %ebx. This is needed to + * get position-independent code (PIC). + */ + call .L_PIC +.L_PIC: + popl %ebx + addl $_GLOBAL_OFFSET_TABLE_+[.-.L_PIC], %ebx + movl lzma_crc64_table@GOT(%ebx), %ebx + + /* Complement the initial value. */ + notl %eax + notl %edx + +.L_align: + /* + * Check if there is enough input to use slicing-by-four. + * We need eight bytes, because the loop pre-reads four bytes. + */ + cmpl $8, %edi + jl .L_rest + + /* Check if we have reached alignment of four bytes. */ + testl $3, %esi + jz .L_slice + + /* Calculate CRC of the next input byte. */ + movzbl (%esi), %ebp + incl %esi + movzbl %al, %ecx + xorl %ecx, %ebp + shrdl $8, %edx, %eax + xorl (%ebx, %ebp, 8), %eax + shrl $8, %edx + xorl 4(%ebx, %ebp, 8), %edx + decl %edi + jmp .L_align + +.L_slice: + /* + * If we get here, there's at least eight bytes of aligned input + * available. Make %edi multiple of four bytes. Store the possible + * remainder over the "size" variable in the argument stack. + */ + movl %edi, 0x18(%esp) + andl $-4, %edi + subl %edi, 0x18(%esp) + + /* + * Let %edi be buf + size - 4 while running the main loop. This way + * we can compare for equality to determine when exit the loop. + */ + addl %esi, %edi + subl $4, %edi + + /* Read in the first four aligned bytes. */ + movl (%esi), %ecx + +.L_loop: + xorl %eax, %ecx + movzbl %cl, %ebp + movl 0x1800(%ebx, %ebp, 8), %eax + xorl %edx, %eax + movl 0x1804(%ebx, %ebp, 8), %edx + movzbl %ch, %ebp + xorl 0x1000(%ebx, %ebp, 8), %eax + xorl 0x1004(%ebx, %ebp, 8), %edx + shrl $16, %ecx + movzbl %cl, %ebp + xorl 0x0800(%ebx, %ebp, 8), %eax + xorl 0x0804(%ebx, %ebp, 8), %edx + movzbl %ch, %ebp + addl $4, %esi + xorl (%ebx, %ebp, 8), %eax + xorl 4(%ebx, %ebp, 8), %edx + + /* Check for end of aligned input. */ + cmpl %edi, %esi + + /* + * Copy the next input byte to %ecx. It is slightly faster to + * read it here than at the top of the loop. + */ + movl (%esi), %ecx + jl .L_loop + + /* + * Process the remaining four bytes, which we have already + * copied to %ecx. + */ + xorl %eax, %ecx + movzbl %cl, %ebp + movl 0x1800(%ebx, %ebp, 8), %eax + xorl %edx, %eax + movl 0x1804(%ebx, %ebp, 8), %edx + movzbl %ch, %ebp + xorl 0x1000(%ebx, %ebp, 8), %eax + xorl 0x1004(%ebx, %ebp, 8), %edx + shrl $16, %ecx + movzbl %cl, %ebp + xorl 0x0800(%ebx, %ebp, 8), %eax + xorl 0x0804(%ebx, %ebp, 8), %edx + movzbl %ch, %ebp + addl $4, %esi + xorl (%ebx, %ebp, 8), %eax + xorl 4(%ebx, %ebp, 8), %edx + + /* Copy the number of remaining bytes to %edi. */ + movl 0x18(%esp), %edi + +.L_rest: + /* Check for end of input. */ + testl %edi, %edi + jz .L_return + + /* Calculate CRC of the next input byte. */ + movzbl (%esi), %ebp + incl %esi + movzbl %al, %ecx + xorl %ecx, %ebp + shrdl $8, %edx, %eax + xorl (%ebx, %ebp, 8), %eax + shrl $8, %edx + xorl 4(%ebx, %ebp, 8), %edx + decl %edi + jmp .L_rest + +.L_return: + /* Complement the final value. */ + notl %eax + notl %edx + + popl %ebp + popl %edi + popl %esi + popl %ebx + ret + + .size lzma_crc32, .-lzma_crc32 diff --git a/src/liblzma/check/crc_macros.h b/src/liblzma/check/crc_macros.h new file mode 100644 index 00000000..5fbecf07 --- /dev/null +++ b/src/liblzma/check/crc_macros.h @@ -0,0 +1,33 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_macros +/// \brief Some endian-dependent macros for CRC32 and CRC64 +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifdef WORDS_BIGENDIAN +# include "check_byteswap.h" + +# define A(x) ((x) >> 24) +# define B(x) (((x) >> 16) & 0xFF) +# define C(x) (((x) >> 8) & 0xFF) +# define D(x) ((x) & 0xFF) + +# define S8(x) ((x) << 8) +# define S32(x) ((x) << 32) + +#else +# define A(x) ((x) & 0xFF) +# define B(x) (((x) >> 8) & 0xFF) +# define C(x) (((x) >> 16) & 0xFF) +# define D(x) ((x) >> 24) + +# define S8(x) ((x) >> 8) +# define S32(x) ((x) >> 32) +#endif diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c new file mode 100644 index 00000000..8e3d375a --- /dev/null +++ b/src/liblzma/check/sha256.c @@ -0,0 +1,203 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sha256.c +/// \brief SHA256 +// +// Based on the public domain code found from Wei Dai's Crypto++ library +// version 5.5.1: http://www.cryptopp.com/ +// This code has been put into the public domain. +// +/// \todo Crypto++ has x86 ASM optimizations. They use SSE so if they +/// are imported to liblzma, SSE instructions need to be used +/// conditionally to keep the code working on older boxes. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "check.h" + +#ifndef WORDS_BIGENDIAN +# include "check_byteswap.h" +#endif + +// At least on x86, GCC is able to optimize this to a rotate instruction. +#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount))) + +#define blk0(i) (W[i] = data[i]) +#define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + + s0(W[(i - 15) & 15])) + +#define Ch(x, y, z) (z ^ (x & (y ^ z))) +#define Maj(x, y, z) ((x & y) | (z & (x | y))) + +#define a(i) T[(0 - i) & 7] +#define b(i) T[(1 - i) & 7] +#define c(i) T[(2 - i) & 7] +#define d(i) T[(3 - i) & 7] +#define e(i) T[(4 - i) & 7] +#define f(i) T[(5 - i) & 7] +#define g(i) T[(6 - i) & 7] +#define h(i) T[(7 - i) & 7] + +#define R(i) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \ + + (j ? blk2(i) : blk0(i)); \ + d(i) += h(i); \ + h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) + +#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22)) +#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25)) +#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3)) +#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10)) + + +static const uint32_t SHA256_K[64] = { + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, + 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, + 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, + 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, + 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, + 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, + 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, + 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, + 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, +}; + + +static void +transform(uint32_t state[static 8], const uint32_t data[static 16]) +{ + uint32_t W[16]; + uint32_t T[8]; + + // Copy state[] to working vars. + memcpy(T, state, sizeof(T)); + + // 64 operations, partially loop unrolled + for (unsigned int j = 0; j < 64; j += 16) { + R( 0); R( 1); R( 2); R( 3); + R( 4); R( 5); R( 6); R( 7); + R( 8); R( 9); R(10); R(11); + R(12); R(13); R(14); R(15); + } + + // Add the working vars back into state[]. + state[0] += a(0); + state[1] += b(0); + state[2] += c(0); + state[3] += d(0); + state[4] += e(0); + state[5] += f(0); + state[6] += g(0); + state[7] += h(0); +} + + +static void +process(lzma_sha256 *sha256) +{ +#ifdef WORDS_BIGENDIAN + transform(sha256->state, (uint32_t *)(sha256->buffer)); + +#else + uint32_t data[16]; + + for (size_t i = 0; i < 16; ++i) + data[i] = bswap_32(*((uint32_t*)(sha256->buffer) + i)); + + transform(sha256->state, data); +#endif + + return; +} + + +extern void +lzma_sha256_init(lzma_sha256 *sha256) +{ + static const uint32_t s[8] = { + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, + }; + + memcpy(sha256->state, s, sizeof(s)); + sha256->size = 0; + + return; +} + + +extern void +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_sha256 *sha256) +{ + // Copy the input data into a properly aligned temporary buffer. + // This way we can be called with arbitrarily sized buffers + // (no need to be multiple of 64 bytes), and the code works also + // on architectures that don't allow unaligned memory access. + while (size > 0) { + const size_t copy_start = sha256->size & 0x3F; + size_t copy_size = 64 - copy_start; + if (copy_size > size) + copy_size = size; + + memcpy(sha256->buffer + copy_start, buf, copy_size); + + buf += copy_size; + size -= copy_size; + sha256->size += copy_size; + + if ((sha256->size & 0x3F) == 0) + process(sha256); + } + + return; +} + + +extern void +lzma_sha256_finish(lzma_sha256 *sha256) +{ + // Add padding as described in RFC 3174 (it describes SHA-1 but + // the same padding style is used for SHA-256 too). + size_t pos = sha256->size & 0x3F; + sha256->buffer[pos++] = 0x80; + + while (pos != 64 - 8) { + if (pos == 64) { + process(sha256); + pos = 0; + } + + sha256->buffer[pos++] = 0x00; + } + + // Convert the message size from bytes to bits. + sha256->size *= 8; + +#ifdef WORDS_BIGENDIAN + *(uint64_t *)(sha256->buffer + 64 - 8) = sha256->size; +#else + *(uint64_t *)(sha256->buffer + 64 - 8) = bswap_64(sha256->size); +#endif + + process(sha256); + + for (size_t i = 0; i < 8; ++i) +#ifdef WORDS_BIGENDIAN + ((uint32_t *)(sha256->buffer))[i] = sha256->state[i]; +#else + ((uint32_t *)(sha256->buffer))[i] = bswap_32(sha256->state[i]); +#endif + + return; +} diff --git a/src/liblzma/common/Makefile.am b/src/liblzma/common/Makefile.am new file mode 100644 index 00000000..4eb9d54e --- /dev/null +++ b/src/liblzma/common/Makefile.am @@ -0,0 +1,94 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = libcommon.la +libcommon_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/check \ + -I@top_srcdir@/src/liblzma/lz \ + -I@top_srcdir@/src/liblzma/lzma \ + -I@top_srcdir@/src/liblzma/simple \ + -I@top_srcdir@/src/liblzma/subblock \ + -I@top_srcdir@/src/liblzma/rangecoder +libcommon_la_SOURCES = \ + common.h \ + sysdefs.h \ + allocator.c \ + block_private.h \ + extra.c \ + features.c \ + index.c \ + info.c \ + init.c \ + memory_limitter.c \ + memory_usage.c \ + next_coder.c \ + raw_common.c \ + raw_common.h \ + code.c \ + version.c + +if COND_FILTER_COPY +libcommon_la_SOURCES += \ + copy_coder.c \ + copy_coder.h +endif + +if COND_FILTER_DELTA +libcommon_la_SOURCES += \ + delta_coder.c \ + delta_coder.h +endif + +if COND_MAIN_ENCODER +libcommon_la_SOURCES += \ + alignment.c \ + auto_decoder.c \ + alone_encoder.c \ + block_encoder.c \ + block_encoder.h \ + block_header_encoder.c \ + filter_flags_encoder.c \ + init_encoder.c \ + metadata_encoder.c \ + metadata_encoder.h \ + raw_encoder.c \ + raw_encoder.h \ + stream_common.c \ + stream_common.h \ + stream_encoder_single.c \ + stream_encoder_multi.c \ + stream_flags_encoder.c \ + vli_encoder.c +endif + +if COND_MAIN_DECODER +libcommon_la_SOURCES += \ + alone_decoder.c \ + alone_decoder.h \ + block_decoder.c \ + block_decoder.h \ + block_header_decoder.c \ + filter_flags_decoder.c \ + init_decoder.c \ + metadata_decoder.c \ + metadata_decoder.h \ + raw_decoder.c \ + raw_decoder.h \ + stream_decoder.c \ + stream_flags_decoder.c \ + stream_flags_decoder.h \ + vli_decoder.c \ + vli_reverse_decoder.c +endif diff --git a/src/liblzma/common/alignment.c b/src/liblzma/common/alignment.c new file mode 100644 index 00000000..2d468fe5 --- /dev/null +++ b/src/liblzma/common/alignment.c @@ -0,0 +1,118 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alignment.c +/// \brief Calculates preferred alignments of different filters +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API uint32_t +lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess) +{ + for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { + switch (filters[i].id) { + case LZMA_FILTER_COPY: + case LZMA_FILTER_DELTA: + // The same as the input, check the next filter. + continue; + + case LZMA_FILTER_SUBBLOCK: + if (filters[i].options == NULL) + return LZMA_SUBBLOCK_ALIGNMENT_DEFAULT; + else + return ((const lzma_options_subblock *)( + filters[i].options))->alignment; + + case LZMA_FILTER_X86: + return 1; + + case LZMA_FILTER_ARMTHUMB: + return 2; + + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_ARM: + case LZMA_FILTER_SPARC: + return 4; + + case LZMA_FILTER_IA64: + return 16; + + case LZMA_FILTER_LZMA: { + const lzma_options_lzma *lzma = filters[i].options; + return 1 << MAX(lzma->pos_bits, + lzma->literal_pos_bits); + } + + default: + return UINT32_MAX; + } + } + + return guess; +} + + +extern LZMA_API uint32_t +lzma_alignment_output(const lzma_options_filter *filters, uint32_t guess) +{ + // Check if there is only an implicit Copy filter. + if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN) + return guess; + + // Find the last filter in the chain. + size_t i = 0; + while (filters[i + 1].id != LZMA_VLI_VALUE_UNKNOWN) + ++i; + + do { + switch (filters[i].id) { + case LZMA_FILTER_COPY: + case LZMA_FILTER_DELTA: + // It's the same as the input alignment, so + // check the next filter. + continue; + + case LZMA_FILTER_SUBBLOCK: + if (filters[i].options == NULL) + return LZMA_SUBBLOCK_ALIGNMENT_DEFAULT; + else + return ((const lzma_options_subblock *)( + filters[i].options))->alignment; + + case LZMA_FILTER_X86: + case LZMA_FILTER_LZMA: + return 1; + + case LZMA_FILTER_ARMTHUMB: + return 2; + + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_ARM: + case LZMA_FILTER_SPARC: + return 4; + + case LZMA_FILTER_IA64: + return 16; + + default: + return UINT32_MAX; + } + } while (i-- != 0); + + // If we get here, we have the same alignment as the input data. + return guess; +} diff --git a/src/liblzma/common/allocator.c b/src/liblzma/common/allocator.c new file mode 100644 index 00000000..edea0f68 --- /dev/null +++ b/src/liblzma/common/allocator.c @@ -0,0 +1,57 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file allocator.c +/// \brief Allocating and freeing memory +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + +#undef lzma_free + +extern void * lzma_attribute((malloc)) +lzma_alloc(size_t size, lzma_allocator *allocator) +{ + // Some malloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) + ptr = allocator->alloc(allocator->opaque, 1, size); + else + ptr = malloc(size); + +#if !defined(NDEBUG) && defined(HAVE_MEMSET) + // This helps to catch some stupid mistakes. + if (ptr != NULL) + memset(ptr, 0xFD, size); +#endif + + return ptr; +} + + +extern void +lzma_free(void *ptr, lzma_allocator *allocator) +{ + if (allocator != NULL && allocator->free != NULL) + allocator->free(allocator->opaque, ptr); + else + free(ptr); + + return; +} diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c new file mode 100644 index 00000000..092047b4 --- /dev/null +++ b/src/liblzma/common/alone_decoder.c @@ -0,0 +1,197 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.c +/// \brief Decoder for LZMA_Alone files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "alone_decoder.h" +#include "lzma_decoder.h" + + +struct lzma_coder_s { + lzma_next_coder next; + + enum { + SEQ_PROPERTIES, + SEQ_DICTIONARY_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_CODER_INIT, + SEQ_CODE, + } sequence; + + size_t pos; + + lzma_options_alone options; +}; + + +static lzma_ret +alone_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action) +{ + while (*out_pos < out_size + && (coder->sequence == SEQ_CODE || *in_pos < in_size)) + switch (coder->sequence) { + case SEQ_PROPERTIES: + if (lzma_lzma_decode_properties( + &coder->options.lzma, in[*in_pos])) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_DICTIONARY_SIZE; + ++*in_pos; + break; + + case SEQ_DICTIONARY_SIZE: + coder->options.lzma.dictionary_size + |= (size_t)(in[*in_pos]) << (coder->pos * 8); + + if (++coder->pos == 4) { + // A hack to ditch tons of false positives: We allow + // only dictionary sizes that are a power of two. + // LZMA_Alone didn't create other kinds of files, + // although it's not impossible that files with + // other dictionary sizes exist. Well, if someone + // complains, this will be reconsidered. + size_t count = 0; + for (size_t i = 0; i < 32; ++i) + if (coder->options.lzma.dictionary_size + & (UINT32_C(1) << i)) + ++count; + + if (count > 1) + return LZMA_DATA_ERROR; + + coder->pos = 0; + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + } + + ++*in_pos; + break; + + case SEQ_UNCOMPRESSED_SIZE: + coder->options.uncompressed_size + |= (lzma_vli)(in[*in_pos]) << (coder->pos * 8); + + if (++coder->pos == 8) { + // Another hack to ditch false positives: Assume that + // if the uncompressed size is known, it must be less + // than 256 GiB. Again, if someone complains, this + // will be reconsidered. + if (coder->options.uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->options.uncompressed_size + >= (LZMA_VLI_C(1) << 38)) + return LZMA_DATA_ERROR; + + coder->pos = 0; + coder->sequence = SEQ_CODER_INIT; + } + + ++*in_pos; + break; + + case SEQ_CODER_INIT: { + // Two is enough because there won't be implicit filters. + lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options.lzma, + .uncompressed_size = coder->options + .uncompressed_size, + }, { + .init = NULL, + } + }; + + const lzma_ret ret = lzma_next_filter_init(&coder->next, + allocator, filters); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_CODE; + } + + // Fall through + + case SEQ_CODE: { + return coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &alone_decode; + next->end = &alone_decoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + next->coder->sequence = SEQ_PROPERTIES; + next->coder->pos = 0; + next->coder->options.lzma.dictionary_size = 0; + next->coder->options.uncompressed_size = 0; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +{ + // We need to use _init2 because we don't pass any varadic args. + lzma_next_coder_init2(next, allocator, alone_decoder_init, + alone_decoder_init, allocator); +} + + +extern LZMA_API lzma_ret +lzma_alone_decoder(lzma_stream *strm) +{ + lzma_next_strm_init2(strm, alone_decoder_init, + alone_decoder_init, strm->allocator); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/alone_decoder.h b/src/liblzma/common/alone_decoder.h new file mode 100644 index 00000000..a9b7e84b --- /dev/null +++ b/src/liblzma/common/alone_decoder.h @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.h +/// \brief Decoder for LZMA_Alone files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern lzma_ret lzma_alone_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator); diff --git a/src/liblzma/common/alone_encoder.c b/src/liblzma/common/alone_encoder.c new file mode 100644 index 00000000..7629aa77 --- /dev/null +++ b/src/liblzma/common/alone_encoder.c @@ -0,0 +1,167 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.c +/// \brief Decoder for LZMA_Alone files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "lzma_encoder.h" + + +struct lzma_coder_s { + lzma_next_coder next; + + enum { + SEQ_PROPERTIES, + SEQ_DICTIONARY_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_CODE, + } sequence; + + size_t pos; + + lzma_options_alone options; +}; + + +static lzma_ret +alone_encode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action) +{ + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_PROPERTIES: + if (lzma_lzma_encode_properties( + &coder->options.lzma, out + *out_pos)) { + return LZMA_PROG_ERROR; + } + + coder->sequence = SEQ_DICTIONARY_SIZE; + ++*out_pos; + break; + + case SEQ_DICTIONARY_SIZE: + out[*out_pos] = coder->options.lzma.dictionary_size + >> (coder->pos * 8); + + if (++coder->pos == 4) { + coder->pos = 0; + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + } + + ++*out_pos; + break; + + case SEQ_UNCOMPRESSED_SIZE: + out[*out_pos] = coder->options.uncompressed_size + >> (coder->pos * 8); + + if (++coder->pos == 8) { + coder->pos = 0; + coder->sequence = SEQ_CODE; + } + + ++*out_pos; + break; + + case SEQ_CODE: { + return coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +// At least for now, this is not used by any internal function. +static lzma_ret +alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_alone *options) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &alone_encode; + next->end = &alone_encoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Initialize the LZMA_Alone coder variables. + next->coder->sequence = SEQ_PROPERTIES; + next->coder->pos = 0; + next->coder->options = *options; + + // Verify uncompressed_size since the other functions assume that + // it is valid. + if (!lzma_vli_is_valid(next->coder->options.uncompressed_size)) + return LZMA_PROG_ERROR; + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = &next->coder->options.lzma, + .uncompressed_size = next->coder->options + .uncompressed_size, + }, { + .init = NULL, + } + }; + + return lzma_next_filter_init(&next->coder->next, allocator, filters); +} + + +/* +extern lzma_ret +lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_alone *options) +{ + lzma_next_coder_init(alone_encoder_init, next, allocator, options); +} +*/ + + +extern LZMA_API lzma_ret +lzma_alone_encoder(lzma_stream *strm, const lzma_options_alone *options) +{ + lzma_next_strm_init(strm, alone_encoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c new file mode 100644 index 00000000..7e92df9a --- /dev/null +++ b/src/liblzma/common/auto_decoder.c @@ -0,0 +1,113 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file auto_decoder.c +/// \brief Autodetect between .lzma Stream and LZMA_Alone formats +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "alone_decoder.h" + + +struct lzma_coder_s { + lzma_next_coder next; + + lzma_extra **header; + lzma_extra **footer; + bool initialized; +}; + + +static lzma_ret +auto_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + if (!coder->initialized) { + if (*in_pos >= in_size) + return LZMA_OK; + + lzma_ret ret; + + if (in[*in_pos] == 0xFF) + ret = lzma_stream_decoder_init(&coder->next, allocator, + coder->header, coder->footer); + else + ret = lzma_alone_decoder_init(&coder->next, allocator); + + if (ret != LZMA_OK) + return ret; + + coder->initialized = true; + } + + return coder->next.code(coder->next.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, action); +} + + +static void +auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_extra **header, lzma_extra **footer) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &auto_decode; + next->end = &auto_decoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + next->coder->header = header; + next->coder->footer = footer; + next->coder->initialized = false; + + return LZMA_OK; +} + + +/* +extern lzma_ret +lzma_auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_extra **header, lzma_extra **footer) +{ + lzma_next_coder_init( + auto_decoder_init, next, allocator, header, footer); +} +*/ + + +extern LZMA_API lzma_ret +lzma_auto_decoder(lzma_stream *strm, lzma_extra **header, lzma_extra **footer) +{ + lzma_next_strm_init(strm, auto_decoder_init, header, footer); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/block_decoder.c b/src/liblzma/common/block_decoder.c new file mode 100644 index 00000000..b9dcde49 --- /dev/null +++ b/src/liblzma/common/block_decoder.c @@ -0,0 +1,405 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_decoder.c +/// \brief Decodes .lzma Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_decoder.h" +#include "block_private.h" +#include "raw_decoder.h" +#include "check.h" + + +struct lzma_coder_s { + enum { + SEQ_CODE, + SEQ_CHECK, + SEQ_UNCOMPRESSED_SIZE, + SEQ_BACKWARD_SIZE, + SEQ_PADDING, + SEQ_END, + } sequence; + + /// The filters in the chain; initialized with lzma_raw_decoder_init(). + lzma_next_coder next; + + /// Decoding options; we also write Total Size, Compressed Size, and + /// Uncompressed Size back to this structure when the encoding has + /// been finished. + lzma_options_block *options; + + /// Position in variable-length integers (and in some other places). + size_t pos; + + /// Check of the uncompressed data + lzma_check check; + + /// Total Size calculated while encoding + lzma_vli total_size; + + /// Compressed Size calculated while encoding + lzma_vli compressed_size; + + /// Uncompressed Size calculated while encoding + lzma_vli uncompressed_size; + + /// Maximum allowed total_size + lzma_vli total_limit; + + /// Maximum allowed uncompressed_size + lzma_vli uncompressed_limit; + + /// Temporary location for the Uncompressed Size and Backward Size + /// fields in Block Footer. + lzma_vli tmp; + + /// Size of the Backward Size field - This is needed so that we + /// can verify the Backward Size and still keep updating total_size. + size_t size_of_backward_size; +}; + + +static lzma_ret +update_sequence(lzma_coder *coder) +{ + switch (coder->sequence) { + case SEQ_CODE: + if (coder->options->check != LZMA_CHECK_NONE) { + lzma_check_finish(&coder->check, + coder->options->check); + coder->sequence = SEQ_CHECK; + break; + } + + // Fall through + + case SEQ_CHECK: + if (coder->options->has_uncompressed_size_in_footer) { + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + break; + } + + // Fall through + + case SEQ_UNCOMPRESSED_SIZE: + if (coder->options->has_backward_size) { + coder->sequence = SEQ_BACKWARD_SIZE; + break; + } + + // Fall through + + case SEQ_BACKWARD_SIZE: + if (coder->options->handle_padding) { + coder->sequence = SEQ_PADDING; + break; + } + + case SEQ_PADDING: + if (!is_size_valid(coder->total_size, + coder->options->total_size) + || !is_size_valid(coder->compressed_size, + coder->options->compressed_size) + || !is_size_valid(coder->uncompressed_size, + coder->options->uncompressed_size)) + return LZMA_DATA_ERROR; + + // Copy the values into coder->options. The caller + // may use this information to construct Index. + coder->options->total_size = coder->total_size; + coder->options->compressed_size = coder->compressed_size; + coder->options->uncompressed_size = coder->uncompressed_size; + + return LZMA_STREAM_END; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +block_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Special case when the Block has only Block Header. + if (coder->sequence == SEQ_END) + return LZMA_STREAM_END; + + // FIXME: Termination condition should work but could be cleaner. + while (*out_pos < out_size && (*in_pos < in_size + || coder->sequence == SEQ_CODE)) + switch (coder->sequence) { + case SEQ_CODE: { + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + const size_t in_used = *in_pos - in_start; + const size_t out_used = *out_pos - out_start; + + if (update_size(&coder->total_size, in_used, + coder->total_limit) + || update_size(&coder->compressed_size, + in_used, + coder->options->compressed_size) + || update_size(&coder->uncompressed_size, + out_used, coder->uncompressed_limit)) + return LZMA_DATA_ERROR; + + lzma_check_update(&coder->check, coder->options->check, + out + out_start, out_used); + + if (ret != LZMA_STREAM_END) + return ret; + + ret = update_sequence(coder); + if (ret != LZMA_OK) + return ret; + + break; + } + + case SEQ_CHECK: + switch (coder->options->check) { + case LZMA_CHECK_CRC32: + if (((coder->check.crc32 >> (coder->pos * 8)) + & 0xFF) != in[*in_pos]) + return LZMA_DATA_ERROR; + break; + + case LZMA_CHECK_CRC64: + if (((coder->check.crc64 >> (coder->pos * 8)) + & 0xFF) != in[*in_pos]) + return LZMA_DATA_ERROR; + break; + + case LZMA_CHECK_SHA256: + if (coder->check.sha256.buffer[coder->pos] + != in[*in_pos]) + return LZMA_DATA_ERROR; + break; + + default: + assert(coder->options->check != LZMA_CHECK_NONE); + assert(coder->options->check <= LZMA_CHECK_ID_MAX); + break; + } + + if (update_size(&coder->total_size, 1, coder->total_limit)) + return LZMA_DATA_ERROR; + + ++*in_pos; + + if (++coder->pos == lzma_check_sizes[coder->options->check]) { + const lzma_ret ret = update_sequence(coder); + if (ret != LZMA_OK) + return ret; + + coder->pos = 0; + } + + break; + + case SEQ_UNCOMPRESSED_SIZE: { + const size_t in_start = *in_pos; + + lzma_ret ret = lzma_vli_decode(&coder->tmp, + &coder->pos, in, in_pos, in_size); + + if (update_size(&coder->total_size, *in_pos - in_start, + coder->total_limit)) + return LZMA_DATA_ERROR; + + if (ret != LZMA_STREAM_END) + return ret; + + if (coder->tmp != coder->uncompressed_size) + return LZMA_DATA_ERROR; + + coder->pos = 0; + coder->tmp = 0; + + ret = update_sequence(coder); + if (ret != LZMA_OK) + return ret; + + break; + } + + case SEQ_BACKWARD_SIZE: { + const size_t in_start = *in_pos; + + lzma_ret ret = lzma_vli_decode(&coder->tmp, + &coder->pos, in, in_pos, in_size); + + const size_t in_used = *in_pos - in_start; + + if (update_size(&coder->total_size, in_used, + coder->total_limit)) + return LZMA_DATA_ERROR; + + coder->size_of_backward_size += in_used; + + if (ret != LZMA_STREAM_END) + return ret; + + if (coder->tmp != coder->total_size + - coder->size_of_backward_size) + return LZMA_DATA_ERROR; + + ret = update_sequence(coder); + if (ret != LZMA_OK) + return ret; + + break; + } + + case SEQ_PADDING: + if (in[*in_pos] == 0x00) { + if (update_size(&coder->total_size, 1, + coder->total_limit)) + return LZMA_DATA_ERROR; + + ++*in_pos; + break; + } + + return update_sequence(coder); + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options) +{ + // This is pretty similar to lzma_block_encoder_init(). + // See comments there. + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &block_decode; + next->end = &block_decoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + if (!lzma_vli_is_valid(options->total_size) + || !lzma_vli_is_valid(options->compressed_size) + || !lzma_vli_is_valid(options->uncompressed_size) + || !lzma_vli_is_valid(options->total_size) + || !lzma_vli_is_valid(options->total_limit) + || !lzma_vli_is_valid(options->uncompressed_limit) + || (options->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && options->uncompressed_size + > options->uncompressed_limit) + || (options->total_size != LZMA_VLI_VALUE_UNKNOWN + && options->total_size + > options->total_limit) + || (!options->has_eopm && options->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN) + || options->header_size > options->total_size + || (options->handle_padding + && (options->has_uncompressed_size_in_footer + || options->has_backward_size))) + return LZMA_PROG_ERROR; + + { + const lzma_ret ret = lzma_check_init( + &next->coder->check, options->check); + if (ret != LZMA_OK) + return ret; + } + + if (!options->has_eopm && options->uncompressed_size == 0) { + if (!is_size_valid(0, options->compressed_size)) + return LZMA_PROG_ERROR; + + if (options->check != LZMA_CHECK_NONE) { + lzma_check_finish(&next->coder->check, options->check); + next->coder->sequence = SEQ_CHECK; + } else if (options->handle_padding) { + next->coder->sequence = SEQ_PADDING; + } else { + next->coder->sequence = SEQ_END; + } + } else { + next->coder->sequence = SEQ_CODE; + } + + { + const lzma_ret ret = lzma_raw_decoder_init( + &next->coder->next, allocator, + options->filters, options->has_eopm + ? LZMA_VLI_VALUE_UNKNOWN + : options->uncompressed_size, + true); + if (ret != LZMA_OK) + return ret; + } + + next->coder->options = options; + next->coder->pos = 0; + next->coder->total_size = options->header_size; + next->coder->compressed_size = 0; + next->coder->uncompressed_size = 0; + next->coder->total_limit + = MIN(options->total_size, options->total_limit); + next->coder->uncompressed_limit = MIN(options->uncompressed_size, + options->uncompressed_limit); + next->coder->tmp = 0; + next->coder->size_of_backward_size = 0; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_block_decoder(lzma_stream *strm, lzma_options_block *options) +{ + lzma_next_strm_init(strm, lzma_block_decoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/block_decoder.h b/src/liblzma/common/block_decoder.h new file mode 100644 index 00000000..af71128d --- /dev/null +++ b/src/liblzma/common/block_decoder.h @@ -0,0 +1,29 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_decoder.h +/// \brief Decodes .lzma Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_DECODER_H +#define LZMA_BLOCK_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_block *options); + +#endif diff --git a/src/liblzma/common/block_encoder.c b/src/liblzma/common/block_encoder.c new file mode 100644 index 00000000..77ff78ea --- /dev/null +++ b/src/liblzma/common/block_encoder.c @@ -0,0 +1,375 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_encoder.c +/// \brief Encodes .lzma Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_encoder.h" +#include "block_private.h" +#include "raw_encoder.h" +#include "check.h" + + +struct lzma_coder_s { + /// The filters in the chain; initialized with lzma_raw_decoder_init(). + lzma_next_coder next; + + /// Encoding options; we also write Total Size, Compressed Size, and + /// Uncompressed Size back to this structure when the encoding has + /// been finished. + lzma_options_block *options; + + enum { + SEQ_CODE, + SEQ_CHECK_FINISH, + SEQ_CHECK_COPY, + SEQ_UNCOMPRESSED_SIZE, + SEQ_BACKWARD_SIZE, + SEQ_PADDING, + } sequence; + + /// Position in .header and .check. + size_t pos; + + /// Check of the uncompressed data + lzma_check check; + + /// Total Size calculated while encoding + lzma_vli total_size; + + /// Compressed Size calculated while encoding + lzma_vli compressed_size; + + /// Uncompressed Size calculated while encoding + lzma_vli uncompressed_size; + + /// Maximum allowed total_size + lzma_vli total_limit; + + /// Maximum allowed uncompressed_size + lzma_vli uncompressed_limit; + + /// Backward Size - This is a copy of total_size right before + /// the Backward Size field. + lzma_vli backward_size; +}; + + +static lzma_ret +block_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Check that our amount of input stays in proper limits. + if (coder->options->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + if (action == LZMA_FINISH) { + if (coder->options->uncompressed_size + - coder->uncompressed_size + != (lzma_vli)(in_size - *in_pos)) + return LZMA_DATA_ERROR; + } else { + if (coder->options->uncompressed_size + - coder->uncompressed_size + < (lzma_vli)(in_size - *in_pos)) + return LZMA_DATA_ERROR; + } + } else if (LZMA_VLI_VALUE_MAX - coder->uncompressed_size + < (lzma_vli)(in_size - *in_pos)) { + return LZMA_DATA_ERROR; + } + + // Main loop + while (*out_pos < out_size + && (*in_pos < in_size || action == LZMA_FINISH)) + switch (coder->sequence) { + case SEQ_CODE: { + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + const size_t in_used = *in_pos - in_start; + const size_t out_used = *out_pos - out_start; + + if (update_size(&coder->total_size, out_used, + coder->total_limit) + || update_size(&coder->compressed_size, + out_used, + coder->options->compressed_size)) + return LZMA_DATA_ERROR; + + // No need to check for overflow because we have already + // checked it at the beginning of this function. + coder->uncompressed_size += in_used; + + lzma_check_update(&coder->check, coder->options->check, + in + in_start, in_used); + + if (ret != LZMA_STREAM_END) + return ret; + + assert(*in_pos == in_size); + + // Compressed and Uncompressed Sizes are now at their final + // values. Verify that they match the values give to us. + if (!is_size_valid(coder->compressed_size, + coder->options->compressed_size) + || !is_size_valid(coder->uncompressed_size, + coder->options->uncompressed_size)) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_CHECK_FINISH; + break; + } + + case SEQ_CHECK_FINISH: + if (coder->options->check == LZMA_CHECK_NONE) { + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + break; + } + + lzma_check_finish(&coder->check, coder->options->check); + coder->sequence = SEQ_CHECK_COPY; + + // Fall through + + case SEQ_CHECK_COPY: + assert(lzma_check_sizes[coder->options->check] > 0); + + switch (coder->options->check) { + case LZMA_CHECK_CRC32: + out[*out_pos] = coder->check.crc32 >> (coder->pos * 8); + break; + + case LZMA_CHECK_CRC64: + out[*out_pos] = coder->check.crc64 >> (coder->pos * 8); + break; + + case LZMA_CHECK_SHA256: + out[*out_pos] = coder->check.sha256.buffer[coder->pos]; + break; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + ++*out_pos; + + if (update_size(&coder->total_size, 1, coder->total_limit)) + return LZMA_DATA_ERROR; + + if (++coder->pos == lzma_check_sizes[coder->options->check]) { + coder->pos = 0; + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + } + + break; + + case SEQ_UNCOMPRESSED_SIZE: + if (coder->options->has_uncompressed_size_in_footer) { + const size_t out_start = *out_pos; + + const lzma_ret ret = lzma_vli_encode( + coder->uncompressed_size, + &coder->pos, 1, + out, out_pos, out_size); + + // Updating the size this way instead of doing in a + // single chunk using lzma_vli_size(), because this + // way we detect when exactly we are going out of + // our limits. + if (update_size(&coder->total_size, + *out_pos - out_start, + coder->total_limit)) + return LZMA_DATA_ERROR; + + if (ret != LZMA_STREAM_END) + return ret; + + coder->pos = 0; + } + + coder->backward_size = coder->total_size; + coder->sequence = SEQ_BACKWARD_SIZE; + break; + + case SEQ_BACKWARD_SIZE: + if (coder->options->has_backward_size) { + const size_t out_start = *out_pos; + + const lzma_ret ret = lzma_vli_encode( + coder->backward_size, &coder->pos, 1, + out, out_pos, out_size); + + if (update_size(&coder->total_size, + *out_pos - out_start, + coder->total_limit)) + return LZMA_DATA_ERROR; + + if (ret != LZMA_STREAM_END) + return ret; + } + + coder->sequence = SEQ_PADDING; + break; + + case SEQ_PADDING: + if (coder->options->handle_padding) { + assert(!coder->options + ->has_uncompressed_size_in_footer); + assert(!coder->options->has_backward_size); + assert(coder->options->total_size != LZMA_VLI_VALUE_UNKNOWN); + + if (coder->total_size < coder->options->total_size) { + out[*out_pos] = 0x00; + ++*out_pos; + + if (update_size(&coder->total_size, 1, + coder->total_limit)) + return LZMA_DATA_ERROR; + + break; + } + } + + // Now also Total Size is known. Verify it. + if (!is_size_valid(coder->total_size, + coder->options->total_size)) + return LZMA_DATA_ERROR; + + // Copy the values into coder->options. The caller + // may use this information to construct Index. + coder->options->total_size = coder->total_size; + coder->options->compressed_size = coder->compressed_size; + coder->options->uncompressed_size = coder->uncompressed_size; + + return LZMA_STREAM_END; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options) +{ + // Validate some options. + if (options == NULL + || !lzma_vli_is_valid(options->total_size) + || !lzma_vli_is_valid(options->compressed_size) + || !lzma_vli_is_valid(options->uncompressed_size) + || !lzma_vli_is_valid(options->total_size) + || !lzma_vli_is_valid(options->total_limit) + || !lzma_vli_is_valid(options->uncompressed_limit) + || (options->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && options->uncompressed_size + > options->uncompressed_limit) + || (options->total_size != LZMA_VLI_VALUE_UNKNOWN + && options->total_size + > options->total_limit) + || (!options->has_eopm && options->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN) + || (options->handle_padding && (options->total_size + == LZMA_VLI_VALUE_UNKNOWN + || options->has_uncompressed_size_in_footer + || options->has_backward_size)) + || options->header_size > options->total_size) + return LZMA_PROG_ERROR; + + // Allocate and initialize *next->coder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &block_encode; + next->end = &block_encoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Initialize the check. + return_if_error(lzma_check_init(&next->coder->check, options->check)); + + // If End of Payload Marker is not used and Uncompressed Size is zero, + // Compressed Data is empty. That is, we don't call the encoder at all. + // We initialize it though; it allows detecting invalid options. + if (!options->has_eopm && options->uncompressed_size == 0) { + // Also Compressed Size must also be zero if it has been + // given to us. + if (!is_size_valid(options->compressed_size, 0)) + return LZMA_PROG_ERROR; + + next->coder->sequence = SEQ_CHECK_FINISH; + } else { + next->coder->sequence = SEQ_CODE; + } + + // Other initializations + next->coder->options = options; + next->coder->pos = 0; + next->coder->total_size = options->header_size; + next->coder->compressed_size = 0; + next->coder->uncompressed_size = 0; + next->coder->total_limit + = MIN(options->total_size, options->total_limit); + next->coder->uncompressed_limit = MIN(options->uncompressed_size, + options->uncompressed_limit); + + // Initialize the requested filters. + return lzma_raw_encoder_init(&next->coder->next, allocator, + options->filters, options->has_eopm + ? LZMA_VLI_VALUE_UNKNOWN + : options->uncompressed_size, + true); +} + + +extern lzma_ret +lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options) +{ + lzma_next_coder_init(block_encoder_init, next, allocator, options); +} + + +extern LZMA_API lzma_ret +lzma_block_encoder(lzma_stream *strm, lzma_options_block *options) +{ + lzma_next_strm_init(strm, block_encoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/block_encoder.h b/src/liblzma/common/block_encoder.h new file mode 100644 index 00000000..eafcc618 --- /dev/null +++ b/src/liblzma/common/block_encoder.h @@ -0,0 +1,29 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_encoder.h +/// \brief Encodes .lzma Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_ENCODER_H +#define LZMA_BLOCK_ENCODER_H + +#include "common.h" + + +extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_block *options); + +#endif diff --git a/src/liblzma/common/block_header_decoder.c b/src/liblzma/common/block_header_decoder.c new file mode 100644 index 00000000..7676c795 --- /dev/null +++ b/src/liblzma/common/block_header_decoder.c @@ -0,0 +1,373 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_header_decoder.c +/// \brief Decodes Block Header from .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "check.h" + + +struct lzma_coder_s { + lzma_options_block *options; + + enum { + SEQ_FLAGS_1, + SEQ_FLAGS_2, + SEQ_COMPRESSED_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_FILTER_FLAGS_INIT, + SEQ_FILTER_FLAGS_DECODE, + SEQ_CRC32, + SEQ_PADDING + } sequence; + + /// Position in variable-length integers + size_t pos; + + /// CRC32 of the Block Header + uint32_t crc32; + + lzma_next_coder filter_flags_decoder; +}; + + +static bool +update_sequence(lzma_coder *coder) +{ + switch (coder->sequence) { + case SEQ_FLAGS_2: + if (coder->options->compressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + coder->pos = 0; + coder->sequence = SEQ_COMPRESSED_SIZE; + break; + } + + // Fall through + + case SEQ_COMPRESSED_SIZE: + if (coder->options->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + coder->pos = 0; + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + break; + } + + // Fall through + + case SEQ_UNCOMPRESSED_SIZE: + coder->pos = 0; + + // Fall through + + case SEQ_FILTER_FLAGS_DECODE: + if (coder->options->filters[coder->pos].id + != LZMA_VLI_VALUE_UNKNOWN) { + coder->sequence = SEQ_FILTER_FLAGS_INIT; + break; + } + + if (coder->options->has_crc32) { + coder->pos = 0; + coder->sequence = SEQ_CRC32; + break; + } + + case SEQ_CRC32: + if (coder->options->padding != 0) { + coder->pos = 0; + coder->sequence = SEQ_PADDING; + break; + } + + return true; + + default: + assert(0); + return true; + } + + return false; +} + + +static lzma_ret +block_header_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + while (*in_pos < in_size) + switch (coder->sequence) { + case SEQ_FLAGS_1: + // Check that the reserved bit is unset. Use HEADER_ERROR + // because newer version of liblzma may support the reserved + // bit, although it is likely that this is just a broken file. + if (in[*in_pos] & 0x40) + return LZMA_HEADER_ERROR; + + // Number of filters: we prepare appropriate amount of + // variables for variable-length integer parsing. The + // initialization function has already reset the rest + // of the values to LZMA_VLI_VALUE_UNKNOWN, which allows + // us to later know how many filters there are. + for (int i = (int)(in[*in_pos] & 0x07) - 1; i >= 0; --i) + coder->options->filters[i].id = 0; + + // End of Payload Marker flag + coder->options->has_eopm = (in[*in_pos] & 0x08) != 0; + + // Compressed Size: Prepare for variable-length integer + // parsing if it is known. + if (in[*in_pos] & 0x10) + coder->options->compressed_size = 0; + + // Uncompressed Size: the same. + if (in[*in_pos] & 0x20) + coder->options->uncompressed_size = 0; + + // Is Metadata Block flag + coder->options->is_metadata = (in[*in_pos] & 0x80) != 0; + + // We need at least one: Uncompressed Size or EOPM. + if (coder->options->uncompressed_size == LZMA_VLI_VALUE_UNKNOWN + && !coder->options->has_eopm) + return LZMA_DATA_ERROR; + + // Update header CRC32. + coder->crc32 = lzma_crc32(in + *in_pos, 1, coder->crc32); + + ++*in_pos; + coder->sequence = SEQ_FLAGS_2; + break; + + case SEQ_FLAGS_2: + // Check that the reserved bits are unset. + if (in[*in_pos] & 0xE0) + return LZMA_DATA_ERROR; + + // Get the size of Header Padding. + coder->options->padding = in[*in_pos] & 0x1F; + + coder->crc32 = lzma_crc32(in + *in_pos, 1, coder->crc32); + + ++*in_pos; + + if (update_sequence(coder)) + return LZMA_STREAM_END; + + break; + + case SEQ_COMPRESSED_SIZE: { + // Store the old input position to be used when + // updating coder->header_crc32. + const size_t in_start = *in_pos; + + const lzma_ret ret = lzma_vli_decode( + &coder->options->compressed_size, + &coder->pos, in, in_pos, in_size); + + const size_t in_used = *in_pos - in_start; + + coder->options->compressed_reserve += in_used; + assert(coder->options->compressed_reserve + <= LZMA_VLI_BYTES_MAX); + + coder->options->header_size += in_used; + + coder->crc32 = lzma_crc32(in + in_start, in_used, + coder->crc32); + + if (ret != LZMA_STREAM_END) + return ret; + + if (update_sequence(coder)) + return LZMA_STREAM_END; + + break; + } + + case SEQ_UNCOMPRESSED_SIZE: { + const size_t in_start = *in_pos; + + const lzma_ret ret = lzma_vli_decode( + &coder->options->uncompressed_size, + &coder->pos, in, in_pos, in_size); + + const size_t in_used = *in_pos - in_start; + + coder->options->uncompressed_reserve += in_used; + assert(coder->options->uncompressed_reserve + <= LZMA_VLI_BYTES_MAX); + + coder->options->header_size += in_used; + + coder->crc32 = lzma_crc32(in + in_start, in_used, + coder->crc32); + + if (ret != LZMA_STREAM_END) + return ret; + + if (update_sequence(coder)) + return LZMA_STREAM_END; + + break; + } + + case SEQ_FILTER_FLAGS_INIT: { + assert(coder->options->filters[coder->pos].id + != LZMA_VLI_VALUE_UNKNOWN); + + const lzma_ret ret = lzma_filter_flags_decoder_init( + &coder->filter_flags_decoder, allocator, + &coder->options->filters[coder->pos]); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_FILTER_FLAGS_DECODE; + } + + // Fall through + + case SEQ_FILTER_FLAGS_DECODE: { + const size_t in_start = *in_pos; + + const lzma_ret ret = coder->filter_flags_decoder.code( + coder->filter_flags_decoder.coder, + allocator, in, in_pos, in_size, + NULL, NULL, 0, LZMA_RUN); + + const size_t in_used = *in_pos - in_start; + coder->options->header_size += in_used; + coder->crc32 = lzma_crc32(in + in_start, + in_used, coder->crc32); + + if (ret != LZMA_STREAM_END) + return ret; + + ++coder->pos; + + if (update_sequence(coder)) + return LZMA_STREAM_END; + + break; + } + + case SEQ_CRC32: + assert(coder->options->has_crc32); + + if (in[*in_pos] != ((coder->crc32 >> (coder->pos * 8)) & 0xFF)) + return LZMA_DATA_ERROR; + + ++*in_pos; + ++coder->pos; + + // Check if we reached end of the CRC32 field. + if (coder->pos == 4) { + coder->options->header_size += 4; + + if (update_sequence(coder)) + return LZMA_STREAM_END; + } + + break; + + case SEQ_PADDING: + if (in[*in_pos] != 0x00) + return LZMA_DATA_ERROR; + + ++*in_pos; + ++coder->options->header_size; + ++coder->pos; + + if (coder->pos < (size_t)(coder->options->padding)) + break; + + return LZMA_STREAM_END; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +block_header_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->filter_flags_decoder, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_block_header_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_block *options) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &block_header_decode; + next->end = &block_header_decoder_end; + next->coder->filter_flags_decoder = LZMA_NEXT_CODER_INIT; + } + + // Assume that Compressed Size and Uncompressed Size are unknown. + options->compressed_size = LZMA_VLI_VALUE_UNKNOWN; + options->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + + // We will calculate the sizes of these fields too so that the + // application may rewrite the header if it wishes so. + options->compressed_reserve = 0; + options->uncompressed_reserve = 0; + + // The Block Flags field is always present, so include its size here + // and we don't need to worry about it in block_header_decode(). + options->header_size = 2; + + // Reset filters[] to indicate empty list of filters. + // See SEQ_FLAGS_1 in block_header_decode() for reasoning of this. + for (size_t i = 0; i < 8; ++i) { + options->filters[i].id = LZMA_VLI_VALUE_UNKNOWN; + options->filters[i].options = NULL; + } + + next->coder->options = options; + next->coder->sequence = SEQ_FLAGS_1; + next->coder->pos = 0; + next->coder->crc32 = 0; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_block_header_decoder(lzma_stream *strm, + lzma_options_block *options) +{ + lzma_next_strm_init(strm, lzma_block_header_decoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c new file mode 100644 index 00000000..594b4fc0 --- /dev/null +++ b/src/liblzma/common/block_header_encoder.c @@ -0,0 +1,211 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_header_encoder.c +/// \brief Encodes Block Header for .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "check.h" + + +extern LZMA_API lzma_ret +lzma_block_header_size(lzma_options_block *options) +{ + // Block Flags take two bytes. + size_t size = 2; + + // Compressed Size + if (!lzma_vli_is_valid(options->compressed_size)) { + return LZMA_PROG_ERROR; + + } else if (options->compressed_reserve != 0) { + // Make sure that the known Compressed Size fits into the + // reserved space. Note that lzma_vli_size() will return zero + // if options->compressed_size is LZMA_VLI_VALUE_UNKNOWN, so + // we don't need to handle that special case separately. + if (options->compressed_reserve > LZMA_VLI_BYTES_MAX + || lzma_vli_size(options->compressed_size) + > (size_t)(options->compressed_reserve)) + return LZMA_PROG_ERROR; + + size += options->compressed_reserve; + + } else if (options->compressed_size != LZMA_VLI_VALUE_UNKNOWN) { + // Compressed Size is known. We have already checked + // that is is a valid VLI, and since it isn't + // LZMA_VLI_VALUE_UNKNOWN, we can be sure that + // lzma_vli_size() will succeed. + size += lzma_vli_size(options->compressed_size); + } + + // Uncompressed Size + if (!lzma_vli_is_valid(options->uncompressed_size)) { + return LZMA_PROG_ERROR; + + } else if (options->uncompressed_reserve != 0) { + if (options->uncompressed_reserve > LZMA_VLI_BYTES_MAX + || lzma_vli_size(options->uncompressed_size) + > (size_t)(options->uncompressed_reserve)) + return LZMA_PROG_ERROR; + + size += options->uncompressed_reserve; + + } else if (options->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + size += lzma_vli_size(options->uncompressed_size); + } + + // List of Filter Flags + for (size_t i = 0; options->filters[i].id != LZMA_VLI_VALUE_UNKNOWN; + ++i) { + // Don't allow too many filters. + if (i == 7) + return LZMA_PROG_ERROR; + + uint32_t tmp; + const lzma_ret ret = lzma_filter_flags_size(&tmp, + options->filters + i); + if (ret != LZMA_OK) + return ret; + + size += tmp; + } + + // CRC32 + if (options->has_crc32) + size += 4; + + // Padding + int32_t padding; + if (options->padding == LZMA_BLOCK_HEADER_PADDING_AUTO) { + const uint32_t preferred = lzma_alignment_output( + options->filters, 1); + const uint32_t unaligned = size + options->alignment; + padding = (int32_t)(unaligned % preferred); + if (padding != 0) + padding = preferred - padding; + } else if (options->padding >= LZMA_BLOCK_HEADER_PADDING_MIN + && options->padding <= LZMA_BLOCK_HEADER_PADDING_MAX) { + padding = options->padding; + } else { + return LZMA_PROG_ERROR; + } + + // All success. Copy the calculated values to the options structure. + options->padding = padding; + options->header_size = size + (size_t)(padding); + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_block_header_encode(uint8_t *out, const lzma_options_block *options) +{ + // We write the Block Flags later. + if (options->header_size < 2) + return LZMA_PROG_ERROR; + + const size_t out_size = options->header_size; + size_t out_pos = 2; + + // Compressed Size + if (options->compressed_size != LZMA_VLI_VALUE_UNKNOWN + || options->compressed_reserve != 0) { + const lzma_vli size = options->compressed_size + != LZMA_VLI_VALUE_UNKNOWN + ? options->compressed_size : 0; + size_t vli_pos = 0; + if (lzma_vli_encode( + size, &vli_pos, options->compressed_reserve, + out, &out_pos, out_size) != LZMA_STREAM_END) + return LZMA_PROG_ERROR; + + } + + // Uncompressed Size + if (options->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN + || options->uncompressed_reserve != 0) { + const lzma_vli size = options->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + ? options->uncompressed_size : 0; + size_t vli_pos = 0; + if (lzma_vli_encode( + size, &vli_pos, options->uncompressed_reserve, + out, &out_pos, out_size) != LZMA_STREAM_END) + return LZMA_PROG_ERROR; + + } + + // Filter Flags + size_t filter_count; + for (filter_count = 0; options->filters[filter_count].id + != LZMA_VLI_VALUE_UNKNOWN; ++filter_count) { + // There can be at maximum of seven filters. + if (filter_count == 7) + return LZMA_PROG_ERROR; + + const lzma_ret ret = lzma_filter_flags_encode(out, &out_pos, + out_size, options->filters + filter_count); + // FIXME: Don't return LZMA_BUF_ERROR. + if (ret != LZMA_OK) + return ret; + } + + // Block Flags 1 + out[0] = filter_count; + + if (options->has_eopm) + out[0] |= 0x08; + else if (options->uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_PROG_ERROR; + + if (options->compressed_size != LZMA_VLI_VALUE_UNKNOWN + || options->compressed_reserve != 0) + out[0] |= 0x10; + + if (options->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN + || options->uncompressed_reserve != 0) + out[0] |= 0x20; + + if (options->is_metadata) + out[0] |= 0x80; + + // Block Flags 2 + if (options->padding < LZMA_BLOCK_HEADER_PADDING_MIN + || options->padding > LZMA_BLOCK_HEADER_PADDING_MAX) + return LZMA_PROG_ERROR; + + out[1] = (uint8_t)(options->padding); + + // CRC32 + if (options->has_crc32) { + if (out_size - out_pos < 4) + return LZMA_PROG_ERROR; + + const uint32_t crc = lzma_crc32(out, out_pos, 0); + for (size_t i = 0; i < 4; ++i) + out[out_pos++] = crc >> (i * 8); + } + + // Padding - the amount of available space must now match with + // the size of the Padding field. + if (out_size - out_pos != (size_t)(options->padding)) + return LZMA_PROG_ERROR; + + memzero(out + out_pos, (size_t)(options->padding)); + + return LZMA_OK; +} diff --git a/src/liblzma/common/block_private.h b/src/liblzma/common/block_private.h new file mode 100644 index 00000000..8e2db319 --- /dev/null +++ b/src/liblzma/common/block_private.h @@ -0,0 +1,46 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_private.h +/// \brief Common stuff for Block encoder and decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_COMMON_H +#define LZMA_BLOCK_COMMON_H + +#include "common.h" + +static inline bool +update_size(lzma_vli *size, lzma_vli add, lzma_vli limit) +{ + if (limit > LZMA_VLI_VALUE_MAX) + limit = LZMA_VLI_VALUE_MAX; + + if (limit < *size || limit - *size < add) + return true; + + *size += add; + + return false; +} + + +static inline bool +is_size_valid(lzma_vli size, lzma_vli reference) +{ + return reference == LZMA_VLI_VALUE_UNKNOWN || reference == size; +} + +#endif diff --git a/src/liblzma/common/chunk_size.c b/src/liblzma/common/chunk_size.c new file mode 100644 index 00000000..042201d2 --- /dev/null +++ b/src/liblzma/common/chunk_size.c @@ -0,0 +1,74 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file chunk_size.c +/// \brief Finds out the minimal reasonable chunk size for a filter chain +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/** + * \brief Finds out the minimal reasonable chunk size for a filter chain + * + * This function helps determining the Uncompressed Sizes of the Blocks when + * doing multi-threaded encoding. + * + * When compressing a large file on a system having multiple CPUs or CPU + * cores, the file can be splitted in smaller chunks, that are compressed + * independently into separate Blocks in the same .lzma Stream. + * + * \return Minimum reasonable Uncompressed Size of a Block. The + * recommended minimum Uncompressed Size is between this value + * and the value times two. + + Zero if the Uncompressed Sizes of Blocks don't matter + */ +extern LZMA_API size_t +lzma_chunk_size(const lzma_options_filter *filters) +{ + while (filters->id != LZMA_VLI_VALUE_UNKNOWN) { + switch (filters->id) { + // TODO LZMA_FILTER_SPARSE + + case LZMA_FILTER_COPY: + case LZMA_FILTER_SUBBLOCK: + case LZMA_FILTER_X86: + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_IA64: + case LZMA_FILTER_ARM: + case LZMA_FILTER_ARMTHUMB: + case LZMA_FILTER_SPARC: + // These are very fast, thus there is no point in + // splitting the data in smaller blocks. + break; + + case LZMA_FILTER_LZMA: + // The block sizes of the possible next filters in + // the chain are irrelevant after the LZMA filter. + return ((lzma_options_lzma *)(filters->options)) + ->dictionary_size; + + default: + // Unknown filters + return 0; + } + + ++filters; + } + + // Indicate that splitting would be useless. + return SIZE_MAX; +} diff --git a/src/liblzma/common/code.c b/src/liblzma/common/code.c new file mode 100644 index 00000000..0e3929b6 --- /dev/null +++ b/src/liblzma/common/code.c @@ -0,0 +1,203 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file code.c +/// \brief zlib-like API wrapper for liblzma's internal API +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +LZMA_API const lzma_stream LZMA_STREAM_INIT_VAR = { + .next_in = NULL, + .avail_in = 0, + .total_in = 0, + .next_out = NULL, + .avail_out = 0, + .total_out = 0, + .allocator = NULL, + .internal = NULL, +}; + + +extern lzma_ret +lzma_strm_init(lzma_stream *strm) +{ + if (strm == NULL) + return LZMA_PROG_ERROR; + + if (strm->internal == NULL) { + strm->internal = lzma_alloc(sizeof(lzma_internal), + strm->allocator); + if (strm->internal == NULL) + return LZMA_MEM_ERROR; + + strm->internal->next = LZMA_NEXT_CODER_INIT; + } + + strm->internal->supported_actions[LZMA_RUN] = false; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; + strm->internal->supported_actions[LZMA_FINISH] = false; + strm->internal->sequence = ISEQ_RUN; + + strm->total_in = 0; + strm->total_out = 0; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_code(lzma_stream *strm, lzma_action action) +{ + // Sanity checks + if ((strm->next_in == NULL && strm->avail_in != 0) + || (strm->next_out == NULL && strm->avail_out != 0) + || strm->internal == NULL + || strm->internal->next.code == NULL + || (unsigned int)(action) > LZMA_FINISH + || !strm->internal->supported_actions[action]) + return LZMA_PROG_ERROR; + + switch (strm->internal->sequence) { + case ISEQ_RUN: + switch (action) { + case LZMA_RUN: + break; + + case LZMA_SYNC_FLUSH: + strm->internal->sequence = ISEQ_SYNC_FLUSH; + break; + + case LZMA_FULL_FLUSH: + strm->internal->sequence = ISEQ_FULL_FLUSH; + break; + + case LZMA_FINISH: + strm->internal->sequence = ISEQ_FINISH; + break; + } + + break; + + case ISEQ_SYNC_FLUSH: + if (action != LZMA_SYNC_FLUSH) + return LZMA_PROG_ERROR; + + // Check that application doesn't change avail_in once + // LZMA_SYNC_FLUSH has been used. + if (strm->internal->avail_in != strm->avail_in) + return LZMA_DATA_ERROR; + + break; + + case ISEQ_FULL_FLUSH: + if (action != LZMA_FULL_FLUSH) + return LZMA_PROG_ERROR; + + // Check that application doesn't change avail_in once + // LZMA_FULL_FLUSH has been used. + if (strm->internal->avail_in != strm->avail_in) + return LZMA_DATA_ERROR; + + break; + + case ISEQ_FINISH: + if (action != LZMA_FINISH) + return LZMA_PROG_ERROR; + + if (strm->internal->avail_in != strm->avail_in) + return LZMA_DATA_ERROR; + + break; + + case ISEQ_END: + return LZMA_STREAM_END; + + case ISEQ_ERROR: + default: + return LZMA_PROG_ERROR; + } + + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( + strm->internal->next.coder, strm->allocator, + strm->next_in, &in_pos, strm->avail_in, + strm->next_out, &out_pos, strm->avail_out, action); + + strm->next_in += in_pos; + strm->avail_in -= in_pos; + strm->total_in += in_pos; + + strm->next_out += out_pos; + strm->avail_out -= out_pos; + strm->total_out += out_pos; + + strm->internal->avail_in = strm->avail_in; + + switch (ret) { + case LZMA_OK: + // Don't return LZMA_BUF_ERROR when it happens the first time. + // This is to avoid returning LZMA_BUF_ERROR when avail_out + // was zero but still there was no more data left to written + // to next_out. + if (out_pos == 0 && in_pos == 0) { + if (strm->internal->allow_buf_error) + ret = LZMA_BUF_ERROR; + else + strm->internal->allow_buf_error = true; + } else { + strm->internal->allow_buf_error = false; + } + break; + + case LZMA_STREAM_END: + if (strm->internal->sequence == ISEQ_SYNC_FLUSH + || strm->internal->sequence == ISEQ_FULL_FLUSH) + strm->internal->sequence = ISEQ_RUN; + else + strm->internal->sequence = ISEQ_END; + break; + + case LZMA_UNSUPPORTED_CHECK: + strm->internal->allow_buf_error = false; + break; + + default: + // All the other errors are fatal; coding cannot be continued. + strm->internal->sequence = ISEQ_ERROR; + break; + } + + return ret; +} + + +extern LZMA_API void +lzma_end(lzma_stream *strm) +{ + if (strm != NULL && strm->internal != NULL) { + if (strm->internal->next.end != NULL) + strm->internal->next.end(strm->internal->next.coder, + strm->allocator); + + lzma_free(strm->internal, strm->allocator); + strm->internal = NULL; + } + + return; +} diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h new file mode 100644 index 00000000..ca9c2f23 --- /dev/null +++ b/src/liblzma/common/common.h @@ -0,0 +1,271 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file common.h +/// \brief Definitions common to the whole liblzma library +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_COMMON_H +#define LZMA_COMMON_H + +#include "sysdefs.h" + +// Don't use ifdef... +#if HAVE_VISIBILITY +# define LZMA_API __attribute__((__visibility__("default"))) +#else +# define LZMA_API +#endif + + +/// Size of temporary buffers needed in some filters +#define LZMA_BUFFER_SIZE 4096 + + +/// Internal helper filter used by Subblock decoder. It is mapped to an +/// otherwise invalid Filter ID, which is impossible to get from any input +/// file (even if malicious file). +#define LZMA_FILTER_SUBBLOCK_HELPER (UINT64_MAX - 2) + + +/////////// +// Types // +/////////// + +typedef struct lzma_coder_s lzma_coder; + +typedef struct lzma_next_coder_s lzma_next_coder; + +typedef struct lzma_filter_info_s lzma_filter_info; + + +typedef lzma_ret (*lzma_init_function)( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters); + +typedef lzma_ret (*lzma_code_function)( + lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action); + +typedef void (*lzma_end_function)( + lzma_coder *coder, lzma_allocator *allocator); + + +/// Hold data and function pointers of the next filter in the chain. +struct lzma_next_coder_s { + /// Pointer to coder-specific data + lzma_coder *coder; + + /// "Pointer" to init function. This is never called here. + /// We need only to detect if we are initializing a coder + /// that was allocated earlier. See code.c and next_coder.c. + uintptr_t init; + + /// Pointer to function to do the actual coding + lzma_code_function code; + + /// Pointer to function to free lzma_next_coder.coder + lzma_end_function end; +}; + +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = 0, \ + .code = NULL, \ + .end = NULL, \ + } + + +struct lzma_internal_s { + lzma_next_coder next; + + enum { + ISEQ_RUN, + ISEQ_SYNC_FLUSH, + ISEQ_FULL_FLUSH, + ISEQ_FINISH, + ISEQ_END, + ISEQ_ERROR, + } sequence; + + bool supported_actions[4]; + bool allow_buf_error; + size_t avail_in; +}; + + +struct lzma_filter_info_s { + /// Pointer to function used to initialize the filter. + /// This is NULL to indicate end of array. + lzma_init_function init; + + /// Pointer to filter's options structure + void *options; + + /// Uncompressed size of the filter, or LZMA_VLI_VALUE_UNKNOWN + /// if unknown. + lzma_vli uncompressed_size; +}; + + +/* +typedef struct { + lzma_init_function init; + uint32_t (*input_alignment)(lzma_vli id, const void *options); + uint32_t (*output_alignment)(lzma_vli id, const void *options); + bool changes_uncompressed_size; + bool supports_eopm; +} lzma_filter_hook; +*/ + + +/////////////// +// Functions // +/////////////// + +/// Allocates memory +extern void *lzma_alloc(size_t size, lzma_allocator *allocator) + lzma_attribute((malloc)); + +/// Frees memory +extern void lzma_free(void *ptr, lzma_allocator *allocator); + +/// Initializes lzma_stream FIXME desc +extern lzma_ret lzma_strm_init(lzma_stream *strm); + +/// +extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +/// +extern void lzma_next_coder_end(lzma_next_coder *next, + lzma_allocator *allocator); + + +extern lzma_ret lzma_filter_flags_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_filter *options); + +extern lzma_ret lzma_block_header_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_block *options); + +extern lzma_ret lzma_stream_encoder_single_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_stream *options); + +extern lzma_ret lzma_stream_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + lzma_extra **header, lzma_extra **footer); + + +/// \brief Wrapper for memcpy() +/// +/// This function copies as much data as possible from in[] to out[] and +/// updates *in_pos and *out_pos accordingly. +/// +static inline size_t +bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + const size_t in_avail = in_size - *in_pos; + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(in_avail, out_avail); + + memcpy(out + *out_pos, in + *in_pos, copy_size); + + *in_pos += copy_size; + *out_pos += copy_size; + + return copy_size; +} + + +/// \brief Initializing the next coder +/// +/// lzma_next_coder can point to different types of coders. The existing +/// coder may be different than what we are initializing now. In that case +/// we must git rid of the old coder first. Otherwise we reuse the existing +/// coder structure. +/// +#define lzma_next_coder_init2(next, allocator, cmpfunc, func, ...) \ +do { \ + if ((uintptr_t)(&cmpfunc) != (next)->init) \ + lzma_next_coder_end(next, allocator); \ + const lzma_ret ret = func(next, __VA_ARGS__); \ + if (ret == LZMA_OK) { \ + (next)->init = (uintptr_t)(&cmpfunc); \ + assert((next)->code != NULL); \ + assert((next)->end != NULL); \ + } else { \ + lzma_next_coder_end(next, allocator); \ + } \ + return ret; \ +} while (0) + +/// \brief Initializing lzma_next_coder +/// +/// Call the initialization function, which must take at least one +/// argument in addition to lzma_next_coder and lzma_allocator. +#define lzma_next_coder_init(func, next, allocator, ...) \ + lzma_next_coder_init2(next, allocator, \ + func, func, allocator, __VA_ARGS__) + + +/// \brief Initializing lzma_stream +/// +/// lzma_strm initialization with more detailed options. +#define lzma_next_strm_init2(strm, cmpfunc, func, ...) \ +do { \ + lzma_ret ret = lzma_strm_init(strm); \ + if (ret != LZMA_OK) \ + return ret; \ + if ((uintptr_t)(&cmpfunc) != (strm)->internal->next.init) \ + lzma_next_coder_end(\ + &(strm)->internal->next, (strm)->allocator); \ + ret = func(&(strm)->internal->next, __VA_ARGS__); \ + if (ret != LZMA_OK) { \ + lzma_end(strm); \ + return ret; \ + } \ + (strm)->internal->next.init = (uintptr_t)(&cmpfunc); \ + assert((strm)->internal->next.code != NULL); \ + assert((strm)->internal->next.end != NULL); \ +} while (0) + +/// \brief Initializing lzma_stream +/// +/// Call the initialization function, which must take at least one +/// argument in addition to lzma_next_coder and lzma_allocator. +#define lzma_next_strm_init(strm, func, ...) \ + lzma_next_strm_init2(strm, func, func, (strm)->allocator, __VA_ARGS__) + + +/// \brief Return if expression doesn't evaluate to LZMA_OK +/// +/// There are several situations where we want to return immediatelly +/// with the value of expr if it isn't LZMA_OK. This macro shortens +/// the code a bit. +/// +#define return_if_error(expr) \ +do { \ + const lzma_ret ret_ = expr; \ + if (ret_ != LZMA_OK) \ + return ret_; \ +} while (0) + +#endif diff --git a/src/liblzma/common/copy_coder.c b/src/liblzma/common/copy_coder.c new file mode 100644 index 00000000..64864f60 --- /dev/null +++ b/src/liblzma/common/copy_coder.c @@ -0,0 +1,143 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file copy_coder.c +/// \brief The Copy filter encoder and decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "copy_coder.h" + + +struct lzma_coder_s { + lzma_next_coder next; + lzma_vli uncompressed_size; + bool is_encoder; +}; + + +static lzma_ret +copy_code(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // If we aren't the last filter in the chain, the Copy filter + // is totally useless. Note that it is job of the next coder to + // take care of Uncompressed Size, so we don't need to update our + // coder->uncompressed_size at all. + if (coder->next.code != NULL) + return coder->next.code(coder->next.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + + // If we get here, we are the last filter in the chain. + + const size_t in_avail = in_size - *in_pos; + + if (coder->is_encoder) { + // Check that we don't have too much input. + if ((lzma_vli)(in_avail) > coder->uncompressed_size) + return LZMA_DATA_ERROR; + + // Check that once LZMA_FINISH has been given, the + // amount of input matches uncompressed_size if it + // is known. + if (action == LZMA_FINISH && coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->uncompressed_size + != (lzma_vli)(in_avail)) + return LZMA_DATA_ERROR; + + } else { + // Limit in_size so that we don't copy too much. + if ((lzma_vli)(in_avail) > coder->uncompressed_size) + in_size = *in_pos + (size_t)(coder->uncompressed_size); + } + + // Store the old input position, which is needed to update + // coder->uncompressed_size. + const size_t in_start = *in_pos; + + // We are the last coder in the chain. + // Just copy as much data as possible. + bufcpy(in, in_pos, in_size, out, out_pos, out_size); + + // Update uncompressed_size if it is known. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_size -= *in_pos - in_start; + + // action can be LZMA_FINISH only in the encoder. + if ((action == LZMA_FINISH && *in_pos == in_size) + || coder->uncompressed_size == 0) + return LZMA_STREAM_END; + + return LZMA_OK; +} + + +static void +copy_coder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +copy_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + // Allocate memory for the decoder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = ©_code; + next->end = ©_coder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Copy Uncompressed Size which is used to limit the output size. + next->coder->uncompressed_size = filters[0].uncompressed_size; + + // The coder acts slightly differently as encoder and decoder. + next->coder->is_encoder = is_encoder; + + // Initialize the next decoder in the chain, if any. + return lzma_next_filter_init( + &next->coder->next, allocator, filters + 1); +} + + +#ifdef HAVE_ENCODER +extern lzma_ret +lzma_copy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_next_coder_init(copy_coder_init, next, allocator, filters, true); +} +#endif + + +#ifdef HAVE_DECODER +extern lzma_ret +lzma_copy_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_next_coder_init(copy_coder_init, next, allocator, filters, false); +} +#endif diff --git a/src/liblzma/common/copy_coder.h b/src/liblzma/common/copy_coder.h new file mode 100644 index 00000000..b8d0295d --- /dev/null +++ b/src/liblzma/common/copy_coder.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file copy_coder.h +/// \brief The Copy filter encoder and decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_COPY_CODER_H +#define LZMA_COPY_CODER_H + +#include "common.h" + +extern lzma_ret lzma_copy_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_copy_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif diff --git a/src/liblzma/common/delta_coder.c b/src/liblzma/common/delta_coder.c new file mode 100644 index 00000000..ec8c6d59 --- /dev/null +++ b/src/liblzma/common/delta_coder.c @@ -0,0 +1,210 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file delta_coder.c +/// \brief Encoder and decoder for the Delta filter +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "delta_coder.h" + + +struct lzma_coder_s { + /// Next coder in the chain + lzma_next_coder next; + + /// Uncompressed size - This is needed when we are the last + /// filter in the chain. + lzma_vli uncompressed_size; + + /// Delta distance + size_t distance; + + /// True if we are encoding; false if decoding + bool is_encoder; + + /// Position in history[] + uint8_t pos; + + /// Buffer to hold history of the original data + uint8_t history[LZMA_DELTA_DISTANCE_MAX]; +}; + + +static void +encode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +{ + const size_t distance = coder->distance; + + for (size_t i = 0; i < size; ++i) { + const uint8_t tmp = coder->history[ + (distance + coder->pos) & 0xFF]; + coder->history[coder->pos--] = buffer[i]; + buffer[i] -= tmp; + } + + return; +} + + +static void +decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +{ + const size_t distance = coder->distance; + + for (size_t i = 0; i < size; ++i) { + buffer[i] += coder->history[(distance + coder->pos) & 0xFF]; + coder->history[coder->pos--] = buffer[i]; + } + + return; +} + + +static lzma_ret +delta_code(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + const size_t out_start = *out_pos; + size_t size; + lzma_ret ret; + + if (coder->next.code == NULL) { + const size_t in_avail = in_size - *in_pos; + + if (coder->is_encoder) { + // Check that we don't have too much input. + if ((lzma_vli)(in_avail) > coder->uncompressed_size) + return LZMA_DATA_ERROR; + + // Check that once LZMA_FINISH has been given, the + // amount of input matches uncompressed_size if it + // is known. + if (action == LZMA_FINISH && coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->uncompressed_size + != (lzma_vli)(in_avail)) + return LZMA_DATA_ERROR; + + } else { + // Limit in_size so that we don't copy too much. + if ((lzma_vli)(in_avail) > coder->uncompressed_size) + in_size = *in_pos + (size_t)( + coder->uncompressed_size); + } + + size = bufcpy(in, in_pos, in_size, out, out_pos, out_size); + + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_size -= size; + + // action can be LZMA_FINISH only in the encoder. + ret = (action == LZMA_FINISH && *in_pos == in_size) + || coder->uncompressed_size == 0 + ? LZMA_STREAM_END : LZMA_OK; + + } else { + ret = coder->next.code(coder->next.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + return ret; + + size = *out_pos - out_start; + } + + if (coder->is_encoder) + encode_buffer(coder, out + out_start, size); + else + decode_buffer(coder, out + out_start, size); + + return ret; +} + + +static void +delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + // Allocate memory for the decoder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &delta_code; + next->end = &delta_coder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Copy Uncompressed Size which is used to limit the output size. + next->coder->uncompressed_size = filters[0].uncompressed_size; + + // The coder acts slightly differently as encoder and decoder. + next->coder->is_encoder = is_encoder; + + // Set the delta distance. + if (filters[0].options == NULL) + return LZMA_PROG_ERROR; + next->coder->distance = ((lzma_options_delta *)(filters[0].options)) + ->distance; + if (next->coder->distance < LZMA_DELTA_DISTANCE_MIN + || next->coder->distance > LZMA_DELTA_DISTANCE_MAX) + return LZMA_HEADER_ERROR; + + // Initialize the rest of the variables. + next->coder->pos = 0; + memzero(next->coder->history, LZMA_DELTA_DISTANCE_MAX); + + // Initialize the next decoder in the chain, if any. + { + const lzma_ret ret = lzma_next_filter_init(&next->coder->next, + allocator, filters + 1); + if (ret != LZMA_OK) + return ret; + } + + return LZMA_OK; +} + + +#ifdef HAVE_ENCODER +extern lzma_ret +lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return delta_coder_init(next, allocator, filters, true); +} +#endif + + +#ifdef HAVE_DECODER +extern lzma_ret +lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return delta_coder_init(next, allocator, filters, false); +} +#endif diff --git a/src/liblzma/common/delta_coder.h b/src/liblzma/common/delta_coder.h new file mode 100644 index 00000000..60cea95c --- /dev/null +++ b/src/liblzma/common/delta_coder.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file delta_coder.h +/// \brief The Delta filter encoder and decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_DELTA_CODER_H +#define LZMA_DELTA_CODER_H + +#include "common.h" + +extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif diff --git a/src/liblzma/common/extra.c b/src/liblzma/common/extra.c new file mode 100644 index 00000000..b743a439 --- /dev/null +++ b/src/liblzma/common/extra.c @@ -0,0 +1,33 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file extra.c +/// \brief Handling of Extra in Metadata +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API void +lzma_extra_free(lzma_extra *extra, lzma_allocator *allocator) +{ + while (extra != NULL) { + lzma_extra *tmp = extra->next; + lzma_free(extra, allocator); + extra = tmp; + } + + return; +} diff --git a/src/liblzma/common/features.c b/src/liblzma/common/features.c new file mode 100644 index 00000000..33b2e0a2 --- /dev/null +++ b/src/liblzma/common/features.c @@ -0,0 +1,70 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file features.c +/// \brief Information about features enabled at compile time +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +static const lzma_vli filters[] = { +#ifdef HAVE_FILTER_COPY + LZMA_FILTER_COPY, +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + LZMA_FILTER_SUBBLOCK, +#endif + +#ifdef HAVE_FILTER_X86 + LZMA_FILTER_X86, +#endif + +#ifdef HAVE_FILTER_POWERPC + LZMA_FILTER_POWERPC, +#endif + +#ifdef HAVE_FILTER_IA64 + LZMA_FILTER_IA64, +#endif + +#ifdef HAVE_FILTER_ARM + LZMA_FILTER_ARM, +#endif + +#ifdef HAVE_FILTER_ARMTHUMB + LZMA_FILTER_ARMTHUMB, +#endif + +#ifdef HAVE_FILTER_SPARC + LZMA_FILTER_SPARC, +#endif + +#ifdef HAVE_FILTER_DELTA + LZMA_FILTER_DELTA, +#endif + +#ifdef HAVE_FILTER_LZMA + LZMA_FILTER_LZMA, +#endif + + LZMA_VLI_VALUE_UNKNOWN +}; + + +LZMA_API const lzma_vli *const lzma_available_filter_encoders = filters; + +LZMA_API const lzma_vli *const lzma_available_filter_decoders = filters; diff --git a/src/liblzma/common/filter_flags_decoder.c b/src/liblzma/common/filter_flags_decoder.c new file mode 100644 index 00000000..515f9346 --- /dev/null +++ b/src/liblzma/common/filter_flags_decoder.c @@ -0,0 +1,382 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_flags_decoder.c +/// \brief Decodes a Filter Flags field +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "lzma_decoder.h" + + +struct lzma_coder_s { + lzma_options_filter *options; + + enum { + SEQ_MISC, + SEQ_ID, + SEQ_SIZE, + SEQ_PROPERTIES, + } sequence; + + /// \brief Position in variable-length integers + size_t pos; + + /// \brief Size of Filter Properties + lzma_vli properties_size; +}; + + +#ifdef HAVE_FILTER_SUBBLOCK +static lzma_ret +properties_subblock(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in lzma_attribute((unused)), + size_t *in_pos lzma_attribute((unused)), + size_t in_size lzma_attribute((unused))) +{ + if (coder->properties_size != 0) + return LZMA_HEADER_ERROR; + + coder->options->options = lzma_alloc( + sizeof(lzma_options_subblock), allocator); + if (coder->options->options == NULL) + return LZMA_MEM_ERROR; + + ((lzma_options_subblock *)(coder->options->options)) + ->allow_subfilters = true; + return LZMA_STREAM_END; +} +#endif + + +#ifdef HAVE_FILTER_SIMPLE +static lzma_ret +properties_simple(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) +{ + if (coder->properties_size == 0) + return LZMA_STREAM_END; + + if (coder->properties_size != 4) + return LZMA_HEADER_ERROR; + + lzma_options_simple *options = coder->options->options; + + if (options == NULL) { + options = lzma_alloc(sizeof(lzma_options_simple), allocator); + if (options == NULL) + return LZMA_MEM_ERROR; + + options->start_offset = 0; + coder->options->options = options; + } + + while (coder->pos < 4) { + if (*in_pos == in_size) + return LZMA_OK; + + options->start_offset + |= (uint32_t)(in[*in_pos]) << (8 * coder->pos); + ++*in_pos; + ++coder->pos; + } + + // Don't leave an options structure allocated if start_offset is zero. + if (options->start_offset == 0) { + lzma_free(options, allocator); + coder->options->options = NULL; + } + + return LZMA_STREAM_END; +} +#endif + + +#ifdef HAVE_FILTER_DELTA +static lzma_ret +properties_delta(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) +{ + if (coder->properties_size != 1) + return LZMA_HEADER_ERROR; + + if (*in_pos == in_size) + return LZMA_OK; + + lzma_options_delta *options = lzma_alloc( + sizeof(lzma_options_delta), allocator); + if (options == NULL) + return LZMA_MEM_ERROR; + + coder->options->options = options; + + options->distance = (uint32_t)(in[*in_pos]) + 1; + ++*in_pos; + + return LZMA_STREAM_END; +} +#endif + + +#ifdef HAVE_FILTER_LZMA +static lzma_ret +properties_lzma(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) +{ + // LZMA properties are always two bytes (at least for now). + if (coder->properties_size != 2) + return LZMA_HEADER_ERROR; + + assert(coder->pos < 2); + + while (*in_pos < in_size) { + switch (coder->pos) { + case 0: + // Allocate the options structure. + coder->options->options = lzma_alloc( + sizeof(lzma_options_lzma), allocator); + if (coder->options->options == NULL) + return LZMA_MEM_ERROR; + + // Decode lc, lp, and pb. + if (lzma_lzma_decode_properties( + coder->options->options, in[*in_pos])) + return LZMA_HEADER_ERROR; + + ++*in_pos; + ++coder->pos; + break; + + case 1: { + lzma_options_lzma *options = coder->options->options; + + // Check that reserved bits are unset. + if (in[*in_pos] & 0xC0) + return LZMA_HEADER_ERROR; + + // Decode the dictionary size. See the file format + // specification section 4.3.4.2 to understand this. + if (in[*in_pos] == 0) { + options->dictionary_size = 1; + + } else if (in[*in_pos] > 59) { + // Dictionary size is over 1 GiB. + // It's not supported at the moment. + return LZMA_HEADER_ERROR; +# if LZMA_DICTIONARY_SIZE_MAX != UINT32_C(1) << 30 +# error Update the if()-condition a few lines +# error above to match LZMA_DICTIONARY_SIZE_MAX. +# endif + + } else { + options->dictionary_size + = 2 | ((in[*in_pos] + 1) & 1); + options->dictionary_size + <<= (in[*in_pos] - 1) / 2; + } + + ++*in_pos; + return LZMA_STREAM_END; + } + } + } + + assert(coder->pos < 2); + return LZMA_OK; +} +#endif + + +static lzma_ret +filter_flags_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + while (*in_pos < in_size || coder->sequence == SEQ_PROPERTIES) + switch (coder->sequence) { + case SEQ_MISC: + // Determine the Filter ID and Size of Filter Properties. + if (in[*in_pos] >= 0xE0) { + // Using External ID. Prepare the ID + // for variable-length integer parsing. + coder->options->id = 0; + + if (in[*in_pos] == 0xFF) { + // Mark that Size of Filter Properties is + // unknown, so we know later that there is + // external Size of Filter Properties present. + coder->properties_size + = LZMA_VLI_VALUE_UNKNOWN; + } else { + // Take Size of Filter Properties from Misc. + coder->properties_size = in[*in_pos] - 0xE0; + } + + coder->sequence = SEQ_ID; + + } else { + // The Filter ID is the same as Misc. + coder->options->id = in[*in_pos]; + + // The Size of Filter Properties can be calculated + // from Misc too. + coder->properties_size = in[*in_pos] / 0x20; + + coder->sequence = SEQ_PROPERTIES; + } + + ++*in_pos; + break; + + case SEQ_ID: { + const lzma_ret ret = lzma_vli_decode(&coder->options->id, + &coder->pos, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + + if (coder->properties_size == LZMA_VLI_VALUE_UNKNOWN) { + // We have also external Size of Filter + // Properties. Prepare the size for + // variable-length integer parsing. + coder->properties_size = 0; + coder->sequence = SEQ_SIZE; + } else { + coder->sequence = SEQ_PROPERTIES; + } + + // Reset pos for its next job. + coder->pos = 0; + break; + } + + case SEQ_SIZE: { + const lzma_ret ret = lzma_vli_decode(&coder->properties_size, + &coder->pos, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + + coder->pos = 0; + coder->sequence = SEQ_PROPERTIES; + break; + } + + case SEQ_PROPERTIES: { + lzma_ret (*get_properties)(lzma_coder *coder, + lzma_allocator *allocator, const uint8_t *in, + size_t *in_pos, size_t in_size); + + switch (coder->options->id) { +#ifdef HAVE_FILTER_COPY + case LZMA_FILTER_COPY: + return coder->properties_size > 0 + ? LZMA_HEADER_ERROR : LZMA_STREAM_END; +#endif +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK: + get_properties = &properties_subblock; + break; +#endif +#ifdef HAVE_FILTER_SIMPLE +# ifdef HAVE_FILTER_X86 + case LZMA_FILTER_X86: +# endif +# ifdef HAVE_FILTER_POWERPC + case LZMA_FILTER_POWERPC: +# endif +# ifdef HAVE_FILTER_IA64 + case LZMA_FILTER_IA64: +# endif +# ifdef HAVE_FILTER_ARM + case LZMA_FILTER_ARM: +# endif +# ifdef HAVE_FILTER_ARMTHUMB + case LZMA_FILTER_ARMTHUMB: +# endif +# ifdef HAVE_FILTER_SPARC + case LZMA_FILTER_SPARC: +# endif + get_properties = &properties_simple; + break; +#endif +#ifdef HAVE_FILTER_DELTA + case LZMA_FILTER_DELTA: + get_properties = &properties_delta; + break; +#endif +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: + get_properties = &properties_lzma; + break; +#endif + default: + return LZMA_HEADER_ERROR; + } + + return get_properties(coder, allocator, in, in_pos, in_size); + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +filter_flags_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_filter_flags_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_options_filter *options) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &filter_flags_decode; + next->end = &filter_flags_decoder_end; + } + + options->id = 0; + options->options = NULL; + + next->coder->options = options; + next->coder->sequence = SEQ_MISC; + next->coder->pos = 0; + next->coder->properties_size = 0; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_filter_flags_decoder(lzma_stream *strm, lzma_options_filter *options) +{ + lzma_next_strm_init(strm, lzma_filter_flags_decoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/filter_flags_encoder.c b/src/liblzma/common/filter_flags_encoder.c new file mode 100644 index 00000000..d8f260a1 --- /dev/null +++ b/src/liblzma/common/filter_flags_encoder.c @@ -0,0 +1,359 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_flags_encoder.c +/// \brief Decodes a Filter Flags field +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "lzma_encoder.h" + + +/// \brief Calculates the size of the Filter Properties field +/// +/// This currently can return only LZMA_OK or LZMA_HEADER_ERROR, but +/// with some new filters it may return also LZMA_PROG_ERROR. +static lzma_ret +get_properties_size(uint32_t *size, const lzma_options_filter *options) +{ + lzma_ret ret = LZMA_OK; + + switch (options->id) { +#ifdef HAVE_FILTER_COPY + case LZMA_FILTER_COPY: + *size = 0; + break; +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK: + *size = 0; + break; +#endif + +#ifdef HAVE_FILTER_SIMPLE +# ifdef HAVE_FILTER_X86 + case LZMA_FILTER_X86: +# endif +# ifdef HAVE_FILTER_POWERPC + case LZMA_FILTER_POWERPC: +# endif +# ifdef HAVE_FILTER_IA64 + case LZMA_FILTER_IA64: +# endif +# ifdef HAVE_FILTER_ARM + case LZMA_FILTER_ARM: +# endif +# ifdef HAVE_FILTER_ARMTHUMB + case LZMA_FILTER_ARMTHUMB: +# endif +# ifdef HAVE_FILTER_SPARC + case LZMA_FILTER_SPARC: +# endif + if (options->options == NULL || ((const lzma_options_simple *)( + options->options))->start_offset == 0) + *size = 0; + else + *size = 4; + break; +#endif + +#ifdef HAVE_FILTER_DELTA + case LZMA_FILTER_DELTA: + *size = 1; + break; +#endif + +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: + *size = 2; + break; +#endif + + default: + // Unknown filter - if the Filter ID is a proper VLI, + // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR, + // because it's possible that we just don't have support + // compiled in for the requested filter. + ret = options->id <= LZMA_VLI_VALUE_MAX + ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR; + break; + } + + return ret; +} + + +extern LZMA_API lzma_ret +lzma_filter_flags_size(uint32_t *size, const lzma_options_filter *options) +{ + // Get size of Filter Properties. + uint32_t prop_size; + const lzma_ret ret = get_properties_size(&prop_size, options); + if (ret != LZMA_OK) + return ret; + + // Size of Filter ID field if it exists. + size_t id_size; + size_t prop_size_size; + if (options->id < 0xE0 + && (lzma_vli)(prop_size) == options->id / 0x20) { + // ID and Size of Filter Properties fit into Misc. + id_size = 0; + prop_size_size = 0; + + } else { + // At least Filter ID is stored using the External ID field. + id_size = lzma_vli_size(options->id); + if (id_size == 0) + return LZMA_PROG_ERROR; + + if (prop_size <= 30) { + // Size of Filter Properties fits into Misc still. + prop_size_size = 0; + } else { + // The Size of Filter Properties field is used too. + prop_size_size = lzma_vli_size(prop_size); + if (prop_size_size == 0) + return LZMA_PROG_ERROR; + } + } + + // 1 is for the Misc field. + *size = 1 + id_size + prop_size_size + prop_size; + + return LZMA_OK; +} + + +#ifdef HAVE_FILTER_SIMPLE +/// Encodes Filter Properties of the so called simple filters +static lzma_ret +properties_simple(uint8_t *out, size_t *out_pos, size_t out_size, + const lzma_options_simple *options) +{ + if (options == NULL || options->start_offset == 0) + return LZMA_OK; + + if (out_size - *out_pos < 4) + return LZMA_BUF_ERROR; + + for (size_t i = 0; i < 4; ++i) + out[(*out_pos)++] = options->start_offset >> (i * 8); + + return LZMA_OK; +} +#endif + + +#ifdef HAVE_FILTER_DELTA +/// Encodes Filter Properties of the Delta filter +static lzma_ret +properties_delta(uint8_t *out, size_t *out_pos, size_t out_size, + const lzma_options_delta *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + // It's possible that newer liblzma versions will support larger + // distance values. + if (options->distance < LZMA_DELTA_DISTANCE_MIN + || options->distance > LZMA_DELTA_DISTANCE_MAX) + return LZMA_HEADER_ERROR; + + if (out_size - *out_pos < 1) + return LZMA_BUF_ERROR; + + out[*out_pos] = options->distance - LZMA_DELTA_DISTANCE_MIN; + ++*out_pos; + + return LZMA_OK; +} +#endif + + +#ifdef HAVE_FILTER_LZMA +/// Encodes LZMA Properties and Dictionary Flags (two bytes) +static lzma_ret +properties_lzma(uint8_t *out, size_t *out_pos, size_t out_size, + const lzma_options_lzma *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (out_size - *out_pos < 2) + return LZMA_BUF_ERROR; + + // LZMA Properties + if (lzma_lzma_encode_properties(options, out + *out_pos)) + return LZMA_HEADER_ERROR; + + ++*out_pos; + + // Dictionary flags + // + // Dictionary size is encoded using six bits of + // which one is mantissa and five are exponent. + // + // There are some limits that must hold to keep + // this coding working. +# if LZMA_DICTIONARY_SIZE_MAX > UINT32_MAX / 2 +# error LZMA_DICTIONARY_SIZE_MAX is too big. +# endif +# if LZMA_DICTIONARY_SIZE_MIN < 1 +# error LZMA_DICTIONARY_SIZE_MIN cannot be zero. +# endif + + // Validate it: + if (options->dictionary_size < LZMA_DICTIONARY_SIZE_MIN + || options->dictionary_size > LZMA_DICTIONARY_SIZE_MAX) + return LZMA_HEADER_ERROR; + + if (options->dictionary_size == 1) { + // Special case + out[*out_pos] = 0x00; + } else { + // TODO This could be more elegant. + uint32_t i = 1; + while (((2 | ((i + 1) & 1)) << ((i - 1) / 2)) + < options->dictionary_size) + ++i; + out[*out_pos] = i; + } + + ++*out_pos; + + return LZMA_OK; +} +#endif + + +extern LZMA_API lzma_ret +lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, size_t out_size, + const lzma_options_filter *options) +{ + // Minimum output is one byte (everything fits into Misc). + // The caller should have checked that there is enough output space, + // so we return LZMA_PROG_ERROR instead of LZMA_BUF_ERROR. + if (*out_pos >= out_size) + return LZMA_PROG_ERROR; + + // Get size of Filter Properties. + uint32_t prop_size; + lzma_ret ret = get_properties_size(&prop_size, options); + if (ret != LZMA_OK) + return ret; + + // Misc, External ID, and Size of Properties + if (options->id < 0xE0 + && (lzma_vli)(prop_size) == options->id / 0x20) { + // ID and Size of Filter Properties fit into Misc. + out[*out_pos] = options->id; + ++*out_pos; + + } else if (prop_size <= 30) { + // Size of Filter Properties fits into Misc. + out[*out_pos] = prop_size + 0xE0; + ++*out_pos; + + // External ID is used to encode the Filter ID. If encoding + // the VLI fails, it's because the caller has given as too + // little output space, which it should have checked already. + // So return LZMA_PROG_ERROR, not LZMA_BUF_ERROR. + size_t dummy = 0; + if (lzma_vli_encode(options->id, &dummy, 1, + out, out_pos, out_size) != LZMA_STREAM_END) + return LZMA_PROG_ERROR; + + } else { + // Nothing fits into Misc. + out[*out_pos] = 0xFF; + ++*out_pos; + + // External ID is used to encode the Filter ID. + size_t dummy = 0; + if (lzma_vli_encode(options->id, &dummy, 1, + out, out_pos, out_size) != LZMA_STREAM_END) + return LZMA_PROG_ERROR; + + // External Size of Filter Properties + dummy = 0; + if (lzma_vli_encode(prop_size, &dummy, 1, + out, out_pos, out_size) != LZMA_STREAM_END) + return LZMA_PROG_ERROR; + } + + // Filter Properties + switch (options->id) { +#ifdef HAVE_FILTER_COPY + case LZMA_FILTER_COPY: + assert(prop_size == 0); + ret = options->options == NULL ? LZMA_OK : LZMA_HEADER_ERROR; + break; +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK: + assert(prop_size == 0); + ret = LZMA_OK; + break; +#endif + +#ifdef HAVE_FILTER_SIMPLE +# ifdef HAVE_FILTER_X86 + case LZMA_FILTER_X86: +# endif +# ifdef HAVE_FILTER_POWERPC + case LZMA_FILTER_POWERPC: +# endif +# ifdef HAVE_FILTER_IA64 + case LZMA_FILTER_IA64: +# endif +# ifdef HAVE_FILTER_ARM + case LZMA_FILTER_ARM: +# endif +# ifdef HAVE_FILTER_ARMTHUMB + case LZMA_FILTER_ARMTHUMB: +# endif +# ifdef HAVE_FILTER_SPARC + case LZMA_FILTER_SPARC: +# endif + ret = properties_simple(out, out_pos, out_size, + options->options); + break; +#endif + +#ifdef HAVE_FILTER_DELTA + case LZMA_FILTER_DELTA: + ret = properties_delta(out, out_pos, out_size, + options->options); + break; +#endif + +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: + ret = properties_lzma(out, out_pos, out_size, + options->options); + break; +#endif + + default: + assert(0); + ret = LZMA_PROG_ERROR; + break; + } + + return ret; +} diff --git a/src/liblzma/common/index.c b/src/liblzma/common/index.c new file mode 100644 index 00000000..6816b37a --- /dev/null +++ b/src/liblzma/common/index.c @@ -0,0 +1,140 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index.c +/// \brief Handling of Index in Metadata +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/** + * \brief Duplicates an Index list + * + * \return A copy of the Index list, or NULL if memory allocation + * failed or the original Index was empty. + */ +extern LZMA_API lzma_index * +lzma_index_dup(const lzma_index *old_current, lzma_allocator *allocator) +{ + lzma_index *new_head = NULL; + lzma_index *new_current = NULL; + + while (old_current != NULL) { + lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); + if (i == NULL) { + lzma_index_free(new_head, allocator); + return NULL; + } + + i->total_size = old_current->total_size; + i->uncompressed_size = old_current->uncompressed_size; + i->next = NULL; + + if (new_head == NULL) + new_head = i; + else + new_current->next = i; + + new_current = i; + old_current = old_current->next; + } + + return new_head; +} + + +/** + * \brief Frees an Index list + * + * All Index Recors in the list are freed. This function is convenient when + * getting rid of lzma_metadata structures containing an Index. + */ +extern LZMA_API void +lzma_index_free(lzma_index *i, lzma_allocator *allocator) +{ + while (i != NULL) { + lzma_index *tmp = i->next; + lzma_free(i, allocator); + i = tmp; + } + + return; +} + + +/** + * \brief Calculates properties of an Index list + * + * + */ +extern LZMA_API lzma_ret +lzma_index_count(const lzma_index *i, size_t *count, + lzma_vli *lzma_restrict total_size, + lzma_vli *lzma_restrict uncompressed_size) +{ + *count = 0; + *total_size = 0; + *uncompressed_size = 0; + + while (i != NULL) { + if (i->total_size == LZMA_VLI_VALUE_UNKNOWN) { + *total_size = LZMA_VLI_VALUE_UNKNOWN; + } else if (i->total_size > LZMA_VLI_VALUE_MAX) { + return LZMA_PROG_ERROR; + } else if (*total_size != LZMA_VLI_VALUE_UNKNOWN) { + *total_size += i->total_size; + if (*total_size > LZMA_VLI_VALUE_MAX) + return LZMA_PROG_ERROR; + } + + if (i->uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) { + *uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + } else if (i->uncompressed_size > LZMA_VLI_VALUE_MAX) { + return LZMA_PROG_ERROR; + } else if (*uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + *uncompressed_size += i->uncompressed_size; + if (*uncompressed_size > LZMA_VLI_VALUE_MAX) + return LZMA_PROG_ERROR; + } + + ++*count; + i = i->next; + } + + // FIXME ? + if (*total_size == LZMA_VLI_VALUE_UNKNOWN + || *uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_HEADER_ERROR; + + return LZMA_OK; +} + + + +extern LZMA_API lzma_bool +lzma_index_is_equal(const lzma_index *a, const lzma_index *b) +{ + while (a != NULL && b != NULL) { + if (a->total_size != b->total_size || a->uncompressed_size + != b->uncompressed_size) + return false; + + a = a->next; + b = b->next; + } + + return a == b; +} diff --git a/src/liblzma/common/info.c b/src/liblzma/common/info.c new file mode 100644 index 00000000..2a59a029 --- /dev/null +++ b/src/liblzma/common/info.c @@ -0,0 +1,823 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file info.c +/// \brief Collects and verifies integrity of Stream size information +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +struct lzma_info_s { + struct { + /// Known Size of Header Metadata Block; here's some + /// special things: + /// - LZMA_VLI_VALUE_UNKNOWN indicates that we don't know + /// if Header Metadata Block is present. + /// - 0 indicates that Header Metadata Block is not present. + lzma_vli header_metadata_size; + + /// Known Total Size of the Data Blocks in the Stream + lzma_vli total_size; + + /// Known Uncompressed Size of the Data Blocks in the Stream + lzma_vli uncompressed_size; + + /// Known Size of Footer Metadata Block + lzma_vli footer_metadata_size; + } known; + + struct { + /// Sum of Total Size fields stored to the Index so far + lzma_vli total_size; + + /// Sum of Uncompressed Size fields stored to the Index so far + lzma_vli uncompressed_size; + + /// First Index Record in the list, or NULL if Index is empty. + lzma_index *head; + + /// Number of Index Records + size_t record_count; + + /// Number of Index Records + size_t incomplete_count; + + /// True when we know that no more Records will get added + /// to the Index. + bool is_final; + } index; + + /// Start offset of the Stream. This is needed to calculate + /// lzma_info_iter.stream_offset. + lzma_vli stream_start_offset; + + /// True if Index is present in Header Metadata Block + bool has_index_in_header_metadata; +}; + + +////////////////////// +// Create/Reset/End // +////////////////////// + +static void +index_init(lzma_info *info) +{ + info->index.total_size = 0; + info->index.uncompressed_size = 0; + info->index.head = NULL; + info->index.record_count = 0; + info->index.incomplete_count = 0; + info->index.is_final = false; + return; +} + + +static void +info_init(lzma_info *info) +{ + info->known.header_metadata_size = LZMA_VLI_VALUE_UNKNOWN; + info->known.total_size = LZMA_VLI_VALUE_UNKNOWN; + info->known.uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + info->known.footer_metadata_size = LZMA_VLI_VALUE_UNKNOWN; + info->stream_start_offset = 0; + info->has_index_in_header_metadata = false; + + index_init(info); + + return; +} + + +extern LZMA_API lzma_info * +lzma_info_init(lzma_info *info, lzma_allocator *allocator) +{ + if (info == NULL) + info = lzma_alloc(sizeof(lzma_info), allocator); + else + lzma_index_free(info->index.head, allocator); + + if (info != NULL) + info_init(info); + + return info; +} + + +extern LZMA_API void +lzma_info_free(lzma_info *info, lzma_allocator *allocator) +{ + lzma_index_free(info->index.head, allocator); + lzma_free(info, allocator); + return; +} + + +///////// +// Set // +///////// + +static lzma_ret +set_size(lzma_vli new_size, lzma_vli *known_size, lzma_vli index_size, + bool forbid_zero) +{ + assert(new_size <= LZMA_VLI_VALUE_MAX); + + lzma_ret ret = LZMA_OK; + + if (forbid_zero && new_size == 0) + ret = LZMA_PROG_ERROR; + else if (index_size > new_size) + ret = LZMA_DATA_ERROR; + else if (*known_size == LZMA_VLI_VALUE_UNKNOWN) + *known_size = new_size; + else if (*known_size != new_size) + ret = LZMA_DATA_ERROR; + + return ret; +} + + +extern LZMA_API lzma_ret +lzma_info_size_set(lzma_info *info, lzma_info_size type, lzma_vli size) +{ + if (size > LZMA_VLI_VALUE_MAX) + return LZMA_PROG_ERROR; + + switch (type) { + case LZMA_INFO_STREAM_START: + info->stream_start_offset = size; + return LZMA_OK; + + case LZMA_INFO_HEADER_METADATA: + return set_size(size, &info->known.header_metadata_size, + 0, false); + + case LZMA_INFO_TOTAL: + return set_size(size, &info->known.total_size, + info->index.total_size, true); + + case LZMA_INFO_UNCOMPRESSED: + return set_size(size, &info->known.uncompressed_size, + info->index.uncompressed_size, false); + + case LZMA_INFO_FOOTER_METADATA: + return set_size(size, &info->known.footer_metadata_size, + 0, true); + } + + return LZMA_PROG_ERROR; +} + + +extern LZMA_API lzma_ret +lzma_info_index_set(lzma_info *info, lzma_allocator *allocator, + lzma_index *i_new, lzma_bool eat_index) +{ + if (i_new == NULL) + return LZMA_PROG_ERROR; + + lzma_index *i_old = info->index.head; + + if (i_old != NULL) { + while (true) { + // If the new Index has fewer Records than the old one, + // the new Index cannot be valid. + if (i_new == NULL) + return LZMA_DATA_ERROR; + + // The new Index must be complete i.e. no unknown + // values. + if (i_new->total_size > LZMA_VLI_VALUE_MAX + || i_new->uncompressed_size + > LZMA_VLI_VALUE_MAX) { + if (eat_index) + lzma_index_free(i_new, allocator); + + return LZMA_PROG_ERROR; + } + + // Compare the values from the new Index with the old + // Index. The old Index may be incomplete; in that + // case we + // - use the value from the new Index as is; + // - update the appropriate info->index.foo_size; and + // - decrease the count of incomplete Index Records. + bool was_incomplete = false; + + if (i_old->total_size == LZMA_VLI_VALUE_UNKNOWN) { + assert(!info->index.is_final); + was_incomplete = true; + + i_old->total_size = i_new->total_size; + + if (lzma_vli_add(info->index.total_size, + i_new->total_size)) { + if (eat_index) + lzma_index_free(i_new, + allocator); + + return LZMA_PROG_ERROR; + } + } else if (i_old->total_size != i_new->total_size) { + if (eat_index) + lzma_index_free(i_new, allocator); + + return LZMA_DATA_ERROR; + } + + if (i_old->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN) { + assert(!info->index.is_final); + was_incomplete = true; + + i_old->uncompressed_size + = i_new->uncompressed_size; + + if (lzma_vli_add(info->index.uncompressed_size, + i_new->uncompressed_size)) { + if (eat_index) + lzma_index_free(i_new, + allocator); + + return LZMA_PROG_ERROR; + } + } else if (i_old->uncompressed_size + != i_new->uncompressed_size) { + if (eat_index) + lzma_index_free(i_new, allocator); + + return LZMA_DATA_ERROR; + } + + if (was_incomplete) { + assert(!info->index.is_final); + assert(info->index.incomplete_count > 0); + --info->index.incomplete_count; + } + + // Get rid of *i_new. It's now identical with *i_old. + lzma_index *tmp = i_new->next; + if (eat_index) + lzma_free(i_new, allocator); + + i_new = tmp; + + // We want to leave i_old pointing to the last + // Index Record in the old Index. This way we can + // concatenate the possible new Records from i_new. + if (i_old->next == NULL) + break; + + i_old = i_old->next; + } + } + + assert(info->index.incomplete_count == 0); + + // If Index was already known to be final, i_new must be NULL now. + // The new Index cannot contain more Records that we already have. + if (info->index.is_final) { + assert(info->index.head != NULL); + + if (i_new != NULL) { + if (eat_index) + lzma_index_free(i_new, allocator); + + return LZMA_DATA_ERROR; + } + + return LZMA_OK; + } + + // The rest of the new Index is merged to the old Index. Keep the + // current i_new pointer in available. We need it when merging the + // new Index with the old one, and if an error occurs so we can + // get rid of the broken part of the new Index. + lzma_index *i_start = i_new; + while (i_new != NULL) { + // The new Index must be complete i.e. no unknown values. + if (i_new->total_size > LZMA_VLI_VALUE_MAX + || i_new->uncompressed_size + > LZMA_VLI_VALUE_MAX) { + if (eat_index) + lzma_index_free(i_start, allocator); + + return LZMA_PROG_ERROR; + } + + // Update info->index.foo_sizes. + if (lzma_vli_add(info->index.total_size, i_new->total_size) + || lzma_vli_add(info->index.uncompressed_size, + i_new->uncompressed_size)) { + if (eat_index) + lzma_index_free(i_start, allocator); + + return LZMA_PROG_ERROR; + } + + ++info->index.record_count; + i_new = i_new->next; + } + + // All the Records in the new Index are good, and info->index.foo_sizes + // were successfully updated. + if (lzma_info_index_finish(info) != LZMA_OK) { + if (eat_index) + lzma_index_free(i_start, allocator); + + return LZMA_DATA_ERROR; + } + + // The Index is ready to be merged. If we aren't supposed to eat + // the Index, make a copy of it first. + if (!eat_index && i_start != NULL) { + i_start = lzma_index_dup(i_start, allocator); + if (i_start == NULL) + return LZMA_MEM_ERROR; + } + + // Concatenate the new Index with the old one. Note that it is + // possible that we don't have any old Index. + if (info->index.head == NULL) + info->index.head = i_start; + else + i_old->next = i_start; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_info_metadata_set(lzma_info *info, lzma_allocator *allocator, + lzma_metadata *metadata, lzma_bool is_header_metadata, + lzma_bool eat_index) +{ + // Validate *metadata. + if (!lzma_vli_is_valid(metadata->header_metadata_size) + || !lzma_vli_is_valid(metadata->total_size) + || !lzma_vli_is_valid(metadata->uncompressed_size)) { + if (eat_index) { + lzma_index_free(metadata->index, allocator); + metadata->index = NULL; + } + + return LZMA_PROG_ERROR; + } + + // Index + if (metadata->index != NULL) { + if (is_header_metadata) + info->has_index_in_header_metadata = true; + + const lzma_ret ret = lzma_info_index_set( + info, allocator, metadata->index, eat_index); + if (ret != LZMA_OK) + return ret; + + } else if (!is_header_metadata + && (metadata->total_size == LZMA_VLI_VALUE_UNKNOWN + || !info->has_index_in_header_metadata)) { + // Either Total Size or Index must be present in Footer + // Metadata Block. If Index is not present, it must have + // already been in the Header Metadata Block. Since we + // got here, these conditions weren't met. + return LZMA_DATA_ERROR; + } + + // Size of Header Metadata + if (!is_header_metadata) { + // If it is marked unknown in Metadata, it means that + // it's not present. + const lzma_vli size = metadata->header_metadata_size + != LZMA_VLI_VALUE_UNKNOWN + ? metadata->header_metadata_size : 0; + const lzma_ret ret = lzma_info_size_set( + info, LZMA_INFO_HEADER_METADATA, size); + if (ret != LZMA_OK) + return ret; + } + + // Total Size + if (metadata->total_size != LZMA_VLI_VALUE_UNKNOWN) { + const lzma_ret ret = lzma_info_size_set(info, + LZMA_INFO_TOTAL, metadata->total_size); + if (ret != LZMA_OK) + return ret; + } + + // Uncompressed Size + if (metadata->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + const lzma_ret ret = lzma_info_size_set(info, + LZMA_INFO_UNCOMPRESSED, + metadata->uncompressed_size); + if (ret != LZMA_OK) + return ret; + } + + return LZMA_OK; +} + + +///////// +// Get // +///////// + +extern LZMA_API lzma_vli +lzma_info_size_get(const lzma_info *info, lzma_info_size type) +{ + switch (type) { + case LZMA_INFO_STREAM_START: + return info->stream_start_offset; + + case LZMA_INFO_HEADER_METADATA: + return info->known.header_metadata_size; + + case LZMA_INFO_TOTAL: + return info->known.total_size; + + case LZMA_INFO_UNCOMPRESSED: + return info->known.uncompressed_size; + + case LZMA_INFO_FOOTER_METADATA: + return info->known.footer_metadata_size; + } + + return LZMA_VLI_VALUE_UNKNOWN; +} + + +extern LZMA_API lzma_index * +lzma_info_index_get(lzma_info *info, lzma_bool detach) +{ + lzma_index *i = info->index.head; + + if (detach) + index_init(info); + + return i; +} + + +extern LZMA_API size_t +lzma_info_index_count_get(const lzma_info *info) +{ + return info->index.record_count; +} + + +///////////////// +// Incremental // +///////////////// + +enum { + ITER_INFO, + ITER_INDEX, + ITER_RESERVED_1, + ITER_RESERVED_2, +}; + + +#define iter_info ((lzma_info *)(iter->internal[ITER_INFO])) + +#define iter_index ((lzma_index *)(iter->internal[ITER_INDEX])) + + +extern LZMA_API void +lzma_info_iter_begin(lzma_info *info, lzma_info_iter *iter) +{ + *iter = (lzma_info_iter){ + .total_size = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_size = LZMA_VLI_VALUE_UNKNOWN, + .stream_offset = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_offset = LZMA_VLI_VALUE_UNKNOWN, + .internal = { info, NULL, NULL, NULL }, + }; + + return; +} + + +extern LZMA_API lzma_ret +lzma_info_iter_next(lzma_info_iter *iter, lzma_allocator *allocator) +{ + // FIXME debug remove + lzma_info *info = iter_info; + (void)info; + + if (iter_index == NULL) { + // The first call after lzma_info_iter_begin(). + if (iter_info->known.header_metadata_size + == LZMA_VLI_VALUE_UNKNOWN) + iter->stream_offset = LZMA_VLI_VALUE_UNKNOWN; + else if (lzma_vli_sum3(iter->stream_offset, + iter_info->stream_start_offset, + LZMA_STREAM_HEADER_SIZE, + iter_info->known.header_metadata_size)) + return LZMA_PROG_ERROR; + + iter->uncompressed_offset = 0; + + if (iter_info->index.head != NULL) { + // The first Index Record has already been allocated. + iter->internal[ITER_INDEX] = iter_info->index.head; + iter->total_size = iter_index->total_size; + iter->uncompressed_size + = iter_index->uncompressed_size; + return LZMA_OK; + } + } else { + // Update iter->*_offsets. + if (iter->stream_offset != LZMA_VLI_VALUE_UNKNOWN) { + if (iter_index->total_size == LZMA_VLI_VALUE_UNKNOWN) + iter->stream_offset = LZMA_VLI_VALUE_UNKNOWN; + else if (lzma_vli_add(iter->stream_offset, + iter_index->total_size)) + return LZMA_DATA_ERROR; + } + + if (iter->uncompressed_offset != LZMA_VLI_VALUE_UNKNOWN) { + if (iter_index->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN) + iter->uncompressed_offset + = LZMA_VLI_VALUE_UNKNOWN; + else if (lzma_vli_add(iter->uncompressed_offset, + iter_index->uncompressed_size)) + return LZMA_DATA_ERROR; + } + + if (iter_index->next != NULL) { + // The next Record has already been allocated. + iter->internal[ITER_INDEX] = iter_index->next; + iter->total_size = iter_index->total_size; + iter->uncompressed_size + = iter_index->uncompressed_size; + return LZMA_OK; + } + } + + // Don't add new Records to a final Index. + if (iter_info->index.is_final) + return LZMA_DATA_ERROR; + + // Allocate and initialize a new Index Record. + lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); + if (i == NULL) + return LZMA_MEM_ERROR; + + i->total_size = LZMA_VLI_VALUE_UNKNOWN; + i->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + i->next = NULL; + + iter->total_size = LZMA_VLI_VALUE_UNKNOWN; + iter->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + + // Decide where to put the new Index Record. + if (iter_info->index.head == NULL) + iter_info->index.head = i; + + if (iter_index != NULL) + iter_index->next = i; + + iter->internal[ITER_INDEX] = i; + + ++iter_info->index.record_count; + ++iter_info->index.incomplete_count; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_info_iter_set(lzma_info_iter *iter, + lzma_vli total_size, lzma_vli uncompressed_size) +{ + // FIXME debug remove + lzma_info *info = iter_info; + (void)info; + + if (iter_index == NULL || !lzma_vli_is_valid(total_size) + || !lzma_vli_is_valid(uncompressed_size)) + return LZMA_PROG_ERROR; + + const bool was_incomplete = iter_index->total_size + == LZMA_VLI_VALUE_UNKNOWN + || iter_index->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN; + + if (total_size != LZMA_VLI_VALUE_UNKNOWN) { + if (iter_index->total_size == LZMA_VLI_VALUE_UNKNOWN) { + iter_index->total_size = total_size; + + if (lzma_vli_add(iter_info->index.total_size, + total_size) + || iter_info->index.total_size + > iter_info->known.total_size) + return LZMA_DATA_ERROR; + + } else if (iter_index->total_size != total_size) { + return LZMA_DATA_ERROR; + } + } + + if (uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + if (iter_index->uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) { + iter_index->uncompressed_size = uncompressed_size; + + if (lzma_vli_add(iter_info->index.uncompressed_size, + uncompressed_size) + || iter_info->index.uncompressed_size + > iter_info->known.uncompressed_size) + return LZMA_DATA_ERROR; + + } else if (iter_index->uncompressed_size + != uncompressed_size) { + return LZMA_DATA_ERROR; + } + } + + // Check if the new information we got managed to finish this + // Index Record. If so, update the count of incomplete Index Records. + if (was_incomplete && iter_index->total_size + != LZMA_VLI_VALUE_UNKNOWN + && iter_index->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + assert(iter_info->index.incomplete_count > 0); + --iter_info->index.incomplete_count; + } + + // Make sure that the known sizes are now available in *iter. + iter->total_size = iter_index->total_size; + iter->uncompressed_size = iter_index->uncompressed_size; + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_info_index_finish(lzma_info *info) +{ + if (info->index.record_count == 0 || info->index.incomplete_count > 0 + || lzma_info_size_set(info, LZMA_INFO_TOTAL, + info->index.total_size) + || lzma_info_size_set(info, LZMA_INFO_UNCOMPRESSED, + info->index.uncompressed_size)) + return LZMA_DATA_ERROR; + + info->index.is_final = true; + + return LZMA_OK; +} + + +////////////// +// Locating // +////////////// + +extern LZMA_API lzma_vli +lzma_info_metadata_locate(const lzma_info *info, lzma_bool is_header_metadata) +{ + bool error = false; + lzma_vli size = 0; + + if (info->known.header_metadata_size == LZMA_VLI_VALUE_UNKNOWN) { + // We don't know if Header Metadata Block is present, thus + // we cannot locate it either. + // + // Well, you could say that just assume that it is present. + // I'm not sure if this is useful. But it can be useful to + // be able to use this function and get LZMA_VLI_VALUE_UNKNOWN + // to detect that Header Metadata Block wasn't present. + error = true; + } else if (is_header_metadata) { + error = lzma_vli_sum(size, info->stream_start_offset, + LZMA_STREAM_HEADER_SIZE); + } else if (!info->index.is_final) { + // Since we don't know if we have all the Index Records yet, + // we cannot know where the Footer Metadata Block is. + error = true; + } else { + error = lzma_vli_sum4(size, info->stream_start_offset, + LZMA_STREAM_HEADER_SIZE, + info->known.header_metadata_size, + info->known.total_size); + } + + return error ? LZMA_VLI_VALUE_UNKNOWN : size; +} + + +extern LZMA_API uint32_t +lzma_info_metadata_alignment_get( + const lzma_info *info, lzma_bool is_header_metadata) +{ + uint32_t alignment; + + if (is_header_metadata) { + alignment = info->stream_start_offset + + LZMA_STREAM_HEADER_SIZE; + } else { + alignment = info->stream_start_offset + LZMA_STREAM_HEADER_SIZE + + info->known.header_metadata_size + + info->known.total_size; + } + + return alignment; +} + + +extern LZMA_API lzma_ret +lzma_info_iter_locate(lzma_info_iter *iter, lzma_allocator *allocator, + lzma_vli uncompressed_offset, lzma_bool allow_alloc) +{ + if (iter == NULL || uncompressed_offset > LZMA_VLI_VALUE_MAX) + return LZMA_PROG_ERROR; + + // Quick check in case Index is final. + if (iter_info->index.is_final) { + assert(iter_info->known.uncompressed_size + == iter_info->index.uncompressed_size); + if (uncompressed_offset >= iter_info->index.uncompressed_size) + return LZMA_DATA_ERROR; + } + + // TODO: Optimize so that it uses existing info from *iter when + // seeking forward. + + // Initialize *iter + if (iter_info->known.header_metadata_size != LZMA_VLI_VALUE_UNKNOWN) { + if (lzma_vli_sum3(iter->stream_offset, + iter_info->stream_start_offset, + LZMA_STREAM_HEADER_SIZE, + iter_info->known.header_metadata_size)) + return LZMA_PROG_ERROR; + } else { + // We don't know the Size of Header Metadata Block, thus + // we cannot calculate the Stream offset either. + iter->stream_offset = LZMA_VLI_VALUE_UNKNOWN; + } + + iter->uncompressed_offset = 0; + + // If we have no Index Records, it's obvious that we need to + // add a new one. + if (iter_info->index.head == NULL) { + assert(!iter_info->index.is_final); + if (!allow_alloc) + return LZMA_DATA_ERROR; + + return lzma_info_iter_next(iter, allocator); + } + + // Locate an appropriate Index Record. + lzma_index *i = iter_info->index.head; + while (true) { + // - If Uncompressed Size in the Record is unknown, + // we have no chance to search further. + // - If the next Record would go past the requested offset, + // we have found our target Data Block. + if (i->uncompressed_size == LZMA_VLI_VALUE_UNKNOWN + || iter->uncompressed_offset + + i->uncompressed_size > uncompressed_offset) { + iter->total_size = i->total_size; + iter->uncompressed_size = i->uncompressed_size; + iter->internal[ITER_INDEX] = i; + return LZMA_OK; + } + + // Update the stream offset. It may be unknown if we didn't + // know the size of Header Metadata Block. + if (iter->stream_offset != LZMA_VLI_VALUE_UNKNOWN) + if (lzma_vli_add(iter->stream_offset, i->total_size)) + return LZMA_PROG_ERROR; + + // Update the uncompressed offset. This cannot overflow since + // the Index is known to be valid. + iter->uncompressed_offset += i->uncompressed_size; + + // Move to the next Block. + if (i->next == NULL) { + assert(!iter_info->index.is_final); + if (!allow_alloc) + return LZMA_DATA_ERROR; + + iter->internal[ITER_INDEX] = i; + return lzma_info_iter_next(iter, allocator); + } + + i = i->next; + } +} diff --git a/src/liblzma/common/init.c b/src/liblzma/common/init.c new file mode 100644 index 00000000..fb377f5a --- /dev/null +++ b/src/liblzma/common/init.c @@ -0,0 +1,39 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file init.c +/// \brief Static internal initializations +/// +/// The initializations have been splitted to so many small files to prevent +/// an application needing only decoder functions from statically linking +/// also the encoder functions. +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API void +lzma_init(void) +{ +#ifdef HAVE_ENCODER + lzma_init_encoder(); +#endif + +#ifdef HAVE_DECODER + lzma_init_decoder(); +#endif + + return; +} diff --git a/src/liblzma/common/init_decoder.c b/src/liblzma/common/init_decoder.c new file mode 100644 index 00000000..2d61b451 --- /dev/null +++ b/src/liblzma/common/init_decoder.c @@ -0,0 +1,33 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file init_decoder.c +/// \brief Static internal initializations +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API void +lzma_init_decoder(void) +{ + // So far there's no decoder-specific stuff to initialize. + +#ifdef HAVE_CHECK + lzma_init_check(); +#endif + + return; +} diff --git a/src/liblzma/common/init_encoder.c b/src/liblzma/common/init_encoder.c new file mode 100644 index 00000000..4d3da506 --- /dev/null +++ b/src/liblzma/common/init_encoder.c @@ -0,0 +1,44 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file init_encoder.c +/// \brief Static internal initializations +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "range_encoder.h" +#include "lzma_encoder.h" + + +extern LZMA_API void +lzma_init_encoder(void) +{ + static bool already_initialized = false; + if (already_initialized) + return; + +#ifdef HAVE_CHECK + lzma_init_check(); +#endif + +// FIXME TODO Create precalculated tables. +#if defined(HAVE_ENCODER) && defined(HAVE_FILTER_LZMA) + lzma_rc_init(); + lzma_fastpos_init(); +#endif + + already_initialized = true; + return; +} diff --git a/src/liblzma/common/memory_limitter.c b/src/liblzma/common/memory_limitter.c new file mode 100644 index 00000000..19cdefc2 --- /dev/null +++ b/src/liblzma/common/memory_limitter.c @@ -0,0 +1,200 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memory_limitter.c +/// \brief Limitting memory usage +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/// Rounds an unsigned integer upwards to the next multiple. +#define my_ceil(num, multiple) \ + ((num) + (((multiple) - ((num) % (multiple))) % (multiple))) + + +/// Rounds upwards to the next multiple of 2 * sizeof(void*). +/// malloc() tends to align allocations this way. +#define malloc_ceil(num) my_ceil(num, 2 * sizeof(void *)) + + +typedef struct lzma_memlimit_list_s lzma_memlimit_list; +struct lzma_memlimit_list_s { + lzma_memlimit_list *next; + void *ptr; + size_t size; +}; + + +struct lzma_memlimit_s { + size_t used; + size_t limit; + lzma_memlimit_list *list; +}; + + +extern LZMA_API lzma_memlimit * +lzma_memlimit_create(size_t limit) +{ + if (limit < sizeof(lzma_memlimit)) + return NULL; + + lzma_memlimit *mem = malloc(sizeof(lzma_memlimit)); + + if (mem != NULL) { + mem->used = sizeof(lzma_memlimit); + mem->limit = limit; + mem->list = NULL; + } + + return mem; +} + + +extern LZMA_API void +lzma_memlimit_set(lzma_memlimit *mem, size_t limit) +{ + mem->limit = limit; + return; +} + + +extern LZMA_API size_t +lzma_memlimit_get(const lzma_memlimit *mem) +{ + return mem->limit; +} + + +extern LZMA_API size_t +lzma_memlimit_used(const lzma_memlimit *mem) +{ + return mem->used; +} + + +extern LZMA_API void +lzma_memlimit_end(lzma_memlimit *mem, lzma_bool free_allocated) +{ + if (mem == NULL) + return; + + lzma_memlimit_list *record = mem->list; + while (record != NULL) { + if (free_allocated) + free(record->ptr); + + lzma_memlimit_list *tmp = record; + record = record->next; + free(tmp); + } + + free(mem); + + return; +} + + +extern LZMA_API void * +lzma_memlimit_alloc(lzma_memlimit *mem, size_t nmemb, size_t size) +{ + // While liblzma always sets nmemb to one, do this multiplication + // to make these functions usable e.g. with zlib and libbzip2. + // Making sure that this doesn't overflow is up to the application. + size *= nmemb; + + // Some malloc() implementations return NULL on malloc(0). We like + // to get a non-NULL value. + if (size == 0) + size = 1; + + // Calculate how much memory we are going to allocate in reality. + // TODO: We should add some rough estimate how much malloc() needs + // for its internal structures. + const size_t total_size = malloc_ceil(size) + + malloc_ceil(sizeof(lzma_memlimit_list)); + + // Integer overflow protection + if (SIZE_MAX - size <= total_size) + return NULL; + + if (mem->limit < mem->used || mem->limit - mem->used < total_size) + return NULL; + + lzma_memlimit_list *record = malloc(sizeof(lzma_memlimit_list)); + void *ptr = malloc(size); + + if (record == NULL || ptr == NULL) { + free(record); + free(ptr); + return NULL; + } + + // Add the new entry to the beginning of the list. This should be + // more efficient when freeing memory, because usually it is + // "last allocated, first freed". + record->next = mem->list; + record->ptr = ptr; + record->size = total_size; + + mem->list = record; + mem->used += total_size; + + return ptr; +} + + +extern LZMA_API void +lzma_memlimit_detach(lzma_memlimit *mem, void *ptr) +{ + if (ptr == NULL || mem->list == NULL) + return; + + lzma_memlimit_list *record = mem->list; + lzma_memlimit_list *prev = NULL; + + while (record->ptr != ptr) { + prev = record; + record = record->next; + if (record == NULL) + return; + } + + if (prev != NULL) + prev->next = record->next; + else + mem->list = record->next; + + assert(mem->used >= record->size); + mem->used -= record->size; + + free(record); + + return; +} + + +extern LZMA_API void +lzma_memlimit_free(lzma_memlimit *mem, void *ptr) +{ + if (ptr == NULL) + return; + + lzma_memlimit_detach(mem, ptr); + + free(ptr); + + return; +} diff --git a/src/liblzma/common/memory_usage.c b/src/liblzma/common/memory_usage.c new file mode 100644 index 00000000..b6f27957 --- /dev/null +++ b/src/liblzma/common/memory_usage.c @@ -0,0 +1,113 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memory_usage.c +/// \brief Calculate rough amount of memory required by filters +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "lz_encoder.h" +#include "lzma_literal.h" + + +static uint64_t +get_usage(const lzma_options_filter *filter, bool is_encoder) +{ + uint64_t ret; + + switch (filter->id) { + case LZMA_FILTER_COPY: + case LZMA_FILTER_X86: + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_IA64: + case LZMA_FILTER_ARM: + case LZMA_FILTER_ARMTHUMB: + case LZMA_FILTER_SPARC: + case LZMA_FILTER_DELTA: + // These don't require any significant amount of memory. + ret = 0; + break; + + case LZMA_FILTER_SUBBLOCK: + if (is_encoder) { + const lzma_options_subblock *options = filter->options; + ret = options->subblock_data_size; + } else { + ret = 0; + } + break; + +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: { + const lzma_options_lzma *options = filter->options; + + // Literal coder - this can be signficant if both values are + // big, or if sizeof(probability) is big. + ret = literal_states(options->literal_context_bits, + options->literal_pos_bits) * LIT_SIZE + * sizeof(probability); + + // Dictionary base size + ret += options->dictionary_size; + + if (is_encoder) { +# ifdef HAVE_ENCODER + // This is rough, but should be accurate enough + // in practice. + ret += options->dictionary_size / 2; + + uint32_t dummy1; + uint32_t dummy2; + uint32_t num_items; + if (lzma_lz_encoder_hash_properties( + options->match_finder, + options->dictionary_size, + &dummy1, &dummy2, &num_items)) + return UINT64_MAX; + + ret += (uint64_t)(num_items) * sizeof(uint32_t); +# else + return UINT64_MAX; +# endif + } + + break; + } +#endif + + default: + return UINT64_MAX; + } + + return ret; +} + + +extern LZMA_API uint32_t +lzma_memory_usage(const lzma_options_filter *filters, lzma_bool is_encoder) +{ + uint64_t usage = 0; + + for (size_t i = 0; filters[i].id != UINT64_MAX; ++i) { + const uint64_t ret = get_usage(filters + i, is_encoder); + if (ret == UINT64_MAX) + return UINT32_MAX; + + usage += ret; + } + + // Convert to mebibytes with rounding. + return usage / (1024 * 1024) + (usage % (1024 * 1024) >= 512 ? 1 : 0); +} diff --git a/src/liblzma/common/metadata_decoder.c b/src/liblzma/common/metadata_decoder.c new file mode 100644 index 00000000..f2ac6c1d --- /dev/null +++ b/src/liblzma/common/metadata_decoder.c @@ -0,0 +1,555 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file metadata_decoder.c +/// \brief Decodes metadata stored in Metadata Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "metadata_decoder.h" +#include "block_decoder.h" + + +/// Maximum size of a single Extra Record. Again, this is mostly to make +/// sure that the parsed lzma_vli fits into size_t. Still, maybe this should +/// be smaller. +#define EXTRA_SIZE_MAX (SIZE_MAX / 4) + + +struct lzma_coder_s { + enum { + SEQ_FLAGS, + SEQ_HEADER_METADATA_SIZE, + SEQ_TOTAL_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_INDEX_COUNT, + SEQ_INDEX_ALLOC, + SEQ_INDEX_TOTAL_SIZE, + SEQ_INDEX_UNCOMPRESSED_SIZE, + SEQ_EXTRA_PREPARE, + SEQ_EXTRA_ALLOC, + SEQ_EXTRA_ID, + SEQ_EXTRA_SIZE, + SEQ_EXTRA_DATA_ALLOC, + SEQ_EXTRA_DATA_COPY, + SEQ_EXTRA_DUMMY_ALLOC, + SEQ_EXTRA_DUMMY_ID, + SEQ_EXTRA_DUMMY_SIZE, + SEQ_EXTRA_DUMMY_COPY, + } sequence; + + /// Number of "things" left to be parsed. If we hit end of input + /// when this isn't zero, we have corrupt Metadata Block. + size_t todo_count; + + /// Position in variable-length integers + size_t pos; + + /// Temporary variable needed to decode variables whose type + /// is size_t instead of lzma_vli. + lzma_vli tmp; + + /// Pointer to target structure to hold the parsed results. + lzma_metadata *metadata; + + /// The Index Record we currently are parsing + lzma_index *index_current; + + /// Number of Records in Index + size_t index_count; + + /// Sum of Total Size fields in the Index + lzma_vli index_total_size; + + /// Sum of Uncompressed Size fields in the Index + lzma_vli index_uncompressed_size; + + /// True if Extra is present. + bool has_extra; + + /// True if we have been requested to store the Extra to *metadata. + bool want_extra; + + /// Pointer to the end of the Extra Record list. + lzma_extra *extra_tail; + + /// Dummy Extra Record used when only verifying integrity of Extra + /// (not storing it to RAM). + lzma_extra extra_dummy; + + /// Block decoder + lzma_next_coder block_decoder; + + /// buffer[buffer_pos] is the next byte to process. + size_t buffer_pos; + + /// buffer[buffer_size] is the first byte to not process. + size_t buffer_size; + + /// Temporary buffer to which encoded Metadata is read before + /// it is parsed. + uint8_t buffer[LZMA_BUFFER_SIZE]; +}; + + +/// Reads a variable-length integer to coder->num. +#define read_vli(num) \ +do { \ + const lzma_ret ret = lzma_vli_decode( \ + &num, &coder->pos, \ + coder->buffer, &coder->buffer_pos, \ + coder->buffer_size); \ + if (ret != LZMA_STREAM_END) \ + return ret; \ + \ + coder->pos = 0; \ +} while (0) + + +static lzma_ret +process(lzma_coder *coder, lzma_allocator *allocator) +{ + while (coder->buffer_pos < coder->buffer_size) + switch (coder->sequence) { + case SEQ_FLAGS: + // Reserved bits must be unset. + if (coder->buffer[coder->buffer_pos] & 0x70) + return LZMA_HEADER_ERROR; + + // If Size of Header Metadata is present, prepare the + // variable for variable-length integer decoding. Otherwise + // set it to LZMA_VLI_VALUE_UNKNOWN to indicate that the + // field isn't present. + if (coder->buffer[coder->buffer_pos] & 0x01) { + coder->metadata->header_metadata_size = 0; + ++coder->todo_count; + } + + if (coder->buffer[coder->buffer_pos] & 0x02) { + coder->metadata->total_size = 0; + ++coder->todo_count; + } + + if (coder->buffer[coder->buffer_pos] & 0x04) { + coder->metadata->uncompressed_size = 0; + ++coder->todo_count; + } + + if (coder->buffer[coder->buffer_pos] & 0x08) { + // Setting index_count to 1 is just to indicate that + // Index is present. The real size is parsed later. + coder->index_count = 1; + ++coder->todo_count; + } + + coder->has_extra = (coder->buffer[coder->buffer_pos] & 0x80) + != 0; + + ++coder->buffer_pos; + coder->sequence = SEQ_HEADER_METADATA_SIZE; + break; + + case SEQ_HEADER_METADATA_SIZE: + if (coder->metadata->header_metadata_size + != LZMA_VLI_VALUE_UNKNOWN) { + read_vli(coder->metadata->header_metadata_size); + + if (coder->metadata->header_metadata_size == 0) + return LZMA_DATA_ERROR; + + --coder->todo_count; + } + + coder->sequence = SEQ_TOTAL_SIZE; + break; + + case SEQ_TOTAL_SIZE: + if (coder->metadata->total_size != LZMA_VLI_VALUE_UNKNOWN) { + read_vli(coder->metadata->total_size); + + if (coder->metadata->total_size == 0) + return LZMA_DATA_ERROR; + + --coder->todo_count; + } + + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + break; + + case SEQ_UNCOMPRESSED_SIZE: + if (coder->metadata->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + read_vli(coder->metadata->uncompressed_size); + --coder->todo_count; + } + + coder->sequence = SEQ_INDEX_COUNT; + break; + + case SEQ_INDEX_COUNT: + if (coder->index_count == 0) { + coder->sequence = SEQ_EXTRA_PREPARE; + break; + } + + read_vli(coder->tmp); + + // Index must not be empty nor far too big (wouldn't fit + // in RAM). + if (coder->tmp == 0 || coder->tmp + >= SIZE_MAX / sizeof(lzma_index)) + return LZMA_DATA_ERROR; + + coder->index_count = (size_t)(coder->tmp); + coder->tmp = 0; + + coder->sequence = SEQ_INDEX_ALLOC; + break; + + case SEQ_INDEX_ALLOC: { + lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); + if (i == NULL) + return LZMA_MEM_ERROR; + + i->total_size = 0; + i->uncompressed_size = 0; + i->next = NULL; + + if (coder->metadata->index == NULL) + coder->metadata->index = i; + else + coder->index_current->next = i; + + coder->index_current = i; + + coder->sequence = SEQ_INDEX_TOTAL_SIZE; + } + + // Fall through + + case SEQ_INDEX_TOTAL_SIZE: { + read_vli(coder->index_current->total_size); + + coder->index_total_size += coder->index_current->total_size; + if (coder->index_total_size > LZMA_VLI_VALUE_MAX) + return LZMA_DATA_ERROR; + + // No Block can have Total Size of zero bytes. + if (coder->index_current->total_size == 0) + return LZMA_DATA_ERROR; + + if (--coder->index_count == 0) { + // If Total Size is present, it must match the sum + // of Total Sizes in Index. + if (coder->metadata->total_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->metadata->total_size + != coder->index_total_size) + return LZMA_DATA_ERROR; + + coder->index_current = coder->metadata->index; + coder->sequence = SEQ_INDEX_UNCOMPRESSED_SIZE; + } else { + coder->sequence = SEQ_INDEX_ALLOC; + } + + break; + } + + case SEQ_INDEX_UNCOMPRESSED_SIZE: { + read_vli(coder->index_current->uncompressed_size); + + coder->index_uncompressed_size + += coder->index_current->uncompressed_size; + if (coder->index_uncompressed_size > LZMA_VLI_VALUE_MAX) + return LZMA_DATA_ERROR; + + coder->index_current = coder->index_current->next; + if (coder->index_current == NULL) { + if (coder->metadata->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->metadata->uncompressed_size + != coder->index_uncompressed_size) + return LZMA_DATA_ERROR; + + --coder->todo_count; + coder->sequence = SEQ_EXTRA_PREPARE; + } + + break; + } + + case SEQ_EXTRA_PREPARE: + assert(coder->todo_count == 0); + + // If we get here, we have at least one byte of input left. + // If "Extra is present" flag is unset in Metadata Flags, + // it means that there is some garbage and we return an error. + if (!coder->has_extra) + return LZMA_DATA_ERROR; + + if (!coder->want_extra) { + coder->extra_tail = &coder->extra_dummy; + coder->sequence = SEQ_EXTRA_DUMMY_ALLOC; + break; + } + + coder->sequence = SEQ_EXTRA_ALLOC; + + // Fall through + + case SEQ_EXTRA_ALLOC: { + lzma_extra *e = lzma_alloc(sizeof(lzma_extra), allocator); + if (e == NULL) + return LZMA_MEM_ERROR; + + e->next = NULL; + e->id = 0; + e->size = 0; + e->data = NULL; + + if (coder->metadata->extra == NULL) + coder->metadata->extra = e; + else + coder->extra_tail->next = e; + + coder->extra_tail = e; + + coder->todo_count = 1; + coder->sequence = SEQ_EXTRA_ID; + } + + // Fall through + + case SEQ_EXTRA_ID: + case SEQ_EXTRA_DUMMY_ID: + read_vli(coder->extra_tail->id); + + if (coder->extra_tail->id == 0) { + coder->extra_tail->size = 0; + coder->extra_tail->data = NULL; + coder->todo_count = 0; + --coder->sequence; + } else { + ++coder->sequence; + } + + break; + + case SEQ_EXTRA_SIZE: + case SEQ_EXTRA_DUMMY_SIZE: + read_vli(coder->tmp); + ++coder->sequence; + break; + + case SEQ_EXTRA_DATA_ALLOC: { + if (coder->tmp > EXTRA_SIZE_MAX) + return LZMA_DATA_ERROR; + + coder->extra_tail->size = (size_t)(coder->tmp); + coder->tmp = 0; + + uint8_t *d = lzma_alloc((size_t)(coder->extra_tail->size), + allocator); + if (d == NULL) + return LZMA_MEM_ERROR; + + coder->extra_tail->data = d; + coder->sequence = SEQ_EXTRA_DATA_COPY; + } + + // Fall through + + case SEQ_EXTRA_DATA_COPY: + bufcpy(coder->buffer, &coder->buffer_pos, coder->buffer_size, + coder->extra_tail->data, &coder->pos, + (size_t)(coder->extra_tail->size)); + + if ((size_t)(coder->extra_tail->size) == coder->pos) { + coder->pos = 0; + coder->todo_count = 0; + coder->sequence = SEQ_EXTRA_ALLOC; + } + + break; + + case SEQ_EXTRA_DUMMY_ALLOC: + // Not really alloc, just initialize the dummy entry. + coder->extra_dummy = (lzma_extra){ + .next = NULL, + .id = 0, + .size = 0, + .data = NULL, + }; + + coder->todo_count = 1; + coder->sequence = SEQ_EXTRA_DUMMY_ID; + break; + + case SEQ_EXTRA_DUMMY_COPY: { + // Simply skip as many bytes as indicated by Extra Record Size. + // We don't check lzma_extra_size_max because we don't + // allocate any memory to hold the data. + const size_t in_avail = coder->buffer_size - coder->buffer_pos; + const size_t skip = MIN((lzma_vli)(in_avail), coder->tmp); + coder->buffer_pos += skip; + coder->tmp -= skip; + + if (coder->tmp == 0) { + coder->todo_count = 0; + coder->sequence = SEQ_EXTRA_DUMMY_ALLOC; + } + + break; + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +metadata_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + bool end_was_reached = false; + + while (true) { + // Fill the buffer if it is empty. + if (coder->buffer_pos == coder->buffer_size) { + coder->buffer_pos = 0; + coder->buffer_size = 0; + + const lzma_ret ret = coder->block_decoder.code( + coder->block_decoder.coder, allocator, + in, in_pos, in_size, coder->buffer, + &coder->buffer_size, LZMA_BUFFER_SIZE, + LZMA_RUN); + + switch (ret) { + case LZMA_OK: + // Return immediatelly if we got no new data. + if (coder->buffer_size == 0) + return LZMA_OK; + + break; + + case LZMA_STREAM_END: + end_was_reached = true; + break; + + default: + return ret; + } + } + + // Process coder->buffer. + const lzma_ret ret = process(coder, allocator); + if (ret != LZMA_OK) + return ret; + + // On success, process() eats all the input. + assert(coder->buffer_pos == coder->buffer_size); + + if (end_was_reached) { + // Check that the sequence is not in the + // middle of anything. + if (coder->todo_count != 0) + return LZMA_DATA_ERROR; + + return LZMA_STREAM_END; + } + } +} + + +static void +metadata_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +metadata_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, lzma_metadata *metadata, + bool want_extra) +{ + if (options == NULL || metadata == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &metadata_decode; + next->end = &metadata_decoder_end; + next->coder->block_decoder = LZMA_NEXT_CODER_INIT; + } + + metadata->header_metadata_size = LZMA_VLI_VALUE_UNKNOWN; + metadata->total_size = LZMA_VLI_VALUE_UNKNOWN; + metadata->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + metadata->index = NULL; + metadata->extra = NULL; + + next->coder->sequence = SEQ_FLAGS; + next->coder->todo_count = 0; + next->coder->pos = 0; + next->coder->tmp = 0; + next->coder->metadata = metadata; + next->coder->index_current = NULL; + next->coder->index_count = 0; + next->coder->index_total_size = 0; + next->coder->index_uncompressed_size = 0; + next->coder->want_extra = want_extra; + next->coder->extra_tail = NULL; + next->coder->buffer_pos = 0; + next->coder->buffer_size = 0; + + return lzma_block_decoder_init( + &next->coder->block_decoder, allocator, options); +} + + +extern lzma_ret +lzma_metadata_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, lzma_metadata *metadata, + bool want_extra) +{ + lzma_next_coder_init(metadata_decoder_init, next, allocator, + options, metadata, want_extra); +} + + +extern LZMA_API lzma_ret +lzma_metadata_decoder(lzma_stream *strm, lzma_options_block *options, + lzma_metadata *metadata, lzma_bool want_extra) +{ + lzma_next_strm_init(strm, lzma_metadata_decoder_init, + options, metadata, want_extra); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/metadata_decoder.h b/src/liblzma/common/metadata_decoder.h new file mode 100644 index 00000000..1fba2179 --- /dev/null +++ b/src/liblzma/common/metadata_decoder.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file metadata_decoder.h +/// \brief Decodes metadata stored in Metadata Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_METADATA_DECODER_H +#define LZMA_METADATA_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_metadata_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, lzma_metadata *metadata, + bool want_extra); + +#endif diff --git a/src/liblzma/common/metadata_encoder.c b/src/liblzma/common/metadata_encoder.c new file mode 100644 index 00000000..17587c5c --- /dev/null +++ b/src/liblzma/common/metadata_encoder.c @@ -0,0 +1,436 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file metadata_encoder.c +/// \brief Encodes metadata to be stored into Metadata Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "metadata_encoder.h" +#include "block_encoder.h" + + +struct lzma_coder_s { + enum { + SEQ_FLAGS, + SEQ_HEADER_METADATA_SIZE, + SEQ_TOTAL_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_INDEX_COUNT, + SEQ_INDEX_TOTAL, + SEQ_INDEX_UNCOMPRESSED, + SEQ_EXTRA_ID, + SEQ_EXTRA_SIZE, + SEQ_EXTRA_DATA, + SEQ_END, + } sequence; + + /// Position in variable-length integers + size_t pos; + + /// Local copy of the Metadata structure. Note that we keep + /// a copy only of the main structure, not Index or Extra Records. + lzma_metadata metadata; + + /// Number of Records in Index + size_t index_count; + + /// Index Record currently being processed + const lzma_index *index_current; + + /// Block encoder for the encoded Metadata + lzma_next_coder block_encoder; + + /// True once everything except compression has been done. + bool end_was_reached; + + /// buffer[buffer_pos] is the first byte that needs to be compressed. + size_t buffer_pos; + + /// buffer[buffer_size] is the next position where a byte will be + /// written by process(). + size_t buffer_size; + + /// Temporary buffer to which encoded Metadata is written before + /// it is compressed. + uint8_t buffer[LZMA_BUFFER_SIZE]; +}; + + +#define write_vli(num) \ +do { \ + const lzma_ret ret = lzma_vli_encode(num, &coder->pos, 1, \ + coder->buffer, &coder->buffer_size, \ + LZMA_BUFFER_SIZE); \ + if (ret != LZMA_STREAM_END) \ + return ret; \ + coder->pos = 0; \ +} while (0) + + +static lzma_ret +process(lzma_coder *coder) +{ + while (coder->buffer_size < LZMA_BUFFER_SIZE) + switch (coder->sequence) { + case SEQ_FLAGS: + coder->buffer[coder->buffer_size] = 0; + + if (coder->metadata.header_metadata_size + != LZMA_VLI_VALUE_UNKNOWN) + coder->buffer[coder->buffer_size] |= 0x01; + + if (coder->metadata.total_size != LZMA_VLI_VALUE_UNKNOWN) + coder->buffer[coder->buffer_size] |= 0x02; + + if (coder->metadata.uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) + coder->buffer[coder->buffer_size] |= 0x04; + + if (coder->index_count > 0) + coder->buffer[coder->buffer_size] |= 0x08; + + if (coder->metadata.extra != NULL) + coder->buffer[coder->buffer_size] |= 0x80; + + ++coder->buffer_size; + coder->sequence = SEQ_HEADER_METADATA_SIZE; + break; + + case SEQ_HEADER_METADATA_SIZE: + if (coder->metadata.header_metadata_size + != LZMA_VLI_VALUE_UNKNOWN) + write_vli(coder->metadata.header_metadata_size); + + coder->sequence = SEQ_TOTAL_SIZE; + break; + + case SEQ_TOTAL_SIZE: + if (coder->metadata.total_size != LZMA_VLI_VALUE_UNKNOWN) + write_vli(coder->metadata.total_size); + + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + break; + + case SEQ_UNCOMPRESSED_SIZE: + if (coder->metadata.uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) + write_vli(coder->metadata.uncompressed_size); + + coder->sequence = SEQ_INDEX_COUNT; + break; + + case SEQ_INDEX_COUNT: + if (coder->index_count == 0) { + if (coder->metadata.extra == NULL) { + coder->sequence = SEQ_END; + return LZMA_STREAM_END; + } + + coder->sequence = SEQ_EXTRA_ID; + break; + } + + write_vli(coder->index_count); + coder->sequence = SEQ_INDEX_TOTAL; + break; + + case SEQ_INDEX_TOTAL: + write_vli(coder->index_current->total_size); + + coder->index_current = coder->index_current->next; + if (coder->index_current == NULL) { + coder->index_current = coder->metadata.index; + coder->sequence = SEQ_INDEX_UNCOMPRESSED; + } + + break; + + case SEQ_INDEX_UNCOMPRESSED: + write_vli(coder->index_current->uncompressed_size); + + coder->index_current = coder->index_current->next; + if (coder->index_current != NULL) + break; + + if (coder->metadata.extra != NULL) { + coder->sequence = SEQ_EXTRA_ID; + break; + } + + coder->sequence = SEQ_END; + return LZMA_STREAM_END; + + case SEQ_EXTRA_ID: { + const lzma_ret ret = lzma_vli_encode( + coder->metadata.extra->id, &coder->pos, 1, + coder->buffer, &coder->buffer_size, + LZMA_BUFFER_SIZE); + switch (ret) { + case LZMA_OK: + break; + + case LZMA_STREAM_END: + coder->pos = 0; + + // Handle the special ID 0. + if (coder->metadata.extra->id == 0) { + coder->metadata.extra + = coder->metadata.extra->next; + if (coder->metadata.extra == NULL) { + coder->sequence = SEQ_END; + return LZMA_STREAM_END; + } + + coder->sequence = SEQ_EXTRA_ID; + + } else { + coder->sequence = SEQ_EXTRA_SIZE; + } + + break; + + default: + return ret; + } + + break; + } + + case SEQ_EXTRA_SIZE: + if (coder->metadata.extra->size >= (lzma_vli)(SIZE_MAX)) + return LZMA_HEADER_ERROR; + + write_vli(coder->metadata.extra->size); + coder->sequence = SEQ_EXTRA_DATA; + break; + + case SEQ_EXTRA_DATA: + bufcpy(coder->metadata.extra->data, &coder->pos, + coder->metadata.extra->size, + coder->buffer, &coder->buffer_size, + LZMA_BUFFER_SIZE); + + if ((size_t)(coder->metadata.extra->size) == coder->pos) { + coder->metadata.extra = coder->metadata.extra->next; + if (coder->metadata.extra == NULL) { + coder->sequence = SEQ_END; + return LZMA_STREAM_END; + } + + coder->pos = 0; + coder->sequence = SEQ_EXTRA_ID; + } + + break; + + case SEQ_END: + // Everything is encoded. Let the compression code finish + // its work now. + return LZMA_STREAM_END; + } + + return LZMA_OK; +} + + +static lzma_ret +metadata_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in lzma_attribute((unused)), + size_t *restrict in_pos lzma_attribute((unused)), + size_t in_size lzma_attribute((unused)), uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action lzma_attribute((unused))) +{ + while (!coder->end_was_reached) { + // Flush coder->buffer if it isn't empty. + if (coder->buffer_size > 0) { + const lzma_ret ret = coder->block_encoder.code( + coder->block_encoder.coder, allocator, + coder->buffer, &coder->buffer_pos, + coder->buffer_size, + out, out_pos, out_size, LZMA_RUN); + if (coder->buffer_pos < coder->buffer_size + || ret != LZMA_OK) + return ret; + + coder->buffer_pos = 0; + coder->buffer_size = 0; + } + + const lzma_ret ret = process(coder); + + switch (ret) { + case LZMA_OK: + break; + + case LZMA_STREAM_END: + coder->end_was_reached = true; + break; + + default: + return ret; + } + } + + // Finish + return coder->block_encoder.code(coder->block_encoder.coder, allocator, + coder->buffer, &coder->buffer_pos, coder->buffer_size, + out, out_pos, out_size, LZMA_FINISH); +} + + +static void +metadata_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->block_encoder, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +metadata_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, const lzma_metadata *metadata) +{ + if (options == NULL || metadata == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &metadata_encode; + next->end = &metadata_encoder_end; + next->coder->block_encoder = LZMA_NEXT_CODER_INIT; + } + + next->coder->sequence = SEQ_FLAGS; + next->coder->pos = 0; + next->coder->metadata = *metadata; + next->coder->index_count = 0; + next->coder->index_current = metadata->index; + next->coder->end_was_reached = false; + next->coder->buffer_pos = 0; + next->coder->buffer_size = 0; + + // Count and validate the Index Records. + { + const lzma_index *i = metadata->index; + while (i != NULL) { + if (i->total_size > LZMA_VLI_VALUE_MAX + || i->uncompressed_size + > LZMA_VLI_VALUE_MAX) + return LZMA_PROG_ERROR; + + ++next->coder->index_count; + i = i->next; + } + } + + // Initialize the Block encoder. + return lzma_block_encoder_init( + &next->coder->block_encoder, allocator, options); +} + + +extern lzma_ret +lzma_metadata_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, const lzma_metadata *metadata) +{ + lzma_next_coder_init(metadata_encoder_init, next, allocator, + options, metadata); +} + + +extern LZMA_API lzma_ret +lzma_metadata_encoder(lzma_stream *strm, lzma_options_block *options, + const lzma_metadata *metadata) +{ + lzma_next_strm_init(strm, metadata_encoder_init, options, metadata); + + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +extern LZMA_API lzma_vli +lzma_metadata_size(const lzma_metadata *metadata) +{ + lzma_vli size = 1; // Metadata Flags + + // Validate header_metadata_size, total_size, and uncompressed_size. + if (!lzma_vli_is_valid(metadata->header_metadata_size) + || !lzma_vli_is_valid(metadata->total_size) + || !lzma_vli_is_valid(metadata->uncompressed_size)) + return 0; + + // Add the sizes of these three fields. + if (metadata->header_metadata_size != LZMA_VLI_VALUE_UNKNOWN) + size += lzma_vli_size(metadata->header_metadata_size); + + if (metadata->total_size != LZMA_VLI_VALUE_UNKNOWN) + size += lzma_vli_size(metadata->total_size); + + if (metadata->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + size += lzma_vli_size(metadata->uncompressed_size); + + // Index + if (metadata->index != NULL) { + const lzma_index *i = metadata->index; + size_t count = 1; + + do { + const size_t x = lzma_vli_size(i->total_size); + const size_t y = lzma_vli_size(i->uncompressed_size); + if (x == 0 || y == 0) + return 0; + + size += x + y; + ++count; + i = i->next; + + } while (i != NULL); + + const size_t tmp = lzma_vli_size(count); + if (tmp == 0) + return 0; + + size += tmp; + } + + // Extra + { + const lzma_extra *e = metadata->extra; + while (e != NULL) { + // Validate the numbers. + if (e->id > LZMA_VLI_VALUE_MAX + || e->size >= (lzma_vli)(SIZE_MAX)) + return 0; + + // Add the sizes. + size += lzma_vli_size(e->id); + if (e->id != 0) { + size += lzma_vli_size(e->size); + size += e->size; + } + + e = e->next; + } + } + + return size; +} diff --git a/src/liblzma/common/metadata_encoder.h b/src/liblzma/common/metadata_encoder.h new file mode 100644 index 00000000..20357fe6 --- /dev/null +++ b/src/liblzma/common/metadata_encoder.h @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file metadata_encoder.h +/// \brief Encodes metadata to be stored into Metadata Blocks +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_METADATA_ENCODER_H +#define LZMA_METADATA_ENCODER_H + +#include "common.h" + + +extern lzma_ret lzma_metadata_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + lzma_options_block *options, const lzma_metadata *metadata); + +#endif diff --git a/src/liblzma/common/next_coder.c b/src/liblzma/common/next_coder.c new file mode 100644 index 00000000..c10fe24d --- /dev/null +++ b/src/liblzma/common/next_coder.c @@ -0,0 +1,65 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file next_coder.c +/// \brief Initializing and freeing the next coder in the chain +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + +extern lzma_ret +lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_ret ret = LZMA_OK; + + // Free the existing coder if it is different than the current one. + if ((uintptr_t)(filters[0].init) != next->init) + lzma_next_coder_end(next, allocator); + + if (filters[0].init != NULL) { + // Initialize the new coder. + ret = filters[0].init(next, allocator, filters); + + // Set the init function pointer if initialization was + // successful. next->code and next->end are set by the + // initialization function itself. + if (ret == LZMA_OK) { + next->init = (uintptr_t)(filters[0].init); + assert(next->code != NULL); + assert(next->end != NULL); + } else { + lzma_next_coder_end(next, allocator); + } + } + + return ret; +} + + +extern void +lzma_next_coder_end(lzma_next_coder *next, lzma_allocator *allocator) +{ + if (next != NULL) { + if (next->end != NULL) + next->end(next->coder, allocator); + + // Reset the variables so the we don't accidentally think + // that it is an already initialized coder. + *next = LZMA_NEXT_CODER_INIT; + } + + return; +} diff --git a/src/liblzma/common/raw_common.c b/src/liblzma/common/raw_common.c new file mode 100644 index 00000000..394903bc --- /dev/null +++ b/src/liblzma/common/raw_common.c @@ -0,0 +1,175 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_common.c +/// \brief Stuff shared between raw encoder and raw decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "raw_common.h" + + +/// \brief Prepares the filter chain +/// +/// Prepares the filter chain by setting uncompressed sizes for each filter, +/// and adding implicit Subblock filter when needed. +/// +/// \return true if error occurred, false on success. +/// +static bool +prepare(lzma_vli *id, lzma_vli *uncompressed_size, bool implicit) +{ + bool needs_end_of_input = false; + + switch (id[0]) { + case LZMA_FILTER_COPY: + case LZMA_FILTER_X86: + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_IA64: + case LZMA_FILTER_ARM: + case LZMA_FILTER_ARMTHUMB: + case LZMA_FILTER_SPARC: + case LZMA_FILTER_DELTA: + uncompressed_size[1] = uncompressed_size[0]; + needs_end_of_input = true; + break; + + case LZMA_FILTER_SUBBLOCK: + case LZMA_FILTER_LZMA: + // These change the size of the data unpredictably. + uncompressed_size[1] = LZMA_VLI_VALUE_UNKNOWN; + break; + + case LZMA_FILTER_SUBBLOCK_HELPER: + uncompressed_size[1] = uncompressed_size[0]; + break; + + default: + // Unknown filter. + return true; + } + + // Is this the last filter in the chain? + if (id[1] == LZMA_VLI_VALUE_UNKNOWN) { + if (!needs_end_of_input || !implicit || uncompressed_size[0] + != LZMA_VLI_VALUE_UNKNOWN) + return false; + + // Add implicit Subblock filter. + id[1] = LZMA_FILTER_SUBBLOCK; + uncompressed_size[1] = LZMA_VLI_VALUE_UNKNOWN; + id[2] = LZMA_VLI_VALUE_UNKNOWN; + } + + return prepare(id + 1, uncompressed_size + 1, implicit); +} + + +extern lzma_ret +lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_filter *options, lzma_vli uncompressed_size, + lzma_init_function (*get_function)(lzma_vli id), + bool allow_implicit, bool is_encoder) +{ + if (options == NULL || !lzma_vli_is_valid(uncompressed_size)) + return LZMA_PROG_ERROR; + + // Count the number of filters in the chain. + size_t count = 0; + while (options[count].id != LZMA_VLI_VALUE_UNKNOWN) + ++count; + + // Allocate enough space from the stack for IDs and uncompressed + // sizes. We need two extra: possible implicit Subblock and end + // of array indicator. + lzma_vli ids[count + 2]; + lzma_vli uncompressed_sizes[count + 2]; + bool using_implicit = false; + + uncompressed_sizes[0] = uncompressed_size; + + if (count == 0) { + if (!allow_implicit) + return LZMA_PROG_ERROR; + + count = 1; + using_implicit = true; + + // Special case: no filters were specified, so an implicit + // Copy or Subblock filter is used. + if (uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) + ids[0] = LZMA_FILTER_SUBBLOCK; + else + ids[0] = LZMA_FILTER_COPY; + + ids[1] = LZMA_VLI_VALUE_UNKNOWN; + + } else { + // Prepare the ids[] and uncompressed_sizes[]. + for (size_t i = 0; i < count; ++i) + ids[i] = options[i].id; + + ids[count] = LZMA_VLI_VALUE_UNKNOWN; + + if (prepare(ids, uncompressed_sizes, allow_implicit)) + return LZMA_HEADER_ERROR; + + // Check if implicit Subblock filter was added. + if (ids[count] != LZMA_VLI_VALUE_UNKNOWN) { + assert(ids[count] == LZMA_FILTER_SUBBLOCK); + ++count; + using_implicit = true; + } + } + + // Set the filter functions, and copy uncompressed sizes and options. + lzma_filter_info filters[count + 1]; + if (is_encoder) { + for (size_t i = 0; i < count; ++i) { + // The order of the filters is reversed in the + // encoder. It allows more efficient handling + // of the uncompressed data. + const size_t j = count - i - 1; + + filters[j].init = get_function(ids[i]); + if (filters[j].init == NULL) + return LZMA_HEADER_ERROR; + + filters[j].options = options[i].options; + filters[j].uncompressed_size = uncompressed_sizes[i]; + } + + if (using_implicit) + filters[0].options = NULL; + + } else { + for (size_t i = 0; i < count; ++i) { + filters[i].init = get_function(ids[i]); + if (filters[i].init == NULL) + return LZMA_HEADER_ERROR; + + filters[i].options = options[i].options; + filters[i].uncompressed_size = uncompressed_sizes[i]; + } + + if (using_implicit) + filters[count - 1].options = NULL; + } + + // Terminate the array. + filters[count].init = NULL; + + // Initialize the filters. + return lzma_next_filter_init(next, allocator, filters); +} diff --git a/src/liblzma/common/raw_common.h b/src/liblzma/common/raw_common.h new file mode 100644 index 00000000..172223cb --- /dev/null +++ b/src/liblzma/common/raw_common.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_common.h +/// \brief Stuff shared between raw encoder and raw decoder +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RAW_COMMON_H +#define LZMA_RAW_COMMON_H + +#include "common.h" + +extern lzma_ret lzma_raw_coder_init(lzma_next_coder *next, + lzma_allocator *allocator, + const lzma_options_filter *options, lzma_vli uncompressed_size, + lzma_init_function (*get_function)(lzma_vli id), + bool allow_implicit, bool is_encoder); + +#endif diff --git a/src/liblzma/common/raw_decoder.c b/src/liblzma/common/raw_decoder.c new file mode 100644 index 00000000..a11cb5c4 --- /dev/null +++ b/src/liblzma/common/raw_decoder.c @@ -0,0 +1,127 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_decoder.c +/// \brief Raw decoder initialization API +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "raw_decoder.h" +#include "copy_coder.h" +#include "simple_coder.h" +#include "subblock_decoder.h" +#include "subblock_decoder_helper.h" +#include "delta_coder.h" +#include "lzma_decoder.h" +#include "metadata_decoder.h" + + +static lzma_init_function +get_function(lzma_vli id) +{ + switch (id) { +#ifdef HAVE_FILTER_COPY + case LZMA_FILTER_COPY: + return &lzma_copy_decoder_init; +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK: + return &lzma_subblock_decoder_init; +#endif + +#ifdef HAVE_FILTER_X86 + case LZMA_FILTER_X86: + return &lzma_simple_x86_decoder_init; +#endif + +#ifdef HAVE_FILTER_POWERPC + case LZMA_FILTER_POWERPC: + return &lzma_simple_powerpc_decoder_init; +#endif + +#ifdef HAVE_FILTER_IA64 + case LZMA_FILTER_IA64: + return &lzma_simple_ia64_decoder_init; +#endif + +#ifdef HAVE_FILTER_ARM + case LZMA_FILTER_ARM: + return &lzma_simple_arm_decoder_init; +#endif + +#ifdef HAVE_FILTER_ARMTHUMB + case LZMA_FILTER_ARMTHUMB: + return &lzma_simple_armthumb_decoder_init; +#endif + +#ifdef HAVE_FILTER_SPARC + case LZMA_FILTER_SPARC: + return &lzma_simple_sparc_decoder_init; +#endif + +#ifdef HAVE_FILTER_DELTA + case LZMA_FILTER_DELTA: + return &lzma_delta_decoder_init; +#endif + +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: + return &lzma_lzma_decoder_init; +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK_HELPER: + return &lzma_subblock_decoder_helper_init; +#endif + } + + return NULL; +} + + +extern lzma_ret +lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_filter *options, + lzma_vli uncompressed_size, bool allow_implicit) +{ + const lzma_ret ret = lzma_raw_coder_init(next, allocator, + options, uncompressed_size, &get_function, + allow_implicit, false); + + if (ret != LZMA_OK) + lzma_next_coder_end(next, allocator); + + return ret; +} + + +extern LZMA_API lzma_ret +lzma_raw_decoder(lzma_stream *strm, const lzma_options_filter *options, + lzma_vli uncompressed_size, lzma_bool allow_implicit) +{ + return_if_error(lzma_strm_init(strm)); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, + strm->allocator, options, uncompressed_size, + &get_function, allow_implicit, false); + + if (ret != LZMA_OK) + lzma_end(strm); + + return ret; +} diff --git a/src/liblzma/common/raw_decoder.h b/src/liblzma/common/raw_decoder.h new file mode 100644 index 00000000..9d48074b --- /dev/null +++ b/src/liblzma/common/raw_decoder.h @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_decoder.h +/// \brief Raw decoder initialization API +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RAW_DECODER_H +#define LZMA_RAW_DECODER_H + +#include "raw_common.h" + + +extern lzma_ret lzma_raw_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_filter *options, + lzma_vli uncompressed_size, bool implicit); + +#endif diff --git a/src/liblzma/common/raw_encoder.c b/src/liblzma/common/raw_encoder.c new file mode 100644 index 00000000..c2cd0a51 --- /dev/null +++ b/src/liblzma/common/raw_encoder.c @@ -0,0 +1,124 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_encoder.c +/// \brief Raw encoder initialization API +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "raw_encoder.h" +#include "copy_coder.h" +#include "simple_coder.h" +#include "subblock_encoder.h" +#include "delta_coder.h" +#include "lzma_encoder.h" + + +static lzma_init_function +get_function(lzma_vli id) +{ + switch (id) { +#ifdef HAVE_FILTER_COPY + case LZMA_FILTER_COPY: + return &lzma_copy_encoder_init; +#endif + +#ifdef HAVE_FILTER_SUBBLOCK + case LZMA_FILTER_SUBBLOCK: + return &lzma_subblock_encoder_init; +#endif + +#ifdef HAVE_FILTER_X86 + case LZMA_FILTER_X86: + return &lzma_simple_x86_encoder_init; +#endif + +#ifdef HAVE_FILTER_POWERPC + case LZMA_FILTER_POWERPC: + return &lzma_simple_powerpc_encoder_init; +#endif + +#ifdef HAVE_FILTER_IA64 + case LZMA_FILTER_IA64: + return &lzma_simple_ia64_encoder_init; +#endif + +#ifdef HAVE_FILTER_ARM + case LZMA_FILTER_ARM: + return &lzma_simple_arm_encoder_init; +#endif + +#ifdef HAVE_FILTER_ARMTHUMB + case LZMA_FILTER_ARMTHUMB: + return &lzma_simple_armthumb_encoder_init; +#endif + +#ifdef HAVE_FILTER_SPARC + case LZMA_FILTER_SPARC: + return &lzma_simple_sparc_encoder_init; +#endif + +#ifdef HAVE_FILTER_DELTA + case LZMA_FILTER_DELTA: + return &lzma_delta_encoder_init; +#endif + +#ifdef HAVE_FILTER_LZMA + case LZMA_FILTER_LZMA: + return &lzma_lzma_encoder_init; +#endif + } + + return NULL; +} + + +extern lzma_ret +lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_filter *options, + lzma_vli uncompressed_size, bool allow_implicit) +{ + // lzma_raw_coder_init() accesses get_function() via function pointer, + // because this way linker doesn't statically link both encoder and + // decoder functions if user needs only encoder or decoder. + const lzma_ret ret = lzma_raw_coder_init(next, allocator, + options, uncompressed_size, &get_function, + allow_implicit, true); + + if (ret != LZMA_OK) + lzma_next_coder_end(next, allocator); + + return ret; +} + + +extern LZMA_API lzma_ret +lzma_raw_encoder(lzma_stream *strm, const lzma_options_filter *options, + lzma_vli uncompressed_size, lzma_bool allow_implicit) +{ + return_if_error(lzma_strm_init(strm)); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, + strm->allocator, options, uncompressed_size, + &get_function, allow_implicit, true); + + if (ret != LZMA_OK) + lzma_end(strm); + + return ret; +} diff --git a/src/liblzma/common/raw_encoder.h b/src/liblzma/common/raw_encoder.h new file mode 100644 index 00000000..b0aab61a --- /dev/null +++ b/src/liblzma/common/raw_encoder.h @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file raw_encoder.h +/// \brief Raw encoder initialization API +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RAW_ENCODER_H +#define LZMA_RAW_ENCODER_H + +#include "raw_common.h" + + +extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_filter *options, + lzma_vli uncompressed_size, bool allow_implicit); + +#endif diff --git a/src/liblzma/common/stream_common.c b/src/liblzma/common/stream_common.c new file mode 100644 index 00000000..121a6674 --- /dev/null +++ b/src/liblzma/common/stream_common.c @@ -0,0 +1,23 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_common.c +/// \brief Common stuff for Stream coders +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_common.h" + +const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 }; +const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; diff --git a/src/liblzma/common/stream_common.h b/src/liblzma/common/stream_common.h new file mode 100644 index 00000000..b2f37f37 --- /dev/null +++ b/src/liblzma/common/stream_common.h @@ -0,0 +1,28 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_common.h +/// \brief Common stuff for Stream coders +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_STREAM_COMMON_H +#define LZMA_STREAM_COMMON_H + +#include "common.h" + +extern const uint8_t lzma_header_magic[6]; +extern const uint8_t lzma_footer_magic[2]; + +#endif diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c new file mode 100644 index 00000000..d8000c3d --- /dev/null +++ b/src/liblzma/common/stream_decoder.c @@ -0,0 +1,454 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_decoder.c +/// \brief Decodes .lzma Streams +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_common.h" +#include "check.h" +#include "stream_flags_decoder.h" +#include "block_decoder.h" +#include "metadata_decoder.h" + + +struct lzma_coder_s { + enum { + SEQ_STREAM_HEADER_CODE, + SEQ_BLOCK_HEADER_INIT, + SEQ_BLOCK_HEADER_CODE, + SEQ_METADATA_CODE, + SEQ_DATA_CODE, + SEQ_STREAM_TAIL_INIT, + SEQ_STREAM_TAIL_CODE, + } sequence; + + /// Position in variable-length integers and in some other things. + size_t pos; + + /// Block or Metadata decoder. This takes little memory and the same + /// data structure can be used to decode every Block Header, so it's + /// a good idea to have a separate lzma_next_coder structure for it. + lzma_next_coder block_decoder; + + /// Block Header decoder; this is separate + lzma_next_coder block_header_decoder; + + lzma_options_block block_options; + + /// Information about the sizes of the Blocks + lzma_info *info; + + /// Current Block in *info + lzma_info_iter iter; + + /// Number of bytes not yet processed from Data Blocks in the Stream. + /// This can be LZMA_VLI_VALUE_UNKNOWN. If it is known, it is + /// decremented while decoding and verified to match the reality. + lzma_vli total_left; + + /// Like uncompressed_left above but for uncompressed data from + /// Data Blocks. + lzma_vli uncompressed_left; + + /// Stream Flags from Stream Header + lzma_stream_flags header_flags; + + /// Stream Flags from Stream tail + lzma_stream_flags tail_flags; + + /// Decoder for Stream Header and Stream tail. This takes very + /// little memory and the same data structure can be used for + /// both Header and tail, so it's a good idea to have a separate + /// lzma_next_coder structure for it. + lzma_next_coder flags_decoder; + + /// Temporary destination for the decoded Metadata. + lzma_metadata metadata; + + /// Pointer to application-supplied pointer where to store the list + /// of Extra Records from the Header Metadata Block. + lzma_extra **header_extra; + + /// Same as above but Footer Metadata Block + lzma_extra **footer_extra; +}; + + +static lzma_ret +metadata_init(lzma_coder *coder, lzma_allocator *allocator) +{ + assert(coder->metadata.index == NULL); + assert(coder->metadata.extra == NULL); + + // Single-Block Streams don't have Metadata Blocks. + if (!coder->header_flags.is_multi) + return LZMA_DATA_ERROR; + + coder->block_options.total_limit = LZMA_VLI_VALUE_UNKNOWN; + + // Limit the Uncompressed Size of a Metadata Block. This is to + // prevent security issues where input file would have very huge + // Metadata. + // + // FIXME: Hardcoded constant is ugly. Maybe we should provide + // some way to specify this from the application. + coder->block_options.uncompressed_limit = LZMA_VLI_C(1) << 23; + + lzma_info_size size_type; + bool want_extra; + + // If we haven't decoded any Data Blocks yet, this is Header + // Metadata Block. + if (lzma_info_index_count_get(coder->info) == 0) { + coder->block_options.has_backward_size = false; + coder->block_options.handle_padding = true; + size_type = LZMA_INFO_HEADER_METADATA; + want_extra = coder->header_extra != NULL; + } else { + if (lzma_info_index_finish(coder->info)) + return LZMA_DATA_ERROR; + + coder->block_options.has_backward_size = true; + coder->block_options.handle_padding = false; + size_type = LZMA_INFO_FOOTER_METADATA; + want_extra = coder->footer_extra != NULL; + } + + coder->block_options.has_uncompressed_size_in_footer = false; + coder->block_options.total_size = lzma_info_size_get( + coder->info, size_type); + + coder->sequence = SEQ_METADATA_CODE; + + return lzma_metadata_decoder_init(&coder->block_decoder, allocator, + &coder->block_options, &coder->metadata, want_extra); +} + + +static lzma_ret +data_init(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_ret ret = lzma_info_iter_next(&coder->iter, allocator); + if (ret != LZMA_OK) + return ret; + + ret = lzma_info_iter_set(&coder->iter, LZMA_VLI_VALUE_UNKNOWN, + coder->block_options.uncompressed_size); + if (ret != LZMA_OK) + return ret; + + coder->block_options.total_size = coder->iter.total_size; + coder->block_options.uncompressed_size = coder->iter.uncompressed_size; + coder->block_options.total_limit = coder->total_left; + coder->block_options.uncompressed_limit = coder->uncompressed_left; + + if (coder->header_flags.is_multi) { + coder->block_options.has_uncompressed_size_in_footer = false; + coder->block_options.has_backward_size = false; + coder->block_options.handle_padding = true; + } else { + coder->block_options.has_uncompressed_size_in_footer + = coder->iter.uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN; + coder->block_options.has_backward_size = true; + coder->block_options.handle_padding = false; + } + + coder->sequence = SEQ_DATA_CODE; + + return lzma_block_decoder_init(&coder->block_decoder, allocator, + &coder->block_options); +} + + +static lzma_ret +stream_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + while (*out_pos < out_size && (*in_pos < in_size + || coder->sequence == SEQ_DATA_CODE)) + switch (coder->sequence) { + case SEQ_STREAM_HEADER_CODE: { + const lzma_ret ret = coder->flags_decoder.code( + coder->flags_decoder.coder, + allocator, in, in_pos, in_size, + NULL, NULL, 0, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + coder->sequence = SEQ_BLOCK_HEADER_INIT; + + // Detect if the Check type is supported and give appropriate + // warning if it isn't. We don't warn every time a new Block + // is started. + lzma_check tmp; + if (lzma_check_init(&tmp, coder->header_flags.check)) + return LZMA_UNSUPPORTED_CHECK; + + break; + } + + case SEQ_BLOCK_HEADER_INIT: { + coder->block_options.check = coder->header_flags.check; + coder->block_options.has_crc32 = coder->header_flags.has_crc32; + + const lzma_ret ret = lzma_block_header_decoder_init( + &coder->block_header_decoder, allocator, + &coder->block_options); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_BLOCK_HEADER_CODE; + } + + // Fall through + + case SEQ_BLOCK_HEADER_CODE: { + lzma_ret ret = coder->block_header_decoder.code( + coder->block_header_decoder.coder, + allocator, in, in_pos, in_size, + NULL, NULL, 0, LZMA_RUN); + + if (ret != LZMA_STREAM_END) + return ret; + + if (coder->block_options.is_metadata) + ret = metadata_init(coder, allocator); + else + ret = data_init(coder, allocator); + + if (ret != LZMA_OK) + return ret; + + break; + } + + case SEQ_METADATA_CODE: { + lzma_ret ret = coder->block_decoder.code( + coder->block_decoder.coder, allocator, + in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + const bool is_header_metadata = lzma_info_index_count_get( + coder->info) == 0; + + if (is_header_metadata) { + if (coder->header_extra != NULL) { + *coder->header_extra = coder->metadata.extra; + coder->metadata.extra = NULL; + } + + if (lzma_info_size_set(coder->info, + LZMA_INFO_HEADER_METADATA, + coder->block_options.total_size) + != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_BLOCK_HEADER_INIT; + } else { + if (coder->footer_extra != NULL) { + *coder->footer_extra = coder->metadata.extra; + coder->metadata.extra = NULL; + } + + coder->sequence = SEQ_STREAM_TAIL_INIT; + } + + assert(coder->metadata.extra == NULL); + + ret = lzma_info_metadata_set(coder->info, allocator, + &coder->metadata, is_header_metadata, true); + if (ret != LZMA_OK) + return ret; + + // Intialize coder->total_size and coder->uncompressed_size + // from Header Metadata. + if (is_header_metadata) { + coder->total_left = lzma_info_size_get( + coder->info, LZMA_INFO_TOTAL); + coder->uncompressed_left = lzma_info_size_get( + coder->info, LZMA_INFO_UNCOMPRESSED); + } + + break; + } + + case SEQ_DATA_CODE: { + lzma_ret ret = coder->block_decoder.code( + coder->block_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + + if (ret != LZMA_STREAM_END) + return ret; + + ret = lzma_info_iter_set(&coder->iter, + coder->block_options.total_size, + coder->block_options.uncompressed_size); + if (ret != LZMA_OK) + return ret; + + // These won't overflow since lzma_info_iter_set() succeeded. + if (coder->total_left != LZMA_VLI_VALUE_UNKNOWN) + coder->total_left -= coder->block_options.total_size; + if (coder->uncompressed_left != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_left -= coder->block_options + .uncompressed_size; + + if (!coder->header_flags.is_multi) { + ret = lzma_info_index_finish(coder->info); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_STREAM_TAIL_INIT; + break; + } + + coder->sequence = SEQ_BLOCK_HEADER_INIT; + break; + } + + case SEQ_STREAM_TAIL_INIT: { + lzma_ret ret = lzma_info_index_finish(coder->info); + if (ret != LZMA_OK) + return ret; + + ret = lzma_stream_tail_decoder_init(&coder->flags_decoder, + allocator, &coder->tail_flags); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_STREAM_TAIL_CODE; + } + + // Fall through + + case SEQ_STREAM_TAIL_CODE: { + const lzma_ret ret = coder->flags_decoder.code( + coder->flags_decoder.coder, allocator, + in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + if (!lzma_stream_flags_is_equal( + coder->header_flags, coder->tail_flags)) + return LZMA_DATA_ERROR; + + return LZMA_STREAM_END; + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->block_decoder, allocator); + lzma_next_coder_end(&coder->block_header_decoder, allocator); + lzma_next_coder_end(&coder->flags_decoder, allocator); + lzma_info_free(coder->info, allocator); + lzma_index_free(coder->metadata.index, allocator); + lzma_extra_free(coder->metadata.extra, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_extra **header, lzma_extra **footer) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &stream_decode; + next->end = &stream_decoder_end; + + next->coder->block_decoder = LZMA_NEXT_CODER_INIT; + next->coder->block_header_decoder = LZMA_NEXT_CODER_INIT; + next->coder->info = NULL; + next->coder->flags_decoder = LZMA_NEXT_CODER_INIT; + next->coder->metadata.index = NULL; + next->coder->metadata.extra = NULL; + } else { + lzma_index_free(next->coder->metadata.index, allocator); + next->coder->metadata.index = NULL; + + lzma_extra_free(next->coder->metadata.extra, allocator); + next->coder->metadata.extra = NULL; + } + + next->coder->info = lzma_info_init(next->coder->info, allocator); + if (next->coder->info == NULL) + return LZMA_MEM_ERROR; + + lzma_info_iter_begin(next->coder->info, &next->coder->iter); + + // Initialize Stream Header decoder. + return_if_error(lzma_stream_header_decoder_init( + &next->coder->flags_decoder, allocator, + &next->coder->header_flags)); + + // Reset the *foo_extra pointers to NULL. This way the caller knows + // if there were no Extra Records. (We don't support appending + // Records to Extra list.) + if (header != NULL) + *header = NULL; + if (footer != NULL) + *footer = NULL; + + // Reset some variables. + next->coder->sequence = SEQ_STREAM_HEADER_CODE; + next->coder->pos = 0; + next->coder->uncompressed_left = LZMA_VLI_VALUE_UNKNOWN; + next->coder->total_left = LZMA_VLI_VALUE_UNKNOWN; + next->coder->header_extra = header; + next->coder->footer_extra = footer; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_extra **header, lzma_extra **footer) +{ + lzma_next_coder_init( + stream_decoder_init, next, allocator, header, footer); +} + + +extern LZMA_API lzma_ret +lzma_stream_decoder(lzma_stream *strm, + lzma_extra **header, lzma_extra **footer) +{ + lzma_next_strm_init(strm, stream_decoder_init, header, footer); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/stream_encoder_multi.c b/src/liblzma/common/stream_encoder_multi.c new file mode 100644 index 00000000..5955f858 --- /dev/null +++ b/src/liblzma/common/stream_encoder_multi.c @@ -0,0 +1,460 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_multi.c +/// \brief Encodes Multi-Block .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_common.h" +#include "block_encoder.h" +#include "metadata_encoder.h" + + +struct lzma_coder_s { + enum { + SEQ_STREAM_HEADER_COPY, + SEQ_HEADER_METADATA_INIT, + SEQ_HEADER_METADATA_COPY, + SEQ_HEADER_METADATA_CODE, + SEQ_DATA_INIT, + SEQ_DATA_COPY, + SEQ_DATA_CODE, + SEQ_FOOTER_METADATA_INIT, + SEQ_FOOTER_METADATA_COPY, + SEQ_FOOTER_METADATA_CODE, + SEQ_STREAM_FOOTER_INIT, + SEQ_STREAM_FOOTER_COPY, + } sequence; + + /// Block or Metadata encoder + lzma_next_coder next; + + /// Options for the Block encoder + lzma_options_block block_options; + + /// Information about the Stream + lzma_info *info; + + /// Information about the current Data Block + lzma_info_iter iter; + + /// Pointer to user-supplied options structure. We don't write to + /// it, only read instructions from the application, thus this is + /// const even though the user-supplied pointer from + /// lzma_options_filter structure isn't. + const lzma_options_stream *stream_options; + + /// Stream Header or Stream Footer in encoded form + uint8_t *header; + size_t header_pos; + size_t header_size; +}; + + +typedef enum { + BLOCK_HEADER_METADATA, + BLOCK_DATA, + BLOCK_FOOTER_METADATA, +} block_type; + + +static lzma_ret +block_header_encode(lzma_coder *coder, lzma_allocator *allocator, + lzma_vli uncompressed_size, block_type type) +{ + assert(coder->header == NULL); + + coder->block_options = (lzma_options_block){ + .check = coder->stream_options->check, + .has_crc32 = coder->stream_options->has_crc32, + .has_eopm = true, + .is_metadata = type != BLOCK_DATA, + .has_uncompressed_size_in_footer = false, + .has_backward_size = type == BLOCK_FOOTER_METADATA, + .handle_padding = false, + .total_size = LZMA_VLI_VALUE_UNKNOWN, + .compressed_size = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_size = uncompressed_size, + .compressed_reserve = 0, + .uncompressed_reserve = 0, + .total_limit = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_limit = LZMA_VLI_VALUE_UNKNOWN, + .padding = LZMA_BLOCK_HEADER_PADDING_AUTO, + }; + + if (type == BLOCK_DATA) { + memcpy(coder->block_options.filters, + coder->stream_options->filters, + sizeof(coder->stream_options->filters)); + coder->block_options.alignment = coder->iter.stream_offset; + } else { + memcpy(coder->block_options.filters, + coder->stream_options->metadata_filters, + sizeof(coder->stream_options->filters)); + coder->block_options.alignment + = lzma_info_metadata_alignment_get( + coder->info, type == BLOCK_HEADER_METADATA); + } + + lzma_ret ret = lzma_block_header_size(&coder->block_options); + if (ret != LZMA_OK) + return ret; + + coder->header_size = coder->block_options.header_size; + coder->header = lzma_alloc(coder->header_size, allocator); + if (coder->header == NULL) + return LZMA_MEM_ERROR; + + ret = lzma_block_header_encode(coder->header, &coder->block_options); + if (ret != LZMA_OK) + return ret; + + coder->header_pos = 0; + return LZMA_OK; +} + + +static lzma_ret +metadata_encoder_init(lzma_coder *coder, lzma_allocator *allocator, + lzma_metadata *metadata, block_type type) +{ + lzma_ret ret = lzma_info_metadata_set(coder->info, allocator, + metadata, type == BLOCK_HEADER_METADATA, false); + if (ret != LZMA_OK) + return ret; + + const lzma_vli metadata_size = lzma_metadata_size(metadata); + if (metadata_size == 0) + return LZMA_PROG_ERROR; + + ret = block_header_encode(coder, allocator, metadata_size, type); + if (ret != LZMA_OK) + return ret; + + return lzma_metadata_encoder_init(&coder->next, allocator, + &coder->block_options, metadata); +} + + +static lzma_ret +data_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_ret ret = lzma_info_iter_next(&coder->iter, allocator); + if (ret != LZMA_OK) + return ret; + + ret = block_header_encode(coder, allocator, + LZMA_VLI_VALUE_UNKNOWN, BLOCK_DATA); + if (ret != LZMA_OK) + return ret; + + return lzma_block_encoder_init(&coder->next, allocator, + &coder->block_options); +} + + +static lzma_ret +stream_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Main loop + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_STREAM_HEADER_COPY: + case SEQ_HEADER_METADATA_COPY: + case SEQ_DATA_COPY: + case SEQ_FOOTER_METADATA_COPY: + case SEQ_STREAM_FOOTER_COPY: + bufcpy(coder->header, &coder->header_pos, coder->header_size, + out, out_pos, out_size); + if (coder->header_pos < coder->header_size) + return LZMA_OK; + + lzma_free(coder->header, allocator); + coder->header = NULL; + + switch (coder->sequence) { + case SEQ_STREAM_HEADER_COPY: + // Write Header Metadata Block if we have Extra for it + // or known Uncompressed Size. + if (coder->stream_options->header != NULL + || coder->stream_options + ->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + coder->sequence = SEQ_HEADER_METADATA_INIT; + } else { + // Mark that Header Metadata Block doesn't + // exist. + if (lzma_info_size_set(coder->info, + LZMA_INFO_HEADER_METADATA, 0) + != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_DATA_INIT; + } + break; + + case SEQ_HEADER_METADATA_COPY: + case SEQ_DATA_COPY: + case SEQ_FOOTER_METADATA_COPY: + ++coder->sequence; + break; + + case SEQ_STREAM_FOOTER_COPY: + return LZMA_STREAM_END; + + default: + assert(0); + } + + break; + + case SEQ_HEADER_METADATA_INIT: { + lzma_metadata metadata = { + .header_metadata_size = LZMA_VLI_VALUE_UNKNOWN, + .total_size = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_size = coder->stream_options + ->uncompressed_size, + .index = NULL, + .extra = coder->stream_options->header, + }; + + const lzma_ret ret = metadata_encoder_init(coder, allocator, + &metadata, BLOCK_HEADER_METADATA); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_HEADER_METADATA_COPY; + break; + } + + case SEQ_FOOTER_METADATA_INIT: { + lzma_metadata metadata = { + .header_metadata_size + = lzma_info_size_get(coder->info, + LZMA_INFO_HEADER_METADATA), + .total_size = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_size = LZMA_VLI_VALUE_UNKNOWN, + .index = lzma_info_index_get(coder->info, false), + .extra = coder->stream_options->footer, + }; + + const lzma_ret ret = metadata_encoder_init(coder, allocator, + &metadata, BLOCK_FOOTER_METADATA); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_FOOTER_METADATA_COPY; + break; + } + + case SEQ_HEADER_METADATA_CODE: + case SEQ_FOOTER_METADATA_CODE: { + size_t dummy = 0; + lzma_ret ret = coder->next.code(coder->next.coder, + allocator, NULL, &dummy, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + ret = lzma_info_size_set(coder->info, + coder->sequence == SEQ_HEADER_METADATA_CODE + ? LZMA_INFO_HEADER_METADATA + : LZMA_INFO_FOOTER_METADATA, + coder->block_options.total_size); + if (ret != LZMA_OK) + return ret; + + ++coder->sequence; + break; + } + + case SEQ_DATA_INIT: { + // Don't create an empty Block unless it would be + // the only Data Block. + if (*in_pos == in_size) { + if (action != LZMA_FINISH) + return LZMA_OK; + + if (lzma_info_index_count_get(coder->info) != 0) { + if (lzma_info_index_finish(coder->info)) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_FOOTER_METADATA_INIT; + break; + } + } + + const lzma_ret ret = data_encoder_init(coder, allocator); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_DATA_COPY; + break; + } + + case SEQ_DATA_CODE: { + static const lzma_action convert[4] = { + LZMA_RUN, + LZMA_SYNC_FLUSH, + LZMA_FINISH, + LZMA_FINISH, + }; + + lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, convert[action]); + if (ret != LZMA_STREAM_END || action == LZMA_SYNC_FLUSH) + return ret; + + ret = lzma_info_iter_set(&coder->iter, + coder->block_options.total_size, + coder->block_options.uncompressed_size); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_DATA_INIT; + break; + } + + case SEQ_STREAM_FOOTER_INIT: { + assert(coder->header == NULL); + + lzma_stream_flags flags = { + .check = coder->stream_options->check, + .has_crc32 = coder->stream_options->has_crc32, + .is_multi = true, + }; + + coder->header = lzma_alloc(LZMA_STREAM_TAIL_SIZE, allocator); + if (coder->header == NULL) + return LZMA_MEM_ERROR; + + const lzma_ret ret = lzma_stream_tail_encode( + coder->header, &flags); + if (ret != LZMA_OK) + return ret; + + coder->header_size = LZMA_STREAM_TAIL_SIZE; + coder->header_pos = 0; + + coder->sequence = SEQ_STREAM_FOOTER_COPY; + break; + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_info_free(coder->info, allocator); + lzma_free(coder->header, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +stream_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_stream *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &stream_encode; + next->end = &stream_encoder_end; + + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->info = NULL; + } else { + lzma_free(next->coder->header, allocator); + } + + next->coder->header = NULL; + + next->coder->info = lzma_info_init(next->coder->info, allocator); + if (next->coder->info == NULL) + return LZMA_MEM_ERROR; + + next->coder->sequence = SEQ_STREAM_HEADER_COPY; + next->coder->stream_options = options; + + // Encode Stream Flags + { + lzma_stream_flags flags = { + .check = options->check, + .has_crc32 = options->has_crc32, + .is_multi = true, + }; + + next->coder->header = lzma_alloc(LZMA_STREAM_HEADER_SIZE, + allocator); + if (next->coder->header == NULL) + return LZMA_MEM_ERROR; + + return_if_error(lzma_stream_header_encode( + next->coder->header, &flags)); + + next->coder->header_pos = 0; + next->coder->header_size = LZMA_STREAM_HEADER_SIZE; + } + + if (lzma_info_size_set(next->coder->info, LZMA_INFO_STREAM_START, + options->alignment) != LZMA_OK) + return LZMA_PROG_ERROR; + + lzma_info_iter_begin(next->coder->info, &next->coder->iter); + + return LZMA_OK; +} + + +/* +extern lzma_ret +lzma_stream_encoder_multi_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_stream *options) +{ + lzma_next_coder_init(stream_encoder_init, next, allocator, options); +} +*/ + + +extern LZMA_API lzma_ret +lzma_stream_encoder_multi( + lzma_stream *strm, const lzma_options_stream *options) +{ + lzma_next_strm_init(strm, stream_encoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/stream_encoder_single.c b/src/liblzma/common/stream_encoder_single.c new file mode 100644 index 00000000..e8efd004 --- /dev/null +++ b/src/liblzma/common/stream_encoder_single.c @@ -0,0 +1,220 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_single.c +/// \brief Encodes Single-Block .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_common.h" +#include "block_encoder.h" + + +struct lzma_coder_s { + /// Uncompressed Size, Backward Size, and Footer Magic Bytes are + /// part of Block in the file format specification, but it is simpler + /// to implement them as part of Stream. + enum { + SEQ_HEADERS, + SEQ_DATA, + SEQ_FOOTER, + } sequence; + + /// Block encoder + lzma_next_coder block_encoder; + + /// Block encoder options + lzma_options_block block_options; + + /// Stream Flags; we need to have these in this struct so that we + /// can encode Stream Footer. + lzma_stream_flags stream_flags; + + /// Stream Header + Block Header, or Stream Footer + uint8_t *header; + size_t header_pos; + size_t header_size; +}; + + +static lzma_ret +stream_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, size_t *out_pos, + size_t out_size, lzma_action action) +{ + // NOTE: We don't check if the amount of input is in the proper limits, + // because the Block encoder will do it for us. + + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_HEADERS: + bufcpy(coder->header, &coder->header_pos, coder->header_size, + out, out_pos, out_size); + + if (coder->header_pos == coder->header_size) { + coder->header_pos = 0; + coder->sequence = SEQ_DATA; + } + + break; + + case SEQ_DATA: { + lzma_ret ret = coder->block_encoder.code( + coder->block_encoder.coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, action); + if (ret != LZMA_STREAM_END || action == LZMA_SYNC_FLUSH) + return ret; + + assert(*in_pos == in_size); + + assert(coder->header_size >= LZMA_STREAM_TAIL_SIZE); + coder->header_size = LZMA_STREAM_TAIL_SIZE; + + ret = lzma_stream_tail_encode( + coder->header, &coder->stream_flags); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_FOOTER; + break; + } + + case SEQ_FOOTER: + bufcpy(coder->header, &coder->header_pos, coder->header_size, + out, out_pos, out_size); + + return coder->header_pos == coder->header_size + ? LZMA_STREAM_END : LZMA_OK; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->block_encoder, allocator); + lzma_free(coder->header, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +stream_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_stream *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &stream_encode; + next->end = &stream_encoder_end; + next->coder->block_encoder = LZMA_NEXT_CODER_INIT; + } else { + // Free the previous buffer, if any. + lzma_free(next->coder->header, allocator); + } + + // At this point, next->coder->header points to nothing useful. + next->coder->header = NULL; + + // Basic initializations + next->coder->sequence = SEQ_HEADERS; + next->coder->header_pos = 0; + + // Initialize next->coder->stream_flags. + next->coder->stream_flags = (lzma_stream_flags){ + .check = options->check, + .has_crc32 = options->has_crc32, + .is_multi = false, + }; + + // Initialize next->coder->block_options. + next->coder->block_options = (lzma_options_block){ + .check = options->check, + .has_crc32 = options->has_crc32, + .has_eopm = options->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN, + .is_metadata = false, + .has_uncompressed_size_in_footer = options->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN, + .has_backward_size = true, + .handle_padding = false, + .compressed_size = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_size = options->uncompressed_size, + .compressed_reserve = 0, + .uncompressed_reserve = 0, + .total_size = LZMA_VLI_VALUE_UNKNOWN, + .total_limit = LZMA_VLI_VALUE_UNKNOWN, + .uncompressed_limit = LZMA_VLI_VALUE_UNKNOWN, + .padding = LZMA_BLOCK_HEADER_PADDING_AUTO, + .alignment = options->alignment + LZMA_STREAM_HEADER_SIZE, + }; + memcpy(next->coder->block_options.filters, options->filters, + sizeof(options->filters)); + + return_if_error(lzma_block_header_size(&next->coder->block_options)); + + // Encode Stream Flags and Block Header into next->coder->header. + next->coder->header_size = (size_t)(LZMA_STREAM_HEADER_SIZE) + + next->coder->block_options.header_size; + next->coder->header = lzma_alloc(next->coder->header_size, allocator); + if (next->coder->header == NULL) + return LZMA_MEM_ERROR; + + return_if_error(lzma_stream_header_encode(next->coder->header, + &next->coder->stream_flags)); + + return_if_error(lzma_block_header_encode( + next->coder->header + LZMA_STREAM_HEADER_SIZE, + &next->coder->block_options)); + + // Initialize the Block encoder. + return lzma_block_encoder_init(&next->coder->block_encoder, allocator, + &next->coder->block_options); +} + + +/* +extern lzma_ret +lzma_stream_encoder_single_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_options_stream *options) +{ + lzma_next_coder_init(stream_encoder_init, allocator, options); +} +*/ + + +extern LZMA_API lzma_ret +lzma_stream_encoder_single( + lzma_stream *strm, const lzma_options_stream *options) +{ + lzma_next_strm_init(strm, stream_encoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/stream_flags_decoder.c b/src/liblzma/common/stream_flags_decoder.c new file mode 100644 index 00000000..d9c847ac --- /dev/null +++ b/src/liblzma/common/stream_flags_decoder.c @@ -0,0 +1,258 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_decoder.c +/// \brief Decodes Stream Header and tail from .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_flags_decoder.h" +#include "stream_common.h" + + +//////////// +// Common // +//////////// + +struct lzma_coder_s { + enum { + SEQ_HEADER_MAGIC, + SEQ_HEADER_FLAGS, + SEQ_HEADER_CRC32, + + SEQ_FOOTER_FLAGS, + SEQ_FOOTER_MAGIC, + } sequence; + + size_t pos; + uint32_t crc32; + + lzma_stream_flags *options; +}; + + +static void +stream_header_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder, allocator); + return; +} + + +static bool +stream_flags_decode(const uint8_t *in, lzma_stream_flags *options) +{ + // Reserved bits must be unset. + if (*in & 0xE0) + return true; + + options->check = *in & 0x07; + options->has_crc32 = (*in & 0x08) != 0; + options->is_multi = (*in & 0x10) != 0; + + return false; +} + + +//////////// +// Header // +//////////// + +static lzma_ret +stream_header_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + while (*in_pos < in_size) + switch (coder->sequence) { + case SEQ_HEADER_MAGIC: + if (in[*in_pos] != lzma_header_magic[coder->pos]) + return LZMA_DATA_ERROR; + + ++*in_pos; + + if (++coder->pos == sizeof(lzma_header_magic)) { + coder->pos = 0; + coder->sequence = SEQ_HEADER_FLAGS; + } + + break; + + case SEQ_HEADER_FLAGS: + if (stream_flags_decode(in + *in_pos, coder->options)) + return LZMA_HEADER_ERROR; + + coder->crc32 = lzma_crc32(in + *in_pos, 1, 0); + + ++*in_pos; + coder->sequence = SEQ_HEADER_CRC32; + break; + + case SEQ_HEADER_CRC32: + if (in[*in_pos] != ((coder->crc32 >> (coder->pos * 8)) & 0xFF)) + return LZMA_DATA_ERROR; + + ++*in_pos; + + if (++coder->pos == 4) + return LZMA_STREAM_END; + + break; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +stream_header_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + } + + // Set the function pointers unconditionally, because they may + // have been pointing to footer decoder too. + next->code = &stream_header_decode; + next->end = &stream_header_decoder_end; + + next->coder->sequence = SEQ_HEADER_MAGIC; + next->coder->pos = 0; + next->coder->crc32 = 0; + next->coder->options = options; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_stream_header_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options) +{ + lzma_next_coder_init( + stream_header_decoder_init, next, allocator, options); +} + + +extern LZMA_API lzma_ret +lzma_stream_header_decoder(lzma_stream *strm, lzma_stream_flags *options) +{ + lzma_next_strm_init(strm, stream_header_decoder_init, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} + + +////////// +// Tail // +////////// + +static lzma_ret +stream_tail_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + while (*in_pos < in_size) + switch (coder->sequence) { + case SEQ_FOOTER_FLAGS: + if (stream_flags_decode(in + *in_pos, coder->options)) + return LZMA_HEADER_ERROR; + + ++*in_pos; + coder->sequence = SEQ_FOOTER_MAGIC; + break; + + case SEQ_FOOTER_MAGIC: + if (in[*in_pos] != lzma_footer_magic[coder->pos]) + return LZMA_DATA_ERROR; + + ++*in_pos; + + if (++coder->pos == sizeof(lzma_footer_magic)) + return LZMA_STREAM_END; + + break; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +stream_tail_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + } + + // Set the function pointers unconditionally, because they may + // have been pointing to footer decoder too. + next->code = &stream_tail_decode; + next->end = &stream_header_decoder_end; + + next->coder->sequence = SEQ_FOOTER_FLAGS; + next->coder->pos = 0; + next->coder->options = options; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_stream_tail_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options) +{ + lzma_next_coder_init2(next, allocator, stream_header_decoder_init, + stream_tail_decoder_init, allocator, options); +} + + +extern LZMA_API lzma_ret +lzma_stream_tail_decoder(lzma_stream *strm, lzma_stream_flags *options) +{ + lzma_next_strm_init2(strm, stream_header_decoder_init, + stream_tail_decoder_init, strm->allocator, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} diff --git a/src/liblzma/common/stream_flags_decoder.h b/src/liblzma/common/stream_flags_decoder.h new file mode 100644 index 00000000..e4b8e3c5 --- /dev/null +++ b/src/liblzma/common/stream_flags_decoder.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_decoder.h +/// \brief Decodes Stream Header and Footer from .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_STREAM_FLAGS_DECODER_H +#define LZMA_STREAM_FLAGS_DECODER_H + +#include "common.h" + +extern lzma_ret lzma_stream_header_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options); + +extern lzma_ret lzma_stream_tail_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_stream_flags *options); + +#endif diff --git a/src/liblzma/common/stream_flags_encoder.c b/src/liblzma/common/stream_flags_encoder.c new file mode 100644 index 00000000..55468580 --- /dev/null +++ b/src/liblzma/common/stream_flags_encoder.c @@ -0,0 +1,75 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_encoder.c +/// \brief Encodes Stream Header and Footer for .lzma files +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_common.h" + + +static bool +stream_flags_encode(uint8_t *flags_byte, const lzma_stream_flags *options) +{ + // Check type + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return true; + + *flags_byte = options->check; + + // Usage of CRC32 in Block Headers + if (options->has_crc32) + *flags_byte |= 0x08; + + // Single- or Multi-Block + if (options->is_multi) + *flags_byte |= 0x10; + + return false; +} + + +extern LZMA_API lzma_ret +lzma_stream_header_encode(uint8_t *out, const lzma_stream_flags *options) +{ + // Magic + memcpy(out, lzma_header_magic, sizeof(lzma_header_magic)); + + // Stream Flags + if (stream_flags_encode(out + sizeof(lzma_header_magic), options)) + return LZMA_PROG_ERROR;; + + // CRC32 of the Stream Header + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), 1, 0); + + for (size_t i = 0; i < 4; ++i) + out[sizeof(lzma_header_magic) + 1 + i] = crc >> (i * 8); + + return LZMA_OK; +} + + +extern LZMA_API lzma_ret +lzma_stream_tail_encode(uint8_t *out, const lzma_stream_flags *options) +{ + // Stream Flags + if (stream_flags_encode(out, options)) + return LZMA_PROG_ERROR; + + // Magic + memcpy(out + 1, lzma_footer_magic, sizeof(lzma_footer_magic)); + + return LZMA_OK; +} diff --git a/src/liblzma/common/sysdefs.h b/src/liblzma/common/sysdefs.h new file mode 120000 index 00000000..c6cb6768 --- /dev/null +++ b/src/liblzma/common/sysdefs.h @@ -0,0 +1 @@ +../../common/sysdefs.h
\ No newline at end of file diff --git a/src/liblzma/common/version.c b/src/liblzma/common/version.c new file mode 100644 index 00000000..dffec7ff --- /dev/null +++ b/src/liblzma/common/version.c @@ -0,0 +1,25 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file version.c +/// \brief liblzma version number +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +LZMA_API const uint32_t lzma_version_number = LZMA_VERSION; + +LZMA_API const char *const lzma_version_string = PACKAGE_VERSION; diff --git a/src/liblzma/common/vli_decoder.c b/src/liblzma/common/vli_decoder.c new file mode 100644 index 00000000..2b89c1a7 --- /dev/null +++ b/src/liblzma/common/vli_decoder.c @@ -0,0 +1,69 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_decoder.c +/// \brief Decodes variable-length integers +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API lzma_ret +lzma_vli_decode(lzma_vli *restrict vli, size_t *restrict vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size) +{ + if (*vli > LZMA_VLI_VALUE_MAX || *vli_pos >= 9 + || (*vli >> (7 * *vli_pos)) != 0) + return LZMA_PROG_ERROR; + + if (*in_pos >= in_size) + return LZMA_BUF_ERROR; + + if (*vli_pos == 0) { + *vli_pos = 1; + + if (in[*in_pos] <= 0x7F) { + // Single-byte integer + *vli = in[*in_pos]; + ++*in_pos; + return LZMA_STREAM_END; + } + + *vli = in[*in_pos] & 0x7F; + ++*in_pos; + } + + while (*in_pos < in_size) { + // Read in the next byte. + *vli |= (lzma_vli)(in[*in_pos] & 0x7F) << (*vli_pos * 7); + ++*vli_pos; + + // Check if this is the last byte of a multibyte integer. + if (in[*in_pos] & 0x80) { + ++*in_pos; + return LZMA_STREAM_END; + } + + // Limit variable-length representation to nine bytes. + if (*vli_pos == 9) + return LZMA_DATA_ERROR; + + // Increment input position only when the byte was accepted. + ++*in_pos; + } + + return LZMA_OK; +} diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c new file mode 100644 index 00000000..1ecdb0d2 --- /dev/null +++ b/src/liblzma/common/vli_encoder.c @@ -0,0 +1,81 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_encoder.c +/// \brief Encodes variable-length integers +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API lzma_ret +lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos, size_t vli_size, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + if (vli > LZMA_VLI_VALUE_MAX || *vli_pos >= 9 || vli_size > 9 + || (vli != 0 && (vli >> (7 * *vli_pos)) == 0)) + return LZMA_PROG_ERROR; + + if (*out_pos >= out_size) + return LZMA_BUF_ERROR; + + if (*vli_pos == 0) { + *vli_pos = 1; + + if (vli <= 0x7F && *vli_pos >= vli_size) { + // Single-byte integer + out[(*out_pos)++] = vli; + return LZMA_STREAM_END; + } + + // First byte of a multibyte integer + out[(*out_pos)++] = (vli & 0x7F) | 0x80; + } + + while (*out_pos < out_size) { + const lzma_vli b = vli >> (7 * *vli_pos); + ++*vli_pos; + + if (b <= 0x7F && *vli_pos >= vli_size) { + // Last byte of a multibyte integer + out[(*out_pos)++] = (b & 0xFF) | 0x80; + return LZMA_STREAM_END; + } + + // Middle byte of a multibyte integer + out[(*out_pos)++] = b & 0x7F; + } + + // vli is not yet completely written out. + return LZMA_OK; +} + + +extern LZMA_API size_t +lzma_vli_size(lzma_vli vli) +{ + if (vli > LZMA_VLI_VALUE_MAX) + return 0; + + size_t i = 0; + do { + vli >>= 7; + ++i; + } while (vli != 0); + + assert(i <= 9); + return i; +} diff --git a/src/liblzma/common/vli_reverse_decoder.c b/src/liblzma/common/vli_reverse_decoder.c new file mode 100644 index 00000000..68ca6a42 --- /dev/null +++ b/src/liblzma/common/vli_reverse_decoder.c @@ -0,0 +1,55 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_reverse_decoder.c +/// \brief Decodes variable-length integers starting at end of the buffer +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API lzma_ret +lzma_vli_reverse_decode(lzma_vli *vli, const uint8_t *in, size_t *in_size) +{ + if (*in_size == 0) + return LZMA_BUF_ERROR; + + size_t i = *in_size - 1; + *vli = in[i] & 0x7F; + + if (!(in[i] & 0x80)) { + *in_size = i; + return LZMA_OK; + } + + const size_t end = *in_size > LZMA_VLI_BYTES_MAX + ? *in_size - LZMA_VLI_BYTES_MAX : 0; + + do { + if (i-- == end) { + if (*in_size < LZMA_VLI_BYTES_MAX) + return LZMA_BUF_ERROR; + + return LZMA_DATA_ERROR; + } + + *vli <<= 7; + *vli = in[i] & 0x7F; + + } while (!(in[i] & 0x80)); + + *in_size = i; + return LZMA_OK; +} diff --git a/src/liblzma/lz/Makefile.am b/src/liblzma/lz/Makefile.am new file mode 100644 index 00000000..5c27e2f2 --- /dev/null +++ b/src/liblzma/lz/Makefile.am @@ -0,0 +1,63 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = liblz.la +liblz_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common \ + -I@top_srcdir@/src/liblzma/check +liblz_la_SOURCES = + + +if COND_MAIN_ENCODER +liblz_la_SOURCES += \ + lz_encoder.c \ + lz_encoder.h \ + lz_encoder_private.h \ + match_c.h \ + match_h.h + +if COND_MF_HC3 +liblz_la_SOURCES += hc3.c hc3.h +liblz_la_CPPFLAGS += -DHAVE_HC3 +endif + +if COND_MF_HC4 +liblz_la_SOURCES += hc4.c hc4.h +liblz_la_CPPFLAGS += -DHAVE_HC4 +endif + +if COND_MF_BT2 +liblz_la_SOURCES += bt2.c bt2.h +liblz_la_CPPFLAGS += -DHAVE_BT2 +endif + +if COND_MF_BT3 +liblz_la_SOURCES += bt3.c bt3.h +liblz_la_CPPFLAGS += -DHAVE_BT3 +endif + +if COND_MF_BT4 +liblz_la_SOURCES += bt4.c bt4.h +liblz_la_CPPFLAGS += -DHAVE_BT4 +endif + +endif + + +if COND_MAIN_DECODER +liblz_la_SOURCES += \ + lz_decoder.c \ + lz_decoder.h +endif diff --git a/src/liblzma/lz/bt2.c b/src/liblzma/lz/bt2.c new file mode 100644 index 00000000..7dc4cb80 --- /dev/null +++ b/src/liblzma/lz/bt2.c @@ -0,0 +1,27 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt2.c +/// \brief Binary Tree 2 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "bt2.h" + +#undef IS_HASH_CHAIN +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#include "match_c.h" diff --git a/src/liblzma/lz/bt2.h b/src/liblzma/lz/bt2.h new file mode 100644 index 00000000..33cb52cd --- /dev/null +++ b/src/liblzma/lz/bt2.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt2.h +/// \brief Binary Tree 2 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BT2_H +#define LZMA_BT2_H + +#undef LZMA_MATCH_FINDER_NAME_LOWER +#undef LZMA_MATCH_FINDER_NAME_UPPER +#define LZMA_MATCH_FINDER_NAME_LOWER bt2 +#define LZMA_MATCH_FINDER_NAME_UPPER BT2 + +#include "match_h.h" + +#endif diff --git a/src/liblzma/lz/bt3.c b/src/liblzma/lz/bt3.c new file mode 100644 index 00000000..d44310f3 --- /dev/null +++ b/src/liblzma/lz/bt3.c @@ -0,0 +1,29 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt3.c +/// \brief Binary Tree 3 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "bt3.h" + +#undef IS_HASH_CHAIN +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#define HASH_ARRAY_2 + +#include "match_c.h" diff --git a/src/liblzma/lz/bt3.h b/src/liblzma/lz/bt3.h new file mode 100644 index 00000000..247c7e5f --- /dev/null +++ b/src/liblzma/lz/bt3.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt3.h +/// \brief Binary Tree 3 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BT3_H +#define LZMA_BT3_H + +#undef LZMA_MATCH_FINDER_NAME_LOWER +#undef LZMA_MATCH_FINDER_NAME_UPPER +#define LZMA_MATCH_FINDER_NAME_LOWER bt3 +#define LZMA_MATCH_FINDER_NAME_UPPER BT3 + +#include "match_h.h" + +#endif diff --git a/src/liblzma/lz/bt4.c b/src/liblzma/lz/bt4.c new file mode 100644 index 00000000..6e1042c9 --- /dev/null +++ b/src/liblzma/lz/bt4.c @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt4.c +/// \brief Binary Tree 4 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "bt4.h" + +#undef IS_HASH_CHAIN +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "match_c.h" diff --git a/src/liblzma/lz/bt4.h b/src/liblzma/lz/bt4.h new file mode 100644 index 00000000..e3fcf6ac --- /dev/null +++ b/src/liblzma/lz/bt4.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file bt4.h +/// \brief Binary Tree 4 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BT4_H +#define LZMA_BT4_H + +#undef LZMA_MATCH_FINDER_NAME_LOWER +#undef LZMA_MATCH_FINDER_NAME_UPPER +#define LZMA_MATCH_FINDER_NAME_LOWER bt4 +#define LZMA_MATCH_FINDER_NAME_UPPER BT4 + +#include "match_h.h" + +#endif diff --git a/src/liblzma/lz/hc3.c b/src/liblzma/lz/hc3.c new file mode 100644 index 00000000..22b5689b --- /dev/null +++ b/src/liblzma/lz/hc3.c @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hc3.c +/// \brief Hash Chain 3 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "hc3.h" + +#undef IS_HASH_CHAIN +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#define IS_HASH_CHAIN +#define HASH_ARRAY_2 + +#include "match_c.h" diff --git a/src/liblzma/lz/hc3.h b/src/liblzma/lz/hc3.h new file mode 100644 index 00000000..97be0b1d --- /dev/null +++ b/src/liblzma/lz/hc3.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hc3.h +/// \brief Hash Chain 3 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_HC3_H +#define LZMA_HC3_H + +#undef LZMA_MATCH_FINDER_NAME_LOWER +#undef LZMA_MATCH_FINDER_NAME_UPPER +#define LZMA_MATCH_FINDER_NAME_LOWER hc3 +#define LZMA_MATCH_FINDER_NAME_UPPER HC3 + +#include "match_h.h" + +#endif diff --git a/src/liblzma/lz/hc4.c b/src/liblzma/lz/hc4.c new file mode 100644 index 00000000..a55cfd09 --- /dev/null +++ b/src/liblzma/lz/hc4.c @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hc4.c +/// \brief Hash Chain 4 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "hc4.h" + +#undef IS_HASH_CHAIN +#undef HASH_ARRAY_2 +#undef HASH_ARRAY_3 + +#define IS_HASH_CHAIN +#define HASH_ARRAY_2 +#define HASH_ARRAY_3 + +#include "match_c.h" diff --git a/src/liblzma/lz/hc4.h b/src/liblzma/lz/hc4.h new file mode 100644 index 00000000..dc072e2f --- /dev/null +++ b/src/liblzma/lz/hc4.h @@ -0,0 +1,31 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hc4.h +/// \brief Hash Chain 4 +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_HC4_H +#define LZMA_HC4_H + +#undef LZMA_MATCH_FINDER_NAME_LOWER +#undef LZMA_MATCH_FINDER_NAME_UPPER +#define LZMA_MATCH_FINDER_NAME_LOWER hc4 +#define LZMA_MATCH_FINDER_NAME_UPPER HC4 + +#include "match_h.h" + +#endif diff --git a/src/liblzma/lz/lz_decoder.c b/src/liblzma/lz/lz_decoder.c new file mode 100644 index 00000000..9c110dec --- /dev/null +++ b/src/liblzma/lz/lz_decoder.c @@ -0,0 +1,462 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_decoder.c +/// \brief LZ out window +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_decoder.h" + + +/// Minimum size of allocated dictionary +#define DICT_SIZE_MIN 8192 + +/// When there is less than this amount of data available for decoding, +/// it is moved to the temporary buffer which +/// - protects from reads past the end of the buffer; and +/// - stored the incomplete data between lzma_code() calls. +/// +/// \note TEMP_LIMIT must be at least as much as +/// REQUIRED_IN_BUFFER_SIZE defined in lzma_decoder.c. +#define TEMP_LIMIT 32 + +// lzma_lz_decoder.dict[] must be three times the size of TEMP_LIMIT. +// 2 * TEMP_LIMIT is used for the actual data, and the third TEMP_LIMIT +// bytes is needed for safety to allow decode_dummy() in lzma_decoder.c +// to read past end of the buffer. This way it should be both fast and simple. +#if LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT +# error LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT +#endif + + +struct lzma_coder_s { + lzma_next_coder next; + lzma_lz_decoder lz; + + // There are more members in this structure but they are not + // visible in LZ coder. +}; + + +/// - Copy as much data as possible from lz->dict[] to out[]. +/// - Update *out_pos, lz->start, and lz->end accordingly. +/// - Wrap lz-pos to the beginning of lz->dict[] if there is a danger that +/// it may go past the end of the buffer (lz->pos >= lz->must_flush_pos). +static inline bool +flush(lzma_lz_decoder *restrict lz, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) +{ + // Flush uncompressed data from the history buffer to + // the output buffer. This is done in two phases. + + assert(lz->start <= lz->end); + + // Flush if pos < start < end. + if (lz->pos < lz->start && lz->start < lz->end) { + bufcpy(lz->dict, &lz->start, lz->end, out, out_pos, out_size); + + // If we reached end of the data in history buffer, + // wrap to the beginning. + if (lz->start == lz->end) + lz->start = 0; + } + + // Flush if start start < pos <= end. This is not as `else' for + // previous `if' because the previous one may make this one true. + if (lz->start < lz->pos) { + bufcpy(lz->dict, &lz->start, + lz->pos, out, out_pos, out_size); + + if (lz->pos >= lz->must_flush_pos) { + // Wrap the flushing position if we have + // flushed the whole history buffer. + if (lz->pos == lz->start) + lz->start = 0; + + // Wrap the write position and store to lz.end + // how much there is new data available. + lz->end = lz->pos; + lz->pos = 0; + lz->is_full = true; + } + } + + assert(lz->pos < lz->must_flush_pos); + + return *out_pos == out_size; +} + + +/// Calculate safe value for lz->limit. If no safe value can be found, +/// set lz->limit to zero. When flushing, only as little data will be +/// decoded as is needed to fill the output buffer (lowers both latency +/// and throughput). +/// +/// \return true if there is no space for new uncompressed data. +/// +static inline bool +set_limit(lzma_lz_decoder *lz, size_t out_avail, bool flushing) +{ + // Set the limit so that writing to dict[limit + match_max_len - 1] + // doesn't overwrite any unflushed data and doesn't write past the + // end of the dict buffer. + if (lz->start <= lz->pos) { + // We can fill the buffer from pos till the end + // of the dict buffer. + lz->limit = lz->must_flush_pos; + } else if (lz->pos + lz->match_max_len < lz->start) { + // There's some unflushed data between pos and end of the + // buffer. Limit so that we don't overwrite the unflushed data. + lz->limit = lz->start - lz->match_max_len; + } else { + // Buffer is too full. + lz->limit = 0; + return true; + } + + // Finetune the limit a bit if it isn't zero. + + assert(lz->limit > lz->pos); + const size_t dict_avail = lz->limit - lz->pos; + + if (lz->uncompressed_size < dict_avail) { + // Finishing a stream that doesn't have + // an end of stream marker. + lz->limit = lz->pos + lz->uncompressed_size; + + } else if (flushing && out_avail < dict_avail) { + // Flushing enabled, decoding only as little as needed to + // fill the out buffer (if there's enough input, of course). + lz->limit = lz->pos + out_avail; + } + + return lz->limit == lz->pos; +} + + +/// Takes care of wrapping the data into temporary buffer when needed, +/// and calls the actual decoder. +/// +/// \return true if error occurred +/// +static inline bool +call_process(lzma_coder *restrict coder, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) +{ + // It would be nice and simple if we could just give in[] to the + // decoder, but the requirement of zlib-like API forces us to be + // able to make *in_pos == in_size whenever there is enough output + // space. If needed, we will append a few bytes from in[] to + // a temporary buffer and decode enough to reach the part that + // was copied from in[]. Then we can continue with the real in[]. + + bool error; + const size_t dict_old_pos = coder->lz.pos; + const size_t in_avail = in_size - *in_pos; + + if (coder->lz.temp_size + in_avail < 2 * TEMP_LIMIT) { + // Copy all the available input from in[] to temp[]. + memcpy(coder->lz.temp + coder->lz.temp_size, + in + *in_pos, in_avail); + coder->lz.temp_size += in_avail; + *in_pos += in_avail; + assert(*in_pos == in_size); + + // Decode as much as possible. + size_t temp_used = 0; + error = coder->lz.process(coder, coder->lz.temp, &temp_used, + coder->lz.temp_size, true); + assert(temp_used <= coder->lz.temp_size); + + // Move the remaining data to the beginning of temp[]. + coder->lz.temp_size -= temp_used; + memmove(coder->lz.temp, coder->lz.temp + temp_used, + coder->lz.temp_size); + + } else if (coder->lz.temp_size > 0) { + // Fill temp[] unless it is already full because we aren't + // the last filter in the chain. + size_t copy_size = 0; + if (coder->lz.temp_size < 2 * TEMP_LIMIT) { + assert(*in_pos < in_size); + copy_size = 2 * TEMP_LIMIT - coder->lz.temp_size; + memcpy(coder->lz.temp + coder->lz.temp_size, + in + *in_pos, copy_size); + // NOTE: We don't update lz.temp_size or *in_pos yet. + } + + size_t temp_used = 0; + error = coder->lz.process(coder, coder->lz.temp, &temp_used, + coder->lz.temp_size + copy_size, false); + + if (temp_used < coder->lz.temp_size) { + // Only very little input data was consumed. Move + // the unprocessed data to the beginning temp[]. + coder->lz.temp_size += copy_size - temp_used; + memmove(coder->lz.temp, coder->lz.temp + temp_used, + coder->lz.temp_size); + *in_pos += copy_size; + assert(*in_pos <= in_size); + + } else { + // We were able to decode so much data that next time + // we can decode directly from in[]. That is, we can + // consider temp[] to be empty now. + *in_pos += temp_used - coder->lz.temp_size; + coder->lz.temp_size = 0; + assert(*in_pos <= in_size); + } + + } else { + // Decode directly from in[]. + error = coder->lz.process(coder, in, in_pos, in_size, false); + assert(*in_pos <= in_size); + } + + assert(coder->lz.pos >= dict_old_pos); + if (coder->lz.uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + // Update uncompressed size. + coder->lz.uncompressed_size -= coder->lz.pos - dict_old_pos; + + // Check that End of Payload Marker hasn't been detected + // since it must not be present because uncompressed size + // is known. + if (coder->lz.eopm_detected) + error = true; + } + + return error; +} + + +static lzma_ret +decode_buffer(lzma_coder *coder, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + bool flushing) +{ + bool stop = false; + + while (true) { + // Flush from coder->lz.dict to out[]. + flush(&coder->lz, out, out_pos, out_size); + + // All done? + if (*out_pos == out_size + || stop + || coder->lz.eopm_detected + || coder->lz.uncompressed_size == 0) + break; + + // Set write limit in the dictionary. + if (set_limit(&coder->lz, out_size - *out_pos, flushing)) + break; + + // Decode more data. + if (call_process(coder, in, in_pos, in_size)) + return LZMA_DATA_ERROR; + + // Set stop to true if we must not call call_process() again + // during this function call. + // FIXME: Can this make the loop exist too early? It wouldn't + // cause data corruption so not a critical problem. It can + // happen if dictionary gets full and lz.temp still contains + // a few bytes data that we could decode right now. + if (*in_pos == in_size && coder->lz.temp_size <= TEMP_LIMIT + && coder->lz.pos < coder->lz.limit) + stop = true; + } + + // If we have decoded everything (EOPM detected or uncompressed_size + // bytes were processed) to the history buffer, and also flushed + // everything from the history buffer, our job is done. + if ((coder->lz.eopm_detected + || coder->lz.uncompressed_size == 0) + && coder->lz.start == coder->lz.pos) + return LZMA_STREAM_END; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_lz_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action) +{ + if (coder->next.code == NULL) { + const lzma_ret ret = decode_buffer(coder, in, in_pos, in_size, + out, out_pos, out_size, + action == LZMA_SYNC_FLUSH); + + if (*out_pos == out_size || ret == LZMA_STREAM_END) { + // Unread to make coder->temp[] empty. This is easy, + // because we know that all the data currently in + // coder->temp[] has been copied form in[] during this + // call to the decoder. + // + // If we didn't do this, we could have data left in + // coder->temp[] when end of stream is reached. That + // data could be left there from *previous* call to + // the decoder; in that case we wouldn't know where + // to put that data. + assert(*in_pos >= coder->lz.temp_size); + *in_pos -= coder->lz.temp_size; + coder->lz.temp_size = 0; + } + + return ret; + } + + // We aren't the last coder in the chain, we need to decode + // our input to a temporary buffer. + const bool flushing = action == LZMA_SYNC_FLUSH; + while (*out_pos < out_size) { + if (!coder->lz.next_finished + && coder->lz.temp_size < LZMA_BUFFER_SIZE) { + const lzma_ret ret = coder->next.code( + coder->next.coder, + allocator, in, in_pos, in_size, + coder->lz.temp, &coder->lz.temp_size, + LZMA_BUFFER_SIZE, action); + + if (ret == LZMA_STREAM_END) + coder->lz.next_finished = true; + else if (coder->lz.temp_size < LZMA_BUFFER_SIZE + || ret != LZMA_OK) + return ret; + } + + if (coder->lz.this_finished) { + if (coder->lz.temp_size != 0) + return LZMA_DATA_ERROR; + + if (coder->lz.next_finished) + return LZMA_STREAM_END; + + return LZMA_OK; + } + + size_t dummy = 0; + const lzma_ret ret = decode_buffer(coder, NULL, &dummy, 0, + out, out_pos, out_size, flushing); + + if (ret == LZMA_STREAM_END) + coder->lz.this_finished = true; + else if (ret != LZMA_OK) + return ret; + else if (coder->lz.next_finished && *out_pos < out_size) + return LZMA_DATA_ERROR; + } + + return LZMA_OK; +} + + +/// \brief Initializes LZ part of the LZMA decoder or Inflate +/// +/// \param history_size Number of bytes the LZ out window is +/// supposed keep available from the output +/// history. +/// \param match_max_len Number of bytes a single decoding loop +/// can advance the write position (lz->pos) +/// in the history buffer (lz->dict). +/// +/// \note This function is called by LZMA decoder and Inflate init()s. +/// It's up to those functions allocate *lz and initialize it +/// with LZMA_LZ_DECODER_INIT. +extern lzma_ret +lzma_lz_decoder_reset(lzma_lz_decoder *lz, lzma_allocator *allocator, + bool (*process)(lzma_coder *restrict coder, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, bool has_safe_buffer), + lzma_vli uncompressed_size, + size_t history_size, size_t match_max_len) +{ + // Set uncompressed size. + lz->uncompressed_size = uncompressed_size; + + // Limit the history size to roughly sane values. This is primarily + // to prevent integer overflows. + if (history_size > UINT32_MAX / 2) + return LZMA_HEADER_ERROR; + + // Store the value actually requested. We use it for sanity checks + // when repeating data from the history buffer. + lz->requested_size = history_size; + + // Avoid tiny history buffer sizes for performance reasons. + // TODO: Test if this actually helps... + if (history_size < DICT_SIZE_MIN) + history_size = DICT_SIZE_MIN; + + // The real size of the history buffer is a bit bigger than + // requested by our caller. This allows us to do some optimizations, + // which help not only speed but simplicity of the code; specifically, + // we can make sure that there is always at least match_max_len + // bytes immediatelly available for writing without a need to wrap + // the history buffer. + const size_t dict_real_size = history_size + 2 * match_max_len + 1; + + // Reallocate memory if needed. + if (history_size != lz->size || match_max_len != lz->match_max_len) { + // Destroy the old buffer. + lzma_lz_decoder_end(lz, allocator); + + lz->size = history_size; + lz->match_max_len = match_max_len; + lz->must_flush_pos = history_size + match_max_len + 1; + + lz->dict = lzma_alloc(dict_real_size, allocator); + if (lz->dict == NULL) + return LZMA_MEM_ERROR; + } + + // Clean up the buffers to make it very sure that there are + // no information leaks when multiple steams are decoded + // with the same decoder structures. + memzero(lz->dict, dict_real_size); + memzero(lz->temp, LZMA_BUFFER_SIZE); + + // Reset the variables so that lz_get_byte(lz, 0) will return '\0'. + lz->pos = 0; + lz->start = 0; + lz->end = dict_real_size; + lz->is_full = false; + lz->eopm_detected = false; + lz->next_finished = false; + lz->this_finished = false; + + // Set the process function pointer. + lz->process = process; + + return LZMA_OK; +} + + +extern void +lzma_lz_decoder_end(lzma_lz_decoder *lz, lzma_allocator *allocator) +{ + lzma_free(lz->dict, allocator); + lz->dict = NULL; + lz->size = 0; + lz->match_max_len = 0; + return; +} diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h new file mode 100644 index 00000000..a8a585cd --- /dev/null +++ b/src/liblzma/lz/lz_decoder.h @@ -0,0 +1,214 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_decoder.h +/// \brief LZ out window +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZ_OUT_H +#define LZMA_LZ_OUT_H + +#include "common.h" + + +/// Get a byte from the history buffer. +#define lz_get_byte(lz, distance) \ + ((distance) < (lz).pos \ + ? (lz).dict[(lz).pos - (distance) - 1] \ + : (lz).dict[(lz).pos - (distance) - 1 + (lz).end]) + + +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ .dict = NULL, .size = 0, .match_max_len = 0 } + + +typedef struct { + /// Function to do the actual decoding (LZMA or Inflate) + bool (*process)(lzma_coder *restrict coder, const uint8_t *restrict in, + size_t *restrict in_pos, size_t size_in, + bool has_safe_buffer); + + /// Pointer to dictionary (history) buffer. + /// \note Not 'restrict' because can alias next_out. + uint8_t *dict; + + /// Next write goes to dict[pos]. + size_t pos; + + /// Next byte to flush is buffer[start]. + size_t start; + + /// First byte to not flush is buffer[end]. + size_t end; + + /// First position to which data must not be written. + size_t limit; + + /// True if dictionary has needed wrapping. + bool is_full; + + /// True if process() has detected End of Payload Marker. + bool eopm_detected; + + /// True if the next coder in the chain has returned LZMA_STREAM_END. + bool next_finished; + + /// True if the LZ decoder (e.g. LZMA) has detected End of Payload + /// Marker. This may become true before next_finished becomes true. + bool this_finished; + + /// When pos >= must_flush_pos, we must not call process(). + size_t must_flush_pos; + + /// Maximum number of bytes that a single decoding loop inside + /// process() can produce data into dict. This amount is kept + /// always available at dict + pos i.e. it is safe to write a byte + /// to dict[pos + match_max_len - 1]. + size_t match_max_len; + + /// Number of bytes allocated to dict. + size_t size; + + /// Requested size of the dictionary. This is needed because we avoid + /// using extremely tiny history buffers. + size_t requested_size; + + /// Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN if unknown. + lzma_vli uncompressed_size; + + /// Number of bytes currently in temp[]. + size_t temp_size; + + /// Temporary buffer needed when + /// 1) we cannot make the input buffer completely empty; or + /// 2) we are not the last filter in the chain. + uint8_t temp[LZMA_BUFFER_SIZE]; + +} lzma_lz_decoder; + + +///////////////////////// +// Function prototypes // +///////////////////////// + +extern lzma_ret lzma_lz_decoder_reset(lzma_lz_decoder *lz, + lzma_allocator *allocator, bool (*process)( + lzma_coder *restrict coder, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + bool has_safe_buffer), + lzma_vli uncompressed_size, + size_t history_size, size_t match_max_len); + +extern lzma_ret lzma_lz_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action); + +/// Deallocates the history buffer if one exists. +extern void lzma_lz_decoder_end( + lzma_lz_decoder *lz, lzma_allocator *allocator); + +////////////////////// +// Inline functions // +////////////////////// + +// Repeat a block of data from the history. Because memcpy() is faster +// than copying byte by byte in a loop, the copying process gets split +// into three cases: +// 1. distance < length +// Source and target areas overlap, thus we can't use memcpy() +// (nor memmove()) safely. +// TODO: If this is common enough, it might be worth optimizing this +// more e.g. by checking if distance > sizeof(uint8_t*) and using +// memcpy in small chunks. +// 2. distance < pos +// This is the easiest and the fastest case. The block being copied +// is a contiguous piece in the history buffer. The buffer offset +// doesn't need wrapping. +// 3. distance >= pos +// We need to wrap the position, because otherwise we would try copying +// behind the first byte of the allocated buffer. It is possible that +// the block is fragmeneted into two pieces, thus we might need to call +// memcpy() twice. +// NOTE: The function using this macro must ensure that length is positive +// and that distance is FIXME +static inline bool +lzma_lz_out_repeat(lzma_lz_decoder *lz, size_t distance, size_t length) +{ + // Validate offset of the block to be repeated. It doesn't + // make sense to copy data behind the beginning of the stream. + // Leaving this check away would lead to a security problem, + // in which e.g. the data of the previously decoded file(s) + // would be leaked (or whatever happens to be in unused + // part of the dictionary buffer). + if (distance >= lz->pos && !lz->is_full) + return false; + + // It also doesn't make sense to copy data farer than + // the dictionary size. + if (distance >= lz->requested_size) + return false; + + // The caller must have checked these! + assert(distance <= lz->size); + assert(length > 0); + assert(length <= lz->match_max_len); + + // Copy the amount of data requested by the decoder. + if (distance < length) { + // Source and target areas overlap, thus we can't use + // memcpy() nor even memmove() safely. :-( + // TODO: Copying byte by byte is slow. It might be + // worth optimizing this more if this case is common. + do { + lz->dict[lz->pos] = lz_get_byte(*lz, distance); + ++lz->pos; + } while (--length > 0); + + } else if (distance < lz->pos) { + // The easiest and fastest case + memcpy(lz->dict + lz->pos, + lz->dict + lz->pos - distance - 1, + length); + lz->pos += length; + + } else { + // The bigger the dictionary, the more rare this + // case occurs. We need to "wrap" the dict, thus + // we might need two memcpy() to copy all the data. + assert(lz->is_full); + const uint32_t copy_pos = lz->pos - distance - 1 + lz->end; + uint32_t copy_size = lz->end - copy_pos; + + if (copy_size < length) { + memcpy(lz->dict + lz->pos, lz->dict + copy_pos, + copy_size); + lz->pos += copy_size; + copy_size = length - copy_size; + memcpy(lz->dict + lz->pos, lz->dict, copy_size); + lz->pos += copy_size; + } else { + memcpy(lz->dict + lz->pos, lz->dict + copy_pos, + length); + lz->pos += length; + } + } + + return true; +} + +#endif diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c new file mode 100644 index 00000000..bc38cae2 --- /dev/null +++ b/src/liblzma/lz/lz_encoder.c @@ -0,0 +1,481 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder.c +/// \brief LZ in window +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder_private.h" + +// Hash Chains +#ifdef HAVE_HC3 +# include "hc3.h" +#endif +#ifdef HAVE_HC4 +# include "hc4.h" +#endif + +// Binary Trees +#ifdef HAVE_BT2 +# include "bt2.h" +#endif +#ifdef HAVE_BT3 +# include "bt3.h" +#endif +#ifdef HAVE_BT4 +# include "bt4.h" +#endif + + +/// This is needed in two places so provide a macro. +#define get_cyclic_buffer_size(history_size) ((history_size) + 1) + + +/// Calculate certain match finder properties and validate the calculated +/// values. This is as its own function, because *num_items is needed to +/// calculate memory requirements in common/memory.c. +extern uint32_t +lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, + uint32_t history_size, uint32_t *restrict hash_mask, + uint32_t *restrict hash_size_sum, uint32_t *restrict num_items) +{ + uint32_t fix_hash_size; + uint32_t sons; + + switch (match_finder) { +#ifdef HAVE_HC3 + case LZMA_MF_HC3: + fix_hash_size = LZMA_HC3_FIX_HASH_SIZE; + sons = 1; + break; +#endif +#ifdef HAVE_HC4 + case LZMA_MF_HC4: + fix_hash_size = LZMA_HC4_FIX_HASH_SIZE; + sons = 1; + break; +#endif +#ifdef HAVE_BT2 + case LZMA_MF_BT2: + fix_hash_size = LZMA_BT2_FIX_HASH_SIZE; + sons = 2; + break; +#endif +#ifdef HAVE_BT3 + case LZMA_MF_BT3: + fix_hash_size = LZMA_BT3_FIX_HASH_SIZE; + sons = 2; + break; +#endif +#ifdef HAVE_BT4 + case LZMA_MF_BT4: + fix_hash_size = LZMA_BT4_FIX_HASH_SIZE; + sons = 2; + break; +#endif + default: + return true; + } + + uint32_t hs; + +#ifdef HAVE_LZMA_BT2 + if (match_finder == LZMA_BT2) { + // NOTE: hash_mask is not used by the BT2 match finder, + // but it is initialized just in case. + hs = LZMA_BT2_HASH_SIZE; + *hash_mask = 0; + } else +#endif + { + hs = history_size - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + + if (hs > (UINT32_C(1) << 24)) { + if (match_finder == LZMA_MF_HC4 + || match_finder == LZMA_MF_BT4) + hs >>= 1; + else + hs = (1 << 24) - 1; + } + + *hash_mask = hs; + ++hs; + } + + *hash_size_sum = hs + fix_hash_size; + + *num_items = *hash_size_sum + + get_cyclic_buffer_size(history_size) * sons; + + return false; +} + + +extern lzma_ret +lzma_lz_encoder_reset(lzma_lz_encoder *lz, lzma_allocator *allocator, + bool (*process)(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size), + lzma_vli uncompressed_size, + size_t history_size, size_t additional_buffer_before, + size_t match_max_len, size_t additional_buffer_after, + lzma_match_finder match_finder, uint32_t match_finder_cycles, + const uint8_t *preset_dictionary, + size_t preset_dictionary_size) +{ + // Set uncompressed size. + lz->uncompressed_size = uncompressed_size; + + /////////////// + // In Window // + /////////////// + + // Validate history size. + if (history_size < LZMA_DICTIONARY_SIZE_MIN + || history_size > LZMA_DICTIONARY_SIZE_MAX) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_HEADER_ERROR; + } + + assert(history_size <= MAX_VAL_FOR_NORMALIZE - 256); + assert(LZMA_DICTIONARY_SIZE_MAX <= MAX_VAL_FOR_NORMALIZE - 256); + + // Calculate the size of the history buffer to allocate. + // TODO: Get a reason for magic constant of 256. + const size_t size_reserv = (history_size + additional_buffer_before + + match_max_len + additional_buffer_after) / 2 + 256; + + lz->keep_size_before = history_size + additional_buffer_before; + lz->keep_size_after = match_max_len + additional_buffer_after; + + const size_t buffer_size = lz->keep_size_before + lz->keep_size_after + + size_reserv; + + // Allocate history buffer if its size has changed. + if (buffer_size != lz->size) { + lzma_free(lz->buffer, allocator); + lz->buffer = lzma_alloc(buffer_size, allocator); + if (lz->buffer == NULL) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_MEM_ERROR; + } + } + + // Allocation successful. Store the new size and calculate + // must_move_pos. + lz->size = buffer_size; + lz->must_move_pos = lz->size - lz->keep_size_after; + + // Reset in window variables. + lz->offset = 0; + lz->read_pos = 0; + lz->read_limit = 0; + lz->write_pos = 0; + lz->stream_end_was_reached = false; + + + ////////////////// + // Match Finder // + ////////////////// + + // Validate match_finder, set function pointers and a few match + // finder specific variables. + switch (match_finder) { +#ifdef HAVE_HC3 + case LZMA_MF_HC3: + lz->get_matches = &lzma_hc3_get_matches; + lz->skip = &lzma_hc3_skip; + lz->cut_value = 8 + (match_max_len >> 2); + break; +#endif +#ifdef HAVE_HC4 + case LZMA_MF_HC4: + lz->get_matches = &lzma_hc4_get_matches; + lz->skip = &lzma_hc4_skip; + lz->cut_value = 8 + (match_max_len >> 2); + break; +#endif +#ifdef HAVE_BT2 + case LZMA_MF_BT2: + lz->get_matches = &lzma_bt2_get_matches; + lz->skip = &lzma_bt2_skip; + lz->cut_value = 16 + (match_max_len >> 1); + break; +#endif +#ifdef HAVE_BT3 + case LZMA_MF_BT3: + lz->get_matches = &lzma_bt3_get_matches; + lz->skip = &lzma_bt3_skip; + lz->cut_value = 16 + (match_max_len >> 1); + break; +#endif +#ifdef HAVE_BT4 + case LZMA_MF_BT4: + lz->get_matches = &lzma_bt4_get_matches; + lz->skip = &lzma_bt4_skip; + lz->cut_value = 16 + (match_max_len >> 1); + break; +#endif + default: + lzma_lz_encoder_end(lz, allocator); + return LZMA_HEADER_ERROR; + } + + // Check if we have been requested to use a non-default cut_value. + if (match_finder_cycles > 0) + lz->cut_value = match_finder_cycles; + + lz->match_max_len = match_max_len; + lz->cyclic_buffer_size = get_cyclic_buffer_size(history_size); + + uint32_t hash_size_sum; + uint32_t num_items; + if (lzma_lz_encoder_hash_properties(match_finder, history_size, + &lz->hash_mask, &hash_size_sum, &num_items)) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_HEADER_ERROR; + } + + if (num_items != lz->num_items) { +#if UINT32_MAX >= SIZE_MAX / 4 + // Check for integer overflow. (Huge dictionaries are not + // possible on 32-bit CPU.) + if (num_items > SIZE_MAX / sizeof(uint32_t)) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_MEM_ERROR; + } +#endif + + const size_t size_in_bytes + = (size_t)(num_items) * sizeof(uint32_t); + + lzma_free(lz->hash, allocator); + lz->hash = lzma_alloc(size_in_bytes, allocator); + if (lz->hash == NULL) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_MEM_ERROR; + } + + lz->num_items = num_items; + } + + lz->son = lz->hash + hash_size_sum; + + // Reset the hash table to empty hash values. + { + uint32_t *restrict items = lz->hash; + + for (uint32_t i = 0; i < hash_size_sum; ++i) + items[i] = EMPTY_HASH_VALUE; + } + + lz->cyclic_buffer_pos = 0; + + // Because zero is used as empty hash value, make the first byte + // appear at buffer[1 - offset]. + ++lz->offset; + + // If we are using a preset dictionary, read it now. + // TODO: This isn't implemented yet so return LZMA_HEADER_ERROR. + if (preset_dictionary != NULL && preset_dictionary_size > 0) { + lzma_lz_encoder_end(lz, allocator); + return LZMA_HEADER_ERROR; + } + + // Set the process function pointer. + lz->process = process; + + return LZMA_OK; +} + + +extern void +lzma_lz_encoder_end(lzma_lz_encoder *lz, lzma_allocator *allocator) +{ + lzma_free(lz->hash, allocator); + lz->hash = NULL; + lz->num_items = 0; + + lzma_free(lz->buffer, allocator); + lz->buffer = NULL; + lz->size = 0; + + return; +} + + +/// \brief Moves the data in the input window to free space for new data +/// +/// lz->buffer is a sliding input window, which keeps lz->keep_size_before +/// bytes of input history available all the time. Now and then we need to +/// "slide" the buffer to make space for the new data to the end of the +/// buffer. At the same time, data older than keep_size_before is dropped. +/// +static void +move_window(lzma_lz_encoder *lz) +{ + // buffer[move_offset] will become buffer[0]. + assert(lz->read_pos > lz->keep_size_after); + size_t move_offset = lz->read_pos - lz->keep_size_before; + + // We need one additional byte, since move_pos() moves on 1 byte. + // TODO: Clean up? At least document more. + if (move_offset > 0) + --move_offset; + + assert(lz->write_pos > move_offset); + const size_t move_size = lz->write_pos - move_offset; + + assert(move_offset + move_size <= lz->size); + + memmove(lz->buffer, lz->buffer + move_offset, move_size); + + lz->offset += move_offset; + lz->read_pos -= move_offset; + lz->read_limit -= move_offset; + lz->write_pos -= move_offset; + + return; +} + + +/// \brief Tries to fill the input window (lz->buffer) +/// +/// If we are the last encoder in the chain, our input data is in in[]. +/// Otherwise we call the next filter in the chain to process in[] and +/// write its output to lz->buffer. +/// +/// This function must not be called once it has returned LZMA_STREAM_END. +/// +static lzma_ret +fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, + size_t *in_pos, size_t in_size, lzma_action action) +{ + assert(coder->lz.read_pos <= coder->lz.write_pos); + lzma_ret ret; + + // Move the sliding window if needed. + if (coder->lz.read_pos >= coder->lz.must_move_pos) + move_window(&coder->lz); + + if (coder->next.code == NULL) { + // Not using a filter, simply memcpy() as much as possible. + bufcpy(in, in_pos, in_size, coder->lz.buffer, + &coder->lz.write_pos, coder->lz.size); + + if (action == LZMA_FINISH && *in_pos == in_size) + ret = LZMA_STREAM_END; + else + ret = LZMA_OK; + + } else { + ret = coder->next.code(coder->next.coder, allocator, + in, in_pos, in_size, + coder->lz.buffer, &coder->lz.write_pos, + coder->lz.size, action); + } + + // If end of stream has been reached, we allow the encoder to process + // all the input (that is, read_pos is allowed to reach write_pos). + // Otherwise we keep keep_size_after bytes available as prebuffer. + if (ret == LZMA_STREAM_END) { + coder->lz.stream_end_was_reached = true; + coder->lz.read_limit = coder->lz.write_pos; + + } else if (coder->lz.write_pos > coder->lz.keep_size_after) { + // This needs to be done conditionally, because if we got + // only little new input, there may be too little input + // to do any encoding yet. + coder->lz.read_limit = coder->lz.write_pos + - coder->lz.keep_size_after; + } + + return ret; +} + + +extern lzma_ret +lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_action action) +{ + while (*out_pos < out_size + && (*in_pos < in_size || action == LZMA_FINISH)) { + // Fill the input window if there is no more usable data. + if (!coder->lz.stream_end_was_reached && coder->lz.read_pos + >= coder->lz.read_limit) { + const lzma_ret ret = fill_window(coder, allocator, + in, in_pos, in_size, action); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + return ret; + } + + // Encode + if (coder->lz.process(coder, out, out_pos, out_size)) + return LZMA_STREAM_END; + } + + return LZMA_OK; +} + + +/// \brief Normalizes hash values +/// +/// lzma_lz_normalize is called when lz->pos hits MAX_VAL_FOR_NORMALIZE, +/// which currently happens once every 2 GiB of input data (to be exact, +/// after the first 2 GiB it happens once every 2 GiB minus dictionary_size +/// bytes). lz->pos is incremented by lzma_lz_move_pos(). +/// +/// lz->hash contains big amount of offsets relative to lz->buffer. +/// The offsets are stored as uint32_t, which is the only reasonable +/// datatype for these offsets; uint64_t would waste far too much RAM +/// and uint16_t would limit the dictionary to 64 KiB (far too small). +/// +/// When compressing files over 2 GiB, lz->buffer needs to be moved forward +/// to avoid integer overflows. We scan the lz->hash array and fix every +/// value to match the updated lz->buffer. +extern void +lzma_lz_encoder_normalize(lzma_lz_encoder *lz) +{ + const uint32_t subvalue = lz->read_pos - lz->cyclic_buffer_size; + assert(subvalue <= INT32_MAX); + + { + const uint32_t num_items = lz->num_items; + uint32_t *restrict items = lz->hash; + + for (uint32_t i = 0; i < num_items; ++i) { + // If the distance is greater than the dictionary + // size, we can simply mark the item as empty. + if (items[i] <= subvalue) + items[i] = EMPTY_HASH_VALUE; + else + items[i] -= subvalue; + } + } + + // Update offset to match the new locations. + lz->offset -= subvalue; + + return; +} diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h new file mode 100644 index 00000000..b39c88e5 --- /dev/null +++ b/src/liblzma/lz/lz_encoder.h @@ -0,0 +1,161 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder.h +/// \brief LZ in window and match finder API +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZ_ENCODER_H +#define LZMA_LZ_ENCODER_H + +#include "common.h" + + +typedef struct lzma_lz_encoder_s lzma_lz_encoder; +struct lzma_lz_encoder_s { + enum { + SEQ_INIT, + SEQ_RUN, + SEQ_FINISH, + SEQ_END + } sequence; + + bool (*process)(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); + + lzma_vli uncompressed_size; + + /////////////// + // In Window // + /////////////// + + /// Pointer to buffer with data to be compressed + uint8_t *buffer; + + /// Total size of the allocated buffer (that is, including all + /// the extra space) + size_t size; + + /// Match finders store locations of matches using 32-bit integers. + /// To avoid adjusting several megabytes of integers every time the + /// input window is moved with move_window(), we only adjust the + /// offset of the buffer. Thus, buffer[match_finder_pos - offset] + /// is the byte pointed by match_finder_pos. + size_t offset; + + /// buffer[read_pos] is the current byte. + size_t read_pos; + + /// As long as read_pos is less than read_limit, there is enough + /// input available in buffer for at least one encoding loop. + /// + /// Because of the stateful API, read_limit may and will get greater + /// than read_pos quite often. This is taken into account when + /// calculating the value for keep_size_after. + size_t read_limit; + + /// buffer[write_pos] is the first byte that doesn't contain valid + /// uncompressed data; that is, the next input byte will be copied + /// to buffer[write_pos]. + size_t write_pos; + + /// When read_pos >= must_move_pos, move_window() must be called + /// to make more space for the input data. + size_t must_move_pos; + + /// Number of bytes that must be kept available in our input history. + /// That is, once keep_size_before bytes have been processed, + /// buffer[read_pos - keep_size_before] is the oldest byte that + /// must be available for reading. + size_t keep_size_before; + + /// Number of bytes that must be kept in buffer after read_pos. + /// That is, read_pos <= write_pos - keep_size_after as long as + /// stream_end_was_reached is false (once it is true, read_pos + /// is allowed to reach write_pos). + size_t keep_size_after; + + /// This is set to true once the last byte of the input data has + /// been copied to buffer. + bool stream_end_was_reached; + + ////////////////// + // Match Finder // + ////////////////// + + // Pointers to match finder functions + void (*get_matches)(lzma_lz_encoder *restrict lz, + uint32_t *restrict distances); + void (*skip)(lzma_lz_encoder *restrict lz, uint32_t num); + + // Match finder data + uint32_t *hash; // TODO: Check if hash aliases son + uint32_t *son; // and add 'restrict' if possible. + uint32_t cyclic_buffer_pos; + uint32_t cyclic_buffer_size; // Must be dictionary_size + 1. + uint32_t hash_mask; + uint32_t cut_value; + uint32_t hash_size_sum; + uint32_t num_items; + uint32_t match_max_len; +}; + + +#define LZMA_LZ_ENCODER_INIT \ + (lzma_lz_encoder){ \ + .buffer = NULL, \ + .size = 0, \ + .hash = NULL, \ + .num_items = 0, \ + } + + +/// Calculates +extern uint32_t lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, + uint32_t history_size, uint32_t *restrict hash_mask, + uint32_t *restrict hash_size_sum, + uint32_t *restrict num_items); + +// NOTE: liblzma doesn't use callback API like LZMA SDK does. The caller +// must make sure that keep_size_after is big enough for single encoding pass +// i.e. keep_size_after >= maximum number of bytes possibly needed after +// the current position between calls to lzma_lz_read(). +extern lzma_ret lzma_lz_encoder_reset(lzma_lz_encoder *lz, + lzma_allocator *allocator, + bool (*process)(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size), + lzma_vli uncompressed_size, + size_t history_size, size_t additional_buffer_before, + size_t match_max_len, size_t additional_buffer_after, + lzma_match_finder match_finder, uint32_t match_finder_cycles, + const uint8_t *preset_dictionary, + size_t preset_dictionary_size); + +/// Frees memory allocated for in window and match finder buffers. +extern void lzma_lz_encoder_end( + lzma_lz_encoder *lz, lzma_allocator *allocator); + +extern lzma_ret lzma_lz_encode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action); + +/// This should not be called directly, but only via move_pos() macro. +extern void lzma_lz_encoder_normalize(lzma_lz_encoder *lz); + +#endif diff --git a/src/liblzma/lz/lz_encoder_private.h b/src/liblzma/lz/lz_encoder_private.h new file mode 100644 index 00000000..638fcb2d --- /dev/null +++ b/src/liblzma/lz/lz_encoder_private.h @@ -0,0 +1,40 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder_private.h +/// \brief Private definitions for LZ encoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZ_ENCODER_PRIVATE_H +#define LZMA_LZ_ENCODER_PRIVATE_H + +#include "lz_encoder.h" + +/// Value used to indicate unused slot +#define EMPTY_HASH_VALUE 0 + +/// When the dictionary and hash variables need to be adjusted to prevent +/// integer overflows. Since we use uint32_t to store the offsets, half +/// of it is the biggest safe limit. +#define MAX_VAL_FOR_NORMALIZE (UINT32_MAX / 2) + + +struct lzma_coder_s { + lzma_next_coder next; + lzma_lz_encoder lz; +}; + +#endif diff --git a/src/liblzma/lz/match_c.h b/src/liblzma/lz/match_c.h new file mode 100644 index 00000000..68766385 --- /dev/null +++ b/src/liblzma/lz/match_c.h @@ -0,0 +1,401 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file match_c.h +/// \brief Template for different match finders +/// +/// This file is included by hc3.c, hc4, bt2.c, bt3.c and bt4.c. Each file +/// sets slighly different #defines, resulting the different match finders. +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +////////////// +// Includes // +////////////// + +#include "check.h" + + +/////////////// +// Constants // +/////////////// + +#define START_MAX_LEN 1 + +#ifdef HASH_ARRAY_2 +# define NUM_HASH_DIRECT_BYTES 0 +# define HASH_2_SIZE (1 << 10) +# ifdef HASH_ARRAY_3 +# define NUM_HASH_BYTES 4 +# define HASH_3_SIZE (1 << 16) +# define HASH_3_OFFSET HASH_2_SIZE +# define FIX_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE) +# else +# define NUM_HASH_BYTES 3 +# define FIX_HASH_SIZE HASH_2_SIZE +# endif +# define HASH_SIZE 0 +# define MIN_MATCH_CHECK NUM_HASH_BYTES +#else +# define NUM_HASH_DIRECT_BYTES 2 +# define NUM_HASH_BYTES 2 +# define HASH_SIZE (1 << (8 * NUM_HASH_BYTES)) +# define MIN_MATCH_CHECK (NUM_HASH_BYTES + 1) +# define FIX_HASH_SIZE 0 +#endif + + +//////////// +// Macros // +//////////// + +#ifdef HASH_ARRAY_2 +# ifdef HASH_ARRAY_3 +# define HASH_CALC() \ + do { \ + const uint32_t temp = lzma_crc32_table[0][ \ + cur[0]] ^ cur[1]; \ + hash_2_value = temp & (HASH_2_SIZE - 1); \ + hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ + & (HASH_3_SIZE - 1); \ + hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + ^ (lzma_crc32_table[0][cur[3]] << 5)) \ + & lz->hash_mask; \ + } while (0) +# else +# define HASH_CALC() \ + do { \ + const uint32_t temp = lzma_crc32_table[0][ \ + cur[0]] ^ cur[1]; \ + hash_2_value = temp & (HASH_2_SIZE - 1); \ + hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ + & lz->hash_mask; \ + } while (0) +# endif +#else +# define HASH_CALC() hash_value = cur[0] ^ ((uint32_t)(cur[1]) << 8) +#endif + + +// Moves the current read position forward by one byte. In LZMA SDK, +// CLZInWindow::MovePos() can read more input data if needed, because of +// the callback style API. In liblzma we must have ensured earlier, that +// there is enough data available in lz->buffer. +#define move_pos() \ +do { \ + if (++lz->cyclic_buffer_pos == lz->cyclic_buffer_size) \ + lz->cyclic_buffer_pos = 0; \ + ++lz->read_pos; \ + assert(lz->read_pos <= lz->write_pos); \ + if (lz->read_pos == MAX_VAL_FOR_NORMALIZE) \ + lzma_lz_encoder_normalize(lz); \ +} while (0) + + +////////////////////// +// Global constants // +////////////////////// + +LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = HASH_SIZE; +LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = FIX_HASH_SIZE; + + +/////////////////// +// API functions // +/////////////////// + +LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER) +{ + uint32_t len_limit; + if (lz->read_pos + lz->match_max_len <= lz->write_pos) { + len_limit = lz->match_max_len; + } else { + assert(lz->stream_end_was_reached); + len_limit = lz->write_pos - lz->read_pos; + if (len_limit < MIN_MATCH_CHECK) { + distances[0] = 0; + move_pos(); + return; + } + } + + int32_t offset = 1; + const uint32_t match_min_pos + = lz->read_pos + lz->offset > lz->cyclic_buffer_size + ? lz->read_pos + lz->offset - lz->cyclic_buffer_size + : 0; + const uint8_t *cur = lz->buffer + lz->read_pos; + uint32_t max_len = START_MAX_LEN; // to avoid items for len < hash_size + +#ifdef HASH_ARRAY_2 + uint32_t hash_2_value; +# ifdef HASH_ARRAY_3 + uint32_t hash_3_value; +# endif +#endif + uint32_t hash_value; + HASH_CALC(); + + uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; +#ifdef HASH_ARRAY_2 + uint32_t cur_match2 = lz->hash[hash_2_value]; +# ifdef HASH_ARRAY_3 + uint32_t cur_match3 = lz->hash[HASH_3_OFFSET + hash_3_value]; +# endif + lz->hash[hash_2_value] = lz->read_pos + lz->offset; + + if (cur_match2 > match_min_pos) { + if (lz->buffer[cur_match2 - lz->offset] == cur[0]) { + max_len = 2; + distances[offset++] = 2; + distances[offset++] = lz->read_pos + lz->offset + - cur_match2 - 1; + } + } + +# ifdef HASH_ARRAY_3 + lz->hash[HASH_3_OFFSET + hash_3_value] = lz->read_pos + lz->offset; + if (cur_match3 > match_min_pos) { + if (lz->buffer[cur_match3 - lz->offset] == cur[0]) { + if (cur_match3 == cur_match2) + offset -= 2; + + max_len = 3; + distances[offset++] = 3; + distances[offset++] = lz->read_pos + lz->offset + - cur_match3 - 1; + cur_match2 = cur_match3; + } + } +# endif + + if (offset != 1 && cur_match2 == cur_match) { + offset -= 2; + max_len = START_MAX_LEN; + } +#endif + + lz->hash[FIX_HASH_SIZE + hash_value] = lz->read_pos + lz->offset; + +#ifdef IS_HASH_CHAIN + lz->son[lz->cyclic_buffer_pos] = cur_match; +#else + uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); + + uint32_t len0 = NUM_HASH_DIRECT_BYTES; + uint32_t len1 = NUM_HASH_DIRECT_BYTES; +#endif + +#if NUM_HASH_DIRECT_BYTES != 0 + if (cur_match > match_min_pos) { + if (lz->buffer[cur_match + NUM_HASH_DIRECT_BYTES - lz->offset] + != cur[NUM_HASH_DIRECT_BYTES]) { + max_len = NUM_HASH_DIRECT_BYTES; + distances[offset++] = NUM_HASH_DIRECT_BYTES; + distances[offset++] = lz->read_pos + lz->offset + - cur_match - 1; + } + } +#endif + + uint32_t count = lz->cut_value; + + while (true) { + if (cur_match <= match_min_pos || count-- == 0) { +#ifndef IS_HASH_CHAIN + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; +#endif + break; + } + + const uint32_t delta = lz->read_pos + lz->offset - cur_match; + const uint32_t cyclic_pos = delta <= lz->cyclic_buffer_pos + ? lz->cyclic_buffer_pos - delta + : lz->cyclic_buffer_pos - delta + + lz->cyclic_buffer_size; + uint32_t *pair = lz->son + +#ifdef IS_HASH_CHAIN + cyclic_pos; +#else + (cyclic_pos << 1); +#endif + + const uint8_t *pb = lz->buffer + cur_match - lz->offset; + uint32_t len = +#ifdef IS_HASH_CHAIN + NUM_HASH_DIRECT_BYTES; + if (pb[max_len] == cur[max_len]) +#else + MIN(len0, len1); +#endif + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (max_len < len) { + max_len = len; + distances[offset++] = len; + distances[offset++] = delta - 1; + if (len == len_limit) { +#ifndef IS_HASH_CHAIN + *ptr1 = pair[0]; + *ptr0 = pair[1]; +#endif + break; + } + } + } + +#ifdef IS_HASH_CHAIN + cur_match = *pair; +#else + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } +#endif + } + + distances[0] = offset - 1; + + move_pos(); + + return; +} + + +LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER) +{ + do { +#ifdef IS_HASH_CHAIN + if (lz->write_pos - lz->read_pos < NUM_HASH_BYTES) { + move_pos(); + continue; + } +#else + uint32_t len_limit; + if (lz->read_pos + lz->match_max_len <= lz->write_pos) { + len_limit = lz->match_max_len; + } else { + assert(lz->stream_end_was_reached == true); + len_limit = lz->write_pos - lz->read_pos; + if (len_limit < MIN_MATCH_CHECK) { + move_pos(); + continue; + } + } + const uint32_t match_min_pos + = lz->read_pos + lz->offset > lz->cyclic_buffer_size + ? lz->read_pos + lz->offset - lz->cyclic_buffer_size + : 0; +#endif + + const uint8_t *cur = lz->buffer + lz->read_pos; + +#ifdef HASH_ARRAY_2 + uint32_t hash_2_value; +# ifdef HASH_ARRAY_3 + uint32_t hash_3_value; + uint32_t hash_value; + HASH_CALC(); + lz->hash[HASH_3_OFFSET + hash_3_value] + = lz->read_pos + lz->offset; +# else + uint32_t hash_value; + HASH_CALC(); +# endif + lz->hash[hash_2_value] = lz->read_pos + lz->offset; +#else + uint32_t hash_value; + HASH_CALC(); +#endif + + uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; + lz->hash[FIX_HASH_SIZE + hash_value] + = lz->read_pos + lz->offset; + +#ifdef IS_HASH_CHAIN + lz->son[lz->cyclic_buffer_pos] = cur_match; +#else + uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); + + uint32_t len0 = NUM_HASH_DIRECT_BYTES; + uint32_t len1 = NUM_HASH_DIRECT_BYTES; + uint32_t count = lz->cut_value; + + while (true) { + if (cur_match <= match_min_pos || count-- == 0) { + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; + break; + } + + const uint32_t delta = lz->read_pos + + lz->offset - cur_match; + const uint32_t cyclic_pos + = delta <= lz->cyclic_buffer_pos + ? lz->cyclic_buffer_pos - delta + : lz->cyclic_buffer_pos - delta + + lz->cyclic_buffer_size; + uint32_t *pair = lz->son + (cyclic_pos << 1); + + const uint8_t *pb = lz->buffer + cur_match + - lz->offset; + uint32_t len = MIN(len0, len1); + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len == len_limit) { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + break; + } + } + + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } + } +#endif + + move_pos(); + + } while (--num != 0); + + return; +} diff --git a/src/liblzma/lz/match_h.h b/src/liblzma/lz/match_h.h new file mode 100644 index 00000000..2eae90ba --- /dev/null +++ b/src/liblzma/lz/match_h.h @@ -0,0 +1,69 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file match_h.h +/// \brief Header template for different match finders +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder_private.h" + + +////////////////////// +// Global constants // +////////////////////// + +#undef LZMA_HASH_SIZE +#undef LZMA_FIX_HASH_SIZE +#undef LZMA_HASH_SIZE_C +#undef LZMA_FIX_HASH_SIZE_C + +#define LZMA_HASH_SIZE(mf_name) LZMA_HASH_SIZE_C(mf_name) +#define LZMA_FIX_HASH_SIZE(mf_name) LZMA_FIX_HASH_SIZE_C(mf_name) + +#define LZMA_HASH_SIZE_C(mf_name) \ + const uint32_t LZMA_ ## mf_name ## _HASH_SIZE + +#define LZMA_FIX_HASH_SIZE_C(mf_name) \ + const uint32_t LZMA_ ## mf_name ## _FIX_HASH_SIZE + +extern LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); +extern LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); + + +/////////////// +// Functions // +/////////////// + +#undef LZMA_GET_MATCHES +#undef LZMA_SKIP +#undef LZMA_GET_MATCHES_C +#undef LZMA_SKIP_C + +#define LZMA_GET_MATCHES(mf_name) LZMA_GET_MATCHES_C(mf_name) +#define LZMA_SKIP(mf_name) LZMA_SKIP_C(mf_name) + +#define LZMA_GET_MATCHES_C(mf_name) \ + extern void lzma_ ## mf_name ## _get_matches( \ + lzma_lz_encoder *restrict lz, \ + uint32_t *restrict distances) + +#define LZMA_SKIP_C(mf_name) \ + extern void lzma_ ## mf_name ## _skip( \ + lzma_lz_encoder *lz, uint32_t num) + +LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER); + +LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER); diff --git a/src/liblzma/lzma.pc.in b/src/liblzma/lzma.pc.in new file mode 100644 index 00000000..5bf9bb10 --- /dev/null +++ b/src/liblzma/lzma.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: liblzma +Description: LZMA compression library +URL: http://tukaani.org/lzma/ +Version: @PACKAGE_VERSION@ +Cflags: -I${includedir} +Libs: -L${libdir} -llzma diff --git a/src/liblzma/lzma/Makefile.am b/src/liblzma/lzma/Makefile.am new file mode 100644 index 00000000..48f3bb23 --- /dev/null +++ b/src/liblzma/lzma/Makefile.am @@ -0,0 +1,43 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = liblzma4.la +liblzma4_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common \ + -I@top_srcdir@/src/liblzma/lz \ + -I@top_srcdir@/src/liblzma/rangecoder + +liblzma4_la_SOURCES = \ + lzma_common.h \ + lzma_literal.c \ + lzma_literal.h + +if COND_MAIN_ENCODER +liblzma4_la_SOURCES += \ + lzma_encoder.h \ + lzma_encoder.c \ + lzma_encoder_presets.c \ + lzma_encoder_private.h \ + lzma_encoder_init.c \ + lzma_encoder_features.c \ + lzma_encoder_getoptimum.c \ + lzma_encoder_getoptimumfast.c +endif + +if COND_MAIN_DECODER +liblzma4_la_SOURCES += \ + lzma_decoder.c \ + lzma_decoder.h +endif diff --git a/src/liblzma/lzma/lzma_common.h b/src/liblzma/lzma/lzma_common.h new file mode 100644 index 00000000..4ff59ae6 --- /dev/null +++ b/src/liblzma/lzma/lzma_common.h @@ -0,0 +1,128 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_common.h +/// \brief Private definitions common to LZMA encoder and decoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA_COMMON_H +#define LZMA_LZMA_COMMON_H + +#include "common.h" +#include "lzma_literal.h" +#include "range_common.h" + + +/////////////// +// Constants // +/////////////// + +#define REP_DISTANCES 4 +#define STATES 12 +#define LIT_STATES 7 + +#define POS_SLOT_BITS 6 +#define DICT_LOG_SIZE_MAX 30 +#define DIST_TABLE_SIZE_MAX (DICT_LOG_SIZE_MAX * 2) +#if (UINT32_C(1) << DICT_LOG_SIZE_MAX) != LZMA_DICTIONARY_SIZE_MAX +# error DICT_LOG_SIZE_MAX is inconsistent with LZMA_DICTIONARY_SIZE_MAX +#endif + +// 2 is for speed optimization +#define LEN_TO_POS_STATES_BITS 2 +#define LEN_TO_POS_STATES (1 << LEN_TO_POS_STATES_BITS) + +#define MATCH_MIN_LEN 2 + +#define ALIGN_BITS 4 +#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) + +#define START_POS_MODEL_INDEX 4 +#define END_POS_MODEL_INDEX 14 +#define POS_MODELS (END_POS_MODEL_INDEX - START_POS_MODEL_INDEX) + +#define FULL_DISTANCES (1 << (END_POS_MODEL_INDEX / 2)) + +#define LIT_POS_STATES_BITS_MAX LZMA_LITERAL_POS_BITS_MAX +#define LIT_CONTEXT_BITS_MAX LZMA_LITERAL_CONTEXT_BITS_MAX + +#define POS_STATES_BITS_MAX LZMA_POS_BITS_MAX +#define POS_STATES_MAX (1 << POS_STATES_BITS_MAX) + + +// Length coder & Length price table encoder +#define LEN_LOW_BITS 3 +#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS) +#define LEN_MID_BITS 3 +#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS) +#define LEN_HIGH_BITS 8 +#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS) +#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS) +#define LEN_SPEC_SYMBOLS (LOW_LOW_SYMBOLS + LEN_MID_LEN_SYMBOLS) +#define MATCH_MAX_LEN (MATCH_MIN_LEN + LEN_SYMBOLS - 1) + +// Total number of probs in a Len Encoder +#define LEN_CODER_TOTAL_PROBS (LEN_HIGH_CODER + LEN_HIGH_SYMBOLS) + +// Price table size of Len Encoder +#define LEN_PRICES (LEN_SYMBOLS << POS_STATES_BITS_MAX) + + +// Optimal - Number of entries in the optimum array. +#define OPTS (1 << 12) + + +// Miscellaneous +#define INFINITY_PRICE 0x0FFFFFFF + + +//////////// +// Macros // +//////////// + +#define get_len_to_pos_state(len) \ + ((len) < LEN_TO_POS_STATES + MATCH_MIN_LEN \ + ? (len) - MATCH_MIN_LEN \ + : LEN_TO_POS_STATES - 1) + + +/////////// +// State // +/////////// + +// Used for updating strm->data->state in both encoder and decoder. + +#define update_char(index) \ + index = ((index) < 4 \ + ? 0 \ + : ((index) < 10 \ + ? (index) - 3 \ + : (index) - 6)) + +#define update_match(index) \ + index = ((index) < LIT_STATES ? 7 : 10) + +#define update_rep(index) \ + index = ((index) < LIT_STATES ? 8 : 11) + +#define update_short_rep(index) \ + index = ((index) < LIT_STATES ? 9 : 11) + +#define is_char_state(index) \ + ((index) < LIT_STATES) + +#endif diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c new file mode 100644 index 00000000..6e2c166d --- /dev/null +++ b/src/liblzma/lzma/lzma_decoder.c @@ -0,0 +1,844 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_decoder.c +/// \brief LZMA decoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_common.h" +#include "lzma_decoder.h" +#include "lz_decoder.h" +#include "range_decoder.h" + + +/// REQUIRED_IN_BUFFER_SIZE is the number of required input bytes +/// for the worst case: longest match with longest distance. +/// LZMA_IN_BUFFER_SIZE must be larger than REQUIRED_IN_BUFFER_SIZE. +/// 23 bits = 2 (match select) + 10 (len) + 6 (distance) + 4 (align) +/// + 1 (rc_normalize) +/// +/// \todo Is this correct for sure? +/// +#define REQUIRED_IN_BUFFER_SIZE \ + ((23 * (BIT_MODEL_TOTAL_BITS - MOVE_BITS + 1) + 26 + 9) / 8) + + +// Length decoders are easiest to have as macros, because they use range +// decoder macros, which use local variables rc_range and rc_code. + +#define length_decode(target, len_decoder, pos_state) \ +do { \ + if_bit_0(len_decoder.choice) { \ + update_bit_0(len_decoder.choice); \ + target = MATCH_MIN_LEN; \ + bittree_decode(target, \ + len_decoder.low[pos_state], LEN_LOW_BITS); \ + } else { \ + update_bit_1(len_decoder.choice); \ + if_bit_0(len_decoder.choice2) { \ + update_bit_0(len_decoder.choice2); \ + target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ + bittree_decode(target, len_decoder.mid[pos_state], \ + LEN_MID_BITS); \ + } else { \ + update_bit_1(len_decoder.choice2); \ + target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS \ + + LEN_MID_SYMBOLS; \ + bittree_decode(target, len_decoder.high, \ + LEN_HIGH_BITS); \ + } \ + } \ +} while (0) + + +#define length_decode_dummy(target, len_decoder, pos_state) \ +do { \ + if_bit_0(len_decoder.choice) { \ + update_bit_0_dummy(); \ + target = MATCH_MIN_LEN; \ + bittree_decode_dummy(target, \ + len_decoder.low[pos_state], LEN_LOW_BITS); \ + } else { \ + update_bit_1_dummy(); \ + if_bit_0(len_decoder.choice2) { \ + update_bit_0_dummy(); \ + target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ + bittree_decode_dummy(target, \ + len_decoder.mid[pos_state], \ + LEN_MID_BITS); \ + } else { \ + update_bit_1_dummy(); \ + target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS \ + + LEN_MID_SYMBOLS; \ + bittree_decode_dummy(target, len_decoder.high, \ + LEN_HIGH_BITS); \ + } \ + } \ +} while (0) + + +typedef struct { + probability choice; + probability choice2; + probability low[POS_STATES_MAX][LEN_LOW_SYMBOLS]; + probability mid[POS_STATES_MAX][LEN_MID_SYMBOLS]; + probability high[LEN_HIGH_SYMBOLS]; +} lzma_length_decoder; + + +struct lzma_coder_s { + /// Data of the next coder, if any. + lzma_next_coder next; + + /// Sliding output window a.k.a. dictionary a.k.a. history buffer. + lzma_lz_decoder lz; + + // Range coder + lzma_range_decoder rc; + + // State + uint32_t state; + uint32_t rep0; ///< Distance of the latest match + uint32_t rep1; ///< Distance of second latest match + uint32_t rep2; ///< Distance of third latest match + uint32_t rep3; ///< Distance of fourth latest match + uint32_t pos_bits; + uint32_t pos_mask; + uint32_t now_pos; // Lowest 32-bits are enough here. + + lzma_literal_coder *literal_coder; + + /// If 1, it's a match. Otherwise it's a single 8-bit literal. + probability is_match[STATES][POS_STATES_MAX]; + + /// If 1, it's a repeated match. The distance is one of rep0 .. rep3. + probability is_rep[STATES]; + + /// If 0, distance of a repeated match is rep0. + /// Otherwise check is_rep1. + probability is_rep0[STATES]; + + /// If 0, distance of a repeated match is rep1. + /// Otherwise check is_rep2. + probability is_rep1[STATES]; + + /// If 0, distance of a repeated match is rep2. Otherwise it is rep3. + probability is_rep2[STATES]; + + /// If 1, the repeated match has length of one byte. Otherwise + /// the length is decoded from rep_match_len_decoder. + probability is_rep0_long[STATES][POS_STATES_MAX]; + + probability pos_slot_decoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; + probability pos_decoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_align_decoder[1 << ALIGN_BITS]; + + /// Length of a match + lzma_length_decoder len_decoder; + + /// Length of a repeated match. + lzma_length_decoder rep_match_len_decoder; + + /// The first five bytes of LZMA compressed data are treated + /// specially. Once they are read, this stays at zero. + size_t init_bytes_left; +}; + + +/// \brief Check if the next iteration of the decoder loop can be run. +/// +/// \note There must always be REQUIRED_IN_BUFFER_SIZE bytes +/// readable space! +/// +static bool lzma_attribute((pure)) +decode_dummy(const lzma_coder *restrict coder, + const uint8_t *restrict in, size_t in_pos_local, + const size_t in_size, uint32_t rc_range, uint32_t rc_code, + uint32_t state, uint32_t rep0, const uint32_t now_pos) +{ + uint32_t rc_bound; + + do { + const uint32_t pos_state = now_pos & coder->pos_mask; + + if_bit_0(coder->is_match[state][pos_state]) { + // It's a literal i.e. a single 8-bit byte. + + update_bit_0_dummy(); + + const probability *subcoder = literal_get_subcoder( + coder->literal_coder, + now_pos, lz_get_byte(coder->lz, 0)); + uint32_t symbol = 1; + + if (!is_char_state(state)) { + // Decode literal with match byte. + + assert(rep0 != UINT32_MAX); + uint32_t match_byte + = lz_get_byte(coder->lz, rep0); + + do { + match_byte <<= 1; + const uint32_t match_bit + = match_byte & 0x100; + const uint32_t subcoder_index = 0x100 + + match_bit + symbol; + + if_bit_0(subcoder[subcoder_index]) { + update_bit_0_dummy(); + symbol <<= 1; + if (match_bit != 0) + break; + } else { + update_bit_1_dummy(); + symbol = (symbol << 1) | 1; + if (match_bit == 0) + break; + } + } while (symbol < 0x100); + } + + // Decode literal without match byte. This is also + // the tail of the with-match-byte function. + while (symbol < 0x100) { + if_bit_0(subcoder[symbol]) { + update_bit_0_dummy(); + symbol <<= 1; + } else { + update_bit_1_dummy(); + symbol = (symbol << 1) | 1; + } + } + + break; + } + + update_bit_1_dummy(); + uint32_t len; + + if_bit_0(coder->is_rep[state]) { + update_bit_0_dummy(); + length_decode_dummy(len, coder->len_decoder, pos_state); + update_match(state); + + const uint32_t len_to_pos_state + = get_len_to_pos_state(len); + uint32_t pos_slot = 0; + bittree_decode_dummy(pos_slot, coder->pos_slot_decoder[ + len_to_pos_state], POS_SLOT_BITS); + assert(pos_slot <= 63); + + if (pos_slot >= START_POS_MODEL_INDEX) { + uint32_t direct_bits = (pos_slot >> 1) - 1; + assert(direct_bits >= 1 && direct_bits <= 31); + rep0 = 2 | (pos_slot & 1); + + if (pos_slot < END_POS_MODEL_INDEX) { + assert(direct_bits <= 5); + rep0 <<= direct_bits; + assert(rep0 <= 96); + // -1 is fine, because + // bittree_reverse_decode() + // starts from table index [1] + // (not [0]). + assert((int32_t)(rep0 - pos_slot - 1) + >= -1); + assert((int32_t)(rep0 - pos_slot - 1) + <= 82); + // We add the result to rep0, so rep0 + // must not be part of second argument + // of the macro. + const int32_t offset + = rep0 - pos_slot - 1; + bittree_reverse_decode_dummy( + coder->pos_decoders + offset, + direct_bits); + } else { + // Decode direct bits + assert(pos_slot >= 14); + assert(direct_bits >= 6); + direct_bits -= ALIGN_BITS; + assert(direct_bits >= 2); + do { + rc_normalize(); + rc_range >>= 1; + const uint32_t t + = (rc_code - rc_range) + >> 31; + rc_code -= rc_range & (t - 1); + } while (--direct_bits > 0); + rep0 <<= ALIGN_BITS; + + bittree_reverse_decode_dummy( + coder->pos_align_decoder, + ALIGN_BITS); + } + } + + } else { + update_bit_1_dummy(); + + if_bit_0(coder->is_rep0[state]) { + update_bit_0_dummy(); + + if_bit_0(coder->is_rep0_long[state][ + pos_state]) { + update_bit_0_dummy(); + break; + } else { + update_bit_1_dummy(); + } + + } else { + update_bit_1_dummy(); + + if_bit_0(coder->is_rep1[state]) { + update_bit_0_dummy(); + } else { + update_bit_1_dummy(); + + if_bit_0(coder->is_rep2[state]) { + update_bit_0_dummy(); + } else { + update_bit_1_dummy(); + } + } + } + + length_decode_dummy(len, coder->rep_match_len_decoder, + pos_state); + } + } while (0); + + rc_normalize(); + + // Validate the buffer position. + if (in_pos_local > in_size) + return false; + + return true; +} + + +static bool +decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, bool has_safe_buffer) +{ + //////////////////// + // Initialization // + //////////////////// + + while (coder->init_bytes_left > 0) { + if (*in_pos == in_size) + return false; + + coder->rc.code = (coder->rc.code << 8) | in[*in_pos]; + ++*in_pos; + --coder->init_bytes_left; + } + + + /////////////// + // Variables // + /////////////// + + // Making local copies of often-used variables improves both + // speed and readability. + + // Range decoder + rc_to_local(coder->rc); + + // State + uint32_t state = coder->state; + uint32_t rep0 = coder->rep0; + uint32_t rep1 = coder->rep1; + uint32_t rep2 = coder->rep2; + uint32_t rep3 = coder->rep3; + + // Misc + uint32_t now_pos = coder->now_pos; + + // Variables derived from decoder settings + const uint32_t pos_mask = coder->pos_mask; + + size_t in_pos_local = *in_pos; // Local copy + size_t in_limit; + if (in_size <= REQUIRED_IN_BUFFER_SIZE) + in_limit = 0; + else + in_limit = in_size - REQUIRED_IN_BUFFER_SIZE; + + + while (coder->lz.pos < coder->lz.limit && (in_pos_local < in_limit + || (has_safe_buffer && decode_dummy( + coder, in, in_pos_local, in_size, + rc_range, rc_code, state, rep0, now_pos)))) { + + ///////////////////// + // Actual decoding // + ///////////////////// + + const uint32_t pos_state = now_pos & pos_mask; + + if_bit_0(coder->is_match[state][pos_state]) { + update_bit_0(coder->is_match[state][pos_state]); + + // It's a literal i.e. a single 8-bit byte. + + probability *subcoder = literal_get_subcoder( + coder->literal_coder, + now_pos, lz_get_byte(coder->lz, 0)); + uint32_t symbol = 1; + + if (!is_char_state(state)) { + // Decode literal with match byte. + + assert(rep0 != UINT32_MAX); + uint32_t match_byte + = lz_get_byte(coder->lz, rep0); + + do { + match_byte <<= 1; + const uint32_t match_bit + = match_byte & 0x100; + const uint32_t subcoder_index = 0x100 + + match_bit + symbol; + + if_bit_0(subcoder[subcoder_index]) { + update_bit_0(subcoder[ + subcoder_index]); + symbol <<= 1; + if (match_bit != 0) + break; + } else { + update_bit_1(subcoder[ + subcoder_index]); + symbol = (symbol << 1) | 1; + if (match_bit == 0) + break; + } + } while (symbol < 0x100); + } + + // Decode literal without match byte. This is also + // the tail of the with-match-byte function. + while (symbol < 0x100) { + if_bit_0(subcoder[symbol]) { + update_bit_0(subcoder[symbol]); + symbol <<= 1; + } else { + update_bit_1(subcoder[symbol]); + symbol = (symbol << 1) | 1; + } + } + + // Put the decoded byte to the dictionary, update the + // decoder state, and start a new decoding loop. + coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol); + ++now_pos; + update_char(state); + continue; + } + + // Instead of a new byte we are going to get a byte range + // (distance and length) which will be repeated from our + // output history. + + update_bit_1(coder->is_match[state][pos_state]); + uint32_t len; + + if_bit_0(coder->is_rep[state]) { + update_bit_0(coder->is_rep[state]); + + // Not a repeated match + // + // We will decode a new distance and store + // the value to rep0. + + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + + // Decode the length of the match. + length_decode(len, coder->len_decoder, pos_state); + + update_match(state); + + const uint32_t len_to_pos_state + = get_len_to_pos_state(len); + uint32_t pos_slot = 0; + bittree_decode(pos_slot, coder->pos_slot_decoder[ + len_to_pos_state], POS_SLOT_BITS); + assert(pos_slot <= 63); + + if (pos_slot >= START_POS_MODEL_INDEX) { + uint32_t direct_bits = (pos_slot >> 1) - 1; + assert(direct_bits >= 1 && direct_bits <= 30); + rep0 = 2 | (pos_slot & 1); + + if (pos_slot < END_POS_MODEL_INDEX) { + assert(direct_bits <= 5); + rep0 <<= direct_bits; + assert(rep0 <= 96); + // -1 is fine, because + // bittree_reverse_decode() + // starts from table index [1] + // (not [0]). + assert((int32_t)(rep0 - pos_slot - 1) + >= -1); + assert((int32_t)(rep0 - pos_slot - 1) + <= 82); + // We add the result to rep0, so rep0 + // must not be part of second argument + // of the macro. + const int32_t offset + = rep0 - pos_slot - 1; + bittree_reverse_decode(rep0, + coder->pos_decoders + offset, + direct_bits); + } else { + // Decode direct bits + assert(pos_slot >= 14); + assert(direct_bits >= 6); + direct_bits -= ALIGN_BITS; + assert(direct_bits >= 2); + do { + rc_normalize(); + rc_range >>= 1; + const uint32_t t + = (rc_code - rc_range) + >> 31; + rc_code -= rc_range & (t - 1); + rep0 = (rep0 << 1) | (1 - t); + } while (--direct_bits > 0); + rep0 <<= ALIGN_BITS; + + bittree_reverse_decode(rep0, + coder->pos_align_decoder, + ALIGN_BITS); + + if (rep0 == UINT32_MAX) { + // End of Payload Marker found. + coder->lz.eopm_detected = true; + break; + } + } + } else { + rep0 = pos_slot; + } + + } else { + update_bit_1(coder->is_rep[state]); + + // Repeated match + // + // The match distance is a value that we have had + // earlier. The latest four match distances are + // available as rep0, rep1, rep2 and rep3. We will + // now decode which of them is the new distance. + + if_bit_0(coder->is_rep0[state]) { + update_bit_0(coder->is_rep0[state]); + + // The distance is rep0. + + if_bit_0(coder->is_rep0_long[state][ + pos_state]) { + update_bit_0(coder->is_rep0_long[ + state][pos_state]); + + // Repeating exactly one byte. For + // simplicity, it is done here inline + // instead of at the end of the main + // loop. + + update_short_rep(state); + + // Security/sanity checks. See the end + // of the main loop for explanation + // of these. + if ((rep0 >= coder->lz.pos + && !coder->lz.is_full) + || in_pos_local + > in_size) + goto error; + + // Repeat one byte and start a new + // decoding loop. + coder->lz.dict[coder->lz.pos] + = lz_get_byte( + coder->lz, rep0); + ++coder->lz.pos; + ++now_pos; + continue; + + } else { + update_bit_1(coder->is_rep0_long[ + state][pos_state]); + + // Repeating more than one byte at + // distance of rep0. + } + + } else { + update_bit_1(coder->is_rep0[state]); + + // The distance is rep1, rep2 or rep3. Once + // we find out which one of these three, it + // is stored to rep0 and rep1, rep2 and rep3 + // are updated accordingly. + + uint32_t distance; + + if_bit_0(coder->is_rep1[state]) { + update_bit_0(coder->is_rep1[state]); + distance = rep1; + } else { + update_bit_1(coder->is_rep1[state]); + + if_bit_0(coder->is_rep2[state]) { + update_bit_0(coder->is_rep2[ + state]); + distance = rep2; + } else { + update_bit_1(coder->is_rep2[ + state]); + distance = rep3; + rep3 = rep2; + } + + rep2 = rep1; + } + + rep1 = rep0; + rep0 = distance; + } + + // Decode the length of the repeated match. + length_decode(len, coder->rep_match_len_decoder, + pos_state); + + update_rep(state); + } + + + ///////////////////////////////// + // Repeat from history buffer. // + ///////////////////////////////// + + // The length is always between these limits. There is no way + // to trigger the algorithm to set len outside this range. + assert(len >= MATCH_MIN_LEN); + assert(len <= MATCH_MAX_LEN); + + now_pos += len; + + // Validate the buffer position to avoid buffer overflows + // on corrupted input data. + if (in_pos_local > in_size) + goto error; + + // Repeat len bytes from distance of rep0. + if (!lzma_lz_out_repeat(&coder->lz, rep0, len)) + goto error; + } + + rc_normalize(); + + + ///////////////////////////////// + // Update the *data structure. // + ///////////////////////////////// + + // Range decoder + rc_from_local(coder->rc); + + // State + coder->state = state; + coder->rep0 = rep0; + coder->rep1 = rep1; + coder->rep2 = rep2; + coder->rep3 = rep3; + + // Misc + coder->now_pos = now_pos; + *in_pos = in_pos_local; + + return false; + +error: + return true; +} + + +static void +lzma_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_lz_decoder_end(&coder->lz, allocator); + lzma_literal_end(&coder->literal_coder, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // Validate pos_bits. Other options are validated by the + // respective initialization functions. + const lzma_options_lzma *options = filters[0].options; + if (options->pos_bits > LZMA_POS_BITS_MAX) + return LZMA_HEADER_ERROR; + + // Allocate memory for the decoder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + // Initialize variables so that we know later that we don't + // have an existing decoder initialized. + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->lz = LZMA_LZ_DECODER_INIT; + next->coder->literal_coder = NULL; + } + + // Store the pos_bits and calculate pos_mask. + next->coder->pos_bits = options->pos_bits; + next->coder->pos_mask = (1U << next->coder->pos_bits) - 1; + + // Allocate (if needed) and initialize the literal decoder. + { + const lzma_ret ret = lzma_literal_init( + &next->coder->literal_coder, allocator, + options->literal_context_bits, + options->literal_pos_bits); + if (ret != LZMA_OK) { + lzma_free(next->coder, allocator); + next->coder = NULL; + return ret; + } + } + + // Allocate and initialize the LZ decoder. + { + const lzma_ret ret = lzma_lz_decoder_reset( + &next->coder->lz, allocator, &decode_real, + filters[0].uncompressed_size, + options->dictionary_size, MATCH_MAX_LEN); + if (ret != LZMA_OK) { + lzma_literal_end(&next->coder->literal_coder, + allocator); + lzma_free(next->coder, allocator); + next->coder = NULL; + return ret; + } + } + + // State + next->coder->state = 0; + next->coder->rep0 = 0; + next->coder->rep1 = 0; + next->coder->rep2 = 0; + next->coder->rep3 = 0; + next->coder->pos_bits = options->pos_bits; + next->coder->pos_mask = (1 << next->coder->pos_bits) - 1; + next->coder->now_pos = 0; + next->coder->init_bytes_left = 5; + + // Range decoder + rc_reset(next->coder->rc); + + // Bit and bittree decoders + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= next->coder->pos_mask; ++j) { + bit_reset(next->coder->is_match[i][j]); + bit_reset(next->coder->is_rep0_long[i][j]); + } + + bit_reset(next->coder->is_rep[i]); + bit_reset(next->coder->is_rep0[i]); + bit_reset(next->coder->is_rep1[i]); + bit_reset(next->coder->is_rep2[i]); + } + + for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i) + bittree_reset(next->coder->pos_slot_decoder[i], POS_SLOT_BITS); + + for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + bit_reset(next->coder->pos_decoders[i]); + + bittree_reset(next->coder->pos_align_decoder, ALIGN_BITS); + + // Len decoders (also bit/bittree) + const uint32_t num_pos_states = 1 << next->coder->pos_bits; + bit_reset(next->coder->len_decoder.choice); + bit_reset(next->coder->len_decoder.choice2); + bit_reset(next->coder->rep_match_len_decoder.choice); + bit_reset(next->coder->rep_match_len_decoder.choice2); + + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { + bittree_reset(next->coder->len_decoder.low[pos_state], + LEN_LOW_BITS); + bittree_reset(next->coder->len_decoder.mid[pos_state], + LEN_MID_BITS); + + bittree_reset(next->coder->rep_match_len_decoder.low[ + pos_state], LEN_LOW_BITS); + bittree_reset(next->coder->rep_match_len_decoder.mid[ + pos_state], LEN_MID_BITS); + } + + bittree_reset(next->coder->len_decoder.high, LEN_HIGH_BITS); + bittree_reset(next->coder->rep_match_len_decoder.high, LEN_HIGH_BITS); + + // Initialize the next decoder in the chain, if any. + { + const lzma_ret ret = lzma_next_filter_init(&next->coder->next, + allocator, filters + 1); + if (ret != LZMA_OK) { + lzma_decoder_end(next->coder, allocator); + return ret; + } + } + + // Initialization successful. Set the function pointers. + next->code = &lzma_lz_decode; + next->end = &lzma_decoder_end; + + return LZMA_OK; +} + + +extern bool +lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte) +{ + if (byte > (4 * 5 + 4) * 9 + 8) + return true; + + // See the file format specification to understand this. + options->pos_bits = byte / (9 * 5); + byte -= options->pos_bits * 9 * 5; + options->literal_pos_bits = byte / 9; + options->literal_context_bits = byte - options->literal_pos_bits * 9; + + return false; +} diff --git a/src/liblzma/lzma/lzma_decoder.h b/src/liblzma/lzma/lzma_decoder.h new file mode 100644 index 00000000..929c2bff --- /dev/null +++ b/src/liblzma/lzma/lzma_decoder.h @@ -0,0 +1,41 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_decoder.h +/// \brief LZMA decoder API +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA_DECODER_H +#define LZMA_LZMA_DECODER_H + +#include "common.h" + + +/// \brief Allocates and initializes LZMA decoder +extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +/// \brief Decodes the LZMA Properties byte (lc/lp/pb) +/// +/// \return true if error occorred, false on success +/// +extern bool lzma_lzma_decode_properties( + lzma_options_lzma *options, uint8_t byte); + +// There is no public lzma_lzma_encode() because lzma_lz_encode() works +// as a wrapper for it. + +#endif diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c new file mode 100644 index 00000000..f9c1e3fe --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder.c @@ -0,0 +1,413 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder.c +/// \brief LZMA encoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +// NOTE: If you want to keep the line length in 80 characters, set +// tab width to 4 or less in your editor when editing this file. + + +#include "lzma_encoder_private.h" + + +//////////// +// Macros // +//////////// + +// These are as macros mostly because they use local range encoder variables. + +#define literal_encode(subcoder, symbol) \ +do { \ + uint32_t context = 1; \ + int i = 8; \ + do { \ + --i; \ + const uint32_t bit = ((symbol) >> i) & 1; \ + bit_encode(subcoder[context], bit); \ + context = (context << 1) | bit; \ + } while (i != 0); \ +} while (0) + + +#define literal_encode_matched(subcoder, match_byte, symbol) \ +do { \ + uint32_t context = 1; \ + int i = 8; \ + do { \ + --i; \ + uint32_t bit = ((symbol) >> i) & 1; \ + const uint32_t match_bit = ((match_byte) >> i) & 1; \ + const uint32_t subcoder_index = 0x100 + (match_bit << 8) + context; \ + bit_encode(subcoder[subcoder_index], bit); \ + context = (context << 1) | bit; \ + if (match_bit != bit) { \ + while (i != 0) { \ + --i; \ + bit = ((symbol) >> i) & 1; \ + bit_encode(subcoder[context], bit); \ + context = (context << 1) | bit; \ + } \ + break; \ + } \ + } while (i != 0); \ +} while (0) + + +#define length_encode(length_encoder, symbol, pos_state, update_price) \ +do { \ + \ + if ((symbol) < LEN_LOW_SYMBOLS) { \ + bit_encode_0((length_encoder).choice); \ + bittree_encode((length_encoder).low[pos_state], \ + LEN_LOW_BITS, symbol); \ + } else { \ + bit_encode_1((length_encoder).choice); \ + if ((symbol) < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS) { \ + bit_encode_0((length_encoder).choice2); \ + bittree_encode((length_encoder).mid[pos_state], \ + LEN_MID_BITS, \ + (symbol) - LEN_LOW_SYMBOLS); \ + } else { \ + bit_encode_1((length_encoder).choice2); \ + bittree_encode((length_encoder).high, LEN_HIGH_BITS, \ + (symbol) - LEN_LOW_SYMBOLS \ + - LEN_MID_SYMBOLS); \ + } \ + } \ + if (update_price) \ + if (--(length_encoder).counters[pos_state] == 0) \ + lzma_length_encoder_update_table(&(length_encoder), pos_state); \ +} while (0) + + +/////////////// +// Functions // +/////////////// + +/// \brief Updates price table of the length encoder +/// +/// All all the other prices in LZMA, these are used by lzma_get_optimum(). +/// +extern void +lzma_length_encoder_update_table(lzma_length_encoder *lencoder, + const uint32_t pos_state) +{ + const uint32_t num_symbols = lencoder->table_size; + const uint32_t a0 = bit_get_price_0(lencoder->choice); + const uint32_t a1 = bit_get_price_1(lencoder->choice); + const uint32_t b0 = a1 + bit_get_price_0(lencoder->choice2); + const uint32_t b1 = a1 + bit_get_price_1(lencoder->choice2); + + uint32_t *prices = lencoder->prices[pos_state]; + uint32_t i = 0; + + for (i = 0; i < num_symbols && i < LEN_LOW_SYMBOLS; ++i) { + prices[i] = a0; + bittree_get_price(prices[i], lencoder->low[pos_state], + LEN_LOW_BITS, i); + } + + for (; i < num_symbols && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) { + prices[i] = b0; + bittree_get_price(prices[i], lencoder->mid[pos_state], + LEN_MID_BITS, i - LEN_LOW_SYMBOLS); + } + + for (; i < num_symbols; ++i) { + prices[i] = b1; + bittree_get_price(prices[i], lencoder->high, LEN_HIGH_BITS, + i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); + } + + lencoder->counters[pos_state] = num_symbols; + + return; +} + + +/** + * \brief LZMA encoder + * + * \return true if end of stream was reached, false otherwise. + */ +extern bool +lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) +{ + // Flush the range encoder's temporary buffer to out[]. + // Return immediatelly if not everything could be flushed. + if (rc_flush_buffer(&coder->rc, out, out_pos, out_size)) + return false; + + // Return immediatelly if we have already finished our work. + if (coder->lz.stream_end_was_reached + && coder->is_initialized + && coder->lz.read_pos == coder->lz.write_pos + && coder->additional_offset == 0) + return true; + + // Local copies + rc_to_local(coder->rc); + size_t out_pos_local = *out_pos; + const uint32_t pos_mask = coder->pos_mask; + const bool best_compression = coder->best_compression; + + // Initialize the stream if no data has been encoded yet. + if (!coder->is_initialized) { + if (coder->lz.read_pos == coder->lz.read_limit) { + // Cannot initialize, because there is no input data. + if (!coder->lz.stream_end_was_reached) + return false; + + // If we get here, we are encoding an empty file. + // Initialization is skipped completely. + assert(coder->lz.write_pos == coder->lz.read_pos); + + } else { + // Do the actual initialization. + uint32_t len; + uint32_t num_distance_pairs; + lzma_read_match_distances(coder, &len, &num_distance_pairs); + + bit_encode_0(coder->is_match[coder->state][0]); + update_char(coder->state); + + const uint8_t cur_byte = coder->lz.buffer[ + coder->lz.read_pos - coder->additional_offset]; + probability *subcoder = literal_get_subcoder(coder->literal_coder, + coder->now_pos, coder->previous_byte); + literal_encode(subcoder, cur_byte); + + coder->previous_byte = cur_byte; + --coder->additional_offset; + ++coder->now_pos; + + assert(coder->additional_offset == 0); + } + + // Initialization is done (except if empty file). + coder->is_initialized = true; + } + + // Encoding loop + while (true) { + // Check that there is free output space. + if (out_pos_local == out_size) + break; + + assert(rc_buffer_size == 0); + + // Check that there is some input to process. + if (coder->lz.read_pos >= coder->lz.read_limit) { + // If end of input has been reached, we must keep + // encoding until additional_offset becomes zero. + if (!coder->lz.stream_end_was_reached + || coder->additional_offset == 0) + break; + } + + assert(coder->lz.read_pos <= coder->lz.write_pos); + +#ifndef NDEBUG + if (coder->lz.stream_end_was_reached) { + assert(coder->lz.read_limit == coder->lz.write_pos); + } else { + assert(coder->lz.read_limit + coder->lz.keep_size_after + == coder->lz.write_pos); + } +#endif + + const uint32_t pos_state = coder->now_pos & pos_mask; + + uint32_t pos; + uint32_t len; + + // Get optimal match (repeat position and length). + // Value ranges for pos: + // - [0, REP_DISTANCES): repeated match + // - [REP_DISTANCES, UINT32_MAX): match at (pos - REP_DISTANCES) + // - UINT32_MAX: not a match but a literal + // Value ranges for len: + // - [MATCH_MIN_LEN, MATCH_MAX_LEN] + if (best_compression) + lzma_get_optimum(coder, &pos, &len); + else + lzma_get_optimum_fast(coder, &pos, &len); + + if (len == 1 && pos == UINT32_MAX) { + // It's a literal. + bit_encode_0(coder->is_match[coder->state][pos_state]); + + const uint8_t cur_byte = coder->lz.buffer[ + coder->lz.read_pos - coder->additional_offset]; + probability *subcoder = literal_get_subcoder(coder->literal_coder, + coder->now_pos, coder->previous_byte); + + if (is_char_state(coder->state)) { + literal_encode(subcoder, cur_byte); + } else { + const uint8_t match_byte = coder->lz.buffer[ + coder->lz.read_pos + - coder->rep_distances[0] - 1 + - coder->additional_offset]; + literal_encode_matched(subcoder, match_byte, cur_byte); + } + + update_char(coder->state); + coder->previous_byte = cur_byte; + + } else { + // It's a match. + bit_encode_1(coder->is_match[coder->state][pos_state]); + + if (pos < REP_DISTANCES) { + // It's a repeated match i.e. the same distance + // has been used earlier. + bit_encode_1(coder->is_rep[coder->state]); + + if (pos == 0) { + bit_encode_0(coder->is_rep0[coder->state]); + const uint32_t symbol = (len == 1) ? 0 : 1; + bit_encode(coder->is_rep0_long[coder->state][pos_state], + symbol); + } else { + const uint32_t distance = coder->rep_distances[pos]; + bit_encode_1(coder->is_rep0[coder->state]); + + if (pos == 1) { + bit_encode_0(coder->is_rep1[coder->state]); + } else { + bit_encode_1(coder->is_rep1[coder->state]); + bit_encode(coder->is_rep2[coder->state], pos - 2); + + if (pos == 3) + coder->rep_distances[3] = coder->rep_distances[2]; + + coder->rep_distances[2] = coder->rep_distances[1]; + } + + coder->rep_distances[1] = coder->rep_distances[0]; + coder->rep_distances[0] = distance; + } + + if (len == 1) { + update_short_rep(coder->state); + } else { + length_encode(coder->rep_match_len_encoder, + len - MATCH_MIN_LEN, pos_state, + best_compression); + update_rep(coder->state); + } + + } else { + bit_encode_0(coder->is_rep[coder->state]); + update_match(coder->state); + length_encode(coder->len_encoder, len - MATCH_MIN_LEN, + pos_state, best_compression); + pos -= REP_DISTANCES; + + const uint32_t pos_slot = get_pos_slot(pos); + const uint32_t len_to_pos_state = get_len_to_pos_state(len); + bittree_encode(coder->pos_slot_encoder[len_to_pos_state], + POS_SLOT_BITS, pos_slot); + + if (pos_slot >= START_POS_MODEL_INDEX) { + const uint32_t footer_bits = (pos_slot >> 1) - 1; + const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + const uint32_t pos_reduced = pos - base; + + if (pos_slot < END_POS_MODEL_INDEX) { + bittree_reverse_encode( + coder->pos_encoders + base - pos_slot - 1, + footer_bits, pos_reduced); + } else { + rc_encode_direct_bits(pos_reduced >> ALIGN_BITS, + footer_bits - ALIGN_BITS); + bittree_reverse_encode(coder->pos_align_encoder, + ALIGN_BITS, pos_reduced & ALIGN_MASK); + ++coder->align_price_count; + } + } + + coder->rep_distances[3] = coder->rep_distances[2]; + coder->rep_distances[2] = coder->rep_distances[1]; + coder->rep_distances[1] = coder->rep_distances[0]; + coder->rep_distances[0] = pos; + ++coder->match_price_count; + } + + coder->previous_byte = coder->lz.buffer[ + coder->lz.read_pos + len - 1 + - coder->additional_offset]; + } + + assert(coder->additional_offset >= len); + coder->additional_offset -= len; + coder->now_pos += len; + } + + // Check if everything is done. + bool all_done = false; + if (coder->lz.stream_end_was_reached + && coder->lz.read_pos == coder->lz.write_pos + && coder->additional_offset == 0) { + // Write end of stream marker. It is encoded as a match with + // distance of UINT32_MAX. Match length is needed but it is + // ignored by the decoder. + if (coder->lz.uncompressed_size == LZMA_VLI_VALUE_UNKNOWN) { + const uint32_t pos_state = coder->now_pos & pos_mask; + bit_encode_1(coder->is_match[coder->state][pos_state]); + bit_encode_0(coder->is_rep[coder->state]); + update_match(coder->state); + + const uint32_t len = MATCH_MIN_LEN; // MATCH_MAX_LEN; + length_encode(coder->len_encoder, len - MATCH_MIN_LEN, + pos_state, best_compression); + + const uint32_t pos_slot = (1 << POS_SLOT_BITS) - 1; + const uint32_t len_to_pos_state = get_len_to_pos_state(len); + bittree_encode(coder->pos_slot_encoder[len_to_pos_state], + POS_SLOT_BITS, pos_slot); + + const uint32_t footer_bits = 30; + const uint32_t pos_reduced + = (UINT32_C(1) << footer_bits) - 1; + rc_encode_direct_bits(pos_reduced >> ALIGN_BITS, + footer_bits - ALIGN_BITS); + + bittree_reverse_encode(coder->pos_align_encoder, ALIGN_BITS, + pos_reduced & ALIGN_MASK); + } + + // Flush the last bytes of compressed data from + // the range coder to the output buffer. + rc_flush(); + + // All done. Note that some output bytes might be + // pending in coder->buffer. lzma_encode() will + // take care of those bytes. + if (rc_buffer_size == 0) + all_done = true; + } + + // Store local variables back to *coder. + rc_from_local(coder->rc); + *out_pos = out_pos_local; + + return all_done; +} diff --git a/src/liblzma/lzma/lzma_encoder.h b/src/liblzma/lzma/lzma_encoder.h new file mode 100644 index 00000000..1c57f80a --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder.h @@ -0,0 +1,35 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder.h +/// \brief LZMA method handler API +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA_ENCODER_H +#define LZMA_LZMA_ENCODER_H + +#include "common.h" + +extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern bool lzma_lzma_encode_properties( + const lzma_options_lzma *options, uint8_t *byte); + +/// Initializes the lzma_fastpos[] array. +extern void lzma_fastpos_init(void); + +#endif diff --git a/src/liblzma/lzma/lzma_encoder_features.c b/src/liblzma/lzma/lzma_encoder_features.c new file mode 100644 index 00000000..56e59c6a --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_features.c @@ -0,0 +1,59 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_features.c +/// \brief Information about features enabled at compile time +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +static lzma_mode modes[] = { + LZMA_MODE_FAST, + LZMA_MODE_BEST, + LZMA_MODE_INVALID +}; + + +LZMA_API const lzma_mode *const lzma_available_modes = modes; + + +static lzma_match_finder match_finders[] = { +#ifdef HAVE_MF_HC3 + LZMA_MF_HC3, +#endif + +#ifdef HAVE_MF_HC4 + LZMA_MF_HC4, +#endif + +#ifdef HAVE_MF_BT2 + LZMA_MF_BT2, +#endif + +#ifdef HAVE_MF_BT3 + LZMA_MF_BT3, +#endif + +#ifdef HAVE_MF_BT4 + LZMA_MF_BT4, +#endif + + LZMA_MF_INVALID +}; + + +LZMA_API const lzma_match_finder *const lzma_available_match_finders + = match_finders; diff --git a/src/liblzma/lzma/lzma_encoder_getoptimum.c b/src/liblzma/lzma/lzma_encoder_getoptimum.c new file mode 100644 index 00000000..cdeb3145 --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_getoptimum.c @@ -0,0 +1,893 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_getoptimum.c +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +// NOTE: If you want to keep the line length in 80 characters, set +// tab width to 4 or less in your editor when editing this file. + + +// "Would you love the monster code? +// Could you understand beauty of the beast?" +// --Adapted from Lordi's "Would you love a monster man". + + +#include "lzma_encoder_private.h" + + +#define length_get_price(length_encoder, symbol, pos_state) \ + (length_encoder).prices[pos_state][symbol] + + +#define get_rep_len_1_price(state, pos_state) \ + bit_get_price_0(coder->is_rep0[state]) \ + + bit_get_price_0(coder->is_rep0_long[state][pos_state]) + + +// Adds to price_target. +#define get_pure_rep_price(price_target, rep_index, state, pos_state) \ +do { \ + if ((rep_index) == 0) { \ + price_target += bit_get_price_0(coder->is_rep0[state]); \ + price_target += bit_get_price_1( \ + coder->is_rep0_long[state][pos_state]); \ + } else { \ + price_target += bit_get_price_1(coder->is_rep0[state]); \ + if ((rep_index) == 1) { \ + price_target += bit_get_price_0(coder->is_rep1[state]); \ + } else { \ + price_target += bit_get_price_1(coder->is_rep1[state]); \ + price_target += bit_get_price( \ + coder->is_rep2[state], (rep_index) - 2); \ + } \ + } \ +} while (0) + + +// Adds to price_target. +#define get_rep_price(price_target, rep_index, len, state, pos_state) \ +do { \ + get_pure_rep_price(price_target, rep_index, state, pos_state); \ + price_target += length_get_price(coder->rep_match_len_encoder, \ + (len) - MATCH_MIN_LEN, pos_state); \ +} while (0) + + +// Adds to price_target. +#define get_pos_len_price(price_target, pos, len, pos_state) \ +do { \ + const uint32_t len_to_pos_state_tmp = get_len_to_pos_state(len); \ + if ((pos) < FULL_DISTANCES) { \ + price_target += distances_prices[len_to_pos_state_tmp][pos]; \ + } else { \ + price_target \ + += pos_slot_prices[len_to_pos_state_tmp][get_pos_slot_2(pos)] \ + + align_prices[(pos) & ALIGN_MASK]; \ + } \ + price_target += length_get_price( \ + coder->len_encoder, (len) - MATCH_MIN_LEN, pos_state); \ +} while (0) + + +// Three macros to manipulate lzma_optimal structures: +#define make_as_char(opt) \ +do { \ + (opt).back_prev = UINT32_MAX; \ + (opt).prev_1_is_char = false; \ +} while (0) + + +#define make_as_short_rep(opt) \ +do { \ + (opt).back_prev = 0; \ + (opt).prev_1_is_char = false; \ +} while (0) + + +#define is_short_rep(opt) \ + ((opt).back_prev == 0) + + +static void +fill_distances_prices(lzma_coder *coder) +{ + uint32_t temp_prices[FULL_DISTANCES]; + + for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { + const uint32_t pos_slot = get_pos_slot(i); + const uint32_t footer_bits = ((pos_slot >> 1) - 1); + const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + temp_prices[i] = 0; + bittree_reverse_get_price(temp_prices[i], + coder->pos_encoders + base - pos_slot - 1, + footer_bits, i - base); + } + + const uint32_t dist_table_size = coder->dist_table_size; + + for (uint32_t len_to_pos_state = 0; + len_to_pos_state < LEN_TO_POS_STATES; + ++len_to_pos_state) { + + const probability *encoder = coder->pos_slot_encoder[len_to_pos_state]; + uint32_t *pos_slot_prices = coder->pos_slot_prices[len_to_pos_state]; + + for (uint32_t pos_slot = 0; + pos_slot < dist_table_size; + ++pos_slot) { + pos_slot_prices[pos_slot] = 0; + bittree_get_price(pos_slot_prices[pos_slot], encoder, + POS_SLOT_BITS, pos_slot); + } + + for (uint32_t pos_slot = END_POS_MODEL_INDEX; + pos_slot < dist_table_size; + ++pos_slot) + pos_slot_prices[pos_slot] += (((pos_slot >> 1) - 1) + - ALIGN_BITS) << BIT_PRICE_SHIFT_BITS; + + + uint32_t *distances_prices + = coder->distances_prices[len_to_pos_state]; + + uint32_t i; + for (i = 0; i < START_POS_MODEL_INDEX; ++i) + distances_prices[i] = pos_slot_prices[i]; + + for (; i < FULL_DISTANCES; ++i) + distances_prices[i] = pos_slot_prices[get_pos_slot(i)] + + temp_prices[i]; + } + + coder->match_price_count = 0; + + return; +} + + +static void +fill_align_prices(lzma_coder *coder) +{ + for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i) { + uint32_t tmp = 0; + bittree_reverse_get_price(tmp, coder->pos_align_encoder, + ALIGN_BITS, i); + coder->align_prices[i] = tmp; + } + + coder->align_price_count = 0; +} + + +// The first argument is a pointer returned by literal_get_subcoder(). +static uint32_t +literal_get_price(const probability *encoders, const bool match_mode, + const uint8_t match_byte, const uint8_t symbol) +{ + uint32_t price = 0; + uint32_t context = 1; + int i = 8; + + if (match_mode) { + do { + --i; + const uint32_t match_bit = (match_byte >> i) & 1; + const uint32_t bit = (symbol >> i) & 1; + const uint32_t subcoder_index + = 0x100 + (match_bit << 8) + context; + + price += bit_get_price(encoders[subcoder_index], bit); + context = (context << 1) | bit; + + if (match_bit != bit) + break; + + } while (i != 0); + } + + while (i != 0) { + --i; + const uint32_t bit = (symbol >> i) & 1; + price += bit_get_price(encoders[context], bit); + context = (context << 1) | bit; + } + + return price; +} + + +static void +backward(lzma_coder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) +{ + coder->optimum_end_index = cur; + + uint32_t pos_mem = coder->optimum[cur].pos_prev; + uint32_t back_mem = coder->optimum[cur].back_prev; + + do { + if (coder->optimum[cur].prev_1_is_char) { + make_as_char(coder->optimum[pos_mem]); + coder->optimum[pos_mem].pos_prev = pos_mem - 1; + + if (coder->optimum[cur].prev_2) { + coder->optimum[pos_mem - 1].prev_1_is_char = false; + coder->optimum[pos_mem - 1].pos_prev + = coder->optimum[cur].pos_prev_2; + coder->optimum[pos_mem - 1].back_prev + = coder->optimum[cur].back_prev_2; + } + } + + uint32_t pos_prev = pos_mem; + uint32_t back_cur = back_mem; + + back_mem = coder->optimum[pos_prev].back_prev; + pos_mem = coder->optimum[pos_prev].pos_prev; + + coder->optimum[pos_prev].back_prev = back_cur; + coder->optimum[pos_prev].pos_prev = cur; + cur = pos_prev; + + } while (cur != 0); + + coder->optimum_current_index = coder->optimum[0].pos_prev; + *len_res = coder->optimum[0].pos_prev; + *back_res = coder->optimum[0].back_prev; + + return; +} + + +extern void +lzma_get_optimum(lzma_coder *restrict coder, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + // Update the price tables. In the C++ LZMA SDK 4.42 this was done in both + // initialization function and in the main loop. In liblzma they were + // moved into this single place. + if (coder->additional_offset == 0) { + if (coder->match_price_count >= (1 << 7)) + fill_distances_prices(coder); + + if (coder->align_price_count >= ALIGN_TABLE_SIZE) + fill_align_prices(coder); + } + + + if (coder->optimum_end_index != coder->optimum_current_index) { + *len_res = coder->optimum[coder->optimum_current_index].pos_prev + - coder->optimum_current_index; + *back_res = coder->optimum[coder->optimum_current_index].back_prev; + coder->optimum_current_index = coder->optimum[ + coder->optimum_current_index].pos_prev; + return; + } + + coder->optimum_current_index = 0; + coder->optimum_end_index = 0; + + + const uint32_t fast_bytes = coder->fast_bytes; + uint32_t *match_distances = coder->match_distances; + + uint32_t len_main; + uint32_t num_distance_pairs; + + if (!coder->longest_match_was_found) { + lzma_read_match_distances(coder, &len_main, &num_distance_pairs); + } else { + len_main = coder->longest_match_length; + num_distance_pairs = coder->num_distance_pairs; + coder->longest_match_was_found = false; + } + + + const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; + uint32_t num_available_bytes + = coder->lz.write_pos - coder->lz.read_pos + 1; + if (num_available_bytes < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return; + } + + if (num_available_bytes > MATCH_MAX_LEN) + num_available_bytes = MATCH_MAX_LEN; + + + uint32_t reps[REP_DISTANCES]; + uint32_t rep_lens[REP_DISTANCES]; + uint32_t rep_max_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + reps[i] = coder->rep_distances[i]; + const uint32_t back_offset = reps[i] + 1; + + if (buf[0] != *(buf - back_offset) + || buf[1] != *(buf + 1 - back_offset)) { + rep_lens[i] = 0; + continue; + } + + uint32_t len_test; + for (len_test = 2; len_test < num_available_bytes + && buf[len_test] == *(buf + len_test - back_offset); + ++len_test) ; + + rep_lens[i] = len_test; + if (len_test > rep_lens[rep_max_index]) + rep_max_index = i; + } + + if (rep_lens[rep_max_index] >= fast_bytes) { + *back_res = rep_max_index; + *len_res = rep_lens[rep_max_index]; + move_pos(*len_res - 1); + return; + } + + + if (len_main >= fast_bytes) { + *back_res = match_distances[num_distance_pairs] + REP_DISTANCES; + *len_res = len_main; + move_pos(len_main - 1); + return; + } + + uint8_t current_byte = *buf; + uint8_t match_byte = *(buf - reps[0] - 1); + + if (len_main < 2 && current_byte != match_byte + && rep_lens[rep_max_index] < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return; + } + + const uint32_t pos_mask = coder->pos_mask; + + coder->optimum[0].state = coder->state; + + uint32_t position = coder->now_pos; + uint32_t pos_state = (position & pos_mask); + + coder->optimum[1].price = bit_get_price_0( + coder->is_match[coder->state][pos_state]) + + literal_get_price( + literal_get_subcoder(coder->literal_coder, + position, coder->previous_byte), + !is_char_state(coder->state), match_byte, current_byte); + + make_as_char(coder->optimum[1]); + + uint32_t match_price + = bit_get_price_1(coder->is_match[coder->state][pos_state]); + uint32_t rep_match_price + = match_price + bit_get_price_1(coder->is_rep[coder->state]); + + + if (match_byte == current_byte) { + const uint32_t short_rep_price = rep_match_price + + get_rep_len_1_price(coder->state, pos_state); + + if (short_rep_price < coder->optimum[1].price) { + coder->optimum[1].price = short_rep_price; + make_as_short_rep(coder->optimum[1]); + } + } + + uint32_t len_end = (len_main >= rep_lens[rep_max_index]) + ? len_main + : rep_lens[rep_max_index]; + + if (len_end < 2) { + *back_res = coder->optimum[1].back_prev; + *len_res = 1; + return; + } + + coder->optimum[1].pos_prev = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->optimum[0].backs[i] = reps[i]; + + uint32_t len = len_end; + do { + coder->optimum[len].price = INFINITY_PRICE; + } while (--len >= 2); + + + uint32_t (*distances_prices)[FULL_DISTANCES] = coder->distances_prices; + uint32_t (*pos_slot_prices)[DIST_TABLE_SIZE_MAX] = coder->pos_slot_prices; + uint32_t *align_prices = coder->align_prices; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + uint32_t rep_len = rep_lens[i]; + if (rep_len < 2) + continue; + + uint32_t price = rep_match_price; + get_pure_rep_price(price, i, coder->state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + length_get_price( + coder->rep_match_len_encoder, + rep_len - 2, pos_state); + + if (cur_and_len_price < coder->optimum[rep_len].price) { + coder->optimum[rep_len].price = cur_and_len_price; + coder->optimum[rep_len].pos_prev = 0; + coder->optimum[rep_len].back_prev = i; + coder->optimum[rep_len].prev_1_is_char = false; + } + } while (--rep_len >= 2); + } + + + uint32_t normal_match_price = match_price + + bit_get_price_0(coder->is_rep[coder->state]); + + len = (rep_lens[0] >= 2) ? rep_lens[0] + 1 : 2; + + if (len <= len_main) { + uint32_t offs = 0; + + while (len > match_distances[offs + 1]) + offs += 2; + + for(; ; ++len) { + const uint32_t distance = match_distances[offs + 2]; + uint32_t cur_and_len_price = normal_match_price; + get_pos_len_price(cur_and_len_price, distance, len, pos_state); + + if (cur_and_len_price < coder->optimum[len].price) { + coder->optimum[len].price = cur_and_len_price; + coder->optimum[len].pos_prev = 0; + coder->optimum[len].back_prev = distance + REP_DISTANCES; + coder->optimum[len].prev_1_is_char = false; + } + + if (len == match_distances[offs + 1]) { + offs += 2; + if (offs == num_distance_pairs) + break; + } + } + } + + + ////////////////// + // Big loop ;-) // + ////////////////// + + uint32_t cur = 0; + + // The rest of this function is a huge while-loop. To avoid extreme + // indentation, the indentation level is not increased here. + while (true) { + + ++cur; + + assert(cur < OPTS); + + if (cur == len_end) { + backward(coder, len_res, back_res, cur); + return; + } + + uint32_t new_len; + + lzma_read_match_distances(coder, &new_len, &num_distance_pairs); + + if (new_len >= fast_bytes) { + coder->num_distance_pairs = num_distance_pairs; + coder->longest_match_length = new_len; + coder->longest_match_was_found = true; + backward(coder, len_res, back_res, cur); + return; + } + + + ++position; + + uint32_t pos_prev = coder->optimum[cur].pos_prev; + uint32_t state; + + if (coder->optimum[cur].prev_1_is_char) { + --pos_prev; + + if (coder->optimum[cur].prev_2) { + state = coder->optimum[coder->optimum[cur].pos_prev_2].state; + + if (coder->optimum[cur].back_prev_2 < REP_DISTANCES) + update_rep(state); + else + update_match(state); + + } else { + state = coder->optimum[pos_prev].state; + } + + update_char(state); + + } else { + state = coder->optimum[pos_prev].state; + } + + if (pos_prev == cur - 1) { + if (is_short_rep(coder->optimum[cur])) + update_short_rep(state); + else + update_char(state); + } else { + uint32_t pos; + if (coder->optimum[cur].prev_1_is_char && coder->optimum[cur].prev_2) { + pos_prev = coder->optimum[cur].pos_prev_2; + pos = coder->optimum[cur].back_prev_2; + update_rep(state); + } else { + pos = coder->optimum[cur].back_prev; + if (pos < REP_DISTANCES) + update_rep(state); + else + update_match(state); + } + + if (pos < REP_DISTANCES) { + reps[0] = coder->optimum[pos_prev].backs[pos]; + + uint32_t i; + for (i = 1; i <= pos; ++i) + reps[i] = coder->optimum[pos_prev].backs[i - 1]; + + for (; i < REP_DISTANCES; ++i) + reps[i] = coder->optimum[pos_prev].backs[i]; + + } else { + reps[0] = pos - REP_DISTANCES; + + for (uint32_t i = 1; i < REP_DISTANCES; ++i) + reps[i] = coder->optimum[pos_prev].backs[i - 1]; + } + } + + coder->optimum[cur].state = state; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->optimum[cur].backs[i] = reps[i]; + + const uint32_t cur_price = coder->optimum[cur].price; + + buf = coder->lz.buffer + coder->lz.read_pos - 1; + current_byte = *buf; + match_byte = *(buf - reps[0] - 1); + + pos_state = position & pos_mask; + + const uint32_t cur_and_1_price = cur_price + + bit_get_price_0(coder->is_match[state][pos_state]) + + literal_get_price( + literal_get_subcoder(coder->literal_coder, + position, buf[-1]), + !is_char_state(state), match_byte, current_byte); + + bool next_is_char = false; + + if (cur_and_1_price < coder->optimum[cur + 1].price) { + coder->optimum[cur + 1].price = cur_and_1_price; + coder->optimum[cur + 1].pos_prev = cur; + make_as_char(coder->optimum[cur + 1]); + next_is_char = true; + } + + match_price = cur_price + + bit_get_price_1(coder->is_match[state][pos_state]); + rep_match_price = match_price + + bit_get_price_1(coder->is_rep[state]); + + if (match_byte == current_byte + && !(coder->optimum[cur + 1].pos_prev < cur + && coder->optimum[cur + 1].back_prev == 0)) { + + const uint32_t short_rep_price = rep_match_price + + get_rep_len_1_price(state, pos_state); + + if (short_rep_price <= coder->optimum[cur + 1].price) { + coder->optimum[cur + 1].price = short_rep_price; + coder->optimum[cur + 1].pos_prev = cur; + make_as_short_rep(coder->optimum[cur + 1]); + next_is_char = true; + } + } + + uint32_t num_available_bytes_full + = coder->lz.write_pos - coder->lz.read_pos + 1; + num_available_bytes_full = MIN(OPTS - 1 - cur, num_available_bytes_full); + num_available_bytes = num_available_bytes_full; + + if (num_available_bytes < 2) + continue; + + if (num_available_bytes > fast_bytes) + num_available_bytes = fast_bytes; + + if (!next_is_char && match_byte != current_byte) { // speed optimization + // try literal + rep0 + const uint32_t back_offset = reps[0] + 1; + const uint32_t limit = MIN(num_available_bytes_full, fast_bytes + 1); + + uint32_t temp; + for (temp = 1; temp < limit + && buf[temp] == *(buf + temp - back_offset); + ++temp) ; + + const uint32_t len_test_2 = temp - 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_char(state_2); + + const uint32_t pos_state_next = (position + 1) & pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + + bit_get_price_1(coder->is_match[state_2][pos_state_next]) + + bit_get_price_1(coder->is_rep[state_2]); + + // for (; len_test_2 >= 2; --len_test_2) { + const uint32_t offset = cur + 1 + len_test_2; + + while (len_end < offset) + coder->optimum[++len_end].price = INFINITY_PRICE; + + uint32_t cur_and_len_price = next_rep_match_price; + get_rep_price(cur_and_len_price, + 0, len_test_2, state_2, pos_state_next); + + if (cur_and_len_price < coder->optimum[offset].price) { + coder->optimum[offset].price = cur_and_len_price; + coder->optimum[offset].pos_prev = cur + 1; + coder->optimum[offset].back_prev = 0; + coder->optimum[offset].prev_1_is_char = true; + coder->optimum[offset].prev_2 = false; + } +// } + } + } + + + uint32_t start_len = 2; // speed optimization + + for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { + const uint32_t back_offset = reps[rep_index] + 1; + + if (buf[0] != *(buf - back_offset) || buf[1] != *(buf + 1 - back_offset)) + continue; + + uint32_t len_test; + for (len_test = 2; len_test < num_available_bytes + && buf[len_test] == *(buf + len_test - back_offset); + ++len_test) ; + + while (len_end < cur + len_test) + coder->optimum[++len_end].price = INFINITY_PRICE; + + const uint32_t len_test_temp = len_test; + uint32_t price = rep_match_price; + get_pure_rep_price(price, rep_index, state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + length_get_price(coder->rep_match_len_encoder, + len_test - 2, pos_state); + + if (cur_and_len_price < coder->optimum[cur + len_test].price) { + coder->optimum[cur + len_test].price = cur_and_len_price; + coder->optimum[cur + len_test].pos_prev = cur; + coder->optimum[cur + len_test].back_prev = rep_index; + coder->optimum[cur + len_test].prev_1_is_char = false; + } + } while (--len_test >= 2); + + len_test = len_test_temp; + + if (rep_index == 0) + start_len = len_test + 1; + + + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(num_available_bytes_full, + len_test_2 + fast_bytes); + for (; len_test_2 < limit + && buf[len_test_2] == *(buf + len_test_2 - back_offset); + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_rep(state_2); + + uint32_t pos_state_next = (position + len_test) & pos_mask; + + const uint32_t cur_and_len_char_price = price + + length_get_price(coder->rep_match_len_encoder, + len_test - 2, pos_state) + + bit_get_price_0(coder->is_match[state_2][pos_state_next]) + + literal_get_price( + literal_get_subcoder(coder->literal_coder, + position + len_test, buf[len_test - 1]), + true, *(buf + len_test - back_offset), buf[len_test]); + + update_char(state_2); + + pos_state_next = (position + len_test + 1) & pos_mask; + + const uint32_t next_rep_match_price = cur_and_len_char_price + + bit_get_price_1(coder->is_match[state_2][pos_state_next]) + + bit_get_price_1(coder->is_rep[state_2]); + +// for(; len_test_2 >= 2; len_test_2--) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->optimum[++len_end].price = INFINITY_PRICE; + + uint32_t cur_and_len_price = next_rep_match_price; + get_rep_price(cur_and_len_price, + 0, len_test_2, state_2, pos_state_next); + + if (cur_and_len_price < coder->optimum[offset].price) { + coder->optimum[offset].price = cur_and_len_price; + coder->optimum[offset].pos_prev = cur + len_test + 1; + coder->optimum[offset].back_prev = 0; + coder->optimum[offset].prev_1_is_char = true; + coder->optimum[offset].prev_2 = true; + coder->optimum[offset].pos_prev_2 = cur; + coder->optimum[offset].back_prev_2 = rep_index; + } +// } + } + } + + +// for (uint32_t len_test = 2; len_test <= new_len; ++len_test) + if (new_len > num_available_bytes) { + new_len = num_available_bytes; + + for (num_distance_pairs = 0; + new_len > match_distances[num_distance_pairs + 1]; + num_distance_pairs += 2) ; + + match_distances[num_distance_pairs + 1] = new_len; + num_distance_pairs += 2; + } + + + if (new_len >= start_len) { + normal_match_price = match_price + + bit_get_price_0(coder->is_rep[state]); + + while (len_end < cur + new_len) + coder->optimum[++len_end].price = INFINITY_PRICE; + + uint32_t offs = 0; + while (start_len > match_distances[offs + 1]) + offs += 2; + + uint32_t cur_back = match_distances[offs + 2]; + uint32_t pos_slot = get_pos_slot_2(cur_back); + + for (uint32_t len_test = start_len; ; ++len_test) { + uint32_t cur_and_len_price = normal_match_price; + const uint32_t len_to_pos_state = get_len_to_pos_state(len_test); + + if (cur_back < FULL_DISTANCES) + cur_and_len_price += distances_prices[ + len_to_pos_state][cur_back]; + else + cur_and_len_price += pos_slot_prices[ + len_to_pos_state][pos_slot] + + align_prices[cur_back & ALIGN_MASK]; + + cur_and_len_price += length_get_price(coder->len_encoder, + len_test - MATCH_MIN_LEN, pos_state); + + if (cur_and_len_price < coder->optimum[cur + len_test].price) { + coder->optimum[cur + len_test].price = cur_and_len_price; + coder->optimum[cur + len_test].pos_prev = cur; + coder->optimum[cur + len_test].back_prev + = cur_back + REP_DISTANCES; + coder->optimum[cur + len_test].prev_1_is_char = false; + } + + if (len_test == match_distances[offs + 1]) { + // Try Match + Literal + Rep0 + const uint32_t back_offset = cur_back + 1; + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(num_available_bytes_full, + len_test_2 + fast_bytes); + + for (; len_test_2 < limit && + buf[len_test_2] == *(buf + len_test_2 - back_offset); + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_match(state_2); + uint32_t pos_state_next + = (position + len_test) & pos_mask; + + const uint32_t cur_and_len_char_price = cur_and_len_price + + bit_get_price_0( + coder->is_match[state_2][pos_state_next]) + + literal_get_price( + literal_get_subcoder( + coder->literal_coder, + position + len_test, + buf[len_test - 1]), + true, + *(buf + len_test - back_offset), + buf[len_test]); + + update_char(state_2); + pos_state_next = (pos_state_next + 1) & pos_mask; + + const uint32_t next_rep_match_price + = cur_and_len_char_price + + bit_get_price_1( + coder->is_match[state_2][pos_state_next]) + + bit_get_price_1(coder->is_rep[state_2]); + + // for(; len_test_2 >= 2; --len_test_2) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->optimum[++len_end].price = INFINITY_PRICE; + + cur_and_len_price = next_rep_match_price; + get_rep_price(cur_and_len_price, + 0, len_test_2, state_2, pos_state_next); + + if (cur_and_len_price < coder->optimum[offset].price) { + coder->optimum[offset].price = cur_and_len_price; + coder->optimum[offset].pos_prev = cur + len_test + 1; + coder->optimum[offset].back_prev = 0; + coder->optimum[offset].prev_1_is_char = true; + coder->optimum[offset].prev_2 = true; + coder->optimum[offset].pos_prev_2 = cur; + coder->optimum[offset].back_prev_2 + = cur_back + REP_DISTANCES; + } +// } + } + + offs += 2; + if (offs == num_distance_pairs) + break; + + cur_back = match_distances[offs + 2]; + if (cur_back >= FULL_DISTANCES) + pos_slot = get_pos_slot_2(cur_back); + } + } + } + + } // Closes: while (true) +} diff --git a/src/liblzma/lzma/lzma_encoder_getoptimumfast.c b/src/liblzma/lzma/lzma_encoder_getoptimumfast.c new file mode 100644 index 00000000..e6cee19d --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_getoptimumfast.c @@ -0,0 +1,201 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_getoptimumfast.c +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +// NOTE: If you want to keep the line length in 80 characters, set +// tab width to 4 or less in your editor when editing this file. + + +#include "lzma_encoder_private.h" + + +#define change_pair(small_dist, big_dist) \ + (((big_dist) >> 7) > (small_dist)) + + +extern void +lzma_get_optimum_fast(lzma_coder *restrict coder, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + // Local copies + const uint32_t fast_bytes = coder->fast_bytes; + + uint32_t len_main; + uint32_t num_distance_pairs; + if (!coder->longest_match_was_found) { + lzma_read_match_distances(coder, &len_main, &num_distance_pairs); + } else { + len_main = coder->longest_match_length; + num_distance_pairs = coder->num_distance_pairs; + coder->longest_match_was_found = false; + } + + const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; + uint32_t num_available_bytes + = coder->lz.write_pos - coder->lz.read_pos + 1; + + if (num_available_bytes < 2) { + // There's not enough input left to encode a match. + *back_res = UINT32_MAX; + *len_res = 1; + return; + } + + if (num_available_bytes > MATCH_MAX_LEN) + num_available_bytes = MATCH_MAX_LEN; + + + // Look for repetitive matches; scan the previous four match distances + uint32_t rep_lens[REP_DISTANCES]; + uint32_t rep_max_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint32_t back_offset = coder->rep_distances[i] + 1; + + // If the first two bytes (2 == MATCH_MIN_LEN) do not match, + // this rep_distance[i] is not useful. This is indicated + // using zero as the length of the repetitive match. + if (buf[0] != *(buf - back_offset) + || buf[1] != *(buf + 1 - back_offset)) { + rep_lens[i] = 0; + continue; + } + + // The first two bytes matched. + // Calculate the length of the match. + uint32_t len; + for (len = 2; len < num_available_bytes + && buf[len] == *(buf + len - back_offset); + ++len) ; + + // If we have found a repetitive match that is at least + // as long as fast_bytes, return it immediatelly. + if (len >= fast_bytes) { + *back_res = i; + *len_res = len; + move_pos(len - 1); + return; + } + + rep_lens[i] = len; + + // After this loop, rep_lens[rep_max_index] is the biggest + // value of all values in rep_lens[]. + if (len > rep_lens[rep_max_index]) + rep_max_index = i; + } + + + if (len_main >= fast_bytes) { + *back_res = coder->match_distances[num_distance_pairs] + + REP_DISTANCES; + *len_res = len_main; + move_pos(len_main - 1); + return; + } + + uint32_t back_main = 0; + if (len_main >= 2) { + back_main = coder->match_distances[num_distance_pairs]; + + while (num_distance_pairs > 2 && len_main == + coder->match_distances[num_distance_pairs - 3] + 1) { + if (!change_pair(coder->match_distances[ + num_distance_pairs - 2], back_main)) + break; + + num_distance_pairs -= 2; + len_main = coder->match_distances[num_distance_pairs - 1]; + back_main = coder->match_distances[num_distance_pairs]; + } + + if (len_main == 2 && back_main >= 0x80) + len_main = 1; + } + + if (rep_lens[rep_max_index] >= 2) { + if (rep_lens[rep_max_index] + 1 >= len_main + || (rep_lens[rep_max_index] + 2 >= len_main + && (back_main > (1 << 9))) + || (rep_lens[rep_max_index] + 3 >= len_main + && (back_main > (1 << 15)))) { + *back_res = rep_max_index; + *len_res = rep_lens[rep_max_index]; + move_pos(*len_res - 1); + return; + } + } + + if (len_main >= 2 && num_available_bytes > 2) { + lzma_read_match_distances(coder, &coder->longest_match_length, + &coder->num_distance_pairs); + + if (coder->longest_match_length >= 2) { + const uint32_t new_distance = coder->match_distances[ + coder->num_distance_pairs]; + + if ((coder->longest_match_length >= len_main + && new_distance < back_main) + || (coder->longest_match_length == len_main + 1 + && !change_pair(back_main, new_distance)) + || (coder->longest_match_length > len_main + 1) + || (coder->longest_match_length + 1 >= len_main + && len_main >= 3 + && change_pair(new_distance, back_main))) { + coder->longest_match_was_found = true; + *back_res = UINT32_MAX; + *len_res = 1; + return; + } + } + + ++buf; + --num_available_bytes; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint32_t back_offset = coder->rep_distances[i] + 1; + + if (buf[1] != *(buf + 1 - back_offset) + || buf[2] != *(buf + 2 - back_offset)) { + rep_lens[i] = 0; + continue; + } + + uint32_t len; + for (len = 2; len < num_available_bytes + && buf[len] == *(buf + len - back_offset); + ++len) ; + + if (len + 1 >= len_main) { + coder->longest_match_was_found = true; + *back_res = UINT32_MAX; + *len_res = 1; + return; + } + } + + *back_res = back_main + REP_DISTANCES; + *len_res = len_main; + move_pos(len_main - 2); + return; + } + + *back_res = UINT32_MAX; + *len_res = 1; + return; +} diff --git a/src/liblzma/lzma/lzma_encoder_init.c b/src/liblzma/lzma/lzma_encoder_init.c new file mode 100644 index 00000000..d7807529 --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_init.c @@ -0,0 +1,245 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_init.c +/// \brief Creating, resetting and destroying the LZMA encoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder_private.h" + + +uint8_t lzma_fastpos[1 << 11]; + +extern void +lzma_fastpos_init(void) +{ + static const uint8_t fast_slots = 22; + + int c = 2; + lzma_fastpos[0] = 0; + lzma_fastpos[1] = 1; + + for (uint8_t slot_fast = 2; slot_fast < fast_slots; ++slot_fast) { + const uint32_t k = (1 << ((slot_fast >> 1) - 1)); + + for (uint32_t j = 0; j < k; ++j, ++c) + lzma_fastpos[c] = slot_fast; + } + + return; +} + + +/// \brief Initializes the length encoder +static void +length_encoder_reset(lzma_length_encoder *lencoder, + const uint32_t num_pos_states, const uint32_t table_size) +{ + // NLength::CPriceTableEncoder::SetTableSize() + lencoder->table_size = table_size; + + // NLength::CEncoder::Init() + bit_reset(lencoder->choice); + bit_reset(lencoder->choice2); + + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { + bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); + bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); + } + + bittree_reset(lencoder->high, LEN_HIGH_BITS); + + // NLength::CPriceTableEncoder::UpdateTables() + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) + lzma_length_encoder_update_table(lencoder, pos_state); + + return; +} + + +static void +lzma_lzma_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_lz_encoder_end(&coder->lz, allocator); + lzma_literal_end(&coder->literal_coder, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->lz = LZMA_LZ_ENCODER_INIT; + next->coder->literal_coder = NULL; + } + + // Validate options that aren't validated elsewhere. + const lzma_options_lzma *options = filters[0].options; + if (options->pos_bits > LZMA_POS_BITS_MAX + || options->fast_bytes < LZMA_FAST_BYTES_MIN + || options->fast_bytes > LZMA_FAST_BYTES_MAX) { + lzma_lzma_encoder_end(next->coder, allocator); + return LZMA_HEADER_ERROR; + } + + // Set compression mode. + switch (options->mode) { + case LZMA_MODE_FAST: + next->coder->best_compression = false; + break; + + case LZMA_MODE_BEST: + next->coder->best_compression = true; + break; + + default: + lzma_lzma_encoder_end(next->coder, allocator); + return LZMA_HEADER_ERROR; + } + + // Initialize literal coder. + { + const lzma_ret ret = lzma_literal_init( + &next->coder->literal_coder, allocator, + options->literal_context_bits, + options->literal_pos_bits); + if (ret != LZMA_OK) { + lzma_lzma_encoder_end(next->coder, allocator); + return ret; + } + } + + // Initialize LZ encoder. + { + const lzma_ret ret = lzma_lz_encoder_reset( + &next->coder->lz, allocator, &lzma_lzma_encode, + filters[0].uncompressed_size, + options->dictionary_size, OPTS, + options->fast_bytes, MATCH_MAX_LEN + 1 + OPTS, + options->match_finder, + options->match_finder_cycles, + options->preset_dictionary, + options->preset_dictionary_size); + if (ret != LZMA_OK) { + lzma_lzma_encoder_end(next->coder, allocator); + return ret; + } + } + + // Set dist_table_size. + { + // Round the dictionary size up to next 2^n. + uint32_t log_size; + for (log_size = 0; (UINT32_C(1) << log_size) + < options->dictionary_size; ++log_size) ; + + next->coder->dist_table_size = log_size * 2; + } + + // Misc FIXME desc + next->coder->dictionary_size = options->dictionary_size; + next->coder->pos_mask = (1U << options->pos_bits) - 1; + next->coder->fast_bytes = options->fast_bytes; + + // Range coder + rc_reset(next->coder->rc); + + // State + next->coder->state = 0; + next->coder->previous_byte = 0; + for (size_t i = 0; i < REP_DISTANCES; ++i) + next->coder->rep_distances[i] = 0; + + // Bit encoders + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= next->coder->pos_mask; ++j) { + bit_reset(next->coder->is_match[i][j]); + bit_reset(next->coder->is_rep0_long[i][j]); + } + + bit_reset(next->coder->is_rep[i]); + bit_reset(next->coder->is_rep0[i]); + bit_reset(next->coder->is_rep1[i]); + bit_reset(next->coder->is_rep2[i]); + } + + for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + bit_reset(next->coder->pos_encoders[i]); + + // Bit tree encoders + for (size_t i = 0; i < LEN_TO_POS_STATES; ++i) + bittree_reset(next->coder->pos_slot_encoder[i], POS_SLOT_BITS); + + bittree_reset(next->coder->pos_align_encoder, ALIGN_BITS); + + // Length encoders + length_encoder_reset(&next->coder->len_encoder, 1U << options->pos_bits, + options->fast_bytes + 1 - MATCH_MIN_LEN); + + length_encoder_reset(&next->coder->rep_match_len_encoder, + 1U << options->pos_bits, + next->coder->fast_bytes + 1 - MATCH_MIN_LEN); + + // Misc + next->coder->longest_match_was_found = false; + next->coder->optimum_end_index = 0; + next->coder->optimum_current_index = 0; + next->coder->additional_offset = 0; + + next->coder->now_pos = 0; + next->coder->is_initialized = false; + + // Initialize the next decoder in the chain, if any. + { + const lzma_ret ret = lzma_next_filter_init(&next->coder->next, + allocator, filters + 1); + if (ret != LZMA_OK) { + lzma_lzma_encoder_end(next->coder, allocator); + return ret; + } + } + + // Initialization successful. Set the function pointers. + next->code = &lzma_lz_encode; + next->end = &lzma_lzma_encoder_end; + + return LZMA_OK; +} + + +extern bool +lzma_lzma_encode_properties(const lzma_options_lzma *options, uint8_t *byte) +{ + if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX + || options->literal_pos_bits + > LZMA_LITERAL_POS_BITS_MAX + || options->pos_bits > LZMA_POS_BITS_MAX) + return true; + + *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9 + + options->literal_context_bits; + assert(*byte <= (4 * 5 + 4) * 9 + 8); + + return false; +} diff --git a/src/liblzma/lzma/lzma_encoder_presets.c b/src/liblzma/lzma/lzma_encoder_presets.c new file mode 100644 index 00000000..966c7c86 --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_presets.c @@ -0,0 +1,34 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_presets.c +/// \brief Encoder presets +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +LZMA_API const lzma_options_lzma lzma_preset_lzma[9] = { +// dictionary_size lc lp pb mode fb mf mfc +{ UINT32_C(1) << 16, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0 }, +{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0 }, +{ UINT32_C(1) << 19, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 21, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 22, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 23, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 24, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, +{ UINT32_C(1) << 25, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, +}; diff --git a/src/liblzma/lzma/lzma_encoder_private.h b/src/liblzma/lzma/lzma_encoder_private.h new file mode 100644 index 00000000..7fb1566a --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_private.h @@ -0,0 +1,225 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_private.h +/// \brief Private definitions for LZMA encoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA_ENCODER_PRIVATE_H +#define LZMA_LZMA_ENCODER_PRIVATE_H + +#include "lzma_encoder.h" +#include "lzma_common.h" +#include "lz_encoder.h" + +// We need space for about two encoding loops, because there is no check +// for available buffer space before end of payload marker gets written. +// 2*26 bytes should be enough for this... but Lasse isn't very sure about +// the exact value. 64 bytes certainly is enough. :-) +#define RC_BUFFER_SIZE 64 +#include "range_encoder.h" + + +#define move_pos(num) \ +do { \ + assert((int32_t)(num) >= 0); \ + if ((num) != 0) { \ + coder->additional_offset += num; \ + coder->lz.skip(&coder->lz, num); \ + } \ +} while (0) + + +#define get_pos_slot(pos) \ + ((pos) < (1 << 11) \ + ? lzma_fastpos[pos] \ + : ((pos) < (1 << 21) \ + ? lzma_fastpos[(pos) >> 10] + 20 \ + : lzma_fastpos[(pos) >> 20] + 40)) + + +#define get_pos_slot_2(pos) \ + ((pos) < (1 << 17) \ + ? lzma_fastpos[(pos) >> 6] + 12 \ + : ((pos) < (1 << 27) \ + ? lzma_fastpos[(pos) >> 16] + 32 \ + : lzma_fastpos[(pos) >> 26] + 52)) + + +/// This isn't modified once its contents have been +/// initialized by lzma_fastpos_init(). +extern uint8_t lzma_fastpos[1 << 11]; + + +typedef struct { + probability choice; + probability choice2; + probability low[POS_STATES_MAX][LEN_LOW_SYMBOLS]; + probability mid[POS_STATES_MAX][LEN_MID_SYMBOLS]; + probability high[LEN_HIGH_SYMBOLS]; + + uint32_t prices[POS_STATES_MAX][LEN_SYMBOLS]; + uint32_t table_size; + uint32_t counters[POS_STATES_MAX]; + +} lzma_length_encoder; + + +typedef struct { + uint32_t state; + + bool prev_1_is_char; + bool prev_2; + + uint32_t pos_prev_2; + uint32_t back_prev_2; + + uint32_t price; + uint32_t pos_prev; // pos_next; + uint32_t back_prev; + + uint32_t backs[4]; + +} lzma_optimal; + + +struct lzma_coder_s { + // Next coder in the chain + lzma_next_coder next; + + // In window and match finder + lzma_lz_encoder lz; + + // Range encoder + lzma_range_encoder rc; + + // State + uint32_t state; + uint8_t previous_byte; + uint32_t rep_distances[REP_DISTANCES]; + + // Misc + uint32_t match_distances[MATCH_MAX_LEN * 2 + 2 + 1]; + uint32_t num_distance_pairs; + uint32_t additional_offset; + uint32_t now_pos; // Lowest 32 bits are enough here. + bool best_compression; ///< True when LZMA_MODE_BEST is used + bool is_initialized; + + // Literal encoder + lzma_literal_coder *literal_coder; + + // Bit encoders + probability is_match[STATES][POS_STATES_MAX]; + probability is_rep[STATES]; + probability is_rep0[STATES]; + probability is_rep1[STATES]; + probability is_rep2[STATES]; + probability is_rep0_long[STATES][POS_STATES_MAX]; + probability pos_encoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; + + // Bit Tree Encoders + probability pos_slot_encoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; + probability pos_align_encoder[1 << ALIGN_BITS]; + + // Length encoders + lzma_length_encoder len_encoder; + lzma_length_encoder rep_match_len_encoder; + + // Optimal + lzma_optimal optimum[OPTS]; + uint32_t optimum_end_index; + uint32_t optimum_current_index; + uint32_t longest_match_length; + bool longest_match_was_found; + + // Prices + uint32_t pos_slot_prices[LEN_TO_POS_STATES][DIST_TABLE_SIZE_MAX]; + uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; + uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_price_count; + uint32_t dist_table_size; + uint32_t match_price_count; + + // LZMA specific settings + uint32_t dictionary_size; ///< Size in bytes + uint32_t fast_bytes; + uint32_t pos_state_bits; + uint32_t pos_mask; ///< (1 << pos_state_bits) - 1 +}; + + +extern void lzma_length_encoder_update_table(lzma_length_encoder *lencoder, + const uint32_t pos_state); + +extern bool lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); + +extern void lzma_get_optimum(lzma_coder *restrict coder, + uint32_t *restrict back_res, uint32_t *restrict len_res); + +extern void lzma_get_optimum_fast(lzma_coder *restrict coder, + uint32_t *restrict back_res, uint32_t *restrict len_res); + + +// NOTE: Don't add 'restrict'. +static inline void +lzma_read_match_distances(lzma_coder *coder, + uint32_t *len_res, uint32_t *num_distance_pairs) +{ + *len_res = 0; + + coder->lz.get_matches(&coder->lz, coder->match_distances); + + *num_distance_pairs = coder->match_distances[0]; + + if (*num_distance_pairs > 0) { + *len_res = coder->match_distances[*num_distance_pairs - 1]; + assert(*len_res <= MATCH_MAX_LEN); + + if (*len_res == coder->fast_bytes) { + uint32_t offset = *len_res - 1; + const uint32_t distance = coder->match_distances[ + *num_distance_pairs] + 1; + uint32_t limit = MATCH_MAX_LEN - *len_res; + + assert(offset + limit < coder->lz.keep_size_after); + + // If we are close to end of the stream, we may need + // to limit the length of the match. + if (coder->lz.stream_end_was_reached + && coder->lz.write_pos + < coder->lz.read_pos + offset + limit) + limit = coder->lz.write_pos + - (coder->lz.read_pos + offset); + + offset += coder->lz.read_pos; + uint32_t i = 0; + while (i < limit && coder->lz.buffer[offset + i] + == coder->lz.buffer[ + offset + i - distance]) + ++i; + + *len_res += i; + } + } + + ++coder->additional_offset; + + return; +} + +#endif diff --git a/src/liblzma/lzma/lzma_literal.c b/src/liblzma/lzma/lzma_literal.c new file mode 100644 index 00000000..8f650fbf --- /dev/null +++ b/src/liblzma/lzma/lzma_literal.c @@ -0,0 +1,74 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_literal.c +/// \brief Literal Coder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_literal.h" + + +extern lzma_ret +lzma_literal_init(lzma_literal_coder **coder, lzma_allocator *allocator, + uint32_t literal_context_bits, uint32_t literal_pos_bits) +{ + // Verify that arguments are sane. + if (literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX + || literal_pos_bits > LZMA_LITERAL_POS_BITS_MAX) + return LZMA_HEADER_ERROR; + + // Calculate the number of states the literal coder must store. + const uint32_t states = literal_states( + literal_pos_bits, literal_context_bits); + + // Allocate a new literal coder, if needed. + if (*coder == NULL || (**coder).literal_context_bits + != literal_context_bits + || (**coder).literal_pos_bits != literal_pos_bits) { + // Free the old coder, if any. + lzma_free(*coder, allocator); + + // Allocate a new one. + *coder = lzma_alloc(sizeof(lzma_literal_coder) + + states * LIT_SIZE * sizeof(probability), + allocator); + if (*coder == NULL) + return LZMA_MEM_ERROR; + + // Store the new settings. + (**coder).literal_context_bits = literal_context_bits; + (**coder).literal_pos_bits = literal_pos_bits; + + // Calculate also the literal_pos_mask. It's not changed + // anywhere else than here. + (**coder).literal_pos_mask = (1 << literal_pos_bits) - 1; + } + + // Reset the literal coder. + for (uint32_t i = 0; i < states; ++i) + for (uint32_t j = 0; j < LIT_SIZE; ++j) + bit_reset((**coder).coders[i][j]); + + return LZMA_OK; +} + + +extern void +lzma_literal_end(lzma_literal_coder **coder, lzma_allocator *allocator) +{ + lzma_free(*coder, allocator); + *coder = NULL; +} diff --git a/src/liblzma/lzma/lzma_literal.h b/src/liblzma/lzma/lzma_literal.h new file mode 100644 index 00000000..174f5ed4 --- /dev/null +++ b/src/liblzma/lzma/lzma_literal.h @@ -0,0 +1,74 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_literal.h +/// \brief Literal Coder +/// +/// This is used as is by both LZMA encoder and decoder. +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LITERAL_H +#define LZMA_LITERAL_H + +#include "common.h" + +// We need typedef of `probability'. +#include "range_common.h" + + +/// Each literal coder is divided in three sections: +/// - 0x001-0x0FF: Without match byte +/// - 0x101-0x1FF: With match byte; match bit is 0 +/// - 0x201-0x2FF: With match byte; match bit is 1 +#define LIT_SIZE 0x300 + +/// Calculate how many states are needed. Each state has +/// LIT_SIZE `probability' variables. +#define literal_states(literal_context_bits, literal_pos_bits) \ + (1U << ((literal_context_bits) + (literal_pos_bits))) + +/// Locate the literal coder for the next literal byte. The choice depends on +/// - the lowest literal_pos_bits bits of the position of the current +/// byte; and +/// - the highest literal_context_bits bits of the previous byte. +#define literal_get_subcoder(literal_coder, pos, prev_byte) \ + (literal_coder)->coders[(((pos) & (literal_coder)->literal_pos_mask) \ + << (literal_coder)->literal_context_bits) \ + + ((prev_byte) >> (8 - (literal_coder)->literal_context_bits))] + + +typedef struct { + uint32_t literal_context_bits; + uint32_t literal_pos_bits; + + /// literal_pos_mask is always (1 << literal_pos_bits) - 1. + uint32_t literal_pos_mask; + + /// There are (1 << (literal_pos_bits + literal_context_bits)) + /// literal coders. + probability coders[][LIT_SIZE]; + +} lzma_literal_coder; + + +extern lzma_ret lzma_literal_init( + lzma_literal_coder **coder, lzma_allocator *allocator, + uint32_t literal_context_bits, uint32_t literal_pos_bits); + +extern void lzma_literal_end( + lzma_literal_coder **coder, lzma_allocator *allocator); + +#endif diff --git a/src/liblzma/rangecoder/Makefile.am b/src/liblzma/rangecoder/Makefile.am new file mode 100644 index 00000000..ef5d1464 --- /dev/null +++ b/src/liblzma/rangecoder/Makefile.am @@ -0,0 +1,28 @@ +## +## Copyright (C) 2006 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = librangecoder.la + +librangecoder_la_SOURCES = range_common.h +librangecoder_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common + +if COND_MAIN_ENCODER +librangecoder_la_SOURCES += range_encoder.c range_encoder.h +endif + +if COND_MAIN_DECODER +librangecoder_la_SOURCES += range_decoder.h +endif diff --git a/src/liblzma/rangecoder/range_common.h b/src/liblzma/rangecoder/range_common.h new file mode 100644 index 00000000..9e8f89a2 --- /dev/null +++ b/src/liblzma/rangecoder/range_common.h @@ -0,0 +1,68 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file range_common.h +/// \brief Common things for range encoder and decoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2006 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RANGE_COMMON_H +#define LZMA_RANGE_COMMON_H + +#include "common.h" + + +/////////////// +// Constants // +/////////////// + +#define SHIFT_BITS 8 +#define TOP_BITS 24 +#define TOP_VALUE (UINT32_C(1) << TOP_BITS) +#define BIT_MODEL_TOTAL_BITS 11 +#define BIT_MODEL_TOTAL (UINT32_C(1) << BIT_MODEL_TOTAL_BITS) +#define MOVE_BITS 5 + +#define MOVE_REDUCING_BITS 2 +#define BIT_PRICE_SHIFT_BITS 6 + + +//////////// +// Macros // +//////////// + +// Resets the probability so that both 0 and 1 have probability of 50 % +#define bit_reset(prob) \ + prob = BIT_MODEL_TOTAL >> 1 + +// This does the same for a complete bit tree. +// (A tree represented as an array.) +#define bittree_reset(probs, bit_levels) \ + for (uint32_t bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ + bit_reset((probs)[bt_i]) + + +////////////////////// +// Type definitions // +////////////////////// + +// Bit coder speed optimization +// uint16_t is enough for probability, but usually uint32_t is faster and it +// doesn't waste too much memory. If uint64_t is fastest on 64-bit CPU, you +// probably want to use that instead of uint32_t. With uint64_t you will +// waste RAM _at maximum_ of 4.5 MiB (same for both encoding and decoding). +typedef uint32_t probability; + +#endif diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h new file mode 100644 index 00000000..0583faaf --- /dev/null +++ b/src/liblzma/rangecoder/range_decoder.h @@ -0,0 +1,189 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file range_decoder.h +/// \brief Range Decoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RANGE_DECODER_H +#define LZMA_RANGE_DECODER_H + +#include "range_common.h" + + +typedef struct { + uint32_t range; + uint32_t code; +} lzma_range_decoder; + + +/// Makes local copies of range decoder variables. +#define rc_to_local(rc) \ + uint32_t rc_range = (rc).range; \ + uint32_t rc_code = (rc).code; \ + uint32_t rc_bound + +/// Stores the local copes back to the range decoder structure. +#define rc_from_local(rc) \ +do {\ + (rc).range = rc_range; \ + (rc).code = rc_code; \ +} while (0) + +/// Resets the range decoder structure. +#define rc_reset(rc) \ +do { \ + (rc).range = UINT32_MAX; \ + (rc).code = 0; \ +} while (0) + + +// All of the macros in this file expect the following variables being defined: +// - uint32_t rc_range; +// - uint32_t rc_code; +// - uint32_t rc_bound; // Temporary variable +// - uint8_t *in; +// - size_t in_pos_local; // Local alias for *in_pos + + +////////////////// +// Buffer "I/O" // +////////////////// + +// Read the next byte of compressed data from buffer_in, if needed. +#define rc_normalize() \ +do { \ + if (rc_range < TOP_VALUE) { \ + rc_range <<= SHIFT_BITS; \ + rc_code = (rc_code << SHIFT_BITS) | in[in_pos_local++]; \ + } \ +} while (0) + + +////////////////// +// Bit decoding // +////////////////// + +// Range decoder's DecodeBit() is splitted into three macros: +// if_bit_0(prob) { +// update_bit_0(prob) +// ... +// } else { +// update_bit_1(prob) +// ... +// } + +#define if_bit_0(prob) \ + rc_normalize(); \ + rc_bound = (rc_range >> BIT_MODEL_TOTAL_BITS) * (prob); \ + if (rc_code < rc_bound) + + +#define update_bit_0(prob) \ +do { \ + rc_range = rc_bound; \ + prob += (BIT_MODEL_TOTAL - (prob)) >> MOVE_BITS; \ +} while (0) + + +#define update_bit_1(prob) \ +do { \ + rc_range -= rc_bound; \ + rc_code -= rc_bound; \ + prob -= (prob) >> MOVE_BITS; \ +} while (0) + + +// Dummy versions don't update prob. +#define update_bit_0_dummy() \ + rc_range = rc_bound + + +#define update_bit_1_dummy() \ +do { \ + rc_range -= rc_bound; \ + rc_code -= rc_bound; \ +} while (0) + + +/////////////////////// +// Bit tree decoding // +/////////////////////// + +#define bittree_decode(target, probs, bit_levels) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ + if_bit_0((probs)[model_index]) { \ + update_bit_0((probs)[model_index]); \ + model_index <<= 1; \ + } else { \ + update_bit_1((probs)[model_index]); \ + model_index = (model_index << 1) | 1; \ + } \ + } \ + target += model_index - (1 << bit_levels); \ +} while (0) + + +#define bittree_reverse_decode(target, probs, bit_levels) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ + if_bit_0((probs)[model_index]) { \ + update_bit_0((probs)[model_index]); \ + model_index <<= 1; \ + } else { \ + update_bit_1((probs)[model_index]); \ + model_index = (model_index << 1) | 1; \ + target += 1 << bit_index; \ + } \ + } \ +} while (0) + + +// Dummy versions don't update prob. +#define bittree_decode_dummy(target, probs, bit_levels) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ + if_bit_0((probs)[model_index]) { \ + update_bit_0_dummy(); \ + model_index <<= 1; \ + } else { \ + update_bit_1_dummy(); \ + model_index = (model_index << 1) | 1; \ + } \ + } \ + target += model_index - (1 << bit_levels); \ +} while (0) + + +#define bittree_reverse_decode_dummy(probs, bit_levels) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ + if_bit_0((probs)[model_index]) { \ + update_bit_0_dummy(); \ + model_index <<= 1; \ + } else { \ + update_bit_1_dummy(); \ + model_index = (model_index << 1) | 1; \ + } \ + } \ +} while (0) + +#endif diff --git a/src/liblzma/rangecoder/range_encoder.c b/src/liblzma/rangecoder/range_encoder.c new file mode 100644 index 00000000..f03bd873 --- /dev/null +++ b/src/liblzma/rangecoder/range_encoder.c @@ -0,0 +1,46 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file range_encoder.c +/// \brief Static initializations for the range encoder's prices array +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "range_encoder.h" + + +#define NUM_BITS (BIT_MODEL_TOTAL_BITS - MOVE_REDUCING_BITS) + + +uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; + + +extern void +lzma_rc_init(void) +{ + // Initialize lzma_rc_prob_prices[]. + for (int i = NUM_BITS - 1; i >= 0; --i) { + const uint32_t start = 1 << (NUM_BITS - i - 1); + const uint32_t end = 1 << (NUM_BITS - i); + + for (uint32_t j = start; j < end; ++j) { + lzma_rc_prob_prices[j] = (i << BIT_PRICE_SHIFT_BITS) + + (((end - j) << BIT_PRICE_SHIFT_BITS) + >> (NUM_BITS - i - 1)); + } + } + + return; +} diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h new file mode 100644 index 00000000..d513cfd1 --- /dev/null +++ b/src/liblzma/rangecoder/range_encoder.h @@ -0,0 +1,317 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file range_encoder.h +/// \brief Range Encoder +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_RANGE_ENCODER_H +#define LZMA_RANGE_ENCODER_H + +#include "range_common.h" + + +// Allow #including this file even if RC_TEMP_BUFFER_SIZE isn't defined. +#ifdef RC_BUFFER_SIZE +typedef struct { + uint64_t low; + uint32_t range; + uint32_t cache_size; + uint8_t cache; + uint8_t buffer[RC_BUFFER_SIZE]; + size_t buffer_size; +} lzma_range_encoder; +#endif + + +/// Makes local copies of range encoder variables. +#define rc_to_local(rc) \ + uint64_t rc_low = (rc).low; \ + uint32_t rc_range = (rc).range; \ + uint32_t rc_cache_size = (rc).cache_size; \ + uint8_t rc_cache = (rc).cache; \ + uint8_t *rc_buffer = (rc).buffer; \ + size_t rc_buffer_size = (rc).buffer_size + +/// Stores the local copes back to the range encoder structure. +#define rc_from_local(rc) \ +do { \ + (rc).low = rc_low; \ + (rc).range = rc_range; \ + (rc).cache_size = rc_cache_size; \ + (rc).cache = rc_cache; \ + (rc).buffer_size = rc_buffer_size; \ +} while (0) + +/// Resets the range encoder structure. +#define rc_reset(rc) \ +do { \ + (rc).low = 0; \ + (rc).range = 0xFFFFFFFF; \ + (rc).cache_size = 1; \ + (rc).cache = 0; \ + (rc).buffer_size = 0; \ +} while (0) + + +////////////////// +// Bit encoding // +////////////////// + +// These macros expect that the following variables are defined: +// - uint64_t rc_low; +// - uint32_t rc_range; +// - uint8_t rc_cache; +// - uint32_t rc_cache_size; +// - uint8_t *out; +// - size_t out_pos_local; // Local copy of *out_pos +// - size_t size_out; + + +// Combined from NRangeCoder::CEncoder::Encode() +// and NRangeCoder::CEncoder::UpdateModel(). +#define bit_encode(prob, symbol) \ +do { \ + probability rc_prob = prob; \ + const uint32_t rc_bound \ + = (rc_range >> BIT_MODEL_TOTAL_BITS) * rc_prob; \ + if ((symbol) == 0) { \ + rc_range = rc_bound; \ + rc_prob += (BIT_MODEL_TOTAL - rc_prob) >> MOVE_BITS; \ + } else { \ + rc_low += rc_bound; \ + rc_range -= rc_bound; \ + rc_prob -= rc_prob >> MOVE_BITS; \ + } \ + prob = rc_prob; \ + rc_normalize(); \ +} while (0) + + +// Optimized version of bit_encode(prob, 0) +#define bit_encode_0(prob) \ +do { \ + probability rc_prob = prob; \ + rc_range = (rc_range >> BIT_MODEL_TOTAL_BITS) * rc_prob; \ + rc_prob += (BIT_MODEL_TOTAL - rc_prob) >> MOVE_BITS; \ + prob = rc_prob; \ + rc_normalize(); \ +} while (0) + + +// Optimized version of bit_encode(prob, 1) +#define bit_encode_1(prob) \ +do { \ + probability rc_prob = prob; \ + const uint32_t rc_bound = (rc_range >> BIT_MODEL_TOTAL_BITS) \ + * rc_prob; \ + rc_low += rc_bound; \ + rc_range -= rc_bound; \ + rc_prob -= rc_prob >> MOVE_BITS; \ + prob = rc_prob; \ + rc_normalize(); \ +} while (0) + + +/////////////////////// +// Bit tree encoding // +/////////////////////// + +#define bittree_encode(probs, bit_levels, symbol) \ +do { \ + uint32_t model_index = 1; \ + for (int32_t bit_index = bit_levels - 1; \ + bit_index >= 0; --bit_index) { \ + const uint32_t bit = ((symbol) >> bit_index) & 1; \ + bit_encode((probs)[model_index], bit); \ + model_index = (model_index << 1) | bit; \ + } \ +} while (0) + + +#define bittree_reverse_encode(probs, bit_levels, symbol) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ + const uint32_t bit = ((symbol) >> bit_index) & 1; \ + bit_encode((probs)[model_index], bit); \ + model_index = (model_index << 1) | bit; \ + } \ +} while (0) + + +///////////////// +// Direct bits // +///////////////// + +#define rc_encode_direct_bits(value, num_total_bits) \ +do { \ + for (int32_t rc_i = (num_total_bits) - 1; rc_i >= 0; --rc_i) { \ + rc_range >>= 1; \ + if ((((value) >> rc_i) & 1) == 1) \ + rc_low += rc_range; \ + rc_normalize(); \ + } \ +} while (0) + + +////////////////// +// Buffer "I/O" // +////////////////// + +// Calls rc_shift_low() to write out a byte if needed. +#define rc_normalize() \ +do { \ + if (rc_range < TOP_VALUE) { \ + rc_range <<= SHIFT_BITS; \ + rc_shift_low(); \ + } \ +} while (0) + + +// Flushes all the pending output. +#define rc_flush() \ + for (int32_t rc_i = 0; rc_i < 5; ++rc_i) \ + rc_shift_low() + + +// Writes the compressed data to next_out. +// TODO: Notation change? +// (uint32_t)(0xFF000000) => ((uint32_t)(0xFF) << TOP_BITS) +// TODO: Another notation change? +// rc_low = (uint32_t)(rc_low) << SHIFT_BITS; +// => +// rc_low &= TOP_VALUE - 1; +// rc_low <<= SHIFT_BITS; +#define rc_shift_low() \ +do { \ + if ((uint32_t)(rc_low) < (uint32_t)(0xFF000000) \ + || (uint32_t)(rc_low >> 32) != 0) { \ + uint8_t rc_temp = rc_cache; \ + do { \ + rc_write_byte(rc_temp + (uint8_t)(rc_low >> 32)); \ + rc_temp = 0xFF; \ + } while(--rc_cache_size != 0); \ + rc_cache = (uint8_t)((uint32_t)(rc_low) >> 24); \ + } \ + ++rc_cache_size; \ + rc_low = (uint32_t)(rc_low) << SHIFT_BITS; \ +} while (0) + + +// Write one byte of compressed data to *next_out. Updates out_pos_local. +// If out_pos_local == out_size, the byte is appended to rc_buffer. +#define rc_write_byte(b) \ +do { \ + if (out_pos_local == out_size) { \ + rc_buffer[rc_buffer_size++] = (uint8_t)(b); \ + assert(rc_buffer_size < RC_BUFFER_SIZE); \ + } else { \ + assert(rc_buffer_size == 0); \ + out[out_pos_local++] = (uint8_t)(b); \ + } \ +} while (0) + + +////////////////// +// Price macros // +////////////////// + +// These macros expect that the following variables are defined: +// - uint32_t lzma_rc_prob_prices; + +#define bit_get_price(prob, symbol) \ + lzma_rc_prob_prices[((((prob) - (symbol)) ^ (-(symbol))) \ + & (BIT_MODEL_TOTAL - 1)) >> MOVE_REDUCING_BITS] + + +#define bit_get_price_0(prob) \ + lzma_rc_prob_prices[(prob) >> MOVE_REDUCING_BITS] + + +#define bit_get_price_1(prob) \ + lzma_rc_prob_prices[(BIT_MODEL_TOTAL - (prob)) >> MOVE_REDUCING_BITS] + + +// Adds price to price_target. TODO Optimize/Cleanup? +#define bittree_get_price(price_target, probs, bit_levels, symbol) \ +do { \ + uint32_t bittree_symbol = (symbol) | (UINT32_C(1) << bit_levels); \ + while (bittree_symbol != 1) { \ + price_target += bit_get_price((probs)[bittree_symbol >> 1], \ + bittree_symbol & 1); \ + bittree_symbol >>= 1; \ + } \ +} while (0) + + +// Adds price to price_target. +#define bittree_reverse_get_price(price_target, probs, bit_levels, symbol) \ +do { \ + uint32_t model_index = 1; \ + for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ + const uint32_t bit = ((symbol) >> bit_index) & 1; \ + price_target += bit_get_price((probs)[model_index], bit); \ + model_index = (model_index << 1) | bit; \ + } \ +} while (0) + + +////////////////////// +// Global variables // +////////////////////// + +// Probability prices used by *_get_price() macros. This is initialized +// by lzma_rc_init() and is not modified later. +extern uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; + + +/////////////// +// Functions // +/////////////// + +/// Initializes lzma_rc_prob_prices[]. This needs to be called only once. +extern void lzma_rc_init(void); + + +#ifdef RC_BUFFER_SIZE +/// Flushes data from rc->temp[] to out[] as much as possible. If everything +/// cannot be flushed, returns true; false otherwise. +static inline bool +rc_flush_buffer(lzma_range_encoder *rc, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + if (rc->buffer_size > 0) { + const size_t out_avail = out_size - *out_pos; + if (rc->buffer_size > out_avail) { + memcpy(out + *out_pos, rc->buffer, out_avail); + *out_pos += out_avail; + rc->buffer_size -= out_avail; + memmove(rc->buffer, rc->buffer + out_avail, + rc->buffer_size); + return true; + } + + memcpy(out + *out_pos, rc->buffer, rc->buffer_size); + *out_pos += rc->buffer_size; + rc->buffer_size = 0; + } + + return false; +} +#endif + +#endif diff --git a/src/liblzma/simple/Makefile.am b/src/liblzma/simple/Makefile.am new file mode 100644 index 00000000..a37f1eb5 --- /dev/null +++ b/src/liblzma/simple/Makefile.am @@ -0,0 +1,46 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = libsimple.la +libsimple_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common +libsimple_la_SOURCES = \ + simple_coder.c \ + simple_coder.h \ + simple_private.h + +if COND_FILTER_X86 +libsimple_la_SOURCES += x86.c +endif + +if COND_FILTER_POWERPC +libsimple_la_SOURCES += powerpc.c +endif + +if COND_FILTER_IA64 +libsimple_la_SOURCES += ia64.c +endif + +if COND_FILTER_ARM +libsimple_la_SOURCES += arm.c +endif + +if COND_FILTER_ARMTHUMB +libsimple_la_SOURCES += armthumb.c +endif + +if COND_FILTER_SPARC +libsimple_la_SOURCES += sparc.c +endif diff --git a/src/liblzma/simple/arm.c b/src/liblzma/simple/arm.c new file mode 100644 index 00000000..284371c3 --- /dev/null +++ b/src/liblzma/simple/arm.c @@ -0,0 +1,76 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file arm.c +/// \brief Filter for ARM binaries +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +arm_code(lzma_simple *simple lzma_attribute((unused)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + uint32_t i; + for (i = 0; i + 4 <= size; i += 4) { + if (buffer[i + 3] == 0xEB) { + uint32_t src = (buffer[i + 2] << 16) + | (buffer[i + 1] << 8) + | (buffer[i + 0]); + src <<= 2; + + uint32_t dest; + if (is_encoder) + dest = now_pos + (uint32_t)(i) + 8 + src; + else + dest = src - (now_pos + (uint32_t)(i) + 8); + + dest >>= 2; + buffer[i + 2] = (dest >> 16); + buffer[i + 1] = (dest >> 8); + buffer[i + 0] = dest; + } + } + + return i; +} + + +static lzma_ret +arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &arm_code, 0, 4, is_encoder); +} + + +extern lzma_ret +lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return arm_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return arm_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/simple/armthumb.c b/src/liblzma/simple/armthumb.c new file mode 100644 index 00000000..51ed2c43 --- /dev/null +++ b/src/liblzma/simple/armthumb.c @@ -0,0 +1,81 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file armthumb.c +/// \brief Filter for ARM-Thumb binaries +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +armthumb_code(lzma_simple *simple lzma_attribute((unused)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + uint32_t i; + for (i = 0; i + 4 <= size; i += 2) { + if ((buffer[i + 1] & 0xF8) == 0xF0 + && (buffer[i + 3] & 0xF8) == 0xF8) { + uint32_t src = ((buffer[i + 1] & 0x7) << 19) + | (buffer[i + 0] << 11) + | ((buffer[i + 3] & 0x7) << 8) + | (buffer[i + 2]); + + src <<= 1; + + uint32_t dest; + if (is_encoder) + dest = now_pos + (uint32_t)(i) + 4 + src; + else + dest = src - (now_pos + (uint32_t)(i) + 4); + + dest >>= 1; + buffer[i + 1] = 0xF0 | ((dest >> 19) & 0x7); + buffer[i + 0] = (dest >> 11); + buffer[i + 3] = 0xF8 | ((dest >> 8) & 0x7); + buffer[i + 2] = (dest); + i += 2; + } + } + + return i; +} + + +static lzma_ret +armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &armthumb_code, 0, 4, is_encoder); +} + + +extern lzma_ret +lzma_simple_armthumb_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return armthumb_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_armthumb_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return armthumb_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/simple/ia64.c b/src/liblzma/simple/ia64.c new file mode 100644 index 00000000..59830521 --- /dev/null +++ b/src/liblzma/simple/ia64.c @@ -0,0 +1,117 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file ia64.c +/// \brief Filter for IA64 (Itanium) binaries +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +ia64_code(lzma_simple *simple lzma_attribute((unused)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + static const uint32_t BRANCH_TABLE[32] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 6, 6, 0, 0, 7, 7, + 4, 4, 0, 0, 4, 4, 0, 0 + }; + + size_t i; + for (i = 0; i + 16 <= size; i += 16) { + const uint32_t instr_template = buffer[i] & 0x1F; + const uint32_t mask = BRANCH_TABLE[instr_template]; + uint32_t bit_pos = 5; + + for (size_t slot = 0; slot < 3; ++slot, bit_pos += 41) { + if (((mask >> slot) & 1) == 0) + continue; + + const size_t byte_pos = (bit_pos >> 3); + const uint32_t bit_res = bit_pos & 0x7; + uint64_t instruction = 0; + + for (size_t j = 0; j < 6; ++j) + instruction += (uint64_t)( + buffer[i + j + byte_pos]) + << (8 * j); + + uint64_t inst_norm = instruction >> bit_res; + + if (((inst_norm >> 37) & 0xF) == 0x5 + && ((inst_norm >> 9) & 0x7) == 0 + /* && (inst_norm & 0x3F)== 0 */ + ) { + uint32_t src = (uint32_t)( + (inst_norm >> 13) & 0xFFFFF); + src |= ((inst_norm >> 36) & 1) << 20; + + src <<= 4; + + uint32_t dest; + if (is_encoder) + dest = now_pos + (uint32_t)(i) + src; + else + dest = src - (now_pos + (uint32_t)(i)); + + dest >>= 4; + + inst_norm &= ~((uint64_t)(0x8FFFFF) << 13); + inst_norm |= (uint64_t)(dest & 0xFFFFF) << 13; + inst_norm |= (uint64_t)(dest & 0x100000) + << (36 - 20); + + instruction &= (1 << bit_res) - 1; + instruction |= (inst_norm << bit_res); + + for (size_t j = 0; j < 6; j++) + buffer[i + j + byte_pos] = (uint8_t)( + instruction + >> (8 * j)); + } + } + } + + return i; +} + + +static lzma_ret +ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &ia64_code, 0, 4, is_encoder); +} + + +extern lzma_ret +lzma_simple_ia64_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return ia64_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_ia64_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return ia64_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/simple/powerpc.c b/src/liblzma/simple/powerpc.c new file mode 100644 index 00000000..a3089fca --- /dev/null +++ b/src/liblzma/simple/powerpc.c @@ -0,0 +1,80 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file powerpc.c +/// \brief Filter for PowerPC (big endian) binaries +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +powerpc_code(lzma_simple *simple lzma_attribute((unused)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + size_t i; + for (i = 0; i + 4 <= size; i += 4) { + // PowerPC branch 6(48) 24(Offset) 1(Abs) 1(Link) + if ((buffer[i] >> 2) == 0x12 + && ((buffer[i + 3] & 3) == 1)) { + + const uint32_t src = ((buffer[i + 0] & 3) << 24) + | (buffer[i + 1] << 16) + | (buffer[i + 2] << 8) + | (buffer[i + 3] & (~3)); + + uint32_t dest; + if (is_encoder) + dest = now_pos + (uint32_t)(i) + src; + else + dest = src - (now_pos + (uint32_t)(i)); + + buffer[i + 0] = 0x48 | ((dest >> 24) & 0x03); + buffer[i + 1] = (dest >> 16); + buffer[i + 2] = (dest >> 8); + buffer[i + 3] &= 0x03; + buffer[i + 3] |= dest; + } + } + + return i; +} + + +static lzma_ret +powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &powerpc_code, 0, 4, is_encoder); +} + + +extern lzma_ret +lzma_simple_powerpc_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return powerpc_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_powerpc_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return powerpc_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c new file mode 100644 index 00000000..f60dff34 --- /dev/null +++ b/src/liblzma/simple/simple_coder.c @@ -0,0 +1,306 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_coder.c +/// \brief Wrapper for simple filters +/// +/// Simple filters don't change the size of the data i.e. number of bytes +/// in equals the number of bytes out. +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +/// Copied or encodes/decodes more data to out[]. Checks and updates +/// uncompressed_size when we are the last coder in the chain. +/// If we aren't the last filter in the chain, we don't need to care about +/// uncompressed size, since we don't change it; the next filter in the +/// chain will check it anyway. +static lzma_ret +copy_or_code(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + assert(!coder->end_was_reached); + + if (coder->next.code == NULL) { + const size_t in_avail = in_size - *in_pos; + + if (coder->is_encoder) { + if (action == LZMA_FINISH) { + // If uncompressed size is known and the + // amount of available input doesn't match + // the uncompressed size, return an error. + if (coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN + && coder->uncompressed_size + != in_avail) + return LZMA_DATA_ERROR; + + } else if (coder->uncompressed_size + < (lzma_vli)(in_avail)) { + // There is too much input available. + return LZMA_DATA_ERROR; + } + } else { + // Limit in_size so that we don't copy too much. + if ((lzma_vli)(in_avail) > coder->uncompressed_size) + in_size = *in_pos + (size_t)( + coder->uncompressed_size); + } + + // Store the old position so we can update uncompressed_size. + const size_t out_start = *out_pos; + + // Copy the data + bufcpy(in, in_pos, in_size, out, out_pos, out_size); + + // Update uncompressed_size. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_size -= *out_pos - out_start; + + // Check if end of stream was reached. + if (coder->is_encoder) { + if (action == LZMA_FINISH && *in_pos == in_size) + coder->end_was_reached = true; + } else { + if (coder->uncompressed_size == 0) + coder->end_was_reached = true; + } + + } else { + // Call the next coder in the chain to provide us some data. + // We don't care about uncompressed_size here, because + // the next filter in the chain will do it for us (since + // we don't change the size of the data). + const lzma_ret ret = coder->next.code( + coder->next.coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, action); + + if (ret == LZMA_STREAM_END) { + assert(!coder->is_encoder + || action == LZMA_FINISH); + coder->end_was_reached = true; + + } else if (ret != LZMA_OK) { + return ret; + } + } + + return LZMA_OK; +} + + +static size_t +call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) +{ + const size_t filtered = coder->filter(coder->simple, + coder->now_pos, coder->is_encoder, + buffer, size); + coder->now_pos += filtered; + return filtered; +} + + +static lzma_ret +simple_code(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Flush already filtered data from coder->buffer[] to out[]. + if (coder->pos < coder->filtered) { + bufcpy(coder->buffer, &coder->pos, coder->filtered, + out, out_pos, out_size); + + // If we couldn't flush all the filtered data, return to + // application immediatelly. + if (coder->pos < coder->filtered) + return LZMA_OK; + + if (coder->end_was_reached) { + assert(coder->filtered == coder->size); + return LZMA_STREAM_END; + } + } + + // If we get here, there is no filtered data left in the buffer. + coder->filtered = 0; + + assert(!coder->end_was_reached); + + // If there is more output space left than there is unfiltered data + // in coder->buffer[], flush coder->buffer[] to out[], and copy/code + // more data to out[] hopefully filling it completely. Then filter + // the data in out[]. This step is where most of the data gets + // filtered if the buffer sizes used by the application are reasonable. + const size_t out_avail = out_size - *out_pos; + const size_t buf_avail = coder->size - coder->pos; + if (out_avail > buf_avail) { + // Store the old position so that we know from which byte + // to start filtering. + const size_t out_start = *out_pos; + + // Flush data from coder->buffer[] to out[], but don't reset + // coder->pos and coder->size yet. This way the coder can be + // restarted if the next filter in the chain returns e.g. + // LZMA_MEM_ERROR. + memcpy(out + *out_pos, coder->buffer + coder->pos, buf_avail); + *out_pos += buf_avail; + + // Copy/Encode/Decode more data to out[]. + { + const lzma_ret ret = copy_or_code(coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, action); + assert(ret != LZMA_STREAM_END); + if (ret != LZMA_OK) + return ret; + } + + // Filter out[]. + const size_t size = *out_pos - out_start; + const size_t filtered = call_filter( + coder, out + out_start, size); + + const size_t unfiltered = size - filtered; + assert(unfiltered <= coder->allocated / 2); + + // Now we can update coder->pos and coder->size, because + // the next coder in the chain (if any) was successful. + coder->pos = 0; + coder->size = unfiltered; + + if (coder->end_was_reached) { + // The last byte has been copied to out[] already. + // They are left as is. + coder->size = 0; + + } else if (unfiltered > 0) { + // There is unfiltered data left in out[]. Copy it to + // coder->buffer[] and rewind *out_pos appropriately. + *out_pos -= unfiltered; + memcpy(coder->buffer, out + *out_pos, unfiltered); + } + } else if (coder->pos > 0) { + memmove(coder->buffer, coder->buffer + coder->pos, buf_avail); + coder->size -= coder->pos; + coder->pos = 0; + } + + assert(coder->pos == 0); + + // If coder->buffer[] isn't empty, try to fill it by copying/decoding + // more data. Then filter coder->buffer[] and copy the successfully + // filtered data to out[]. It is probable, that some filtered and + // unfiltered data will be left to coder->buffer[]. + if (coder->size > 0) { + { + const lzma_ret ret = copy_or_code(coder, allocator, + in, in_pos, in_size, + coder->buffer, &coder->size, + coder->allocated, action); + assert(ret != LZMA_STREAM_END); + if (ret != LZMA_OK) + return ret; + } + + coder->filtered = call_filter( + coder, coder->buffer, coder->size); + + // Everything is considered to be filtered if coder->buffer[] + // contains the last bytes of the data. + if (coder->end_was_reached) + coder->filtered = coder->size; + + // Flush as much as possible. + bufcpy(coder->buffer, &coder->pos, coder->filtered, + out, out_pos, out_size); + } + + // Check if we got everything done. + if (coder->end_was_reached && coder->pos == coder->size) + return LZMA_STREAM_END; + + return LZMA_OK; +} + + +static void +simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_free(coder->simple, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + bool is_encoder, uint8_t *buffer, size_t size), + size_t simple_size, size_t unfiltered_max, bool is_encoder) +{ + // Allocate memory for the lzma_coder structure if needed. + if (next->coder == NULL) { + // Here we allocate space also for the temporary buffer. We + // need twice the size of unfiltered_max, because then it + // is always possible to filter at least unfiltered_max bytes + // more data in coder->buffer[] if it can be filled completely. + next->coder = lzma_alloc(sizeof(lzma_coder) + + 2 * unfiltered_max, allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &simple_code; + next->end = &simple_coder_end; + + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->filter = filter; + next->coder->allocated = 2 * unfiltered_max; + + // Allocate memory for filter-specific data structure. + if (simple_size > 0) { + next->coder->simple = lzma_alloc( + simple_size, allocator); + if (next->coder->simple == NULL) + return LZMA_MEM_ERROR; + } else { + next->coder->simple = NULL; + } + } + + if (filters[0].options != NULL) { + const lzma_options_simple *simple = filters[0].options; + next->coder->now_pos = simple->start_offset; + } else { + next->coder->now_pos = 0; + } + + // Reset variables. + next->coder->is_encoder = is_encoder; + next->coder->end_was_reached = false; + next->coder->uncompressed_size = filters[0].uncompressed_size; + next->coder->pos = 0; + next->coder->filtered = 0; + next->coder->size = 0; + + return lzma_next_filter_init( + &next->coder->next, allocator, filters + 1); +} diff --git a/src/liblzma/simple/simple_coder.h b/src/liblzma/simple/simple_coder.h new file mode 100644 index 00000000..b682f84f --- /dev/null +++ b/src/liblzma/simple/simple_coder.h @@ -0,0 +1,68 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_coder.h +/// \brief Wrapper for simple filters +// +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_SIMPLE_CODER_H +#define LZMA_SIMPLE_CODER_H + +#include "common.h" + + +extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + + +extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + + +extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + + +extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + + +extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + + +extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif diff --git a/src/liblzma/simple/simple_private.h b/src/liblzma/simple/simple_private.h new file mode 100644 index 00000000..a512396c --- /dev/null +++ b/src/liblzma/simple/simple_private.h @@ -0,0 +1,86 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_private.h +/// \brief Private definitions for so called simple filters +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_SIMPLE_PRIVATE_H +#define LZMA_SIMPLE_PRIVATE_H + +#include "simple_coder.h" + + +typedef struct lzma_simple_s lzma_simple; + +struct lzma_coder_s { + /// Next filter in the chain + lzma_next_coder next; + + /// True if the next coder in the chain has returned LZMA_STREAM_END + /// or if we have processed uncompressed_size bytes. + bool end_was_reached; + + /// True if filter() should encode the data; false to decode. + /// Currently all simple filters use the same function for encoding + /// and decoding, because the difference between encoders and decoders + /// is very small. + bool is_encoder; + + /// Size of the data *left* to be processed, or LZMA_VLI_VALUE_UNKNOWN + /// if unknown. + lzma_vli uncompressed_size; + + /// Pointer to filter-specific function, which does + /// the actual filtering. + size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + bool is_encoder, uint8_t *buffer, size_t size); + + /// Pointer to filter-specific data, or NULL if filter doesn't need + /// any extra data. + lzma_simple *simple; + + /// The lowest 32 bits of the current position in the data. Most + /// filters need this to do conversions between absolute and relative + /// addresses. + uint32_t now_pos; + + /// Size of the memory allocated for the buffer. + size_t allocated; + + /// Flushing position in the temporary buffer. buffer[pos] is the + /// next byte to be copied to out[]. + size_t pos; + + /// buffer[filtered] is the first unfiltered byte. When pos is smaller + /// than filtered, there is unflushed filtered data in the buffer. + size_t filtered; + + /// Total number of bytes (both filtered and unfiltered) currently + /// in the temporary buffer. + size_t size; + + /// Temporary buffer + uint8_t buffer[]; +}; + + +extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters, + size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + bool is_encoder, uint8_t *buffer, size_t size), + size_t simple_size, size_t unfiltered_max, bool is_encoder); + +#endif diff --git a/src/liblzma/simple/sparc.c b/src/liblzma/simple/sparc.c new file mode 100644 index 00000000..09bc67f1 --- /dev/null +++ b/src/liblzma/simple/sparc.c @@ -0,0 +1,88 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sparc.c +/// \brief Filter for SPARC binaries +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +static size_t +sparc_code(lzma_simple *simple lzma_attribute((unused)), + uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + size_t i; + for (i = 0; i + 4 <= size; i += 4) { + + if ((buffer[i] == 0x40 && (buffer[i + 1] & 0xC0) == 0x00) + || (buffer[i] == 0x7F + && (buffer[i + 1] & 0xC0) == 0xC0)) { + + uint32_t src = ((uint32_t)buffer[i + 0] << 24) + | ((uint32_t)buffer[i + 1] << 16) + | ((uint32_t)buffer[i + 2] << 8) + | ((uint32_t)buffer[i + 3]); + + src <<= 2; + + uint32_t dest; + if (is_encoder) + dest = now_pos + (uint32_t)(i) + src; + else + dest = src - (now_pos + (uint32_t)(i)); + + dest >>= 2; + + dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) + | (dest & 0x3FFFFF) + | 0x40000000; + + buffer[i + 0] = (uint8_t)(dest >> 24); + buffer[i + 1] = (uint8_t)(dest >> 16); + buffer[i + 2] = (uint8_t)(dest >> 8); + buffer[i + 3] = (uint8_t)(dest); + } + } + + return i; +} + + +static lzma_ret +sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + return lzma_simple_coder_init(next, allocator, filters, + &sparc_code, 0, 4, is_encoder); +} + + +extern lzma_ret +lzma_simple_sparc_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return sparc_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_sparc_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + return sparc_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/simple/x86.c b/src/liblzma/simple/x86.c new file mode 100644 index 00000000..a48a5750 --- /dev/null +++ b/src/liblzma/simple/x86.c @@ -0,0 +1,161 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file x86.c +/// \brief Filter for x86 binaries (BCJ filter) +// +// Copyright (C) 1999-2006 Igor Pavlov +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_private.h" + + +#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) + + +struct lzma_simple_s { + uint32_t prev_mask; + uint32_t prev_pos; +}; + + +static size_t +x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, + uint8_t *buffer, size_t size) +{ + static const bool MASK_TO_ALLOWED_STATUS[8] + = { true, true, true, false, true, false, false, false }; + + static const uint32_t MASK_TO_BIT_NUMBER[8] + = { 0, 1, 2, 2, 3, 3, 3, 3 }; + + uint32_t prev_mask = simple->prev_mask; + uint32_t prev_pos = simple->prev_pos; + + if (size < 5) + return 0; + + if (now_pos - prev_pos > 5) + prev_pos = now_pos - 5; + + const size_t limit = size - 5; + size_t buffer_pos = 0; + + while (buffer_pos <= limit) { + uint8_t b = buffer[buffer_pos]; + if (b != 0xE8 && b != 0xE9) { + ++buffer_pos; + continue; + } + + const uint32_t offset = now_pos + (uint32_t)(buffer_pos) + - prev_pos; + prev_pos = now_pos + (uint32_t)(buffer_pos); + + if (offset > 5) { + prev_mask = 0; + } else { + for (uint32_t i = 0; i < offset; ++i) { + prev_mask &= 0x77; + prev_mask <<= 1; + } + } + + b = buffer[buffer_pos + 4]; + + if (Test86MSByte(b) + && MASK_TO_ALLOWED_STATUS[(prev_mask >> 1) & 0x7] + && (prev_mask >> 1) < 0x10) { + + uint32_t src = ((uint32_t)(b) << 24) + | ((uint32_t)(buffer[buffer_pos + 3]) << 16) + | ((uint32_t)(buffer[buffer_pos + 2]) << 8) + | (buffer[buffer_pos + 1]); + + uint32_t dest; + while (true) { + if (is_encoder) + dest = src + (now_pos + (uint32_t)( + buffer_pos) + 5); + else + dest = src - (now_pos + (uint32_t)( + buffer_pos) + 5); + + if (prev_mask == 0) + break; + + const uint32_t i = MASK_TO_BIT_NUMBER[ + prev_mask >> 1]; + + b = (uint8_t)(dest >> (24 - i * 8)); + + if (!Test86MSByte(b)) + break; + + src = dest ^ ((1 << (32 - i * 8)) - 1); + } + + buffer[buffer_pos + 4] + = (uint8_t)(~(((dest >> 24) & 1) - 1)); + buffer[buffer_pos + 3] = (uint8_t)(dest >> 16); + buffer[buffer_pos + 2] = (uint8_t)(dest >> 8); + buffer[buffer_pos + 1] = (uint8_t)(dest); + buffer_pos += 5; + prev_mask = 0; + + } else { + ++buffer_pos; + prev_mask |= 1; + if (Test86MSByte(b)) + prev_mask |= 0x10; + } + } + + simple->prev_mask = prev_mask; + simple->prev_pos = prev_pos; + + return buffer_pos; +} + + +static lzma_ret +x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, bool is_encoder) +{ + const lzma_ret ret = lzma_simple_coder_init(next, allocator, filters, + &x86_code, sizeof(lzma_simple), 5, is_encoder); + + if (ret == LZMA_OK) { + next->coder->simple->prev_mask = 0; + next->coder->simple->prev_pos = (uint32_t)(-5); + } + + return ret; +} + + +extern lzma_ret +lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return x86_coder_init(next, allocator, filters, true); +} + + +extern lzma_ret +lzma_simple_x86_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return x86_coder_init(next, allocator, filters, false); +} diff --git a/src/liblzma/subblock/Makefile.am b/src/liblzma/subblock/Makefile.am new file mode 100644 index 00000000..8f2daf59 --- /dev/null +++ b/src/liblzma/subblock/Makefile.am @@ -0,0 +1,33 @@ +## +## Copyright (C) 2007 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = libsubblock.la +libsubblock_la_SOURCES = +libsubblock_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common + +if COND_MAIN_ENCODER +libsubblock_la_SOURCES += \ + subblock_encoder.c \ + subblock_encoder.h +endif + +if COND_MAIN_DECODER +libsubblock_la_SOURCES += \ + subblock_decoder.c \ + subblock_decoder.h \ + subblock_decoder_helper.c \ + subblock_decoder_helper.h +endif diff --git a/src/liblzma/subblock/subblock_decoder.c b/src/liblzma/subblock/subblock_decoder.c new file mode 100644 index 00000000..ef3c289f --- /dev/null +++ b/src/liblzma/subblock/subblock_decoder.c @@ -0,0 +1,681 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_decoder.c +/// \brief Decoder of the Subblock filter +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "subblock_decoder.h" +#include "subblock_decoder_helper.h" +#include "raw_decoder.h" + + +/// Maximum number of consecutive Subblocks with Subblock Type Padding +#define PADDING_MAX 31 + + +struct lzma_coder_s { + lzma_next_coder next; + + enum { + SEQ_FLAGS, + SEQ_SIZE_1, + SEQ_SIZE_2, + SEQ_SIZE_3, + SEQ_DATA, + SEQ_REPEAT_COUNT_1, + SEQ_REPEAT_COUNT_2, + SEQ_REPEAT_COUNT_3, + SEQ_REPEAT_SIZE, + SEQ_REPEAT_READ_DATA, + SEQ_REPEAT_FAST, + SEQ_REPEAT_NORMAL, + SEQ_FILTER_FLAGS, + SEQ_FILTER_END, + } sequence; + + /// Number of bytes left in the current Subblock Data field. + size_t size; + + /// Uncompressed Size, or LZMA_VLI_VALUE_UNKNOWN if unknown. + lzma_vli uncompressed_size; + + /// Number of consecutive Subblocks with Subblock Type Padding + uint32_t padding; + + /// True when .next.code() has returned LZMA_STREAM_END. + bool next_finished; + + /// True when the Subblock decoder has detected End of Payload Marker. + /// This may become true before next_finished becomes true. + bool this_finished; + + /// True if Subfilters are allowed. + bool allow_subfilters; + + /// Indicates if at least one byte of decoded output has been + /// produced after enabling Subfilter. + bool got_output_with_subfilter; + + /// Possible subfilter + lzma_next_coder subfilter; + + /// Filter Flags decoder is needed to parse the ID and Properties + /// of the subfilter. + lzma_next_coder filter_flags_decoder; + + /// The filter_flags_decoder stores its results here. + lzma_options_filter filter_flags; + + /// Options for the Subblock decoder helper. This is used to tell + /// the helper when it should return LZMA_STREAM_END to the subfilter. + lzma_options_subblock_helper helper; + + struct { + /// How many times buffer should be repeated + size_t count; + + /// Size of the buffer + size_t size; + + /// Position in the buffer + size_t pos; + + /// Buffer to hold the data to be repeated + uint8_t buffer[LZMA_SUBBLOCK_RLE_MAX]; + } repeat; + + /// Temporary buffer needed when the Subblock filter is not the last + /// filter in the chain. The output of the next filter is first + /// decoded into buffer[], which is then used as input for the actual + /// Subblock decoder. + struct { + size_t pos; + size_t size; + uint8_t buffer[LZMA_BUFFER_SIZE]; + } temp; +}; + + +/// Values of valid Subblock Flags +enum { + FLAG_PADDING, + FLAG_EOPM, + FLAG_DATA, + FLAG_REPEAT, + FLAG_SET_SUBFILTER, + FLAG_END_SUBFILTER, +}; + + +/// Substracts size from coder->uncompressed_size uncompressed size is known +/// and size isn't bigger than coder->uncompressed_size. +static inline bool +update_uncompressed_size(lzma_coder *coder, size_t size) +{ + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + if ((lzma_vli)(size) > coder->uncompressed_size) + return true; + + coder->uncompressed_size -= size; + } + + return false; +} + + +/// Calls the subfilter and updates coder->uncompressed_size. +static lzma_ret +subfilter_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + assert(coder->subfilter.code != NULL); + + const size_t out_start = *out_pos; + + // Call the subfilter. + const lzma_ret ret = coder->subfilter.code( + coder->subfilter.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, action); + + // Update uncompressed_size. + if (update_uncompressed_size(coder, *out_pos - out_start)) + return LZMA_DATA_ERROR; + + return ret; +} + + +static lzma_ret +decode_buffer(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + while (*out_pos < out_size && (*in_pos < in_size + || coder->sequence == SEQ_DATA)) + switch (coder->sequence) { + case SEQ_FLAGS: { + if ((in[*in_pos] >> 4) != FLAG_PADDING) + coder->padding = 0; + + // Do the correct action depending on the Subblock Type. + switch (in[*in_pos] >> 4) { + case FLAG_PADDING: + // Only check that reserved bits are zero. +// if (++coder->padding > PADDING_MAX +// || in[*in_pos] & 0x0F) +// return LZMA_DATA_ERROR; + ++*in_pos; + break; + + case FLAG_EOPM: + // Check that reserved bits are zero. + if (in[*in_pos] & 0x0F) + return LZMA_DATA_ERROR; + + // There must be no Subfilter enabled. + if (coder->subfilter.code != NULL) + return LZMA_DATA_ERROR; + + // End of Payload Marker must not be used if + // uncompressed size is known. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + return LZMA_DATA_ERROR; + + ++*in_pos; + return LZMA_STREAM_END; + + case FLAG_DATA: + // First four bits of the Subblock Data size. + coder->size = in[*in_pos] & 0x0F; + ++*in_pos; + coder->got_output_with_subfilter = true; + coder->sequence = SEQ_SIZE_1; + break; + + case FLAG_REPEAT: + // First four bits of the Repeat Count. We use + // coder->size as a temporary place for it. + coder->size = in[*in_pos] & 0x0F; + ++*in_pos; + coder->got_output_with_subfilter = true; + coder->sequence = SEQ_REPEAT_COUNT_1; + break; + + case FLAG_SET_SUBFILTER: { + if ((in[*in_pos] & 0x0F) + || coder->subfilter.code != NULL + || !coder->allow_subfilters) + return LZMA_DATA_ERROR; + + assert(coder->filter_flags.options == NULL); + const lzma_ret ret = lzma_filter_flags_decoder_init( + &coder->filter_flags_decoder, + allocator, &coder->filter_flags); + if (ret != LZMA_OK) + return ret; + + coder->got_output_with_subfilter = false; + + ++*in_pos; + coder->sequence = SEQ_FILTER_FLAGS; + break; + } + + case FLAG_END_SUBFILTER: + if (coder->subfilter.code == NULL + || !coder->got_output_with_subfilter) + return LZMA_DATA_ERROR; + + // Tell the helper filter to indicate End of Input + // to our subfilter. + coder->helper.end_was_reached = true; + + size_t dummy = 0; + const lzma_ret ret = subfilter_decode(coder, allocator, + NULL, &dummy, 0, out, out_pos,out_size, + action); + + // If we didn't reach the end of the subfilter's output + // yet, return to the application. On the next call we + // will get to this same switch-case again, because we + // haven't updated *in_pos yet. + if (ret != LZMA_STREAM_END) + return ret; + + // Free Subfilter's memory. This is a bit debatable, + // since we could avoid some malloc()/free() calls + // if the same Subfilter gets used soon again. But + // if Subfilter isn't used again, we could leave + // a memory-hogging filter dangling until someone + // frees Subblock filter itself. + lzma_next_coder_end(&coder->subfilter, allocator); + + // Free memory used for subfilter options. This is + // safe, because we don't support any Subfilter that + // would allow pointers in the options structure. + lzma_free(coder->filter_flags.options, allocator); + coder->filter_flags.options = NULL; + + ++*in_pos; + + if (coder->uncompressed_size == 0) + return LZMA_STREAM_END; + + break; + + default: + return LZMA_DATA_ERROR; + } + + break; + } + + case SEQ_SIZE_1: + case SEQ_REPEAT_COUNT_1: + // We use the same code to parse + // - the Size (28 bits) in Subblocks of type Data; and + // - the Repeat count (28 bits) in Subblocks of type + // Repeating Data. + coder->size |= (size_t)(in[*in_pos]) << 4; + ++*in_pos; + ++coder->sequence; + break; + + case SEQ_SIZE_2: + case SEQ_REPEAT_COUNT_2: + coder->size |= (size_t)(in[*in_pos]) << 12; + ++*in_pos; + ++coder->sequence; + break; + + case SEQ_SIZE_3: + case SEQ_REPEAT_COUNT_3: + coder->size |= (size_t)(in[*in_pos]) << 20; + + // The real value is the stored value plus one. + ++coder->size; + + ++*in_pos; + ++coder->sequence; + break; + + case SEQ_REPEAT_SIZE: + // Move the Repeat Count to the correct variable and parse + // the Size of the Data to be repeated. + coder->repeat.count = coder->size; + coder->repeat.size = (size_t)(in[*in_pos]) + 1; + coder->repeat.pos = 0; + ++*in_pos; + coder->sequence = SEQ_REPEAT_READ_DATA; + break; + + case SEQ_REPEAT_READ_DATA: { + // Fill coder->repeat.buffer[]. + const size_t in_avail = in_size - *in_pos; + const size_t out_avail + = coder->repeat.size - coder->repeat.pos; + const size_t copy_size = MIN(in_avail, out_avail); + + memcpy(coder->repeat.buffer + coder->repeat.pos, + in + *in_pos, copy_size); + *in_pos += copy_size; + coder->repeat.pos += copy_size; + + if (coder->repeat.pos == coder->repeat.size) { + coder->repeat.pos = 0; + + if (coder->repeat.size == 1 + && coder->subfilter.code == NULL) + coder->sequence = SEQ_REPEAT_FAST; + else + coder->sequence = SEQ_REPEAT_NORMAL; + } + + break; + } + + case SEQ_REPEAT_FAST: { + // Optimization for cases when there is only one byte to + // repeat and no Subfilter. + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(coder->repeat.count, out_avail); + + memset(out + *out_pos, coder->repeat.buffer[0], copy_size); + + *out_pos += copy_size; + coder->repeat.count -= copy_size; + + if (update_uncompressed_size(coder, copy_size)) + return LZMA_DATA_ERROR; + + if (coder->repeat.count == 0) { + if (coder->uncompressed_size == 0) + return LZMA_STREAM_END; + } else { + return LZMA_OK; + } + + coder->sequence = SEQ_FLAGS; + break; + } + + case SEQ_REPEAT_NORMAL: + do { + // Cycle the repeat buffer if needed. + if (coder->repeat.pos == coder->repeat.size) { + if (--coder->repeat.count == 0) { + coder->sequence = SEQ_FLAGS; + break; + } + + coder->repeat.pos = 0; + } + + if (coder->subfilter.code == NULL) { + const size_t copy_size = bufcpy( + coder->repeat.buffer, + &coder->repeat.pos, + coder->repeat.size, + out, out_pos, out_size); + + if (update_uncompressed_size(coder, copy_size)) + return LZMA_DATA_ERROR; + + } else { + const lzma_ret ret = subfilter_decode( + coder, allocator, + coder->repeat.buffer, + &coder->repeat.pos, + coder->repeat.size, + out, out_pos, out_size, + action); + + if (ret == LZMA_STREAM_END) { + // End of Subfilter can occur only at + // a Subblock boundary. + if (coder->repeat.pos + != coder->repeat.size + || --coder->repeat + .count != 0) + return LZMA_DATA_ERROR; + + // We need a Subblock with Unset + // Subfilter before more data. + coder->sequence = SEQ_FILTER_END; + break; + + } else if (ret != LZMA_OK) { + return ret; + } + } + } while (*out_pos < out_size); + + break; + + case SEQ_DATA: { + // Limit the amount of input to match the available + // Subblock Data size. + size_t in_limit; + if (in_size - *in_pos > coder->size) + in_limit = *in_pos + coder->size; + else + in_limit = in_size; + + if (coder->subfilter.code == NULL) { + const size_t copy_size = bufcpy( + in, in_pos, in_limit, + out, out_pos, out_size); + + coder->size -= copy_size; + + if (update_uncompressed_size(coder, copy_size)) + return LZMA_DATA_ERROR; + + } else { + const size_t in_start = *in_pos; + const lzma_ret ret = subfilter_decode( + coder, allocator, + in, in_pos, in_limit, + out, out_pos, out_size, + action); + + // Update the number of unprocessed bytes left in + // this Subblock. This assert() is true because + // in_limit prevents *in_pos getting too big. + assert(*in_pos - in_start <= coder->size); + coder->size -= *in_pos - in_start; + + if (ret == LZMA_STREAM_END) { + // End of Subfilter can occur only at + // a Subblock boundary. + if (coder->size != 0) + return LZMA_DATA_ERROR; + + // We need a Subblock with Unset + // Subfilter before more data. + coder->sequence = SEQ_FILTER_END; + break; + } + + if (ret != LZMA_OK) + return ret; + } + + // If we couldn't process the whole Subblock Data yet, return. + if (coder->size > 0) + return LZMA_OK; + + // Check if we have decoded all the data. + if (coder->uncompressed_size == 0 + && coder->subfilter.code == NULL) + return LZMA_STREAM_END; + + coder->sequence = SEQ_FLAGS; + break; + } + + case SEQ_FILTER_FLAGS: { + lzma_ret ret = coder->filter_flags_decoder.code( + coder->filter_flags_decoder.coder, allocator, + in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret == LZMA_HEADER_ERROR + ? LZMA_DATA_ERROR : ret; + + // Don't free the filter_flags_decoder. It doesn't take much + // memory and we may need it again. + + // Initialize the Subfilter. Subblock and Copy filters are + // not allowed. + if (coder->filter_flags.id == LZMA_FILTER_COPY + || coder->filter_flags.id + == LZMA_FILTER_SUBBLOCK) + return LZMA_DATA_ERROR; + + coder->helper.end_was_reached = false; + + lzma_options_filter filters[3] = { + { + .id = coder->filter_flags.id, + .options = coder->filter_flags.options, + }, { + .id = LZMA_FILTER_SUBBLOCK_HELPER, + .options = &coder->helper, + }, { + .id = LZMA_VLI_VALUE_UNKNOWN, + .options = NULL, + } + }; + + // Optimization: We know that LZMA uses End of Payload Marker + // (not End of Input), so we can omit the helper filter. + if (filters[0].id == LZMA_FILTER_LZMA) + filters[1].id = LZMA_VLI_VALUE_UNKNOWN; + + ret = lzma_raw_decoder_init(&coder->subfilter, allocator, + filters, LZMA_VLI_VALUE_UNKNOWN, false); + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_FLAGS; + break; + } + + case SEQ_FILTER_END: + // We are in the beginning of a Subblock. The next Subblock + // whose type is not Padding, must indicate end of Subfilter. + if (in[*in_pos] == (FLAG_PADDING << 4)) { + ++*in_pos; + break; + } + + if (in[*in_pos] != (FLAG_END_SUBFILTER << 4)) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_FLAGS; + break; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +subblock_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + if (coder->next.code == NULL) + return decode_buffer(coder, allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + while (*out_pos < out_size) { + if (!coder->next_finished + && coder->temp.pos == coder->temp.size) { + coder->temp.pos = 0; + coder->temp.size = 0; + + const lzma_ret ret = coder->next.code( + coder->next.coder, + allocator, in, in_pos, in_size, + coder->temp.buffer, &coder->temp.size, + LZMA_BUFFER_SIZE, action); + + if (ret == LZMA_STREAM_END) + coder->next_finished = true; + else if (coder->temp.size == 0 || ret != LZMA_OK) + return ret; + } + + if (coder->this_finished) { + if (coder->temp.pos != coder->temp.size) + return LZMA_DATA_ERROR; + + if (coder->next_finished) + return LZMA_STREAM_END; + + return LZMA_OK; + } + + const lzma_ret ret = decode_buffer(coder, allocator, + coder->temp.buffer, &coder->temp.pos, + coder->temp.size, + out, out_pos, out_size, action); + + if (ret == LZMA_STREAM_END) + // The next coder in the chain hasn't finished + // yet. If the input data is valid, there + // must be no more output coming, but the + // next coder may still need a litle more + // input to detect End of Payload Marker. + coder->this_finished = true; + else if (ret != LZMA_OK) + return ret; + else if (coder->next_finished && *out_pos < out_size) + return LZMA_DATA_ERROR; + } + + return LZMA_OK; +} + + +static void +subblock_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_next_coder_end(&coder->subfilter, allocator); + lzma_next_coder_end(&coder->filter_flags_decoder, allocator); + lzma_free(coder->filter_flags.options, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_subblock_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->subfilter = LZMA_NEXT_CODER_INIT; + next->coder->filter_flags_decoder = LZMA_NEXT_CODER_INIT; + + } else { + lzma_next_coder_end(&next->coder->subfilter, allocator); + lzma_free(next->coder->filter_flags.options, allocator); + } + + next->coder->filter_flags.options = NULL; + + next->coder->sequence = SEQ_FLAGS; + next->coder->uncompressed_size = filters[0].uncompressed_size; + next->coder->padding = 0; + next->coder->next_finished = false; + next->coder->this_finished = false; + next->coder->temp.pos = 0; + next->coder->temp.size = 0; + + if (filters[0].options != NULL) + next->coder->allow_subfilters = ((lzma_options_subblock *)( + filters[0].options))->allow_subfilters; + else + next->coder->allow_subfilters = false; + + { + const lzma_ret ret = lzma_next_filter_init(&next->coder->next, + allocator, filters + 1); + if (ret != LZMA_OK) { + subblock_decoder_end(next->coder, allocator); + return ret; + } + } + + next->code = &subblock_decode; + next->end = &subblock_decoder_end; + + return LZMA_OK; +} diff --git a/src/liblzma/subblock/subblock_decoder.h b/src/liblzma/subblock/subblock_decoder.h new file mode 100644 index 00000000..724ef357 --- /dev/null +++ b/src/liblzma/subblock/subblock_decoder.h @@ -0,0 +1,29 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_decoder.h +/// \brief Decoder of the Subblock filter +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_SUBBLOCK_DECODER_H +#define LZMA_SUBBLOCK_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_subblock_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif diff --git a/src/liblzma/subblock/subblock_decoder_helper.c b/src/liblzma/subblock/subblock_decoder_helper.c new file mode 100644 index 00000000..5ff24431 --- /dev/null +++ b/src/liblzma/subblock/subblock_decoder_helper.c @@ -0,0 +1,80 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_decoder_helper.c +/// \brief Helper filter for the Subblock decoder +/// +/// This filter is used to indicate End of Input for subfilters needing it. +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "subblock_decoder_helper.h" + + +struct lzma_coder_s { + const lzma_options_subblock_helper *options; +}; + + +static lzma_ret +helper_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action lzma_attribute((unused))) +{ + // If end_was_reached is true, we cannot have any input. + assert(!coder->options->end_was_reached || *in_pos == in_size); + + // We can safely copy as much as possible, because we are never + // given more data than a single Subblock Data field. + bufcpy(in, in_pos, in_size, out, out_pos, out_size); + + // Return LZMA_STREAM_END when instructed so by the Subblock decoder. + return coder->options->end_was_reached ? LZMA_STREAM_END : LZMA_OK; +} + + +static void +helper_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_subblock_decoder_helper_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters) +{ + // This is always the last filter in the chain. + assert(filters[1].init == NULL); + + // We never know uncompressed size. + assert(filters[0].uncompressed_size == LZMA_VLI_VALUE_UNKNOWN); + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &helper_decode; + next->end = helper_end; + } + + next->coder->options = filters[0].options; + + return LZMA_OK; +} diff --git a/src/liblzma/subblock/subblock_decoder_helper.h b/src/liblzma/subblock/subblock_decoder_helper.h new file mode 100644 index 00000000..2a3e999e --- /dev/null +++ b/src/liblzma/subblock/subblock_decoder_helper.h @@ -0,0 +1,36 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_decoder_helper.h +/// \brief Helper filter for the Subblock decoder +/// +/// This filter is used to indicate End of Input for subfilters needing it. +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_SUBBLOCK_DECODER_HELPER_H +#define LZMA_SUBBLOCK_DECODER_HELPER_H + +#include "common.h" + + +typedef struct { + bool end_was_reached; +} lzma_options_subblock_helper; + + +extern lzma_ret lzma_subblock_decoder_helper_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif diff --git a/src/liblzma/subblock/subblock_encoder.c b/src/liblzma/subblock/subblock_encoder.c new file mode 100644 index 00000000..9fa95b24 --- /dev/null +++ b/src/liblzma/subblock/subblock_encoder.c @@ -0,0 +1,841 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_encoder.c +/// \brief Encoder of the Subblock filter +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "subblock_encoder.h" +#include "raw_encoder.h" + + +#define REPEAT_COUNT_MAX (1U << 28) + +/// Number of bytes the data chunk being repeated must be before we care +/// about alignment. This is somewhat arbitrary. It just doesn't make sense +/// to waste bytes for alignment when the data chunk is very small. +/// +/// TODO Rename and use this also for Subblock Data? +#define RLE_MIN_SIZE_FOR_ALIGN 3 + +#define write_byte(b) \ +do { \ + out[*out_pos] = b; \ + ++*out_pos; \ + ++coder->alignment.out_pos; \ +} while (0) + + +struct lzma_coder_s { + lzma_next_coder next; + bool next_finished; + + enum { + SEQ_FILL, + SEQ_FLUSH, + SEQ_RLE_COUNT_0, + SEQ_RLE_COUNT_1, + SEQ_RLE_COUNT_2, + SEQ_RLE_COUNT_3, + SEQ_RLE_SIZE, + SEQ_RLE_DATA, + SEQ_DATA_SIZE_0, + SEQ_DATA_SIZE_1, + SEQ_DATA_SIZE_2, + SEQ_DATA_SIZE_3, + SEQ_DATA, + SEQ_SUBFILTER_INIT, + SEQ_SUBFILTER_FLAGS, + } sequence; + + lzma_options_subblock *options; + + lzma_vli uncompressed_size; + + size_t pos; + uint32_t tmp; + + struct { + uint32_t multiple; + uint32_t in_pending; + uint32_t in_pos; + uint32_t out_pos; + } alignment; + + struct { + uint8_t *data; + size_t size; + size_t limit; + } subblock; + + struct { + uint8_t buffer[LZMA_SUBBLOCK_RLE_MAX]; + size_t size; + lzma_vli count; + } rle; + + struct { + enum { + SUB_NONE, + SUB_SET, + SUB_RUN, + SUB_FINISH, + SUB_END_MARKER, + } mode; + + bool got_input; + + uint8_t *flags; + size_t flags_size; + + lzma_next_coder subcoder; + + } subfilter; + + struct { + size_t pos; + size_t size; + uint8_t buffer[LZMA_BUFFER_SIZE]; + } temp; +}; + + +/// \brief Aligns the output buffer +/// +/// Aligns the output buffer so that after skew bytes the output position is +/// a multiple of coder->alignment.multiple. +static bool +subblock_align(lzma_coder *coder, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, uint32_t skew) +{ + assert(*out_pos < out_size); + + const uint32_t target = coder->alignment.in_pos + % coder->alignment.multiple; + + while ((coder->alignment.out_pos + skew) + % coder->alignment.multiple != target) { + // Zero indicates padding. + write_byte(0x00); + + // Check if output buffer got full and indicate it to + // the caller. + if (*out_pos == out_size) + return true; + } + + coder->alignment.in_pos += coder->alignment.in_pending; + coder->alignment.in_pending = 0; + + // Output buffer is not full. + return false; +} + + +/// \brief Checks if buffer contains repeated data +/// +/// \param needle Buffer containing a single repeat chunk +/// \param needle_size Size of needle in bytes +/// \param buf Buffer to search for repeated needles +/// \param buf_chunks Buffer size is buf_chunks * needle_size. +/// +/// \return True if the whole buf is filled with repeated needles. +/// +static bool +is_repeating(const uint8_t *restrict needle, size_t needle_size, + const uint8_t *restrict buf, size_t buf_chunks) +{ + while (buf_chunks-- != 0) { + if (memcmp(buf, needle, needle_size) != 0) + return false; + + buf += needle_size; + } + + return true; +} + + +/// \brief Optimizes the repeating style and updates coder->sequence +static void +subblock_rle_flush(lzma_coder *coder) +{ + // The Subblock decoder can use memset() when the size of the data + // being repeated is one byte, so we check if the RLE buffer is + // filled with a single repeating byte. + if (coder->rle.size > 1) { + const uint8_t b = coder->rle.buffer[0]; + size_t i = 0; + while (true) { + if (coder->rle.buffer[i] != b) + break; + + if (++i == coder->rle.size) { + // TODO Integer overflow check maybe, + // although this needs at least 2**63 bytes + // of input until it gets triggered... + coder->rle.count *= coder->rle.size; + coder->rle.size = 1; + break; + } + } + } + + if (coder->rle.count > REPEAT_COUNT_MAX) + coder->tmp = REPEAT_COUNT_MAX - 1; + else + coder->tmp = coder->rle.count - 1; + + coder->sequence = SEQ_RLE_COUNT_0; + + return; +} + + +/// \brief Resizes coder->subblock.data for a new size limit +static lzma_ret +subblock_data_size(lzma_coder *coder, lzma_allocator *allocator, + size_t new_limit) +{ + // Verify that the new limit is valid. + if (new_limit < LZMA_SUBBLOCK_DATA_SIZE_MIN + || new_limit > LZMA_SUBBLOCK_DATA_SIZE_MAX) + return LZMA_HEADER_ERROR; + + // Ff the new limit is different than the previous one, we need + // to reallocate the data buffer. + if (new_limit != coder->subblock.limit) { + lzma_free(coder->subblock.data, allocator); + coder->subblock.data = lzma_alloc(new_limit, allocator); + if (coder->subblock.data == NULL) + return LZMA_MEM_ERROR; + } + + coder->subblock.limit = new_limit; + + return LZMA_OK; +} + + +static lzma_ret +subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Verify that there is a sane amount of input. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + const lzma_vli in_avail = in_size - *in_pos; + if (action == LZMA_FINISH) { + if (in_avail != coder->uncompressed_size) + return LZMA_DATA_ERROR; + } else { + if (in_avail > coder->uncompressed_size) + return LZMA_DATA_ERROR; + } + } + + // Check if we need to do something special with the Subfilter. + if (coder->options != NULL && coder->options->allow_subfilters) { + switch (coder->options->subfilter_mode) { + case LZMA_SUBFILTER_NONE: + if (coder->subfilter.mode != SUB_NONE) + return LZMA_PROG_ERROR; + break; + + case LZMA_SUBFILTER_SET: + if (coder->subfilter.mode != SUB_NONE) + return LZMA_HEADER_ERROR; + + coder->subfilter.mode = SUB_SET; + coder->subfilter.got_input = false; + + if (coder->sequence == SEQ_FILL) + coder->sequence = SEQ_FLUSH; + + break; + + case LZMA_SUBFILTER_RUN: + if (coder->subfilter.mode != SUB_RUN) + return LZMA_PROG_ERROR; + break; + + case LZMA_SUBFILTER_FINISH: + if (coder->subfilter.mode == SUB_RUN) + coder->subfilter.mode = SUB_FINISH; + else if (coder->subfilter.mode != SUB_FINISH) + return LZMA_PROG_ERROR; + + if (!coder->subfilter.got_input) + return LZMA_PROG_ERROR; + + break; + + default: + return LZMA_HEADER_ERROR; + } + } + + // Main loop + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_FILL: { + // Grab the new Subblock Data Size and reallocate the buffer. + if (coder->subblock.size == 0 && coder->options != NULL + && coder->options->subblock_data_size + != coder->subblock.limit) { + const lzma_ret ret = subblock_data_size(coder, + allocator, coder->options + ->subblock_data_size); + if (ret != LZMA_OK) + return ret; + } + + if (coder->subfilter.mode == SUB_NONE) { + assert(coder->subfilter.subcoder.code == NULL); + + // No Subfilter is enabled, just copy the data as is. + // NOTE: uncompressed_size cannot overflow because we + // have checked/ it in the beginning of this function. + const size_t in_used = bufcpy(in, in_pos, in_size, + coder->subblock.data, + &coder->subblock.size, + coder->subblock.limit); + + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_size -= in_used; + + coder->alignment.in_pending += in_used; + + } else { + const size_t in_start = *in_pos; + lzma_ret ret; + + if (coder->subfilter.mode == SUB_FINISH) { + // Let the Subfilter write out pending data, + // but don't give it any new input anymore. + size_t dummy = 0; + ret = coder->subfilter.subcoder.code(coder + ->subfilter.subcoder.coder, + allocator, NULL, &dummy, 0, + coder->subblock.data, + &coder->subblock.size, + coder->subblock.limit, + LZMA_FINISH); + } else { + // Give our input data to the Subfilter. Note + // that action can be LZMA_FINISH. In that + // case, we filter everything until the end + // of the input. The application isn't required + // to separately set LZMA_SUBBLOCK_FINISH. + ret = coder->subfilter.subcoder.code(coder + ->subfilter.subcoder.coder, + allocator, in, in_pos, in_size, + coder->subblock.data, + &coder->subblock.size, + coder->subblock.limit, + action); + } + + const size_t in_used = *in_pos - in_start; + + if (in_used > 0) + coder->subfilter.got_input = true; + + // NOTE: uncompressed_size cannot overflow because we + // have checked it in the beginning of this function. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) + coder->uncompressed_size -= *in_pos - in_start; + + coder->alignment.in_pending += in_used; + + if (ret == LZMA_STREAM_END) { + // We don't strictly need to do this, but + // doing it sounds like a good idea, because + // otherwise the Subfilter's memory could be + // left allocated for long time, and would + // just waste memory. + lzma_next_coder_end(&coder->subfilter.subcoder, + allocator); + + assert(coder->options != NULL); + coder->options->subfilter_mode + = LZMA_SUBFILTER_NONE; + + assert(coder->subfilter.mode == SUB_FINISH + || action == LZMA_FINISH); + coder->subfilter.mode = SUB_END_MARKER; + + // Flush now. Even if coder->subblock.size + // happens to be zero, we still need to go + // to SEQ_FLUSH to write the Subfilter Unset + // indicator. + coder->sequence = SEQ_FLUSH; + break; + } + + // Return if an error occurred. + if (ret != LZMA_OK) + return ret; + } + + // If we ran out of input before the whole buffer + // was filled, return to application. + if (coder->subblock.size < coder->subblock.limit + && action != LZMA_FINISH) + return LZMA_OK; + + coder->sequence = SEQ_FLUSH; + } + + // Fall through + + case SEQ_FLUSH: + if (coder->options != NULL) { + // Update the alignment variable. + coder->alignment.multiple = coder->options->alignment; + if (coder->alignment.multiple + < LZMA_SUBBLOCK_ALIGNMENT_MIN + || coder->alignment.multiple + > LZMA_SUBBLOCK_ALIGNMENT_MAX) + return LZMA_HEADER_ERROR; + + // Run-length encoder + // + // First check if there is some data pending and we + // have an obvious need to flush it immediatelly. + if (coder->rle.count > 0 + && (coder->rle.size + != coder->options->rle + || coder->subblock.size + % coder->rle.size)) { + subblock_rle_flush(coder); + break; + } + + // Grab the (possibly new) RLE chunk size and + // validate it. + coder->rle.size = coder->options->rle; + if (coder->rle.size > LZMA_SUBBLOCK_RLE_MAX) + return LZMA_HEADER_ERROR; + + if (coder->subblock.size != 0 + && coder->rle.size + != LZMA_SUBBLOCK_RLE_OFF + && coder->subblock.size + % coder->rle.size == 0) { + + // Initialize coder->rle.buffer if we don't + // have RLE already running. + if (coder->rle.count == 0) + memcpy(coder->rle.buffer, + coder->subblock.data, + coder->rle.size); + + // Test if coder->subblock.data is repeating. + const size_t count = coder->subblock.size + / coder->rle.size; + if (is_repeating(coder->rle.buffer, + coder->rle.size, + coder->subblock.data, count)) { + if (LZMA_VLI_VALUE_MAX - count + < coder->rle.count) + return LZMA_PROG_ERROR; + + coder->rle.count += count; + coder->subblock.size = 0; + + } else if (coder->rle.count > 0) { + // It's not repeating or at least not + // with the same byte sequence as the + // earlier Subblock Data buffers. We + // have some data pending in the RLE + // buffer already, so do a flush. + // Once flushed, we will check again + // if the Subblock Data happens to + // contain a different repeating + // sequence. + subblock_rle_flush(coder); + break; + } + } + } + + // If we now have some data left in coder->subblock, the RLE + // buffer is empty and we must write a regular Subblock Data. + if (coder->subblock.size > 0) { + assert(coder->rle.count == 0); + coder->tmp = coder->subblock.size - 1; + coder->sequence = SEQ_DATA_SIZE_0; + break; + } + + // Check if we should enable Subfilter. + if (coder->subfilter.mode == SUB_SET) { + if (coder->rle.count > 0) + subblock_rle_flush(coder); + else + coder->sequence = SEQ_SUBFILTER_INIT; + break; + } + + // Check if we have just finished Subfiltering. + if (coder->subfilter.mode == SUB_END_MARKER) { + if (coder->rle.count > 0) { + subblock_rle_flush(coder); + break; + } + + write_byte(0x50); + coder->subfilter.mode = SUB_NONE; + if (*out_pos == out_size) + return LZMA_OK; + } + + // Check if we have already written everything. + if (action == LZMA_FINISH && *in_pos == in_size + && coder->subfilter.mode == SUB_NONE) { + if (coder->rle.count > 0) { + subblock_rle_flush(coder); + break; + } + + if (coder->uncompressed_size + == LZMA_VLI_VALUE_UNKNOWN) { + // NOTE: No need to use write_byte() here + // since we are finishing. + out[*out_pos] = 0x10; + ++*out_pos; + } else if (coder->uncompressed_size != 0) { + return LZMA_DATA_ERROR; + } + + return LZMA_STREAM_END; + } + + // Otherwise we have more work to do. + coder->sequence = SEQ_FILL; + break; + + case SEQ_RLE_COUNT_0: + // Make the Data field properly aligned, but only if the data + // chunk to be repeated isn't extremely small. We have four + // bytes for Count and one byte for Size, thus the number five. + if (coder->rle.size >= RLE_MIN_SIZE_FOR_ALIGN + && subblock_align( + coder, out, out_pos, out_size, 5)) + return LZMA_OK; + + assert(coder->rle.count > 0); + + write_byte(0x30 | (coder->tmp & 0x0F)); + + coder->sequence = SEQ_RLE_COUNT_1; + break; + + case SEQ_RLE_COUNT_1: + write_byte(coder->tmp >> 4); + coder->sequence = SEQ_RLE_COUNT_2; + break; + + case SEQ_RLE_COUNT_2: + write_byte(coder->tmp >> 12); + coder->sequence = SEQ_RLE_COUNT_3; + break; + + case SEQ_RLE_COUNT_3: + write_byte(coder->tmp >> 20); + + if (coder->rle.count > REPEAT_COUNT_MAX) + coder->rle.count -= REPEAT_COUNT_MAX; + else + coder->rle.count = 0; + + coder->sequence = SEQ_RLE_SIZE; + break; + + case SEQ_RLE_SIZE: + assert(coder->rle.size >= LZMA_SUBBLOCK_RLE_MIN); + assert(coder->rle.size <= LZMA_SUBBLOCK_RLE_MAX); + write_byte(coder->rle.size - 1); + coder->sequence = SEQ_RLE_DATA; + break; + + case SEQ_RLE_DATA: + bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size, + out, out_pos, out_size); + if (coder->pos < coder->rle.size) + return LZMA_OK; + + coder->alignment.out_pos += coder->rle.size; + + coder->pos = 0; + coder->sequence = SEQ_FLUSH; + break; + + case SEQ_DATA_SIZE_0: + // We need four bytes for the Size field. + if (subblock_align(coder, out, out_pos, out_size, 4)) + return LZMA_OK; + + write_byte(0x20 | (coder->tmp & 0x0F)); + coder->sequence = SEQ_DATA_SIZE_1; + break; + + case SEQ_DATA_SIZE_1: + write_byte(coder->tmp >> 4); + coder->sequence = SEQ_DATA_SIZE_2; + break; + + case SEQ_DATA_SIZE_2: + write_byte(coder->tmp >> 12); + coder->sequence = SEQ_DATA_SIZE_3; + break; + + case SEQ_DATA_SIZE_3: + write_byte(coder->tmp >> 20); + coder->sequence = SEQ_DATA; + break; + + case SEQ_DATA: + bufcpy(coder->subblock.data, &coder->pos, + coder->subblock.size, out, out_pos, out_size); + if (coder->pos < coder->subblock.size) + return LZMA_OK; + + coder->alignment.out_pos += coder->subblock.size; + + coder->subblock.size = 0; + coder->pos = 0; + coder->sequence = SEQ_FLUSH; + break; + + case SEQ_SUBFILTER_INIT: { + assert(coder->subblock.size == 0); + assert(coder->rle.count == 0); + assert(coder->subfilter.mode == SUB_SET); + assert(coder->options != NULL); + + // There must be a filter specified. + if (coder->options->subfilter_options.id + == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_HEADER_ERROR; + + // Initialize a raw encoder to work as a Subfilter. + lzma_options_filter options[2]; + options[0] = coder->options->subfilter_options; + options[1].id = LZMA_VLI_VALUE_UNKNOWN; + + lzma_ret ret = lzma_raw_encoder_init( + &coder->subfilter.subcoder, allocator, + options, LZMA_VLI_VALUE_UNKNOWN, false); + if (ret != LZMA_OK) + return ret; + + // Encode the Filter Flags field into a buffer. This should + // never fail since we have already successfully initialized + // the Subfilter itself. Check it still, and return + // LZMA_PROG_ERROR instead of whatever the ret would say. + ret = lzma_filter_flags_size( + &coder->subfilter.flags_size, options); + assert(ret == LZMA_OK); + if (ret != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->subfilter.flags = lzma_alloc( + coder->subfilter.flags_size, allocator); + if (coder->subfilter.flags == NULL) + return LZMA_MEM_ERROR; + + // Now we have a big-enough buffer. Encode the Filter Flags. + // Like above, this should never fail. + size_t dummy = 0; + ret = lzma_filter_flags_encode(coder->subfilter.flags, + &dummy, coder->subfilter.flags_size, options); + assert(ret == LZMA_OK); + assert(dummy == coder->subfilter.flags_size); + if (ret != LZMA_OK || dummy != coder->subfilter.flags_size) + return LZMA_PROG_ERROR; + + // Write a Subblock indicating a new Subfilter. + write_byte(0x40); + + coder->options->subfilter_mode = LZMA_SUBFILTER_RUN; + coder->subfilter.mode = SUB_RUN; + coder->sequence = SEQ_SUBFILTER_FLAGS; + } + + // Fall through + + case SEQ_SUBFILTER_FLAGS: + // Copy the Filter Flags to the output stream. + bufcpy(coder->subfilter.flags, &coder->pos, + coder->subfilter.flags_size, + out, out_pos, out_size); + if (coder->pos < coder->subfilter.flags_size) + return LZMA_OK; + + lzma_free(coder->subfilter.flags, allocator); + coder->subfilter.flags = NULL; + + coder->pos = 0; + coder->sequence = SEQ_FILL; + break; + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static lzma_ret +subblock_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + if (coder->next.code == NULL) + return subblock_buffer(coder, allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + while (*out_pos < out_size + && (*in_pos < in_size || action == LZMA_FINISH)) { + if (!coder->next_finished + && coder->temp.pos == coder->temp.size) { + coder->temp.pos = 0; + coder->temp.size = 0; + + const lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + coder->temp.buffer, &coder->temp.size, + LZMA_BUFFER_SIZE, action); + if (ret == LZMA_STREAM_END) { + assert(action == LZMA_FINISH); + coder->next_finished = true; + } else if (coder->temp.size == 0 || ret != LZMA_OK) { + return ret; + } + } + + const lzma_ret ret = subblock_buffer(coder, allocator, + coder->temp.buffer, &coder->temp.pos, + coder->temp.size, out, out_pos, out_size, + coder->next_finished ? LZMA_FINISH : LZMA_RUN); + if (ret == LZMA_STREAM_END) { + assert(action == LZMA_FINISH); + assert(coder->next_finished); + return LZMA_STREAM_END; + } + + if (ret != LZMA_OK) + return ret; + } + + return LZMA_OK; +} + + +static void +subblock_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_coder_end(&coder->next, allocator); + lzma_next_coder_end(&coder->subfilter.subcoder, allocator); + lzma_free(coder->subblock.data, allocator); + lzma_free(coder->subfilter.flags, allocator); + return; +} + + +extern lzma_ret +lzma_subblock_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->coder->next = LZMA_NEXT_CODER_INIT; + next->coder->subblock.data = NULL; + next->coder->subblock.limit = 0; + next->coder->subfilter.subcoder = LZMA_NEXT_CODER_INIT; + } else { + lzma_next_coder_end(&next->coder->subfilter.subcoder, + allocator); + lzma_free(next->coder->subfilter.flags, allocator); + } + + next->coder->subfilter.flags = NULL; + + next->coder->next_finished = false; + next->coder->sequence = SEQ_FILL; + next->coder->options = filters[0].options; + next->coder->uncompressed_size = filters[0].uncompressed_size; + next->coder->pos = 0; + + next->coder->alignment.in_pending = 0; + next->coder->alignment.in_pos = 0; + next->coder->alignment.out_pos = 0; + next->coder->subblock.size = 0; + next->coder->rle.count = 0; + next->coder->subfilter.mode = SUB_NONE; + + next->coder->temp.pos = 0; + next->coder->temp.size = 0; + + // Grab some values from the options structure if it is available. + size_t subblock_size_limit; + if (next->coder->options != NULL) { + if (next->coder->options->alignment + < LZMA_SUBBLOCK_ALIGNMENT_MIN + || next->coder->options->alignment + > LZMA_SUBBLOCK_ALIGNMENT_MAX) { + subblock_encoder_end(next->coder, allocator); + return LZMA_HEADER_ERROR; + } + next->coder->alignment.multiple + = next->coder->options->alignment; + subblock_size_limit = next->coder->options->subblock_data_size; + } else { + next->coder->alignment.multiple + = LZMA_SUBBLOCK_ALIGNMENT_DEFAULT; + subblock_size_limit = LZMA_SUBBLOCK_DATA_SIZE_DEFAULT; + } + + { + const lzma_ret ret = subblock_data_size(next->coder, allocator, + subblock_size_limit); + if (ret != LZMA_OK) { + subblock_encoder_end(next->coder, allocator); + return ret; + } + } + + { + const lzma_ret ret = lzma_next_filter_init(&next->coder->next, + allocator, filters + 1); + if (ret != LZMA_OK) { + subblock_encoder_end(next->coder, allocator); + return ret; + } + } + + next->code = &subblock_encode; + next->end = &subblock_encoder_end; + + return LZMA_OK; +} diff --git a/src/liblzma/subblock/subblock_encoder.h b/src/liblzma/subblock/subblock_encoder.h new file mode 100644 index 00000000..3ef9e2e4 --- /dev/null +++ b/src/liblzma/subblock/subblock_encoder.h @@ -0,0 +1,28 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file subblock_encoder.h +/// \brief Encoder of the Subblock filter +// +// Copyright (C) 2007 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_SUBBLOCK_ENCODER_H +#define LZMA_SUBBLOCK_ENCODER_H + +#include "common.h" + +extern lzma_ret lzma_subblock_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +#endif |