1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
///////////////////////////////////////////////////////////////////////////////
//
/// \file 04_compress_easy_mt.c
/// \brief Compress in multi-call mode using LZMA2 in multi-threaded mode
///
/// Usage: ./04_compress_easy_mt < INFILE > OUTFILE
///
/// Example: ./04_compress_easy_mt < foo > foo.xz
//
// Author: Lasse Collin
//
// This file has been put into the public domain.
// You can do whatever you want with this file.
//
///////////////////////////////////////////////////////////////////////////////
#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <lzma.h>
static bool
init_encoder(lzma_stream *strm)
{
// The threaded encoder takes the options as pointer to
// a lzma_mt structure.
lzma_mt mt = {
// No flags are needed.
.flags = 0,
// Let liblzma determine a sane block size.
.block_size = 0,
// Use no timeout for lzma_code() calls by setting timeout
// to zero. That is, sometimes lzma_code() might block for
// a long time (from several seconds to even minutes).
// If this is not OK, for example due to progress indicator
// needing updates, specify a timeout in milliseconds here.
// See the documentation of lzma_mt in lzma/container.h for
// information how to choose a reasonable timeout.
.timeout = 0,
// Use the default preset (6) for LZMA2.
// To use a preset, filters must be set to NULL.
.preset = LZMA_PRESET_DEFAULT,
.filters = NULL,
// Use CRC64 for integrity checking. See also
// 01_compress_easy.c about choosing the integrity check.
.check = LZMA_CHECK_CRC64,
};
// Detect how many threads the CPU supports.
mt.threads = lzma_cputhreads();
// If the number of CPU cores/threads cannot be detected,
// use one thread. Note that this isn't the same as the normal
// single-threaded mode as this will still split the data into
// blocks and use more RAM than the normal single-threaded mode.
// You may want to consider using lzma_easy_encoder() or
// lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
// lzma_cputhreads() returns 0 or 1.
if (mt.threads == 0)
mt.threads = 1;
// If the number of CPU cores/threads exceeds threads_max,
// limit the number of threads to keep memory usage lower.
// The number 8 is arbitrarily chosen and may be too low or
// high depending on the compression preset and the computer
// being used.
//
// FIXME: A better way could be to check the amount of RAM
// (or available RAM) and use lzma_stream_encoder_mt_memusage()
// to determine if the number of threads should be reduced.
const uint32_t threads_max = 8;
if (mt.threads > threads_max)
mt.threads = threads_max;
// Initialize the threaded encoder.
lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
if (ret == LZMA_OK)
return true;
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR:
msg = "Memory allocation failed";
break;
case LZMA_OPTIONS_ERROR:
// We are no longer using a plain preset so this error
// message has been edited accordingly compared to
// 01_compress_easy.c.
msg = "Specified filter chain is not supported";
break;
case LZMA_UNSUPPORTED_CHECK:
msg = "Specified integrity check is not supported";
break;
default:
msg = "Unknown error, possibly a bug";
break;
}
fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
msg, ret);
return false;
}
// This function is identical to the one in 01_compress_easy.c.
static bool
compress(lzma_stream *strm, FILE *infile, FILE *outfile)
{
lzma_action action = LZMA_RUN;
uint8_t inbuf[BUFSIZ];
uint8_t outbuf[BUFSIZ];
strm->next_in = NULL;
strm->avail_in = 0;
strm->next_out = outbuf;
strm->avail_out = sizeof(outbuf);
while (true) {
if (strm->avail_in == 0 && !feof(infile)) {
strm->next_in = inbuf;
strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
infile);
if (ferror(infile)) {
fprintf(stderr, "Read error: %s\n",
strerror(errno));
return false;
}
if (feof(infile))
action = LZMA_FINISH;
}
lzma_ret ret = lzma_code(strm, action);
if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
size_t write_size = sizeof(outbuf) - strm->avail_out;
if (fwrite(outbuf, 1, write_size, outfile)
!= write_size) {
fprintf(stderr, "Write error: %s\n",
strerror(errno));
return false;
}
strm->next_out = outbuf;
strm->avail_out = sizeof(outbuf);
}
if (ret != LZMA_OK) {
if (ret == LZMA_STREAM_END)
return true;
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR:
msg = "Memory allocation failed";
break;
case LZMA_DATA_ERROR:
msg = "File size limits exceeded";
break;
default:
msg = "Unknown error, possibly a bug";
break;
}
fprintf(stderr, "Encoder error: %s (error code %u)\n",
msg, ret);
return false;
}
}
}
extern int
main(void)
{
lzma_stream strm = LZMA_STREAM_INIT;
bool success = init_encoder(&strm);
if (success)
success = compress(&strm, stdin, stdout);
lzma_end(&strm);
if (fclose(stdout)) {
fprintf(stderr, "Write error: %s\n", strerror(errno));
success = false;
}
return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
|
