*
* Application may modify the values of total_in and total_out as it wants.
* They are updated by liblzma to match the amount of data read and
- * written, but aren't used for anything else.
+ * written but aren't used for anything else except as a possible return
+ * values from lzma_get_progress().
*/
typedef struct {
const uint8_t *next_in; /**< Pointer to the next input byte. */
extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow;
+/**
+ * \brief Get progress information
+ *
+ * In single-threaded mode, applications can get progress information from
+ * strm->total_in and strm->total_out. In multi-threaded mode this is less
+ * useful because a significant amount of both input and output data gets
+ * buffered internally by liblzma. This makes total_in and total_out give
+ * misleading information and also makes the progress indicator updates
+ * non-smooth.
+ *
+ * This function gives realistic progress information also in multi-threaded
+ * mode by taking into account the progress made by each thread. In
+ * single-threaded mode *progress_in and *progress_out are set to
+ * strm->total_in and strm->total_out, respectively.
+ */
+extern LZMA_API(void) lzma_get_progress(lzma_stream *strm,
+ uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow;
+
+
/**
* \brief Get the memory usage of decoder filter chain
*
/// allocator before calling lzma_end().
const lzma_allocator *allocator;
+ /// Amount of uncompressed data that has already been compressed.
+ uint64_t progress_in;
+
+ /// Amount of compressed data that is ready.
+ uint64_t progress_out;
+
/// Block encoder
lzma_next_coder block_encoder;
/// the new input from the application.
worker_thread *thr;
+
+ /// Amount of uncompressed data in Blocks that have already
+ /// been finished.
+ uint64_t progress_in;
+
+ /// Amount of compressed data in Stream Header + Blocks that
+ /// have already been finished.
+ uint64_t progress_out;
+
+
pthread_mutex_t mutex;
mythread_cond cond;
};
static worker_state
worker_encode(worker_thread *thr, worker_state state)
{
+ assert(thr->progress_in == 0);
+ assert(thr->progress_out == 0);
+
// Set the Block options.
thr->block_options = (lzma_block){
.version = 0,
do {
mythread_sync(thr->mutex) {
+ // Store in_pos and out_pos into *thr so that
+ // an application may read them via
+ // lzma_get_progress() to get progress information.
+ //
+ // NOTE: These aren't updated when the encoding
+ // finishes. Instead, the final values are taken
+ // later from thr->outbuf.
+ thr->progress_in = in_pos;
+ thr->progress_out = thr->outbuf->size;
+
while (in_size == thr->in_size
&& thr->state == THR_RUN)
pthread_cond_wait(&thr->cond, &thr->mutex);
state = thr->state;
in_size = thr->in_size;
-
- // TODO? Store in_pos and out_pos into *thr here
- // so that the application may read them via
- // some currently non-existing function to get
- // progress information.
}
// Return if we were asked to stop or exit.
// no errors occurred.
thr->outbuf->finished = state == THR_FINISH;
+ // Update the main progress info.
+ thr->coder->progress_in
+ += thr->outbuf->uncompressed_size;
+ thr->coder->progress_out += thr->outbuf->size;
+ thr->progress_in = 0;
+ thr->progress_out = 0;
+
// Return this thread to the stack of free threads.
thr->next = thr->coder->threads_free;
thr->coder->threads_free = thr;
thr->state = THR_IDLE;
thr->allocator = allocator;
thr->coder = coder;
+ thr->progress_in = 0;
+ thr->progress_out = 0;
thr->block_encoder = LZMA_NEXT_CODER_INIT;
if (mythread_create(&thr->thread_id, &worker_start, thr))
&coder->index_encoder, allocator,
coder->index));
coder->sequence = SEQ_INDEX;
+
+ // Update the progress info to take the Index and
+ // Stream Footer into account. Those are very fast to encode
+ // so in terms of progress information they can be thought
+ // to be ready to be copied out.
+ coder->progress_out += lzma_index_size(coder->index)
+ + LZMA_STREAM_HEADER_SIZE;
}
// Fall through
}
+static void
+get_progress(lzma_coder *coder, uint64_t *progress_in, uint64_t *progress_out)
+{
+ // Lock coder->mutex to prevent finishing threads from moving their
+ // progress info from the worker_thread structure to lzma_coder.
+ mythread_sync(coder->mutex) {
+ *progress_in = coder->progress_in;
+ *progress_out = coder->progress_out;
+
+ for (size_t i = 0; i < coder->threads_initialized; ++i) {
+ mythread_sync(coder->threads[i].mutex) {
+ *progress_in += coder->threads[i].progress_in;
+ *progress_out += coder->threads[i]
+ .progress_out;
+ }
+ }
+ }
+
+ return;
+}
+
+
static lzma_ret
stream_encoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator,
const lzma_mt *options)
next->code = &stream_encode_mt;
next->end = &stream_encoder_mt_end;
+ next->get_progress = &get_progress;
// next->update = &stream_encoder_mt_update;
next->coder->filters[0].id = LZMA_VLI_UNKNOWN;
next->coder->header_pos = 0;
+ // Progress info
+ next->coder->progress_in = 0;
+ next->coder->progress_out = LZMA_STREAM_HEADER_SIZE;
+
return LZMA_OK;
}
}
-/// Get information about position in the stream. This is currently simple,
-/// but it will become more complicated once we have multithreading support.
+/// Get how much uncompressed and compressed data has been processed.
static void
progress_pos(uint64_t *in_pos,
uint64_t *compressed_pos, uint64_t *uncompressed_pos)
{
- *in_pos = progress_strm->total_in;
+ uint64_t out_pos;
+ lzma_get_progress(progress_strm, in_pos, &out_pos);
+
+ // It cannot have processed more input than it has been given.
+ assert(*in_pos <= progress_strm->total_in);
+
+ // It cannot have produced more output than it claims to have ready.
+ assert(out_pos >= progress_strm->total_out);
if (opt_mode == MODE_COMPRESS) {
- *compressed_pos = progress_strm->total_out;
- *uncompressed_pos = progress_strm->total_in;
+ *compressed_pos = out_pos;
+ *uncompressed_pos = *in_pos;
} else {
- *compressed_pos = progress_strm->total_in;
- *uncompressed_pos = progress_strm->total_out;
+ *compressed_pos = *in_pos;
+ *uncompressed_pos = out_pos;
}
return;
projects / thirdparty / xz.git / commitdiff
