Add ldm options to 'man zstd'

diff --git a/lib/zstd.h b/lib/zstd.h
index e7d4fbdf476dc187f696f61bd156c1bc35dbb4ec..4879c4a63183a309f1692d9e19678284566823ae 100644 (file)
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -979,24 +979,34 @@ typedef enum {
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
                               * even when referencing into Dictionary content (default:0) */
-    ZSTD_p_enableLongDistanceMatching,  /* Enable long distance matching. This increases the memory
-                                         * usage as well as window size. Note: setting this
-                                         * parameter resets all the LDM parameters as well as
-                                         * ZSTD_p_windowLog. It should be set after
+    ZSTD_p_enableLongDistanceMatching,  /* Enable long distance matching.
+                                         * This parameter is designed to improve the compression
+                                         * ratio for large inputs with long distance matches.
+                                         * This increases the memory usage as well as window size.
+                                         * Note: setting this parameter sets all the LDM parameters
+                                         * as well as ZSTD_p_windowLog. It should be set after
                                          * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
                                          * other LDM parameters. Setting the compression level
                                          * after this parameter overrides the window log, though LDM
                                          * will remain enabled until explicitly disabled. */
-    ZSTD_p_ldmHashLog,   /* Size of the table for long distance matching.
-                          * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. */
+    ZSTD_p_ldmHashLog,   /* Size of the table for long distance matching, as a power of 2.
+                          * Larger values increase memory usage and compression ratio, but decrease
+                          * compression speed.
+                          * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+                          * (default: 20). */
     ZSTD_p_ldmMinMatch,  /* Minimum size of searched matches for long distance matcher.
+                          * Larger/too small values usually decrease compression ratio.
                           * Must be clamped between ZSTD_LDM_SEARCHLENGTH_MIN
-                          * and ZSTD_LDM_SEARCHLENGTH_MAX. */
-    ZSTD_p_ldmBucketSizeLog,  /* Log size of each bucket in the hash table for collision resolution.
-                               * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. */
+                          * and ZSTD_LDM_SEARCHLENGTH_MAX (default: 64). */
+    ZSTD_p_ldmBucketSizeLog,  /* Log size of each bucket in the LDM hash table for collision resolution.
+                               * Larger values usually improve collision resolution but may decrease
+                               * compression speed.
+                               * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). */
     ZSTD_p_ldmHashEveryLog,  /* Frequency of inserting/looking up entries in the LDM hash table.
                               * The default is MAX(0, (windowLog - ldmHashLog)) to
                               * optimize hash table usage.
+                              * Larger values improve compression speed. Deviating far from the
+                              * default value will likely result in a decrease in compression ratio.
                               * Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */
 
 } ZSTD_cParameter;

diff --git a/programs/zstd.1 b/programs/zstd.1
index 5a91eea281f49f3f321c6307b64e35e72a764677..13c804ae5fd5df5d168af5862b4ef3d00dabc2c6 100644 (file)
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "August 2017" "zstd 1.3.1" "User Commands"
+.TH "ZSTD" "1" "September 2017" "zstd 1.3.1" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -104,6 +104,10 @@ Display information related to a zstd compressed file, such as size, ratio, and
 unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
 .
 .TP
+\fB\-\-long\fR
+enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
+.
+.TP
 \fB\-T#\fR, \fB\-\-threads=#\fR
 Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==256\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
@@ -322,6 +326,58 @@ Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This pa
 .IP
 The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
 .
+.TP
+\fBldmHashLog\fR=\fIldmHlog\fR, \fBldmHlog\fR=\fIldmHlog\fR
+Specify the maximum size for a hash table used for long distance matching\.
+.
+.IP
+This option is ignored unless long distance matching is enabled\.
+.
+.IP
+Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
+.
+.IP
+The minimum \fIldmHlog\fR is 6 and the maximum is 26 (default: 20)\.
+.
+.TP
+\fBldmSearchLength\fR=\fIldmSlen\fR, \fBldmSlen\fR=\fIldmSlen\fR
+Specify the minimum searched length of a match for long distance matching\.
+.
+.IP
+This option is ignored unless long distance matching is enabled\.
+.
+.IP
+Larger/very small values usually decrease compression ratio\.
+.
+.IP
+The minumum \fIldmSlen\fR is 4 and the maximum is 4096 (default: 64)\.
+.
+.TP
+\fBldmBucketSizeLog\fR=\fIldmBucketSizeLog\fR
+Specify the size of each bucket for the hash table used for long distance matching\.
+.
+.IP
+This option is ignored unless long distance matching is enabled\.
+.
+.IP
+Larger bucket sizes improve collision resolution but decrease compression speed\.
+.
+.IP
+The minimum \fIldmBucketSizeLog\fR is 0 and the maximum is 8 (default: 3)\.
+.
+.TP
+\fBldmHashEveryLog\fR=\fIldmHashEveryLog\fR
+Specify the frequency of inserting entries into the long distance matching hash table\.
+.
+.IP
+This option is ignored unless long distance matching is enabled\.
+.
+.IP
+Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
+.
+.IP
+The default value is \fBwLog \- ldmHlog\fR\.
+.
 .SS "\-B#:"
 Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
 .

diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 4310afa1aaf8397014c1daeee2f21d71adc1ae06..5f6aa4500f721105ce0f4e8aecd558f7e34af22b 100644 (file)
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -105,6 +105,12 @@ the last one takes effect.
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
+* `--long`:
+    enables long distance matching.
+    This increases the window size (`windowLog`) and memory usage for both the
+    compressor and decompressor. This setting is designed to improve the
+    compression ratio for files with long matches at a large distance
+    (up to the maximum window size, 128 MiB).
 * `-T#`, `--threads=#`:
     Compress using `#` threads (default: 1).
     If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -327,6 +333,47 @@ The list of available _options_:
     Default _ovlog_ is 6, which means "reload `windowSize / 8`".
     Exception : the maximum compression level (22) has a default _ovlog_ of 9.
 
+- `ldmHashLog`=_ldmHlog_, `ldmHlog`=_ldmHlog_:
+    Specify the maximum size for a hash table used for long distance matching.
+
+    This option is ignored unless long distance matching is enabled.
+
+    Bigger hash tables usually improve compression ratio at the expense of more
+    memory during compression and a decrease in compression speed.
+
+    The minimum _ldmHlog_ is 6 and the maximum is 26 (default: 20).
+
+- `ldmSearchLength`=_ldmSlen_, `ldmSlen`=_ldmSlen_:
+    Specify the minimum searched length of a match for long distance matching.
+
+    This option is ignored unless long distance matching is enabled.
+
+    Larger/very small values usually decrease compression ratio.
+
+    The minumum _ldmSlen_ is 4 and the maximum is 4096 (default: 64).
+
+- `ldmBucketSizeLog`=_ldmBucketSizeLog_:
+    Specify the size of each bucket for the hash table used for long distance
+    matching.
+
+    This option is ignored unless long distance matching is enabled.
+
+    Larger bucket sizes improve collision resolution but decrease compression
+    speed.
+
+    The minimum _ldmBucketSizeLog_ is 0 and the maximum is 8 (default: 3).
+
+- `ldmHashEveryLog`=_ldmHashEveryLog_:
+    Specify the frequency of inserting entries into the long distance matching
+    hash table.
+
+    This option is ignored unless long distance matching is enabled.
+
+    Larger values will improve compression speed. Deviating far from the
+    default value will likely result in a decrease in compression ratio.
+
+    The default value is `wLog - ldmHlog`.
+
 ### -B#:
 Select the size of each compression job.
 This parameter is available only when multi-threading is enabled.