From 274b60e6e6571ccce5982e36ac82eabe606c1364 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 27 Aug 2018 17:08:44 -0700
Subject: [PATCH] largeNbDicts can compress and compare dict vs noDict

---
 contrib/largeNbDicts/Makefile       |   2 +
 contrib/largeNbDicts/largeNbDicts   | Bin 13626 -> 14034 bytes
 contrib/largeNbDicts/largeNbDicts.c | 146 +++++++++++++++++++++-------
 3 files changed, 115 insertions(+), 33 deletions(-)

diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile
index 082f0102a..026d76f12 100644
--- a/contrib/largeNbDicts/Makefile
+++ b/contrib/largeNbDicts/Makefile
@@ -21,6 +21,8 @@ CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 
 default: largeNbDicts
 
+all : largeNbDicts
+
 largeNbDicts: LDFLAGS += -lzstd
 largeNbDicts: largeNbDicts.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts
index 40416f050fa594cb085444ebb1ca433629d59318..c057a2b78aa551a2de831b6e304f8747a6ea3d0f 100755
GIT binary patch
literal 14034
zc-rk-eRNdC6`%cz3kbVV)KC!~HnE{%$f8)Vgl5Suyu}v`lBfm3vY%v6H@i8zZ%a_I
z!DL%rpRqY@@o?H|qg6|-t*M_#1!F;o=TxN#r&y{`Ywb&sLkg!zk+QvaX7=kPP|xY<
ze>*2{=HB_u+_`h-&V4WQ-oAbA#D&WUF_;MHG!jDWaP?XUNfUu!ngW*ym)G0qT;g8o
zzWp{bA{oekPbjC#h;o|TcQuXDU(j{nJm^M19a6@ty<R!gA?KTokY{7^y2j;sbD>-_
z=`3Wu-lUxJCod|m`}0iReL6mytXy7Mkvwl}DB2!Mj3Uo`HS7OD0dKsdet~!_DPL5-
zVJ@$#fHz-a9Y9`EPDEnsE+VgdDwB7&DbE)s;|O`F*!oB;=#9j}@loXcU^<hxwtzSO
z7eAS2KshdFle5?#sNL<w{*mR;PfYw`daaAgMS6}d&sxBn3SeB%D~kHZqTb|&Hh(<o
zZI=@x<XO2qdjYSPcV&^hsP9tp%DKF41$?1AoNu|#Zfu`Qo@KYXU7qY3)adM1!-|7t
zc{zptd@aO*8r}|VW39>i7Oe(;J9N+-gmiF!Kt0;$5CTg}CeI~g0C3!XjRihq<TgN=
zS6yQL*WN_P4Wrag;7fivT(eWj#O!FqKRdV~8szxZT#pm;mzdu9XYHzO{wbc;S@(Rg
zB0Kf5`1hp(LlCxWI<eRAl;Y8FlsYaEH8-0K)J|;Vc)%B(9dDOs-<_0$R1vt4u69&|
zN)bQpKd-R=oDa?U{;b1_4M{oFR=tRKq3JrV-`Y8>oXL5#ZYh;gDV0(wl~O5{QYroK
zsaaJ1>b%o=hqKw)v_kCqxLj0Di^|6RqUr|HJt+0ari|l&qzsFnoc1XDq@Ie`v7e;;
zP3wn8T#xcPl21@_Peb~Iyrd!hp}fd5|9#v2bK#j$>~Y&gm5A!bLD8^JQZ9&@w`|?V
zL4>F0&I`&vy8e8b*mdr*WS!%Hn0~{S{xu*&zoR3Z)E13nBi#ZeXtZD4S4SUSp|rA+
zPGV>$(lg(39O(5Zu0x>7)PthoFw$0!vihKvGj?R`oGC}9oHGJ40%xj_sp3oxGBuo;
zhs-=+#IE03#jd?)Q{M%pWx>;P(?L&9(;?C;Dn?P6MqBK)(pH%_QhyicpIyIHoVo8-
z#r5yAw<`CZKZ~=GlqyMCYn7A^yV$j-e3f&xv&Ff_>2299_SD{kQ|_7DNsIBDq1e#_
zv(zSz#fsY14E8a2J;GZMJ~f7ra5!9To8CQgeH%T>+aBd0EU^dHA$>$tE$1hoW#`G$
zK-dzH;usK>KWe+ipxr+amG`yZfRN*>T?*(lcOedgkK>L3?MKk$Riw3_kD+dWYrl9G
z*1*b1$_JA2FQ;-?n+~W5y_KU{BarEVR9%Bo?NQcPJ^L2hS@U6-zO{JzMmE1qbhhIy
zIPfoVev`(~>Hme+&-F9-Ur_9L4&l>QI{i`R58l1$<6iU=oc(#R=Y_^hCWC67w4z$J
z^fP!?@>049sw>GbA+>l8VM}ktKKa(K@z$G0Yn_ToYO3GUbs-}MLA~_lINVV!Vnqw>
zYe;A01~99(-pXu}J|YXd3{<D1l5z?xrY!|Xnm;U$$A0y|xE*I4M<gYqO`}4lRx~Jb
zg%vaV@y~nqxhm*}K*Guha|PWQoA%&j(Wum<qxu6_M8iJH8BzIEQfp6wz8>|fp!us~
zvaaylWjOtvYw7gg*PgxrrJ81}dz%QR=>CO?b~|mmucsC2!#4l;Ew~Q0)#D$g<-Kp5
zve_+<QhH_&TJn_bYp=mD-7rkQLAwpdP}A*HD9k83fJ)_wid1106>|lsAplQ(OsoHt
zDk3mN2B;!V6UuqG2I|-}A|U@|=CCjott{)Zy=639iAqbqQ+ZEP{^GGc^B!9Ky|U~|
z*}5m0&^LE>&=FcT!_^79c5_xkTCX;A7OFQ7>LleO?a6agIb^9~9~p6tA%3Y3S7BYx
zPAhGBMpT=(iORc@>h9N;pNA^-1w<OsXKda3X{VM!N!i#dsZZFUeus$$#!qmJ4=q&Y
zb~?arXe+5Z>!?m!L@ct7pl;lcQCDg118oOjq(M<_>Gvq_YIZc2@*0NS?niMj)pBAi
z4F_S-7JCEN@y#-}j)%(FI)-gkX|{fAaQ&WuC8avSB7Vn0WAt`v3OeY<tvIln9~cMy
zkuf`{M>%Ti7TLI|v1~`EU&#%9J0E)G=tFl}aa5P`fi@8acAUZ6#Cv8~J=-&{b1?%T
z#xjnHv3!n+8350-TvbM+$2Yl%bJ|<bacR$j6KF8)U31<C24Qz%VDmVRgl*GZIpep{
zHs+($_z$3&GrOhv#}mgS<<p}01FC4ZLu0XPx7Cw*Gj$TT^4HCDUAAj;%v7Sbl2z>2
zmNR0jCNiQ|`zj*_w3`_*sNKMbZCZtyZqiMAG``ad?jBY;!$lSvAHRpo^aq^VJ0==F
zhiU!XFEChxV#g#r(t4i3zh|(S!Dg`5j;j#f!r*5a{8uxDzl`v7gulVy?=yHWgTILI
zbqL?dV41-`V{jG1vk;!g;ARGIX0U+pY=oyU_+|!27;Hm$4#MY*6uyeVOBpO7jAyCZ
z5e5foFq?ZlgQLLhcn8h$iV>Xy*H`~M1b1k3>4pK1a!6ECgZOOJF#k<kT0pTYjpzn@
z>2|GVu43!P8<FFP+mR8wHrk2YqVorg0yiBN={}r{<$$POj^%bqH6KR*QXA~rIRlMQ
z=+#q<^tLHC82-k+Y{&8D&nnOPG7lNrV|j`D3{;{sFLC>E6DTYxXW>p<RqRT;nMpsP
z794Lylb%PLZKe-*wU0CSFoTECmChg>VQ`wkFEjXC2oE8=l)+&J|B%6tAp9}H*8`ki
zk2_Y-`_tUpFgmLfpx-%I!CiPFx&v>Y8ZL%MT?mqf&Dk5*Il|6qVP~uv<^Bh&tDKh9
z8@n=w`zj^%9;@5&VOPc|H+N;ssmY>xj~%4F2GX=c1g$04iRv|P(HJ)l6xdL#z)#p#
zhc0od0Nl;B4N&KN1>xg%$C=}1QB9WP?%!}WlW{4>a<}H*Oyz?hH&fY5^Y7982Q+_(
z=38n0Lz@4X=8w?4kLHij{BfE;N%L(qe~RW$)BIO7@2B~*G=HAvFIwoce9tu&Ae<{Q
zmH!0odt2!<^+ynETdzUtPe^?ksRKyiq4v-qQte3XLh9Q{J&zPxV(2NPUP7u5DYW3w
zLrCFs|Ih<S8Ne_@_ak)$Qg}CQorzS8j)qhaDK}Cpk!nT??*Od<q!v>RkXnG$H<6ly
z)PqRjZMOBNNKFH(@IRiKdbr%h{{>+kEqQ(a!eo0iA}>mX!=VI;`V#9x%lxiLKu%T%
zNX)+|8V`IuDJ-lN93&Q6PuWH}AuOmBl8Jyj79`QQFBplf6N1nu5|8;38-&WFFqQ-o
zA)j0f%o5^ZLB=6uAsM+lBvhsZ{{}gfYaVjt+7j%>v=V023d@=$iOg8JqRHirB;AQb
zJW+)WR)Cb5ITb3C)z#G?%O8pbS_$aK`d7Ehc^Pb+AZjo8nRrl&X;bY83k{1v0CvTh
zvi`|x)-ah9LQN(_V(lr+<v5)PvlIwx^L31vUN9*)7H9$NB?rh)zF2vz&vQ+2*%w6<
zEQ`B1P(uRow)R9QnM8y5WFZ>*MktD!tdGd8anQG5oo*3shb4;FR#wjm3qn;^-I=%?
zIoZWh!Ez}kIzPM5CUIRT7D_OWfa+YINGLgri_F>|O&Ds6gSfFo@cVE?-?|XU1{Pc^
z9EpZvzP6CC>IPTA*ew9x;35fsPFLo;BWw*$&8=Naop^MR1Ojr$MLdsnVZO)x7VTdj
zk48gO9TG&n3YL?yt1w^YZ^>1&w{cVXd!q`zLn}Bxah%!k`@1X#e5<{HunXTv6Hc4x
zyDJM3*gHMLG3YdGB8I-P#5fEOnKOv-v%IV)llfcaY1y~Syqv?!1-x9$%SK+V<YkbT
zF<y4?@_t@Ez{`ht*~iPLc=<dpckyzNmj`(Hr>j5PR|ln3N~KgvrBq6#R7#~(N~Kgv
zrBq6#M99Qz%h%#p&XelA_~Cjmo|4<i@&xIv!%u73r|kYnEEs|+Y@aBfSm#~o^3*qZ
z<%BO1Tbziu@ngbXk`NHYLy_EROFR_|B2uMC>g(kW1Zwnvm!A97<Jk$~^Kv*nQ^|I#
z(^Y4ii^cV>Y<*+4K9rOrZCM0QYY@FFi-r@SysWi_wK++x*;;<^gLUoHe7<Bdl#sm%
zIfiu|P!|qF<4HO~rvce`dx&D2Dc7Ee#N;sYy_5%ysjaqhj&|&$%=$z`rtCIk+kDYz
zJb-LJvPsz|S9q4)=5pVB6IO0#m5D%GJ6kk*667ToV)3@MdP5zdKuQjITVXg@w)(=O
z-X$YWTqQqQ-UbUz+KlvgsdE}+$7T>>r01ytU(icW;z+BB3|9iycOzuOH}YXSryVnO
z8hed>V|z__aK6Pr$OO2KO=RacPjlX0&UhTeW?)}vBY!333F(`{_!*oZo|OO91M<i0
zjK3v|^Y(R|&;BZ;fyvDs9R+oKhmKcu{1ZC<MIAq=<Nv7RPw9B0p_ref;|0d#elSt3
z<LB#mr;ZnO{Bj+CmyTbf<HI^WrsMC`@!dKeKgt+G=*Rc;7XkQP{CL8?=7%Mr-?mT0
zdBgQN!hUgqYmUDVz;EaAvwS?>#IO7DOcY<pOonS36JPiLkS`GLDEq|g&$+sny-V<p
z*eqVUQ@n*c>LS5%v@khJfV_BU91#>phQpD8@M0lQAN=%TuQ=cphq{9MMmFB@K$nYf
mJhZWxmJMdvC{`T4vT<v~-G|SEnGYnz+ZIQnAVqhskLf>uRQ9z1

literal 13626
zc-rk-eQ+Da6<>Wg5fmba@?nMm7ba$$OsZHVh9tzPl}K<3=U`&zqa@)XS*KW3mOOW-
ziNg@aU~49as@7#nCo`0plt~L^XmOcBQxja5gg7uva7}=e@L}2jCq9PHhLjJh@9nN+
zNp?d2==869liT-x`*z>Gef!=yvv+#o%MT}J5@K@@(qkutRKwL@MM#<m1nFY9M7U(R
z#dm$+=D^w;$rQ4YZzr@<XGc42fsJj`%oi#qa16c?aEHv`-LkAJ-TLIPN_+JeFnd?d
zoW#yjyH3(m3T0W-Q(^72_SVl~_M#QIKrU`?MwvaiLy31O>NNKD^ZA1%*nUR;+LH-Q
zKdpZ~moj^GCD?g}eE@r!uErAEPGj#OZqH$##HD1PVlS209!o^zSR$I7#@-{BGkYy1
z*#0efvc!gV{Ju6{c|Op%SC{8U0poT%_{IF%mekAqoZg<R1UpL9xSzAj<|pE^wxctg
zjLTiRI>nxg+p8|YmG;gnvlkDY$zBb&x2pt~+QappUlBJ}r&4gu4FP|!um|-OajR!+
zdDWzy(tN%GQmCGf!`Qg$Cv%H-18#u{Jzhe#@_eX)%k(n-^;Lv4f&M!sdtRU_etq{0
z``-ccYp3a-#kc$%xE7@}bx}MPUKH68k8u4Z+>gD-A3ONqP0RM)^TfWlm#@C}k$Hus
zpN;<(lmvpXs_DU5WKQ|r@Gec<BCgQsu+ccNJ15&i@kPllebH^29-)rFg|f%%0heN7
zI)6#&{O_M|PR?f&T6bu=(&<^rr%<|*`#0`o(Vx#TM%N;XEV9TVi!8FpB8x1t$Rdj@
z`v1^6(Rkap!MEPG&ezr|_I^+!8lQ>A&QUQF05Nb_%Ic21cT6%q7Qgr`XpBhNdC%ZH
z$#~annn_5|cm~A=DjsZ3f2d#IoIati3@&}ceOH5x5HTC57BfW5>^v;mMkHfG%pY<0
zy$>dW*$orMC%tdY6nnp%sWo}W#PqZ7^dCTzd!F9mBXev9yVG;P!J|=eq=_DO0CzD?
z7m<4k<@ravWBozH{}OmI=cs5qhO#SYY(DDZpc+9n2Q>(4I1msB9MmDG<DecvJqJq=
zECC?)KJ5~F51mT=6r9$DV0Q7*V7Bcg(k~iz(YTC`IOL+E^3SH;6PKRazFM3=a)aUj
z=F|<w?O&h5)ksF2WNdXwMt8N?d$4AcZ?kWU?-rlD<$#!NJdR7wHjGntt_f@~i{~c4
zs=|)O{Y?E=rhWz0_oDhPP)DOtkNfhzY4-Qd59)ABjIX53_r@TC!OTvy@_U%>9TtsO
z&2`lJ<D(P#ylA{>M)2Hx*qz>NC&c}y^ik-pB_9*gh)25abiiIV`f5JfGyUj`pj*m3
zUZ2nBgS``Zci#*;z2V?Ybm>j=MJMLaNOKxg@7r+Wzc62S^1M2s3qv-Rcarg`Wc=G4
z0F|`#V|^AT$Zr7zye9+R6OxfPdr@Wl8vT-t$B!{K4h-zzlHGm30b9|aJ8>xL4p)g7
zGxz4I7RqlQ{?z@$sx}&khNplH*vu79%)j(FU8sK^By^fCI+Hy{yQFz8(QwWavrl!y
z>W1)y%Y5K#9FWEiD5Qr(h)mx-xZ=#ZT|VO#9{LC227ku!82@#n=1~VBf1&e#;=rXJ
zl8n!S?nmFm0F6QqHFGPPF#-c{LBeg>1J`ays6E^_=*%?lF7;|_9z$@_oJD=QAEB5&
z?(S={7jvMu3XkxqBj$}oC$=yrn$suY9{b#VyRpzR%{_F9utAcc3`@p8M5Fnz1Q~6<
zXs7#yTk@$Lvq9e;%rs-dt_Ax;c&0gs+h|UJ_DPg-)wmVz4Z~6Qd=}IZ8fQ#i?%L(f
zLV7+?m}MLj?>vaPTv9YtA!T;<<nRgQJ&8_iLVw(kj-aKe=uQmdes6TJ{od-B-0z5k
z?)PCQ?l)}FsJR-Z4mt>5<~yd71$}gNcKJ8xoav)$eFmO+?!F7*-jKNZRo>%#_;DL*
z{jcMW>34|v1Jcs3)PG9G`(ox=a}=wv_kb&ye<Ae|KIn4Tk=Z$F4uAkQG7wgHVqys#
zvC*>Wz&*?C+-DB)(s+VR88Cmxgon(%RFgM<X=6{#p99a0VMw!$VJBVR&uGb=dn=&9
zCvg#L=_9)FOIC_1Q{&bR9z*q2OnscGTbX)3L}hRs)#o$yex_c`)EA@rT~z;zQ1$&x
zeF0P7hicqj^X0<xX?-hwBsKg1t35LdrrZOAhGrGBWt1kv3NW(EQOJddp>H2*H~DVa
za-i_M>&@F!XNj3RTsiw!`Fz28<}$GUmZM;Oak+K1XnjG^x>RO8U%MYP`Pw1c4%7B&
z+8(6sA=)0M?Q^t!p0-D5J4)Lz+8(9t%d|a4+gEA(8njaO(q>@3)_iRv@cs_^x9uI+
z_zrw$m3s}PYfu_P3Ex}h4x@yxzH&n-rBV7LO824kFiP|<>K>G^E^@y>>F+507$w}O
z++8SrjuJkuI`G{<E`bsjXD&iLK<Q?b7NXRG(zPgEmuJOVp!!vC1<L=Ifqj(o{(%)*
zS3IV#Ohu!LO5!1Po3bYCkG1QXr=29iE91%bA8EphM!`!G%61A{bX8d1C}?VXAQ2()
zWGE6#Y!hlVkH;gNO$0?%ld7O9tUDTuE1&~KJ9Ozv1m_81dOIN*6|~rGp!ayALi0)(
z6jl=L9R!BZJ29^`4;!uta~p*<>m-R#mvuBBOLV1lX!WE}n-apzTQJ)h>gJ}QcZaT^
z0})L}*9%4fp7AEsU%W>Iq7h3bLh6o%Xj;b@=~Jw;Wlu8+bla4KqOyLNQB=hgZK06T
zXg7}lx~_C4>8#Rmo(Q5z=Y()50#ifV6k!<#9U_d8PJ}uYfNGdW+mAcBDjAO})D&S>
zHetIayrpQ3zNIU?rYhz5ehJ@lFDC5Tz?CTD1Nj^naXx$JYU_djy38eZ+ut@;o$;mp
z^ZYIK%bbsM{vPL_b3Vbji@!BKkMlX4U&(nr=gT<vbH0Z2jhu%$k6-%VeN|wQMHX3P
zkwq3+WRXP{S!9t#7JWO-zM^I;e&u{#lZ+p(N0KSMi`=A={wDmimVL?|jwK=rbm4fl
zW_FW&vp=}1P1e;=EODKh?BvH%{X`W|+1lop+moGLs-kHtlc_`mHFZS|J@QzEM;_Q;
zU*7L8^e-v&E1DkbENEI|w<)M?DrlpsGHFXJ?JZhrDfGs)0G|F}-$uIJkftfBE~|P1
z`?f+~v^}2G=p8x`CA$==>O#<^#u9oIVK+iNIz!N7V~Eb$MZtD8rc>CDurm~oC)*M3
zMyTl_eO_?QjsC#Zi!r1&nN$&{+}R;3-Aa2(SL6<`4B_;a9{Dwd$+AueH(76|hdKib
z2(i;sI4Km7=@ApT$wB%(sJjN5o+bR`aKX4^&si<>^v7*uUq0ix%7qkz{t|}ga$J4U
z<gW?b{J;sBhvUMpE3RdBibphSD{!;|cUR!MEARsq_^}H7bOjz|xcEbYH!5(h0)JA0
zPgP)>tsJ{5@T>}aVFkXd0xzt<g<r7X*Y@}-_AEj_NT*-P<G14HVtfhv)|=KNVPDDP
z7ycLU{lXLYi<r5l|Ajq)$kXJV94=;*%no{Gnt5_*KAz?gFkSIaE&Hhjf4WkyEb_|o
jE-&ozk}i3k73!K-bg`Cs6|?d!)NP?)%PV!t(-7!C@XCQ?

diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c
index 749d9660d..536e45ffe 100644
--- a/contrib/largeNbDicts/largeNbDicts.c
+++ b/contrib/largeNbDicts/largeNbDicts.c
@@ -159,6 +159,33 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize)
 }
 
 
+/*---  dictionary creation  ---*/
+
+buffer_t createDictionary(const char* dictionary,
+                        const void* srcBuffer, size_t* srcBlockSizes, unsigned nbBlocks)
+{
+    if (dictionary) {
+        DISPLAYLEVEL(3, "loading dictionary %s \n", dictionary);
+        return createBuffer_fromFile(dictionary);
+    } else {
+        DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE);
+        void* const dictBuffer = malloc(DICTSIZE);
+        assert(dictBuffer != NULL);
+
+        size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE,
+                                                    srcBuffer,
+                                                    srcBlockSizes,
+                                                    nbBlocks);
+        assert(!ZSTD_isError(dictSize));
+
+        buffer_t result;
+        result.ptr = dictBuffer;
+        result.capacity = DICTSIZE;
+        result.size = dictSize;
+        return result;
+    }
+}
+
 
 /*---  ddict_collection_t  ---*/
 
@@ -181,6 +208,7 @@ static void freeDDictCollection(ddict_collection_t ddictc)
 static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict)
 {
     ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*));
+    assert(ddicts != NULL);
     if (ddicts==NULL) return kNullDDictCollection;
     for (size_t dictNb=0; dictNb < nbDDict; dictNb++) {
         ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize);
@@ -193,29 +221,64 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
 }
 
 
+/* ---   Compression  --- */
 
-/*---  Benchmark  --- */
+/* compressBlocks() :
+ * @return : total compressed size of all blocks,
+ *        or 0 if error.
+ */
+static size_t compressBlocks(buffer_collection_t dstBlockBuffers, buffer_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel)
+{
+    size_t const nbBlocks = srcBlockBuffers.nbBuffers;
+    assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers);
+
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    assert(cctx != NULL);
+
+    size_t totalCSize = 0;
+    for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) {
+        size_t cBlockSize;
+        if (cdict == NULL) {
+            cBlockSize = ZSTD_compressCCtx(cctx,
+                            dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb],
+                            srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb],
+                            cLevel);
+            assert(!ZSTD_isError(cBlockSize));
+        } else {
+            cBlockSize = ZSTD_compress_usingCDict(cctx,
+                            dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb],
+                            srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb],
+                            cdict);
+            assert(!ZSTD_isError(cBlockSize));
+        }
+        totalCSize += cBlockSize;
+    }
+    return totalCSize;
+}
+
+
+/* ---  Benchmark  --- */
 
 
 /* bench() :
+ * fileName : file to load for benchmarking purpose
+ * dictionary : optional (can be NULL), file to load as dictionary,
+ *              if none provided : will be calculated on the fly by the program.
  * @return : 0 is success, 1+ otherwise */
-int bench(const char* fileName)
+int bench(const char* fileName, const char* dictionary)
 {
     int result = 0;
 
     DISPLAYLEVEL(3, "loading %s... \n", fileName);
     buffer_t const srcBuffer = createBuffer_fromFile(fileName);
-    if (srcBuffer.ptr == NULL) {
-        DISPLAYLEVEL(1," error reading file %s \n", fileName);
-        return 1;
-    }
+    assert(srcBuffer.ptr != NULL);
     DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n",
                     (double)(srcBuffer.size) / (1 MB));
 
     buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE);
     assert(srcBlockBuffers.buffers != NULL);
     unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers;
-    DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n",
+    DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n",
                     nbBlocks, BLOCKSIZE);
 
     size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE);
@@ -230,36 +293,44 @@ int bench(const char* fileName)
     buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize);
     assert(dstBlockBuffers.buffers != NULL);
 
-    DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE);
-    void* const dictBuffer = malloc(DICTSIZE);
-    if (dictBuffer == NULL) { result = 1; goto _cleanup; }
-
-    size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE,
-                                                srcBuffer.ptr,
-                                                srcBlockBuffers.capacities,
-                                                nbBlocks);
-    if (ZSTD_isError(dictSize)) {
-        DISPLAYLEVEL(1, "error creating dictionary \n");
-        result = 1;
-        goto _cleanup;
-    }
+    /* dictionary determination */
+    buffer_t const dictBuffer = createDictionary(dictionary,
+                                srcBuffer.ptr,
+                                srcBlockBuffers.capacities, nbBlocks);
+    assert(dictBuffer.ptr != NULL);
+
+    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL);
+    assert(cdict != NULL);
+
+    size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL);
+    assert(cTotalSizeNoDict != 0);
+    DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f  (%u bytes) \n",
+                    COMP_LEVEL,
+                    (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
+
+    size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL);
+    assert(cTotalSize != 0);
+    DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f  (%u bytes) \n",
+                    (unsigned)dictBuffer.size,
+                    (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize);
 
-    size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
+    size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
     size_t const allDictMem = dictMem * nbBlocks;
     DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
                     nbBlocks, (double)allDictMem / (1 MB));
 
-    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL);
-    do {
-        ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks);
-        assert(dictionaries.ddicts != NULL);
+    ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks);
+    assert(dictionaries.ddicts != NULL);
 
-        freeDDictCollection(dictionaries);
-    } while(0);
-    ZSTD_freeCDict(cdict);
 
-_cleanup:
-    free(dictBuffer);
+
+    //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);;
+
+
+
+    freeDDictCollection(dictionaries);
+    ZSTD_freeCDict(cdict);
+    freeBuffer(dictBuffer);
     freeCollection(dstBlockBuffers);
     freeBuffer(dstBuffer);
     freeCollection(srcBlockBuffers);
@@ -276,7 +347,7 @@ _cleanup:
 int bad_usage(const char* exeName)
 {
     DISPLAY (" bad usage : \n");
-    DISPLAY (" %s filename \n", exeName);
+    DISPLAY (" %s filename [-D dictionary] \n", exeName);
     return 1;
 }
 
@@ -284,6 +355,15 @@ int main (int argc, const char** argv)
 {
     const char* const exeName = argv[0];
 
-    if (argc != 2) return bad_usage(exeName);
-    return bench(argv[1]);
+    if (argc < 2) return bad_usage(exeName);
+    const char* const fileName = argv[1];
+
+    const char* dictionary = NULL;
+    if (argc > 2) {
+        if (argc != 4) return bad_usage(exeName);
+        if (strcmp(argv[2], "-D")) return bad_usage(exeName);
+        dictionary = argv[3];
+    }
+
+    return bench(fileName, dictionary);
 }
-- 
2.47.3