From 4b222712eaab3311d8202c03f9f56b0f95f40522 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Wed, 24 Jan 2018 15:45:26 +0100 Subject: [PATCH 1/6] Updated melanoma example --- src/code-examples/exampleMelanomaTM.m | 57 ++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/code-examples/exampleMelanomaTM.m b/src/code-examples/exampleMelanomaTM.m index f904067..c3c17f8 100644 --- a/src/code-examples/exampleMelanomaTM.m +++ b/src/code-examples/exampleMelanomaTM.m @@ -226,4 +226,59 @@ fprintf('SVORIM+SVOREX+POM Accuracy: %f\n', CCR.calculateMetric(test.targets,info.predictedTest)); fprintf('SVORIM+SVOREX+POM MAE: %f\n', MAE.calculateMetric(test.targets,info.predictedTest)); -scatter(newTrain.patterns(:,1),newTrain.patterns(:,2),7,newTrain.targets); \ No newline at end of file +scatter(newTrain.patterns(:,1),newTrain.patterns(:,2),7,newTrain.targets); + +%% Apply the REDSVM model +% Create the REDSVM object +algorithmObj = REDSVM(); + +% Train REDSVM +info = algorithmObj.runAlgorithm(train,test,struct('C',10,'k',0.001)); + +% Evaluate the model +fprintf('REDSVM method\n---------------\n'); +fprintf('REDSVM Accuracy: %f\n', CCR.calculateMetric(test.targets,info.predictedTest)); +fprintf('REDSVM MAE: %f\n', MAE.calculateMetric(test.targets,info.predictedTest)); + + +%% REDSVM optimization +clear T Ts; + +CVO = cvpartition(train.targets,'KFold',kFold); +Metrics = {@MZE,@AMAE}; +Ts = cell(size(Metrics,2),1); +for m = 1:size(Metrics,2) + mObj = Metrics{m}(); + fprintf('Grid search to optimize %s for REDSVM\n', mObj.name); + bestError=Inf; + T = table(); + for C=10.^(-3:1:3) + for k=10.^(-3:1:3) + param = struct('C',C,'k',k); + info = algorithmObj.runAlgorithm(train,test,param); + error = mObj.calculateMetric(test.targets,info.predictedTest); + + if error < bestError + bestError = error; + bestParam = param; + end + param.error = error; + T = [T; struct2table(param)]; + fprintf('.'); + end + end + Ts{m} = T; + fprintf('\nBest Results REDSVM C %f, k %f --> %s: %f\n', bestParam.C, bestParam.k, mObj.name, bestError); +end + +fprintf('Generating heat maps\n'); +figure; +subplot(2,1,1) +h = heatmap(Ts{1},'C','k','ColorVariable','error'); +title('MZE optimization for REDSVM'); + +subplot(2,1,2) +h = heatmap(Ts{2},'C','k','ColorVariable','error'); +title('AMAE optimization for REDSVM'); + + From 06ce4146a3e7755fe492ef57e5ec7436837aa08a Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Wed, 24 Jan 2018 15:48:43 +0100 Subject: [PATCH 2/6] Updated crossval --- src/code-examples/exampleMelanomaTM.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code-examples/exampleMelanomaTM.m b/src/code-examples/exampleMelanomaTM.m index d567da8..d6b6267 100644 --- a/src/code-examples/exampleMelanomaTM.m +++ b/src/code-examples/exampleMelanomaTM.m @@ -296,7 +296,7 @@ %% REDSVM optimization clear T Ts; -CVO = cvpartition(train.targets,'KFold',kFold); + Metrics = {@MZE,@AMAE}; Ts = cell(size(Metrics,2),1); for m = 1:size(Metrics,2) From 90e6dc4ae44b058092df9c24c07c8271bb761211 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Wed, 24 Jan 2018 22:49:26 +0100 Subject: [PATCH 3/6] Corrected and completed cross validation for the third tutorial --- doc/orca-tutorial-3.md | 71 ++++++++---------- .../images/redsvm-melanoma-contour.png | Bin 0 -> 25976 bytes src/code-examples/exampleMelanomaTM.m | 54 +++++++++---- 3 files changed, 74 insertions(+), 51 deletions(-) create mode 100644 doc/tutorial/images/redsvm-melanoma-contour.png diff --git a/doc/orca-tutorial-3.md b/doc/orca-tutorial-3.md index 2e85655..639130b 100644 --- a/doc/orca-tutorial-3.md +++ b/doc/orca-tutorial-3.md @@ -383,50 +383,45 @@ fprintf('REDSVM Accuracy: %f\n', CCR.calculateMetric(test.targets,info.predicted fprintf('REDSVM MAE: %f\n', MAE.calculateMetric(test.targets,info.predictedTest)); ``` -To better understand the relevance of parameters selection process, the following code optimizes parameters `k` and `k` using a pair of train and validation data. Then, it plots corresponding heatmaps for `Acc` and `AMAE`. Note that the optimal combination may differ depending of the selected performance metric. +To better understand the relevance of parameters selection process, the following code optimizes parameters `k` and `C` using a 3Fold for each combination. Then, it plots corresponding validation results for `Acc` and `AMAE`. Note that the optimal combination may differ depending of the selected performance metric. Depending on your version of Matlab, a `contourf` or a `heatmap` is used for each metric. ```MATLAB -%% REDSVM optimization -clear T Ts; - -Metrics = {@MZE,@AMAE}; -Ts = cell(size(Metrics,2),1); -for m = 1:size(Metrics,2) - mObj = Metrics{m}(); - fprintf('Grid search to optimize %s for REDSVM\n', mObj.name); - bestError=Inf; - T = table(); - for C=10.^(-3:1:3) - for k=10.^(-3:1:3) - param = struct('C',C,'k',k); - info = algorithmObj.runAlgorithm(train,test,param); - error = mObj.calculateMetric(test.targets,info.predictedTest); - - if error < bestError - bestError = error; - bestParam = param; - end - param.error = error; - T = [T; struct2table(param)]; - fprintf('.'); - end +if verLessThan('matlab', '2017a') + % Use contours + figure; + hold on; + for m = 1:size(Metrics,2) + mObj = Metrics{m}(); + subplot(size(Metrics,2),1,m) + x = Ts{m}{:,1}; + y = Ts{m}{:,2}; + z = Ts{m}{:,3}; + numPoints=100; + [xi, yi] = meshgrid(linspace(min(x),max(x),numPoints),linspace(min(y),max(y),numPoints)); + zi = griddata(x,y,z, xi,yi); + contourf(xi,yi,zi,15); + set(gca, 'XScale', 'log'); + set(gca, 'YScale', 'log'); + colorbar; + title([mObj.name ' optimization for REDSVM']); end - Ts{m} = T; - fprintf('\nBest Results REDSVM C %f, k %f --> %s: %f\n', bestParam.C, bestParam.k, mObj.name, bestError); + hold off; +else + % Use heatmaps + fprintf('Generating heat maps\n'); + figure; + subplot(2,1,1) + heatmap(Ts{1},'C','k','ColorVariable','error'); + title('MZE optimization for REDSVM'); + + subplot(2,1,2) + heatmap(Ts{2},'C','k','ColorVariable','error'); + title('AMAE optimization for REDSVM'); end - -fprintf('Generating heat maps\n'); -figure; -subplot(2,1,1) -h = heatmap(Ts{1},'C','k','ColorVariable','error'); -title('MZE optimization for REDSVM'); - -subplot(2,1,2) -h = heatmap(Ts{2},'C','k','ColorVariable','error'); -title('AMAE optimization for REDSVM'); ``` +![REDSVM heatmap to show crossvalidation](tutorial/images/redsvm-melanoma-heatmap.png) -![REDSVM heapmap to show crossvalidation](tutorial/images/redsvm-melanoma-heatmap.png) +![REDSVM contourf to show crossvalidation](tutorial/images/redsvm-melanoma-contour.png) ## Kernel discriminant learning for ordinal regression (KDLOR) diff --git a/doc/tutorial/images/redsvm-melanoma-contour.png b/doc/tutorial/images/redsvm-melanoma-contour.png new file mode 100644 index 0000000000000000000000000000000000000000..b5062c51422d04879bb328f54c3c5cceada7bb3b GIT binary patch literal 25976 zcmbTe1yogC+cvsT1VKVYK|)$O1d-mLlyrAWcXz8GNOzZXcejc((%sUHG;He3?eo6x z`M&@E#yR8v8N;En7i-OR&w1B%T{i);((f?Q3D6-3!W0t~l82x>Y!HN$bN>$b&Pr*( zNAMS#ji{PE1Y!L|{6UJL$07vZM0F685=LD@diIoouJiCS;s%rs!m17m)!V9ZapQNEJ;j4Ix%G zMsZy$M%I@soTRL*+-z*zY^>I$Pmv*r6cQ79tLT!vGv}`PWc7yg&%j;NPY7m4#w zD@foz*-yXkXk@+**WcS9{=@Yw`JuplTPzf6a80t$|J6s!OB*Pp2KIQ&+C>Egj{}5c z_%5=pRh-|i`YIQ~JmC(Wo-oMQ*O&L|PaFGm_-<8gt$DRN2L}fp9z<~1l?B;kxMk8B&Tz7eQ7v_Gt)9ijmK-+uPvZnW2E<=~L z()hO`6g)8HXr3@KG8(`~WccUm?Wq&yD~+*XI_a{Lt-8nKYRGp!&P{)t*2m1Kt z<@;|?w*1uOq{~XzqY80N&DnDOPH1s3U9eEOIHM4L*vqV;u0A_Ar?0R7vCaFYi!=AW zv}BqWd~>{5V{UfV%-sBJf5AF$@??9WRJ*xAxlm9*fY|d`&aFkc@Orm)0bHfnJa%5n#ntut>S&;kYkRzyo{sL$ zhuf=l{v0;H`g(~trkLnx$yDA1zRM@LxVTTAiQyC*CCnt7UM5@Tk%lo264ZT!RDI7B%NFB5M6O8}r+s1EQy&1Qwoj&f4!=t0h z%F6CzU?vz0=JV-O-&~sr9~H#uv@-*E4DPRc?zT556R+JCm)4skwq`N5LM&4(0&Iaw-5^*hmemtOy0q40T z!)FATmszVpD7K-Yp`ylcDUE*#`JMCJj_$?b>dG(aG`{B$-_5yxV8*=Hr6uID6hPgT zfv4jU@x;o+B%Z^X*d9+u%*{fv#9Vg18F}DgQWhC{atR z&_G&V{&9j}D@Ena)gxz46&r4^D^3V(jY#GYGAiobZ<`~zYVWAI0$f~Nb|%a97g{{Y z$jG3F#Kh9ER3fFXZJBU zw8P%~eHR^sZM-Lvz@Om9oA1JivuOhPPC&FV>y2exiELJ7$nnXWPkCj#^wU+V#%j*Y z1IKNhaG@l0O;49OLQQIYbtkL1mUR>=jJc z69TtNCTxj?Dy2pS21NW`$yckW_wGdqACow}eTyV4r>3U1vigq1(;&;WzP^5MZ*PUU zprj-;bwOW$RsF1q4x@z$lrnL9<37)W=gMWaBB#3Y1%&iw|gw2IGcJ;}cJx8af zH`iwin-?NjVZ<{=J;ug+EH%~(eDWWDLWLfZ(JgHqb90`{9U%lE9isSa>De5d5$8x+ z0fb{?V-7;OAtd~%2}o~$^mRp0&<6JO^-&sj#XTMO%S#0&$>p4b<$~N3LddZ<*N6v= z7pb*@C8>&1v0F|H?fo1sC@6pyK|uH1t)5Vwa8hM%ZfX(<8cP##O>9hrrG}i` z5C~B63JTqQ#%Z3)*4G`k2yj+&cBHHK6M2A! zM$5oJi3bPU*kJUFAYJyn)cZQ9<+r5%+$_65z9N%q*&Di;JKw(hpmxJtaR>!vMyM6W z)y=9CC9A5dBhZ!yzDOppes;L$P|lw2+!8SJS}{f~W>Dg!#GHJN=LjsdHDH@by^4l{ zqHS*O8pHsHz`y{1|3^9APB|p{v6es1JAWR7^D+?6qCeN*NCFa-h)4&|e$&xfl66JT zGp;Jtjs6$){6f6&jZ6dTxR{uT-grYp!{gzn^vV`9HJDHfe5gZTqZ~a{eBuJRsah@#A)o*{2cri~HF)^&>7v1F` zC!yt1^VvMbSqaEZw(I@R$K{v(V+GRlN=t7+8KJMI*CqA!5rxRruC@2w4;=F@6pV}^ zp_#`GeQZsa`>ko3eAhdb16x~L5Gfzuyev=1?FEIE?Xjb<`+IE%qoL*vN$R)1Qeiw~ zWPfwf&xxTS+mkJ&3K|9m1`vH&StDl;a?%nE=aOG~2`9DGrsLH=w9aB_NXi^CmrCYj z0YSjwksl^u4#%q4ehAV=1G66D=z~ZaAd{3RAMl3s&!0btgtkF!ATExsmeOKM_Ba6K z7aSZMM-D0D?TEM)a%?D^&})4ljhgvvcP97JD2y1^QZO+nzB<9<#{&Fx5^#+>AK#Ei`1<(`r)VjwBhM||fzoBWnzG*|T&#!H z-EHM5yI$wN6_b`eJw4@f+#01$^Qf?1XsM{EXb-|`tg8zqhPicyKELzvYhB%Wlnh@2 zI32D%1FSik@-K&j^Z>~ zRwqN2MsyE1muZ#Ew+PDz2Q(c0whjI^j|B2KQeD^P`ESb9(Yxzxd#qb7N~_JbWnOZU z`G(`)oVT5ry)8{`F($xaDpyvzmh{GwP7h;vJf`F5K>tv{vu)D0=Q_txj}1M&n9v7q zZv#Fl*0%XERQys<)=hH3>}6c_9ueQU&V?Hu=8cEk^(~i$O}q8NDDSg&Z-7r}kDVj-cHV*G^stMlJ z{=mQv?eNdzL{{j21o|v#?>(*)E=FAx)FL6}QR4caLsB;AXy!${PEVX=UDO6n%hrFA zJxksZB>w!poA~i~Jgf#Lz_!B$%#y%ewr*fQY5%ut`}@}`+JipRe}CuM?IVFMUXK^^ z_7z-+>mSyFWdEGUNivZQWa+)WgjWss3|OZ?smcfHor|OO{i({%8^)@;|6IH?U0qmK z_L71kv!a5PkB{&5>(>kn!D^%+CLC`Jfz~d%+h71RDWLj_W7gTr8=t;^3OdIw-a9ct z)i;~P{I6d-f@lnq5XHMt&9e7z)zt|Kzv*~izc%hC`uOJIlP6D}K2=gyHoD}nv_|xt zUj#uwnx38p1$$TT#>PfZkGOII@I(?Gr=pUQlDs?`4vw>K>a;5NSC&}J+rR*>=O}Kx zYw+_jb!Q_nv4Fxt`m}{9`-cHSpe-n3MJx_fW1z=*ir)U|-h+&cjG|#;>F=rF)NyqL z1sx;TfBi}tA|lwTNEZX5S|2#ZPMgEo%&k{;PEJnR+NqLMGxhelzh=yw|62FxkLATf zCN$xvPoE07eFZLhWim+W|9ZlxY#nsoYrpd? zE+1OS%MS-WrMI)UFDxtsVLeAGNo0+L(*{(x{B(3%bIuE%t1&7+e*DOi&-eH51bN$N zuwOS>S|GlPJP8f0UDUAuHv5Rx^<^s6b2p$}lOT2u^!)#9 zAYDk;kkrQQ-(`HU|7SIugG(^#q&9@@$ol`9FyY-tW9gZHe`gT^DAx4MW`{0DjXyR<%vPT%`8ZCb$(1!gmb zOq_P7V^UN1wznIt<{C^#a>~)0Knv9u7sh+o1uU~O`NsVl;^Av@A1ram!PQEts__vK zpm@b6AxTS2B*4Y(U?{VgVs&!^r?D^Y8f6~gFjitxQd+fEDY?1ppi0DJPz{fb?Vx*^ z9VAOb#A&mbB^E_AES>ZN@xBS2IcLfL{b(K^I|zP8G=y$jw0x~3POtukDSd-0)pVZ9 ztgN47V=XF7X2H_2tW#!9m$VHa~N?uJJ=MUpUj_Ksq{ZA7xip*EerI%t!iTlXz|A z=k83fUj9Jbk?6r>z~E9kcqXmjEqbg=n&PSLBNlTdzbS79o=fa3G{m&{;x+n z{!|cy*Lr8Q{L1)$IbR#=$yh|3i~IW|PoLg{{IQ-Va#*K|QdXEs5PkWZMc&iv>gtX? zkmhW7+QqVBdcbt^7`oQWMG_l!DUnor}1JmPCF-Ebmdlt9<)> z_52EDx`kWVsQ4TS5$*jyad*Gu^cPD@jm);d48~m`Svf;GbU!|}s-=ol4?W*xA{6SC zC+G9uJ@((jvvXMe9dhH#1DUZN&vV2+EsXytu5{yHQ{J#VEB6bVcDlpthT?%s%g3vC zxPLr-kRK@DtzU|BlyqOCEqttJh=0--cUGA*Zw5r#b@|V3$pf7ZR-(_UhcE zr0UO+v5M+6*$u|e;q->P6)2F4O~20t^^xNY$+JZ{3iHYn?JW+A=EiA|&D`sPB3?bl zNFaU2`q4{FFQ$O?-GMu+<9e*kq-x7K@o!Ikv7Z@1j!rf4AL-8W)_$-&&VxSOZg~9e zxV;Bi+A@tQnPYijI*xstI23_VssemXc3gaw-}p`O=q=yYw%H7_VExRkZ zepbpXcVlYG)q#<=N;SvkYe$3MtR9~I;$8S^^^xvd+B)@b=9|WbMVS!quWJ@4C))`~ zVKR5`TNCfdIOlGaI&4cLeRzC6|JYh^*Lg97o<9H&X1P5Av)5?on{KFZmKmx7(ev~r z*`0La_$1XK+s(Y3^oHeOIis+^6@`soSzpm_vdpp-e;XDkb1xp4LW@iL3#Z%ozjkN9 zn_z}M(*_4oi#6{?JV9wbG=kKai6lTKs;XPP`XsVFBwes^6YY_=OhDy(LAUExywLXo zzS{}!pt|*2Cd;q6b1!@t3Ju5IQgN+{}6q#mce1BmtvqY$fH}99QHkc2PBGc z{EeUJ6k;FxexmyXcx&NrF%jOzA=6nNyx@-d#P3)FcM9p?m$l!xm8-5r<>f2hH2Y{G z>B3AXliFg3@8}$Qb!s`HVo*R$4|hA3{TAKxZq8g!OlxaTBf>dEw6j3L@+%8dX2!>Q z96IZSI<2N6W7N8!M-|EQlyByKqPK90lD6dfW1PHOKf-N-AVu4fwF!M$mAQmN4o9nZ z-_u4#BLwDV#^qHQuKiMVMg@OW8#@@IH;~76th_N~f}@!{pCh!$QRoVZmgsDAI1N4d1z0n0-s0_(~ZGRbSGz*PhBVJN^Pgpqv< zT1clLTWgNNG@S2W5VO_4BS*OS;W@iMz}z)-`R-w8l?Q4<26IJ^u?USf3lf z^5JINRr&o*Zow8R?)7hS{|v;kbt5-@&k5zb2l48pD-WqDWQGpbfkZkqk)f<|L4*%rxt0708}VM1G0LbE6!h8 z+Q$T0f8E=hJhmN2`%bFAda*=0hkj3s@F`ZMO(s@nV4XvrQU3&+Vj<~7ky-h=0V+73 zOGL_g_X3pq?tqXY2&D2jl%KH)cPd=(zpVevS; zB4#n9XKKR~!TS9MhIi)n)O1)zy@jcwhL0#2RAoBuyVU^P(P?zDT<=d=nIL<#6$j!t z<;PFmKXF(Fs-o(wSe8$mHKt$x`cB#-@Pj%ar%%>Q`2{HKvEVi~;^N|N`^{$^q0f_| zqN1Xs9hQ(o6@B5BpmtT1bfPfdy z$83|~lQisT#Nidl!MsS;_P^J3J-l;BZvEJ}KDQ9d<`i?>FwZ{P-bYWJAZqq+Maqy){;dib+uAdb9@2zSvL_-2uIMY>R;)kx@$TGyGov)a&N4vxQmOW4svPxwx-vLn3nqwkHdb8wS` z(bjzWrr@KJYIG43+I2+YUQ&ujF zG=C`6fS|et)NnMnP?JIZl%YIxtRdwKdFv!gs*3>0xdhP5fgb62ndNUi?W^{+xEK@S zJ74tSSnyCNR<@q563dp4u-?5!Q~4Ano97+907w_Z-q?suu_!9qm>zHl!9S3`fqiMT zUG6|7cKKRXW^=YT=ca-!9)|QGw(LB%%w>j+h|D}Px82;6WUX^YuRqJI0u_~!tt!wk zp4xD&Dj*s?L7pkoN9|oy)k%jbRL7&Bcg~kEza2Ga>|~zm{vkC%YlQqv_SqBhPO`a3 zSZmxGP5}>oP8=$|_IT2(=d6 z{(6e^*D1!rPM-4S>K5*MN@y^>R~DZ;@Fjhf(PiK7t>?>I&+mqAaBT_;$O#L#18xiH zYf}?1>DQTl{Fm=PBqb$@Qj#Me?!-if3s1MuWd|E_ezN^_=QHuLpr8jwJonI4uuSj; zl(Q{MfO!OkGE$=tD%v95(`SE3xSJcC(7Nd@ zF2J1te;^DV-`<|WWpC6IO$*roJ_2})Mj>qW`=*?CS2e{j@I}U> zhP9NQ=^|WHf=KsPnG6+#OK#`A*`@X%gd+nE5YCl*7ZvO3d^rsI1JH4uRxcPkJ3AFs zPEE}z;7!=ob<%!rrbfxqV4$NH z+d;EouFvPtN;=-^POX_!rcGBrzd!Be<>ltz1fV|bgg*xdfa~~LJ^f8e6HQvZSUGbj zT*8Si2ECy{3Q*60^hyDgrzoX303(8fv#j5ejZV;3@+rpN_Dh1t++4l4S!94{>gqIq zl6+NkY^)wkH_OU(8B5>zw-#WZDr+-HJ4MZG@A3PUvX3MF9N7Zn)A0*raW(9#^s5zv z53S$T-k)oaS#&sD>Hhgsh!p_DCQzNzZlWxWq^KwjEiFF3S5t0oZf&izxOkvoQb73F ztQ0Y0J9+8f#{`ccU>USZ_kk8MxB*c;_=p&o9`oFH? zlG?>>{HoeRRVO@(LB|ge;SH%+?RV5kqftrcbXjy!(J8+Zny4D~!|g2j57TEdU=F~< zfv<+6&jP$pF_^^KFAzg~`h5e@sdkD*rg1jcojR7(^$e}8jNyZ0fsgPvSP12a|RQ-UbXIAJ%p zVhE*`V;({F#H}ECEC@RRs@hBU*C9RoYF{|&QHR~GTa#FR zcs*+jQj8&=ISNCa^9Z`@#C`4>NBqQ`(dE04IP>rFccm4^B(NTfpsOx7Y}#dpV6q@^JT7MglJ+D{R0uFrxhz9R+VH@kcMM(SHqBKt zFZW|0Pr>$0M(^? z4kVlHkGjyB%9Sg~)Xr;%wUmF581)NLBik8;teO!iMePfZp!d(_?p{(;1tOk#^52%7A`}0BaEiPsx&#<>? z{|A(v)4M#~b^rYt4-lq`0KZ*8nMMN}!HSLF8$(jr8|YX8c0Jk-AM2t=>dw;vx)YaG zkHUY$Vgvg-TwjDTA|oFwC)|60d;haQx@>$(O0z+CB!G!Ao^y}VNiAW4Hanqbsa(^Y z>ep$LHKsW+*2VOGwWR9O2@en7y?gge*$Cj-^7$BOXu2^8uM3X~`}D(L%8H5(06KHr z9-pepKSeZAQO8DLBagAvkmCzFb(!c0m8B6`@ZKf{0L4&|%fc{o-uj>b+G48gE!L;A z-&yEft%V=n9oERHx2Jj|uP&;mAnLhm}qiw!Yph4*m?}HeD)A%lVcw_5vTktFZbCQE9LA*32M-wLB7F z112puO)8w`OUl61vKZ+?hGd;4N8WiG{NS})SwXnP+$58sCZ3X;%b(24Vqh87CQY}%wjBu=c2EvCQM{=E{Puz(GSiYS|PZ9QNr;f9v!4|Cy_RBCLGO$t9 z21g(OvKn@g6CK7ZD^A*1z%Zn{>bpQUWF@$h$l9;x(nX$8%|b$fBLLQhQ{JXp0%HYk z@k?n#KdpnbHxG28?%)%JL3S0#WRJNU@J=27Y=bZ4gJXwsO4K`6N31{w`Frd>sA9i) zp@|${><@>zhptY)!intCAUqNV03a85ZXUk^!tZZWTPdR+L%q z*T!drF)A)jMO_`f5+yU;>U#PSMm>tnWxy~&EL!>EjBi9gA0%BTlLuQb2C%Ak=}Q0C zZbZ#ck#215SJK(e6zF&L8)G?f0V&uDBCFY@1^PNU4y5EZ35q>&u|5iCV&dACp%gWF z`K7I`o^|KcWMOhSF)_5lzSi68eGoe^@L7MZNjVVhaHZe0Md;TP81{q>0ird)7J)B={@-o>1F5h^;{jsx zYJZ-WoauZrsko(b2X#^LoqCAZ5}^{JeDR`KwVa%Tqqd=en~KV4y2{kx=Hp$|tLtmA zwEP7G<;u<)OTUl~OkoUu##f%H4Y7~s7Z)jfjkIffV}n zr;>><8jniad$wH@EkCN;{6M<>_#N#;Y!<(FjB~`F;f~~(;1?txoGo|7iI#27%`y@- zjv!h=+#!(rn^C1IOx?qk?nf(cbCC)DA=4b~q^acF>ALy?ZcSjk6fP{OP0Z=^NB{z- z%=1B~LdR4ORub?c%*qouI`L_;1-83%-as8sta;iONk~zuCIC?yfHEBGBJd$5_{@3{ zlyo8qd*PI?W z?0P!g8)|iDtL-JHEvUk@5_^(QZ&T32TCCgoqC7w0;6M9ue0aU@r+^CRfCs=0ZR1$C z1wlCTBsWZMKPC$@;ao8%5unw0KIeNA_Alz=zcS7~4`k$Rt~JBje~`py;QC*Y;1s7 zxlT|nEG{V-H>(b{67}};XJLI5`fvypeEze&J>ajfOrzEcp&-+t&LM>%(71Y#dW5j_l;sJz1C7B26 zgZzA3a4SOI;0U#soF^{;jk-VIED_6K>mUeE0xT&gRN#iHs;X*g33q2kT7T;XOMW~{ z-VrJsTU%WP50@xbX4b3=ug|Nha=5*@wyXD81MIX>AL%RW**cIgA3#9smDy_DascFe z{@6MV4*4mB3+wTXwT>#a60a`VgPd83JG!0#nw*l0>jG#kfZS3?SC{OFxxTK>r91Kk z(6|w^nGAx^19((WcVR)i#44jQwSAR@GWlc>M#e3302Ybb0z z8P{NLK8}1oJ;kqDRtr8!NlB5CQ*?4VZE`)@nVIM7{*)37w)OdG@{X^AkAtC+5tBwu zu%t{Pn^?Z0k)fefB3ppk8i1W7rKC>&{2{&g1qk?DnY0>5pG&p}r>FlLmT_uso@Tfw z-O=_e;hs+vMfa**wYt=;*`u@pl9%t_NJu;=R0UG#qYRvi;DhGp`1lt<;70od9Z0U> z@^q|m5r3ES0EPNp1`RrNb#=AD1?0s*U1!Ue`PpAQYif<*smnoB1rg8p^492*N550~wdke(j7eLu)~ zz5?mu+2WFtO*X$j{kz5O2`3XI8td}9dsg#&QCfuk^Y;Lpf11x=@3zyA?lV|6)S23U zInpoofBwZ^|9?5+Un2W|_R;^pk2HzqAOrgUs(Lrj$)D;zvFn^Zwl6>NFBvnFN#oPf z(ZPm*-T<5bs*JHl&wO`xSG{=r*RNmg_hgEjMbrc}G?G*qPJ!PX9s*@*jMCQ17a(eN zb);ov%wAt#TN^&o@jOv__j;f!H7qFTE~KNQQ)oRpn+P(eDS}YA!A>N37p6;EwF;-CSL0YisM3*QKX_tnUH(P@#frAVlZ%O0~VejhqW$ z(c!c}QA#~e3(sjF(cY}>;N?^<(G(l-K4`~#KE_Mop33umb2*HkiqXQ}L5Bt+a zEMv)F`SD)V?o4fGdpnSo&Q@@{*EdMb7|lk2z6}@W@#9b;u0tS&0*h|;$Amz@Y7<3T zvo*}~16s0z0y=pkpxrmf0n)O=Lm(0F69;y9P{H=%#pkJ-jSUkB089HD8;j?z9b`o# zXMm7rMgr;IrFXTSDYRfd3a28+JwgFgQ2}a&uI|M5#`z+ev|X zF%}w{D;NaWG&?#v0-;n&xfP(PVq)DO=-j<~*Bhu}|3XZAOG}Rl2)M0gIhEa`#a97D z4JsdsFy5v0b?2KaSLhj+-3mhC?-A7D3E#{G@!Sz;Jc^6wTD@D-_5A22%&JAy`#mZ_ ztOA~h8`NHD^ow<#QqVp1^A7AE0sPt(=qTt_N`Pn{NR8e1>Z!T869-z|2`d2H{`Kpd zVr2=+NLq!Qmw21w3zL&HxIx_u8v6RnK;#LL^73j@MFI7FxE=^k;L({u_){t(77HyY z+JMXtM{D*M_w;PulY~EedAxV5EL_^I=k}s6#rxvWu= AC>*eD9YTeN0(AHP%8jf z^#cPP9i5)%*S@havlB#1M92%U_LpuniIeOj3tnNU_5ezAa$-*)=61--%{7*l4Zr6; z+0}^*fm{N_q-N^s(?E)jjgTEom+9WE{4l>9s02o4Du6W72b-^}zn_VQCLZSQ~K(Ukbx?gCjk3ANfwcqWIdNP6X(djRP|X2nayd z=(LalUD~fC4l!LgA^hKJdZDu*Nf1`G_)W1*Q6SLE!p%uHdkGgZnB44v+!pmi82?Rt zMqcfL7a^4DvV>Og{^ID4}S>V$Hq3srOE-*EuQe>%y(h`lbjz~+r@=%4^$ZTYP)T|U}k1sdt~hq)Ue^R zc7Cu7y)^2379xT}M1%olJ?P>!30ARcT0KrWIXly-w_E-4h$8s**RzGAK6EhUGhV90 z?|Nu(cnH+CInO@sj;ZKe11%%amezV)9D=Ed3MNl^)oQb`SVoPmc879R$$lN9d(>-q z!=E4E@m(K}03Y4`h05}GHb7L#59?|3D=81>2J`ZG)qwdFYm>AMOljyL0YQV^s(867 z21Fjw1?0*J#;Kr6qn4pefzDqe4Zs~ z+>Toa={fw^)ZA0@ST`ynf-J7LI|c<3m6m2-t;@hbrfyI<2IDjePlUbQAEJK-$4o|2 zlG6I4NE&^O#gxDr1|d7fGh>0u{#VjJ(7y-l7#>QkfI`B+6J-UP0tASRytOr>L*qAm zml+x4@-84|X1%Gi0%ogO7^M*&DIlP0QX140TFhDp?hjcIF?G5 zhmFlkVY%ba(TFjyhK5A7;ypeDrt)`YpvT4rXpD$myD_ArTil;DUy#NxEiI{9$GYZi zq?j%>j=;7UCWZZ+TIM}78ud39k0*4z<(>#T$9Iw0ZjI6&9wu403^j5i%#;h}yM8sr z>95G{lw0Psdg5TOd0w7I+;M=)dOC$Rc`U%h6R6JfhDy?PZDO{SwvW8eR(KeKj#ORLGHys3!@x;d?FMfxU)4HIg4 z0U))=JD{{g`2dvKTCHB!fL@`C5eM<}S{qCh^jJ(+1#~VSxHN$i6v?Heq;w3XL3k0e z+*X7R-_ygR1Nb~(Gyppo5yfzQF$9L#9?XWpPMz@7eV7BVLC_h1TH5~ik9%u`39pQ? z623AsXf`?(cQzXY&U-u*6B9#iYyzROPq;`>cwZ7{A!WXxulVO+KgSJ<+D|B6K#dwHW3@G!4=nfy{qrq+qIE+Wf(FCz*f=<%0zjU2%Li6A01AIFUjw)|@(A9` zok~!T^<{U^@0YJ4Z~Ij~0tP;mFIL_zkw}EkShsot3>Eb5pdJA00q&}3WMm{Lc=uXl z!*OPMy36+gPz=`*i_|nT4N@c#P`N z5fQ)l_Pll`S-2N|{uCU#kF(ISiCWp7^zw#<#S>Tw#C+9St2t0>flCki!Dz$9T2kK~ zkx+UqhxN!=FyRJf9x9@GdwRM!+JRixZ4*c%MdcS%bUb5XV^JW)%#sd|YH*ry^1D(u zX)tU^;V#xw9d-T z9RRb6;^s}>b+cN9mC>Uq+*Xy@%ah7hz^oU+8z4Ca?$!3rN zo4MeLJa4Iw`u^WhiPAXA*PGgcpjM=#CBemw1$ho1Urag(RLl+uVCE>2RUmzEZZ1Vy z;sx5Hj@rB+SH^hZGxf5F9(N0_?Oz0~gBCrh)KAfo?_gtxQK!0m+KUMHX9Oq8SKv1o z98)P)j|mTdGfn$AATcQ^!P)?%X23jB&SMj@2UAjPdNCdIVPj%qVq@pF?nJpuAK!C} z5M9~cPKuA8Ov3*3c@}TTy|I1pNOi4+r&%%|;pbH{7=~cp;&`G6z)FD`U3VWJOB)pf z5vrWgHCvA_Ke@uR(Kz4EK}oV5I>KILOE zkmfzFDy_Z_Uo%1=IrV9{8KoE}3zdGtxsC{L#hng)&eJuYwXZasq60@EW#r;YpwxQ| z<|n}16qTIR{z0vNClc^EM^p~{ zaejV&K#dy}33wBr;^0Wg8At*H84lBwVm1}$KJUv(ITJ4Tj#+zR&Ju|`quCF z7m3H`N(=HVlo#32s2h@Sd*F-%{WLl{I--!hBbD zB`AQfpK~KaAej$zciTFQgjc!%X)$QGmbSNh`#Rs=PbeY5O4(jtS5Z(93^RW51bumV z8Dy=^No9a!v1;+}a6MZ6`vpd)K^^C`6^yrXAN}*E6|0E#_duGxRg1vEWEVbbp@SAW zc?5eOF}|?Nd(_wLef4|Vo@*qSF!yRL=~lh4<|d0DHTT?j?^pFai|@m&1-3IxdY*_O z{c`>rn`ZU*7k`)-^nbh@~wM@9I%Ny?Wp ziHkYzK&3J1K!^RC*T-P=%(*9HGP*1Qiv zVCqCl;}e8Ei&9 z1i*;=A!yq7W@-oe|1~561~Az7xBw`92Li1qM)47-%`7L&-jY5FDU7dlCo`e<5LEwQN}OC&W6pB&yVRhqEPgye?LEc6IMUS z^zE<6f2csyQr#PHtN|hnz{+Ge|DgTsmOgF24PgjR@Ikx@YnV8NodXUWxY1u=`0qh* z<*uq}UiXFE|DweA5q}vF-vbOgz{~$DZro1`l>Dg7+fzvY8sPv;JGkQ~e`D$YOlzEn zA;u&AActBB-1;M)_@4(8?Q=~55dQD%2VnLQAEYQCe*TU{(!+?jIA@SvK-&WYzheLl ziB4!xfw2_;3W4GhsOr;huZ&$>>ZFnkya8V+@I?dE0b9$<3V+m>89KPy`NRZ|hShFY1HV6N>vm`?(k1<{c50c* ztfGf9AhWbV^~^wT#^vVY=q`a)|9I6j>y2=T-4D^z)r!6=zq~KdT5=OTD}VPJxM50F`(=7R zj(Jp(>$muaP+qG%6~Xh8h&dGKdlTL-%Yx!jC7yjdlyc z;ku^qm)4N#o_N&Ex_q>EWh}i3nX_-8)h|_|Fh9X0l5AyYu-mq#HxAzn?0u@A7I622 zlj&W^L6aVWe&Jas%qYS|T*S={4ft9f~u09L2W4UrIe4l?p1anDhoIXCIo6*aaI>Bqh5ra@splO#DLl2Aoa zklr*FX|ows!A~tgIBM02pHb9G?9gZ~&ElvE8Q*aUk_ne3T?F0(%~dtF9^uyZYaOKzM-`QUnxB*Ua1Vs=X_IC6M zkZzl53@tn`xX1YREY3q@B~wD3$S-0S{3eaN+B!7yr`Z(s${Z3-tMd=6*yO%aOCp@kFB2aZ3)fQVbG8E0tvNTnr70R~f= zOM6^i;OeW)TxY-ULiRDw>72}fb+H2aJ*vXEKzl95Z9JJk&D{li83kC3lZ(@&U&~;&Wj?ENPPMq8w`<<@U5KW^!@L&F z=xYV3N%hC({aY?Fa011YW(WCj%};4_GQ?q?(dCLTL5}1c*vj|BZpkr1_~2ka9G@c@ zdb7B{V7r@GAF@Nudm{88wIxC18Ae92(i;ISmBHco$r-`ZPv6nFXf=+L``4y{?pefpV!-&P3gs+rJm8S)Aw_Zym9p`MlHIJ#=iJlLkd&+9R z&_3t53M(=%vl}-}Igt0R8X;W4P|3(my&<5AKms>({8v#dcL#_Y}m?Yx?Gt}l{!FPt9;7}BXd1UJ3^&aZ~QaZ{f(LLVufSk z+42EGKh~FJ;N-88`AdljEKV<0J!*JIH<%5diH|+Ncz-SLCzkK_3G%krFd-?iLW1>| zKYyGjx}k?s97>fhPj;8)J>DM-l!3*@@<6WuEf%Zow(qoHRDF?{AKysng8d^SbZ* zx~}`W9@q25er>3gRQrRDtOTp6!2t#O@Ppu#+1u4+1bC`%%TPu-OO`f%WG#P$faUe$ z9yH*$b(gYGP;R{5t}F6RU8PcW^4UK|$8Cz; zB)fW>zh+Ti!$g!UG2@{RmViDTuwaEqJ8}ISh#)~jp@T#J^f#oQl}|q_Le`MH4{-%1 z<2?ZzY560*)Xw)+cy{UqLFRbYB`Dt4e38EQ=py_6Ojd+vih}w}^^%MRflQzpT3f#7 zTn~d{0b~qd)j~V24clefapKUCLh5*O7YnflTcTi;Uy+$5-nj(b^c-%omhCJCwt zap_l@)2vL@!c2(vKR(=-;GZ6`WALtZoy1Nz?vu-*R0iK$nd+U z)2`*IZbkH$_xk@=92*r|U0d6@{uu*J=YUYJdt?Xuh^vW9O6F(=u7DbnjI68^GaizO z;uliO>`7P%>!7|3$F-ax6`sHpPo-RiwzBI^&Z# zk)6~q$+wV7w3NgIX8QKWn2@<}{Fm-!WgR*4`;FnvO*_u<_eE&c0EPEMi_Vw>={GU5 zVvrr5NwE_*%7SGoW;>3v_hJye&8mEhR&{6iS+!^X9JB*~w}y{}m7^aEDU(tHfsmUU z;?fIaCqdzu9&W+e3V8kWReDMVI#*XnP^c=q)(9`OKgI#ri=JpAQ1}L#_VPDXsEn%vvVc-PM$h(w;0s87@)sJ^5qS7Rk=IpXmI;^Ht zh`MCMuSZ$Eg^2n$%iS=tLkC=%q7$=>ZbuaeU=q!(UpW4S0P2Ka`oGie*k-@+d#7`&(LJo2{>~CREj2I&Y)lQKAIV!}0V8z%R z9_D*vjmzu>zw=Ao<)2O1A^loj>m}Iq%>xO&KT^L~A~6Mc7%fCUT)51pH)S2pqIqd; z>VvIj#G7y zAAfm7lY&MS3A_Jd9jgi~$A%y_iZAwU|GO?=U7FJtmrlfQd>mmods5=f&d0%7)yj%o z4bNgdl>y~>c87A61C?>N##Xqs4VBtSZJb9Q$XG7%!H{bBEZLF!r{?~{o(Doj(2!WWaQNBL_ew*q5UMdWg`>3)-$g7G!gof`opSa=zzl9O zk7@CIG`f0MEbGDTq_Ppa!%T?90p`aSFbC8VMyEXh2s3oe?_Mo_WxOVnatK0<&2Wf9m_$OL&&_ zPJIIQ+lr0E!y)UQOf<>Gt?JG@KFIP4=oPdwoEU;8R9#+u$p@S7YI10je4CUQbedwbY<)Kpwhf{}c+KbYvjq z0k#!B5b?eTO_FdM{0ZAXh1hX)g(p3_?Km*#i0@`9H7|E8u{+vwi+kdz?&4tS$YCS$)zbZU{)^;xPXD3e`>E_PRo2`D0O5r+9;QZ z=|-ZA{g#v?B3x(<$fEa8IPz}0B&^x9Kvl74cW0|>El>Bs*Xe0M-53B+-v`+d6Qy`5 z%kV+yd;q1BF5q-G11dZGQgJ zgE<2Bma1OX+QBSh$2LD6dS`{r9eq13Wmaf0@B`y1O|nbg-4N^|^)np|8vcM&l7JMO z9`eYz@BkhBlCyDoqCX-Cy-FA9kB2|*=9$$&{TDK%kP(L(Lfh^E-JOT+1gw5)b}vTd zmte$0dNibTT8ZK8mlSvGl<4rIihcXG$@u=XQA4MrtSC9pnOdjDRMnY7LeR&P-Y`1qbo;mh+NxadF(&G3XvNN`%uoWD9Y4c;e*FmKcI+|aW|1~NSeE5)$ z6JscZ_ssX+60SNq7EyWZUj18p-EKNj*_;L~s{R?V7b){xw5ezu^=(UROV*LHXG1LI zw0^30`6L|BGdnj|-Pu+;gD#Q>GY;gPvPcNDrbk1Q{*0l`acO`X@|IdsfCBP|?GBk*hk6o-k`j~&-(1Jj^z=y%zA1=F zO~^qd%9D9U!&n6D*?6?ckb)&1rXk1Z1p%c{EBJmuOQL;Qhn% z!%3v&1p$jhhJSwomzci56y`T4Jn@K&iDO}@hN zZ!L78dLjd|)r{nPZn6DI&bT%c&|1G)ZC(0f3aJF|k1N|^W!XtGyV+cR^pl9W&q4_n z(IjAzknNSKn#B5;`1h~*gMP9 z5$)S&Vi_1B(g;`~%Zg=vEj=FlsMqdceBK(qyr#XbyMtMSp3?2gbu9d$^~~y#dy^A8 zQ8m8!Ktza`bwFm}at`fU`-oGLtg#~=Cz!D??rCLZGf08>x_ZsRc>Fli5FOl-al4rR@2IHthh0=K#5JU8R&ex_{&tbN1%5sZt{*M zzY!86>ZU)v^f&6{H2=n_h{ux3zx!5u4A2ReX+cF7QCgyp3$S|ViDTnDPAC#?WCUZY z=QLXFtISbzFri~`bQE0iNJB#dC1-#5Pe2fWVuaM=j_`D*L|9w7^>~t+;zEgEOx(Tja(q?Q zDrjxZ74HR{&Z5hOLC+)zYZw@GfWa2lykj=Sc?1#5WtGz}`F#ZQ^y9=aXY`V<6w!u@ zUC#RqkylW#_Ec>*N^7&mo__%A@-=$YkB03_l$Dn^F)=BYxpDqLJ8N10H?CW@>YK&W zr}ME1uTl!1VNESiTCZkGH9K;Zh?0Fwi$A8zwC>k^$7fv{i9Z_K&USlmUOhzD(()s8 z&3zaf%S>8M+fN_G3l!;CZCNbUs6b%Zv*L;T8`XSl^#R8;E~HLSij3kvJ`3tPJ%jKb zE9GL@8J24^%Kgd@L6kBy8jHpH-Ljeu|H?FMDGO7TZ$t2B@}uOI1DFj>p+IK^)T`_X z)Xt(9qn$kHTX=S_Bry%aH)N)Yxl<|k)Z=+oXHRc~kTu^vt&<+JvyrzhcRXQ<9= zd1<`#d5SL)=5#ZlUB354Pmc)Cr3^ROxk?Q~g)a8quTc%NPfn z$oqI*svn7}fp@rR%}-`~p!zgX`^ynDO3N)^eK{y#!#9>`!EipE-@AW=ZJ>$3@V~FA zNiVT~6hwSTjeI4pPQ+(s?WC6o&)c08JT@%s=*q=eUVg^ko5MlotS_UCvL+GlVVkT- z#E)5~9Wq=LNyT5NB|k~c4^E6>d81{iP;jXIjgNa%H#74cp(mI2>3#^tISU9DZuiag zefL;dSm^HXHV+`jJv`%tIh@CT)vn;Xkceo2>7iBUrYlijnXb&P#+-I>4O#eB?dodE zOE#YvM6N$F4~}|JxO)n7roi*cyPA|+gH@T>3Nthh87aD1lYRRZQs{@^NSc6rcW;W2XR3?q zA%FDrVb`LP?B^#ugBTV?9OBMBV0o@2gwmQkSrI%#8RQ|yN#$cr&#`sO;Hj@)fBK}x z$r(pIA0A&EaQ%##U!64PL5>0JY*G`C>FyWq2} z0q-w*czBQlarFqS!QrOz`|%{NNjvf{x7VO#8uE@X%%p2NvoV*Cttv?PlmS-OGx({F zI!E*C2GMfYbDG}$3X^CZ?*dcNAi8pkcM?H!UNv{wq#C*@7}{Fl->Uq;NzrMY`8Ml!@xJP*YU zXlPS4)Md27JGpKY=`Ry1k@rCFixOeXrm-G+I^Qrf`u=99=SMH!5B&o3&OzK}h%O6* zd(ynpged7+z4C!juy@JyK|RP+(x-{$iEnD)CFgfKk@NoS#wgOCjpeJ_1Ou9?nMkE9 z-vPD_;MviSA6?h1*dyq#PAHi``^eRzw1(>pW2uqqqK^Jk`7g4&M!u>C6`lM)e3JDi zq%tS?p5m~ptU^1T-f-!+TclIviFo%-V&)AqB+>G+<%M~;5Gun+KXiTJTkQO*vl9$@ z;k1SZfDyol3m5$DDE3kTF2W`gyhrfh_QYC}^VpiKckxWA>&e15_-{f7I&pDmxofw{ z7pkg8-39fQq8G1C^6uRu-&>oEu71#TF~X%<`K_Q9(|l_Rm(`!JAoCx;KCP;uQU0wnrV~|tmB!&k@Gn^6wCXyzH~COFJn&05tE`kM zigY!*i0eVpdN(&Y=md3d{Kcg4KEdUk+3zTlX)Dit!QpwC1N?! zU__XWVqtOeX~s45j>Klgw^>W;gU>RX!dJLqOTso2ecb3Tn$L|QeP#wvKj!JEN4!|} zLU^_(w4A-2Rkl+TMLJO{W-+|ysRXZiFu$B$s0I%ktB{r$up$N-MtzC> zbho=;&~o?e=4f}fV5Js^R47;yS#1wRXC31Sacfi4gwe$xOZ3O>`f+enk!oT|&9_Y) zbYwXaY4(FQoU*u*Y(K>~xVW6+E-pvup15jhPtw=%6RKK6k@u0nyv3d}XM28T{Ee{u z;c@!Ib4`k!ZYo=CsB_WOMT{&to|>g*TWYhZ#_d@)nJ7ZMeev~L;9;9j_trk@y=>`K zr&5in)K3cwZTZ-(nCJz###YC|rDe{0Yxzs#W)_OaY@XdqoSCW-w!(QE4=vvxyrJD_ z=`VnXIXihxP0KtTqOz3Odz1lQlRWkHUvk=9sWnNAoH_>8aVo<}!wRO;HNqa3z}c1Z z^G%s%n`z^c11m0~NXc%|$tY6sut1<<*_}?eu&YuP6%yys3B7$LdX8yHew29zE#D8O zjrunIg-0>b$`m1(Uibe_FT{d{u~cbSS4q)WCYZL!K<)bS2U6;PY=ufYFPRNYu&C%d zlb7R8O?KT$yVvEyVJuvg<#PXZq_}u>W;ZR4#2AH^gPPT67^=9oq=c|<1X#c@|fjI*_=0;A) zh2)mq`Jqj- zv+UO5KaF|*ub11o9FHokgIgCcrO@vY@aEUICY*J>nohQe@xj!ut1*qRxxa{+7ey} zr=`@@WCkMdoKZz_iC-Iu7jU1$I`~C<{sz`9}ijk`K1+f z^9>3<0DebHv9!M~gD)gv-PhtR zvFRJ^8DU@1$G~O)&OQ#^eQ)e5R!mpg+S(wkJ`uD;0&oJp8Z%=UP~{oX9;z9Gb7G?P zcCTMU7M`(x6*yESr|ov}(JE5N-fm0`YhV>f1;Ux;eW)*QiifFEM1rD6D$8FlYHB26 zzXH$>Hv&kWr+*e`j3kp9fkXwmy>0dG$`2qjx&TqU7V2vg5nmMy9A#df2#OozN$jD2 ztR=oyYEG>S=5zK-H{t9jXR}I3Bf;tJ`G%xvw(Zrcf=7=+9=^~o17tCRcW56q0u~%L zXQ6ePPT)=MlXpMew+^ic(Dr;#=ro0f)rOlNwBV|5ItM*oR*WjkRpXZyz^!ODg6? zJQv@hC+P^8UP~RPwe{Po-dzb52d5+92`4H0cO!3#1>9(xr*&iCP60hVJvhO{lrJ8< z2oojB{@A3%OcQ+1%;TI5_6MyWZgQ)?&}<%&M`Sv6@^IrBIhjO+6RNfZm+xr8pEOv= z4WOGNdjy#e>`YBfP4~vU&Z*2r#wMevH|M-bj6lo31%1^N%#X>&rCX<`mw& %s: %f\n', bestParam.C, bestParam.k, mObj.name, bestError); end -fprintf('Generating heat maps\n'); -figure; -subplot(2,1,1) -h = heatmap(Ts{1},'C','k','ColorVariable','error'); -title('MZE optimization for REDSVM'); - -subplot(2,1,2) -h = heatmap(Ts{2},'C','k','ColorVariable','error'); -title('AMAE optimization for REDSVM'); +if verLessThan('matlab', '2017a') + % Use contours + figure; + hold on; + for m = 1:size(Metrics,2) + mObj = Metrics{m}(); + subplot(size(Metrics,2),1,m) + x = Ts{m}{:,1}; + y = Ts{m}{:,2}; + z = Ts{m}{:,3}; + numPoints=100; + [xi, yi] = meshgrid(linspace(min(x),max(x),numPoints),linspace(min(y),max(y),numPoints)); + zi = griddata(x,y,z, xi,yi); + contourf(xi,yi,zi,15); + set(gca, 'XScale', 'log'); + set(gca, 'YScale', 'log'); + colorbar; + title([mObj.name ' optimization for REDSVM']); + end + hold off; +else + % Use heatmaps + fprintf('Generating heat maps\n'); + figure; + subplot(2,1,1) + heatmap(Ts{1},'C','k','ColorVariable','error'); + title('MZE optimization for REDSVM'); + + subplot(2,1,2) + heatmap(Ts{2},'C','k','ColorVariable','error'); + title('AMAE optimization for REDSVM'); +end %% Apply the KDLOR model From b5729b9d3c5c70639c144b49063b294aeb2f7485 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Wed, 24 Jan 2018 23:22:23 +0100 Subject: [PATCH 4/6] Updated ensembles part --- doc/orca-tutorial-3.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/orca-tutorial-3.md b/doc/orca-tutorial-3.md index 639130b..65b9abc 100644 --- a/doc/orca-tutorial-3.md +++ b/doc/orca-tutorial-3.md @@ -88,6 +88,7 @@ end hold off; ``` which generates the following figure: + ![Projections of POM for melanoma](tutorial/images/POMMelanomaProjections.png) As can be checked no pattern is projected beyond the last threshold, so that the last class is ignored. Note that POM is a linear model and this can limit its accuracy. We can check this in the confusion matrix: @@ -120,6 +121,7 @@ legend(arrayfun(@(num) sprintf('C%d', num), 1:Q, 'UniformOutput', false)) hold off; ``` which generates the plot: + ![Projections of POM for melanoma with colours](tutorial/images/POMMelanomaProjectionsColours.png) As can be observed the three patterns from the last class are never correctly classified. @@ -140,6 +142,7 @@ for i=1:size(info.model.thresholds,1) end hold off; ``` + ![Cumulative probabilities by this set of thresholds](tutorial/images/POMMelanomaCumProb.png) ```MATLAB @@ -154,6 +157,7 @@ for i=1:size(info.model.thresholds,1) end hold off; ``` + ![Individual probabilities by this set of thresholds](tutorial/images/POMMelanomaProb.png) As can be seen, those projections close to the thresholds can be classified in different classes according to the probability distribution. However, following the spirit of threshold models, the implementation of POM included in ORCA classify the patterns according to their position with respect to the thresholds. @@ -342,6 +346,7 @@ end legend('SVOREX'); hold off; ``` + ![Comparison of SVORIM and SVOREX](tutorial/images/SVORIM_SVOREX.png) Fine tuning a bit the parameters, we can improve the results: @@ -418,7 +423,8 @@ else heatmap(Ts{2},'C','k','ColorVariable','error'); title('AMAE optimization for REDSVM'); end -``` + + ![REDSVM heatmap to show crossvalidation](tutorial/images/redsvm-melanoma-heatmap.png) ![REDSVM contourf to show crossvalidation](tutorial/images/redsvm-melanoma-contour.png) @@ -487,6 +493,7 @@ end legend(arrayfun(@(num) sprintf('C%d', num), 1:Q, 'UniformOutput', false)) hold off; ``` + ![Projection of KDLOR for the melanoma dataset](tutorial/images/KDLORProjectionMelanoma.png) --- @@ -553,16 +560,20 @@ If we check the dataset used for POM: >> scatter(newTrain.patterns(:,1),newTrain.patterns(:,2),7,newTrain.targets); ``` + ![Intermediate dataset of the custom ensemble](tutorial/images/ensembleMelanoma.png) we can see that, although the correlation of both projections is quite high, some patterns can be refined by considering both projections. --- -***Exercise 3***: construct a similar ensemble but using different SVORIM projections with different parameters for the `C` value. The number of members of the ensemble should be a parameter. +***Exercise 3***: construct a similar ensemble but using different SVORIM projections with different subsets of input variables (a 40% of randomly chosen variables). The number of members of the ensemble should be as a parameter (try 50). + +---- + +***Exercise 4***: construct a similar ensemble but using different SVORIM projections with different parameters for the `C` value. --- -***Exercise 4***: construct a similar ensemble but using different SVORIM projections with different subsets of patterns and different subsets of input variables (randomization). The number of members of the ensemble should remain as a parameter. # References From 06220fb3d8d0d34c03a3c00cb1c7f2f83186b2b0 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Thu, 25 Jan 2018 09:58:00 +0100 Subject: [PATCH 5/6] Corrected closing code block --- doc/orca-tutorial-3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/orca-tutorial-3.md b/doc/orca-tutorial-3.md index 65b9abc..f6765ac 100644 --- a/doc/orca-tutorial-3.md +++ b/doc/orca-tutorial-3.md @@ -391,7 +391,7 @@ fprintf('REDSVM MAE: %f\n', MAE.calculateMetric(test.targets,info.predictedTest) To better understand the relevance of parameters selection process, the following code optimizes parameters `k` and `C` using a 3Fold for each combination. Then, it plots corresponding validation results for `Acc` and `AMAE`. Note that the optimal combination may differ depending of the selected performance metric. Depending on your version of Matlab, a `contourf` or a `heatmap` is used for each metric. ```MATLAB -if verLessThan('matlab', '2017a') +>> if verLessThan('matlab', '2017a') % Use contours figure; hold on; @@ -423,7 +423,7 @@ else heatmap(Ts{2},'C','k','ColorVariable','error'); title('AMAE optimization for REDSVM'); end - +``` ![REDSVM heatmap to show crossvalidation](tutorial/images/redsvm-melanoma-heatmap.png) From d002acd8b72d1a07a7020192b98571cd0ed6cbc5 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Thu, 25 Jan 2018 09:59:18 +0100 Subject: [PATCH 6/6] Corrected closing code block --- doc/orca-tutorial-3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/orca-tutorial-3.md b/doc/orca-tutorial-3.md index f6765ac..f3bb4be 100644 --- a/doc/orca-tutorial-3.md +++ b/doc/orca-tutorial-3.md @@ -566,11 +566,11 @@ we can see that, although the correlation of both projections is quite high, som --- -***Exercise 3***: construct a similar ensemble but using different SVORIM projections with different subsets of input variables (a 40% of randomly chosen variables). The number of members of the ensemble should be as a parameter (try 50). +***Exercise 4***: construct a similar ensemble but using different SVORIM projections with different subsets of input variables (a 40% of randomly chosen variables). The number of members of the ensemble should be as a parameter (try 50). ---- -***Exercise 4***: construct a similar ensemble but using different SVORIM projections with different parameters for the `C` value. +***Exercise 5***: construct a similar ensemble but using different SVORIM projections with different parameters for the `C` value. ---