pretokenize: move substitution table to external file and use multi-s…

…ubstition function

extras/fr_pretokenize_subs.csv

@@ -0,0 +1,157 @@
+"string§To§Find","string§To§Replace"
+’,'
+J,"Je "
+qu'elle,que elle
+"’","'"
+"J'","Je "
+"j'","je "
+"S'","Se "
+"s'","se "
+"C'","Ce "
+"c'","ce "
+"N'","Ne "
+"n'","ne "
+"D'","De "
+"d'","de "
+"L'","Le "
+"l'","la "
+"T'","tu "
+"t'","tu "
+"-le"," le"
+"-moi"," moi"
+"m'","me "
+"M'","Me "
+"-je"," je"
+"-il"," il"
+"-on"," on"
+"-lui"," lui"
+"-elle"," elle"
+"-nous"," nous"
+"-vous"," vous"
+"-nous"," nous"
+"-ce"," ce"
+"-tu"," tu"
+"-toi"," toi"
+"jusqu'à'","jusque à"
+"aujourd'hui","aujourdhui"
+"-t",""
+"-y"," y"
+"-en"," en"
+"-ci"," ci"
+"-là"," là"
+"Qu'","Que "
+"qu'","que "
+"-même"," même"
+" Il "," il "
+" Ils "," ils "
+" Elles "," elles "
+" Elle "," elle "
+" Je "," je "
+" Tu "," tu "
+" Toi "," toi "
+" Nous "," nous "
+" Vous "," vous "
+" Mais "," mais "
+" Ne "," ne "
+" Et "," et "
+" Pourquoi "," pourquoi "
+" Alors "," alors "
+" Aussi "," aussi "
+" Car "," car "
+" Au "," au "
+" Ses "," ses "
+" Se "," se "
+" Moi "," moi "
+" Toute "," toute "
+" Tout "," tout "
+" Hier "," hier "
+" Non "," non "
+" Comme "," comme "
+" Dans "," dans "
+" Pour "," pour "
+" Voilà "," voilà "
+" Son "," son "
+" Une "," une "
+" Un "," un "
+" Où "," où "
+" De "," de "
+" Qui "," qui "
+" Depuis "," depuis "
+" Ça "," ça "
+" Sur "," sur "
+" Ensuite "," ensuite "
+" Puis "," puis "
+" On "," on "
+" Si "," si "
+" Même "," même "
+" Toutefois "," toutefois "
+" Ainsi "," ainsi "
+" Aucun "," aucun "
+" Ce "," ce "
+" Ces "," ces "
+" Toutes "," toutes "
+" En "," en "
+" Après "," après "
+" Quel "," quel "
+" Quelle "," quelle "
+" Quand "," quand "
+" Celle "," celle "
+" Puisque "," puisque "
+" Tous "," tous "
+" Dès "," dès "
+" Cet "," cet "
+" Lorsque "," lorsque "
+" Lui "," lui "
+" Sauf "," sauf "
+" Moins "," moins "
+" Encore "," encore "
+" Cependant "," cependant "
+" Comment "," comment "
+" Assez "," assez "
+" Ma "," ma "
+" Quelques "," quelques "
+" Leurs "," leurs "
+" Ceux "," ceux "
+" Par "," par "
+" Devant "," devant "
+" Bien "," bien "
+" Personne "," personne "
+" Près "," près "
+" Avant "," avant "
+" Rien "," rien "
+" Partout "," partout "
+" Pourtant "," pourtant "
+" Déjà "," déjà "
+" Enfin "," enfin "
+" Maintenant "," maintenant "
+" Quoi "," quoi "
+" Eh "," eh "
+" Ah "," ah "
+" Oh "," oh "
+" Jamais "," jamais "
+" Mon "," mon "
+" Cela "," cela "
+" Du "," du "
+" Oui "," oui "
+" Ou "," ou "
+" Sa "," sa "
+" Celui "," celui "
+" Cette "," cette "
+" Des "," des "
+" Naturellement "," naturellement "
+" Sans "," sans "
+" Vos "," vos "
+" Votre "," votre "
+" Notre "," notre "
+" Peut-être "," peut-être "
+" Mes "," mes "
+" Celle "," celle "
+" Tant "," tant "
+" Demain "," demain "
+" Qu "," que "
+" qu "," que "
+" quelqu "," quelque "
+" jusqu "," jusque "
+" Jusqu "," jusque "
+" aujourd hui "," aujourd'hui "
+"  ","  "