From 6fe3c19ca22bf9fe102041c28de8e9d4d0f893f2 Mon Sep 17 00:00:00 2001 From: Raivis Dejus Date: Sat, 13 Jan 2024 09:28:29 +0200 Subject: [PATCH] Adding latvian rules --- src/rules/disallowed_words/lv.txt | 3077 +++++++++++++++++++++++++++++ src/rules/lv.toml | 109 + src/segmenter.rs | 26 + 3 files changed, 3212 insertions(+) create mode 100644 src/rules/disallowed_words/lv.txt create mode 100644 src/rules/lv.toml diff --git a/src/rules/disallowed_words/lv.txt b/src/rules/disallowed_words/lv.txt new file mode 100644 index 0000000..7092261 --- /dev/null +++ b/src/rules/disallowed_words/lv.txt @@ -0,0 +1,3077 @@ +a +Aa +AaB +ab +Ab +Abadie +Abarth +Aberdeen +Abū +Acacia +Academia +Acadiocaris +Acanthocinus +Acanthocnema +Acanthuridae +Acaridae +Accademia +Acela +Achema +Achievement +Achillea +Achnanthes +Achzarit +Acidobacter +acidophilus +Acridoidea +Acrophoca +Acropolis +Acta +Actaeonella +Actavis +Actinotrocha +Active +Adiacium +Administration +adress +Adriaticum +Advance +Advanced +Adventureland +Adventurer +Adventures +Aegithalos +Aerobee +Aerosmith +Aerospace +aethiopicus +aethipicus +Affair +Africa +Again +Agaricus +Agrobacterium +Ai +Air +Airlines +Airplane +Aisha +Alaska +alcalica +Alcatel +Alcheimera +Alchemilla +Alcini +Alcmaeonidae +Alcoholic +Alectorobius +alighting +All +Allmusic +Allsafe +Alpha +Alphasat +Alphonso +Amarcord +Amazing +Ambassador +Ambition +America +American +Americano +Americans +Americus +Amica +ammophila +Ammunition +ampellophaga +Amphiops +Amphitheatre +Anchilaos +Ancient +Ancona +anculonamensis +and +And +Anderlecht +Anglianico +animalcules +Anischiidae +Annalen +Annihilator +anocleithrum +Anorthosis +Another +Ansichten +anthracis +Anthracotheriidae +Anthropoidea +Anthus +Apache +Aphidius +Apigee +Appendage +Applications +Aramco +Arbeitsgemeinschaft +Arcade +Archaeplastida +Archaias +Archer +Architects +Architecture +Architektūra +Archive +Archostemata +Archuleta +Arconia +arctica +Arctica +Arctocephalus +arctos +Arctotherium +Arengupartei +Arenicolumba +Areopagitica +Arlecchino +arrosticini +Arsch +Arthašastra +As +Asaphus +ascalonicum +Ascalonicum +Ascaphaburg +Ascaris +Ascoli +Asha +asiaticum +Asiocoleoidea +aspideoides +Asseco +Associated +Association +Associazione +Astronomica +Athena +Athene +Athens +Athinon +Athletic +Atlantic +Atletico +Attention +atthis +Attica +Attractions +Aucella +aurantiacum +Aurantioideae +Australopithecus +Autodoc +Automotive +Autoteollisuus +Avalanche +Avanger +Avicennia +Azadirachta +azotochtli +Azteca +b +Ba +Bacardi +Bachianas +backbone +Backspace +Bactericera +Bagrations +Bahrain +Balaenoptera +Ballklubb +Baltharmonia +Baltic +Baltica +Balticom +Baltimore +Baltische +Baltischer +banco +Bandcamp +Bandersnatch +Barcelona +Barefoot +Barracuda +Basilica +Basketball +Bataclan +Battle +Bauakademie +Beach +beacon +Beagle +bear +Bear +Bears +Beat +Beatbombers +Beats +Beautiful +Beauveria +Becherovka +Bechstein +Beecake +Beep +beetje +Beetle +Begravningsplatsen +Behrstuhki +Beijing +Belgrade +Bellacord +Bellingcat +Beloved +Belzacars +Benedictino +Benfica +Beobachter +berceau +Bercelona +Beresheet +Bethera +Betsafe +Bhagīrathi +Bhāgīrathi +Bianchi +Bibliographisches +Bicarinatina +Bielefeld +Big +Bihavelanc +Bilharziella +Billboard +Biluhuta +Binnenvaartmuseum +Biohazard +Biozec +Biscuit +Bishop +Bistouri +Bitcoin +Bithilani +Black +Blackbirds +Blackbriar +Blackguard +Blackjacks +Blacklash +Blacklight +Blackpool +Blackstone +Blade +Blades +Blagnac +Blazers +Bleeders +Bleeding +Blepharoneura +Blind +Blizzard +Bloemencorso +Blogcritics +Bloodmoon +Bloods +Bloomberg +Blue +Bluetooth +Boardman +Bobblehead +Bobcats +Boca +Boeing +Bonnier +Boogie +Book +Boomers +bootcamp +boppe +boppeline +Borac +Borch +borchgrevinki +Borec +Bosch +Boston +Boulogne +Bournemouth +Brachininae +Brachiosaurus +brancai +Branchiopoda +Brassica +Brassicaceae +Brassicaeae +Breath +Breathing +Brescello +Brescia +Britannia +British +broadcast +Broadcom +Brocchinia +Bronchos +Broncos +Brontotheriidae +Brooks +Bros +Brother +Brotherhood +Brothers +Bruschette +Bubblicious +Bucerotidae +bucket +Buckingham +bucklandi +Bucks +Buclera +Buducnost +Bull–Renault +burchellii +Burghauzene +Bursaphelenchus +Bursche +Burton +businessman +Buthidae +Ca +Cabaret +Cabeljau +cabeza +Cacao +cachau +cache +caching +cachola +cadetti +Cadi +Cadillac +Caedonia +caerimonia +Caesarodunum +Caesars +Caesena +Cafe +Caging +Cagliari +Cainotheriidae +Caissa +Cake +Calamagrostis +Calar +Calcarius +Calcio +Caldirola +Cale +Calendar +Cali +Caliente +California +californica +Calliope +Callipogon +Callistephus +Calliteara +Callosciurinae +Callosciurini +Callosciurus +calor +Caloris +Calpurnia +Caltech +calvescens +Cambria +Cambriae +Camelot +camemberti +camera +Camilion +Camp +Campanella +Campanulotes +Campari +Camper +Campo +Campomaiorense +Camponotus +Camptosaurus +Canada +canadensis +Canadian +Canadiens +Canandaigua +Canaria +Canberra +Candidatus +caniba +Caninae +Canis +cannabis +Cannes +Cannmeat +Cantacaderinae +Cantacaderini +cantando +Cantarell +Cantemus +Canthon +cantina +canto +Canto +Cantus +Canucks +canzonetta +Capcom +capella +Capesize +Capina +Capita +Capital +capitalino +Capitals +capitata +capite +Capitol +Capitolinus +capoeta +capote +cappa +cappella +Cappella +cappellanu +cappula +cappulum +Capra +Capracotta +caprea +Caprimulgus +capsulatum +capture +Car +Carabinae +Carabus +Carales +Caralis +Carbonaria +Carcharocles +Carcharodon +Carcharodontosaurus +Card +cardia +Cardinals +Cargo +Cargotec +Cargradu +Carguru +carinata +carinulatus +carinum +caristi +Carl +Carldrakeaninae +Carlsberg +Carmel +Carmen +Carmina +carnaval +Carnegie +Carnera +carnival +carnosus +carolina +Carousel +Carpathia +Carpe +Carpets +Carpi +Carrarese +Carraria +Carrefour +Carrera +Cars +Carskoje +Cartuja +carunculata +Casa +Casado +Casale +Casanova +Casas +Casino +Cassaena +Cassida +Cassiinae +Cassini +Cassiope +Casteddu +Castelli +Castellina +Casterman +Castilla +castillo +Castle +Castra +Cat +Catalonia +Catania +Catedral +Catepillar +Catholica +Cathures +Catoplatus +Cats +caudatus +Cauvin +Cavalerius +Cavaliers +Cavalleria +Cavese +Cavs +Cd +Cecilia +cecum +Cedar +Cedevita +Cedric +Celebrate +Celebration +Celsius +Celtic +Celtics +cembrae +censi +Centaur +Center +Centipede +Centraalbureau +Central +Centre +Centro +cepaeum +Cepkul +Ceratitis +Ceratocaderini +Ceratosaurus +Cercozoa +Cereiba +Cetema +Cetium +Cetoniinae +ch +Chabanaguncamogue +chacha +Chacornac +Chaetocladus +Chaetomium +Chaetonotida +Chai +Chalcalburnus +chalcites +Challenge +Challenger +Chalten +Chamaecrista +chamber +Chameleon +Chamomilla +Championnat +Championship +Chandra +Chandravāra +Chanel +chankonabe +Channel +Chapman +Chapters +Chara +Characeae +Charcot +Charentes +Charlemagne +Charonia +Charter +Chase +Chaubunagungamaug +Chazal +check +Checkers +Cheerios +Cheetah +Cheilinus +Chelesea +Chelsea +Chemicals +Chenopodiaceae +Cherie +Chersina +Chess +Chesstime +Chevalier +Chevrolet +chi +Chi +chiaroscuro +Chicago +Chicken +Chiefs +Chiesa +Chievo +chilensis +Chili +Chiloglanis +Chinarm +chloris +Chlorops +Chmielaki +cho +Choatian +Chocolat +Choeropsis +chokkarak +Chorale +Chori +Christi +Christian +christianus +chromatiskā +Chrome +Chromium +Chromolaena +chromospermus +Chronica +Chronicle +Church +Churchill +ci +ciānkāliju +cibarius +Cibona +cibus +cicada +Cicindela +Cicindelidae +Cicindelinae +Cicindellinae +Ciclistica +Cimitero +Cinamon +Cinara +Cinema +Cinesite +Cinzano +Circle +Circolo +Circuit +circum +Cirez +Cirrina +Cisco +Cissus +Citilink +Citizens +Cittadella +Ciudad +Ciuman +Cladium +Cladonia +Clarte +classful +Classic +Classics +Claude +clausum +Claviceps +Clavicipitaceae +Clavigerinae +Clean +Clear +cleithrum +Clemantis +Clematis +Clermont +Cleveron +Climacoceras +Climactic +Climate +Clippers +Clitellata +Clostridium +Clovis +Club +Clube +clubstage +Cluedo +C:N +Co +coal +Coast +coat +Coat +Coatonachthodes +Cobra +Coca +cocalero +Coccoloba +Cochlochila +cocksucker +Code +Codelgniter +Codi +Coeini +Coffea +Coffee +Cofidis +Cognitum +Col +Cola +colada +Colder +Colegio +Coleraine +colgadas +Colla +Collective +College +Collegium +Colliers +Collins +collinus +Colonels +Colonia +Colonna +Colorati +Coloratini +Colts +Columbia +Columbus +Colurieae +coma +Comatose +Comcast +Come +Comeback +Comenius +Comet +cometa +Comics +Coming +Commandant +commercial +commersoni +Committee +Commodore +Common +Commons +commune +communication +Communication +Communicator +Communicorp +Como +compact +Compact +Compagnie +Components +Composer +Composites +Comsat +comtulus +Comunale +Conarium +concertante +concerto +Concerto +concinna +conclusus +Concordia +Concur +Condom +Condor +Conepatus +Confessions +Confluentes +Confused +congesta +congolensis +Conjectandi +conjugatum +conntrack +Conrod +Constellation +Consul +Consulting +Continent +Continental +contrapposto +control +Controller +Cookie +Cooliecut +Coolio +Coon +Coop +Cooper +Coors +Copa +Copernicus +Copium +Coppa +Coppi +Coprinaceae +Coptotriche +cor +Cora +Coral +Core +Coregonus +Cori +Corinthians +corium +cork +cornu +Corona +Coronella +coroza +Corp +Corporation +Corps +Corpus +Corsiaceae +Cortina +Cortinarius +Cortoriacum +Cosmic +Cosmographia +Cosmographiae +Cosmopolitan +Cosmos +cossus +Cossus +Costa +costifera +cothurnata +Cottbus +cotton +Cougar +Cougars +Counter +Counts +Courage +Courier +cours +Cours +Coursera +Court +courte +coutumier +Covestro +Crabronidae +Cracovia +crease +Creative +Credit +Credneria +Credo +Creek +Creep +cremoris +Creole +cretaceus +Cricova +Crimson +criollo +crista +Cristatella +Croatia +Croatobranchus +Crombie +Crossing +crossover +Crotram +Crouching +Crunch +crunk +crustosa +Cruzeiro +cruziana +Crvena +Ctenoptana +Cu +Cuba +Cubasa +Cube +Cubic +cucullatus +Cudi +cugzvang +Culture +cumaria +Cummins +Cunard +cunt +Cuora +cup +Cup +Cupola +cuprum +Curonia +curonica +curriculum +Curriculum +cursarius +Cursica +Cursor +Curtipleon +Curva +custos +Čavphrajas +Daei +Daelim +Daeng +Dagbladet +dancehall +Dancer +Danmark +Dare +Darioritum +Darjeeling +Darkchild +Darkthrone +Darmstadt +Dassault +Dastarcus +Dastherdi +de +Dead +Dealer +Dear +death +Death +deathcore +Deathmatch +deathnroll +Debereinera +debilis +deco +Deco +Dedication +Deep +Deepdale +Deere +default +defectus +Deilephila +Delicatesssen +Deloitte +Demokratische +Depeche +dermosphenoticum +Deschu +Design +Desmanthus +Destruction +deutsch +Deutsche +Deutschen +Deutschland +Devastation +Devices +devonicum +Devononchus +dharma +Diamonds +Diaphanopterodea +Dice +dich +Dich +Dichterhalle +Dick +Digital +Dijon +Dilucu +Dimension +Direction +Directions +Director +Dirt +Discina +Disciotis +Disco +Discord +Dissostichus +Distalgesic +Disturbed +Divided +Divisie +Division +Dj +Doc +Docker +Docs +doctor +Dodge +Dolphin +Dolphins +Donauinselfest +Donkervoort +Donnerstag +Dookie +Doom +Dordrecht +Dorotheum +Dorpatensis +douthitti +Dracaenaceae +Drache +Dream +Dreamcatcher +Dreamers +Dreamlanders +dreampop +Dreams +Drive +Drivers +dromedarius +Ducati +ducis +Duck +Ducks +Duncan +Dunctve +Dune +Džaganatha +džāhilijja +Eagle +Eagles +Earth +Eastern +ecash +Ecce +Echeveria +Echis +Echo +Echobrain +Echoes +Echorec +Eclectic +Eclipse +Ecolog +Ecuador +Edelaraudtee +edge +Edition +Edomae +Education +Eduskunta +Eels +Eerste +Eesti +Effector +Ehrenfriedersdorf +Ehrhardt +Eight +Eintracht +el +Elachiptera +elaphus +Elastica +Electric +Electrics +Electronics +Elektriraudtee +Eleutherococcus +Elevation +Elisabeth +Eliteserien +Emacs +Emotions +Empioidae +Empire +Endeavour +Eneco +Energie +Engel +Engine +engineer +Enrico +enterica +Enterprise +Entertainment +entrepreneur +Ephedra +Ephemeroptera +ephippium +Ephiridai +Ephraim +Epolethaizivis +epolethaizivju +epolethaizivs +erchercoga +Erchercogiste +erchercogs +erchercogu +Erdbeermarmelade +Eredivisie +Ericsson +Erzgebirge +Escherichia +Escort +esculenta +Esculenta +esculentum +Esteghlal +Ethereal +ethernet +Ethernet +Ethical +ethos +Eucirripedia +Euclid +Eucopia +Eugenia +Eulachnus +Eunotia +Euphilotes +Euphorbia +Euphoria +Euptoieta +Eureka +Eurema +Eurocontrol +Eurocorps +Eurofighter +Euronics +Europa +europaeus +Europe +Eurosatori +Euroscola +Eurosport +Eurostar +Eurostat +Euskadi +Euskal +Euskaltel +Eutelsat +Eutetrapha +Eutheria +Evanescence +Evangelion +Evangelium +Evening +Event +Ever +Everlast +Evermore +Evernia +Evernote +Everton +Evertons +evil +Evolution +Fabbrica +face +Face +facebook +Facebook +Faces +facie +Facing +factoid +Faith +Falco +Falcon +Falcons +fasciculus +Fauchard +Fe +Feeling +Felicianova +Felicity +Fences +ferrugineum +Festschrift +fffffffffffffffffffft +fibromiomu +Ficedula +fiche +fichet +Fickaraldo +Fidelio +Field +Fieldston +Fierce +Fifteen +Fighter +Fighters +Fighting +Fightstar +Filarmonica +film +Film +Filmfare +Films +Filtronic +Final +Finegana +Finger +First +Fish +Fisht +fitchet +Five +Flashlight +Flintholmene +floricola +fluorescenci +Focus +Foot +Footlights +Footprint +Forbes +foreshortened +foreshortening +Forest +Forests +Forever +Formica +Formicoidea +Formics +Forschungen +forshortened +Fortitude +Fortitudo +Fortune +Fotball +Fourdrinier +fractional +fractus +Francesco +Francisco +Franciscus +Frantic +Fratercula +Frecce +Free +Freedom +Freezers +Freitag +French +fresco +Fuchs +fuck +Fucus +furcata +furfuracea +Fusion +Gaagudju +Gaara +Gaas +Galactic +Galeatus +Galerucella +Gallaecia +Game +Games +Gang +garage +Garbarnia +Gardenback +Gardnerella +Gasteruption +Gear +Geas +Gee +Geenna +Gees +geeza +geezu +General +Generali +Generals +Generation +Generative +Generic +Genesis +Genicanthus +Genie +Genius +Genk +Gennembrud +Geocell +Geocosmos +Geographia +Geonenertes +Geordie +Germ +Germania +Germanica +Germanus +Gernika +Gerolsteiner +Gersicania +Gersicke +Geschichte +Geschichtsinitiativen +Gesellschaft +Gesoriacum +Gespe +Gespenst +Gesta +Gestaltungslehre +gestatoria +Getafe +Getica +Getinge +Geum–Siversia +ghel +Ghostface +Giant +Giants +Gillette +Gilmore +Gimnasia +Ginkgolepis +Ginnastica +Giorgio +Girl +Girls +Give +Glaceau +Glaschu +glaucescens +Glaucus +Globe +Globetrotters +Gloeocapsomorpha +Gloeopeniophorella +Gnathifera +Gnathophausia +Gnorimus +Godflesh +Godmother +Goldhand +Gone +Good +Goodreads +Google +Gorica +gracilicorne +gracilis +graeca +Graetz +Grammatica +Granuloreticulosea +Green +Greenland +Greenpeace +Grizzlies +groove +Grossopterus +Group +Guaranteed +Guldbagge +Guldbaggen +Gummitehdas +Hachette +Hadriaticum +haemorrhoidaria +Hafeet +Hallelujah +Hamiltonella +Handbuch +handsfree +Hannibal +Hanzeatica +Haplic +hardcore +Hardenbergplatz +Harthgrepa +Harthgrepas +Hasenpoth +hash +Hasharon +Hashcash +hatha +Hattrick +heading +heal +Health +healthcare +Heart +heartbeat +Heartbeat +Hearthstone +Heat +Heathen +Hechtsheim +Hecla +Hector +Heels +Heerenveen +Hellendoorn +Helmsing +Helochares +helophoroides +Her +Heracles +heraldica +hercegovinae +Heresiarh +Heroes +Herrenmenschen +Herschel +Hersilia +Hertha +Hesketh +Hessisches +Heterobranchia +Hieracium +High +Highjump +Highland +hircus +Hirsacker +Hitachi +Hodotermes +Hoegaarden +Homecoming +Homesdale +Hooks +Hool +Hosomichi +House +htm +html +Huashuo +Hubble +Huesca +Huez +Hughes +humanitarica +Hungarian +Hungarica +Hunter +Hurricanes +Ibn +Ice +Ichakam +Ichaks +Ichneumon +Icosium +Identified +Igloopol +Illaenus +Illiberis +illudens +Imagine +Imantica +Imatran +Imperii +Impregnable +in +In +Inc +Incentro +Inch +incognito +inconspicua +Independencia +Independent +Independiente +Indestructible +India +Indian +Indiana +Indianapolis +Indonesia +Industrijalec +inertae +Inflicted +Interactive +International +Investigator +Invincible +ipchains +Ipukarea +irmā +Irresistible +Isolierkupplung +israel +Israel +Italico +Italicum +Iudaea +Ivantcha +Ivantchas +Jaan +Jaccuse +Jack +Jackass +Jade +Jagged +Janiosteus +Java +Jazz +Jean +Jeep +Jeeperes +Jehan +Jerometichenoria +Jesi +Jesus +Jet +jetpack +Jets +Jetzt +jiedushi +Jindržichūvhradecu +Jinjer +Johann +Johanneskirche +Joseph +Journalismfund +jr +Jugend +Jugoinspect +Juice +Juilliard +Juive +Jungheinrich +Jungholca +junior +Junior +Juniors +Junkers +Juno +Junun +Jupiler +Jupiter +Jurjunghaja +Kaapeli +Kaapelitehdas +Kahrkling +Kailash +Kallithea +Kampfgruppe +Kaninghems +kannadas +Kapšaghaja +Karaghandi +Kardashian +Katakerghona +Kāthiāvāras +Kebangsaan +Kecheek +Keef +Keeler +Keep +Kehakultuuris +Kenfack +Kenghirota +Kesselberg +Kettenschiff +Kettenschlepper +Kettenschleppschiff +kharošthi +Khoratpithecus +kibuca +kibucos +kibucu +Kickstarter +Kieltolakiliitto +kierchekh +Kinect +Kirche +Kirtharu +Kiuchi +kiviseen +km +Knickerbocker +Knickerbockers +Knicks +Knights +Koch +Kochanku +Kodachrome +Kokemeenjoki +Kollection +komanu +Koncerthalle +Kontho +Koos +Koove +Koovee +Korinthos +Kronenbourg +Kunsthalle +Kuu +Kuzmickaite +Kvarachelija +Kvarachelijas +Laappeenrantas +Laccognathus +Lacerta +Lacetti +Lachistan +Lachnus +Lactarius +lactea +Lactifluus +Lacuania +Lacus +Laeiszhalle +laevis +Laibach +Lamborghini +Lammechinus +Landscape +lapponicus +Last +latgalica +Lattelecom +Launch +Le +Leafs +League +Lecce +Leccinellum +Lecco +Lechia +Leclerc +lecot +Lee +Leen +Leica +Leicester +Leipzig +Leipziger +Leptocephalus +Leptoglossus +Lethal +Lethrus +Lettgallia +Lettische +lhotshampa +Liberibacter +Liberobacter +Licencitorgam +Licensed +Liceu +Lidar +Liebesfreud +Liebesleid +Liebfraumilch +Lieeeneee +Life +Ligariella +Light +lightering +Lightning +Lights +Lightstorm +Lignica +Limerick +Limnognathia +Lincoln +Line +Lionsgate +Liophis +Lithodidae +Lithuana +Lithuanica +littorea +Littorina +Lituanica +Live +Liverpool +livica +Livonicae +Livrustkammaren +Loch +Lochmaea +Lockheed +Lodeacanthus +Lodsch +Logicool +Longhorn +longicornis +Longue +Lophotrochozoa +lorica +Loringhofenu +Louise +Lousanna +Love +Ltd +Lucanus +Lucas +Lucasfilm +Lucchese +Lucus +Ludacris +Lufengpithecus +Lupiae +lusitanicus +Luthien +Mac +Maccabi +Machine +Macintosh +Maclovium +Maconellicoccus +Macpherson +macrocephalus +Macrochelidae +macrodon +Macropocopris +macropodidae +Macrosiphoniella +Macrosiphum +Macrotermes +Maculinea +Mademoiselle +Magazine +Magic +Magica +magick +Magnificent +Magrosiphum +Mahshavah +Mainhattan +Mainhouse +Malachi +Malpighiales +Managament +Management +Manghistau +Mannschaft +Manumatic +Maps +Maracanazo +marathi +Marathon +marathu +Marāthvādas +March +Margherita +Marienhausen +Marinebrigade +Mariscal +Markkleeberg +Marseille +Marshal +Marshall +Marvel +Mastering +Matchroom +Mathematicae +Mathematische +Mathurā +Mathuras +Matt +Mauthauzenas +Mauthauzenes +Maven +Maverick +Mavericks +Mavic +Mavourneen +Mcheta +mclaughlinae +Mecaplast +Mechanics +Mechel +Mechelen +mecotinta +medica +Medical +Medicea +Medicine +Mediomatrici +Mediomatricorum +Megadeth +Megaman +Megascolia +Melodifestivalen +Menace +Menashe +mentomandibulare +Mephisto +Mercatone +Mercedes +Mercedez +Merch +Merci +Mercosur +Messecenter +Messerschmitt +Metalac +metalcore +metallescens +metallica +Metallica +Method +Metteniusales +Mg +Miami +Mich +Michael +Michelin +Michoacanā +Micral +Microcerasus +Micron +Micronas +Microphones +Microsoft +Micul +Middelzee +Middlesborough +Middlesbrough +midget +Midnight +Mielec +millefolium +Millionaire +Mincemeat +Mind +Mine +Minecraft +Minelab +Minor +Minudobrenija +Miotapirus +Miracle +Mirrors +Mission +Mitropolia +Mitsubishi +Mittelrhein +Mobile +Moghbeli +Moguntiacum +monachus +Monaco +Monarchs +Monatsschrift +Monchengladbach +Monferato +Monnaie +Monochamus +Monolith +Monsieur +Montepaschi +Montpellier +Moon +Moondance +Moondog +Moonlight +Moonzund +Moorea +Moose +Mooseheads +Morbillivirus +Morchella +Morcheln +Morph +Morphini +Morricone +Mosaic +Moscoviensis +Mother +motherfucker +mourerae +Mouscron +Mr +Mr.Andersen +Mrs +mucedo +Much +mūlasthāna +Multicar +multishot +multocida +Munerae +Murcia +muscaria +Muscicapa +Museum +Mushroom +Music +musica +Nabeghlavi +nach +Nacht +Nachtigall +Nachtmusik +Nacional +Nactus +Naised +Naissos +Namecoin +Nanhigganeuck +nasicornis +National +Naturforscher +neanderthalensis +Nectiopoda +Nectocader +Neophron +Neotraginae +Neovison +Nephrops +Nerdfighteria +Nerdfighters +Nevermind +nh +Nice +Nicht +Night +Nights +Nivathāmrongs +n,nnn +Noah +Noctiluca +noctua +Noon +norice +norrice +North +Northrop +Notebook +Nothing +Notholithocarpus +Nothorstdālene +Nottingham +npn +nslookup +nuchale +Nudibranchia +Nuphar +nurice +nurrice +Nurse +Obrenovac +obscura +occidentalis +occlusio +Ocean +Ocimum +ocreata +Ocriculum +October +octopus +octree +Oenanthe +of +Of +Offenbach +Office +officer +officiranum +Ogg +Oghre +Oldsmobile +Olimpico +Omphalotus +online +Online +Onthaphagus +oostegītiem +oostegītu +Open +Openchrome +Operation +Opeth +Opetiidae +Ophiura +Opisthokonta +or +Orange +Orbcomm +Orchestra +Orchestre +Orchid +Orchis +Oreochromis +Organisation +orghanismus +Orthopilot +Oscinella +Oscorp +Ost +Othniella +Otodontidae +Outotec +ovambolandicus +Overlook +P +Pacer +Pacers +Paceville +Pacific +Pacifica +Pacis +Pack +Packaging +Packers +Paederus +Pain +pajaca +Pajaci +Palac +Palace +Palaelodidae +Palaeolagus +palaeosinensis +Palaeotragus +Palitzsch +Panasonic +Panathinaikos +Panelboard +panniculus +pannonicum +Panther +Panthera +Panthers +Papeete +Pappelstadion +Paramount +Paranaense +paranuchale +Parasuchus +parathormonam +parathormonu +Park +Parlophone +Parool +parteur +Pascal +Pasteurella +Path +Pathet +Pathfinder +pauciflorum +Pb +Peachocarididae +Pearl +Pearlman +pedunculatus +Peep +peering +Pegasoferae +Peijetheme +Pemphis +Penarmeniaca +People +Perodicticus +Perpetuum +Perseverances +perspicillata +Petroleum +Petrusskirche +Peutingeriana +Pha +Phacochoerus +Phaedo +Phalanta +phalantha +Phaneropleuridae +Phantoms +Pharma +Pharmaca +Pharmacopa +Pharsalia +Phaseolus +Phatnoma +Phatnomatini +PhD +Phelsuma +phenakistes +Phi +Philae +Philharmonia +Philippine +Philips +Phillies +Philomena +Philosophumena +Phlebotominae +Phoca +Phocoena +Phoenicurus +Phoronis +Phoronopsis +Phosphatherium +Photinus +Photoshop +Photuris +Phratora +Piacenza +Piazza +Piccolo +Picken +Picker +Picnosteus +Picromerus +picturata +Picture +Pictures +pictus +Pieazzetta +Pieezere +Pieezeres +Pilskalnee +Pinchas +Pinguine +Pinguins +Pioneer +Piratpartiet +Pisaurum +Piscaria +piscis +Pitcairn +pitch +Pitchfork +Pittsburgh +Pivdenne +Pizza +Plac +Placar +Place +Placentalia +Placentia +Places +Plagiodera +Planck +Platerodrilus +Plat.One +Plohn +Plumbaria +Plurirum +Pochovischali +Podzoluvisols +Poecile +Poecilochirus +Polacanthus +Politehnica +poliziottesco +polonica +Pomacanthidae +Pomacentridae +Pontificalis +Pontificio +Pooh +pool +Pool +Poor +Poptropica +Porsche +Porthedlendā +Portsmouth +Portucale +Portugalete +position +Possibla +Potamotherium +Potiicoara +Prachanda +practica +praearticulare +praecursor +Praesens +Praetuttiorum +Prager +Prague +prearticulare +Precision +Premicrodispus +Presence +prethābsburgu +Preussen +Pristichampsus +Proboscidea +Proceedings +procella +Procellariiformes +Procera +Productions +Project +Prometheus +Prospector +Prost +Proteaceae +proteinoplastiem +Protoceratidae +Protocoleoptera +Protrepticus +Pruzze +Psammolepis +Psammosteus +Pseudoalthenia +Pseudoboletus +pseudomallei +Pseudomonas +Pseudostigmatidae +pseudosulphureus +Psiadia +Pterocomma +Pu +pulchella +punctum +Punctum +Punicum +Purapilpos +purpure +purušārtha +push +Rabbitohs +Race +Racing +Rack +Racom +Racon +Rademachersmedjorna +Radiotechnika +Ralph +Ranck +Rapsodie +Ratiopharm +Raudtee +Razzamatazz +React +Reading +Realsongs +Reamonn +Rebecca +Recife +recitando +Recoil +Recommended +record +Records +reductio +Reebok +Registration +Reiches +Reichsarbeitsdienst +reichstāgi +Rekall +Reliance +Rema +Repack +Research +Resistance +Resolute +Respect +Rh +Rhadinacanthus +Rhaphigaster +Rhapsodies +Rhegium +Rhenus +rhesus +Rhine +Rhinoceros +Rhipiceridae +rhiziogenes +Rhizobium +Rhizocorallium +Rhizoctonia +Rhizodontidae +Rhizodus +Rhizomastigina +Rhodes +Rhombocoleoidea +ri +Ricardo +ricchi +Rienci +Riesling +Rigasche +Right +Righteous +Rihthofena +Rihthofenam +Rihthofenu +Rimac +Ritmico +Roadside +Roadster +Rocco +Rocella +Rochea +Rocher +Rock +Rocket +Rockets +Rocks +Rockstaar +Rocznik +Rogue +Room +Roosevelt +Rooste +Roosters +Root +root.hints +Rosco +Roshan +Roshen +rosica +Rothmans +Rotterdam +Roudourou +Rouge +route +Routemaster +Rovers +rubicunda +rubrilacteus +Rucavietis +Ruch +Ruche +Rue +Rueter +Runaround +Rundschau +Rungholtas +Rupephaps +Rush +Russica +Russulaceae +rusticana +Ruthenia +Rutoideae +Sa +Saccharum +Sacculina +Sachsen +Sachsenring +Sadd +Saddledome +Saeco +Salcomp +Salernitana +salmonicolor +Saloon +Samche +samfundetin +Sampdoria +Samphanu +Sanchez +Sarcophaga +sarkoscīfa +Sarmaticum +Sarsfield +Sasha +Sauleorum +Sb +Sc +scabiei +scabra +Scacchistica +Scaevola +Scala +Scaled +Scalidophora +Scanderbeg +Scandia +Scandinavium +Scandlines +Scarabaeinae +Scaritinae +Scaumenacia +Scelotes +sceptera +Schachberichte +Schachspiels +Schachzeitung +Schaden +Schauspieldirektor +Schauspielhaus +Schedonorus +Scheimeln +Scheldeprijs +Schere +Scherens +Schering +Schiaparelli +Schimmelcultures +Schindleriidae +Schistosoma +Schistosomatidae +Schistosomatium +Schizophora +Schizosteus +Schleifseile +Schlesisches +Schmidt +Schneider +Schola +scholarium +School +Schramocaris +Schriften +Schuld +Schulrucksack +Schultzenhof +Schutzmann +Sciences +Scolia +Scoliidae +Scooter +Scops +Scorpaenidae +Scorpions +Scotia +Scotiabank +Scotland +Scottish +scramjet +Scream +Screaming +Scriptum +Scriptura +Scriptus +Scrobiculati +scrofa +Scrolls +Scuderia +Scuffs +Scuola +Se +Sea +Search +Searchlight +Seas +season +secco +seconda +secret +sectile +Sehnsucht +Selespeed +Semiconductor +semīthamītu +Sentences +Senthūberta +Sephanoides +Serco +Serranidae +Service +Services +Setukeste +Seventeen +Seventh +Severance +Shack +Shaft +Shahed +Shake +Shakhtar +Sharks +she +She +Sheffield +Shelbourne +Shell +Shenfu +Shenlong +Shenzhou +Sheraton +Sheriff +shi +shift +Shimpaku +Shining +Shinkansen +shio +Shippuden +shit +Shkupi +Shock +Shoes +shonen +Shooting +Shopping +Short +Shot +shotgun +Shri +Shrine +Shtimme +Shueisha +Shuttle +Sicherheitsdienst +Siculus +Side +Sidemen +Silmarillionam +Silmarillions +Silverchair +simplicius +Singhs +Siphonotretida +Siphunculina +Sivapithecus +Skateboarder +Skeemella +Sleeping +Slovalco +Smarterphone +Smokepurpp +Smoothie +snaphance +Snoop +Soccer +Solanaceae +Solider +Somnambula +Somniorum +Somphona +Sonderkommando +Sonneratia +Sophia +Sorcerer +Sosiocaris +Sound +Soundcloud +Soundgarden +source +Source +South +Southampton +sp +Space +Spaced +Spacelab +Sparganium +Special +Spectre +Spectrum +Speed +Speedo +Speedster +Spektr.press +Speothos +Spherics +Splash +sporthalle +Spotlight +spp +Sprite +Sr +St +Staatstheater +Stadium +Stadshagen +Staeben +stafetes +Stage +staghaundiem +Starbucks +Starchild +Starcourt +starfish +Starfish +Stargate +Starlight +Starliner +Stasi +Station +Steadicam +Steam +Steamhammer +Steaua +Steel +Steelers +Steinernematidae +Step +Stephanitis +Stiatha +Still +Stockmann +Stone +Stones +Storhamar +Stories +straight +Strasbourg +Strasdenhof +Strasse +Strassenhof +Stream +Streckenbach +Street +Streetshop +Streptococcus +Strethernas +Stuckenia +Studios +Sturmabteilung +Stuttgart +subfurcata +suboperculum +subreticulata +Subtilor +subtorminosus +Subversion +successionis +suckurs +Sucre +Suebicum +Suicidal +Suicide +Sulphur +sulphureus +Summit +Sun +Sunbird +Sunderland +Sunlight +Sunliner +Sunrise +Sunsel +Suolle +suomaa +Suomen +Superbike +Supercoppa +Superettan +Superman +Superunion +Supremus +Surgutneftegaz +surround +Surtees +Sus +Suseja +Suspiro +Suurhall +Svensktoppen +Svetichoveli +Szellemi +Šachmeistara +Šmidchens +Tabernanthe +Tabletop +taccada +Tachiniscinae +Tae +Taenarium +Tagblatt +Tageblatt +Tagesspiegel +Taibhdhearc +Take +Taldikorghana +Talghars +Talvivaara +Tampereen +Tanacetum +Tangkap +Tanheizers +tardigradus +Tarptautinis +Tartuosteus +Tatanagar +tatarica +Tathils +Taugenichts +Tcl +Tealliocaris +Team +Tech +Technigaz +techno +Technocentrs +Technologic +Technologies +technoloģiju +technon +Tecnico +tecti +Tecumseh +Teen +Tegenaria +Tegeus +Tegueste +Telecom +Telecommunications +Telegraph +Teletoon +Telmatobacter +Temple +Teplice +th +Thalassoleon +Thales +thaliana +Thank +Thankful +That +Thaumatococcus +the +The +Theate +Theater +Theatre +Theatrum +thefakesoundofprogress +Thema +Theme +Themognatha +then +Theodossia +Theologia +Theoria +Theresimima +Therma +Thermessu +Thermitanorum +thibetanus +Thief +Thiel +Thieves +thin +Things +Thirteenth +This +Thistle +thiutisk +tholussum +Thomson +Thook +Thor +Thoracotremata +Thottu +thrash +Thrashers +Three +Threshold +thresholding +Thriller +thug +Thule +Thum +Thunder +Thunderbird +Thunderbirds +thung +Tiangong +Tianhe +Tianzhou +Ticket +Tidende +Tiergarten +Tiger +Tigers +timanicus +Timbaland +Timbiriche +Times +Tingini +Tingis +Titanic +tlacahuapahualiztli +Tobacco +Toce +Toddalioideae +Tokatlian +Tomarctus +Tonight +Tonneau +Too +toob +Toolbar +Tools +Toompuiestee +Tooru +toorumees +tormentorii +torminosulus +torminosus +Tosb +Tosca +Toshiba +Tottenham +Touch +Toulouse +Tour +Tournament +Tours +Tourtel +Trabzonspor +Trachodon +trackpad +Trackpoints +trautscholdi +Treadstone +Tree +Treer +Trees +Trefl +Tremarctinae +Tremarctos +Treroninae +Tribute +Triceratops +Trichobilharzia +Trichocolea +Trichoderma +tricinātāju +Tricolor +Tricolori +tricornis +Trin +Trinacria +Trinbago +Triniusa +Trinucleus +Trippie +Trittstufen +Trochetia +Trochidae +Trofeo +Tropicana +trucks +True +Trust +Truth +Tshekardocoleidae +tsillian +tsu +Tudeh +Tuhkatriinu +Tuhkimo +Tumbling +tumefaciens +Turkcell +Turnerschaft +Turnhalle +Tusculan +tz +Uivištathuaha +Ukrhidroenergo +Ullevaal +Ultimate +Umbrella +Unbehagen +Uncle +Uncover +Under +underground +Underground +undulatus +Unfabulous +Unforgiven +unforshortened +Unian +Unicaja +Unichrome +unicode +Unicode +Unified +Union +Uniona +Unions +Unipolise +United +Universidad +universum +Unokomanda +Unreal +Unter +Until +Up +Uponor +Upper +Upsilon +Urbinum +Urdaneta +Urheberrecht +urophthalmoides +Uropodidae +Urskumug +Ursus +Urtica +Usenet +uses +Uses +Usher +Ushuaja +Utrecht +utt +Uzbeghana +Vaanil +Vacansoleil +vacante +vacca +vaccinia +Vahliales +Valvaribifidum +Vammas +Vampanoagu +vampumpeag +vānaprastha +vandermeeschii +Vanellus +Vanemuine +Vanenica +Vasco +Vasconcellea +vaticanus +Vaughans +Vee +Veetamoo +Vehicle +Veikkausliiga +velebitica +Vercelli +Ver.di +versicolora +Vesica +vesiculosus +Vespidae +Vespoidea +Vespucius +Vesthardiana +VfB +VfR +ViA +Viacom +Vigione +virescens +Visual +vitae +vitulina +vjl +Vlogbrothers +Vnešekonombank +Vocal +Vogue +Voices +vokshola +Voldersgracht +Volksbeauftragten +Volucella +volume +Volunteers +Voshod +Vossloh +Vostoc +vs +vulgatissima +Vulture +zacatechichi +Zaffah +Zagodzerno +Zannichellia +Zappanale +Zeelandia +Zenico +Zeppelin +Zetekella +Zhang +ZiM +Zn +Zoantharia +Zodiac +Zoe +Zohar +Zooamata +Zoopagales +Zoosk +Zootropole +Zospeum +zouk +Zvolles +Enough +Gael +Frufru +ji +Iabi +kuce +kuces +kucei +maita +maitas +maitu +maitasgabala +sasodīti +sasodītā +sasodītās +sasodīts +sasodītajai +sasodīta +sasodītais +sekss +seksu +seksa +seksīgs +seksuālu +seksuāli +seksuālo +seksīgos +seksīgāks +seksīgajām +seksuāla +seksuālajās +seksam +seksīgi +seksualitāti +mērglis +mērgli +mērglim +nogalināt +nogalināšu +nogalināja +zāles +zāle +zāli +narkotiku +narkotikas +stroķi +stulbie +stulbās +stulbo +stulbi +stulbu +stulbajiem +stulba +dirsu +nedirsini +dirsina +dirsā +Sadirsi +ZZwordsWeDoNotWantFromEuroparl +Komisija +Komisiju +Komisijas +Komisijai +Padome +Padomei +Padomi +kungs +Komisāre +Komisārs +Eiropa +Eiropas +Eiropai +Eiropu +Lisabonas +stratēģijā +administratīvo +saskaņošana \ No newline at end of file diff --git a/src/rules/lv.toml b/src/rules/lv.toml new file mode 100644 index 0000000..df5ea86 --- /dev/null +++ b/src/rules/lv.toml @@ -0,0 +1,109 @@ +allowed_symbols_regex="[aābcčdeēfgģhiījkķlļmnņoprsštuūvzžAĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ,–:\\.!\\? \"„“]" +matching_symbols = [["„", "“"]] +even_symbols = ['"'] +min_trimmed_length = 6 +min_word_count = 7 +max_word_count = 14 +min_characters = 2 +may_end_with_colon = false +quote_start_with_letter = true +needs_punctuation_end = true +needs_letter_start = true +needs_uppercase_start = true +broken_whitespace = [" ", " ,", " .", " ?", " !", " ;"] +segmenter="python" +replacements = [ + [" ", " "], + ["…", "..."], + ["—", "–"], + ["−", "-"], + ["«", "\""], + ["»", "\""], + ["‘", "\""], + ["’", "\""], + ["“", "\""], + ["”", "\""], + ["'", "\""], + ["\"\"", "\""], + ["\"\"", "\""], + ["\"\"", "\""], + ["‚", ","], + ["ŗ", "r"], + ["Ŗ", "R"], + + # Common abbreviations + ["p.m.ē.", "pirms mūsu ēras"], + ["t.i.", "–"], + ["g.k.", "galvenokārt"], + ["pr.Kr.", "pirms Kristus"], + ["pr. Kr.", "pirms Kristus"], + ["p Kr.", "pirms Kristus"], +] +regex_replacement_list = [ + # Un glue glued sentences + ["\\ [AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž\"]{3,}\\.[AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ][aābcčdeēfgģhiījkķlļmnņoprsštuūvzž,:]{1,}\\ ", ".", ". "], + ["\\.\\.\\.", "...", "... "], + ["\\.\\.", "..", ".. "], + + # Split long sentences + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", bet ", ". Bet "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", un ", ". Un "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", ko ", ". Ko "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", ka ", ". Ka "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kā ", ". Kā "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", jo ", ". Jo "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", ja ", ". Ja "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", lai ", ". Lai "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kas ", ". Kas "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kur ", ". Kur "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kad ", ". Kad "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kam ", ". Kam "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurš ", ". Kurš "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kura ", ". Kura "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuri ", ". Kuri "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurā ", ". Kurā "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuru ", ". Kuru "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurus ", ". Kurus "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuras ", ". Kuras "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuram ", ". Kuram "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurai ", ". Kurai "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurām ", ". Kurām "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuriem ", ". Kuriem "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kuros ", ". Kuros "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kurās ", ". Kurās "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", tomēr ", ". Tomēr "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", taču ", ". Taču "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", kādu ", ". Kādu "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", tāpēc ", ". Tāpēc "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", tātad ", ". Tātad "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", tad ", ". Tad "], + ["\\b(?:\\S+\\s+){15,}\\S+[.!?]", ", tas ", ". Tas "], +] +abbreviation_patterns = [ + "[AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ]+\\.", + "Dz\\.", + "Dž\\.", + "Fr\\.", + "[AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ]+\\.*[AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ]", + # For AS, SIA, NVS and similar + "[AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ]{2,}", + # Coommon abbreviations + "t\\.s\\.|t\\. s\\.|t\\.s|t\\. sk\\.|mēn\\.|m\\.ē\\.|el\\.|[Nn]r\\.|[Dd]r\\.|[Pp]rof\\.|[Ll]td\\.|[Pp]iem\\.", + "t\\.|kand\\.|inž\\.|Sv\\.|Kr\\.|Ād\\.|vsk\\.", +] +other_patterns = [ + # internet addresses + "\\.[com|net|org|lv|lt|ee|de|fi|fm|js|py|ru|am]", + + # roman numbering + "[MCLXVI]\\s", + + # - Sentence delimiter can only be at the end of a sentence + # - No words with only one letter (" a.", " a", " a ") + # - Mixed upper/lowercase in words (LaSi - mostly chemical elements?) + "[\\.|\\?|!].+$", + "(\\s[A-Za-z]{1}[\\.|\\?|!]*$)|\\s[A-Za-z]{1}\\s", + "[a-z][A-Z][a-z]", +] + +disallowed_words = [ ] \ No newline at end of file diff --git a/src/segmenter.rs b/src/segmenter.rs index 56ccf28..1502fc3 100644 --- a/src/segmenter.rs +++ b/src/segmenter.rs @@ -6,6 +6,7 @@ pub fn split_sentences_with_python(language: &str, text: &str) -> Vec { "de" => split_sentences_with_python_de(text), "bn" => split_sentences_with_python_bn(text), "tr" => split_sentences_with_python_tr(text), + "lv" => split_sentences_with_python_lv(text), _ => { panic!("{} is not supported for Python segmenter, please implement it or remove the segmenter rule", language); }, @@ -86,6 +87,23 @@ pub fn split_sentences_with_python_tr(text: &str) -> Vec { ctx.get("split_sentences") } +pub fn split_sentences_with_python_lv(text: &str) -> Vec { + let ctx = Context::new(); + + ctx.run(python! { + import nltk + + try: + nltk.data.load("tokenizers/punkt/english.pickle") + except LookupError: + nltk.download("punkt") + + split_sentences = nltk.sent_tokenize('text) + }); + + ctx.get("split_sentences") +} + #[cfg(test)] mod test { use super::*; @@ -114,6 +132,14 @@ mod test { assert_eq!(split_sentences_with_python(language, text).len(), 2); } + #[test] + fn test_segmenter_lv() { + let language = "lv"; + let text = "Esmu teikums. Es arī!"; + + assert_eq!(split_sentences_with_python(language, text).len(), 2); + } + #[test] #[should_panic] fn test_segmenter_invalid_language() {