Limit names to 100 bytes

src/main/java/dev/morling/onebrc/CreateMeasurements3.java

@@ -21,6 +21,7 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.concurrent.ThreadLocalRandom;
@@ -113,6 +114,9 @@ private static ArrayList<WeatherStation> generateWeatherStations() throws Except
                 while (names.contains(name)) {
                     name = name.substring(1) + readNonSpace(nameSource);
                 }
+                while (name.getBytes(StandardCharsets.UTF_8).length > 100) {
+                    name = name.substring(0, name.length() - 1);
+                }
                 if (name.indexOf(';') != -1) {
                     throw new Exception("Station name contains a semicolon!");
                 }

src/test/resources/samples/measurements-complex-utf8.out

@@ -1 +1 @@
-{ Al=18.5/18.5/18.5, FraijanesOuricur=29.3/29.3/29.3, akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne=19.2/19.2/19.2, arh SisaiOulad AyyadAhirauliyāVilla=7.9/7.9/7.9, arpā=23.7/23.7/23.7, diqabadShahr-e Ko=14.0/14.0/14.0, einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo=11.9/11.9/11.9, e‘utPic=15.9/15.9/15.9, hleyHornchurchLiepājaActonKatiBouiraPort=10.7/10.7/10.7, iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd=18.6/18.6/18.6, ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai=9.8/9.8/9.8, ingtonAlcalá de HenaresTarantoKhowyUr=18.8/18.8/18.8, kohā=28.1/28.1/28.1, l ḨazmĀlampur GonpuraKr=8.0/8.0/8.0, lţiWłocławekMaumere=20.4/20.4/20.4, molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú=5.6/5.6/5.6, n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para=17.2/17.2/17.2, nBanī SuwayfCollege StationBà Rị=-4.7/-4.7/-4.7, ntevedraChilliwackBuuhoodleFarmin=17.5/17.5/17.5, onGwadarCambambeVictoriasPinamalayanKatiolaC=12.0/12.0/12.0, riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam=12.8/12.8/12.8, rlooKaikkudiBernalKamagayaNor=31.8/31.8/31.8, saNāsri=11.8/11.8/11.8, uOn=17.5/17.5/17.5, ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh=2.6/2.6/2.6, yuMekeleGon=15.7/15.7/15.7, ālaV=26.0/26.0/26.0, ž=12.3/12.3/12.3, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ=7.1/10.1/13.0, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè=11.1/17.2/23.3}
+{B=8.9/8.9/8.9, C=38.9/38.9/38.9, CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa=3.0/3.0/3.0, ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS=23.4/23.4/23.4, MirnaPehčevoRopažiGus=16.7/16.7/16.7, PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb=17.5/17.5/17.5, TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka=10.6/10.6/10.6, aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS=25.4/25.4/25.4, burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak=21.5/21.5/21.5, cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS=11.2/11.2/11.2, eLafayetteAsh Shaţ=14.2/14.2/14.2, en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi=11.9/11.9/11.9, epé=28.2/28.2/28.2, hanVarkkallaiPort LokoD=10.9/10.9/10.9, iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM=13.4/13.4/13.4, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul=22.5/22.5/22.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl=11.5/11.5/11.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul=18.5/18.5/18.5, inhoSökeDordrechtPoáLaloG=13.1/13.1/13.1, iudad Melchor MúzquizQuinhámelDa=40.5/40.5/40.5, l ‘=14.6/14.6/14.6, lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait=22.8/22.8/22.8, liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque=12.4/12.4/12.4, lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe=22.5/22.5/22.5, m el Bo=14.6/14.6/14.6, mazunchaleZrenjaninFouchanaSurtPanč=6.7/6.7/6.7, ngoDübendorfC=11.7/11.7/11.7, nt-A=9.2/9.2/9.2, ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon=14.6/14.6/14.6, oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe=17.3/17.3/17.3, oGumlāSamā’=14.9/14.9/14.9, os Reyes de SalgadoCinisello BalsamoKashibaH=20.0/20.0/20.0, picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy=13.0/13.0/13.0, raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat=13.5/13.5/13.5, rhamDera Ghazi KhanMiyazakiBhātpār=21.3/21.3/21.3, rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan=10.9/10.9/10.9, skişeh=12.9/12.9/12.9, venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC=1.9/1.9/1.9, y-le-MoutierSant’ArpinoPljevljaRo=0.8/0.8/0.8, ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK=6.0/6.0/6.0, üSosnowiecTanauanMya=18.4/18.4/18.4, ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E=19.4/19.4/19.4, āng=15.7/15.7/15.7, ġFis=9.6/9.6/9.6, ‘AqabahPembaNowgongQu=12.9/12.9/12.9}

src/test/resources/samples/measurements-complex-utf8.txt

@@ -1,32 +1,45 @@
-kohā;28.1
-lţiWłocławekMaumere;20.4
- Al;18.5
-arh SisaiOulad AyyadAhirauliyāVilla;7.9
-molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú;5.6
-hleyHornchurchLiepājaActonKatiBouiraPort;10.7
-onGwadarCambambeVictoriasPinamalayanKatiolaC;12.0
-saNāsri;11.8
-nBanī SuwayfCollege StationBà Rị;-4.7
-ntevedraChilliwackBuuhoodleFarmin;17.5
-rlooKaikkudiBernalKamagayaNor;31.8
-ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh;2.6
-FraijanesOuricur;29.3
-iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd;18.6
-ālaV;26.0
-ž;12.3
-l ḨazmĀlampur GonpuraKr;8.0
-einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo;11.9
-ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;23.3
-ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;13.0
-ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;11.1
-ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;7.1
-uOn;17.5
-yuMekeleGon;15.7
-riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam;12.8
-e‘utPic;15.9
-akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne;19.2
-arpā;23.7
-ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai;9.8
-n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para;17.2
-ingtonAlcalá de HenaresTarantoKhowyUr;18.8
-diqabadShahr-e Ko;14.0
+aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS;25.4
+picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy;13.0
+lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait;22.8
+āng;15.7
+hanVarkkallaiPort LokoD;10.9
+eLafayetteAsh Shaţ;14.2
+‘AqabahPembaNowgongQu;12.9
+inhoSökeDordrechtPoáLaloG;13.1
+skişeh;12.9
+rhamDera Ghazi KhanMiyazakiBhātpār;21.3
+igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl;11.5
+igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul;18.5
+igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul;22.5
+B;8.9
+C;38.9
+nt-A;9.2
+y-le-MoutierSant’ArpinoPljevljaRo;0.8
+oGumlāSamā’;14.9
+os Reyes de SalgadoCinisello BalsamoKashibaH;20.0
+m el Bo;14.6
+mazunchaleZrenjaninFouchanaSurtPanč;6.7
+ġFis;9.6
+epé;28.2
+ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E;19.4
+iudad Melchor MúzquizQuinhámelDa;40.5
+ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS;23.4
+cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS;11.2
+raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat;13.5
+l ‘;14.6
+TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka;10.6
+MirnaPehčevoRopažiGus;16.7
+üSosnowiecTanauanMya;18.4
+ngoDübendorfC;11.7
+liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque;12.4
+burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak;21.5
+iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM;13.4
+lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe;22.5
+PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb;17.5
+CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa;3.0
+venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC;1.9
+ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon;14.6
+rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan;10.9
+oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe;17.3
+ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK;6.0
+en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi;11.9