diff --git a/src/main/java/dev/morling/onebrc/CreateMeasurements3.java b/src/main/java/dev/morling/onebrc/CreateMeasurements3.java index 23b1b74d89..3910744fa5 100644 --- a/src/main/java/dev/morling/onebrc/CreateMeasurements3.java +++ b/src/main/java/dev/morling/onebrc/CreateMeasurements3.java @@ -21,6 +21,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.concurrent.ThreadLocalRandom; @@ -113,6 +114,9 @@ private static ArrayList generateWeatherStations() throws Except while (names.contains(name)) { name = name.substring(1) + readNonSpace(nameSource); } + while (name.getBytes(StandardCharsets.UTF_8).length > 100) { + name = name.substring(0, name.length() - 1); + } if (name.indexOf(';') != -1) { throw new Exception("Station name contains a semicolon!"); } diff --git a/src/test/resources/samples/measurements-complex-utf8.out b/src/test/resources/samples/measurements-complex-utf8.out index 24436cd584..ac8b7a1493 100644 --- a/src/test/resources/samples/measurements-complex-utf8.out +++ b/src/test/resources/samples/measurements-complex-utf8.out @@ -1 +1 @@ -{ Al=18.5/18.5/18.5, FraijanesOuricur=29.3/29.3/29.3, akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne=19.2/19.2/19.2, arh SisaiOulad AyyadAhirauliyāVilla=7.9/7.9/7.9, arpā=23.7/23.7/23.7, diqabadShahr-e Ko=14.0/14.0/14.0, einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo=11.9/11.9/11.9, e‘utPic=15.9/15.9/15.9, hleyHornchurchLiepājaActonKatiBouiraPort=10.7/10.7/10.7, iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd=18.6/18.6/18.6, ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai=9.8/9.8/9.8, ingtonAlcalá de HenaresTarantoKhowyUr=18.8/18.8/18.8, kohā=28.1/28.1/28.1, l ḨazmĀlampur GonpuraKr=8.0/8.0/8.0, lţiWłocławekMaumere=20.4/20.4/20.4, molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú=5.6/5.6/5.6, n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para=17.2/17.2/17.2, nBanī SuwayfCollege StationBà Rị=-4.7/-4.7/-4.7, ntevedraChilliwackBuuhoodleFarmin=17.5/17.5/17.5, onGwadarCambambeVictoriasPinamalayanKatiolaC=12.0/12.0/12.0, riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam=12.8/12.8/12.8, rlooKaikkudiBernalKamagayaNor=31.8/31.8/31.8, saNāsri=11.8/11.8/11.8, uOn=17.5/17.5/17.5, ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh=2.6/2.6/2.6, yuMekeleGon=15.7/15.7/15.7, ālaV=26.0/26.0/26.0, ž=12.3/12.3/12.3, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ=7.1/10.1/13.0, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè=11.1/17.2/23.3} +{B=8.9/8.9/8.9, C=38.9/38.9/38.9, CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa=3.0/3.0/3.0, ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS=23.4/23.4/23.4, MirnaPehčevoRopažiGus=16.7/16.7/16.7, PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb=17.5/17.5/17.5, TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka=10.6/10.6/10.6, aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS=25.4/25.4/25.4, burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak=21.5/21.5/21.5, cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS=11.2/11.2/11.2, eLafayetteAsh Shaţ=14.2/14.2/14.2, en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi=11.9/11.9/11.9, epé=28.2/28.2/28.2, hanVarkkallaiPort LokoD=10.9/10.9/10.9, iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM=13.4/13.4/13.4, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul=22.5/22.5/22.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl=11.5/11.5/11.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul=18.5/18.5/18.5, inhoSökeDordrechtPoáLaloG=13.1/13.1/13.1, iudad Melchor MúzquizQuinhámelDa=40.5/40.5/40.5, l ‘=14.6/14.6/14.6, lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait=22.8/22.8/22.8, liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque=12.4/12.4/12.4, lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe=22.5/22.5/22.5, m el Bo=14.6/14.6/14.6, mazunchaleZrenjaninFouchanaSurtPanč=6.7/6.7/6.7, ngoDübendorfC=11.7/11.7/11.7, nt-A=9.2/9.2/9.2, ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon=14.6/14.6/14.6, oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe=17.3/17.3/17.3, oGumlāSamā’=14.9/14.9/14.9, os Reyes de SalgadoCinisello BalsamoKashibaH=20.0/20.0/20.0, picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy=13.0/13.0/13.0, raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat=13.5/13.5/13.5, rhamDera Ghazi KhanMiyazakiBhātpār=21.3/21.3/21.3, rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan=10.9/10.9/10.9, skişeh=12.9/12.9/12.9, venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC=1.9/1.9/1.9, y-le-MoutierSant’ArpinoPljevljaRo=0.8/0.8/0.8, ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK=6.0/6.0/6.0, üSosnowiecTanauanMya=18.4/18.4/18.4, ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E=19.4/19.4/19.4, āng=15.7/15.7/15.7, ġFis=9.6/9.6/9.6, ‘AqabahPembaNowgongQu=12.9/12.9/12.9} diff --git a/src/test/resources/samples/measurements-complex-utf8.txt b/src/test/resources/samples/measurements-complex-utf8.txt index 60a9035955..a4b4c2c72c 100644 --- a/src/test/resources/samples/measurements-complex-utf8.txt +++ b/src/test/resources/samples/measurements-complex-utf8.txt @@ -1,32 +1,45 @@ -kohā;28.1 -lţiWłocławekMaumere;20.4 - Al;18.5 -arh SisaiOulad AyyadAhirauliyāVilla;7.9 -molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú;5.6 -hleyHornchurchLiepājaActonKatiBouiraPort;10.7 -onGwadarCambambeVictoriasPinamalayanKatiolaC;12.0 -saNāsri;11.8 -nBanī SuwayfCollege StationBà Rị;-4.7 -ntevedraChilliwackBuuhoodleFarmin;17.5 -rlooKaikkudiBernalKamagayaNor;31.8 -ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh;2.6 -FraijanesOuricur;29.3 -iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd;18.6 -ālaV;26.0 -ž;12.3 -l ḨazmĀlampur GonpuraKr;8.0 -einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo;11.9 -ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;23.3 -ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;13.0 -ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;11.1 -ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;7.1 -uOn;17.5 -yuMekeleGon;15.7 -riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam;12.8 -e‘utPic;15.9 -akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne;19.2 -arpā;23.7 -ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai;9.8 -n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para;17.2 -ingtonAlcalá de HenaresTarantoKhowyUr;18.8 -diqabadShahr-e Ko;14.0 +aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS;25.4 +picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy;13.0 +lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait;22.8 +āng;15.7 +hanVarkkallaiPort LokoD;10.9 +eLafayetteAsh Shaţ;14.2 +‘AqabahPembaNowgongQu;12.9 +inhoSökeDordrechtPoáLaloG;13.1 +skişeh;12.9 +rhamDera Ghazi KhanMiyazakiBhātpār;21.3 +igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl;11.5 +igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul;18.5 +igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul;22.5 +B;8.9 +C;38.9 +nt-A;9.2 +y-le-MoutierSant’ArpinoPljevljaRo;0.8 +oGumlāSamā’;14.9 +os Reyes de SalgadoCinisello BalsamoKashibaH;20.0 +m el Bo;14.6 +mazunchaleZrenjaninFouchanaSurtPanč;6.7 +ġFis;9.6 +epé;28.2 +ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E;19.4 +iudad Melchor MúzquizQuinhámelDa;40.5 +ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS;23.4 +cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS;11.2 +raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat;13.5 +l ‘;14.6 +TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka;10.6 +MirnaPehčevoRopažiGus;16.7 +üSosnowiecTanauanMya;18.4 +ngoDübendorfC;11.7 +liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque;12.4 +burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak;21.5 +iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM;13.4 +lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe;22.5 +PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb;17.5 +CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa;3.0 +venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC;1.9 +ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon;14.6 +rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan;10.9 +oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe;17.3 +ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK;6.0 +en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi;11.9