[sv] Changing character encoding of fsa dict files from ISO-8859-1 to UTF-8

languagetool-language-modules/sv/src/main/java/org/languagetool/tokenizers/sv/SwedishWordTokenizer.java

@@ -0,0 +1,111 @@
+/* LanguageTool, a natural language style checker
+ * Copyright (C) 2008 Daniel Naber (http://www.danielnaber.de)
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+package org.languagetool.tokenizers.sv;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.languagetool.tokenizers.WordTokenizer;
+
+public class SwedishWordTokenizer extends WordTokenizer {
+
+  private static final List<String> QUOTES = Arrays.asList("'", "`", "’",  "‘", "´", ":"); // Jämför vanligt citattecken "”"
+
+  private final String svTokenizingChars;
+
+  public SwedishWordTokenizer() {
+    //remove the apostrophe etc. from the standard tokenizing characters:
+    String chars = super.getTokenizingCharacters() + "_";
+    for (String quote : QUOTES) {
+      chars = chars.replace(quote, "");
+    }
+    svTokenizingChars = chars;
+  }
+
+  /**
+   * Tokenizes just like WordTokenizer with the exception for words such as
+   * "oma's" that contain an apostrophe in their middle.
+   * @param text Text to tokenize
+   * @return List of tokens
+   */
+  @Override
+  public List<String> tokenize(final String text) {
+    final List<String> l = new ArrayList<>();
+    final StringTokenizer st = new StringTokenizer(text, svTokenizingChars, true);
+    while (st.hasMoreElements()) {
+      String token = st.nextToken();
+      String origToken = token;
+      if (token.length() > 1) {
+        if (startsWithQuote(token) && endsWithQuote(token) && token.length() > 2) {
+          l.add(token.substring(0, 1));
+          l.add(token.substring(1, token.length()-1));
+          l.add(token.substring(token.length()-1));
+        } else if (endsWithQuote(token)) {
+          int cnt = 0;
+          while (endsWithQuote(token)) {
+            token = token.substring(0, token.length() - 1);
+            cnt++;
+          }
+          l.add(token);
+          for (int i = origToken.length() - cnt; i < origToken.length(); i++) {
+            l.add(origToken.substring(i, i + 1));
+          }
+        } else if (startsWithQuote(token)) {
+          while (startsWithQuote(token)) {
+            l.add(token.substring(0, 1));
+            token = token.substring(1);
+          }
+          l.add(token);
+        } else {
+          l.add(token);
+        }
+      } else {
+        l.add(token);
+      }
+    }
+    return joinEMailsAndUrls(l);
+  }
+
+  private boolean startsWithQuote(String token) {
+    for (String quote : QUOTES) {
+      if (token.startsWith(quote)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private boolean endsWithQuote(String token) {
+    for (String quote : QUOTES) {
+      if (token.endsWith(quote)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public String getTokenizingCharacters() {
+    return svTokenizingChars;
+  }
+}
+

languagetool-language-modules/sv/src/main/resources/org/languagetool/resource/sv/swedish.info

@@ -1,8 +1,17 @@
 #
-# Dictionary properties.
+# Dictionary properties for swedish.dict (later maybe sv_SE.dict and sv_FI.dict)
 #
 
 fsa.dict.separator=+
-fsa.dict.encoding=iso-8859-1
+fsa.dict.encoding=utf-8
+
+#fsa.dict.speller.runon-words=false
+# Inte säker på att detta är en bra idé, men kan tyskarna ...
+#fsa.dict.speller.ignore-diacritics=false
+# Lägga till så snart som möjligt
+#fsa.dict.frequency-included=true
+
+fsa.dict.speller.replacement-pairs=ß ss, β ss, z s, s z, ai ei, ei ai, oi eu, oi äu, eu oi, eu äu, äu eu, a ä, o ö, u ü, ae ä, e ä, ä e, e é, ee é, é ee, oe ö, ö oe, o oo, ue ü, Ae Ä, Oe Ö, Ue Ü, q qu, Rh R, R Rh, rh r, th t, t th, T Th, Th T, g ch, ch g, fl fl, f ph, ph f, Ph F, F Ph, f pf, pf f, Pf F, F Pf, mft nft, F V, W V, f v, v f, w v, i y, j y, ü y, J Y, d t, t d, b p, p b, g k, k g, z c, c z, k c, c k, k ck, ck k, ck kk, kk k, ks x, x ks, g ğ, s š, c ć, ç c, c ç, o oh, oh o, u ou, ou u, ug ung, ig ing, ing ig, sch sh, sh sch, tje sje, sje tje, k sj, sj k, stj k, k stj, stj sj, sj stj, sch sj, sj sch
 
 fsa.dict.encoder=SUFFIX
+

languagetool-language-modules/sv/src/main/resources/org/languagetool/resource/sv/swedish_synth.dict_tags.txt

@@ -1,3 +1,9 @@
+"
+#
+,
+.
+:
+;
 AB
 IN
 JJ:BF
@@ -26,6 +32,7 @@ NN:BF:SIN:NMNOM:UTR
 NN:BF:SIN:NOM:NEU
 NN:BF:SIN:NOM:NON
 NN:BF:SIN:NOM:UTR
+NN:OF:NON:NOM:UTR
 NN:OF:PLU:GEN:NEU
 NN:OF:PLU:GEN:NON
 NN:OF:PLU:GEN:UTR
@@ -39,8 +46,14 @@ NN:OF:SIN:NOM:NEU
 NN:OF:SIN:NOM:NON
 NN:OF:SIN:NOM:UTR
 PM:GEN
+PM:GEN:ACR
 PM:NOM
+PM:NOM:ACR
 PN
+PN:SIN:ACK:NON:FEM
+PN:SIN:ACK:POS:NON:M
+PN:SIN:ACK:POS:NON:NM
+PN:SIN:AGEN:NON:FEM
 PP
 VB:IMP
 VB:INF
@@ -56,3 +69,4 @@ VB:PRT
 VB:PRT:PF
 VB:SUP
 VB:SUP:PF
+¶

languagetool-language-modules/sv/src/main/resources/org/languagetool/resource/sv/swedish_synth.info

@@ -3,6 +3,6 @@
 #
 
 fsa.dict.separator=+
-fsa.dict.encoding=iso-8859-1
+fsa.dict.encoding=UTF-8
 
 fsa.dict.encoder=SUFFIX

languagetool-language-modules/sv/src/test/java/org/languagetool/synthesis/sv/SwedishSynthesizerTest.java

@@ -37,8 +37,10 @@ public final void testSynthesizeStringString() throws IOException {
         "blablabla").length, 0);
 
     assertEquals("[nyckelns]", Arrays.toString(synth.synthesize(dummyToken("nyckel"), "NN:BF:SIN:GEN:UTR", true)));
+    assertEquals("[FN:s]", Arrays.toString(synth.synthesize(dummyToken("FN"), "PM:GEN:ACR", true)));
     //with regular expressions
-    assertEquals("[störste, största]", Arrays.toString(synth.synthesize(dummyToken("stor"), "JJ:S:BF:.*", true)));    
+    assertEquals("[störste, största]", Arrays.toString(synth.synthesize(dummyToken("stor"), "JJ:S:BF:.*", true)));
+    assertEquals("[VW:s, VW]", Arrays.toString(synth.synthesize(dummyToken("VW"), "PM:.*:ACR", true)));    
 
   }
 

languagetool-language-modules/sv/src/test/java/org/languagetool/tagging/disambiguation/sv/SwedishDisambiguationRuleTest.java

@@ -55,8 +55,10 @@ public void testChunker() throws IOException {
 
     TestTools.myAssert("Att testa disambiguering är, en passant, kul.",
         "/[null]SENT_START Att/[att]KN  /[null]null testa/[testa]VB:IMP|testa/[testa]VB:INF  /[null]null disambiguering/[null]null  /[null]null är/[vara]VB:PRS ,/[null]null  /[null]null en/[en passant]<NN:OF:SIN:NOM:UTR>|en/[en]NN:OF:SIN:NOM:UTR|en/[en]PN  /[null]null passant/[en passant]</NN:OF:SIN:NOM:UTR> ,/[null]null  /[null]null kul/[kul]JJ:PU ./[null]null", tokenizer, sentenceTokenizer, tagger, disambiguator);
-        TestTools.myAssert("Te från Sri Lanka är mycket gott.",
-        "/[null]SENT_START Te/[te]NN:OF:SIN:NOM:NEU|Te/[te]VB:IMP|Te/[te]VB:INF  /[null]null från/[från]PP  /[null]null Sri/[Sri Lanka]<PM:NOM>  /[null]null Lanka/[Sri Lanka]</PM:NOM>  /[null]null är/[vara]VB:PRS  /[null]null mycket/[mycken]JJ:PN|mycket/[mycket]AB  /[null]null gott/[god]JJ:PN|gott/[gott]AB ./[null]null", tokenizer, sentenceTokenizer, tagger, disambiguator);
+    TestTools.myAssert("Te från Sri Lanka är mycket gott.",
+        "/[null]SENT_START Te/[te]NN:OF:NON:NOM:UTR|Te/[te]NN:OF:SIN:NOM:NEU|Te/[te]VB:IMP|Te/[te]VB:INF  /[null]null från/[från]PP  /[null]null Sri/[Sri Lanka]<PM:NOM>  /[null]null Lanka/[Sri Lanka]</PM:NOM>  /[null]null är/[vara]VB:PRS  /[null]null mycket/[mycken]JJ:PN|mycket/[mycket]AB  /[null]null gott/[god]JJ:PN|gott/[gott]AB ./[null]null", tokenizer, sentenceTokenizer, tagger, disambiguator);
+    TestTools.myAssert("VW-skandalen tog fuskandet till en ny nivå.",
+        "VW-skandalen/[null]null -- tog/[ta]VB:PRT -- fuskandet/[null]null -- till/[till]AB|till/[till]PP -- en/[en]NN:OF:SIN:NOM:UTR|en/[en]PN -- ny/[ny]JJ:PU -- nivå/[null]null", tokenizer, tagger);
 
     TestTools.myAssert("Test ...",
         "/[null]SENT_START Test/[test]NN:OF:PLU:NOM:NEU|Test/[test]NN:OF:SIN:NOM:NEU|Test/[test]NN:OF:SIN:NOM:UTR  /[null]null ./[...]<ELLIPS> ./[null]null ./[...]</ELLIPS>", tokenizer, sentenceTokenizer, tagger, disambiguator);

languagetool-language-modules/sv/src/test/java/org/languagetool/tagging/sv/SwedishTaggerTest.java

@@ -51,8 +51,13 @@ public void testTagger() throws IOException {
     TestTools.myAssert("Hon nämnde, en passant, att det inte var klädsamt",
         "Hon/[hon]PN -- nämnde/[nämna]VB:PRT -- en/[en]NN:OF:SIN:NOM:UTR|en/[en]PN -- passant/[null]null -- att/[att]KN -- det/[det]PN -- inte/[inte]AB -- var/[var]AB|var/[var]NN:OF:SIN:NOM:NEU|var/[var]PN|var/[vara]VB:IMP|var/[vara]VB:PRT -- klädsamt/[klädsam]JJ:PN", tokenizer, tagger);
 
+    TestTools.myAssert("Nato-vänliga länder har blivit fler.",
+        "Nato-vänliga/[null]null -- länder/[land]NN:OF:PLU:NOM:NEU|länder/[länd]NN:OF:PLU:NOM:UTR|länder/[lända]VB:PRS -- har/[ha]VB:PRS -- blivit/[bli]VB:SUP -- fler/[mången]JJ:K", tokenizer, tagger);
+    TestTools.myAssert("FN:s nya projekt.",
+        "FN/[FN]PM:NOM:ACR -- s/[null]null -- nya/[ny]JJ:BF|nya/[ny]JJ:P -- projekt/[projekt]NN:OF:PLU:NOM:NEU|projekt/[projekt]NN:OF:SIN:NOM:NEU", tokenizer, tagger);
+
     TestTools.myAssert("Du menar sannolikt \"massera\" om du inte skriver om masarnas era förstås.",
-        "Du/[du]PN -- menar/[mena]VB:PRS -- sannolikt/[sannolik]JJ:PN|sannolikt/[sannolikt]AB -- massera/[massera]VB:IMP|massera/[massera]VB:INF -- om/[om]AB|om/[om]KN|om/[om]PP -- du/[du]PN -- inte/[inte]AB -- skriver/[skriva]VB:PRS -- om/[om]AB|om/[om]KN|om/[om]PP -- masarnas/[mas]NN:BF:PLU:GEN:UTR -- era/[era]NN:OF:SIN:NOM:UTR|era/[era]PN -- förstås/[förstå]VB:INF:PF|förstås/[förstå]VB:PRS:PF|förstås/[förstås]AB", tokenizer, tagger);        
+        "Du/[du]PN -- menar/[mena]VB:PRS -- sannolikt/[sannolik]JJ:PN|sannolikt/[sannolikt]AB -- massera/[massera]VB:IMP|massera/[massera]VB:INF -- om/[om]AB|om/[om]KN|om/[om]PP -- du/[du]PN -- inte/[inte]AB -- skriver/[skriva]VB:PRS -- om/[om]AB|om/[om]KN|om/[om]PP -- masarnas/[mas]NN:BF:PLU:GEN:UTR -- era/[era]NN:OF:SIN:NOM:UTR|era/[era]PN -- förstås/[förstå]VB:INF:PF|förstås/[förstå]VB:PRS:PF|förstås/[förstås]AB", tokenizer, tagger);       
   }
 
 }

languagetool-language-modules/sv/src/test/java/org/languagetool/tokenizers/sv/SwedishWordTokenizerTest.java

@@ -0,0 +1,56 @@
+/* LanguageTool, a natural language style checker
+ * Copyright (C) 2005 Daniel Naber (http://www.danielnaber.de)
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+package org.languagetool.tokenizers.sv;
+
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class SwedishWordTokenizerTest {
+
+  private final SwedishWordTokenizer wordTokenizer = new SwedishWordTokenizer();
+
+  @Test
+  public void testTokenize() {
+    assertTokenize("This is\u00A0a test",
+                   "[This,  , is,  , a,  , test]");
+    assertTokenize("Nato-vänliga länder har blivit fler.",
+                   "[Nato-vänliga,  , länder,  , har,  , blivit,  , fler, .]");
+    assertTokenize("Jag säger de'", "[Jag,  , säger,  , de, ']");
+    assertTokenize("Ja' säger de'", "[Ja, ',  , säger,  , de, ']");
+
+    assertTokenize("Jag 'säger' det", "[Jag,  , ', säger, ',  , det]");
+    assertTokenize("Jag ‘säger’ det", "[Jag,  , ‘, säger, ’,  , det]");
+    assertTokenize("Jag \"säger\" det", "[Jag,  , \", säger, \",  , det]");
+    assertTokenize("Jag ”säger” det", "[Jag,  , ”, säger, ”,  , det]");
+
+    assertTokenize("'", "[']");
+    assertTokenize("''", "[, ', ']");
+    assertTokenize("'x'", "[', x, ']");
+    assertTokenize("`x`", "[`, x, `]");
+  }
+
+  private void assertTokenize(String input, String expected) {
+    List<String> result = wordTokenizer.tokenize(input);
+    assertEquals(expected, result.toString());
+  }
+}