From b8e6906f08443cd45bef508dbf31e8f62f63eaa5 Mon Sep 17 00:00:00 2001
From: martindholmes <mholmes@uvic.ca>
Date: Mon, 6 May 2024 09:05:54 -0700
Subject: [PATCH] Fix with accompanying test.

---
 test/apostrophes.html |  4 ++++
 test/testJs/testJs.js | 16 ++++++++++++++++
 xsl/tokenize.xsl      |  6 ++++--
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/test/apostrophes.html b/test/apostrophes.html
index 4957b88d..5ffa4816 100644
--- a/test/apostrophes.html
+++ b/test/apostrophes.html
@@ -77,6 +77,10 @@
             <cite>Martin Porter, "The apostrophe character," <a href="http://snowball.tartarus.org/texts/apostrophe.html">http://snowball.tartarus.org/texts/apostrophe.html</a></cite>
           <p>This additional paragraph contains a couple of other test items for the tokenizer, including a word containing a combining diaeresis (U+0308, for issue #164). For this we have invented a word which must use a combining diacritic rather than a precomposed character because the precomposed character does not exist; this helps to avoid inadvertent composition. The word puc̈ist is made up for this purpose, along with the word puq̈uist.</p>
             
+            <p>We also add here a test for the character U+A78F, the Latin Sinological Dot, which 
+            is supposed to be a letter character, so should not cause a word to be broken during
+            tokenization. This is the teꞏst.</p>
+            
         </div>
     </body>
 </html>
\ No newline at end of file
diff --git a/test/testJs/testJs.js b/test/testJs/testJs.js
index bad122d6..343db436 100644
--- a/test/testJs/testJs.js
+++ b/test/testJs/testJs.js
@@ -55,6 +55,22 @@ tests.push({
   }
 });
 
+//One word search word-internal sinological dot search
+tests.push({
+  setup: function () {
+    Sch.queryBox.value = "teꞏst";
+  },
+  check: function (num) {
+    console.log("Search hook " + num);
+    console.log(`Testing results for word with word - internal sinological dot "teꞏst".`);
+    checkResults({
+      docsFound: 1,
+      contextsFound: 1,
+      scoreTotal: 1
+    })
+  }
+});
+
 //One word possessive apostrophe search
 tests.push({
   setup: function () {
diff --git a/xsl/tokenize.xsl b/xsl/tokenize.xsl
index cf93a67c..a7f8d9d6 100644
--- a/xsl/tokenize.xsl
+++ b/xsl/tokenize.xsl
@@ -119,9 +119,11 @@
         <xd:desc>Regex to match alphanumeric words. Note that we use 
         Unicode character classes to take our best shot at splitting on
         word boundaries, but this is bound to be fragile where multiple
-        languages are involved.</xd:desc>
+        languages are involved. NOTE: character U+A78F is introduced explicitly,
+        although it should be covered by \p{L}, because of an apparent bug in
+        Java or Saxon regex processing; see GH issue #200.</xd:desc>
     </xd:doc>
-  <xsl:variable name="alphanumeric">[\p{L}\p{M}<xsl:value-of select="string-join($allApos,'')"/>]+</xsl:variable>
+  <xsl:variable name="alphanumeric">[&#xA78F;\p{L}\p{M}<xsl:value-of select="string-join($allApos,'')"/>]+</xsl:variable>
     
     <xd:doc>
         <xd:desc>Regex to match hyphenated words</xd:desc>