Add support for multi-code-point unicode characters.

Previously, `deleteContentBackward|Forward` inputs would delete exactly one code point. This created invalid strings when the last code point was part of a multi-code-point character. To resolve this, we now determine the length of the unicode character, and delete by that many code points, which matches default contentEditable behavior.
nytimes · May 23, 2024 · 53ad39a · 53ad39a
1 parent 7131045
commit 53ad39a
Show file tree

Hide file tree

Showing 8 changed files with 480 additions and 113 deletions.
diff --git a/demo/main.tsx b/demo/main.tsx
@@ -117,7 +117,7 @@ const editorState = EditorState.create({
     schema.nodes.image.create(),
     schema.nodes.paragraph.create(
       {},
-      schema.text("This is the third paragraph")
+      schema.text("This is the third paragraph 🫵")
     ),
     schema.nodes.table.create({}, [
       schema.nodes.table_row.create({}, [

diff --git a/docs/assets/index-43bfecdd.js b/docs/assets/index-43bfecdd.js
diff --git a/docs/assets/index-474d9858.js b/docs/assets/index-474d9858.js
diff --git a/docs/index.html b/docs/index.html
@@ -5,7 +5,7 @@
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>React-ProseMirror Demo</title>
-    <script type="module" crossorigin src="/react-prosemirror/assets/index-474d9858.js"></script>
+    <script type="module" crossorigin src="/react-prosemirror/assets/index-43bfecdd.js"></script>
     <link rel="stylesheet" href="/react-prosemirror/assets/index-523693cd.css">
   </head>
   <body>

diff --git a/src/plugins/beforeInputPlugin.ts b/src/plugins/beforeInputPlugin.ts
@@ -1,44 +1,18 @@
-import { Mark, Node } from "prosemirror-model";
+import { Mark } from "prosemirror-model";
 import { Plugin } from "prosemirror-state";
 import { Decoration, EditorView } from "prosemirror-view";
 
 import { CursorWrapper } from "../components/CursorWrapper.js";
 import { widget } from "../decorations/ReactWidgetType.js";
+import { getEdgeCharacterSize } from "../strings/characters.js";
+import { findDirection } from "../strings/direction.js";
+import {
+  findWordBoundaryBackward,
+  findWordBoundaryForward,
+} from "../strings/words.js";
 
 import { reactKeysPluginKey } from "./reactKeys.js";
 
-const SPACE = /\s/;
-const PUNCTUATION =
-  /[\u0021-\u0023\u0025-\u002A\u002C-\u002F\u003A\u003B\u003F\u0040\u005B-\u005D\u005F\u007B\u007D\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E3B\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]/;
-const CHAMELEON = /['\u2018\u2019]/;
-
-const isWordCharacter = (doc: Node, pos: number, checkDir = -1): boolean => {
-  const $pos = doc.resolve(pos);
-
-  // The position is at the beginning of a node
-  if ($pos.parentOffset === 0) return false;
-
-  const char = doc.textBetween(pos + checkDir, pos, null, " ");
-
-  if (SPACE.test(char)) {
-    return false;
-  }
-
-  // Chameleons count as word characters as long as they're in a word, so
-  // recurse to see if the next one is a word character or not.
-  if (CHAMELEON.test(char)) {
-    if (isWordCharacter(doc, pos - 1)) {
-      return true;
-    }
-  }
-
-  if (PUNCTUATION.test(char)) {
-    return false;
-  }
-
-  return true;
-};
-
 function insertText(
   view: EditorView,
   eventData: string | null,
@@ -82,28 +56,6 @@ function insertText(
   return true;
 }
 
-function findWordBoundaryBackward(doc: Node, start: number) {
-  let pos = start;
-  while (!isWordCharacter(doc, pos)) {
-    pos--;
-  }
-  while (isWordCharacter(doc, pos)) {
-    pos--;
-  }
-  return pos;
-}
-
-function findWordBoundaryForward(doc: Node, start: number) {
-  let pos = start;
-  while (!isWordCharacter(doc, pos, 1)) {
-    pos++;
-  }
-  while (isWordCharacter(doc, pos, 1)) {
-    pos++;
-  }
-  return pos;
-}
-
 export function beforeInputPlugin(
   setCursorWrapper: (deco: Decoration | null) => void
 ) {
@@ -195,8 +147,6 @@ export function beforeInputPlugin(
                   insertText(view, data, { from, to });
                 }
               });
-              // We have to unilaterally prevent default here, because there's no way
-              // to synchronously check the contents of the data.
               break;
             }
             case "insertText": {
@@ -235,6 +185,29 @@ export function beforeInputPlugin(
             }
             case "deleteContentBackward": {
               const { tr, doc, selection } = view.state;
+
+              if (selection.empty) {
+                const textNode = selection.$anchor.nodeBefore;
+                const text = textNode?.text;
+                if (!text) break;
+
+                const characterSize = getEdgeCharacterSize(
+                  text,
+                  "trailing",
+                  findDirection(view, selection.anchor) === "rtl"
+                );
+
+                const to = selection.to;
+                const from = to - characterSize;
+                const storedMarks = doc
+                  .resolve(from)
+                  .marksAcross(doc.resolve(to));
+
+                tr.delete(from, to).setStoredMarks(storedMarks);
+                view.dispatch(tr);
+                break;
+              }
+
               const from = selection.empty
                 ? selection.from - 1
                 : selection.from;
@@ -250,13 +223,36 @@ export function beforeInputPlugin(
             }
             case "deleteContentForward": {
               const { tr, doc, selection } = view.state;
+              if (selection.empty) {
+                const textNode = selection.$anchor.nodeAfter;
+                const text = textNode?.text;
+                if (!text) break;
+
+                const characterEnd = getEdgeCharacterSize(
+                  text,
+                  "leading",
+                  findDirection(view, selection.anchor) === "rtl"
+                );
+
+                const from = selection.from;
+                const to = from + characterEnd;
+                const storedMarks = doc
+                  .resolve(from)
+                  .marksAcross(doc.resolve(to));
+
+                tr.delete(from, to).setStoredMarks(storedMarks);
+                view.dispatch(tr);
+                break;
+              }
+
               const from = selection.from;
-              const to = selection.empty ? selection.to + 1 : selection.to;
+              const to = selection.to;
               const storedMarks = doc
                 .resolve(from)
                 .marksAcross(doc.resolve(to));
 
               tr.delete(from, to).setStoredMarks(storedMarks);
+              view.dispatch(tr);
               break;
             }
             case "deleteContent": {