Add MDX support to the GUI API (#73)

* Create JSX component that processes markdown using MDX 2 * Add markdown to the GUI API and style it on the client * Update Example 14 to be scoped per-client * Move GUI Handles for Folders, Tabs, and Markdown to _gui_handles.py * Use regex literals to resolve 'Unsupported Escape Sequence' warnings * Update interface to intake a image_root Path and replace relative paths in mdx This gets around introducing custom syntax that isn't being processed by mdx * Remove custom placeholder from markdown image As far as I can tell it doesn't work for images without a predefined aspect ratio * Tweak markdown support - Remove <img src="" /> => data URL logic - Add .visible support * Prettier --------- Co-authored-by: Brent Yi <[email protected]>
nerfstudio-project · Aug 8, 2023 · 046b813 · 046b813
1 parent db72bca
commit 046b813
Show file tree

Hide file tree

Showing 16 changed files with 1,698 additions and 156 deletions.
diff --git a/examples/13_theming.py b/examples/13_theming.py
@@ -38,7 +38,9 @@
 
 titlebar_theme = TitlebarConfig(buttons=buttons, image=image)
 
-server.configure_theme(titlebar_content=titlebar_theme, control_layout="fixed")
+server.configure_theme(
+    dark_mode=True, titlebar_content=titlebar_theme, control_layout="fixed"
+)
 server.world_axes.visible = True
 
 while True:

diff --git a/examples/14_markdown.py b/examples/14_markdown.py
@@ -0,0 +1,34 @@
+"""Markdown Demonstration
+
+Viser GUI has MDX 2 support.
+"""
+
+import time
+from pathlib import Path
+
+import viser
+
+server = viser.ViserServer()
+server.world_axes.visible = True
+
+
+@server.on_client_connect
+def _(client: viser.ClientHandle) -> None:
+    here = Path(__file__).absolute().parent
+    markdown_source = (here / "./assets/mdx_example.mdx").read_text()
+    markdown = client.add_gui_markdown(markdown=markdown_source, image_root=here)
+
+    button = client.add_gui_button("Remove Markdown")
+    checkbox = client.add_gui_checkbox("Visibility", initial_value=True)
+
+    @button.on_click
+    def _(_):
+        markdown.remove()
+
+    @checkbox.on_update
+    def _(_):
+        markdown.visible = checkbox.value
+
+
+while True:
+    time.sleep(10.0)
diff --git a/examples/assets/mdx_example.mdx b/examples/assets/mdx_example.mdx
@@ -0,0 +1,95 @@
+## Markdown in Viser
+
+---
+
+Viser has full support for the GFM markdown spec, including **bold**, _italics_,
+~~strikethrough~~, and many other features.
+
+Here's a [masked link](https://github.com/nerfstudio-project/viser). Not a fan?
+Here's a normal one: https://pypi.org/project/viser/
+
+Anywhere where you can insert GUI elements, you can also insert `images`,
+`blockquotes`, `lists`, `tables`, `task lists`, and `(unstyled) code blocks`.
+
+In inline code blocks, you can show off colors with color chips: `#FED363`
+`hsl(0, 0%, 82%)` `rgb(255, 255, 255)`
+
+Adding images from a remote origin is simple.
+
+![Viser Logo](http://nerfstudio-project.github.io/viser/_static/viser.svg)
+
+For local images with relative paths, you can either directly use a
+[data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs)
+or set the `image_root` argument to the `Path` object that you'd like your paths
+relative to. If no such `image_root` is provided, the file system will be scoped
+to the directory that Viser is installed in.
+
+![Cal Logo](../examples/assets/Cal_logo.png)
+![Cal Logo](/Users/brentyi/Documents/code/viser/examples/assets/Cal_logo.png)
+
+Tables follow the standard markdown spec:
+
+| Application                                          | Description                                        |
+| ---------------------------------------------------- | -------------------------------------------------- |
+| [Nerfstudio](https://nerf.studio)                    | A collaboration friendly studio for NeRFs          |
+| [Viser](https://nerfstudio-project.github.io/viser/) | An interactive 3D visualization toolbox for Python |
+
+Code blocks, while being not nearly as exciting as some of the things presented,
+work as expected. Currently, while you can specify a language and metadata in
+your blocks, they will remain unused by the Markdown renderer.
+
+```python
+"""Markdown Demonstration
+
+Viser GUI has MDX 2 support.
+"""
+
+import time
+from pathlib import Path
+
+import viser
+
+server = viser.ViserServer()
+server.world_axes.visible = True
+
+
+@server.on_client_connect
+def _(client: viser.ClientHandle) -> None:
+    with open("./assets/mdx_example.mdx", "r") as mkdn:
+        markdown = client.add_gui_markdown(
+            markdown=mkdn.read(), image_root=Path(__file__).parent
+        )
+
+    button = client.add_gui_button("Remove Markdown")
+
+    @button.on_click
+    def _(_):
+        markdown.remove()
+
+
+while True:
+    time.sleep(10.0)
+```
+
+As a bonus, MDX is extensible and JS capable. This means that you have the
+freedom to do things like:
+
+This page loaded on {(new Date()).toString()}
+
+Or:
+
+> Oh yes, mdx PR would be exciting
+>
+> <Cite> &#8212; Brent Yi </Cite>
+
+**Note**: Be careful when playing with JSX, it's very easy to break markdown.
+
+So that's MDX in Viser. It has support for:
+
+- [x] CommonMark and GFM standards
+  - bold, italics, strikethrough, images, blockquotes, tables, task lists, code
+    blocks, inline code
+- [x] Color chips
+- [x] JSX enhanced components
+- [ ] Prism highlighted code blocks and code block tabs
+- [ ] Exposed Mantine in markdown
diff --git a/viser/__init__.py b/viser/__init__.py
@@ -1,10 +1,11 @@
-from ._gui_api import GuiFolderHandle as GuiFolderHandle
-from ._gui_api import GuiTabGroupHandle as GuiTabGroupHandle
-from ._gui_api import GuiTabHandle as GuiTabHandle
 from ._gui_handles import GuiButtonGroupHandle as GuiButtonGroupHandle
 from ._gui_handles import GuiButtonHandle as GuiButtonHandle
 from ._gui_handles import GuiDropdownHandle as GuiDropdownHandle
+from ._gui_handles import GuiFolderHandle as GuiFolderHandle
 from ._gui_handles import GuiHandle as GuiHandle
+from ._gui_handles import GuiMarkdownHandle as GuiMarkdownHandle
+from ._gui_handles import GuiTabGroupHandle as GuiTabGroupHandle
+from ._gui_handles import GuiTabHandle as GuiTabHandle
 from ._icons_enum import Icon as Icon
 from ._scene_handles import CameraFrustumHandle as CameraFrustumHandle
 from ._scene_handles import FrameHandle as FrameHandle

diff --git a/viser/_client_autobuild.py b/viser/_client_autobuild.py
@@ -1,8 +1,9 @@
 import subprocess
 import sys
+from pathlib import Path
+
 import psutil
 import rich
-from pathlib import Path
 
 client_dir = Path(__file__).absolute().parent / "client"
 build_dir = client_dir / "build"

diff --git a/viser/_gui_api.py b/viser/_gui_api.py
@@ -5,11 +5,12 @@
 from __future__ import annotations
 
 import abc
-import dataclasses
+import re
 import threading
 import time
-import uuid
+import urllib.parse
 import warnings
+from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -22,6 +23,7 @@
     overload,
 )
 
+import imageio.v3 as iio
 import numpy as onp
 from typing_extensions import LiteralString
 
@@ -30,12 +32,14 @@
     GuiButtonGroupHandle,
     GuiButtonHandle,
     GuiDropdownHandle,
+    GuiFolderHandle,
     GuiHandle,
+    GuiMarkdownHandle,
+    GuiTabGroupHandle,
     _GuiHandleState,
+    _make_unique_id,
 )
-from ._icons import base64_from_icon
-from ._icons_enum import Icon
-from ._message_api import MessageApi, cast_vector
+from ._message_api import MessageApi, _encode_image_base64, cast_vector
 
 if TYPE_CHECKING:
     from .infra import ClientId
@@ -46,11 +50,6 @@
 T = TypeVar("T")
 
 
-def _make_unique_id() -> str:
-    """Return a unique ID for referencing GUI elements."""
-    return str(uuid.uuid4())
-
-
 def _compute_step(x: Optional[float]) -> float:  # type: ignore
     """For number inputs: compute an increment size from some number.
 
@@ -80,6 +79,38 @@ def _compute_precision_digits(x: float) -> int:
     return digits
 
 
+def _get_data_url(url: str, image_root: Optional[Path]) -> str:
+    if not url.startswith("http") and not image_root:
+        warnings.warn(
+            "No `image_root` provided. All relative paths will be scoped to viser's installation path.",
+            stacklevel=2,
+        )
+    if url.startswith("http"):
+        return url
+    if image_root is None:
+        image_root = Path(__file__).parent
+    try:
+        image = iio.imread(image_root / url)
+        data_uri = _encode_image_base64(image, "png")
+        url = urllib.parse.quote(f"{data_uri[1]}")
+        return f"data:{data_uri[0]};base64,{url}"
+    except (IOError, FileNotFoundError):
+        warnings.warn(
+            f"Failed to read image {url}, with image_root set to {image_root}.",
+            stacklevel=2,
+        )
+        return url
+
+
+def _parse_markdown(markdown: str, image_root: Optional[Path]) -> str:
+    markdown = re.sub(
+        r"\!\[([^]]*)\]\(([^]]*)\)",
+        lambda match: f"![{match.group(1)}]({_get_data_url(match.group(2), image_root)})",
+        markdown,
+    )
+    return markdown
+
+
 class GuiApi(abc.ABC):
     _target_container_from_thread_id: Dict[int, str] = {}
     """ID of container to put GUI elements into."""
@@ -138,6 +169,27 @@ def add_gui_tab_group(self) -> GuiTabGroupHandle:
             _container_id=self._get_container_id(),
         )
 
+    def add_gui_markdown(
+        self, markdown: str, image_root: Optional[Path]
+    ) -> GuiMarkdownHandle:
+        """Add markdown to the GUI."""
+        markdown = _parse_markdown(markdown, image_root)
+
+        markdown_id = _make_unique_id()
+        self._get_api()._queue(
+            _messages.GuiAddMarkdownMessage(
+                order=time.time(),
+                id=markdown_id,
+                markdown=markdown,
+                container_id=self._get_container_id(),
+            )
+        )
+        return GuiMarkdownHandle(
+            _gui_api=self,
+            _id=markdown_id,
+            _visible=True,
+        )
+
     def add_gui_button(
         self,
         label: str,
@@ -620,111 +672,3 @@ def sync_other_clients(client_id: ClientId, value: Any) -> None:
             handle.visible = visible
 
         return handle
-
-
-@dataclasses.dataclass(frozen=True)
-class GuiTabGroupHandle:
-    _tab_group_id: str
-    _labels: List[str]
-    _icons_base64: List[Optional[str]]
-    _tab_container_ids: List[str]
-    _gui_api: GuiApi
-    _container_id: str
-
-    def add_tab(self, label: str, icon: Optional[Icon] = None) -> GuiTabHandle:
-        """Add a tab. Returns a handle we can use to add GUI elements to it."""
-
-        id = _make_unique_id()
-
-        # We may want to make this thread-safe in the future.
-        self._labels.append(label)
-        self._icons_base64.append(None if icon is None else base64_from_icon(icon))
-        self._tab_container_ids.append(id)
-
-        self._sync_with_client()
-
-        return GuiTabHandle(_parent=self, _container_id=id)
-
-    def remove(self) -> None:
-        """Remove this tab group and all contained GUI elements."""
-        self._gui_api._get_api()._queue(_messages.GuiRemoveMessage(self._tab_group_id))
-        # Containers will be removed automatically by the client.
-        #
-        # for tab_container_id in self._tab_container_ids:
-        #     self._gui_api._get_api()._queue(
-        #         _messages.GuiRemoveContainerChildrenMessage(tab_container_id)
-        #     )
-
-    def _sync_with_client(self) -> None:
-        """Send a message that syncs tab state with the client."""
-        self._gui_api._get_api()._queue(
-            _messages.GuiAddTabGroupMessage(
-                order=time.time(),
-                id=self._tab_group_id,
-                container_id=self._container_id,
-                tab_labels=tuple(self._labels),
-                tab_icons_base64=tuple(self._icons_base64),
-                tab_container_ids=tuple(self._tab_container_ids),
-            )
-        )
-
-
-@dataclasses.dataclass
-class GuiFolderHandle:
-    """Use as a context to place GUI elements into a folder."""
-
-    _gui_api: GuiApi
-    _container_id: str
-    _container_id_restore: Optional[str] = None
-
-    def __enter__(self) -> None:
-        self._container_id_restore = self._gui_api._get_container_id()
-        self._gui_api._set_container_id(self._container_id)
-
-    def __exit__(self, *args) -> None:
-        del args
-        assert self._container_id_restore is not None
-        self._gui_api._set_container_id(self._container_id_restore)
-        self._container_id_restore = None
-
-    def remove(self) -> None:
-        """Permanently remove this folder and all contained GUI elements from the
-        visualizer."""
-        self._gui_api._get_api()._queue(_messages.GuiRemoveMessage(self._container_id))
-
-
-@dataclasses.dataclass
-class GuiTabHandle:
-    """Use as a context to place GUI elements into a tab."""
-
-    _parent: GuiTabGroupHandle
-    _container_id: str
-    _container_id_restore: Optional[str] = None
-
-    def __enter__(self) -> None:
-        self._container_id_restore = self._parent._gui_api._get_container_id()
-        self._parent._gui_api._set_container_id(self._container_id)
-
-    def __exit__(self, *args) -> None:
-        del args
-        assert self._container_id_restore is not None
-        self._parent._gui_api._set_container_id(self._container_id_restore)
-        self._container_id_restore = None
-
-    def remove(self) -> None:
-        """Permanently remove this tab and all contained GUI elements from the
-        visualizer."""
-        # We may want to make this thread-safe in the future.
-        container_index = self._parent._tab_container_ids.index(self._container_id)
-        assert container_index != -1, "Tab already removed!"
-
-        # Container needs to be manually removed.
-        self._parent._gui_api._get_api()._queue(
-            _messages.GuiRemoveContainerChildrenMessage(self._container_id)
-        )
-
-        self._parent._labels.pop(container_index)
-        self._parent._icons_base64.pop(container_index)
-        self._parent._tab_container_ids.pop(container_index)
-
-        self._parent._sync_with_client()