Bug fixes, details

nerfstudio-project · Jul 23, 2023 · 4fb0a76 · 4fb0a76
1 parent 4b7baaa
commit 4fb0a76
Show file tree

Hide file tree

Showing 7 changed files with 139 additions and 62 deletions.
diff --git a/examples/05_camera_commands.py b/examples/05_camera_commands.py
@@ -51,17 +51,17 @@ def _(_):
 
             T_current_target = T_world_current.inverse() @ T_world_target
 
-            for j in range(50):
+            for j in range(20):
                 T_world_set = T_world_current @ tf.SE3.exp(
-                    T_current_target.log() * j / 49.0
+                    T_current_target.log() * j / 19.0
                 )
 
                 # Important bit: we atomically set both the orientation and the position
                 # of the camera.
                 with client.atomic():
                     client.camera.wxyz = T_world_set.rotation().wxyz
                     client.camera.position = T_world_set.translation()
-                time.sleep(0.01)
+                time.sleep(1.0 / 60.0)
 
             # Mouse interactions should orbit around the frame origin.
             client.camera.look_at = frame.position

diff --git a/examples/07_record3d_visualizer.py b/examples/07_record3d_visualizer.py
@@ -18,8 +18,8 @@
 
 def main(
     data_path: Path = Path(__file__).parent / "assets/record3d_dance",
-    downsample_factor: int = 2,
-    max_frames: int = 50,
+    downsample_factor: int = 4,
+    max_frames: int = 100,
 ) -> None:
     server = viser.ViserServer()
 
@@ -30,11 +30,16 @@ def main(
     # Add playback UI.
     with server.add_gui_folder("Playback"):
         gui_timestep = server.add_gui_slider(
-            "Timestep", min=0, max=num_frames - 1, step=1, initial_value=0
+            "Timestep",
+            min=0,
+            max=num_frames - 1,
+            step=1,
+            initial_value=0,
+            disabled=True,
         )
-        gui_next_frame = server.add_gui_button("Next Frame")
-        gui_prev_frame = server.add_gui_button("Prev Frame")
-        gui_playing = server.add_gui_checkbox("Playing", False)
+        gui_next_frame = server.add_gui_button("Next Frame", disabled=True)
+        gui_prev_frame = server.add_gui_button("Prev Frame", disabled=True)
+        gui_playing = server.add_gui_checkbox("Playing", True)
         gui_framerate = server.add_gui_slider(
             "FPS", min=1, max=60, step=0.1, initial_value=loader.fps
         )

diff --git a/viser/client/src/App.tsx b/viser/client/src/App.tsx
@@ -82,6 +82,14 @@ function SingleViewer() {
     // Scene node attributes that aren't placed in the zustand state, for performance reasons.
     nodeAttributesFromName: React.useRef({}),
   };
+
+  // Memoize the websocket interface so it isn't remounted when the theme or
+  // viewer context changes.
+  const memoizedWebsocketInterface = React.useMemo(
+    () => <WebsocketInterface />,
+    []
+  );
+
   const fixed_sidebar = viewer.useGui((state) => state.theme.fixed_sidebar);
   return (
     <ViewerContext.Provider value={viewer}>
@@ -106,9 +114,7 @@ function SingleViewer() {
                 theme.colorScheme === "light" ? "#fff" : theme.colors.dark[9],
             })}
           >
-            <ViewerCanvas>
-              <WebsocketInterface />
-            </ViewerCanvas>
+            <ViewerCanvas>{memoizedWebsocketInterface}</ViewerCanvas>
           </Box>
         </MediaQuery>
         <ControlPanel fixed_sidebar={fixed_sidebar} />

diff --git a/viser/client/src/WebsocketInterface.tsx b/viser/client/src/WebsocketInterface.tsx
@@ -327,7 +327,19 @@ function useMessageHandler() {
           message.position[2]
         ).applyQuaternion(R_threeworld_world);
         camera.up.set(updir.x, updir.y, updir.z);
-        cameraControls.applyCameraUp();
+
+        // Back up position.
+        const prevPosition = new THREE.Vector3();
+        cameraControls.getPosition(prevPosition);
+
+        cameraControls.updateCameraUp();
+
+        // Restore position, which gets unexpectedly mutated in updateCameraUp().
+        cameraControls.setPosition(
+          prevPosition.x,
+          prevPosition.y,
+          prevPosition.z
+        );
         return;
       }
       case "SetCameraPositionMessage": {

diff --git a/viser/client/yarn.lock b/viser/client/yarn.lock
@@ -176,13 +176,20 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.22.5"
 
-"@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.16.7", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.9.2":
+"@babel/runtime@^7.10.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.16.7", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.9.2":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.5.tgz#8564dd588182ce0047d55d7a75e93921107b57ec"
   integrity sha512-ecjvYlnAaZ/KVneE/OdKYBYfgXV3Ptu6zQWmgEF7vwKhQnvVS6bjMD2XYgj+SNvQ1GfK/pjgokfPkC/2CO8CuA==
   dependencies:
     regenerator-runtime "^0.13.11"
 
+"@babel/runtime@^7.11.2":
+  version "7.22.6"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.6.tgz#57d64b9ae3cff1d67eb067ae117dac087f5bd438"
+  integrity sha512-wDb5pWm4WDdF6LFUde3Jl8WzPA+3ZbxYqkC6xAXuD3irdEHN1k0NfTRrJD8ZD378SJ61miMLCqIOXYhd8x+AJQ==
+  dependencies:
+    regenerator-runtime "^0.13.11"
+
 "@babel/template@^7.22.5":
   version "7.22.5"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec"
@@ -614,10 +621,10 @@
   resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-6.0.13.tgz#a7adc128a2e7c07031c7221c1533800d0c80279a"
   integrity sha512-iqIU9wurqAeccVbWjM0yr1JGne5VP+ob55M03QAXOEN4+ck93VDTjCkZJR2RFhDcs5q0twQFoOmU/gULR8aKIA==
 
-"@mediapipe/[email protected]-rc2":
-  version "0.10.2-rc2"
-  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.2-rc2.tgz#e3fa5d84d58b9031a0e975d1e5ef8eb8e4a6fc11"
-  integrity sha512-b9ar6TEUo8I07n/jXSuKDu5HgzkDah9pe4H8BYpcubhCEahlfDD5ixE+9SQyJM4HXHXdF9nN/wRQT7rEnLz7Gg==
+"@mediapipe/[email protected]":
+  version "0.10.2"
+  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.2.tgz#eae193cf4a5c57baf2b235decde288b5152ee433"
+  integrity sha512-d8Q9uRK89ZRWmED2JLI9/blpJcfdbh0iEUuMo8TgkMzNfQBY1/GC0FEJWrairTwHkxIf6Oud1vFBP+aHicWqJA==
 
 "@msgpackr-extract/[email protected]":
   version "3.0.2"
@@ -807,12 +814,12 @@
   integrity sha512-POu8Mk0hIU3lRXB3bGIGe4VHIwwDsQyoD1F394OK7STTiX9w4dG3cTLljjYswkQN+hDSHRrj4O36kuVa7KPU8Q==
 
 "@react-three/drei@^9.64.0":
-  version "9.76.0"
-  resolved "https://registry.yarnpkg.com/@react-three/drei/-/drei-9.76.0.tgz#9d3b9428f8d8d65d6ec7cde3f195a226e1719daf"
-  integrity sha512-pF5jrVTcaAn5qkmvVmLZOUl9mZexNMdZVVWBZlFX7TjhQuBMbEo4lZG8HZgbj4jZ7J/McWXgn0+9UNdmONwILw==
+  version "9.79.3"
+  resolved "https://registry.yarnpkg.com/@react-three/drei/-/drei-9.79.3.tgz#37b74ae0adb8f8b8d3142c17ddd077248b61b627"
+  integrity sha512-aTJVOHzaS8DDlqkT9YsyRcsF2nrRPEoA8tTHY6Lr78gMHDc44HX068Ie3Kq+Tw5FabOfu2SMoFW0IVuDg94qfA==
   dependencies:
     "@babel/runtime" "^7.11.2"
-    "@mediapipe/tasks-vision" "0.10.2-rc2"
+    "@mediapipe/tasks-vision" "0.10.2"
     "@react-spring/three" "~9.6.1"
     "@use-gesture/react" "^10.2.24"
     camera-controls "^2.4.2"
@@ -825,6 +832,7 @@
     meshline "^3.1.6"
     react-composer "^5.0.3"
     react-merge-refs "^1.1.0"
+    stats-gl "^1.0.4"
     stats.js "^0.17.0"
     suspend-react "^0.1.3"
     three-mesh-bvh "^0.6.0"
@@ -1445,9 +1453,9 @@ camelcase@^6.2.0:
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
 camera-controls@^2.4.2:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.6.0.tgz#27bb5398a7600220fba4300debc3dfd3693d1eb0"
-  integrity sha512-65vkZ+FQfRLtq5LHrNuDFeOALN8+gfoRFuIOwYwgwzVY7bjBxP+j3joj6RTgc5Ot+dTJupFWwfcq7ds4Iq4DGg==
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.7.0.tgz#13e2895375fbd8fb3353baeada6c8bc267a60d09"
+  integrity sha512-HONMoMYHieOCQOoweS639bdWHP/P/fvVGR08imnECGVUp04mqGfsX/zp1ZufLeiAA5hA6i1JhP6SrnOwh01C0w==
 
 caniuse-lite@^1.0.30001503:
   version "1.0.30001504"
@@ -1593,9 +1601,9 @@ define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0:
     object-keys "^1.1.1"
 
 detect-gpu@^5.0.28:
-  version "5.0.29"
-  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.29.tgz#87238cbea009ad5509025784f6cb64cb586a3df5"
-  integrity sha512-DEqWxHXAKaoIHxF0rtFPDdWWINGoketQ6fk64KbahK3IC0I0LiZR6NbWZo4pwf7UH8khBLY8w4wS+1MHi81LSQ==
+  version "5.0.34"
+  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.34.tgz#c4a29f99e3d78ac35df44355f0298f5ae29c6fe4"
+  integrity sha512-iHYDY2iy6OVvJVmTXvMrp5+OROP0Q62qTlrsC7wl3kZmm6yVAoOhQx5cIIxTVEEIZe1M1aPVI3RgsbG/Z6/7PQ==
   dependencies:
     webgl-constants "^1.1.1"
 
@@ -3105,6 +3113,11 @@ source-map@^0.5.7:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==
 
+stats-gl@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/stats-gl/-/stats-gl-1.0.4.tgz#a6fb1a9605545558857895c743978ed44258f0b8"
+  integrity sha512-oxo13HHonoMWIYcrIu4xCk8IcFEFaqAOkMOMIyfvZFxNZzGy+jnW8sy0W3VfEjKQd5JX0Kp2KhePAKhtI6/TSw==
+
 stats.js@^0.17.0:
   version "0.17.0"
   resolved "https://registry.yarnpkg.com/stats.js/-/stats.js-0.17.0.tgz#b1c3dc46d94498b578b7fd3985b81ace7131cc7d"
@@ -3213,11 +3226,11 @@ text-table@^0.2.0:
   integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==
 
 three-mesh-bvh@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.6.0.tgz#15523c335383df658dc60063a783fdd52d045dc5"
-  integrity sha512-4/oXeqVMLuN9/P0M3L5ezIVrFiXQXKvjVTErkiSYMjSaPoWfNPAwqulSgLf4bIUPn8/Lq3rmIJwxbCuD8qDobA==
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.6.3.tgz#c387d6ae7faf1c5a1caf0643c19535dab27d707b"
+  integrity sha512-xjuGLSI9nBATIsWcT/DnnNma5xXYyvBiXfUbhGLAFqItOlOKYF5JWsUOX+cuSAnSWovEoHzd5Emx23qKiByrlw==
 
-three-stdlib@^2.23.4, three-stdlib@^2.23.9:
+three-stdlib@^2.23.4:
   version "2.23.10"
   resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.10.tgz#94907a558a00da327bd74308c92078fea72f77fc"
   integrity sha512-y0DlxaN5HZXI9hKjEtqO2xlCEt7XyDCOMvD2M3JJFBmYjwbU+PbJ1n3Z+7Hr/6BeVGE6KZYcqPMnfKrTK5WTJg==
@@ -3234,6 +3247,23 @@ three-stdlib@^2.23.4, three-stdlib@^2.23.9:
     potpack "^1.0.1"
     zstddec "^0.0.2"
 
+three-stdlib@^2.23.9:
+  version "2.23.13"
+  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.13.tgz#2647b9b6b93e9a257efb6d14c1b9af7a0f938439"
+  integrity sha512-WU4XHs4E6szAyoREzTs5bUAc1WFadiz5jRjNlA7aIIqf+raT4jfA320P0XmlyZz/wJwmpZnOuki4kAFsmadkTQ==
+  dependencies:
+    "@types/draco3d" "^1.4.0"
+    "@types/offscreencanvas" "^2019.6.4"
+    "@types/webxr" "^0.5.2"
+    chevrotain "^10.1.2"
+    draco3d "^1.4.1"
+    fflate "^0.6.9"
+    ktx-parse "^0.4.5"
+    mmd-parser "^1.0.4"
+    opentype.js "^1.3.3"
+    potpack "^1.0.1"
+    zstddec "^0.0.2"
+
 three@^0.151.3:
   version "0.151.3"
   resolved "https://registry.yarnpkg.com/three/-/three-0.151.3.tgz#0b3c7de4b070d5b66b15217f42465d67cbfa6004"

diff --git a/viser/infra/_async_message_buffer.py b/viser/infra/_async_message_buffer.py
@@ -2,7 +2,7 @@
 import dataclasses
 import time
 from asyncio.events import AbstractEventLoop
-from typing import AsyncGenerator, Awaitable, Dict, List, Optional, Sequence
+from typing import AsyncGenerator, Awaitable, Dict, Optional, Sequence
 
 from ._messages import Message
 
@@ -39,28 +39,36 @@ def push(self, message: Message) -> None:
         self.event_loop.call_soon_threadsafe(self.message_event.set)
 
     async def window_generator(
-        self, ignore_client_id: int
+        self, client_id: int
     ) -> AsyncGenerator[Sequence[Message], None]:
         """Async iterator over messages. Loops infinitely, and waits when no messages
         are available."""
         # Wait for a first message to arrive.
         if len(self.message_from_id) == 0:
             await self.message_event.wait()
 
-        window = MessageWindow()
+        window = MessageWindow(client_id=client_id)
         last_sent_id = -1
         while True:
             # Wait until there are new messages available.
             most_recent_message_id = self.message_counter - 1
             while last_sent_id >= most_recent_message_id:
-                await self.message_event.wait()
-                most_recent_message_id = self.message_counter - 1
+                next_message = self.message_event.wait()
+                try:
+                    await asyncio.wait_for(
+                        next_message, timeout=window.max_time_until_ready()
+                    )
+                    most_recent_message_id = self.message_counter - 1
+                except asyncio.TimeoutError:
+                    out = window.get_window_to_send()
+                    if out is not None:
+                        yield out
 
             # Try to yield the next message ID. Note that messages can be culled before
             # they're sent.
             last_sent_id += 1
             message = self.message_from_id.get(last_sent_id, None)
-            if message is not None and message.excluded_self_client != ignore_client_id:
+            if message is not None:
                 window.append_to_window(message)
                 self.event_loop.call_soon_threadsafe(self.message_event.clear)
 
@@ -76,31 +84,49 @@ async def window_generator(
 class MessageWindow:
     """Helper for building windows of messages to send to clients."""
 
+    client_id: int
+    """Client that this window will be sent to. Used for ignoring certain messages."""
+
     window_duration_sec: float = 1.0 / 60.0
-    window_max_length: int = 1024
+    window_max_length: int = 256
 
     _window_start_time: float = -1
-    _window: List[Message] = dataclasses.field(default_factory=list)
+    _window: Dict[str, Message] = dataclasses.field(default_factory=dict)
+    """We use a redundancy key -> message dictionary to track our window. This helps us
+    eliminate redundant messages."""
 
     def append_to_window(self, message: Message) -> None:
         """Append a message to our window."""
+        if message.excluded_self_client == self.client_id:
+            return
         if len(self._window) == 0:
             self._window_start_time = time.time()
-        self._window.append(message)
+        self._window[message.redundancy_key()] = message
 
-    async def wait_and_append_to_window(self, message: Awaitable[Message]) -> None:
-        """Async version of `append_to_window()`."""
-        message = asyncio.shield(message)
+    async def wait_and_append_to_window(self, message: Awaitable[Message]) -> bool:
+        """Async version of `append_to_window()`. Returns `True` if successful, `False`
+        if timed out."""
         if len(self._window) == 0:
             self.append_to_window(await message)
-        else:
-            elapsed = time.time() - self._window_start_time
-            (done, pending) = await asyncio.wait(
-                [message], timeout=elapsed - self.window_duration_sec
-            )
-            del pending
-            if message in done:
-                self.append_to_window(await message)
+            return True
+
+        message = asyncio.shield(message)
+        (done, pending) = await asyncio.wait(
+            [message], timeout=self.max_time_until_ready()
+        )
+        del pending
+        if message in done:
+            self.append_to_window(await message)
+            return True
+        return False
+
+    def max_time_until_ready(self) -> Optional[float]:
+        """Returns the maximum amount of time, in seconds, until we're ready to send the
+        current window. If the window is empty, returns `None`."""
+        if len(self._window) == 0:
+            return None
+        elapsed = time.time() - self._window_start_time
+        return max(0.0, self.window_duration_sec - elapsed)
 
     def get_window_to_send(self) -> Optional[Sequence[Message]]:
         """Returns window of messages if ready. Otherwise, returns None."""
@@ -115,9 +141,8 @@ def get_window_to_send(self) -> Optional[Sequence[Message]]:
             ready = True
 
         # Clear window and return if ready.
-        if ready:
-            out = tuple(self._window)
-            self._window.clear()
-            return out
-        else:
+        if not ready:
             return None
+        out = tuple(self._window.values())
+        self._window.clear()
+        return out
diff --git a/viser/infra/_infra.py b/viser/infra/_infra.py
@@ -292,7 +292,6 @@ async def viser_http_server(
                         serve,
                         host,
                         port,
-                        compression=None,
                         process_request=(
                             viser_http_server if http_server_root is not None else None
                         ),
@@ -329,10 +328,11 @@ async def _client_producer(
 ) -> None:
     """Infinite loop to send messages from a buffer to a single client."""
 
-    window = MessageWindow()
-
+    window = MessageWindow(client_id=client_id)
+    message_future = asyncio.ensure_future(get_next())
     while True:
-        await window.wait_and_append_to_window(get_next())
+        if await window.wait_and_append_to_window(message_future):
+            message_future = asyncio.ensure_future(get_next())
         outgoing = window.get_window_to_send()
         if outgoing is not None:
             serialized = msgpack.packb(
@@ -347,7 +347,6 @@ async def _broadcast_producer(
     get_next_window: Callable[[], Awaitable[Sequence[Message]]],
 ) -> None:
     """Infinite loop to broadcast windows of messages from a buffer."""
-
     while True:
         outgoing = await get_next_window()
         serialized = msgpack.packb(