Skip to content

Commit

Permalink
deploy: d228dbe
Browse files Browse the repository at this point in the history
  • Loading branch information
@brentyi
brentyi committed Sep 25, 2024
1 parent f2e1d52 commit 906e24a
Show file tree
Hide file tree
Showing 16 changed files with 733 additions and 628 deletions.
Binary file modified latest/.doctrees/environment.pickle
View file
Binary file not shown.
Binary file modified latest/.doctrees/examples/07_record3d_visualizer.doctree
View file
Binary file not shown.
Binary file modified latest/.doctrees/examples/08_smpl_visualizer.doctree
View file
Binary file not shown.
Binary file modified latest/.doctrees/examples/25_smpl_visualizer_skinned.doctree
View file
Binary file not shown.
Binary file modified latest/.doctrees/scene_api.doctree
View file
Binary file not shown.
13 changes: 10 additions & 3 deletions latest/_modules/viser/_scene_api/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1639,7 +1639,13 @@ <h1>Source code for viser._scene_api</h1><div class="highlight"><pre>
<span class="p">)</span></div>


<span class="k">def</span> <span class="nf">_add_gaussian_splats</span><span class="p">(</span>
<span class="k">def</span> <span class="nf">_add_gaussian_splats</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">GaussianSplatHandle</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Backwards compatibility shim. Use `add_gaussian_splats()` instead.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">add_gaussian_splats</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

<div class="viewcode-block" id="SceneApi.add_gaussian_splats">
<a class="viewcode-back" href="../../../scene_api/#viser.SceneApi.add_gaussian_splats">[docs]</a>
<span class="k">def</span> <span class="nf">add_gaussian_splats</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">centers</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
Expand All @@ -1652,7 +1658,7 @@ <h1>Source code for viser._scene_api</h1><div class="highlight"><pre>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">GaussianSplatHandle</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Add a model to render using Gaussian Splatting.</span>

<span class="sd"> **Work-in-progress.** This feature is experimental and still under</span>
<span class="sd"> **Experimental.** This feature is experimental and still under</span>
<span class="sd"> development. It may be changed or removed.</span>

<span class="sd"> Arguments:</span>
Expand Down Expand Up @@ -1708,7 +1714,8 @@ <h1>Source code for viser._scene_api</h1><div class="highlight"><pre>
<span class="n">node_handle</span> <span class="o">=</span> <span class="n">GaussianSplatHandle</span><span class="o">.</span><span class="n">_make</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">wxyz</span><span class="p">,</span> <span class="n">position</span><span class="p">,</span> <span class="n">visible</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">node_handle</span>
<span class="k">return</span> <span class="n">node_handle</span></div>


<div class="viewcode-block" id="SceneApi.add_box">
<a class="viewcode-back" href="../../../scene_api/#viser.SceneApi.add_box">[docs]</a>
Expand Down
29 changes: 21 additions & 8 deletions latest/_sources/examples/07_record3d_visualizer.rst.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@ Parse and stream record3d captures. To get the demo data, see ``./assets/downloa
import numpy as np
import tyro
from tqdm.auto import tqdm
import viser
import viser.extras
import viser.transforms as tf
from tqdm.auto import tqdm
def main(
Expand All @@ -41,6 +40,13 @@ Parse and stream record3d captures. To get the demo data, see ``./assets/downloa
# Add playback UI.
with server.gui.add_folder("Playback"):
gui_point_size = server.gui.add_slider(
"Point size",
min=0.001,
max=0.02,
step=1e-3,
initial_value=0.01,
)
gui_timestep = server.gui.add_slider(
"Timestep",
min=0,
Expand Down Expand Up @@ -88,6 +94,10 @@ Parse and stream record3d captures. To get the demo data, see ``./assets/downloa
nonlocal prev_timestep
current_timestep = gui_timestep.value
with server.atomic():
# Update point size.
point_nodes[current_timestep].point_size = gui_point_size.value
# Toggle visibility.
frame_nodes[current_timestep].visible = True
frame_nodes[prev_timestep].visible = False
prev_timestep = current_timestep
Expand All @@ -101,6 +111,7 @@ Parse and stream record3d captures. To get the demo data, see ``./assets/downloa
show_axes=False,
)
frame_nodes: list[viser.FrameHandle] = []
point_nodes: list[viser.PointCloudHandle] = []
for i in tqdm(range(num_frames)):
frame = loader.get_frame(i)
position, color = frame.get_point_cloud(downsample_factor)
Expand All @@ -109,12 +120,14 @@ Parse and stream record3d captures. To get the demo data, see ``./assets/downloa
frame_nodes.append(server.scene.add_frame(f"/frames/t{i}", show_axes=False))
# Place the point cloud in the frame.
server.scene.add_point_cloud(
name=f"/frames/t{i}/point_cloud",
points=position,
colors=color,
point_size=0.01,
point_shape="rounded",
point_nodes.append(
server.scene.add_point_cloud(
name=f"/frames/t{i}/point_cloud",
points=position,
colors=color,
point_size=gui_point_size.value,
point_shape="rounded",
)
)
# Place the frustum.
Expand Down
52 changes: 29 additions & 23 deletions latest/_sources/examples/08_smpl_visualizer.rst.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,21 @@ See here for download instructions:
assert model_path.suffix.lower() == ".npz", "Model should be an .npz file!"
body_dict = dict(**np.load(model_path, allow_pickle=True))
self._J_regressor = body_dict["J_regressor"]
self._weights = body_dict["weights"]
self._v_template = body_dict["v_template"]
self._posedirs = body_dict["posedirs"]
self._shapedirs = body_dict["shapedirs"]
self._faces = body_dict["f"]
self.num_joints: int = self._weights.shape[-1]
self.num_betas: int = self._shapedirs.shape[-1]
self.J_regressor = body_dict["J_regressor"]
self.weights = body_dict["weights"]
self.v_template = body_dict["v_template"]
self.posedirs = body_dict["posedirs"]
self.shapedirs = body_dict["shapedirs"]
self.faces = body_dict["f"]
self.num_joints: int = self.weights.shape[-1]
self.num_betas: int = self.shapedirs.shape[-1]
self.parent_idx: np.ndarray = body_dict["kintree_table"][0]
def get_outputs(self, betas: np.ndarray, joint_rotmats: np.ndarray) -> SmplOutputs:
# Get shaped vertices + joint positions, when all local poses are identity.
v_tpose = self._v_template + np.einsum("vxb,b->vx", self._shapedirs, betas)
j_tpose = np.einsum("jv,vx->jx", self._J_regressor, v_tpose)
v_tpose = self.v_template + np.einsum("vxb,b->vx", self.shapedirs, betas)
j_tpose = np.einsum("jv,vx->jx", self.J_regressor, v_tpose)
# Local SE(3) transforms.
T_parent_joint = np.zeros((self.num_joints, 4, 4)) + np.eye(4)
Expand All @@ -73,13 +73,13 @@ See here for download instructions:
# Linear blend skinning.
pose_delta = (joint_rotmats[1:, ...] - np.eye(3)).flatten()
v_blend = v_tpose + np.einsum("byn,n->by", self._posedirs, pose_delta)
v_blend = v_tpose + np.einsum("byn,n->by", self.posedirs, pose_delta)
v_delta = np.ones((v_blend.shape[0], self.num_joints, 4))
v_delta[:, :, :3] = v_blend[:, None, :] - j_tpose[None, :, :]
v_posed = np.einsum(
"jxy,vj,vjy->vx", T_world_joint[:, :3, :], self._weights, v_delta
"jxy,vj,vjy->vx", T_world_joint[:, :3, :], self.weights, v_delta
)
return SmplOutputs(v_posed, self._faces, T_world_joint, T_parent_joint)
return SmplOutputs(v_posed, self.faces, T_world_joint, T_parent_joint)
def main(model_path: Path) -> None:
Expand All @@ -96,6 +96,13 @@ See here for download instructions:
num_joints=model.num_joints,
parent_idx=model.parent_idx,
)
body_handle = server.scene.add_mesh_simple(
"/human",
model.v_template,
model.faces,
wireframe=gui_elements.gui_wireframe.value,
color=gui_elements.gui_rgb.value,
)
while True:
# Do nothing if no change.
time.sleep(0.02)
Expand All @@ -104,21 +111,20 @@ See here for download instructions:
gui_elements.changed = False
# Compute SMPL outputs.
# If anything has changed, re-compute SMPL outputs.
smpl_outputs = model.get_outputs(
betas=np.array([x.value for x in gui_elements.gui_betas]),
joint_rotmats=tf.SO3.exp(
# (num_joints, 3)
np.array([x.value for x in gui_elements.gui_joints])
).as_matrix(),
)
server.scene.add_mesh_simple(
"/human",
smpl_outputs.vertices,
smpl_outputs.faces,
wireframe=gui_elements.gui_wireframe.value,
color=gui_elements.gui_rgb.value,
)
# Update the mesh properties based on the SMPL model output + GUI
# elements.
body_handle.vertices = smpl_outputs.vertices
body_handle.wireframe = gui_elements.gui_wireframe.value
body_handle.color = gui_elements.gui_rgb.value
# Match transform control gizmos to joint positions.
for i, control in enumerate(gui_elements.transform_controls):
Expand Down Expand Up @@ -156,7 +162,7 @@ See here for download instructions:
with tab_group.add_tab("View", viser.Icon.VIEWFINDER):
gui_rgb = server.gui.add_rgb("Color", initial_value=(90, 200, 255))
gui_wireframe = server.gui.add_checkbox("Wireframe", initial_value=False)
gui_show_controls = server.gui.add_checkbox("Handles", initial_value=False)
gui_show_controls = server.gui.add_checkbox("Handles", initial_value=True)
gui_rgb.on_update(set_changed)
gui_wireframe.on_update(set_changed)
Expand Down
86 changes: 50 additions & 36 deletions latest/_sources/examples/25_smpl_visualizer_skinned.rst.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,27 @@ See here for download instructions:
assert model_path.suffix.lower() == ".npz", "Model should be an .npz file!"
body_dict = dict(**np.load(model_path, allow_pickle=True))
self._J_regressor = body_dict["J_regressor"]
self._weights = body_dict["weights"]
self._v_template = body_dict["v_template"]
self._posedirs = body_dict["posedirs"]
self._shapedirs = body_dict["shapedirs"]
self._faces = body_dict["f"]
self.num_joints: int = self._weights.shape[-1]
self.num_betas: int = self._shapedirs.shape[-1]
self.J_regressor = body_dict["J_regressor"]
self.weights = body_dict["weights"]
self.v_template = body_dict["v_template"]
self.posedirs = body_dict["posedirs"]
self.shapedirs = body_dict["shapedirs"]
self.faces = body_dict["f"]
self.num_joints: int = self.weights.shape[-1]
self.num_betas: int = self.shapedirs.shape[-1]
self.parent_idx: np.ndarray = body_dict["kintree_table"][0]
def get_tpose(self, betas: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
# Get shaped vertices + joint positions, when all local poses are identity.
v_tpose = self.v_template + np.einsum("vxb,b->vx", self.shapedirs, betas)
j_tpose = np.einsum("jv,vx->jx", self.J_regressor, v_tpose)
return v_tpose, j_tpose
def get_outputs(self, betas: np.ndarray, joint_rotmats: np.ndarray) -> SmplOutputs:
# Get shaped vertices + joint positions, when all local poses are identity.
v_tpose = self._v_template + np.einsum("vxb,b->vx", self._shapedirs, betas)
j_tpose = np.einsum("jv,vx->jx", self._J_regressor, v_tpose)
v_tpose = self.v_template + np.einsum("vxb,b->vx", self.shapedirs, betas)
j_tpose = np.einsum("jv,vx->jx", self.J_regressor, v_tpose)
# Local SE(3) transforms.
T_parent_joint = np.zeros((self.num_joints, 4, 4)) + np.eye(4)
Expand All @@ -74,13 +80,13 @@ See here for download instructions:
# Linear blend skinning.
pose_delta = (joint_rotmats[1:, ...] - np.eye(3)).flatten()
v_blend = v_tpose + np.einsum("byn,n->by", self._posedirs, pose_delta)
v_blend = v_tpose + np.einsum("byn,n->by", self.posedirs, pose_delta)
v_delta = np.ones((v_blend.shape[0], self.num_joints, 4))
v_delta[:, :, :3] = v_blend[:, None, :] - j_tpose[None, :, :]
v_posed = np.einsum(
"jxy,vj,vjy->vx", T_world_joint[:, :3, :], self._weights, v_delta
"jxy,vj,vjy->vx", T_world_joint[:, :3, :], self.weights, v_delta
)
return SmplOutputs(v_posed, self._faces, T_world_joint, T_parent_joint)
return SmplOutputs(v_posed, self.faces, T_world_joint, T_parent_joint)
def main(model_path: Path) -> None:
Expand All @@ -97,23 +103,14 @@ See here for download instructions:
num_joints=model.num_joints,
parent_idx=model.parent_idx,
)
smpl_outputs = model.get_outputs(
betas=np.array([x.value for x in gui_elements.gui_betas]),
joint_rotmats=np.zeros((model.num_joints, 3, 3)) + np.eye(3),
)
bone_wxyzs = np.array(
[tf.SO3.from_matrix(R).wxyz for R in smpl_outputs.T_world_joint[:, :3, :3]]
)
bone_positions = smpl_outputs.T_world_joint[:, :3, 3]
skinned_handle = server.scene.add_mesh_skinned(
v_tpose, j_tpose = model.get_tpose(np.zeros((model.num_betas,)))
mesh_handle = server.scene.add_mesh_skinned(
"/human",
smpl_outputs.vertices,
smpl_outputs.faces,
bone_wxyzs=bone_wxyzs,
bone_positions=bone_positions,
skin_weights=model._weights,
v_tpose,
model.faces,
bone_wxyzs=tf.SO3.identity(batch_axes=(model.num_joints,)).wxyz,
bone_positions=j_tpose,
skin_weights=model.weights,
wireframe=gui_elements.gui_wireframe.value,
color=gui_elements.gui_rgb.value,
)
Expand All @@ -124,10 +121,19 @@ See here for download instructions:
if not gui_elements.changed:
continue
# Shapes changed: update vertices / joint positions.
if gui_elements.betas_changed:
v_tpose, j_tpose = model.get_tpose(
np.array([gui_beta.value for gui_beta in gui_elements.gui_betas])
)
mesh_handle.vertices = v_tpose
mesh_handle.bone_positions = j_tpose
gui_elements.changed = False
gui_elements.betas_changed = False
# Render as wireframe?
skinned_handle.wireframe = gui_elements.gui_wireframe.value
mesh_handle.wireframe = gui_elements.gui_wireframe.value
# Compute SMPL outputs.
smpl_outputs = model.get_outputs(
Expand All @@ -144,10 +150,10 @@ See here for download instructions:
# Match transform control gizmos to joint positions.
for i, control in enumerate(gui_elements.transform_controls):
control.position = smpl_outputs.T_parent_joint[i, :3, 3]
skinned_handle.bones[i].wxyz = tf.SO3.from_matrix(
mesh_handle.bones[i].wxyz = tf.SO3.from_matrix(
smpl_outputs.T_world_joint[i, :3, :3]
).wxyz
skinned_handle.bones[i].position = smpl_outputs.T_world_joint[i, :3, 3]
mesh_handle.bones[i].position = smpl_outputs.T_world_joint[i, :3, 3]
@dataclass
Expand All @@ -161,7 +167,10 @@ See here for download instructions:
transform_controls: List[viser.TransformControlsHandle]
changed: bool
"""This flag will be flipped to True whenever the mesh needs to be re-generated."""
"""This flag will be flipped to True whenever any input is changed."""
betas_changed: bool
"""This flag will be flipped to True whenever the shape changes."""
def make_gui_elements(
Expand All @@ -175,7 +184,11 @@ See here for download instructions:
tab_group = server.gui.add_tab_group()
def set_changed(_) -> None:
out.changed = True # out is define later!
out.changed = True # out is defined later!
def set_betas_changed(_) -> None:
out.betas_changed = True
out.changed = True
# GUI elements: mesh settings + visibility.
with tab_group.add_tab("View", viser.Icon.VIEWFINDER):
Expand Down Expand Up @@ -225,7 +238,7 @@ See here for download instructions:
f"beta{i}", min=-5.0, max=5.0, step=0.01, initial_value=0.0
)
gui_betas.append(beta)
beta.on_update(set_changed)
beta.on_update(set_betas_changed)
# GUI elements: joint angles.
with tab_group.add_tab("Joints", viser.Icon.ANGLE):
Expand Down Expand Up @@ -300,6 +313,7 @@ See here for download instructions:
gui_joints,
transform_controls=transform_controls,
changed=True,
betas_changed=False,
)
return out
Expand Down
Loading

0 comments on commit 906e24a

Please sign in to comment.