Animated Textures

src/components/model-viewer/ModelViewerInternal.tsx

@@ -148,7 +148,7 @@ async function initialize(
 
   const textureManager = new TextureManager(assetBaseUrl);
 
-  const { cameraProps, group } = await loadScene(
+  const { cameraProps, group, animatedTextureParts } = await loadScene(
     textureManager,
     source,
     abortSignal
@@ -268,14 +268,36 @@ async function initialize(
 
   let disposed = false;
 
-  const animate = function () {
+  let nextTick = 0;
+  const animate = function (time: number) {
     if (disposed) {
       return;
     }
-    requestAnimationFrame(animate);
 
     controls?.update();
 
+    // Update textures
+
+    if (time > nextTick) {
+      nextTick = time + 1000 / 20;
+
+      for (const animatedPart of animatedTextureParts) {
+        const { x, y, frameTextures, frames, currentFrame } = animatedPart;
+        if (++animatedPart.subFrame >= frames[currentFrame].time) {
+          animatedPart.currentFrame = (currentFrame + 1) % frames.length;
+          animatedPart.subFrame = 0;
+        }
+
+        for (const targetTexture of animatedPart.targetTextures) {
+          renderer.copyTextureToTexture(
+            new Vector2(x, y),
+            frameTextures[frames[currentFrame].index],
+            targetTexture
+          );
+        }
+      }
+    }
+
     renderer.render(scene, camera);
 
     // Update what's under the mouse
@@ -287,13 +309,9 @@ async function initialize(
     }
   };
 
-  viewportEl.append(renderer.domElement);
+  renderer.setAnimationLoop(animate);
 
-  try {
-    animate();
-  } catch (e) {
-    console.error("Failed to render %s", source, e);
-  }
+  viewportEl.append(renderer.domElement);
 
   return {
     dispose(): void {

src/components/model-viewer/TextureManager.ts

@@ -23,10 +23,14 @@ export default class TextureManager {
 
   constructor(private readonly assetBaseUrl: string) {}
 
+  getFullUrl(url: string): string {
+    return this.assetBaseUrl + url;
+  }
+
   async getImage(url: string, builtIn = false): Promise<ImageBitmap> {
     let image = this.images[url];
     if (!this.enableCaching || !image) {
-      const fullUrl = builtIn ? url : this.assetBaseUrl + "/" + url;
+      const fullUrl = builtIn ? url : this.assetBaseUrl + url;
       console.debug("Loading image %s", fullUrl);
       try {
         this.loader.setOptions({

src/components/model-viewer/buildInWorldAnnotation.ts

@@ -324,7 +324,26 @@ function buildHitTestGeometry(annotation: InWorldAnnotation): {
   return { geometry, position };
 }
 
+function normalizeAnnotation(annotation: InWorldAnnotation) {
+  if (annotation.type === "box") {
+    const minCorner = annotation.minCorner;
+    const maxCorner = annotation.maxCorner;
+    annotation.minCorner = [
+      Math.min(minCorner[0], maxCorner[0]),
+      Math.min(minCorner[1], maxCorner[1]),
+      Math.min(minCorner[2], maxCorner[2]),
+    ];
+    annotation.maxCorner = [
+      Math.max(minCorner[0], maxCorner[0]),
+      Math.max(minCorner[1], maxCorner[1]),
+      Math.max(minCorner[2], maxCorner[2]),
+    ];
+  }
+}
+
 export function buildInWorldAnnotation(annotation: InWorldAnnotation) {
+  normalizeAnnotation(annotation);
+
   const geometry = buildRenderGeometry(annotation);
 
   const group = new Group();

src/components/model-viewer/loadMaterial.ts

@@ -62,7 +62,8 @@ function setBlending(
 
 export default async function loadMaterial(
   textureManager: TextureManager,
-  expMaterial: ExpMaterial
+  expMaterial: ExpMaterial,
+  texturesById: Map<string, Texture[]>
 ): Promise<Material> {
   const samplers: (Texture | null)[] = [];
   for (let i = 0; i < expMaterial.samplersLength(); ++i) {
@@ -85,6 +86,16 @@ export default async function loadMaterial(
       sampler.useMipmaps()
     );
     samplers.push(texture);
+
+    const textureId = sampler.textureId();
+    if (textureId) {
+      const textures = texturesById.get(textureId);
+      if (!textures) {
+        texturesById.set(textureId, [texture]);
+      } else if (!textures.includes(texture)) {
+        textures.push(texture);
+      }
+    }
   }
 
   // These parameters are usually set via the shader

src/components/model-viewer/loadScene.ts

@@ -1,6 +1,6 @@
 import * as flatbuffers from "flatbuffers";
 import { ExpScene } from "../../generated/scene.ts";
-import { Group, Mesh } from "three";
+import { Group, Mesh, Texture } from "three";
 import TextureManager from "./TextureManager.ts";
 import loadGeometry from "./loadGeometry.ts";
 import loadMaterial from "./loadMaterial.ts";
@@ -9,6 +9,7 @@ import decompress from "../../decompress.ts";
 type LoadedScene = {
   group: Group;
   cameraProps: CameraProps;
+  animatedTextureParts: AnimatedTexturePart[];
 };
 
 export type CameraProps = {
@@ -18,6 +19,24 @@ export type CameraProps = {
   zoom: number;
 };
 
+export type AnimatedTextureFrame = {
+  index: number;
+  time: number;
+};
+
+/**
+ * Implements Minecraft animated sprites, which update parts of a larger atlas texture.
+ */
+export type AnimatedTexturePart = {
+  targetTextures: Texture[];
+  frameTextures: Texture[];
+  frames: AnimatedTextureFrame[];
+  x: number;
+  y: number;
+  currentFrame: number;
+  subFrame: number;
+};
+
 async function decompressResponse(response: Response) {
   const blob = await response.blob();
   response = await decompress(blob);
@@ -49,7 +68,7 @@ export default async function loadScene(
   const buf = new flatbuffers.ByteBuffer(data);
 
   const group = new Group();
-
+  const texturesById = new Map<string, Texture[]>();
   const expScene = ExpScene.getRootAsExpScene(buf);
   for (let i = 0; i < expScene.meshesLength(); i++) {
     const expMesh = expScene.meshes(i);
@@ -63,7 +82,11 @@ export default async function loadScene(
     }
 
     const geometry = loadGeometry(expMesh);
-    const material = await loadMaterial(textureManager, expMaterial);
+    const material = await loadMaterial(
+      textureManager,
+      expMaterial,
+      texturesById
+    );
     const mesh = new Mesh(geometry, material);
     mesh.frustumCulled = false;
     group.add(mesh);
@@ -73,12 +96,93 @@ export default async function loadScene(
   if (!expCamera) {
     throw new Error("Scene is missing camera settings");
   }
+
+  const animatedTextureParts: AnimatedTexturePart[] = [];
+  for (let i = 0; i < expScene.animatedTexturesLength(); i++) {
+    const animatedTexture = expScene.animatedTextures(i);
+    if (!animatedTexture) {
+      continue;
+    }
+
+    const framesPath = animatedTexture.framesPath();
+    if (!framesPath) {
+      continue;
+    }
+
+    const fullUrl = textureManager.getFullUrl(framesPath);
+    const sourceDataResponse = await fetch(fullUrl, { signal: abortSignal });
+    if (!sourceDataResponse.ok) {
+      console.error(
+        "Failed to retrieve animated texture %s: %o",
+        fullUrl,
+        sourceDataResponse
+      );
+      continue;
+    }
+    const sourceData = await sourceDataResponse.blob();
+
+    // Splice it up into frames
+    const sourceFramePromises: Promise<ImageBitmap>[] = [];
+    for (let j = 0; j < animatedTexture.frameCount(); j++) {
+      const frameX =
+        (j % animatedTexture.framesPerRow()) * animatedTexture.width();
+      const frameY =
+        Math.floor(j / animatedTexture.framesPerRow()) *
+        animatedTexture.height();
+      sourceFramePromises.push(
+        createImageBitmap(
+          sourceData,
+          frameX,
+          frameY,
+          animatedTexture.width(),
+          animatedTexture.height()
+        )
+      );
+    }
+    const frameTextures = (await Promise.allSettled(sourceFramePromises)).map(
+      (result) => {
+        if (result.status === "fulfilled") {
+          return new Texture(result.value);
+        } else {
+          return null!; // TODO
+        }
+      }
+    );
+
+    const targetTextures =
+      texturesById.get(animatedTexture.textureId() ?? "") ?? [];
+    if (!targetTextures.length) {
+      continue;
+    }
+
+    const frames: AnimatedTextureFrame[] = [];
+    for (let j = 0; j < animatedTexture.framesLength(); j++) {
+      const expFrame = animatedTexture.frames(j);
+      if (expFrame) {
+        frames.push({
+          index: expFrame.index(),
+          time: expFrame.time(),
+        });
+      }
+    }
+
+    animatedTextureParts.push({
+      frameTextures,
+      targetTextures,
+      frames,
+      x: animatedTexture.x(),
+      y: animatedTexture.y(),
+      currentFrame: 0,
+      subFrame: 0,
+    });
+  }
+
   const cameraProps = {
     yaw: expCamera.yaw(),
     pitch: expCamera.pitch(),
     roll: expCamera.roll(),
     zoom: expCamera.zoom(),
   };
 
-  return { cameraProps, group };
+  return { cameraProps, group, animatedTextureParts };
 }

src/generated/scene.ts

@@ -1,5 +1,7 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
+export { ExpAnimatedTexturePart } from './scene/exp-animated-texture-part.js';
+export { ExpAnimatedTexturePartFrame } from './scene/exp-animated-texture-part-frame.js';
 export { ExpCameraSettings } from './scene/exp-camera-settings.js';
 export { ExpDepthTest } from './scene/exp-depth-test.js';
 export { ExpIndexElementType } from './scene/exp-index-element-type.js';

src/generated/scene/exp-animated-texture-part-frame.ts

@@ -0,0 +1,33 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+import * as flatbuffers from 'flatbuffers';
+
+export class ExpAnimatedTexturePartFrame {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):ExpAnimatedTexturePartFrame {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+index():number {
+  return this.bb!.readUint16(this.bb_pos);
+}
+
+time():number {
+  return this.bb!.readUint16(this.bb_pos + 2);
+}
+
+static sizeOf():number {
+  return 4;
+}
+
+static createExpAnimatedTexturePartFrame(builder:flatbuffers.Builder, index: number, time: number):flatbuffers.Offset {
+  builder.prep(2, 4);
+  builder.writeInt16(time);
+  builder.writeInt16(index);
+  return builder.offset();
+}
+
+}