diff --git a/src/components/model-viewer/ModelViewerInternal.tsx b/src/components/model-viewer/ModelViewerInternal.tsx index 4e86f1d..37e7c74 100644 --- a/src/components/model-viewer/ModelViewerInternal.tsx +++ b/src/components/model-viewer/ModelViewerInternal.tsx @@ -148,7 +148,7 @@ async function initialize( const textureManager = new TextureManager(assetBaseUrl); - const { cameraProps, group } = await loadScene( + const { cameraProps, group, animatedTextureParts } = await loadScene( textureManager, source, abortSignal @@ -268,14 +268,36 @@ async function initialize( let disposed = false; - const animate = function () { + let nextTick = 0; + const animate = function (time: number) { if (disposed) { return; } - requestAnimationFrame(animate); controls?.update(); + // Update textures + + if (time > nextTick) { + nextTick = time + 1000 / 20; + + for (const animatedPart of animatedTextureParts) { + const { x, y, frameTextures, frames, currentFrame } = animatedPart; + if (++animatedPart.subFrame >= frames[currentFrame].time) { + animatedPart.currentFrame = (currentFrame + 1) % frames.length; + animatedPart.subFrame = 0; + } + + for (const targetTexture of animatedPart.targetTextures) { + renderer.copyTextureToTexture( + new Vector2(x, y), + frameTextures[frames[currentFrame].index], + targetTexture + ); + } + } + } + renderer.render(scene, camera); // Update what's under the mouse @@ -287,13 +309,9 @@ async function initialize( } }; - viewportEl.append(renderer.domElement); + renderer.setAnimationLoop(animate); - try { - animate(); - } catch (e) { - console.error("Failed to render %s", source, e); - } + viewportEl.append(renderer.domElement); return { dispose(): void { diff --git a/src/components/model-viewer/TextureManager.ts b/src/components/model-viewer/TextureManager.ts index 08bc9cc..a186960 100644 --- a/src/components/model-viewer/TextureManager.ts +++ b/src/components/model-viewer/TextureManager.ts @@ -23,10 +23,14 @@ export default class TextureManager { constructor(private readonly assetBaseUrl: string) {} + getFullUrl(url: string): string { + return this.assetBaseUrl + url; + } + async getImage(url: string, builtIn = false): Promise { let image = this.images[url]; if (!this.enableCaching || !image) { - const fullUrl = builtIn ? url : this.assetBaseUrl + "/" + url; + const fullUrl = builtIn ? url : this.assetBaseUrl + url; console.debug("Loading image %s", fullUrl); try { this.loader.setOptions({ diff --git a/src/components/model-viewer/buildInWorldAnnotation.ts b/src/components/model-viewer/buildInWorldAnnotation.ts index 637df10..7752354 100644 --- a/src/components/model-viewer/buildInWorldAnnotation.ts +++ b/src/components/model-viewer/buildInWorldAnnotation.ts @@ -324,7 +324,26 @@ function buildHitTestGeometry(annotation: InWorldAnnotation): { return { geometry, position }; } +function normalizeAnnotation(annotation: InWorldAnnotation) { + if (annotation.type === "box") { + const minCorner = annotation.minCorner; + const maxCorner = annotation.maxCorner; + annotation.minCorner = [ + Math.min(minCorner[0], maxCorner[0]), + Math.min(minCorner[1], maxCorner[1]), + Math.min(minCorner[2], maxCorner[2]), + ]; + annotation.maxCorner = [ + Math.max(minCorner[0], maxCorner[0]), + Math.max(minCorner[1], maxCorner[1]), + Math.max(minCorner[2], maxCorner[2]), + ]; + } +} + export function buildInWorldAnnotation(annotation: InWorldAnnotation) { + normalizeAnnotation(annotation); + const geometry = buildRenderGeometry(annotation); const group = new Group(); diff --git a/src/components/model-viewer/loadMaterial.ts b/src/components/model-viewer/loadMaterial.ts index e0554d5..5ac4013 100644 --- a/src/components/model-viewer/loadMaterial.ts +++ b/src/components/model-viewer/loadMaterial.ts @@ -62,7 +62,8 @@ function setBlending( export default async function loadMaterial( textureManager: TextureManager, - expMaterial: ExpMaterial + expMaterial: ExpMaterial, + texturesById: Map ): Promise { const samplers: (Texture | null)[] = []; for (let i = 0; i < expMaterial.samplersLength(); ++i) { @@ -85,6 +86,16 @@ export default async function loadMaterial( sampler.useMipmaps() ); samplers.push(texture); + + const textureId = sampler.textureId(); + if (textureId) { + const textures = texturesById.get(textureId); + if (!textures) { + texturesById.set(textureId, [texture]); + } else if (!textures.includes(texture)) { + textures.push(texture); + } + } } // These parameters are usually set via the shader diff --git a/src/components/model-viewer/loadScene.ts b/src/components/model-viewer/loadScene.ts index b71cca6..42e3adb 100644 --- a/src/components/model-viewer/loadScene.ts +++ b/src/components/model-viewer/loadScene.ts @@ -1,6 +1,6 @@ import * as flatbuffers from "flatbuffers"; import { ExpScene } from "../../generated/scene.ts"; -import { Group, Mesh } from "three"; +import { Group, Mesh, Texture } from "three"; import TextureManager from "./TextureManager.ts"; import loadGeometry from "./loadGeometry.ts"; import loadMaterial from "./loadMaterial.ts"; @@ -9,6 +9,7 @@ import decompress from "../../decompress.ts"; type LoadedScene = { group: Group; cameraProps: CameraProps; + animatedTextureParts: AnimatedTexturePart[]; }; export type CameraProps = { @@ -18,6 +19,24 @@ export type CameraProps = { zoom: number; }; +export type AnimatedTextureFrame = { + index: number; + time: number; +}; + +/** + * Implements Minecraft animated sprites, which update parts of a larger atlas texture. + */ +export type AnimatedTexturePart = { + targetTextures: Texture[]; + frameTextures: Texture[]; + frames: AnimatedTextureFrame[]; + x: number; + y: number; + currentFrame: number; + subFrame: number; +}; + async function decompressResponse(response: Response) { const blob = await response.blob(); response = await decompress(blob); @@ -49,7 +68,7 @@ export default async function loadScene( const buf = new flatbuffers.ByteBuffer(data); const group = new Group(); - + const texturesById = new Map(); const expScene = ExpScene.getRootAsExpScene(buf); for (let i = 0; i < expScene.meshesLength(); i++) { const expMesh = expScene.meshes(i); @@ -63,7 +82,11 @@ export default async function loadScene( } const geometry = loadGeometry(expMesh); - const material = await loadMaterial(textureManager, expMaterial); + const material = await loadMaterial( + textureManager, + expMaterial, + texturesById + ); const mesh = new Mesh(geometry, material); mesh.frustumCulled = false; group.add(mesh); @@ -73,6 +96,87 @@ export default async function loadScene( if (!expCamera) { throw new Error("Scene is missing camera settings"); } + + const animatedTextureParts: AnimatedTexturePart[] = []; + for (let i = 0; i < expScene.animatedTexturesLength(); i++) { + const animatedTexture = expScene.animatedTextures(i); + if (!animatedTexture) { + continue; + } + + const framesPath = animatedTexture.framesPath(); + if (!framesPath) { + continue; + } + + const fullUrl = textureManager.getFullUrl(framesPath); + const sourceDataResponse = await fetch(fullUrl, { signal: abortSignal }); + if (!sourceDataResponse.ok) { + console.error( + "Failed to retrieve animated texture %s: %o", + fullUrl, + sourceDataResponse + ); + continue; + } + const sourceData = await sourceDataResponse.blob(); + + // Splice it up into frames + const sourceFramePromises: Promise[] = []; + for (let j = 0; j < animatedTexture.frameCount(); j++) { + const frameX = + (j % animatedTexture.framesPerRow()) * animatedTexture.width(); + const frameY = + Math.floor(j / animatedTexture.framesPerRow()) * + animatedTexture.height(); + sourceFramePromises.push( + createImageBitmap( + sourceData, + frameX, + frameY, + animatedTexture.width(), + animatedTexture.height() + ) + ); + } + const frameTextures = (await Promise.allSettled(sourceFramePromises)).map( + (result) => { + if (result.status === "fulfilled") { + return new Texture(result.value); + } else { + return null!; // TODO + } + } + ); + + const targetTextures = + texturesById.get(animatedTexture.textureId() ?? "") ?? []; + if (!targetTextures.length) { + continue; + } + + const frames: AnimatedTextureFrame[] = []; + for (let j = 0; j < animatedTexture.framesLength(); j++) { + const expFrame = animatedTexture.frames(j); + if (expFrame) { + frames.push({ + index: expFrame.index(), + time: expFrame.time(), + }); + } + } + + animatedTextureParts.push({ + frameTextures, + targetTextures, + frames, + x: animatedTexture.x(), + y: animatedTexture.y(), + currentFrame: 0, + subFrame: 0, + }); + } + const cameraProps = { yaw: expCamera.yaw(), pitch: expCamera.pitch(), @@ -80,5 +184,5 @@ export default async function loadScene( zoom: expCamera.zoom(), }; - return { cameraProps, group }; + return { cameraProps, group, animatedTextureParts }; } diff --git a/src/generated/scene.ts b/src/generated/scene.ts index 728531d..759222c 100644 --- a/src/generated/scene.ts +++ b/src/generated/scene.ts @@ -1,5 +1,7 @@ // automatically generated by the FlatBuffers compiler, do not modify +export { ExpAnimatedTexturePart } from './scene/exp-animated-texture-part.js'; +export { ExpAnimatedTexturePartFrame } from './scene/exp-animated-texture-part-frame.js'; export { ExpCameraSettings } from './scene/exp-camera-settings.js'; export { ExpDepthTest } from './scene/exp-depth-test.js'; export { ExpIndexElementType } from './scene/exp-index-element-type.js'; diff --git a/src/generated/scene/exp-animated-texture-part-frame.ts b/src/generated/scene/exp-animated-texture-part-frame.ts new file mode 100644 index 0000000..0f943ff --- /dev/null +++ b/src/generated/scene/exp-animated-texture-part-frame.ts @@ -0,0 +1,33 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +import * as flatbuffers from 'flatbuffers'; + +export class ExpAnimatedTexturePartFrame { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):ExpAnimatedTexturePartFrame { + this.bb_pos = i; + this.bb = bb; + return this; +} + +index():number { + return this.bb!.readUint16(this.bb_pos); +} + +time():number { + return this.bb!.readUint16(this.bb_pos + 2); +} + +static sizeOf():number { + return 4; +} + +static createExpAnimatedTexturePartFrame(builder:flatbuffers.Builder, index: number, time: number):flatbuffers.Offset { + builder.prep(2, 4); + builder.writeInt16(time); + builder.writeInt16(index); + return builder.offset(); +} + +} diff --git a/src/generated/scene/exp-animated-texture-part.ts b/src/generated/scene/exp-animated-texture-part.ts new file mode 100644 index 0000000..f6d906d --- /dev/null +++ b/src/generated/scene/exp-animated-texture-part.ts @@ -0,0 +1,152 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +import * as flatbuffers from 'flatbuffers'; + +import { ExpAnimatedTexturePartFrame } from '../scene/exp-animated-texture-part-frame.js'; + + +export class ExpAnimatedTexturePart { + bb: flatbuffers.ByteBuffer|null = null; + bb_pos = 0; + __init(i:number, bb:flatbuffers.ByteBuffer):ExpAnimatedTexturePart { + this.bb_pos = i; + this.bb = bb; + return this; +} + +static getRootAsExpAnimatedTexturePart(bb:flatbuffers.ByteBuffer, obj?:ExpAnimatedTexturePart):ExpAnimatedTexturePart { + return (obj || new ExpAnimatedTexturePart()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +static getSizePrefixedRootAsExpAnimatedTexturePart(bb:flatbuffers.ByteBuffer, obj?:ExpAnimatedTexturePart):ExpAnimatedTexturePart { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new ExpAnimatedTexturePart()).__init(bb.readInt32(bb.position()) + bb.position(), bb); +} + +textureId():string|null +textureId(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null +textureId(optionalEncoding?:any):string|Uint8Array|null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; +} + +x():number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; +} + +y():number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; +} + +width():number { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.readUint32(this.bb_pos + offset) : 0; +} + +height():number { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? this.bb!.readUint32(this.bb_pos + offset) : 0; +} + +framesPath():string|null +framesPath(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null +framesPath(optionalEncoding?:any):string|Uint8Array|null { + const offset = this.bb!.__offset(this.bb_pos, 14); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; +} + +interpolate():boolean { + const offset = this.bb!.__offset(this.bb_pos, 16); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; +} + +frameCount():number { + const offset = this.bb!.__offset(this.bb_pos, 18); + return offset ? this.bb!.readUint32(this.bb_pos + offset) : 0; +} + +framesPerRow():number { + const offset = this.bb!.__offset(this.bb_pos, 20); + return offset ? this.bb!.readUint32(this.bb_pos + offset) : 0; +} + +frames(index: number, obj?:ExpAnimatedTexturePartFrame):ExpAnimatedTexturePartFrame|null { + const offset = this.bb!.__offset(this.bb_pos, 22); + return offset ? (obj || new ExpAnimatedTexturePartFrame()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 4, this.bb!) : null; +} + +framesLength():number { + const offset = this.bb!.__offset(this.bb_pos, 22); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; +} + +static startExpAnimatedTexturePart(builder:flatbuffers.Builder) { + builder.startObject(10); +} + +static addTextureId(builder:flatbuffers.Builder, textureIdOffset:flatbuffers.Offset) { + builder.addFieldOffset(0, textureIdOffset, 0); +} + +static addX(builder:flatbuffers.Builder, x:number) { + builder.addFieldInt32(1, x, 0); +} + +static addY(builder:flatbuffers.Builder, y:number) { + builder.addFieldInt32(2, y, 0); +} + +static addWidth(builder:flatbuffers.Builder, width:number) { + builder.addFieldInt32(3, width, 0); +} + +static addHeight(builder:flatbuffers.Builder, height:number) { + builder.addFieldInt32(4, height, 0); +} + +static addFramesPath(builder:flatbuffers.Builder, framesPathOffset:flatbuffers.Offset) { + builder.addFieldOffset(5, framesPathOffset, 0); +} + +static addInterpolate(builder:flatbuffers.Builder, interpolate:boolean) { + builder.addFieldInt8(6, +interpolate, +false); +} + +static addFrameCount(builder:flatbuffers.Builder, frameCount:number) { + builder.addFieldInt32(7, frameCount, 0); +} + +static addFramesPerRow(builder:flatbuffers.Builder, framesPerRow:number) { + builder.addFieldInt32(8, framesPerRow, 0); +} + +static addFrames(builder:flatbuffers.Builder, framesOffset:flatbuffers.Offset) { + builder.addFieldOffset(9, framesOffset, 0); +} + +static startFramesVector(builder:flatbuffers.Builder, numElems:number) { + builder.startVector(4, numElems, 2); +} + +static endExpAnimatedTexturePart(builder:flatbuffers.Builder):flatbuffers.Offset { + const offset = builder.endObject(); + return offset; +} + +static createExpAnimatedTexturePart(builder:flatbuffers.Builder, textureIdOffset:flatbuffers.Offset, x:number, y:number, width:number, height:number, framesPathOffset:flatbuffers.Offset, interpolate:boolean, frameCount:number, framesPerRow:number, framesOffset:flatbuffers.Offset):flatbuffers.Offset { + ExpAnimatedTexturePart.startExpAnimatedTexturePart(builder); + ExpAnimatedTexturePart.addTextureId(builder, textureIdOffset); + ExpAnimatedTexturePart.addX(builder, x); + ExpAnimatedTexturePart.addY(builder, y); + ExpAnimatedTexturePart.addWidth(builder, width); + ExpAnimatedTexturePart.addHeight(builder, height); + ExpAnimatedTexturePart.addFramesPath(builder, framesPathOffset); + ExpAnimatedTexturePart.addInterpolate(builder, interpolate); + ExpAnimatedTexturePart.addFrameCount(builder, frameCount); + ExpAnimatedTexturePart.addFramesPerRow(builder, framesPerRow); + ExpAnimatedTexturePart.addFrames(builder, framesOffset); + return ExpAnimatedTexturePart.endExpAnimatedTexturePart(builder); +} +} diff --git a/src/generated/scene/exp-sampler.ts b/src/generated/scene/exp-sampler.ts index dbbae86..60ea3ba 100644 --- a/src/generated/scene/exp-sampler.ts +++ b/src/generated/scene/exp-sampler.ts @@ -20,37 +20,48 @@ static getSizePrefixedRootAsExpSampler(bb:flatbuffers.ByteBuffer, obj?:ExpSample return (obj || new ExpSampler()).__init(bb.readInt32(bb.position()) + bb.position(), bb); } +textureId():string|null +textureId(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null +textureId(optionalEncoding?:any):string|Uint8Array|null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; +} + texture():string|null texture(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null texture(optionalEncoding?:any):string|Uint8Array|null { - const offset = this.bb!.__offset(this.bb_pos, 4); + const offset = this.bb!.__offset(this.bb_pos, 6); return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; } linearFiltering():boolean { - const offset = this.bb!.__offset(this.bb_pos, 6); + const offset = this.bb!.__offset(this.bb_pos, 8); return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; } useMipmaps():boolean { - const offset = this.bb!.__offset(this.bb_pos, 8); + const offset = this.bb!.__offset(this.bb_pos, 10); return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; } static startExpSampler(builder:flatbuffers.Builder) { - builder.startObject(3); + builder.startObject(4); +} + +static addTextureId(builder:flatbuffers.Builder, textureIdOffset:flatbuffers.Offset) { + builder.addFieldOffset(0, textureIdOffset, 0); } static addTexture(builder:flatbuffers.Builder, textureOffset:flatbuffers.Offset) { - builder.addFieldOffset(0, textureOffset, 0); + builder.addFieldOffset(1, textureOffset, 0); } static addLinearFiltering(builder:flatbuffers.Builder, linearFiltering:boolean) { - builder.addFieldInt8(1, +linearFiltering, +false); + builder.addFieldInt8(2, +linearFiltering, +false); } static addUseMipmaps(builder:flatbuffers.Builder, useMipmaps:boolean) { - builder.addFieldInt8(2, +useMipmaps, +false); + builder.addFieldInt8(3, +useMipmaps, +false); } static endExpSampler(builder:flatbuffers.Builder):flatbuffers.Offset { @@ -58,8 +69,9 @@ static endExpSampler(builder:flatbuffers.Builder):flatbuffers.Offset { return offset; } -static createExpSampler(builder:flatbuffers.Builder, textureOffset:flatbuffers.Offset, linearFiltering:boolean, useMipmaps:boolean):flatbuffers.Offset { +static createExpSampler(builder:flatbuffers.Builder, textureIdOffset:flatbuffers.Offset, textureOffset:flatbuffers.Offset, linearFiltering:boolean, useMipmaps:boolean):flatbuffers.Offset { ExpSampler.startExpSampler(builder); + ExpSampler.addTextureId(builder, textureIdOffset); ExpSampler.addTexture(builder, textureOffset); ExpSampler.addLinearFiltering(builder, linearFiltering); ExpSampler.addUseMipmaps(builder, useMipmaps); diff --git a/src/generated/scene/exp-scene.ts b/src/generated/scene/exp-scene.ts index 41e51ed..ae6b681 100644 --- a/src/generated/scene/exp-scene.ts +++ b/src/generated/scene/exp-scene.ts @@ -2,6 +2,7 @@ import * as flatbuffers from 'flatbuffers'; +import { ExpAnimatedTexturePart } from '../scene/exp-animated-texture-part.js'; import { ExpCameraSettings } from '../scene/exp-camera-settings.js'; import { ExpMesh } from '../scene/exp-mesh.js'; @@ -39,8 +40,18 @@ meshesLength():number { return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; } +animatedTextures(index: number, obj?:ExpAnimatedTexturePart):ExpAnimatedTexturePart|null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? (obj || new ExpAnimatedTexturePart()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; +} + +animatedTexturesLength():number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; +} + static startExpScene(builder:flatbuffers.Builder) { - builder.startObject(2); + builder.startObject(3); } static addCamera(builder:flatbuffers.Builder, cameraOffset:flatbuffers.Offset) { @@ -63,6 +74,22 @@ static startMeshesVector(builder:flatbuffers.Builder, numElems:number) { builder.startVector(4, numElems, 4); } +static addAnimatedTextures(builder:flatbuffers.Builder, animatedTexturesOffset:flatbuffers.Offset) { + builder.addFieldOffset(2, animatedTexturesOffset, 0); +} + +static createAnimatedTexturesVector(builder:flatbuffers.Builder, data:flatbuffers.Offset[]):flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]!); + } + return builder.endVector(); +} + +static startAnimatedTexturesVector(builder:flatbuffers.Builder, numElems:number) { + builder.startVector(4, numElems, 4); +} + static endExpScene(builder:flatbuffers.Builder):flatbuffers.Offset { const offset = builder.endObject(); return offset; @@ -76,10 +103,11 @@ static finishSizePrefixedExpSceneBuffer(builder:flatbuffers.Builder, offset:flat builder.finish(offset, undefined, true); } -static createExpScene(builder:flatbuffers.Builder, cameraOffset:flatbuffers.Offset, meshesOffset:flatbuffers.Offset):flatbuffers.Offset { +static createExpScene(builder:flatbuffers.Builder, cameraOffset:flatbuffers.Offset, meshesOffset:flatbuffers.Offset, animatedTexturesOffset:flatbuffers.Offset):flatbuffers.Offset { ExpScene.startExpScene(builder); ExpScene.addCamera(builder, cameraOffset); ExpScene.addMeshes(builder, meshesOffset); + ExpScene.addAnimatedTextures(builder, animatedTexturesOffset); return ExpScene.endExpScene(builder); } }