Merge pull request #15558 from CedricGuillemet/MeshHotspots

Mesh HotSpot

packages/dev/core/src/Meshes/abstractMesh.hotSpot.ts

@@ -0,0 +1,105 @@
+import { Vector3, TmpVectors, Matrix } from "../Maths/math.vector";
+import type { AbstractMesh } from "./abstractMesh";
+import { VertexBuffer } from "../Buffers/buffer";
+
+/**
+ * Data for mesh hotspot computation
+ */
+export type HotSpotQuery = {
+    /**
+     * 3 point indices
+     */
+    pointIndex: [number, number, number];
+    /**
+     * 3 barycentric coordinates
+     */
+    barycentric: [number, number, number];
+};
+
+/**
+ * Return a transformed local position from a mesh and vertex index
+ * @param mesh mesh used to get vertex array from
+ * @param index vertex index
+ * @param res resulting local position
+ * @returns false if it was not possible to compute the position for that vertex
+ */
+export function GetTransformedPosition(mesh: AbstractMesh, index: number, res: Vector3): boolean {
+    const data = mesh.getVerticesData(VertexBuffer.PositionKind);
+    if (!data) {
+        return false;
+    }
+    const base = index * 3;
+    const values = [data[base + 0], data[base + 1], data[base + 2]];
+    if (mesh.morphTargetManager) {
+        for (let component = 0; component < 3; component++) {
+            let value = values[component];
+            for (let targetCount = 0; targetCount < mesh.morphTargetManager.numTargets; targetCount++) {
+                const target = mesh.morphTargetManager.getTarget(targetCount);
+                const influence = target.influence;
+                if (influence !== 0) {
+                    const targetData = target.getPositions();
+                    if (targetData) {
+                        value += (targetData[base + component] - data[base + component]) * influence;
+                    }
+                }
+            }
+            values[component] = value;
+        }
+    }
+    res.fromArray(values);
+    if (mesh.skeleton) {
+        const matricesIndicesData = mesh.getVerticesData(VertexBuffer.MatricesIndicesKind);
+        const matricesWeightsData = mesh.getVerticesData(VertexBuffer.MatricesWeightsKind);
+        if (matricesWeightsData && matricesIndicesData) {
+            const needExtras = mesh.numBoneInfluencers > 4;
+            const matricesIndicesExtraData = needExtras ? mesh.getVerticesData(VertexBuffer.MatricesIndicesExtraKind) : null;
+            const matricesWeightsExtraData = needExtras ? mesh.getVerticesData(VertexBuffer.MatricesWeightsExtraKind) : null;
+            const skeletonMatrices = mesh.skeleton.getTransformMatrices(mesh);
+
+            const finalMatrix = TmpVectors.Matrix[0];
+            const tempMatrix = TmpVectors.Matrix[1];
+
+            finalMatrix.reset();
+            const matWeightIdx = index * 4;
+
+            let inf: number;
+            let weight: number;
+            for (inf = 0; inf < 4; inf++) {
+                weight = matricesWeightsData[matWeightIdx + inf];
+                if (weight > 0) {
+                    Matrix.FromFloat32ArrayToRefScaled(skeletonMatrices, Math.floor(matricesIndicesData[matWeightIdx + inf] * 16), weight, tempMatrix);
+                    finalMatrix.addToSelf(tempMatrix);
+                }
+            }
+            if (matricesIndicesExtraData && matricesWeightsExtraData) {
+                for (inf = 0; inf < 4; inf++) {
+                    weight = matricesWeightsExtraData[matWeightIdx + inf];
+                    if (weight > 0) {
+                        Matrix.FromFloat32ArrayToRefScaled(skeletonMatrices, Math.floor(matricesIndicesExtraData[matWeightIdx + inf] * 16), weight, tempMatrix);
+                        finalMatrix.addToSelf(tempMatrix);
+                    }
+                }
+            }
+
+            Vector3.TransformCoordinatesFromFloatsToRef(values[0], values[1], values[2], finalMatrix, res);
+        }
+    }
+
+    return true;
+}
+
+/**
+ * Compute a world space hotspot position
+ * @param mesh mesh used to get hotspot from
+ * @param hotSpotQuery point indices and barycentric
+ * @param res output world position
+ */
+export function GetHotSpotToRef(mesh: AbstractMesh, hotSpotQuery: HotSpotQuery, res: Vector3): void {
+    res.set(0, 0, 0);
+    for (let i = 0; i < 3; i++) {
+        const index = hotSpotQuery.pointIndex[i];
+        GetTransformedPosition(mesh, index, TmpVectors.Vector3[0]);
+        TmpVectors.Vector3[0].scaleInPlace(hotSpotQuery.barycentric[i]);
+        res.addInPlace(TmpVectors.Vector3[0]);
+    }
+}

packages/dev/core/src/Meshes/index.ts

@@ -2,6 +2,7 @@
 /* eslint-disable import/no-internal-modules */
 export * from "./abstractMesh";
 import "./abstractMesh.decalMap";
+export * from "./abstractMesh.hotSpot";
 export * from "./Compression/index";
 export * from "./csg";
 export * from "./meshUVSpaceRenderer";

packages/tools/viewer-alpha/src/viewer.ts

@@ -5,6 +5,7 @@ import type {
     AutoRotationBehavior,
     Camera,
     FramingBehavior,
+    HotSpotQuery,
     IDisposable,
     LoadAssetContainerOptions,
     Mesh,
@@ -21,13 +22,15 @@ import { CubeTexture } from "core/Materials/Textures/cubeTexture";
 import { Texture } from "core/Materials/Textures/texture";
 import { Color4 } from "core/Maths/math.color";
 import { Clamp } from "core/Maths/math.scalar.functions";
-import { Vector3 } from "core/Maths/math.vector";
+import { TmpVectors, Vector3 } from "core/Maths/math.vector";
 import { CreateBox } from "core/Meshes/Builders/boxBuilder";
 import { computeMaxExtents } from "core/Meshes/meshUtils";
 import { AsyncLock } from "core/Misc/asyncLock";
 import { Observable } from "core/Misc/observable";
 import { Scene, ScenePerformancePriority } from "core/scene";
 import { registerBuiltInLoaders } from "loaders/dynamic";
+import { Viewport } from "core/Maths/math.viewport";
+import { GetHotSpotToRef } from "core/Meshes/abstractMesh.hotSpot";
 
 function throwIfAborted(...abortSignals: (Nullable<AbortSignal> | undefined)[]): void {
     for (const signal of abortSignals) {
@@ -91,6 +94,27 @@ export type ViewerOptions = Partial<
         }>
 >;
 
+export type ViewerHotSpotQuery = {
+    /**
+     * The index of the mesh within the loaded model.
+     */
+    meshIndex: number;
+} & HotSpotQuery;
+
+/**
+ * Information computed from the hot spot surface data, canvas and mesh datas
+ */
+export type ViewerHotSpot = {
+    /**
+     * 2D canvas position in pixels
+     */
+    screenPosition: [number, number];
+    /**
+     * 3D world coordinates
+     */
+    worldPosition: [number, number, number];
+};
+
 /**
  * Provides an experience for viewing a single 3D model.
  * @remarks
@@ -483,6 +507,37 @@ export class Viewer implements IDisposable {
         this._isDisposed = true;
     }
 
+    /**
+     * retrun world and canvas coordinates of an hot spot
+     * @param hotSpotQuery mesh index and surface information to query the hot spot positions
+     * @param res Query a Hot Spot and does the conversion for Babylon Hot spot to a more generic HotSpotPositions, without Vector types
+     * @returns true if hotspot found
+     */
+    public getHotSpotToRef(hotSpotQuery: Readonly<ViewerHotSpotQuery>, res: ViewerHotSpot): boolean {
+        if (!this._details.model) {
+            return false;
+        }
+        const worldPos = TmpVectors.Vector3[1];
+        const screenPos = TmpVectors.Vector3[0];
+        const mesh = this._details.model.meshes[hotSpotQuery.meshIndex];
+        if (!mesh) {
+            return false;
+        }
+        GetHotSpotToRef(mesh, hotSpotQuery, worldPos);
+
+        const renderWidth = this._engine.getRenderWidth(); // Get the canvas width
+        const renderHeight = this._engine.getRenderHeight(); // Get the canvas height
+
+        const viewportWidth = this._camera.viewport.width * renderWidth;
+        const viewportHeight = this._camera.viewport.height * renderHeight;
+        const scene = this._details.scene;
+
+        Vector3.ProjectToRef(worldPos, mesh.getWorldMatrix(), scene.getTransformMatrix(), new Viewport(0, 0, viewportWidth, viewportHeight), screenPos);
+        res.screenPosition = [screenPos.x, screenPos.y];
+        res.worldPosition = [worldPos.x, worldPos.y, worldPos.z];
+        return true;
+    }
+
     private _updateCamera(): void {
         // Enable camera's behaviors
         this._camera.useFramingBehavior = true;

packages/tools/viewer-alpha/src/viewerElement.ts

@@ -2,7 +2,7 @@
 import type { Nullable } from "core/index";
 
 import type { PropertyValues } from "lit";
-import type { ViewerDetails } from "./viewer";
+import type { ViewerDetails, ViewerHotSpot, ViewerHotSpotQuery } from "./viewer";
 import type { CanvasViewerOptions } from "./viewerFactory";
 
 import { LitElement, css, html } from "lit";
@@ -66,6 +66,13 @@ export class HTML3DElement extends LitElement {
             height: 100%;
         }
 
+        .children-slot {
+            position: absolute;
+            top: 0;
+            background: transparent;
+            pointer-events: none;
+        }
+
         .tool-bar {
             position: absolute;
             display: flex;
@@ -216,6 +223,24 @@ export class HTML3DElement extends LitElement {
         return this._viewerDetails;
     }
 
+    /**
+     * Get hotspot world and screen values from a named hotspot
+     * @param name slot of the hot spot
+     * @param result resulting world and screen positions
+     * @returns world and screen space coordinates
+     */
+    public queryHotSpot(name: string, result: ViewerHotSpot): boolean {
+        // Retrieve all hotspots inside the viewer element
+        let resultFound = false;
+        // Iterate through each hotspot to get the 'data-surface' and 'data-name' attributes
+        if (this._viewerDetails) {
+            const hotspot = this.hotspots?.[name];
+            if (hotspot) {
+                resultFound = this._viewerDetails.viewer.getHotSpotToRef(hotspot, result);
+            }
+        }
+        return resultFound;
+    }
     /**
      * The engine to use for rendering.
      */
@@ -242,6 +267,36 @@ export class HTML3DElement extends LitElement {
     @property({ reflect: true })
     public environment: Nullable<string> = null;
 
+    /**
+     * A string value that encodes one or more hotspots.
+     */
+    @property({
+        type: "string",
+        converter: (value) => {
+            if (!value) {
+                return null;
+            }
+
+            const array = value.split(" ");
+            if (array.length % 8 !== 0) {
+                throw new Error(
+                    `hotspots should be defined in sets of 8 elements: 'name meshIndex pointIndex1 pointIndex2 pointIndex3 barycentricCoord1 barycentricCoord2 barycentricCoord3', but a total of ${array.length} elements were found in '${value}'`
+                );
+            }
+
+            const hotspots: Record<string, ViewerHotSpotQuery> = {};
+            for (let offset = 0; offset < array.length; offset += 8) {
+                hotspots[array[offset]] = {
+                    meshIndex: Number(array[offset + 1]),
+                    pointIndex: [Number(array[offset + 2]), Number(array[offset + 3]), Number(array[offset + 4])],
+                    barycentric: [Number(array[offset + 5]), Number(array[offset + 6]), Number(array[offset + 7])],
+                };
+            }
+            return hotspots;
+        },
+    })
+    public hotspots: Nullable<Record<string, ViewerHotSpotQuery>> = null;
+
     /**
      * The list of animation names for the currently loaded model.
      */
@@ -334,9 +389,11 @@ export class HTML3DElement extends LitElement {
 
     // eslint-disable-next-line babylonjs/available
     override render() {
+        // NOTE: The unnamed 'slot' element holds all child elements of the <babylon-viewer> that do not specify a 'slot' attribute.
         return html`
             <div class="full-size">
                 <div id="canvasContainer" class="full-size"></div>
+                <slot class="full-size children-slot"></slot>
                 ${this.animations.length === 0
                     ? ""
                     : html`