Dynamic hotspots (#3981)

* switch to three.js reduceVertices

* cleanup and refactor of fromPoint methods

* implementation complete

* updated editor to dynamic hotspots

* position working

* added normals

* fixed editor tests

* added test

* added docs

* added example

* update

Author: elalish

packages/model-viewer/src/features/annotation.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import {Matrix3, Matrix4, Vector3} from 'three';
+import {Matrix4, Vector3} from 'three';
 
 import ModelViewerElementBase, {$needsRender, $scene, $tick, toVector2D, toVector3D, Vector2D, Vector3D} from '../model-viewer-base.js';
 import {Hotspot, HotspotConfiguration} from '../three-components/Hotspot.js';
@@ -27,7 +27,6 @@ const $addHotspot = Symbol('addHotspot');
 const $removeHotspot = Symbol('removeHotspot');
 
 const worldToModel = new Matrix4();
-const worldToModelNormal = new Matrix3();
 
 export declare type HotspotData = {
   position: Vector3D,
@@ -40,7 +39,8 @@ export declare interface AnnotationInterface {
   updateHotspot(config: HotspotConfiguration): void;
   queryHotspot(name: string): HotspotData|null;
   positionAndNormalFromPoint(pixelX: number, pixelY: number):
-      {position: Vector3D, normal: Vector3D, uv: Vector2D|null}|null
+      {position: Vector3D, normal: Vector3D, uv: Vector2D|null}|null;
+  surfaceFromPoint(pixelX: number, pixelY: number): string|null;
 }
 
 /**
@@ -108,7 +108,8 @@ export const AnnotationMixin = <T extends Constructor<ModelViewerElementBase>>(
       const camera = scene.getCamera();
 
       if (scene.shouldRender()) {
-        scene.updateHotspots(camera.position);
+        scene.updateSurfaceHotspots();
+        scene.updateHotspotsVisibility(camera.position);
         annotationRenderer.domElement.style.display = '';
         annotationRenderer.render(scene, camera);
       }
@@ -190,10 +191,7 @@ export const AnnotationMixin = <T extends Constructor<ModelViewerElementBase>>(
 
       worldToModel.copy(scene.target.matrixWorld).invert();
       const position = toVector3D(hit.position.applyMatrix4(worldToModel));
-
-      worldToModelNormal.getNormalMatrix(worldToModel);
-      const normal =
-          toVector3D(hit.normal.applyNormalMatrix(worldToModelNormal));
+      const normal = toVector3D(hit.normal.transformDirection(worldToModel));
 
       let uv = null;
       if (hit.uv != null) {
@@ -203,6 +201,20 @@ export const AnnotationMixin = <T extends Constructor<ModelViewerElementBase>>(
       return {position: position, normal: normal, uv: uv};
     }
 
+    /**
+     * This method returns a dynamic hotspot ID string of the point on the mesh
+     * corresponding to the input pixel coordinates given relative to the
+     * model-viewer element. The ID string can be used in the data-surface
+     * attribute of the hotspot to make it follow this point on the surface even
+     * as the model animates. If the mesh is not hit, the result is null.
+     */
+    surfaceFromPoint(pixelX: number, pixelY: number): string|null {
+      const scene = this[$scene];
+      const ndcPosition = scene.getNDC(pixelX, pixelY);
+
+      return scene.surfaceFromPoint(ndcPosition);
+    }
+
     private[$addHotspot](node: Node) {
       if (!(node instanceof HTMLElement &&
             node.slot.indexOf('hotspot') === 0)) {
@@ -218,6 +230,7 @@ export const AnnotationMixin = <T extends Constructor<ModelViewerElementBase>>(
           name: node.slot,
           position: node.dataset.position,
           normal: node.dataset.normal,
+          surface: node.dataset.surface,
         });
         this[$hotspotMap].set(node.slot, hotspot);
         this[$scene].addHotspot(hotspot);

packages/model-viewer/src/features/scene-graph.ts

@@ -276,11 +276,18 @@ export const SceneGraphMixin = <T extends Constructor<ModelViewerElementBase>>(
     }
 
     materialFromPoint(pixelX: number, pixelY: number): Material|null {
+      const model = this[$model];
+      if (model == null) {
+        return null;
+      }
       const scene = this[$scene];
       const ndcCoords = scene.getNDC(pixelX, pixelY);
-      scene.raycaster.setFromCamera(ndcCoords, scene.getCamera());
+      const hit = scene.hitFromPoint(ndcCoords);
+      if (hit == null || hit.face == null) {
+        return null;
+      }
 
-      return this[$model]![$materialFromPoint](scene.raycaster);
+      return model[$materialFromPoint](hit);
     }
   }
 

packages/model-viewer/src/features/scene-graph/model.ts

@@ -13,14 +13,14 @@
  * limitations under the License.
  */
 
-import {Group, Intersection, Material as ThreeMaterial, Mesh, MeshStandardMaterial, Object3D, Raycaster} from 'three';
+import {Intersection, Material as ThreeMaterial, Mesh, MeshStandardMaterial, Object3D} from 'three';
 
 import {CorrelatedSceneGraph, GLTFElementToThreeObjectMap, ThreeObjectSet} from '../../three-components/gltf-instance/correlated-scene-graph.js';
 import {GLTF, GLTFElement, Material as GLTFMaterial} from '../../three-components/gltf-instance/gltf-2.0.js';
 
 import {Model as ModelInterface} from './api.js';
 import {$setActive, $variantSet, Material} from './material.js';
-import {$children, Node, PrimitiveNode} from './nodes/primitive-node.js';
+import {Node, PrimitiveNode} from './nodes/primitive-node.js';
 import {$correlatedObjects, $sourceObject} from './three-dom-element.js';
 
 
@@ -33,9 +33,9 @@ export const $loadVariant = Symbol('loadVariant');
 export const $correlatedSceneGraph = Symbol('correlatedSceneGraph');
 export const $prepareVariantsForExport = Symbol('prepareVariantsForExport');
 export const $switchVariant = Symbol('switchVariant');
-export const $threeScene = Symbol('threeScene');
-export const $materialsFromPoint = Symbol('materialsFromPoint');
 export const $materialFromPoint = Symbol('materialFromPoint');
+export const $nodeFromPoint = Symbol('nodeFromPoint');
+export const $nodeFromIndex = Symbol('nodeFromIndex');
 export const $variantData = Symbol('variantData');
 export const $availableVariants = Symbol('availableVariants');
 const $modelOnUpdate = Symbol('modelOnUpdate');
@@ -78,7 +78,6 @@ export class Model implements ModelInterface {
   private[$hierarchy] = new Array<Node>();
   private[$roots] = new Array<Node>();
   private[$primitivesList] = new Array<PrimitiveNode>();
-  private[$threeScene]: Object3D|Group;
   private[$modelOnUpdate]: () => void = () => {};
   private[$correlatedSceneGraph]: CorrelatedSceneGraph;
   private[$variantData] = new Map<string, VariantData>();
@@ -89,7 +88,6 @@ export class Model implements ModelInterface {
     this[$modelOnUpdate] = onUpdate;
     this[$correlatedSceneGraph] = correlatedSceneGraph;
     const {gltf, threeGLTF, gltfElementMap} = correlatedSceneGraph;
-    this[$threeScene] = threeGLTF.scene;
 
     for (const [i, material] of gltf.materials!.entries()) {
       const correlatedMaterial =
@@ -168,7 +166,7 @@ export class Model implements ModelInterface {
 
       const parent: Node|undefined = parentMap.get(object);
       if (parent != null) {
-        parent[$children].push(node);
+        parent.children.push(node);
       } else {
         this[$roots].push(node);
       }
@@ -213,46 +211,37 @@ export class Model implements ModelInterface {
     return null;
   }
 
-
-  /**
-   * Intersects a ray with the Model and returns a list of materials whose
-   * objects were intersected.
-   */
-  [$materialsFromPoint](raycaster: Raycaster): Material[] {
-    const hits = raycaster.intersectObject(this[$threeScene], true);
-
-    // Map the object hits to primitives and then to the active material of
-    // the primitive.
-    return hits.map((hit: Intersection<Object3D>) => {
-      const found = this[$hierarchy].find((node: Node) => {
-        if (node instanceof PrimitiveNode) {
-          const primitive = node as PrimitiveNode;
-          if (primitive.mesh === hit.object) {
-            return true;
-          }
+  [$nodeFromIndex](mesh: number, primitive: number): PrimitiveNode|null {
+    const found = this[$hierarchy].find((node: Node) => {
+      if (node instanceof PrimitiveNode) {
+        const {meshes, primitives} = node.mesh.userData.associations;
+        if (meshes == mesh && primitives == primitive) {
+          return true;
         }
-        return false;
-      }) as PrimitiveNode;
+      }
+      return false;
+    });
+    return found == null ? null : found as PrimitiveNode;
+  }
 
-      if (found != null) {
-        return found.getActiveMaterial();
+  [$nodeFromPoint](hit: Intersection<Object3D>): PrimitiveNode {
+    return this[$hierarchy].find((node: Node) => {
+      if (node instanceof PrimitiveNode) {
+        const primitive = node as PrimitiveNode;
+        if (primitive.mesh === hit.object) {
+          return true;
+        }
       }
-      return null;
-    }) as Material[];
+      return false;
+    }) as PrimitiveNode;
   }
 
   /**
    * Intersects a ray with the Model and returns the first material whose
    * object was intersected.
    */
-  [$materialFromPoint](raycaster: Raycaster): Material|null {
-    const materials = this[$materialsFromPoint](raycaster);
-
-    if (materials.length > 0) {
-      return materials[0];
-    }
-
-    return null;
+  [$materialFromPoint](hit: Intersection<Object3D>): Material {
+    return this[$nodeFromPoint](hit).getActiveMaterial();
   }
 
   /**