Video texture (#3886)

* added createElementTexture and example

* moved updates to setTexture

* turn off video updates

* added initial Lottie support

* updated create textures api

* todo

* started dynamic import

* back to bundled lottie for now

* lazy load working

* added docs

* updated docs

Author: elalish

packages/model-viewer/package-lock.json

@@ -103,7 +103,8 @@
     "rollup-plugin-dts": "^4.2.2",
     "rollup-plugin-polyfill": "^3.0.0",
     "rollup-plugin-terser": "^7.0.2",
-    "typescript": "4.8.4"
+    "typescript": "4.8.4",
+    "lottie-web": "5.9.6"
   },
   "publishConfig": {
     "access": "public"

packages/model-viewer/package.json

@@ -33,6 +33,9 @@ const DEFAULT_DRACO_DECODER_LOCATION =
 const DEFAULT_KTX2_TRANSCODER_LOCATION =
     'https://www.gstatic.com/basis-universal/versioned/2021-04-15-ba1c3e4/';
 
+const DEFAULT_LOTTIE_LOADER_LOCATION =
+    'https://cdn.jsdelivr.net/npm/[email protected]/examples/jsm/loaders/LottieLoader.js';
+
 const RevealStrategy: {[index: string]: RevealAttributeValue} = {
   AUTO: 'auto',
   MANUAL: 'manual'
@@ -74,13 +77,15 @@ export declare interface LoadingStaticInterface {
   dracoDecoderLocation: string;
   ktx2TranscoderLocation: string;
   meshoptDecoderLocation: string;
+  lottieLoaderLocation: string;
   mapURLs(callback: (url: string) => string): void;
 }
 
 export interface ModelViewerGlobalConfig {
   dracoDecoderLocation?: string;
   ktx2TranscoderLocation?: string;
   meshoptDecoderLocation?: string;
+  lottieLoaderLocation?: string;
   powerPreference?: string;
 }
 
@@ -161,6 +166,14 @@ export const LoadingMixin = <T extends Constructor<ModelViewerElementBase>>(
       return CachingGLTFLoader.getMeshoptDecoderLocation();
     }
 
+    static set lottieLoaderLocation(value: string) {
+      Renderer.singleton.textureUtils!.lottieLoaderUrl = value;
+    }
+
+    static get lottieLoaderLocation() {
+      return Renderer.singleton.textureUtils!.lottieLoaderUrl
+    }
+
     /**
      * If provided, the callback will be passed each resource URL before a
      * request is sent. The callback may return the original URL, or a new URL
@@ -313,6 +326,10 @@ export const LoadingMixin = <T extends Constructor<ModelViewerElementBase>>(
         CachingGLTFLoader.setMeshoptDecoderLocation(
             ModelViewerElement.meshoptDecoderLocation);
       }
+
+      const lottieLoaderLocation = ModelViewerElement.lottieLoaderLocation ||
+          DEFAULT_LOTTIE_LOADER_LOCATION;
+      Renderer.singleton.textureUtils!.lottieLoaderUrl = lottieLoaderLocation;
     }
 
     connectedCallback() {

packages/model-viewer/src/features/loading.ts

@@ -14,7 +14,7 @@
  */
 
 import {property} from 'lit/decorators.js';
-import {RepeatWrapping, sRGBEncoding, Texture, TextureLoader} from 'three';
+import {CanvasTexture, RepeatWrapping, sRGBEncoding, Texture, VideoTexture} from 'three';
 import {GLTFExporter, GLTFExporterOptions} from 'three/examples/jsm/exporters/GLTFExporter.js';
 
 import ModelViewerElementBase, {$needsRender, $onModelLoad, $progressTracker, $renderer, $scene} from '../model-viewer-base.js';
@@ -34,7 +34,7 @@ export const $currentGLTF = Symbol('currentGLTF');
 export const $originalGltfJson = Symbol('originalGltfJson');
 export const $model = Symbol('model');
 const $getOnUpdateMethod = Symbol('getOnUpdateMethod');
-const $textureLoader = Symbol('textureLoader');
+const $buildTexture = Symbol('buildTexture');
 
 interface SceneExportOptions {
   binary?: boolean, trs?: boolean, onlyVisible?: boolean,
@@ -51,6 +51,10 @@ export interface SceneGraphInterface {
   readonly originalGltfJson: GLTF|null;
   exportScene(options?: SceneExportOptions): Promise<Blob>;
   createTexture(uri: string, type?: string): Promise<ModelViewerTexture|null>;
+  createLottieTexture(uri: string, quality?: number):
+      Promise<ModelViewerTexture|null>;
+  createVideoTexture(uri: string): ModelViewerTexture;
+  createCanvasTexture(): ModelViewerTexture;
   /**
    * Intersects a ray with the scene and returns a list of materials who's
    * objects were intersected.
@@ -70,7 +74,6 @@ export const SceneGraphMixin = <T extends Constructor<ModelViewerElementBase>>(
   class SceneGraphModelViewerElement extends ModelViewerElement {
     protected[$model]: Model|undefined = undefined;
     protected[$currentGLTF]: ModelViewerGLTFInstance|null = null;
-    private[$textureLoader] = new TextureLoader();
     private[$originalGltfJson]: GLTF|null = null;
 
     @property({type: String, attribute: 'variant-name'})
@@ -117,22 +120,48 @@ export const SceneGraphMixin = <T extends Constructor<ModelViewerElementBase>>(
       };
     }
 
-    async createTexture(uri: string, type: string = 'image/png'):
-        Promise<ModelViewerTexture|null> {
-      const currentGLTF = this[$currentGLTF];
-      const texture: Texture = await new Promise<Texture>(
-          (resolve) => this[$textureLoader].load(uri, resolve));
-      if (!currentGLTF || !texture) {
-        return null;
-      }
-      // Applies default settings.
+    private[$buildTexture](texture: Texture): ModelViewerTexture {
+      // Applies glTF default settings.
       texture.encoding = sRGBEncoding;
       texture.wrapS = RepeatWrapping;
       texture.wrapT = RepeatWrapping;
-      texture.flipY = false;
+      return new ModelViewerTexture(this[$getOnUpdateMethod](), texture);
+    }
+
+    async createTexture(uri: string, type: string = 'image/png'):
+        Promise<ModelViewerTexture> {
+      const {textureUtils} = this[$renderer];
+      const texture = await textureUtils!.loadImage(uri);
+
       texture.userData.mimeType = type;
 
-      return new ModelViewerTexture(this[$getOnUpdateMethod](), texture);
+      return this[$buildTexture](texture);
+    }
+
+    async createLottieTexture(uri: string, quality = 1):
+        Promise<ModelViewerTexture> {
+      const {textureUtils} = this[$renderer];
+      const texture = await textureUtils!.loadLottie(uri, quality);
+
+      return this[$buildTexture](texture);
+    }
+
+    createVideoTexture(uri: string): ModelViewerTexture {
+      const video = document.createElement('video');
+      video.src = uri;
+      video.muted = true;
+      video.play();
+      video.loop = true;
+      const texture = new VideoTexture(video);
+
+      return this[$buildTexture](texture);
+    }
+
+    createCanvasTexture(): ModelViewerTexture {
+      const canvas = document.createElement('canvas');
+      const texture = new CanvasTexture(canvas);
+
+      return this[$buildTexture](texture);
     }
 
     async updated(changedProperties: Map<string, any>) {

packages/model-viewer/src/features/scene-graph.ts

@@ -12,6 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import {AnimationItem} from 'lottie-web';
+
 import {AlphaMode, MagFilter, MinFilter, WrapMode} from '../../three-components/gltf-instance/gltf-2.0.js';
 
 
@@ -314,14 +316,30 @@ export declare interface Image {
   /**
    * The bufferView of the image, if it is embedded.
    */
-  readonly bufferView?: number
+  readonly bufferView?: number;
+
+  /**
+   * The backing HTML element, if this is a video or canvas texture.
+   */
+  readonly element?: HTMLVideoElement|HTMLCanvasElement;
+
+  /**
+   * The Lottie animation object, if this is a Lottie texture.
+   */
+  readonly animation?: AnimationItem;
 
   /**
    * A method to create an object URL of this image at the desired
    * resolution. Especially useful for KTX2 textures which are GPU compressed,
    * and so are unreadable on the CPU without a method like this.
    */
   createThumbnail(width: number, height: number): Promise<string>;
+
+  /**
+   * Only applies to canvas textures. Call when the content of the canvas has
+   * been updated and should be reflected in the model.
+   */
+  update(): void;
 }
 
 /**

packages/model-viewer/src/features/scene-graph/api.ts

@@ -13,14 +13,15 @@
  * limitations under the License.
  */
 
+import {AnimationItem} from 'lottie-web';
 import {Mesh, MeshBasicMaterial, OrthographicCamera, PlaneGeometry, Scene, Texture as ThreeTexture, WebGLRenderTarget} from 'three';
 
 import {blobCanvas} from '../../model-viewer-base.js';
 import {Image as GLTFImage} from '../../three-components/gltf-instance/gltf-2.0.js';
 import {Renderer} from '../../three-components/Renderer.js';
 
 import {Image as ImageInterface} from './api.js';
-import {$correlatedObjects, $sourceObject, ThreeDOMElement} from './three-dom-element.js';
+import {$correlatedObjects, $onUpdate, $sourceObject, ThreeDOMElement} from './three-dom-element.js';
 
 
 const quadMaterial = new MeshBasicMaterial();
@@ -67,6 +68,22 @@ export class Image extends ThreeDOMElement implements ImageInterface {
     return (this[$sourceObject] as GLTFImage).bufferView;
   }
 
+  get element(): HTMLVideoElement|HTMLCanvasElement|undefined {
+    const texture = this[$threeTexture] as any;
+    if (texture && (texture.isCanvasTexture || texture.isVideoTexture)) {
+      return texture.image;
+    }
+    return;
+  }
+
+  get animation(): AnimationItem|undefined {
+    const texture = this[$threeTexture] as any;
+    if (texture && texture.isCanvasTexture && texture.animation) {
+      return texture.animation;
+    }
+    return;
+  }
+
   get type(): 'embedded'|'external' {
     return this.uri != null ? 'external' : 'embedded';
   }
@@ -75,6 +92,15 @@ export class Image extends ThreeDOMElement implements ImageInterface {
     (this[$sourceObject] as GLTFImage).name = name;
   }
 
+  update() {
+    const texture = this[$threeTexture] as any;
+    // Applies to non-Lottie canvas textures only
+    if (texture && texture.isCanvasTexture && !texture.animation) {
+      this[$threeTexture].needsUpdate = true;
+      this[$onUpdate]();
+    }
+  }
+
   async createThumbnail(width: number, height: number): Promise<string> {
     const scene = new Scene();
     quadMaterial.map = this[$threeTexture];

packages/model-viewer/src/features/scene-graph/image.ts

@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-import {LinearEncoding, MeshStandardMaterial, sRGBEncoding, Texture as ThreeTexture, TextureEncoding, Vector2} from 'three';
+import {LinearEncoding, MeshStandardMaterial, sRGBEncoding, Texture as ThreeTexture, TextureEncoding, Vector2, VideoTexture} from 'three';
 
 import {GLTF, TextureInfo as GLTFTextureInfo} from '../../three-components/gltf-instance/gltf-2.0.js';
 
@@ -25,6 +25,8 @@ const $texture = Symbol('texture');
 const $transform = Symbol('transform');
 export const $materials = Symbol('materials');
 export const $usage = Symbol('usage');
+const $onUpdate = Symbol('onUpdate');
+const $activeVideo = Symbol('activeVideo');
 
 // Defines what a texture will be used for.
 export enum TextureUsage {
@@ -58,7 +60,8 @@ export class TextureInfo implements TextureInfoInterface {
   // Texture usage defines the how the texture is used (ie Normal, Emissive...
   // etc)
   [$usage]: TextureUsage;
-  onUpdate: () => void;
+  [$onUpdate]: () => void;
+  [$activeVideo] = false;
 
   constructor(
       onUpdate: () => void, usage: TextureUsage,
@@ -83,7 +86,7 @@ export class TextureInfo implements TextureInfoInterface {
           new Texture(onUpdate, threeTexture, gltfTexture, sampler, image);
     }
 
-    this.onUpdate = onUpdate;
+    this[$onUpdate] = onUpdate;
     this[$materials] = material;
     this[$usage] = usage;
   }
@@ -95,9 +98,36 @@ export class TextureInfo implements TextureInfoInterface {
   setTexture(texture: Texture|null): void {
     const threeTexture: ThreeTexture|null =
         texture != null ? texture.source[$threeTexture] : null;
-    let encoding: TextureEncoding = sRGBEncoding;
+
+    const oldTexture = this[$texture] as unknown as VideoTexture;
+    if (oldTexture != null && oldTexture.isVideoTexture) {
+      const element = oldTexture.image;
+      this[$activeVideo] = false;
+      if (element.requestVideoFrameCallback == null) {
+        element.removeEventListener('timeupdate', this[$onUpdate]);
+      }
+    }
+
     this[$texture] = texture;
 
+    if (threeTexture != null && (threeTexture as VideoTexture).isVideoTexture) {
+      const element = threeTexture.image;
+      this[$activeVideo] = true;
+      if (element.requestVideoFrameCallback != null) {
+        const update = () => {
+          if (!this[$activeVideo]) {
+            return;
+          }
+          this[$onUpdate]();
+          element.requestVideoFrameCallback(update);
+        };
+        element.requestVideoFrameCallback(update);
+      } else {
+        element.addEventListener('timeupdate', this[$onUpdate]);
+      }
+    }
+
+    let encoding: TextureEncoding = sRGBEncoding;
     if (this[$materials]) {
       for (const material of this[$materials]!) {
         switch (this[$usage]) {
@@ -133,6 +163,6 @@ export class TextureInfo implements TextureInfoInterface {
       threeTexture.repeat = this[$transform].scale;
       threeTexture.offset = this[$transform].offset;
     }
-    this.onUpdate();
+    this[$onUpdate]();
   }
 }

packages/model-viewer/src/features/scene-graph/texture-info.ts

@@ -24,9 +24,9 @@ import {StagingMixin} from './features/staging.js';
 import ModelViewerElementBase from './model-viewer-base.js';
 import {FocusVisiblePolyfillMixin} from './utilities/focus-visible.js';
 
-// Uncomment these lines to export PMREM textures in Glitch:
-// export {default as TextureUtils} from './three-components/TextureUtils';
-// export * from 'three';
+// Export these to allow lazy-loaded LottieLoader.js to find what it needs.
+// Requires an import map - "three": "path/to/model-viewer.min.js".
+export {CanvasTexture, FileLoader, Loader, NearestFilter} from 'three';
 
 export const ModelViewerElement = AnnotationMixin(SceneGraphMixin(StagingMixin(
     EnvironmentMixin(ControlsMixin(ARMixin(LoadingMixin(AnimationMixin(