feat(VideoTexture): onVideoFrame (#2238)

Author: abernier

docs/loaders/video-texture-use-video-texture.mdx

@@ -24,16 +24,18 @@ export function useVideoTexture(
   {
     unsuspend = 'loadedmetadata',
     start = true,
-    hls: hlsConfig = {},
+    hls = {},
     crossOrigin = 'anonymous',
     muted = true,
     loop = true,
     playsInline = true,
+    onVideoFrame,
     ...videoProps
   }: {
     unsuspend?: keyof HTMLVideoElementEventMap
     start?: boolean
     hls?: Parameters<typeof getHls>[0]
+    onVideoFrame: VideoFrameRequestCallback
   } & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
 )
 ```
@@ -84,3 +86,36 @@ const texture = useVideoTexture('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3
   hls: { abrEwmaFastLive: 1.0, abrEwmaSlowLive: 3.0, enableWorker: true },
 })
 ```
+
+## `requestVideoFrameCallback` (rVFC)
+
+`useVideoTexture` supports [`requestVideoFrameCallback`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement/requestVideoFrameCallback):
+
+```jsx
+useVideoTexture(src, {
+  onVideoFrame: (now, metadata) => {}
+})
+```
+
+## `<VideoTexture>` Component
+
+```tsx
+export type VideoTextureProps = {
+  children?: (texture: THREE.VideoTexture) => React.ReactNode
+  src: UseVideoTextureParams[0]
+} & UseVideoTextureParams[1]
+```
+
+You can access the texture via children's render prop:
+
+```jsx
+<VideoTexture src="/video.mp4">
+  {(texture) => <meshBasicMaterial map={texture} />}
+```
+
+or exposed via `ref`:
+
+```jsx
+const textureRef = useRef()
+<VideoTexture ref={textureRef} src="/video.mp4" />
+```

src/core/VideoTexture.tsx

@@ -1,6 +1,7 @@
+/* eslint react-hooks/exhaustive-deps: 1 */
 import * as React from 'react'
 import * as THREE from 'three'
-import { useEffect, useRef } from 'react'
+import { forwardRef, useEffect, useImperativeHandle, useRef } from 'react'
 import { useThree } from '@react-three/fiber'
 import { suspend } from 'suspend-react'
 import { type default as Hls, Events } from 'hls.js'
@@ -31,11 +32,22 @@ export function useVideoTexture(
     muted = true,
     loop = true,
     playsInline = true,
+    onVideoFrame,
     ...videoProps
   }: {
+    /** Event name that will unsuspend the video */
     unsuspend?: keyof HTMLVideoElementEventMap
+    /** Auto start the video once unsuspended */
     start?: boolean
+    /** HLS config */
     hls?: Parameters<typeof getHls>[0]
+    /**
+     * request Video Frame Callback (rVFC)
+     *
+     * @see https://web.dev/requestvideoframecallback-rvfc/
+     * @see https://www.remotion.dev/docs/video-manipulation
+     * */
+    onVideoFrame?: VideoFrameRequestCallback
   } & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
 ) {
   const gl = useThree((state) => state.gl)
@@ -81,6 +93,9 @@ export function useVideoTexture(
     [srcOrSrcObject]
   )
 
+  const video = texture.source.data as HTMLVideoElement
+  useVideoFrame(video, onVideoFrame)
+
   useEffect(() => {
     start && texture.image.play()
 
@@ -96,22 +111,45 @@ export function useVideoTexture(
 }
 
 //
+// VideoTexture
+//
+
+type UseVideoTextureParams = Parameters<typeof useVideoTexture>
+type VideoTexture = ReturnType<typeof useVideoTexture>
+
+export type VideoTextureProps = {
+  children?: (texture: VideoTexture) => React.ReactNode
+  src: UseVideoTextureParams[0]
+} & UseVideoTextureParams[1]
+
+export const VideoTexture = /* @__PURE__ */ forwardRef<VideoTexture, VideoTextureProps>(
+  ({ children, src, ...config }, fref) => {
+    const texture = useVideoTexture(src, config)
 
-type UseVideoTexture = Parameters<typeof useVideoTexture>
+    useEffect(() => {
+      return () => void texture.dispose()
+    }, [texture])
 
-export const VideoTexture = ({
-  children,
-  src,
-  ...config
-}: {
-  children?: (texture: ReturnType<typeof useVideoTexture>) => React.ReactNode
-  src: UseVideoTexture[0]
-} & UseVideoTexture[1]) => {
-  const ret = useVideoTexture(src, config)
+    useImperativeHandle(fref, () => texture, [texture]) // expose texture through ref
 
+    return <>{children?.(texture)}</>
+  }
+)
+
+// rVFC hook
+
+const useVideoFrame = (video: HTMLVideoElement, f?: VideoFrameRequestCallback) => {
   useEffect(() => {
-    return () => void ret.dispose()
-  }, [ret])
+    if (!f) return
+    if (!video.requestVideoFrameCallback) return
+
+    let handle: ReturnType<(typeof video)['requestVideoFrameCallback']>
+    const callback: VideoFrameRequestCallback = (...args) => {
+      f(...args)
+      handle = video.requestVideoFrameCallback(callback)
+    }
+    video.requestVideoFrameCallback(callback)
 
-  return <>{children?.(ret)}</>
+    return () => video.cancelVideoFrameCallback(handle)
+  }, [video, f])
 }