feat: new TextDisplayer implementation to allow selecting subtitles via native API and controls (#8520)

Close #8519
Fixes #8475

Introduce `NativeTextDisplayer` as a replacement of
`SimpleTextDisplayer`. But keep them both work.
Is MSE mode, `NativeTextDisplayer` creates `<track>` elements for text
streams. And listens to change events on both ends to keep them in sync.
In SRC mode, `NativeTextDisplayer` would do nothing, the player uses
original TextTracks instead.

Advantages of `NativeTextDisplayer`:
- Allow text track selection using the browser built-in UI
- Allow text track manipulation using native APIs
- Avoid transferring and processing cues in SRC mode

---------

Co-authored-by: Álvaro Velad Galván <[email protected]>

Author: xxoo

AUTHORS

@@ -119,3 +119,4 @@ João Nabais <[email protected]>
 Koen Romers <[email protected]>
 Zhenghang Chen <[email protected]>
 Xperi <*@xperi.com>
+Shen Xiao <[email protected]>

CONTRIBUTORS

@@ -168,3 +168,4 @@ Zhenghang Chen <[email protected]>
 Ashley Manners <[email protected]>
 Bidisha Das <[email protected]>
 Chafroud Tarek <[email protected]>
+Shen Xiao <[email protected]>

build/types/core

@@ -64,6 +64,7 @@
 
 +../../lib/text/cue.js
 +../../lib/text/cue_region.js
++../../lib/text/native_text_displayer.js
 +../../lib/text/simple_text_displayer.js
 +../../lib/text/stub_text_displayer.js
 +../../lib/text/text_engine.js
@@ -131,4 +132,6 @@
 +../../third_party/closure-uri/uri.js
 +../../third_party/closure-uri/utils.js
 
++../../third_party/language-mapping-list/language-mapping-list.js
+
 +@lcevc

build/types/ui

@@ -1,6 +1,5 @@
 # UI library.
 
-+../../third_party/language-mapping-list/language-mapping-list.js
 +../../ui/ad_info.js
 +../../ui/ad_statistics_button.js
 +../../ui/audio_language_selection.js

demo/main.js

@@ -1410,7 +1410,7 @@ shakaDemo.Main = class {
       if (this.nativeControlsEnabled_) {
         this.controls_.setEnabledShakaControls(false);
         this.controls_.setEnabledNativeControls(true);
-        // This will force the player to use SimpleTextDisplayer.
+        // This will force the player to use NativeTextDisplayer.
         this.player_.setVideoContainer(null);
       } else {
         this.controls_.setEnabledShakaControls(true);

docs/tutorials/plugins.md

@@ -49,7 +49,7 @@ __Subtitle/caption displayers__
   - Use {@link player.configure} and set the `textDisplayFactory` field
   - Must implement the {@link shaka.extern.TextDisplayer} interface
   - Default TextDisplayer implementation:
-    {@linksource shaka.text.SimpleTextDisplayer}
+    {@linksource shaka.text.NativeTextDisplayer}
 
 __Networking plugins__
   - Selected by URI scheme (http, https, etc.)

docs/tutorials/text-displayer.md

@@ -5,12 +5,12 @@
 Shaka Player supports two implementations of {@link shaka.extern.TextDisplayer}
 which can be used to render & style content subtitles.
 
-#### SimpleTextDisplayer
+#### NativeTextDisplayer
 
-{@link shaka.text.SimpleTextDisplayer} which uses browser's native cue
-renderer. Shaka Player creates a custom text track attached to the video
-element and provides necessary data so video element can render it. This is
-the default displayer when shaka UI is **not** used.
+{@link shaka.text.NativeTextDisplayer} which uses browser's native cue
+renderer. Shaka Player creates corresponding text tracks for text streams on
+the video element and provides necessary data so video element can render it.
+This is the default displayer when shaka UI is **not** used.
 
 #### UITextDisplayer
 

lib/player.js

@@ -35,6 +35,7 @@ goog.require('shaka.media.TimeRangesUtils');
 goog.require('shaka.net.NetworkingEngine');
 goog.require('shaka.net.NetworkingUtils');
 goog.require('shaka.text.Cue');
+goog.require('shaka.text.NativeTextDisplayer');
 goog.require('shaka.text.SimpleTextDisplayer');
 goog.require('shaka.text.StubTextDisplayer');
 goog.require('shaka.text.TextEngine');
@@ -2575,15 +2576,21 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     this.loadEventManager_.listen(mediaElement, 'ended', updateStateHistory);
     this.loadEventManager_.listen(mediaElement, 'ratechange', onRateChange);
     if (mediaElement.remote) {
-      this.loadEventManager_.listen(mediaElement.remote, 'connect',
-          () => this.onTracksChanged_());
+      this.loadEventManager_.listen(mediaElement.remote, 'connect', () => {
+        if (this.streamingEngine_ &&
+            mediaElement.remote.state == 'connected') {
+          this.onTextChanged_();
+        }
+        this.onTracksChanged_();
+      });
       this.loadEventManager_.listen(mediaElement.remote, 'connecting',
           () => this.onTracksChanged_());
       this.loadEventManager_.listen(mediaElement.remote, 'disconnect',
           async () => {
             if (this.streamingEngine_ &&
                 mediaElement.remote.state == 'disconnected') {
               await this.streamingEngine_.resetMediaSource();
+              this.onTextChanged_();
             }
             this.onTracksChanged_();
           });
@@ -2606,6 +2613,13 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     }
 
     if (mediaElement.textTracks) {
+      const trackChange = () => {
+        if (this.loadMode_ === shaka.Player.LoadMode.SRC_EQUALS &&
+            this.textDisplayer_ instanceof shaka.text.NativeTextDisplayer) {
+          this.onTextChanged_();
+        }
+        this.onTracksChanged_();
+      };
       this.loadEventManager_.listen(
           mediaElement.textTracks, 'addtrack', (e) => {
             const trackEvent = /** @type {!TrackEvent} */(e);
@@ -2624,15 +2638,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
                   break;
 
                 default:
-                  this.onTracksChanged_();
+                  trackChange();
                   break;
               }
             }
           });
       this.loadEventManager_.listen(mediaElement.textTracks, 'removetrack',
-          () => this.onTracksChanged_());
+          trackChange);
       this.loadEventManager_.listen(mediaElement.textTracks, 'change',
-          () => this.onTracksChanged_());
+          trackChange);
 
       if (this.config_.streaming.crossBoundaryStrategy !==
         shaka.config.CrossBoundaryStrategy.KEEP) {
@@ -3121,55 +3135,43 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
 
     if (mediaElement.textTracks) {
       this.createTextDisplayer_();
-      const setShowingMode = () => {
-        const track = this.getFilteredTextTracks_()
-            .find((t) => t.mode !== 'disabled');
-        if (track) {
-          track.mode = 'showing';
-        }
-        const generatedTrack = this.getGeneratedTextTrack_();
-        if (generatedTrack) {
-          generatedTrack.mode = 'hidden';
-        }
-      };
-      const setHiddenMode = () => {
-        const track = this.getFilteredTextTracks_()
-            .find((t) => t.mode !== 'disabled');
-        if (track) {
-          track.mode = 'hidden';
-        }
-        const generatedTrack = this.getGeneratedTextTrack_();
-        const isTextVisible = this.textDisplayer_.isTextVisible();
-        if (generatedTrack && isTextVisible) {
-          generatedTrack.mode = 'showing';
+      const setMode = (showing) => {
+        if (!(this.textDisplayer_ instanceof shaka.text.NativeTextDisplayer)) {
+          const track = this.getFilteredTextTracks_()
+              .find((t) => t.mode !== 'disabled');
+          if (track) {
+            track.mode = showing ? 'showing' : 'hidden';
+          }
+          if (this.textDisplayer_ instanceof shaka.text.SimpleTextDisplayer) {
+            const generatedTrack = this.getGeneratedTextTrack_();
+            if (generatedTrack) {
+              generatedTrack.mode =
+                !showing && this.textDisplayer_.isTextVisible() ?
+                'showing' : 'hidden';
+            }
+          }
         }
       };
       this.loadEventManager_.listen(mediaElement, 'enterpictureinpicture',
-          () => setShowingMode());
+          () => setMode(true));
       this.loadEventManager_.listen(mediaElement, 'leavepictureinpicture',
-          () => setHiddenMode());
+          () => setMode(false));
       if (mediaElement.remote) {
         this.loadEventManager_.listen(mediaElement.remote, 'connect',
-            () => setHiddenMode());
+            () => setMode(false));
         this.loadEventManager_.listen(mediaElement.remote, 'connecting',
-            () => setHiddenMode());
+            () => setMode(false));
         this.loadEventManager_.listen(mediaElement.remote, 'disconnect',
-            () => setHiddenMode());
+            () => setMode(false));
       } else if ('webkitCurrentPlaybackTargetIsWireless' in mediaElement) {
         this.loadEventManager_.listen(mediaElement,
             'webkitcurrentplaybacktargetiswirelesschanged',
-            () => setHiddenMode());
+            () => setMode(false));
       }
       const video = /** @type {HTMLVideoElement} */(mediaElement);
       if (video.webkitSupportsFullscreen) {
         this.loadEventManager_.listen(video, 'webkitpresentationmodechanged',
-            () => {
-              if (video.webkitPresentationMode != 'inline') {
-                setShowingMode();
-              } else {
-                setHiddenMode();
-              }
-            });
+            () => setMode(video.webkitPresentationMode !== 'inline'));
       }
     }
     // Add all media element listeners.
@@ -3273,27 +3275,33 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
             this.textDisplayer_.setTextVisibility(true);
           }
 
-          if (textTracks.length) {
-            if (this.textDisplayer_.enableTextDisplayer) {
-              this.textDisplayer_.enableTextDisplayer();
-            } else {
-              shaka.Deprecate.deprecateFeature(5,
-                  'Text displayer w/ enableTextDisplayer',
-                  'Text displayer should have a "enableTextDisplayer" method!');
+          if (
+            !(this.textDisplayer_ instanceof shaka.text.NativeTextDisplayer)
+          ) {
+            if (textTracks.length) {
+              if (this.textDisplayer_.enableTextDisplayer) {
+                this.textDisplayer_.enableTextDisplayer();
+              } else {
+                shaka.Deprecate.deprecateFeature(
+                    5,
+                    'Text displayer w/ enableTextDisplayer',
+                    'Text displayer should have a "enableTextDisplayer" method',
+                );
+              }
             }
-          }
 
-          let enabledNativeTrack = false;
-          for (const track of textTracks) {
-            if (track.mode !== 'disabled') {
-              if (!enabledNativeTrack) {
-                this.enableNativeTrack_(track);
-                enabledNativeTrack = true;
-              } else {
-                track.mode = 'disabled';
-                shaka.log.alwaysWarn(
-                    'Found more than one enabled text track, disabling it',
-                    track);
+            let enabledNativeTrack = false;
+            for (const track of textTracks) {
+              if (track.mode !== 'disabled') {
+                if (!enabledNativeTrack) {
+                  this.enableNativeTrack_(track);
+                  enabledNativeTrack = true;
+                } else {
+                  track.mode = 'disabled';
+                  shaka.log.alwaysWarn(
+                      'Found more than one enabled text track, disabling it',
+                      track);
+                }
               }
             }
           }
@@ -5222,20 +5230,24 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    * @export
    */
   getTextTracks() {
-    if (this.manifest_ && !this.isRemotePlayback()) {
-      const currentTextStream = this.streamingEngine_ ?
+    if (this.manifest_) {
+      if (this.isRemotePlayback()) {
+        return [];
+      } else {
+        const currentTextStream = this.streamingEngine_ ?
           this.streamingEngine_.getCurrentTextStream() : null;
-      const tracks = [];
+        const tracks = [];
 
-      // Convert all selectable text streams to tracks.
-      for (const text of this.manifest_.textStreams) {
-        const track = shaka.util.StreamUtils.textStreamToTrack(text);
-        track.active = text == currentTextStream;
+        // Convert all selectable text streams to tracks.
+        for (const text of this.manifest_.textStreams) {
+          const track = shaka.util.StreamUtils.textStreamToTrack(text);
+          track.active = text == currentTextStream;
 
-        tracks.push(track);
-      }
+          tracks.push(track);
+        }
 
-      return tracks;
+        return tracks;
+      }
     } else if (this.video_ && this.video_.src && this.video_.textTracks) {
       const textTracks = this.getFilteredTextTracks_();
       const StreamUtils = shaka.util.StreamUtils;
@@ -5501,22 +5513,33 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     const selectSrcEqualsMode = () => {
       if (this.video_ && this.video_.textTracks) {
         const textTracks = this.getFilteredTextTracks_();
-        const oldTrack = textTracks.find((textTrack) =>
-          textTrack.mode !== 'disabled');
         const newTrack = textTracks.find((textTrack) =>
           shaka.util.StreamUtils.html5TrackId(textTrack) === track.id);
         if (!newTrack) {
           shaka.log.error('No track with id', track.id);
           return;
         }
-        if (oldTrack !== newTrack) {
-          if (oldTrack) {
-            oldTrack.mode = 'disabled';
-            this.loadEventManager_.unlisten(oldTrack, 'cuechange');
-            this.textDisplayer_.remove(0, Infinity);
+        if (this.textDisplayer_ instanceof shaka.text.NativeTextDisplayer) {
+          for (const texTrack of textTracks) {
+            const mode = texTrack === newTrack ?
+                this.isTextVisible_ ? 'showing' : 'hidden' :
+                'disabled';
+            if (texTrack.mode !== mode) {
+              texTrack.mode = mode;
+            }
           }
-          if (newTrack) {
-            this.enableNativeTrack_(newTrack);
+        } else {
+          const oldTrack = textTracks.find((textTrack) =>
+            textTrack.mode !== 'disabled');
+          if (oldTrack !== newTrack) {
+            if (oldTrack) {
+              oldTrack.mode = 'disabled';
+              this.loadEventManager_.unlisten(oldTrack, 'cuechange');
+              this.textDisplayer_.remove(0, Infinity);
+            }
+            if (newTrack) {
+              this.enableNativeTrack_(newTrack);
+            }
           }
         }
         this.onTextChanged_();
@@ -7388,17 +7411,16 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     // TextDisplay factory must capture a reference to "this".
     config.textDisplayFactory = () => {
       // On iOS where the Fullscreen API is not available we prefer
-      // SimpleTextDisplayer because it works with the Fullscreen API of the
+      // NativeTextDisplayer because it works with the Fullscreen API of the
       // video element itself.
       const Platform = shaka.util.Platform;
       if (this.videoContainer_ &&
           (!Platform.isApple() || document.fullscreenEnabled)) {
         return new shaka.text.UITextDisplayer(
             this.video_, this.videoContainer_);
       } else {
-        if ('addTextTrack' in this.video_) {
-          return new shaka.text.SimpleTextDisplayer(
-              this.video_, shaka.Player.TextTrackLabel);
+        if ('track' in document.createElement('track')) {
+          return new shaka.text.NativeTextDisplayer(this);
         } else {
           shaka.log.warning('Text tracks are not supported by the ' +
                             'browser, disabling.');