feat(HLS): Add I-Frame playlist support (#7230)

Author: avelad

README.md

@@ -140,7 +140,7 @@ HLS features supported:
  - SAMPLE-AES and SAMPLE-AES-CTR (identity) support on browsers with ClearKey support
  - Key rotation
  - Raw AAC, MP3, AC-3 and EC-3 (without an MP4 container)
- - I-frame-only playlists with mjpg codec for thumbnails
+ - I-frame-only playlists (for trick play and thumbnails)
  - #EXT-X-IMAGE-STREAM-INF for thumbnails
  - Interstitials
  - Container change during the playback (eg: MP4 to TS, or AAC to TS)

demo/common/assets.js

@@ -1385,6 +1385,7 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.HLS)
       .addFeature(shakaAssets.Feature.MP4)
       .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.TRICK_MODE)
       .addFeature(shakaAssets.Feature.OFFLINE)
       .addFeature(shakaAssets.Feature.THUMBNAILS),
   new ShakaDemoAssetInfo(
@@ -1461,6 +1462,7 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.HLS)
       .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
       .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.TRICK_MODE)
       .addFeature(shakaAssets.Feature.OFFLINE)
       .addFeature(shakaAssets.Feature.LCEVC)
       .setExtraConfig({
@@ -1479,6 +1481,7 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.HLS)
       .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
       .addFeature(shakaAssets.Feature.MP2TS)
+      .addFeature(shakaAssets.Feature.TRICK_MODE)
       .addFeature(shakaAssets.Feature.OFFLINE)
       .addFeature(shakaAssets.Feature.LCEVC)
       .setExtraConfig({

lib/dash/dash_parser.js

@@ -35,6 +35,7 @@ goog.require('shaka.util.ObjectUtils');
 goog.require('shaka.util.OperationManager');
 goog.require('shaka.util.PeriodCombiner');
 goog.require('shaka.util.PlayerConfiguration');
+goog.require('shaka.util.StreamUtils');
 goog.require('shaka.util.StringUtils');
 goog.require('shaka.util.Timer');
 goog.require('shaka.util.TXml');
@@ -1596,22 +1597,8 @@ shaka.dash.DashParser = class {
       for (const normalSet of normalAdaptationSets) {
         if (targetIds.includes(normalSet.id)) {
           for (const stream of normalSet.streams) {
-            const validStreams = trickModeSet.streams.filter((trickStream) =>
-              shaka.util.MimeUtils.getNormalizedCodec(stream.codecs) ==
-              shaka.util.MimeUtils.getNormalizedCodec(trickStream.codecs))
-                .sort((a, b) => {
-                  return a.bandwidth - b.bandwidth;
-                });
-            stream.trickModeVideo = validStreams[0];
-            if (validStreams.length <= 1) {
-              continue;
-            }
-            const sameResolutionStream = validStreams.find((trickStream) =>
-              stream.width == trickStream.width &&
-              stream.height == trickStream.height);
-            if (sameResolutionStream) {
-              stream.trickModeVideo = sameResolutionStream;
-            }
+            shaka.util.StreamUtils.setBetterIFrameStream(
+                stream, trickModeSet.streams);
           }
         }
       }

lib/hls/hls_parser.js

@@ -43,6 +43,7 @@ goog.require('shaka.util.Timer');
 goog.require('shaka.util.TsParser');
 goog.require('shaka.util.TXml');
 goog.require('shaka.util.Platform');
+goog.require('shaka.util.StreamUtils');
 goog.require('shaka.util.Uint8ArrayUtils');
 goog.requireType('shaka.hls.Segment');
 
@@ -938,9 +939,10 @@ shaka.hls.HlsParser = class {
       this.parseCodecs_(variantTags);
 
       this.parseClosedCaptions_(mediaTags);
+      const iFrameStreams = this.parseIFrames_(iFrameTags);
       variants = await this.createVariantsForTags_(
           variantTags, sessionKeyTags, mediaTags, getUris,
-          this.globalVariables_);
+          this.globalVariables_, iFrameStreams);
       textStreams = this.parseTexts_(mediaTags);
       imageStreams = await this.parseImages_(imageTags, iFrameTags);
     }
@@ -1458,7 +1460,8 @@ shaka.hls.HlsParser = class {
       }
       try {
         const streamInfo = this.createStreamInfoFromIframeTag_(tag);
-        if (streamInfo.stream.codecs !== 'mjpg') {
+        const ContentType = shaka.util.ManifestParserUtils.ContentType;
+        if (streamInfo.stream.type !== ContentType.IMAGE) {
           return null;
         }
         return streamInfo.stream;
@@ -1503,19 +1506,40 @@ shaka.hls.HlsParser = class {
     }
   }
 
+  /**
+   * @param {!Array.<!shaka.hls.Tag>} iFrameTags from the playlist.
+   * @return {!Array.<!shaka.extern.Stream>}
+   * @private
+   */
+  parseIFrames_(iFrameTags) {
+    // Create iFrame stream for each iFrame tag.
+    const iFrameStreams = iFrameTags.map((tag) => {
+      const streamInfo = this.createStreamInfoFromIframeTag_(tag);
+      const ContentType = shaka.util.ManifestParserUtils.ContentType;
+      if (streamInfo.stream.type !== ContentType.VIDEO) {
+        return null;
+      }
+      return streamInfo.stream;
+    });
+
+    // Filter mjpg iFrames
+    return iFrameStreams.filter((s) => s);
+  }
+
   /**
    * @param {!Array.<!shaka.hls.Tag>} tags Variant tags from the playlist.
    * @param {!Array.<!shaka.hls.Tag>} sessionKeyTags EXT-X-SESSION-KEY tags
    * from the playlist.
    * @param {!Array.<!shaka.hls.Tag>} mediaTags EXT-X-MEDIA tags from the
    * playlist.
    * @param {function():!Array.<string>} getUris
-   * @param {?Map.<string, string>=} variables
+   * @param {?Map.<string, string>} variables
+   * @param {!Array.<!shaka.extern.Stream>} iFrameStreams
    * @return {!Promise.<!Array.<!shaka.extern.Variant>>}
    * @private
    */
   async createVariantsForTags_(tags, sessionKeyTags, mediaTags, getUris,
-      variables) {
+      variables, iFrameStreams) {
     // EXT-X-SESSION-KEY processing
     const drmInfos = [];
     const keyIds = new Set();
@@ -1625,7 +1649,8 @@ shaka.hls.HlsParser = class {
           videoRange,
           videoLayout,
           drmInfos,
-          keyIds));
+          keyIds,
+          iFrameStreams));
     }
     return allVariants.filter((variant) => variant != null);
   }
@@ -1960,12 +1985,13 @@ shaka.hls.HlsParser = class {
    * @param {?string} videoLayout
    * @param {!Array.<shaka.extern.DrmInfo>} drmInfos
    * @param {!Set.<string>} keyIds
+   * @param {!Array.<!shaka.extern.Stream>} iFrameStreams
    * @return {!Array.<!shaka.extern.Variant>}
    * @private
    */
   createVariants_(
       audioInfos, videoInfos, bandwidth, width, height, frameRate, videoRange,
-      videoLayout, drmInfos, keyIds) {
+      videoLayout, drmInfos, keyIds, iFrameStreams) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     const DrmUtils = shaka.util.DrmUtils;
 
@@ -2000,6 +2026,8 @@ shaka.hls.HlsParser = class {
         if (videoStream) {
           videoStream.drmInfos = drmInfos;
           videoStream.keyIds = keyIds;
+          shaka.util.StreamUtils.setBetterIFrameStream(
+              videoStream, iFrameStreams);
         }
         if (videoStream && !audioStream) {
           videoStream.bandwidth = bandwidth;
@@ -2267,11 +2295,15 @@ shaka.hls.HlsParser = class {
     goog.asserts.assert(tag.name == 'EXT-X-I-FRAME-STREAM-INF',
         'Should only be called on iframe tags!');
     /** @type {string} */
-    const type = shaka.util.ManifestParserUtils.ContentType.IMAGE;
+    let type = shaka.util.ManifestParserUtils.ContentType.VIDEO;
 
     const verbatimIFramePlaylistUri = tag.getRequiredAttrValue('URI');
     const codecs = tag.getAttributeValue('CODECS') || '';
 
+    if (codecs == 'mjpg') {
+      type = shaka.util.ManifestParserUtils.ContentType.IMAGE;
+    }
+
     // Check if the stream has already been created as part of another Variant
     // and return it if it has.
     if (this.uriToStreamInfosMap_.has(verbatimIFramePlaylistUri)) {

lib/util/stream_utils.js

@@ -1884,6 +1884,37 @@ shaka.util.StreamUtils = class {
   }
 
 
+  /**
+   * Set the best iframe stream to the original stream.
+   *
+   * @param {!shaka.extern.Stream} stream
+   * @param {!Array.<!shaka.extern.Stream>} iFrameStreams
+   */
+  static setBetterIFrameStream(stream, iFrameStreams) {
+    if (!iFrameStreams.length) {
+      return;
+    }
+    const validStreams = iFrameStreams.filter((iFrameStream) =>
+      shaka.util.MimeUtils.getNormalizedCodec(stream.codecs) ==
+      shaka.util.MimeUtils.getNormalizedCodec(iFrameStream.codecs))
+        .sort((a, b) => {
+          if (!a.bandwidth || !b.bandwidth || a.bandwidth == b.bandwidth) {
+            return (a.width || 0) - (b.width || 0);
+          }
+          return a.bandwidth - b.bandwidth;
+        });
+    stream.trickModeVideo = validStreams[0];
+    if (validStreams.length > 1) {
+      const sameResolutionStream = validStreams.find((iFrameStream) =>
+        stream.width == iFrameStream.width &&
+        stream.height == iFrameStream.height);
+      if (sameResolutionStream) {
+        stream.trickModeVideo = sameResolutionStream;
+      }
+    }
+  }
+
+
   /**
    * Returns a string of a variant, with the attribute values of its audio
    * and/or video streams for log printing.

roadmap.md

@@ -22,6 +22,7 @@ v5.0 - 2024 Q4
 
 v4.11 - 2024 Q3
  - HLS: EXT-X-START support
+ - HLS: EXT-X-I-FRAME-STREAM-INF support
  - Basic support of VAST and VMAP without IMA (playback without tracking)
  - DASH: DVB Fonts
  - TTML: IMSC1 (CMAF) image subtitle

test/hls/hls_parser_unit.js

@@ -2136,6 +2136,88 @@ describe('HlsParser', () => {
     expect(thirdThumbnailReference).not.toBe(null);
   });
 
+  it('supports EXT-X-I-FRAME-STREAM-INF for trick play', async () => {
+    const master = [
+      '#EXTM3U\n',
+      '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="sub1",LANGUAGE="eng",',
+      'URI="text"\n',
+      '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="eng",',
+      'CHANNELS="2",URI="audio"\n',
+      '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,mp4a",',
+      'RESOLUTION=960x540,FRAME-RATE=60,AUDIO="aud1",SUBTITLES="sub1"\n',
+      'video\n',
+      '#EXT-X-I-FRAME-STREAM-INF:RESOLUTION=960x540,CODECS="avc1",',
+      'URI="iframe"\n',
+      '#EXT-X-I-FRAME-STREAM-INF:RESOLUTION=240×135,CODECS="avc1",',
+      'URI="iframeAvc"\n',
+    ].join('');
+
+    const video = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+    ].join('');
+
+    const audio = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+    ].join('');
+
+    const text = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXTINF:5,\n',
+      '#EXT-X-BYTERANGE:121090@616\n',
+      'main.vtt',
+    ].join('');
+
+    const iframe = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+      '#EXTINF:5,\n',
+      'main.mp4\n',
+    ].join('');
+
+    fakeNetEngine
+        .setResponseText('test:/master', master)
+        .setResponseText('test:/audio', audio)
+        .setResponseText('test:/video', video)
+        .setResponseText('test:/text', text)
+        .setResponseText('test:/iframe', iframe)
+        .setResponseText('test:/main.vtt', vttText)
+        .setResponseValue('test:/init.mp4', initSegmentData)
+        .setResponseValue('test:/main.mp4', segmentData);
+
+    const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
+
+    expect(actual.textStreams.length).toBe(1);
+    expect(actual.variants.length).toBe(1);
+
+    const trickModeVideo = actual.variants[0].video.trickModeVideo;
+    expect(trickModeVideo).toBeDefined();
+    expect(trickModeVideo.width).toBe(960);
+    expect(trickModeVideo.height).toBe(540);
+  });
+
   it('parse EXT-X-GAP', async () => {
     const master = [
       '#EXTM3U\n',