fix: Mixed clear/encrypted playback on Safari MSE (#8354)

Fixes #8335

---------

Co-authored-by: Álvaro Velad Galván <[email protected]>
Co-authored-by: Gary Katsevman <[email protected]>

Author: 3 people

karma.conf.js

@@ -254,6 +254,7 @@ module.exports = (config) => {
       {pattern: 'test/**/*.js', included: false},
       {pattern: 'test/test/assets/*', included: false},
       {pattern: 'test/test/assets/clear-encrypted/*', included: false},
+      {pattern: 'test/test/assets/clear-encrypted-hls/*', included: false},
       {pattern: 'test/test/assets/dash-multi-codec/*', included: false},
       {pattern: 'test/test/assets/dash-multi-codec-ec3/*', included: false},
       {pattern: 'test/test/assets/3675/*', included: false},

lib/media/content_workarounds.js

@@ -10,6 +10,7 @@ goog.require('goog.asserts');
 goog.require('shaka.log');
 goog.require('shaka.util.BufferUtils');
 goog.require('shaka.util.Error');
+goog.require('shaka.util.Mp4BoxParsers');
 goog.require('shaka.util.Mp4Generator');
 goog.require('shaka.util.Mp4Parser');
 goog.require('shaka.util.Platform');
@@ -149,6 +150,111 @@ shaka.media.ContentWorkarounds = class {
     return modifiedInitSegment;
   }
 
+  /**
+   * @param {!BufferSource} mediaSegmentBuffer
+   * @return {!Uint8Array}
+   */
+  static fakeMediaEncryption(mediaSegmentBuffer) {
+    const mediaSegment = shaka.util.BufferUtils.toUint8(mediaSegmentBuffer);
+    const mdatBoxes = [];
+    new shaka.util.Mp4Parser()
+        .box('mdat', (box) => {
+          mdatBoxes.push(box);
+        })
+        .parse(mediaSegment);
+
+    const newSegmentChunks = [];
+    for (let i = 0; i < mdatBoxes.length; i++) {
+      const prevMdat = mdatBoxes[i - 1];
+      const currMdat = mdatBoxes[i];
+      const chunkStart = prevMdat ? prevMdat.start + prevMdat.size : 0;
+      const chunkEnd = currMdat.start + currMdat.size;
+      const chunk = mediaSegment.subarray(chunkStart, chunkEnd);
+      newSegmentChunks.push(
+          shaka.media.ContentWorkarounds.fakeMediaEncryptionInChunk_(chunk));
+    }
+    return shaka.util.Uint8ArrayUtils.concat(...newSegmentChunks);
+  }
+
+  /**
+   * @param {!Uint8Array} chunk
+   * @return {!Uint8Array}
+   * @private
+   */
+  static fakeMediaEncryptionInChunk_(chunk) {
+    // Which track from stsd we want to use, 1-based.
+    const desiredSampleDescriptionIndex = 2;
+    let tfhdBox;
+    let trunBox;
+    let parsedTfhd;
+    let parsedTrun;
+    const ancestorBoxes = [];
+    const onSimpleAncestorBox = (box) => {
+      ancestorBoxes.push(box);
+      shaka.util.Mp4Parser.children(box);
+    };
+    const onTfhdBox = (box) => {
+      tfhdBox = box;
+      parsedTfhd = shaka.util.Mp4BoxParsers.parseTFHD(box.reader, box.flags);
+    };
+    const onTrunBox = (box) => {
+      trunBox = box;
+      parsedTrun = shaka.util.Mp4BoxParsers.parseTRUN(box.reader, box.version,
+          box.flags);
+    };
+    new shaka.util.Mp4Parser()
+        .box('moof', onSimpleAncestorBox)
+        .box('traf', onSimpleAncestorBox)
+        .fullBox('tfhd', onTfhdBox)
+        .fullBox('trun', onTrunBox)
+        .parse(chunk);
+    if (parsedTfhd && parsedTfhd.sampleDescriptionIndex !==
+        desiredSampleDescriptionIndex) {
+      const sdiPosition = tfhdBox.start +
+          shaka.util.Mp4Parser.headerSize(tfhdBox) +
+          4 + // track_id
+          (parsedTfhd.baseDataOffset !== null ? 8 : 0);
+      const dataview = shaka.util.BufferUtils.toDataView(chunk);
+      if (parsedTfhd.sampleDescriptionIndex !== null) {
+        dataview.setUint32(sdiPosition, desiredSampleDescriptionIndex);
+      } else {
+        const sdiSize = 4; // uint32
+
+        // first, update size & flags of tfhd
+        shaka.media.ContentWorkarounds.updateBoxSize_(chunk,
+            tfhdBox.start, tfhdBox.size + sdiSize);
+        const versionAndFlags = dataview.getUint32(tfhdBox.start + 8);
+        dataview.setUint32(tfhdBox.start + 8, versionAndFlags | 0x000002);
+
+        // second, update trun
+        if (parsedTrun && parsedTrun.dataOffset !== null) {
+          const newDataOffset = parsedTrun.dataOffset + sdiSize;
+          const dataOffsetPosition = trunBox.start +
+              shaka.util.Mp4Parser.headerSize(trunBox) +
+              4; // sample count
+          dataview.setInt32(dataOffsetPosition, newDataOffset);
+        }
+        const beforeSdi = chunk.subarray(0, sdiPosition);
+        const afterSdi = chunk.subarray(sdiPosition);
+        chunk = new Uint8Array(chunk.byteLength + sdiSize);
+        chunk.set(beforeSdi);
+
+        const bytes = [];
+        for (let byte = sdiSize - 1; byte >= 0; byte--) {
+          bytes.push((desiredSampleDescriptionIndex >> (8 * byte)) & 0xff);
+        }
+        chunk.set(new Uint8Array(bytes), sdiPosition);
+        chunk.set(afterSdi, sdiPosition + sdiSize);
+        for (const box of ancestorBoxes) {
+          shaka.media.ContentWorkarounds.updateBoxSize_(chunk, box.start,
+              box.size + sdiSize);
+        }
+      }
+    }
+
+    return chunk;
+  }
+
   /**
    * Insert an encryption metadata box ("encv" or "enca" box) into the MP4 init
    * segment, based on the source box ("mp4a", "avc1", etc).  Returns a new
@@ -178,8 +284,8 @@ shaka.media.ContentWorkarounds = class {
     // For other platforms, we cut and insert at the end of the source box. It's
     // not clear why this is necessary on Xbox One, but it seems to be evidence
     // of another bug in the firmware implementation of MediaSource & EME.
-    const cutPoint =
-      (shaka.util.Platform.isXboxOne() || shaka.util.Platform.isEdge()) ?
+    const cutPoint = (shaka.util.Platform.isApple() ||
+      shaka.util.Platform.isXboxOne() || shaka.util.Platform.isEdge()) ?
         sourceBox.start :
         sourceBox.start + sourceBox.size;
 

lib/media/media_source_engine.js

@@ -1320,8 +1320,7 @@ shaka.media.MediaSourceEngine = class {
     }
 
     data = this.workAroundBrokenPlatforms_(
-        stream, data, reference ? reference.startTime : null, contentType,
-        reference ? reference.getUris()[0] : null);
+        stream, data, reference, contentType);
 
     if (reference && this.sequenceMode_ && contentType != ContentType.TEXT) {
       // In sequence mode, for non-text streams, if we just cleared the buffer
@@ -2141,44 +2140,54 @@ shaka.media.MediaSourceEngine = class {
    *
    * @param {shaka.extern.Stream} stream
    * @param {!BufferSource} segment
-   * @param {?number} startTime
+   * @param {?shaka.media.SegmentReference} reference
    * @param {shaka.util.ManifestParserUtils.ContentType} contentType
-   * @param {?string} uri
    * @return {!BufferSource}
    * @private
    */
-  workAroundBrokenPlatforms_(stream, segment, startTime, contentType, uri) {
+  workAroundBrokenPlatforms_(stream, segment, reference, contentType) {
     const Platform = shaka.util.Platform;
 
-    const isInitSegment = startTime == null;
+    const isMp4 = shaka.util.MimeUtils.getContainerType(
+        this.sourceBufferTypes_.get(contentType)) == 'mp4';
+    if (!isMp4) {
+      return segment;
+    }
+
+    const isInitSegment = reference === null;
     const encryptionExpected = this.expectedEncryption_.get(contentType);
     const keySystem = this.playerInterface_.getKeySystem();
+    let isEncrypted = false;
+    if (reference && reference.initSegmentReference) {
+      isEncrypted = reference.initSegmentReference.encrypted;
+    }
+    const uri = reference ? reference.getUris()[0] : null;
 
     // If:
     //   1. the configuration tells to insert fake encryption,
-    //   2. and this is an init segment,
+    //   2. and this is an init segment or media segment,
     //   3. and encryption is expected,
-    //   4. and the platform requires encryption in all init segments,
-    //   5. and the content is MP4 (mimeType == "video/mp4" or "audio/mp4"),
+    //   4. and the platform requires encryption in all init or media segments
+    //      of current content type,
     // then insert fake encryption metadata for init segments that lack it.
     // The MP4 requirement is because we can currently only do this
     // transformation on MP4 containers.
     // See: https://github.com/shaka-project/shaka-player/issues/2759
-    if (this.config_.insertFakeEncryptionInInit &&
-        isInitSegment &&
-        encryptionExpected &&
-        Platform.requiresEncryptionInfoInAllInitSegments(keySystem) &&
-        shaka.util.MimeUtils.getContainerType(
-            this.sourceBufferTypes_.get(contentType)) == 'mp4') {
-      shaka.log.debug('Forcing fake encryption information in init segment.');
-      segment =
-          shaka.media.ContentWorkarounds.fakeEncryption(stream, segment, uri);
-    }
-
-    if (isInitSegment &&
-        Platform.requiresEC3InitSegments() &&
-        shaka.util.MimeUtils.getContainerType(
-            this.sourceBufferTypes_.get(contentType)) == 'mp4') {
+    if (this.config_.insertFakeEncryptionInInit && encryptionExpected &&
+        Platform.requiresEncryptionInfoInAllInitSegments(keySystem,
+            contentType)) {
+      if (isInitSegment) {
+        shaka.log.debug('Forcing fake encryption information in init segment.');
+        segment =
+            shaka.media.ContentWorkarounds.fakeEncryption(stream, segment, uri);
+      } else if (!isEncrypted && Platform.requiresTfhdFix(contentType)) {
+        shaka.log.debug(
+            'Forcing fake encryption information in media segment.');
+        segment = shaka.media.ContentWorkarounds.fakeMediaEncryption(segment);
+      }
+    }
+
+    if (isInitSegment && Platform.requiresEC3InitSegments()) {
       shaka.log.debug('Forcing fake EC-3 information in init segment.');
       segment = shaka.media.ContentWorkarounds.fakeEC3(segment);
     }

lib/util/mp4_box_parsers.js

@@ -21,17 +21,18 @@ shaka.util.Mp4BoxParsers = class {
     let defaultSampleDuration = null;
     let defaultSampleSize = null;
     let baseDataOffset = null;
+    let sampleDescriptionIndex = null;
 
     const trackId = reader.readUint32(); // Read "track_ID"
 
-    // Skip "base_data_offset" if present.
+    // Read "base_data_offset" if present.
     if (flags & 0x000001) {
       baseDataOffset = reader.readUint64();
     }
 
-    // Skip "sample_description_index" if present.
+    // Read "sample_description_index" if present.
     if (flags & 0x000002) {
-      reader.skip(4);
+      sampleDescriptionIndex = reader.readUint32();
     }
 
     // Read "default_sample_duration" if present.
@@ -49,6 +50,7 @@ shaka.util.Mp4BoxParsers = class {
       defaultSampleDuration,
       defaultSampleSize,
       baseDataOffset,
+      sampleDescriptionIndex,
     };
   }
 
@@ -742,7 +744,8 @@ shaka.util.Mp4BoxParsers = class {
  *    trackId: number,
  *    defaultSampleDuration: ?number,
  *    defaultSampleSize: ?number,
- *    baseDataOffset: ?number
+ *    baseDataOffset: ?number,
+ *    sampleDescriptionIndex: ?number
  * }}
  *
  * @property {number} trackId
@@ -756,7 +759,8 @@ shaka.util.Mp4BoxParsers = class {
  *   size in the Track Extends Box for this fragment
  * @property {?number} baseDataOffset
  *   If specified via flags, this indicate the base data offset
- *
+ * @property {?number} sampleDescriptionIndex
+ *   If specified via flags, this indicate the sample description index
  * @exportDoc
  */
 shaka.util.ParsedTFHDBox;

lib/util/platform.js

@@ -621,16 +621,26 @@ shaka.util.Platform = class {
    * initialization segments.
    *
    * @param {?string} keySystem
+   * @param {string} contentType
    * @return {boolean}
    * @see https://github.com/shaka-project/shaka-player/issues/2759
    */
-  static requiresEncryptionInfoInAllInitSegments(keySystem) {
+  static requiresEncryptionInfoInAllInitSegments(keySystem, contentType) {
     const Platform = shaka.util.Platform;
     const isPlayReady = shaka.drm.DrmUtils.isPlayReadyKeySystem(keySystem);
-    return Platform.isTizen() || Platform.isXboxOne() || Platform.isOrange() ||
+    return (Platform.isApple() && contentType === 'audio') ||
+      Platform.isTizen() || Platform.isXboxOne() || Platform.isOrange() ||
       (Platform.isEdge() && Platform.isWindows() && isPlayReady);
   }
 
+  /**
+   * @param {string} contentType
+   * @return {boolean}
+   */
+  static requiresTfhdFix(contentType) {
+    return shaka.util.Platform.isApple() && contentType === 'audio';
+  }
+
   /**
    * Returns true if the platform requires AC-3 signalling in init
    * segments to be replaced with EC-3 signalling.

lib/util/player_configuration.js

@@ -311,11 +311,6 @@ shaka.util.PlayerConfiguration = class {
           shaka.config.CrossBoundaryStrategy.RESET_TO_ENCRYPTED;
     }
 
-    if (shaka.util.Platform.isApple()) {
-      streaming.crossBoundaryStrategy =
-          shaka.config.CrossBoundaryStrategy.RESET_ON_ENCRYPTION_CHANGE;
-    }
-
     const networking = {
       forceHTTP: false,
       forceHTTPS: false,

test/media/content_workarounds_integration.js

@@ -82,7 +82,28 @@ describe('ContentWorkarounds', () => {
     await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 5, 30);
   });
 
-  for (const keySystem of ['com.widevine.alpha', 'com.microsoft.playready']) {
+  const keySystemsConfigs = new Map()
+      .set('com.widevine.alpha', {
+        servers: {
+          'com.widevine.alpha': 'https://cwip-shaka-proxy.appspot.com/no_auth',
+        },
+      })
+      .set('com.microsoft.playready', {
+        servers: {
+          'com.microsoft.playready': 'http://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(kid:51745386-2d42-56fd-8bad-4f58422004d7,contentkey:UXRThi1CVv2LrU9YQiAE1w==),(kid:26470f42-96d4-5d04-a9ba-bb442e169800,contentkey:JkcPQpbUXQSpurtELhaYAA==)',
+        },
+      })
+      .set('com.apple.fps', {
+        servers: {
+          'com.apple.fps': 'https://fps.ezdrm.com/api/licenses/b99ed9e5-c641-49d1-bfa8-43692b686ddb',
+        },
+        advanced: {
+          'com.apple.fps': {
+            serverCertificate: null, // empty now, fulfilled during actual test
+          },
+        },
+      });
+  for (const [keySystem, drmConfig] of keySystemsConfigs) {
     drmIt(`plays mixed clear encrypted content with ${keySystem}`, async () => {
       if (!shakaSupport.drm[keySystem]) {
         pending('Needed DRM is not supported on this platform');
@@ -91,30 +112,37 @@ describe('ContentWorkarounds', () => {
         pending('Tizen 3 currently does not support mixed clear ' +
             'encrypted content');
       }
+      if (keySystem === 'com.apple.fps' && getClientArg('runningInVM')) {
+        pending('FairPlay is not supported in a VM');
+      }
       const keyStatusSpy = jasmine.createSpy('onKeyStatus');
       eventManager.listen(player, 'keystatuschanged',
           Util.spyFunc(keyStatusSpy));
 
-      const licenseUrl = keySystem == 'com.widevine.alpha' ?
-          'https://cwip-shaka-proxy.appspot.com/no_auth' :
-          'http://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(kid:51745386-2d42-56fd-8bad-4f58422004d7,contentkey:UXRThi1CVv2LrU9YQiAE1w==),(kid:26470f42-96d4-5d04-a9ba-bb442e169800,contentkey:JkcPQpbUXQSpurtELhaYAA==)';
-      player.configure({
-        drm: {
-          servers: {
-            [keySystem]: licenseUrl,
-          },
-        },
-      });
-      await player.load('/base/test/test/assets/clear-encrypted/manifest.mpd');
+      if (keySystem === 'com.apple.fps') {
+        const serverCert = await Util.fetch(
+            '/base/test/test/assets/clear-encrypted-hls/certificate.cer');
+        drmConfig.advanced[keySystem].serverCertificate =
+            shaka.util.BufferUtils.toUint8(serverCert);
+      }
+      player.configure({drm: drmConfig});
+
+      const url = keySystem === 'com.apple.fps' ?
+        '/base/test/test/assets/clear-encrypted-hls/manifest.m3u8' :
+        '/base/test/test/assets/clear-encrypted/manifest.mpd';
+      await player.load(url);
       await video.play();
 
+      // Ensure we're using MediaSource.
+      expect(player.getLoadMode()).toBe(shaka.Player.LoadMode.MEDIA_SOURCE);
+
       // Wait for the video to start playback.  If it takes longer than 10
       // seconds, fail the test.
       await waiter.waitForMovementOrFailOnTimeout(video, 10);
 
-      // Play for 5 seconds, but stop early if the video ends.  If it takes
+      // Play for 10 seconds, but stop early if the video ends.  If it takes
       // longer than 30 seconds, fail the test.
-      await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 5, 30);
+      await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 10, 30);
 
       // Check did we have key status change.
       expect(keyStatusSpy).toHaveBeenCalled();