feat(Transmuxer): Split init segment and segment data when appending to MSE

externs/shaka/transmuxer.js

@@ -53,12 +53,27 @@ shaka.extern.Transmuxer = class {
    *   null for init segments
    * @param {number} duration
    * @param {string} contentType
-   * @return {!Promise<!Uint8Array>}
+   * @return {!Promise<(!shaka.extern.TransmuxerOutput)>}
    */
   transmux(data, stream, reference, duration, contentType) {}
 };
 
 
+/**
+ * @typedef {{
+ *   data: !Uint8Array,
+ *   init: ?Uint8Array
+ * }}
+ *
+ * @property {!Uint8Array} data
+ *   Segment data.
+ * @property {?Uint8Array} init
+ *   Init segment data.
+ * @exportDoc
+ */
+shaka.extern.TransmuxerOutput;
+
+
 /**
  * @typedef {function():!shaka.extern.Transmuxer}
  * @exportDoc

lib/media/media_source_engine.js

@@ -1236,8 +1236,16 @@ shaka.media.MediaSourceEngine = class {
     }
 
     if (this.transmuxers_.has(contentType)) {
-      data = await this.transmuxers_.get(contentType).transmux(
-          data, stream, reference, this.mediaSource_.duration, contentType);
+      const transmuxerOutput =
+          await this.transmuxers_.get(contentType).transmux(
+              data, stream, reference, this.mediaSource_.duration, contentType);
+      if (transmuxerOutput.init != null) {
+        const initData = transmuxerOutput.init;
+        this.enqueueOperation_(contentType, () => {
+          this.append_(contentType, initData, timestampOffset, stream);
+        }, reference ? reference.getUris()[0] : null);
+      }
+      data = transmuxerOutput.data;
     }
 
     data = this.workAroundBrokenPlatforms_(

lib/transmuxer/aac_transmuxer.js

@@ -15,7 +15,6 @@ goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.MimeUtils');
 goog.require('shaka.util.Mp4Generator');
-goog.require('shaka.util.Uint8ArrayUtils');
 
 
 /**
@@ -109,7 +108,6 @@ shaka.transmuxer.AacTransmuxer = class {
    */
   transmux(data, stream, reference, duration) {
     const ADTS = shaka.transmuxer.ADTS;
-    const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
 
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
 
@@ -222,12 +220,10 @@ shaka.transmuxer.AacTransmuxer = class {
     const segmentData = mp4Generator.segmentData();
     this.lastInitSegment_ = initSegment;
     this.frameIndex_++;
-    if (appendInitSegment) {
-      const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
-      return Promise.resolve(transmuxData);
-    } else {
-      return Promise.resolve(segmentData);
-    }
+    return Promise.resolve({
+      data: segmentData,
+      init: appendInitSegment ? initSegment : null,
+    });
   }
 };
 

lib/transmuxer/ac3_transmuxer.js

@@ -15,7 +15,6 @@ goog.require('shaka.util.Error');
 goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.Mp4Generator');
-goog.require('shaka.util.Uint8ArrayUtils');
 
 
 /**
@@ -113,7 +112,6 @@ shaka.transmuxer.Ac3Transmuxer = class {
    */
   transmux(data, stream, reference, duration) {
     const Ac3 = shaka.transmuxer.Ac3;
-    const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
 
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
 
@@ -216,12 +214,10 @@ shaka.transmuxer.Ac3Transmuxer = class {
     const segmentData = mp4Generator.segmentData();
     this.lastInitSegment_ = initSegment;
     this.frameIndex_++;
-    if (appendInitSegment) {
-      const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
-      return Promise.resolve(transmuxData);
-    } else {
-      return Promise.resolve(segmentData);
-    }
+    return Promise.resolve({
+      data: segmentData,
+      init: appendInitSegment ? initSegment : null,
+    });
   }
 };
 

lib/transmuxer/ec3_transmuxer.js

@@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
 goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.Mp4Generator');
-goog.require('shaka.util.Uint8ArrayUtils');
 
 
 /**
@@ -107,7 +106,6 @@ shaka.transmuxer.Ec3Transmuxer = class {
    */
   transmux(data, stream, reference, duration) {
     const Ec3 = shaka.transmuxer.Ec3;
-    const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
 
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
 
@@ -210,12 +208,10 @@ shaka.transmuxer.Ec3Transmuxer = class {
     const segmentData = mp4Generator.segmentData();
     this.lastInitSegment_ = initSegment;
     this.frameIndex_++;
-    if (appendInitSegment) {
-      const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
-      return Promise.resolve(transmuxData);
-    } else {
-      return Promise.resolve(segmentData);
-    }
+    return Promise.resolve({
+      data: segmentData,
+      init: appendInitSegment ? initSegment : null,
+    });
   }
 };
 

lib/transmuxer/h265.js

@@ -78,12 +78,13 @@ shaka.transmuxer.H265 = class {
           spsConfiguration.generalConstraintIndicatorFlags5,
       generalConstraintIndicatorFlags6:
           spsConfiguration.generalConstraintIndicatorFlags6,
-      constantFrameRate: spsConfiguration.constantFrameRate,
       minSpatialSegmentationIdc: spsConfiguration.minSpatialSegmentationIdc,
       chromaFormatIdc: spsConfiguration.chromaFormatIdc,
       bitDepthLumaMinus8: spsConfiguration.bitDepthLumaMinus8,
       bitDepthChromaMinus8: spsConfiguration.bitDepthChromaMinus8,
       parallelismType: ppsConfiguration.parallelismType,
+      frameRateFps: spsConfiguration.frameRateFps,
+      frameRateFixed: spsConfiguration.frameRateFixed,
     };
 
     const videoConfig = H265.getVideoConfiguration_(
@@ -308,6 +309,9 @@ shaka.transmuxer.H265 = class {
     let defaultDisplayWindowFlag = false; // for calc offset
     let sarWidth = 1;
     let sarHeight = 1;
+    let fixedPicRateGeneralFlag = true;
+    let fpsDen = 1;
+    let fpsNum = 0;
     let minSpatialSegmentationIdc = 0; // for hvcC
     gb.readBoolean(); // sps_temporal_mvp_enabled_flag
     gb.readBoolean(); // strong_intra_smoothing_enabled_flag
@@ -355,15 +359,15 @@ shaka.transmuxer.H265 = class {
       gb.readBoolean(); // frame_field_info_present_flag
       defaultDisplayWindowFlag = gb.readBoolean();
       if (defaultDisplayWindowFlag) {
-        gb.readUnsignedExpGolomb();
-        gb.readUnsignedExpGolomb();
-        gb.readUnsignedExpGolomb();
-        gb.readUnsignedExpGolomb();
+        leftOffset += gb.readUnsignedExpGolomb();
+        rightOffset += gb.readUnsignedExpGolomb();
+        topOffset += gb.readUnsignedExpGolomb();
+        bottomOffset += gb.readUnsignedExpGolomb();
       }
       const vuiTimingInfoPresentFlag = gb.readBoolean();
       if (vuiTimingInfoPresentFlag) {
-        gb.readBits(32); // fps_den
-        gb.readBits(32); // fps_num
+        fpsDen = gb.readBits(32);
+        fpsNum = gb.readBits(32);
         const vuiPocProportionalToTimingFlag = gb.readBoolean();
         if (vuiPocProportionalToTimingFlag) {
           gb.readUnsignedExpGolomb();
@@ -396,7 +400,7 @@ shaka.transmuxer.H265 = class {
             }
           }
           for (let i = 0; i <= maxSubLayersMinus1; i++) {
-            const fixedPicRateGeneralFlag = gb.readBoolean();
+            fixedPicRateGeneralFlag = gb.readBoolean();
             let fixedPicRateWithinCvsFlag = true;
             let cpbCnt = 1;
             if (!fixedPicRateGeneralFlag) {
@@ -472,14 +476,15 @@ shaka.transmuxer.H265 = class {
       generalConstraintIndicatorFlags5,
       generalConstraintIndicatorFlags6,
       minSpatialSegmentationIdc,
-      constantFrameRate: 0, // FIXME!!!
       chromaFormatIdc,
       bitDepthLumaMinus8,
       bitDepthChromaMinus8,
       width: codecWidth,
       height: codecHeight,
       sarWidth: sarWidth,
       sarHeight: sarHeight,
+      frameRateFps: fpsNum / fpsDen,
+      frameRateFixed: fixedPicRateGeneralFlag,
     };
   }
 
@@ -545,8 +550,6 @@ shaka.transmuxer.H265 = class {
    * @private
    */
   static getVideoConfiguration_(vps, sps, pps, detail) {
-    const H265 = shaka.transmuxer.H265;
-
     const length = 23 + (3 + 2 + vps.byteLength) +
         (3 + 2 + sps.byteLength) + (3 + 2 + pps.byteLength);
     const data = new Uint8Array(length);
@@ -574,33 +577,34 @@ shaka.transmuxer.H265 = class {
     data[17] = 0xF8 | (detail.bitDepthLumaMinus8 & 0x07);
     data[18] = 0xF8 | (detail.bitDepthChromaMinus8 & 0x07);
     data[19] = 0;
-    data[20] = 0;
-    data[21] = ((detail.constantFrameRate & 0x03) << 6) |
+    data[20] = parseInt(detail.frameRateFps, 10);
+    data[21] = (((detail.frameRateFixed ? 1 : 0) & 0x03) << 6) |
         ((detail.numTemporalLayers & 0x07) << 3) |
         ((detail.temporalIdNested ? 1 : 0) << 2) | 3;
     data[22] = 3;
-    data[23 + 0 + 0] = 0x80 | H265.NALU_TYPE_VPS_;
-    data[23 + 0 + 1] = 0;
-    data[23 + 0 + 2] = 1;
-    data[23 + 0 + 3] = (vps.byteLength & 0xFF00) >> 8;
-    data[23 + 0 + 4] = (vps.byteLength & 0x00FF) >> 0;
-    data.set(vps, 23 + 0 + 5);
-    data[23 + (5 + vps.byteLength) + 0] =
-        0x80 | H265.NALU_TYPE_SPS_;
-    data[23 + (5 + vps.byteLength) + 1] = 0;
-    data[23 + (5 + vps.byteLength) + 2] = 1;
-    data[23 + (5 + vps.byteLength) + 3] = (sps.byteLength & 0xFF00) >> 8;
-    data[23 + (5 + vps.byteLength) + 4] = (sps.byteLength & 0x00FF) >> 0;
-    data.set(sps, 23 + (5 + vps.byteLength) + 5);
-    data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 0] =
-        0x80 | H265.NALU_TYPE_PPS_;
-    data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 1] = 0;
-    data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 2] = 1;
-    data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 3] =
-        (pps.byteLength & 0xFF00) >> 8;
-    data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 4] =
-        (pps.byteLength & 0x00FF) >> 0;
-    data.set(pps, 23 + (5 + vps.byteLength + 5 + sps.byteLength) + 5);
+
+    const units = [vps, sps, pps];
+    let offset = 23;
+    const iMax = units.length - 1;
+    for (let i = 0; i < units.length; i += 1) {
+      data.set(
+          new Uint8Array([
+            (32 + i) | (i === iMax ? 128 : 0),
+            0x00,
+            0x01,
+          ]),
+          offset,
+      );
+      offset += 3;
+      data.set(
+          new Uint8Array([units[i].byteLength >> 8, units[i].byteLength & 255]),
+          offset,
+      );
+      offset += 2;
+      data.set(units[i], offset);
+      offset += units[i].byteLength;
+    }
+
     return data;
   }
 
@@ -788,7 +792,6 @@ shaka.transmuxer.H265.VPSConfiguration;
  *   generalConstraintIndicatorFlags4: number,
  *   generalConstraintIndicatorFlags5: number,
  *   generalConstraintIndicatorFlags6: number,
- *   constantFrameRate: number,
  *   minSpatialSegmentationIdc: number,
  *   chromaFormatIdc: number,
  *   bitDepthLumaMinus8: number,
@@ -797,6 +800,8 @@ shaka.transmuxer.H265.VPSConfiguration;
  *   height: number,
  *   sarWidth: number,
  *   sarHeight: number,
+ *   frameRateFps: number,
+ *   frameRateFixed: boolean,
  * }}
  *
  * @property {number} generalProfileSpace
@@ -813,7 +818,6 @@ shaka.transmuxer.H265.VPSConfiguration;
  * @property {number} generalConstraintIndicatorFlags4
  * @property {number} generalConstraintIndicatorFlags5
  * @property {number} generalConstraintIndicatorFlags6
- * @property {number} constantFrameRate
  * @property {number} minSpatialSegmentationIdc
  * @property {number} chromaFormatIdc
  * @property {number} bitDepthLumaMinus8
@@ -822,6 +826,8 @@ shaka.transmuxer.H265.VPSConfiguration;
  * @property {number} height
  * @property {number} sarWidth
  * @property {number} sarHeight
+ * @property {number} frameRateFps
+ * @property {boolean} frameRateFixed
  */
 shaka.transmuxer.H265.SPSConfiguration;
 
@@ -854,12 +860,13 @@ shaka.transmuxer.H265.PPSConfiguration;
  *   generalConstraintIndicatorFlags4: number,
  *   generalConstraintIndicatorFlags5: number,
  *   generalConstraintIndicatorFlags6: number,
- *   constantFrameRate: number,
  *   minSpatialSegmentationIdc: number,
  *   chromaFormatIdc: number,
  *   bitDepthLumaMinus8: number,
  *   bitDepthChromaMinus8: number,
  *   parallelismType: number,
+ *   frameRateFps: number,
+ *   frameRateFixed: boolean,
  * }}
  *
  * @property {number} numTemporalLayers
@@ -878,11 +885,12 @@ shaka.transmuxer.H265.PPSConfiguration;
  * @property {number} generalConstraintIndicatorFlags4
  * @property {number} generalConstraintIndicatorFlags5
  * @property {number} generalConstraintIndicatorFlags6
- * @property {number} constantFrameRate
  * @property {number} minSpatialSegmentationIdc
  * @property {number} chromaFormatIdc
  * @property {number} bitDepthLumaMinus8
  * @property {number} bitDepthChromaMinus8
  * @property {number} parallelismType
+ * @property {number} frameRateFps
+ * @property {boolean} frameRateFixed
  */
 shaka.transmuxer.H265.DecoderConfigurationRecordType;

lib/transmuxer/mp3_transmuxer.js

@@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
 goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.Mp4Generator');
-goog.require('shaka.util.Uint8ArrayUtils');
 
 
 /**
@@ -107,7 +106,6 @@ shaka.transmuxer.Mp3Transmuxer = class {
    */
   transmux(data, stream, reference, duration) {
     const MpegAudio = shaka.transmuxer.MpegAudio;
-    const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
 
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
 
@@ -203,12 +201,10 @@ shaka.transmuxer.Mp3Transmuxer = class {
     const segmentData = mp4Generator.segmentData();
     this.lastInitSegment_ = initSegment;
     this.frameIndex_++;
-    if (appendInitSegment) {
-      const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
-      return Promise.resolve(transmuxData);
-    } else {
-      return Promise.resolve(segmentData);
-    }
+    return Promise.resolve({
+      data: segmentData,
+      init: appendInitSegment ? initSegment : null,
+    });
   }
 };
 

lib/transmuxer/mpeg_ts_transmuxer.js

@@ -174,7 +174,10 @@ shaka.transmuxer.MpegTsTransmuxer = class {
       }
     }
 
-    return Promise.resolve(transmuxData);
+    return Promise.resolve({
+      data: transmuxData,
+      init: null,
+    });
   }
 };