Skip to content

feat(Transmuxer): Split init segment and segment data when appending to MSE #8855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 9, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion externs/shaka/transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,27 @@ shaka.extern.Transmuxer = class {
* null for init segments
* @param {number} duration
* @param {string} contentType
* @return {!Promise<!Uint8Array>}
* @return {!Promise<(!shaka.extern.TransmuxerOutput)>}
*/
transmux(data, stream, reference, duration, contentType) {}
};


/**
* @typedef {{
* data: !Uint8Array,
* init: ?Uint8Array
* }}
*
* @property {!Uint8Array} data
* Segment data.
* @property {?Uint8Array} init
* Init segment data.
* @exportDoc
*/
shaka.extern.TransmuxerOutput;


/**
* @typedef {function():!shaka.extern.Transmuxer}
* @exportDoc
Expand Down
12 changes: 10 additions & 2 deletions lib/media/media_source_engine.js
Original file line number Diff line number Diff line change
Expand Up @@ -1236,8 +1236,16 @@ shaka.media.MediaSourceEngine = class {
}

if (this.transmuxers_.has(contentType)) {
data = await this.transmuxers_.get(contentType).transmux(
data, stream, reference, this.mediaSource_.duration, contentType);
const transmuxerOutput =
await this.transmuxers_.get(contentType).transmux(
data, stream, reference, this.mediaSource_.duration, contentType);
if (transmuxerOutput.init != null) {
const initData = transmuxerOutput.init;
this.enqueueOperation_(contentType, () => {
this.append_(contentType, initData, timestampOffset, stream);
}, reference ? reference.getUris()[0] : null);
}
data = transmuxerOutput.data;
}

data = this.workAroundBrokenPlatforms_(
Expand Down
12 changes: 4 additions & 8 deletions lib/transmuxer/aac_transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ goog.require('shaka.util.Id3Utils');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.MimeUtils');
goog.require('shaka.util.Mp4Generator');
goog.require('shaka.util.Uint8ArrayUtils');


/**
Expand Down Expand Up @@ -109,7 +108,6 @@ shaka.transmuxer.AacTransmuxer = class {
*/
transmux(data, stream, reference, duration) {
const ADTS = shaka.transmuxer.ADTS;
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;

const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);

Expand Down Expand Up @@ -222,12 +220,10 @@ shaka.transmuxer.AacTransmuxer = class {
const segmentData = mp4Generator.segmentData();
this.lastInitSegment_ = initSegment;
this.frameIndex_++;
if (appendInitSegment) {
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
return Promise.resolve(transmuxData);
} else {
return Promise.resolve(segmentData);
}
return Promise.resolve({
data: segmentData,
init: appendInitSegment ? initSegment : null,
});
}
};

Expand Down
12 changes: 4 additions & 8 deletions lib/transmuxer/ac3_transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ goog.require('shaka.util.Error');
goog.require('shaka.util.Id3Utils');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.Mp4Generator');
goog.require('shaka.util.Uint8ArrayUtils');


/**
Expand Down Expand Up @@ -113,7 +112,6 @@ shaka.transmuxer.Ac3Transmuxer = class {
*/
transmux(data, stream, reference, duration) {
const Ac3 = shaka.transmuxer.Ac3;
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;

const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);

Expand Down Expand Up @@ -216,12 +214,10 @@ shaka.transmuxer.Ac3Transmuxer = class {
const segmentData = mp4Generator.segmentData();
this.lastInitSegment_ = initSegment;
this.frameIndex_++;
if (appendInitSegment) {
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
return Promise.resolve(transmuxData);
} else {
return Promise.resolve(segmentData);
}
return Promise.resolve({
data: segmentData,
init: appendInitSegment ? initSegment : null,
});
}
};

Expand Down
12 changes: 4 additions & 8 deletions lib/transmuxer/ec3_transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
goog.require('shaka.util.Id3Utils');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.Mp4Generator');
goog.require('shaka.util.Uint8ArrayUtils');


/**
Expand Down Expand Up @@ -107,7 +106,6 @@ shaka.transmuxer.Ec3Transmuxer = class {
*/
transmux(data, stream, reference, duration) {
const Ec3 = shaka.transmuxer.Ec3;
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;

const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);

Expand Down Expand Up @@ -210,12 +208,10 @@ shaka.transmuxer.Ec3Transmuxer = class {
const segmentData = mp4Generator.segmentData();
this.lastInitSegment_ = initSegment;
this.frameIndex_++;
if (appendInitSegment) {
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
return Promise.resolve(transmuxData);
} else {
return Promise.resolve(segmentData);
}
return Promise.resolve({
data: segmentData,
init: appendInitSegment ? initSegment : null,
});
}
};

Expand Down
86 changes: 47 additions & 39 deletions lib/transmuxer/h265.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,13 @@ shaka.transmuxer.H265 = class {
spsConfiguration.generalConstraintIndicatorFlags5,
generalConstraintIndicatorFlags6:
spsConfiguration.generalConstraintIndicatorFlags6,
constantFrameRate: spsConfiguration.constantFrameRate,
minSpatialSegmentationIdc: spsConfiguration.minSpatialSegmentationIdc,
chromaFormatIdc: spsConfiguration.chromaFormatIdc,
bitDepthLumaMinus8: spsConfiguration.bitDepthLumaMinus8,
bitDepthChromaMinus8: spsConfiguration.bitDepthChromaMinus8,
parallelismType: ppsConfiguration.parallelismType,
frameRateFps: spsConfiguration.frameRateFps,
frameRateFixed: spsConfiguration.frameRateFixed,
};

const videoConfig = H265.getVideoConfiguration_(
Expand Down Expand Up @@ -308,6 +309,9 @@ shaka.transmuxer.H265 = class {
let defaultDisplayWindowFlag = false; // for calc offset
let sarWidth = 1;
let sarHeight = 1;
let fixedPicRateGeneralFlag = true;
let fpsDen = 1;
let fpsNum = 0;
let minSpatialSegmentationIdc = 0; // for hvcC
gb.readBoolean(); // sps_temporal_mvp_enabled_flag
gb.readBoolean(); // strong_intra_smoothing_enabled_flag
Expand Down Expand Up @@ -355,15 +359,15 @@ shaka.transmuxer.H265 = class {
gb.readBoolean(); // frame_field_info_present_flag
defaultDisplayWindowFlag = gb.readBoolean();
if (defaultDisplayWindowFlag) {
gb.readUnsignedExpGolomb();
gb.readUnsignedExpGolomb();
gb.readUnsignedExpGolomb();
gb.readUnsignedExpGolomb();
leftOffset += gb.readUnsignedExpGolomb();
rightOffset += gb.readUnsignedExpGolomb();
topOffset += gb.readUnsignedExpGolomb();
bottomOffset += gb.readUnsignedExpGolomb();
}
const vuiTimingInfoPresentFlag = gb.readBoolean();
if (vuiTimingInfoPresentFlag) {
gb.readBits(32); // fps_den
gb.readBits(32); // fps_num
fpsDen = gb.readBits(32);
fpsNum = gb.readBits(32);
const vuiPocProportionalToTimingFlag = gb.readBoolean();
if (vuiPocProportionalToTimingFlag) {
gb.readUnsignedExpGolomb();
Expand Down Expand Up @@ -396,7 +400,7 @@ shaka.transmuxer.H265 = class {
}
}
for (let i = 0; i <= maxSubLayersMinus1; i++) {
const fixedPicRateGeneralFlag = gb.readBoolean();
fixedPicRateGeneralFlag = gb.readBoolean();
let fixedPicRateWithinCvsFlag = true;
let cpbCnt = 1;
if (!fixedPicRateGeneralFlag) {
Expand Down Expand Up @@ -472,14 +476,15 @@ shaka.transmuxer.H265 = class {
generalConstraintIndicatorFlags5,
generalConstraintIndicatorFlags6,
minSpatialSegmentationIdc,
constantFrameRate: 0, // FIXME!!!
chromaFormatIdc,
bitDepthLumaMinus8,
bitDepthChromaMinus8,
width: codecWidth,
height: codecHeight,
sarWidth: sarWidth,
sarHeight: sarHeight,
frameRateFps: fpsNum / fpsDen,
frameRateFixed: fixedPicRateGeneralFlag,
};
}

Expand Down Expand Up @@ -545,8 +550,6 @@ shaka.transmuxer.H265 = class {
* @private
*/
static getVideoConfiguration_(vps, sps, pps, detail) {
const H265 = shaka.transmuxer.H265;

const length = 23 + (3 + 2 + vps.byteLength) +
(3 + 2 + sps.byteLength) + (3 + 2 + pps.byteLength);
const data = new Uint8Array(length);
Expand Down Expand Up @@ -574,33 +577,34 @@ shaka.transmuxer.H265 = class {
data[17] = 0xF8 | (detail.bitDepthLumaMinus8 & 0x07);
data[18] = 0xF8 | (detail.bitDepthChromaMinus8 & 0x07);
data[19] = 0;
data[20] = 0;
data[21] = ((detail.constantFrameRate & 0x03) << 6) |
data[20] = parseInt(detail.frameRateFps, 10);
data[21] = (((detail.frameRateFixed ? 1 : 0) & 0x03) << 6) |
((detail.numTemporalLayers & 0x07) << 3) |
((detail.temporalIdNested ? 1 : 0) << 2) | 3;
data[22] = 3;
data[23 + 0 + 0] = 0x80 | H265.NALU_TYPE_VPS_;
data[23 + 0 + 1] = 0;
data[23 + 0 + 2] = 1;
data[23 + 0 + 3] = (vps.byteLength & 0xFF00) >> 8;
data[23 + 0 + 4] = (vps.byteLength & 0x00FF) >> 0;
data.set(vps, 23 + 0 + 5);
data[23 + (5 + vps.byteLength) + 0] =
0x80 | H265.NALU_TYPE_SPS_;
data[23 + (5 + vps.byteLength) + 1] = 0;
data[23 + (5 + vps.byteLength) + 2] = 1;
data[23 + (5 + vps.byteLength) + 3] = (sps.byteLength & 0xFF00) >> 8;
data[23 + (5 + vps.byteLength) + 4] = (sps.byteLength & 0x00FF) >> 0;
data.set(sps, 23 + (5 + vps.byteLength) + 5);
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 0] =
0x80 | H265.NALU_TYPE_PPS_;
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 1] = 0;
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 2] = 1;
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 3] =
(pps.byteLength & 0xFF00) >> 8;
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 4] =
(pps.byteLength & 0x00FF) >> 0;
data.set(pps, 23 + (5 + vps.byteLength + 5 + sps.byteLength) + 5);

const units = [vps, sps, pps];
let offset = 23;
const iMax = units.length - 1;
for (let i = 0; i < units.length; i += 1) {
data.set(
new Uint8Array([
(32 + i) | (i === iMax ? 128 : 0),
0x00,
0x01,
]),
offset,
);
offset += 3;
data.set(
new Uint8Array([units[i].byteLength >> 8, units[i].byteLength & 255]),
offset,
);
offset += 2;
data.set(units[i], offset);
offset += units[i].byteLength;
}

return data;
}

Expand Down Expand Up @@ -788,7 +792,6 @@ shaka.transmuxer.H265.VPSConfiguration;
* generalConstraintIndicatorFlags4: number,
* generalConstraintIndicatorFlags5: number,
* generalConstraintIndicatorFlags6: number,
* constantFrameRate: number,
* minSpatialSegmentationIdc: number,
* chromaFormatIdc: number,
* bitDepthLumaMinus8: number,
Expand All @@ -797,6 +800,8 @@ shaka.transmuxer.H265.VPSConfiguration;
* height: number,
* sarWidth: number,
* sarHeight: number,
* frameRateFps: number,
* frameRateFixed: boolean,
* }}
*
* @property {number} generalProfileSpace
Expand All @@ -813,7 +818,6 @@ shaka.transmuxer.H265.VPSConfiguration;
* @property {number} generalConstraintIndicatorFlags4
* @property {number} generalConstraintIndicatorFlags5
* @property {number} generalConstraintIndicatorFlags6
* @property {number} constantFrameRate
* @property {number} minSpatialSegmentationIdc
* @property {number} chromaFormatIdc
* @property {number} bitDepthLumaMinus8
Expand All @@ -822,6 +826,8 @@ shaka.transmuxer.H265.VPSConfiguration;
* @property {number} height
* @property {number} sarWidth
* @property {number} sarHeight
* @property {number} frameRateFps
* @property {boolean} frameRateFixed
*/
shaka.transmuxer.H265.SPSConfiguration;

Expand Down Expand Up @@ -854,12 +860,13 @@ shaka.transmuxer.H265.PPSConfiguration;
* generalConstraintIndicatorFlags4: number,
* generalConstraintIndicatorFlags5: number,
* generalConstraintIndicatorFlags6: number,
* constantFrameRate: number,
* minSpatialSegmentationIdc: number,
* chromaFormatIdc: number,
* bitDepthLumaMinus8: number,
* bitDepthChromaMinus8: number,
* parallelismType: number,
* frameRateFps: number,
* frameRateFixed: boolean,
* }}
*
* @property {number} numTemporalLayers
Expand All @@ -878,11 +885,12 @@ shaka.transmuxer.H265.PPSConfiguration;
* @property {number} generalConstraintIndicatorFlags4
* @property {number} generalConstraintIndicatorFlags5
* @property {number} generalConstraintIndicatorFlags6
* @property {number} constantFrameRate
* @property {number} minSpatialSegmentationIdc
* @property {number} chromaFormatIdc
* @property {number} bitDepthLumaMinus8
* @property {number} bitDepthChromaMinus8
* @property {number} parallelismType
* @property {number} frameRateFps
* @property {boolean} frameRateFixed
*/
shaka.transmuxer.H265.DecoderConfigurationRecordType;
12 changes: 4 additions & 8 deletions lib/transmuxer/mp3_transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
goog.require('shaka.util.Id3Utils');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.Mp4Generator');
goog.require('shaka.util.Uint8ArrayUtils');


/**
Expand Down Expand Up @@ -107,7 +106,6 @@ shaka.transmuxer.Mp3Transmuxer = class {
*/
transmux(data, stream, reference, duration) {
const MpegAudio = shaka.transmuxer.MpegAudio;
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;

const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);

Expand Down Expand Up @@ -203,12 +201,10 @@ shaka.transmuxer.Mp3Transmuxer = class {
const segmentData = mp4Generator.segmentData();
this.lastInitSegment_ = initSegment;
this.frameIndex_++;
if (appendInitSegment) {
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
return Promise.resolve(transmuxData);
} else {
return Promise.resolve(segmentData);
}
return Promise.resolve({
data: segmentData,
init: appendInitSegment ? initSegment : null,
});
}
};

Expand Down
5 changes: 4 additions & 1 deletion lib/transmuxer/mpeg_ts_transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,10 @@ shaka.transmuxer.MpegTsTransmuxer = class {
}
}

return Promise.resolve(transmuxData);
return Promise.resolve({
data: transmuxData,
init: null,
});
}
};

Expand Down
Loading
Loading