Skip to content

Commit b59066e

Browse files
aveladtykus160
avelad
and
tykus160
authored
feat(Transmuxer): Split init segment and segment data when appending to MSE (#8855)
Also fixes the TS-HEVC transmuxer, SPS parsing was wrong. --------- Co-authored-by: Wojciech Tyczyński <[email protected]>
1 parent c14e6cc commit b59066e

File tree

8 files changed

+102
-81
lines changed

8 files changed

+102
-81
lines changed

‎externs/shaka/transmuxer.js

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,30 @@ shaka.extern.Transmuxer = class {
5353
* null for init segments
5454
* @param {number} duration
5555
* @param {string} contentType
56-
* @return {!Promise<!Uint8Array>}
56+
* @return {!Promise<(!Uint8Array|!shaka.extern.TransmuxerOutput)>} If you
57+
* only want to return the result, use Uint8Array, if you want to separate
58+
* the initialization segment and the data segment, you have to use
59+
* shaka.extern.TransmuxerOutput
5760
*/
5861
transmux(data, stream, reference, duration, contentType) {}
5962
};
6063

6164

65+
/**
66+
* @typedef {{
67+
* data: !Uint8Array,
68+
* init: ?Uint8Array
69+
* }}
70+
*
71+
* @property {!Uint8Array} data
72+
* Segment data.
73+
* @property {?Uint8Array} init
74+
* Init segment data.
75+
* @exportDoc
76+
*/
77+
shaka.extern.TransmuxerOutput;
78+
79+
6280
/**
6381
* @typedef {function():!shaka.extern.Transmuxer}
6482
* @exportDoc

‎lib/media/media_source_engine.js

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,8 +1236,22 @@ shaka.media.MediaSourceEngine = class {
12361236
}
12371237

12381238
if (this.transmuxers_.has(contentType)) {
1239-
data = await this.transmuxers_.get(contentType).transmux(
1240-
data, stream, reference, this.mediaSource_.duration, contentType);
1239+
const transmuxerOutput =
1240+
await this.transmuxers_.get(contentType).transmux(
1241+
data, stream, reference, this.mediaSource_.duration, contentType);
1242+
if (ArrayBuffer.isView(transmuxerOutput)) {
1243+
data = /** @type {!Uint8Array} */(transmuxerOutput);
1244+
} else {
1245+
const output =
1246+
/** @type {!shaka.extern.TransmuxerOutput} */(transmuxerOutput);
1247+
if (output.init != null) {
1248+
const initData = output.init;
1249+
this.enqueueOperation_(contentType, () => {
1250+
this.append_(contentType, initData, timestampOffset, stream);
1251+
}, reference ? reference.getUris()[0] : null);
1252+
}
1253+
data = output.data;
1254+
}
12411255
}
12421256

12431257
data = this.workAroundBrokenPlatforms_(

‎lib/transmuxer/aac_transmuxer.js

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ goog.require('shaka.util.Id3Utils');
1515
goog.require('shaka.util.ManifestParserUtils');
1616
goog.require('shaka.util.MimeUtils');
1717
goog.require('shaka.util.Mp4Generator');
18-
goog.require('shaka.util.Uint8ArrayUtils');
1918

2019

2120
/**
@@ -109,7 +108,6 @@ shaka.transmuxer.AacTransmuxer = class {
109108
*/
110109
transmux(data, stream, reference, duration) {
111110
const ADTS = shaka.transmuxer.ADTS;
112-
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
113111

114112
const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
115113

@@ -222,12 +220,10 @@ shaka.transmuxer.AacTransmuxer = class {
222220
const segmentData = mp4Generator.segmentData();
223221
this.lastInitSegment_ = initSegment;
224222
this.frameIndex_++;
225-
if (appendInitSegment) {
226-
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
227-
return Promise.resolve(transmuxData);
228-
} else {
229-
return Promise.resolve(segmentData);
230-
}
223+
return Promise.resolve({
224+
data: segmentData,
225+
init: appendInitSegment ? initSegment : null,
226+
});
231227
}
232228
};
233229

‎lib/transmuxer/ac3_transmuxer.js

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ goog.require('shaka.util.Error');
1515
goog.require('shaka.util.Id3Utils');
1616
goog.require('shaka.util.ManifestParserUtils');
1717
goog.require('shaka.util.Mp4Generator');
18-
goog.require('shaka.util.Uint8ArrayUtils');
1918

2019

2120
/**
@@ -113,7 +112,6 @@ shaka.transmuxer.Ac3Transmuxer = class {
113112
*/
114113
transmux(data, stream, reference, duration) {
115114
const Ac3 = shaka.transmuxer.Ac3;
116-
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
117115

118116
const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
119117

@@ -216,12 +214,10 @@ shaka.transmuxer.Ac3Transmuxer = class {
216214
const segmentData = mp4Generator.segmentData();
217215
this.lastInitSegment_ = initSegment;
218216
this.frameIndex_++;
219-
if (appendInitSegment) {
220-
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
221-
return Promise.resolve(transmuxData);
222-
} else {
223-
return Promise.resolve(segmentData);
224-
}
217+
return Promise.resolve({
218+
data: segmentData,
219+
init: appendInitSegment ? initSegment : null,
220+
});
225221
}
226222
};
227223

‎lib/transmuxer/ec3_transmuxer.js

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
1414
goog.require('shaka.util.Id3Utils');
1515
goog.require('shaka.util.ManifestParserUtils');
1616
goog.require('shaka.util.Mp4Generator');
17-
goog.require('shaka.util.Uint8ArrayUtils');
1817

1918

2019
/**
@@ -107,7 +106,6 @@ shaka.transmuxer.Ec3Transmuxer = class {
107106
*/
108107
transmux(data, stream, reference, duration) {
109108
const Ec3 = shaka.transmuxer.Ec3;
110-
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
111109

112110
const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
113111

@@ -210,12 +208,10 @@ shaka.transmuxer.Ec3Transmuxer = class {
210208
const segmentData = mp4Generator.segmentData();
211209
this.lastInitSegment_ = initSegment;
212210
this.frameIndex_++;
213-
if (appendInitSegment) {
214-
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
215-
return Promise.resolve(transmuxData);
216-
} else {
217-
return Promise.resolve(segmentData);
218-
}
211+
return Promise.resolve({
212+
data: segmentData,
213+
init: appendInitSegment ? initSegment : null,
214+
});
219215
}
220216
};
221217

‎lib/transmuxer/h265.js

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -78,12 +78,13 @@ shaka.transmuxer.H265 = class {
7878
spsConfiguration.generalConstraintIndicatorFlags5,
7979
generalConstraintIndicatorFlags6:
8080
spsConfiguration.generalConstraintIndicatorFlags6,
81-
constantFrameRate: spsConfiguration.constantFrameRate,
8281
minSpatialSegmentationIdc: spsConfiguration.minSpatialSegmentationIdc,
8382
chromaFormatIdc: spsConfiguration.chromaFormatIdc,
8483
bitDepthLumaMinus8: spsConfiguration.bitDepthLumaMinus8,
8584
bitDepthChromaMinus8: spsConfiguration.bitDepthChromaMinus8,
8685
parallelismType: ppsConfiguration.parallelismType,
86+
frameRateFps: spsConfiguration.frameRateFps,
87+
frameRateFixed: spsConfiguration.frameRateFixed,
8788
};
8889

8990
const videoConfig = H265.getVideoConfiguration_(
@@ -308,6 +309,9 @@ shaka.transmuxer.H265 = class {
308309
let defaultDisplayWindowFlag = false; // for calc offset
309310
let sarWidth = 1;
310311
let sarHeight = 1;
312+
let fixedPicRateGeneralFlag = true;
313+
let fpsDen = 1;
314+
let fpsNum = 0;
311315
let minSpatialSegmentationIdc = 0; // for hvcC
312316
gb.readBoolean(); // sps_temporal_mvp_enabled_flag
313317
gb.readBoolean(); // strong_intra_smoothing_enabled_flag
@@ -355,15 +359,15 @@ shaka.transmuxer.H265 = class {
355359
gb.readBoolean(); // frame_field_info_present_flag
356360
defaultDisplayWindowFlag = gb.readBoolean();
357361
if (defaultDisplayWindowFlag) {
358-
gb.readUnsignedExpGolomb();
359-
gb.readUnsignedExpGolomb();
360-
gb.readUnsignedExpGolomb();
361-
gb.readUnsignedExpGolomb();
362+
leftOffset += gb.readUnsignedExpGolomb();
363+
rightOffset += gb.readUnsignedExpGolomb();
364+
topOffset += gb.readUnsignedExpGolomb();
365+
bottomOffset += gb.readUnsignedExpGolomb();
362366
}
363367
const vuiTimingInfoPresentFlag = gb.readBoolean();
364368
if (vuiTimingInfoPresentFlag) {
365-
gb.readBits(32); // fps_den
366-
gb.readBits(32); // fps_num
369+
fpsDen = gb.readBits(32);
370+
fpsNum = gb.readBits(32);
367371
const vuiPocProportionalToTimingFlag = gb.readBoolean();
368372
if (vuiPocProportionalToTimingFlag) {
369373
gb.readUnsignedExpGolomb();
@@ -396,7 +400,7 @@ shaka.transmuxer.H265 = class {
396400
}
397401
}
398402
for (let i = 0; i <= maxSubLayersMinus1; i++) {
399-
const fixedPicRateGeneralFlag = gb.readBoolean();
403+
fixedPicRateGeneralFlag = gb.readBoolean();
400404
let fixedPicRateWithinCvsFlag = true;
401405
let cpbCnt = 1;
402406
if (!fixedPicRateGeneralFlag) {
@@ -472,14 +476,15 @@ shaka.transmuxer.H265 = class {
472476
generalConstraintIndicatorFlags5,
473477
generalConstraintIndicatorFlags6,
474478
minSpatialSegmentationIdc,
475-
constantFrameRate: 0, // FIXME!!!
476479
chromaFormatIdc,
477480
bitDepthLumaMinus8,
478481
bitDepthChromaMinus8,
479482
width: codecWidth,
480483
height: codecHeight,
481484
sarWidth: sarWidth,
482485
sarHeight: sarHeight,
486+
frameRateFps: fpsNum / fpsDen,
487+
frameRateFixed: fixedPicRateGeneralFlag,
483488
};
484489
}
485490

@@ -545,8 +550,6 @@ shaka.transmuxer.H265 = class {
545550
* @private
546551
*/
547552
static getVideoConfiguration_(vps, sps, pps, detail) {
548-
const H265 = shaka.transmuxer.H265;
549-
550553
const length = 23 + (3 + 2 + vps.byteLength) +
551554
(3 + 2 + sps.byteLength) + (3 + 2 + pps.byteLength);
552555
const data = new Uint8Array(length);
@@ -574,33 +577,34 @@ shaka.transmuxer.H265 = class {
574577
data[17] = 0xF8 | (detail.bitDepthLumaMinus8 & 0x07);
575578
data[18] = 0xF8 | (detail.bitDepthChromaMinus8 & 0x07);
576579
data[19] = 0;
577-
data[20] = 0;
578-
data[21] = ((detail.constantFrameRate & 0x03) << 6) |
580+
data[20] = parseInt(detail.frameRateFps, 10);
581+
data[21] = (((detail.frameRateFixed ? 1 : 0) & 0x03) << 6) |
579582
((detail.numTemporalLayers & 0x07) << 3) |
580583
((detail.temporalIdNested ? 1 : 0) << 2) | 3;
581584
data[22] = 3;
582-
data[23 + 0 + 0] = 0x80 | H265.NALU_TYPE_VPS_;
583-
data[23 + 0 + 1] = 0;
584-
data[23 + 0 + 2] = 1;
585-
data[23 + 0 + 3] = (vps.byteLength & 0xFF00) >> 8;
586-
data[23 + 0 + 4] = (vps.byteLength & 0x00FF) >> 0;
587-
data.set(vps, 23 + 0 + 5);
588-
data[23 + (5 + vps.byteLength) + 0] =
589-
0x80 | H265.NALU_TYPE_SPS_;
590-
data[23 + (5 + vps.byteLength) + 1] = 0;
591-
data[23 + (5 + vps.byteLength) + 2] = 1;
592-
data[23 + (5 + vps.byteLength) + 3] = (sps.byteLength & 0xFF00) >> 8;
593-
data[23 + (5 + vps.byteLength) + 4] = (sps.byteLength & 0x00FF) >> 0;
594-
data.set(sps, 23 + (5 + vps.byteLength) + 5);
595-
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 0] =
596-
0x80 | H265.NALU_TYPE_PPS_;
597-
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 1] = 0;
598-
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 2] = 1;
599-
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 3] =
600-
(pps.byteLength & 0xFF00) >> 8;
601-
data[23 + (5 + vps.byteLength + 5 + sps.byteLength) + 4] =
602-
(pps.byteLength & 0x00FF) >> 0;
603-
data.set(pps, 23 + (5 + vps.byteLength + 5 + sps.byteLength) + 5);
585+
586+
const units = [vps, sps, pps];
587+
let offset = 23;
588+
const iMax = units.length - 1;
589+
for (let i = 0; i < units.length; i += 1) {
590+
data.set(
591+
new Uint8Array([
592+
(32 + i) | (i === iMax ? 128 : 0),
593+
0x00,
594+
0x01,
595+
]),
596+
offset,
597+
);
598+
offset += 3;
599+
data.set(
600+
new Uint8Array([units[i].byteLength >> 8, units[i].byteLength & 255]),
601+
offset,
602+
);
603+
offset += 2;
604+
data.set(units[i], offset);
605+
offset += units[i].byteLength;
606+
}
607+
604608
return data;
605609
}
606610

@@ -788,7 +792,6 @@ shaka.transmuxer.H265.VPSConfiguration;
788792
* generalConstraintIndicatorFlags4: number,
789793
* generalConstraintIndicatorFlags5: number,
790794
* generalConstraintIndicatorFlags6: number,
791-
* constantFrameRate: number,
792795
* minSpatialSegmentationIdc: number,
793796
* chromaFormatIdc: number,
794797
* bitDepthLumaMinus8: number,
@@ -797,6 +800,8 @@ shaka.transmuxer.H265.VPSConfiguration;
797800
* height: number,
798801
* sarWidth: number,
799802
* sarHeight: number,
803+
* frameRateFps: number,
804+
* frameRateFixed: boolean,
800805
* }}
801806
*
802807
* @property {number} generalProfileSpace
@@ -813,7 +818,6 @@ shaka.transmuxer.H265.VPSConfiguration;
813818
* @property {number} generalConstraintIndicatorFlags4
814819
* @property {number} generalConstraintIndicatorFlags5
815820
* @property {number} generalConstraintIndicatorFlags6
816-
* @property {number} constantFrameRate
817821
* @property {number} minSpatialSegmentationIdc
818822
* @property {number} chromaFormatIdc
819823
* @property {number} bitDepthLumaMinus8
@@ -822,6 +826,8 @@ shaka.transmuxer.H265.VPSConfiguration;
822826
* @property {number} height
823827
* @property {number} sarWidth
824828
* @property {number} sarHeight
829+
* @property {number} frameRateFps
830+
* @property {boolean} frameRateFixed
825831
*/
826832
shaka.transmuxer.H265.SPSConfiguration;
827833

@@ -854,12 +860,13 @@ shaka.transmuxer.H265.PPSConfiguration;
854860
* generalConstraintIndicatorFlags4: number,
855861
* generalConstraintIndicatorFlags5: number,
856862
* generalConstraintIndicatorFlags6: number,
857-
* constantFrameRate: number,
858863
* minSpatialSegmentationIdc: number,
859864
* chromaFormatIdc: number,
860865
* bitDepthLumaMinus8: number,
861866
* bitDepthChromaMinus8: number,
862867
* parallelismType: number,
868+
* frameRateFps: number,
869+
* frameRateFixed: boolean,
863870
* }}
864871
*
865872
* @property {number} numTemporalLayers
@@ -878,11 +885,12 @@ shaka.transmuxer.H265.PPSConfiguration;
878885
* @property {number} generalConstraintIndicatorFlags4
879886
* @property {number} generalConstraintIndicatorFlags5
880887
* @property {number} generalConstraintIndicatorFlags6
881-
* @property {number} constantFrameRate
882888
* @property {number} minSpatialSegmentationIdc
883889
* @property {number} chromaFormatIdc
884890
* @property {number} bitDepthLumaMinus8
885891
* @property {number} bitDepthChromaMinus8
886892
* @property {number} parallelismType
893+
* @property {number} frameRateFps
894+
* @property {boolean} frameRateFixed
887895
*/
888896
shaka.transmuxer.H265.DecoderConfigurationRecordType;

‎lib/transmuxer/mp3_transmuxer.js

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ goog.require('shaka.util.Error');
1414
goog.require('shaka.util.Id3Utils');
1515
goog.require('shaka.util.ManifestParserUtils');
1616
goog.require('shaka.util.Mp4Generator');
17-
goog.require('shaka.util.Uint8ArrayUtils');
1817

1918

2019
/**
@@ -107,7 +106,6 @@ shaka.transmuxer.Mp3Transmuxer = class {
107106
*/
108107
transmux(data, stream, reference, duration) {
109108
const MpegAudio = shaka.transmuxer.MpegAudio;
110-
const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils;
111109

112110
const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
113111

@@ -203,12 +201,10 @@ shaka.transmuxer.Mp3Transmuxer = class {
203201
const segmentData = mp4Generator.segmentData();
204202
this.lastInitSegment_ = initSegment;
205203
this.frameIndex_++;
206-
if (appendInitSegment) {
207-
const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
208-
return Promise.resolve(transmuxData);
209-
} else {
210-
return Promise.resolve(segmentData);
211-
}
204+
return Promise.resolve({
205+
data: segmentData,
206+
init: appendInitSegment ? initSegment : null,
207+
});
212208
}
213209
};
214210

0 commit comments

Comments
 (0)