Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Parse TFRF for continuous SmoothStreaming segments refresh #7005

Open
wants to merge 1 commit into
base: dev-v2
Choose a base branch
from
Open

Parse TFRF for continuous SmoothStreaming segments refresh #7005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 43 additions & 10 deletions ...tor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ public class FragmentedMp4Extractor implements Extractor {
public static final ExtractorsFactory FACTORY =
() -> new Extractor[] {new FragmentedMp4Extractor()};

public interface SsAtomCallback {
void onTfrfAtom(long startTime, long duration);
}

/**
* Flags controlling the behavior of the extractor. Possible flag values are {@link
* #FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME}, {@link #FLAG_WORKAROUND_IGNORE_TFDT_BOX},
Expand Down Expand Up @@ -115,6 +119,8 @@ public class FragmentedMp4Extractor implements Extractor {
new byte[] {-94, 57, 79, 82, 90, -101, 79, 20, -94, 68, 108, 66, 124, 100, -115, -12};
private static final Format EMSG_FORMAT =
Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG);
private static final byte[] TFRF_UUID =
new byte[] { -44, -128, 126, -14, -54, 57, 70, -107, -114, 84, 38, -53, -98, 70, -89, -97 };

// Parser states.
private static final int STATE_READING_ATOM_HEADER = 0;
Expand Down Expand Up @@ -151,6 +157,8 @@ public class FragmentedMp4Extractor implements Extractor {
private final ArrayDeque<MetadataSampleInfo> pendingMetadataSampleInfos;
@Nullable private final TrackOutput additionalEmsgTrackOutput;

private SsAtomCallback ssAtomCallback;

private int parserState;
private int atomType;
private long atomSize;
Expand Down Expand Up @@ -267,6 +275,10 @@ public FragmentedMp4Extractor(
enterReadingAtomHeaderState();
}

public void setSsAtomCallback(SsAtomCallback cb) {
ssAtomCallback = cb;
}

@Override
public boolean sniff(ExtractorInput input) throws IOException, InterruptedException {
return Sniffer.sniffFragmented(input);
Expand Down Expand Up @@ -685,7 +697,7 @@ private static long parseMehd(ParsableByteArray mehd) {
return version == 0 ? mehd.readUnsignedInt() : mehd.readUnsignedLongToLong();
}

private static void parseMoof(ContainerAtom moof, SparseArray<TrackBundle> trackBundleArray,
private void parseMoof(ContainerAtom moof, SparseArray<TrackBundle> trackBundleArray,
@Flags int flags, byte[] extendedTypeScratch) throws ParserException {
int moofContainerChildrenSize = moof.containerChildren.size();
for (int i = 0; i < moofContainerChildrenSize; i++) {
Expand All @@ -700,7 +712,7 @@ private static void parseMoof(ContainerAtom moof, SparseArray<TrackBundle> track
/**
* Parses a traf atom (defined in 14496-12).
*/
private static void parseTraf(ContainerAtom traf, SparseArray<TrackBundle> trackBundleArray,
private void parseTraf(ContainerAtom traf, SparseArray<TrackBundle> trackBundleArray,
@Flags int flags, byte[] extendedTypeScratch) throws ParserException {
LeafAtom tfhd = traf.getLeafAtomOfType(Atom.TYPE_tfhd);
@Nullable TrackBundle trackBundle = parseTfhd(tfhd.data, trackBundleArray);
Expand Down Expand Up @@ -1004,20 +1016,41 @@ private static int parseTrun(TrackBundle trackBundle, int index, long decodeTime
return trackRunEnd;
}

private static void parseUuid(ParsableByteArray uuid, TrackFragment out,
private void parseUuid(ParsableByteArray uuid, TrackFragment out,
byte[] extendedTypeScratch) throws ParserException {
uuid.setPosition(Atom.HEADER_SIZE);
uuid.readBytes(extendedTypeScratch, 0, 16);

// Currently this parser only supports Microsoft's PIFF SampleEncryptionBox.
if (!Arrays.equals(extendedTypeScratch, PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE)) {
return;
if (Arrays.equals(extendedTypeScratch, PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE)) {
// Except for the extended type, this box is identical to a SENC box. See "Portable encoding of
// audio-video objects: The Protected Interoperable File Format (PIFF), John A. Bocharov et al,
// Section 5.3.2.1."
parseSenc(uuid, 16, out);
return;
}

// Except for the extended type, this box is identical to a SENC box. See "Portable encoding of
// audio-video objects: The Protected Interoperable File Format (PIFF), John A. Bocharov et al,
// Section 5.3.2.1."
parseSenc(uuid, 16, out);
if (Arrays.equals(extendedTypeScratch, TFRF_UUID)) {
ParsableByteArray data = uuid;
data.setPosition(Atom.HEADER_SIZE+16);
int fullAtom = data.readInt();
int version = Atom.parseFullAtomVersion(fullAtom);

int fragmentCount = data.readUnsignedByte();
for(int i=0; i<fragmentCount; i++) {
long absTime = 0, duration = 0;
if(version == 0) {
absTime = data.readInt();
duration = data.readInt();
} else if(version == 1) {
absTime = data.readLong();
duration = data.readLong();
}
if(ssAtomCallback != null) {
ssAtomCallback.onTfrfAtom(absTime, duration);
}
}
return;
}
}

private static void parseSenc(ParsableByteArray senc, TrackFragment out) throws ParserException {
Expand Down
144 changes: 120 additions & 24 deletions .../main/java/com/google/android/exoplayer2/source/smoothstreaming/DefaultSsChunkSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import android.net.Uri;
import androidx.annotation.Nullable;

import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.SeekParameters;
Expand All @@ -25,6 +26,7 @@
import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
import com.google.android.exoplayer2.source.BehindLiveWindowException;
import com.google.android.exoplayer2.source.chunk.BaseMediaChunkIterator;
import com.google.android.exoplayer2.source.SinglePeriodTimeline;
import com.google.android.exoplayer2.source.chunk.Chunk;
import com.google.android.exoplayer2.source.chunk.ChunkExtractorWrapper;
import com.google.android.exoplayer2.source.chunk.ChunkHolder;
Expand All @@ -40,12 +42,14 @@
import com.google.android.exoplayer2.upstream.TransferListener;
import com.google.android.exoplayer2.util.Assertions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeSet;

/**
* A default {@link SsChunkSource} implementation.
*/
public class DefaultSsChunkSource implements SsChunkSource {
public class DefaultSsChunkSource implements SsChunkSource, FragmentedMp4Extractor.SsAtomCallback {

public static final class Factory implements SsChunkSource.Factory {

Expand All @@ -61,17 +65,39 @@ public SsChunkSource createChunkSource(
SsManifest manifest,
int elementIndex,
TrackSelection trackSelection,
@Nullable TransferListener transferListener) {
@Nullable TransferListener transferListener,
SsMediaSource mediaSource) {
DataSource dataSource = dataSourceFactory.createDataSource();
if (transferListener != null) {
dataSource.addTransferListener(transferListener);
}
return new DefaultSsChunkSource(
manifestLoaderErrorThrower, manifest, elementIndex, trackSelection, dataSource);
manifestLoaderErrorThrower, manifest, elementIndex, trackSelection, dataSource, mediaSource);
}

}

public static class ChunkInfo implements Comparable<ChunkInfo> {
//Times are in timescale base
public final long startTimeTs;
public final long durationTs;
public final int chunkId;
public ChunkInfo(long startTimeTs, long durationTs, int chunkId) {
this.startTimeTs = startTimeTs;
this.durationTs=durationTs;
this.chunkId=chunkId;
}

@Override
public int compareTo(ChunkInfo chunkInfo) {
if(this.startTimeTs > chunkInfo.startTimeTs)
return 1;
else if(this.startTimeTs < chunkInfo.startTimeTs)
return -1;
return 0;
}
}

private final LoaderErrorThrower manifestLoaderErrorThrower;
private final int streamElementIndex;
private final ChunkExtractorWrapper[] extractorWrappers;
Expand All @@ -82,6 +108,9 @@ public SsChunkSource createChunkSource(
private int currentManifestChunkOffset;

@Nullable private IOException fatalError;
private final TreeSet<ChunkInfo> ssChunks = new TreeSet<ChunkInfo>();
@Nullable private final SsMediaSource mediaSource;
private final long tsDeltaUs;

/**
* @param manifestLoaderErrorThrower Throws errors affecting loading of manifests.
Expand All @@ -95,12 +124,14 @@ public DefaultSsChunkSource(
SsManifest manifest,
int streamElementIndex,
TrackSelection trackSelection,
DataSource dataSource) {
DataSource dataSource,
SsMediaSource mediaSource) {
this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
this.manifest = manifest;
this.streamElementIndex = streamElementIndex;
this.trackSelection = trackSelection;
this.dataSource = dataSource;
this.mediaSource = mediaSource;

StreamElement streamElement = manifest.streamElements[streamElementIndex];
extractorWrappers = new ChunkExtractorWrapper[trackSelection.length()];
Expand All @@ -123,7 +154,63 @@ public DefaultSsChunkSource(
/* timestampAdjuster= */ null,
track);
extractorWrappers[i] = new ChunkExtractorWrapper(extractor, streamElement.type, format);
extractor.setSsAtomCallback(this);
}
for(int i=0; i<streamElement.chunkCount; i++) {
ssChunks.add(new ChunkInfo(
streamElement.getStartTime(i),
streamElement.getChunkDuration(i),
i
));
}
//Assume now = lastChunk start + duration
tsDeltaUs =
System.currentTimeMillis()*1000L -
(streamElement.getStartTimeUs(streamElement.chunkCount - 1) +
streamElement.getChunkDurationUs(streamElement.chunkCount - 1));
currentManifestChunkOffset=streamElement.chunkCount;
}

private synchronized boolean clearOldChunks() {
long tsNowUs = System.currentTimeMillis()*1000L - tsDeltaUs;
long tsOld = tsNowUs*10L - manifest.dvrWindowLengthUs*10L;

ArrayList<ChunkInfo> toRemove = new ArrayList<>();
for (ChunkInfo i : ssChunks) {
if (i.startTimeTs < tsOld)
toRemove.add(i);
}
ssChunks.removeAll(toRemove);
return toRemove.isEmpty();
}

private synchronized void updateTimeline() {
ChunkInfo end = ssChunks.last();
ChunkInfo head = ssChunks.first();

long chunksWindowDuration = (end.durationTs + end.startTimeTs - head.startTimeTs)/10L;
long startTime = head.startTimeTs/10L;
long defaultStart = manifest.dvrWindowLengthUs - 10*1000L*1000L;

SinglePeriodTimeline timeline = new SinglePeriodTimeline(C.TIME_UNSET, chunksWindowDuration, startTime,
defaultStart, true /* isSeekable */, true /* isDynamic */, true, null, null);
if(mediaSource != null) {
mediaSource.sourceInfoRefreshed(timeline);
}
}

public synchronized void onTfrfAtom(long start, long duration) {
if(!manifest.isLive) return;

boolean ret =
ssChunks.add(new ChunkInfo(
start,
duration,
currentManifestChunkOffset++));
//If we were already aware of this chunk, don't do anything
if(!ret) return;
clearOldChunks();
updateTimeline();
}

@Override
Expand Down Expand Up @@ -185,6 +272,25 @@ public int getPreferredQueueSize(long playbackPositionUs, List<? extends MediaCh
return trackSelection.evaluateQueueSize(playbackPositionUs, queue);
}

private synchronized ChunkInfo bestChunk(MediaChunk previous, long loadPositionUs) {
// We'll have chunkFloor < chunkCeiling
ChunkInfo chunkCeiling = ssChunks.ceiling(new ChunkInfo(loadPositionUs*10L, 0L, 0));
ChunkInfo chunkFloor = ssChunks.floor(new ChunkInfo(loadPositionUs*10L, 0L, 0));
ChunkInfo chunk = chunkCeiling;
if(chunkFloor == null) return ssChunks.last();
if(chunkCeiling == null) return chunkFloor;

if(previous == null) {
//If it's the first chunk we send, send the closest one
if(Math.abs(chunkCeiling.startTimeTs - loadPositionUs) >
Math.abs(chunkFloor.startTimeTs - loadPositionUs))
chunk = chunkFloor;
else
chunk = chunkCeiling;
}
return chunk;
}

@Override
public final void getNextChunk(
long playbackPositionUs,
Expand All @@ -202,21 +308,12 @@ public final void getNextChunk(
return;
}

int chunkIndex;
if (queue.isEmpty()) {
chunkIndex = streamElement.getChunkIndex(loadPositionUs);
} else {
chunkIndex =
(int) (queue.get(queue.size() - 1).getNextChunkIndex() - currentManifestChunkOffset);
if (chunkIndex < 0) {
// This is before the first chunk in the current manifest.
fatalError = new BehindLiveWindowException();
return;
}
}
clearOldChunks();

if (chunkIndex >= streamElement.chunkCount) {
// This is beyond the last chunk in the current manifest.
MediaChunk previous = queue.isEmpty() ? null : queue.get(queue.size() - 1);
ChunkInfo chunk = bestChunk(previous, loadPositionUs);
if(chunk == null) {
// This is before the first chunk in the current manifest.
out.endOfStream = !manifest.isLive;
return;
}
Expand All @@ -227,28 +324,27 @@ public final void getNextChunk(
MediaChunkIterator[] chunkIterators = new MediaChunkIterator[trackSelection.length()];
for (int i = 0; i < chunkIterators.length; i++) {
int trackIndex = trackSelection.getIndexInTrackGroup(i);
chunkIterators[i] = new StreamElementIterator(streamElement, trackIndex, chunkIndex);
chunkIterators[i] = new StreamElementIterator(streamElement, trackIndex, chunk.chunkId);
}
trackSelection.updateSelectedTrack(
playbackPositionUs, bufferedDurationUs, timeToLiveEdgeUs, queue, chunkIterators);

long chunkStartTimeUs = streamElement.getStartTimeUs(chunkIndex);
long chunkEndTimeUs = chunkStartTimeUs + streamElement.getChunkDurationUs(chunkIndex);
long chunkStartTimeUs = chunk.startTimeTs/10L;
long chunkEndTimeUs = chunkStartTimeUs + chunk.durationTs/10L;
long chunkSeekTimeUs = queue.isEmpty() ? loadPositionUs : C.TIME_UNSET;
int currentAbsoluteChunkIndex = chunkIndex + currentManifestChunkOffset;

int trackSelectionIndex = trackSelection.getSelectedIndex();
ChunkExtractorWrapper extractorWrapper = extractorWrappers[trackSelectionIndex];

int manifestTrackIndex = trackSelection.getIndexInTrackGroup(trackSelectionIndex);
Uri uri = streamElement.buildRequestUri(manifestTrackIndex, chunkIndex);
Uri uri = streamElement.buildRequestUriFromStartTime(manifestTrackIndex, chunk.startTimeTs);

out.chunk =
newMediaChunk(
trackSelection.getSelectedFormat(),
dataSource,
uri,
currentAbsoluteChunkIndex,
chunk.chunkId,
chunkStartTimeUs,
chunkEndTimeUs,
chunkSeekTimeUs,
Expand Down
3 changes: 2 additions & 1 deletion ...ing/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ SsChunkSource createChunkSource(
SsManifest manifest,
int streamElementIndex,
TrackSelection trackSelection,
@Nullable TransferListener transferListener);
@Nullable TransferListener transferListener,
SsMediaSource mediaSource);
}

/**
Expand Down
Loading