I am getting irregular newBufferInfo.presentationTimeUs because of which if I put Thread.sleep to slow down the playback, a lot of frames are dropped.
Actually, with Surface the frames timestamps are synchronized automatically with system timestamp without sleep, however it does not work with giving output to OpenGLES. https://developer.android.com/reference/android/media/MediaCodec#releaseOutputBuffer(int,%20long)
I thought mExtractor.getSampleTime() is the problem but even after removing it, the problem is still there.
package com.example.app;
import android.graphics.PixelFormat;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import java.io.IOException;
import java.nio.ByteBuffer;
public final class MediaCodecDecoder {
private static final String VIDEO = "video/";
private static final String TAG = "MediaCodecDecoder";
private final Surface mSurface;
private final String mClipPath;
private SurfaceHolder mSurfaceHolder = null;
private MediaCodec mVideoDecoder;
private MediaExtractor mExtractor;
private Boolean mVideoDecoderRunning = false;
private Thread mVideoDecoderThread;
private int mDropCount;
private int mRenderCount;
private int mFramerate = 30;
public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {
this.mClipPath = clipPath;
this.mSurface = surface;
this.mSurfaceHolder = surfaceHolder;
}
public void start() {
mDropCount = 0;
mRenderCount = 0;
mExtractor = new MediaExtractor();
try {
mExtractor.setDataSource(mClipPath);
} catch (IOException e) {
e.printStackTrace();
}
for (int index = 0; index <= mExtractor.getTrackCount(); index++) {
MediaFormat format = mExtractor.getTrackFormat(index);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith(VIDEO)) {
mExtractor.selectTrack(index);
try {
mVideoDecoder = MediaCodec.createDecoderByType(mime);
} catch (IOException e) {
e.printStackTrace();
}
try {
Log.i(TAG, "format : " + format);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);
mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);
} catch (Exception e) {
e.printStackTrace();
}
break;
}
}
mVideoDecoder.start();
mVideoDecoderThread = new videoDecoderHandler(false);
mVideoDecoderThread.start();
}
public void stop() {
mVideoDecoderRunning = false;
try {
mVideoDecoderThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
// Clear the surface
if (mSurfaceHolder != null) {
mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
mSurfaceHolder.setFormat(PixelFormat.OPAQUE);
}
Log.v("Extended Stats", "drop Count: " + mDropCount);
Log.v("Extended Stats", "rendered Count: " + mRenderCount);
Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));
}
class videoDecoderHandler extends Thread {
boolean endOfStream;
public videoDecoderHandler(boolean endOfStream) {
this.endOfStream = endOfStream;
}
//method where the thread execution will start
public void run() {
//logic to execute in a thread
mVideoDecoderRunning = true;
MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();
ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();
long startNs = System.nanoTime();
int generateIndex = 0;
while (mVideoDecoderRunning) {
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +
mExtractor.getSampleTime() + " PresentationTime " +
(startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));
// mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
computePresentationTime(generateIndex, 30), 0);
generateIndex++;
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(
index,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM
);
}
}
int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = mVideoDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +
mVideoDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(TAG, "INFO_TRY_AGAIN_LATER");
break;
default:
Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +
newBufferInfo.presentationTimeUs);
boolean render = newBufferInfo.size != 0;
long currentNs = System.nanoTime();
if (!render) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
} else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
mDropCount++; // drop if more than 30ms late
} else {
try {
if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {
Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
mVideoDecoder.releaseOutputBuffer(outIndex, true);
mRenderCount++;
}
}
// All decoded frames have been rendered, we can stop playing now
if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
if (endOfStream) {
break;
}
}
mVideoDecoder.stop();
mVideoDecoder.release();
mExtractor.release();
}
}
/**
* Generates the presentation time for frame N, in microseconds.
*/
private static long computePresentationTime(int frameIndex, int FRAME_RATE) {
return (132 + Long.valueOf(frameIndex) * 1000000
/ Long.valueOf(FRAME_RATE));
}
}
This class takes the mp4 video clipPath, and has start and stop functions which start and stop decoding the mp4 video to the passed surface.
Here are the logs
04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 362231984 04-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 361965317 04-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 362265317 04-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 362131984 04-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected 04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 362298650 04-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 362098650 04-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 362331984 04-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected 04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 362165317 04-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected 04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 362365317 04-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 362065317 04-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 362398650 04-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 362265317 04-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected 04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 362431984 04-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 362231984 04-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 362465317 04-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650 04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 232 04-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 192 04-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424
Here are the logs with
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0); // mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + // computePresentationTime(generateIndex, 30), 0);
2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 907904455 2021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 907735489 2021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 907937788 2021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 907902323 2021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 907971122 2021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 907802223 2021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 908004455 2021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 907935690 2021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 908037788 2021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 907868956 2021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 908071122 2021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 908002423 2021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 908104455 2021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 908069156 2021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 908137788 2021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 907969056 2021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 908171122 2021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 908135890 2021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 908204455 2021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 908035790 2021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 908237788 2021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 908202623 2021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 908271122 2021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 908102523 2021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 908304455 2021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 908269356 2021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 908337788 2021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 908169256 2021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 908371122 2021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 908336089 2021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 908404455 2021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 908235990 2021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 908437788 2021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 908402823 2021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface 2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 909 2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 1044 2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953
Here are the logs with
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);
2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 15515500 2021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 15415400 2021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 15582233 2021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 15482133 2021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 15648966 2021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 15548866 2021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 15682333 2021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 15615600 2021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 15749066 2021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 15815800 2021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 15715700 2021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 15882533 2021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 15782433 2021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 15949266 2021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 15849166 2021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999
By default int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000); should output frames in display order (which is the timestamp order) as I have checked in the documentation but the timestamps are out of order.
For some reason, the video playback seems smooth even with out of order timestamps and I am utterly confused.
Advertisement
Answer
I figured out the issue. It was in dequeueInputBuffer part.
I was doing mExtractor.advance() beforehand in if statement.
The correct implementation is as follows
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (sampleSize >= 0) {
mVideoDecoder.queueInputBuffer(index, 0, sampleSize,
mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(index, 0, 0,
0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
mExtractor.advance();
}