Out of order BufferInfo.presentationTimeUs timestamps in mediacodec decoder

I am getting irregular newBufferInfo.presentationTimeUs because of which if I put Thread.sleep to slow down the playback, a lot of frames are dropped.

Actually, with Surface the frames timestamps are synchronized automatically with system timestamp without sleep, however it does not work with giving output to OpenGLES. https://developer.android.com/reference/android/media/MediaCodec#releaseOutputBuffer(int,%20long)

I thought mExtractor.getSampleTime() is the problem but even after removing it, the problem is still there.

package com.example.app;

import android.graphics.PixelFormat;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;

import java.io.IOException;
import java.nio.ByteBuffer;


public final class MediaCodecDecoder {

    private static final String VIDEO = "video/";
    private static final String TAG = "MediaCodecDecoder";
    private final Surface mSurface;
    private final String mClipPath;
    private SurfaceHolder mSurfaceHolder = null;
    private MediaCodec mVideoDecoder;
    private MediaExtractor mExtractor;
    private Boolean mVideoDecoderRunning = false;
    private Thread mVideoDecoderThread;
    private int mDropCount;
    private int mRenderCount;
    private int mFramerate = 30;

    public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {
        this.mClipPath = clipPath;
        this.mSurface = surface;
        this.mSurfaceHolder = surfaceHolder;
    }

    public void start() {
        mDropCount = 0;
        mRenderCount = 0;
        mExtractor = new MediaExtractor();
        try {
            mExtractor.setDataSource(mClipPath);
        } catch (IOException e) {
            e.printStackTrace();
        }
        for (int index = 0; index <= mExtractor.getTrackCount(); index++) {
            MediaFormat format = mExtractor.getTrackFormat(index);
            String mime = format.getString(MediaFormat.KEY_MIME);
            if (mime != null && mime.startsWith(VIDEO)) {
                mExtractor.selectTrack(index);
                try {
                    mVideoDecoder = MediaCodec.createDecoderByType(mime);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    Log.i(TAG, "format : " + format);
                    format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);
                    mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
                    mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                break;
            }
        }
        mVideoDecoder.start();
        mVideoDecoderThread = new videoDecoderHandler(false);
        mVideoDecoderThread.start();
    }


    public void stop() {
        mVideoDecoderRunning = false;
        try {
            mVideoDecoderThread.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        // Clear the surface
        if (mSurfaceHolder != null) {
            mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
            mSurfaceHolder.setFormat(PixelFormat.OPAQUE);
        }

        Log.v("Extended Stats", "drop Count: " + mDropCount);
        Log.v("Extended Stats", "rendered Count: " + mRenderCount);
        Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));
    }


    class videoDecoderHandler extends Thread {
        boolean endOfStream;

        public videoDecoderHandler(boolean endOfStream) {
            this.endOfStream = endOfStream;
        }

        //method where the thread execution will start
        public void run() {
            //logic to execute in a thread
            mVideoDecoderRunning = true;
            MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();
            ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();
            ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();
            long startNs = System.nanoTime();
            int generateIndex = 0;

            while (mVideoDecoderRunning) {
                int index = mVideoDecoder.dequeueInputBuffer(1000);
                if (index >= 0) {
                    // fill inputBuffers[inputBufferIndex] with valid data
                    ByteBuffer inputBuffer = inputBuffers[index];
                    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);

                    if (mExtractor.advance() && sampleSize > 0) {
                        Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +
                                mExtractor.getSampleTime() + " PresentationTime " +
                                (startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));
                        // mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
                                computePresentationTime(generateIndex, 30), 0);
                        generateIndex++;
                    } else {
                        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
                        mVideoDecoder.queueInputBuffer(
                                index,
                                0,
                                0,
                                0,
                                MediaCodec.BUFFER_FLAG_END_OF_STREAM
                        );
                    }
                }
                int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);

                switch (outIndex) {
                    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                        Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
                        outputBuffers = mVideoDecoder.getOutputBuffers();
                        break;
                    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                        Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +
                                mVideoDecoder.getOutputFormat());
                        break;
                    case MediaCodec.INFO_TRY_AGAIN_LATER:
                        Log.d(TAG, "INFO_TRY_AGAIN_LATER");
                        break;
                    default:
                        Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +
                                newBufferInfo.presentationTimeUs);
                        boolean render = newBufferInfo.size != 0;
                        long currentNs = System.nanoTime();
                        if (!render) {
                            mVideoDecoder.releaseOutputBuffer(outIndex, false);
                        } else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {
                            mVideoDecoder.releaseOutputBuffer(outIndex, false);
                            mDropCount++; // drop if more than 30ms late
                        } else {
                            try {
                                if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {
                                    Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);
                                }
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            mVideoDecoder.releaseOutputBuffer(outIndex, true);
                            mRenderCount++;
                        }
                }

                // All decoded frames have been rendered, we can stop playing now
                if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                    Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
                    break;
                }

                if (endOfStream) {
                    break;
                }
            }
            mVideoDecoder.stop();
            mVideoDecoder.release();
            mExtractor.release();
        }
    }


    /**
     * Generates the presentation time for frame N, in microseconds.
     */
    private static long computePresentationTime(int frameIndex, int FRAME_RATE) {
        return (132 + Long.valueOf(frameIndex) * 1000000
                / Long.valueOf(FRAME_RATE));
    }

}

JavaScript
​x
 
package com.example.app;​import android.graphics.PixelFormat;import android.media.MediaCodec;import android.media.MediaExtractor;import android.media.MediaFormat;import android.util.Log;import android.view.Surface;import android.view.SurfaceHolder;​import java.io.IOException;import java.nio.ByteBuffer;​​public final class MediaCodecDecoder {​    private static final String VIDEO = "video/";    private static final String TAG = "MediaCodecDecoder";    private final Surface mSurface;    private final String mClipPath;    private SurfaceHolder mSurfaceHolder = null;    private MediaCodec mVideoDecoder;    private MediaExtractor mExtractor;    private Boolean mVideoDecoderRunning = false;    private Thread mVideoDecoderThread;    private int mDropCount;    private int mRenderCount;    private int mFramerate = 30;​    public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {        this.mClipPath = clipPath;        this.mSurface = surface;        this.mSurfaceHolder = surfaceHolder;    }​    public void start() {        mDropCount = 0;        mRenderCount = 0;        mExtractor = new MediaExtractor();        try {            mExtractor.setDataSource(mClipPath);        } catch (IOException e) {            e.printStackTrace();        }        for (int index = 0; index <= mExtractor.getTrackCount(); index++) {            MediaFormat format = mExtractor.getTrackFormat(index);            String mime = format.getString(MediaFormat.KEY_MIME);            if (mime != null && mime.startsWith(VIDEO)) {                mExtractor.selectTrack(index);                try {                    mVideoDecoder = MediaCodec.createDecoderByType(mime);                } catch (IOException e) {                    e.printStackTrace();                }                try {                    Log.i(TAG, "format : " + format);                    format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);                    mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);                    mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);                } catch (Exception e) {                    e.printStackTrace();                }                break;            }        }        mVideoDecoder.start();        mVideoDecoderThread = new videoDecoderHandler(false);        mVideoDecoderThread.start();    }​​    public void stop() {        mVideoDecoderRunning = false;        try {            mVideoDecoderThread.join();        } catch (InterruptedException e) {            e.printStackTrace();        }        // Clear the surface        if (mSurfaceHolder != null) {            mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);            mSurfaceHolder.setFormat(PixelFormat.OPAQUE);        }​        Log.v("Extended Stats", "drop Count: " + mDropCount);        Log.v("Extended Stats", "rendered Count: " + mRenderCount);        Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));    }​​    class videoDecoderHandler extends Thread {        boolean endOfStream;​        public videoDecoderHandler(boolean endOfStream) {            this.endOfStream = endOfStream;        }​        //method where the thread execution will start        public void run() {            //logic to execute in a thread            mVideoDecoderRunning = true;            MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();            ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();            ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();            long startNs = System.nanoTime();            int generateIndex = 0;​            while (mVideoDecoderRunning) {                int index = mVideoDecoder.dequeueInputBuffer(1000);                if (index >= 0) {                    // fill inputBuffers[inputBufferIndex] with valid data                    ByteBuffer inputBuffer = inputBuffers[index];                    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);​                    if (mExtractor.advance() && sampleSize > 0) {                        Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +                                mExtractor.getSampleTime() + " PresentationTime " +                                (startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));                        // mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +                                computePresentationTime(generateIndex, 30), 0);                        generateIndex++;                    } else {                        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");                        mVideoDecoder.queueInputBuffer(                                index,                                0,                                0,                                0,                                MediaCodec.BUFFER_FLAG_END_OF_STREAM                        );                    }                }                int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);​                switch (outIndex) {                    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:                        Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");                        outputBuffers = mVideoDecoder.getOutputBuffers();                        break;                    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:                        Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +                                mVideoDecoder.getOutputFormat());                        break;                    case MediaCodec.INFO_TRY_AGAIN_LATER:                        Log.d(TAG, "INFO_TRY_AGAIN_LATER");                        break;                    default:                        Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +                                newBufferInfo.presentationTimeUs);                        boolean render = newBufferInfo.size != 0;                        long currentNs = System.nanoTime();                        if (!render) {                            mVideoDecoder.releaseOutputBuffer(outIndex, false);                        } else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {                            mVideoDecoder.releaseOutputBuffer(outIndex, false);                            mDropCount++; // drop if more than 30ms late                        } else {                            try {                                if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {                                    Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);                                }                            } catch (InterruptedException e) {                                e.printStackTrace();                            }                            mVideoDecoder.releaseOutputBuffer(outIndex, true);                            mRenderCount++;                        }                }​                // All decoded frames have been rendered, we can stop playing now                if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {                    Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");                    break;                }​                if (endOfStream) {                    break;                }            }            mVideoDecoder.stop();            mVideoDecoder.release();            mExtractor.release();        }    }​​    /**     * Generates the presentation time for frame N, in microseconds.     */    private static long computePresentationTime(int frameIndex, int FRAME_RATE) {        return (132 + Long.valueOf(frameIndex) * 1000000                / Long.valueOf(FRAME_RATE));    }​}​​

This class takes the mp4 video clipPath, and has start and stop functions which start and stop decoding the mp4 video to the passed surface.

Here are the logs

04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 362231984
04-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 361965317
04-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 362265317
04-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 362131984
04-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 362298650
04-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 362098650
04-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 362331984
04-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 362165317
04-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 362365317
04-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 362065317
04-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 362398650
04-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 362265317
04-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 362431984
04-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 362231984
04-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 362465317
04-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650



04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 232
04-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 192
04-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424

JavaScript
 
04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 36223198404-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 36196531704-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 36226531704-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 36213198404-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 36229865004-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 36209865004-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 36233198404-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 36216531704-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 36236531704-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 36206531704-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 36239865004-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 36226531704-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 36243198404-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 36223198404-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 36246531704-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650​​​04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 23204-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 19204-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424​

Here are the logs with

                         mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
//                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
//                                computePresentationTime(generateIndex, 30), 0);

JavaScript
 
                         mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);//                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +//                                computePresentationTime(generateIndex, 30), 0);​

2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 907904455
2021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 907735489
2021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 907937788
2021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 907902323
2021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 907971122
2021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 907802223
2021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 908004455
2021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 907935690
2021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 908037788
2021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 907868956
2021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 908071122
2021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 908002423
2021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 908104455
2021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 908069156
2021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 908137788
2021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 907969056
2021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 908171122
2021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 908135890
2021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 908204455
2021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 908035790
2021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 908237788
2021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 908202623
2021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 908271122
2021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 908102523
2021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 908304455
2021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 908269356
2021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 908337788
2021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 908169256
2021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 908371122
2021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 908336089
2021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 908404455
2021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 908235990
2021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 908437788
2021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 908402823
2021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface

2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 909
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 1044
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953

JavaScript
 
2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 9079044552021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 9077354892021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 9079377882021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 9079023232021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 9079711222021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 9078022232021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 9080044552021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 9079356902021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 9080377882021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 9078689562021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 9080711222021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 9080024232021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 9081044552021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 9080691562021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 9081377882021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 9079690562021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 9081711222021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 9081358902021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 9082044552021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 9080357902021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 9082377882021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 9082026232021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 9082711222021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 9081025232021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 9083044552021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 9082693562021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 9083377882021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 9081692562021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 9083711222021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 9083360892021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 9084044552021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 9082359902021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 9084377882021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 9084028232021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface​2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 9092021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 10442021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953​

Here are the logs with

mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);

JavaScript
 
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);​

2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 15515500
2021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 15415400
2021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 15582233
2021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 15482133
2021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 15648966
2021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 15548866
2021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 15682333
2021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 15615600
2021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 15749066
2021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 15815800
2021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 15715700
2021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 15882533
2021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 15782433
2021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 15949266
2021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 15849166
2021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999

JavaScript
 
2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 155155002021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 154154002021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 155822332021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 154821332021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 156489662021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 155488662021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 156823332021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 156156002021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 157490662021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 158158002021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 157157002021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 158825332021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 157824332021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 159492662021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 158491662021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999​

By default int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000); should output frames in display order (which is the timestamp order) as I have checked in the documentation but the timestamps are out of order.

For some reason, the video playback seems smooth even with out of order timestamps and I am utterly confused.

Answer

I figured out the issue. It was in dequeueInputBuffer part. I was doing mExtractor.advance() beforehand in if statement.

The correct implementation is as follows

int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
    // fill inputBuffers[inputBufferIndex] with valid data
    ByteBuffer inputBuffer = inputBuffers[index];
    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);

    if (sampleSize >= 0) {
        mVideoDecoder.queueInputBuffer(index, 0, sampleSize,
                mExtractor.getSampleTime(), 0);
    } else {
        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
        mVideoDecoder.queueInputBuffer(index, 0, 0,
                0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
    }
    mExtractor.advance();
}

JavaScript
 
int index = mVideoDecoder.dequeueInputBuffer(1000);if (index >= 0) {    // fill inputBuffers[inputBufferIndex] with valid data    ByteBuffer inputBuffer = inputBuffers[index];    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);​    if (sampleSize >= 0) {        mVideoDecoder.queueInputBuffer(index, 0, sampleSize,                mExtractor.getSampleTime(), 0);    } else {        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");        mVideoDecoder.queueInputBuffer(index, 0, 0,                0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);    }    mExtractor.advance();}​

Advertisement

Answer