Skip to content
Advertisement

Out of order BufferInfo.presentationTimeUs timestamps in mediacodec decoder

I am getting irregular newBufferInfo.presentationTimeUs because of which if I put Thread.sleep to slow down the playback, a lot of frames are dropped.

Actually, with Surface the frames timestamps are synchronized automatically with system timestamp without sleep, however it does not work with giving output to OpenGLES. https://developer.android.com/reference/android/media/MediaCodec#releaseOutputBuffer(int,%20long)

I thought mExtractor.getSampleTime() is the problem but even after removing it, the problem is still there.

package com.example.app;

import android.graphics.PixelFormat;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;

import java.io.IOException;
import java.nio.ByteBuffer;


public final class MediaCodecDecoder {

    private static final String VIDEO = "video/";
    private static final String TAG = "MediaCodecDecoder";
    private final Surface mSurface;
    private final String mClipPath;
    private SurfaceHolder mSurfaceHolder = null;
    private MediaCodec mVideoDecoder;
    private MediaExtractor mExtractor;
    private Boolean mVideoDecoderRunning = false;
    private Thread mVideoDecoderThread;
    private int mDropCount;
    private int mRenderCount;
    private int mFramerate = 30;

    public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {
        this.mClipPath = clipPath;
        this.mSurface = surface;
        this.mSurfaceHolder = surfaceHolder;
    }

    public void start() {
        mDropCount = 0;
        mRenderCount = 0;
        mExtractor = new MediaExtractor();
        try {
            mExtractor.setDataSource(mClipPath);
        } catch (IOException e) {
            e.printStackTrace();
        }
        for (int index = 0; index <= mExtractor.getTrackCount(); index++) {
            MediaFormat format = mExtractor.getTrackFormat(index);
            String mime = format.getString(MediaFormat.KEY_MIME);
            if (mime != null && mime.startsWith(VIDEO)) {
                mExtractor.selectTrack(index);
                try {
                    mVideoDecoder = MediaCodec.createDecoderByType(mime);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                try {
                    Log.i(TAG, "format : " + format);
                    format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);
                    mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
                    mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                break;
            }
        }
        mVideoDecoder.start();
        mVideoDecoderThread = new videoDecoderHandler(false);
        mVideoDecoderThread.start();
    }


    public void stop() {
        mVideoDecoderRunning = false;
        try {
            mVideoDecoderThread.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        // Clear the surface
        if (mSurfaceHolder != null) {
            mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
            mSurfaceHolder.setFormat(PixelFormat.OPAQUE);
        }

        Log.v("Extended Stats", "drop Count: " + mDropCount);
        Log.v("Extended Stats", "rendered Count: " + mRenderCount);
        Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));
    }


    class videoDecoderHandler extends Thread {
        boolean endOfStream;

        public videoDecoderHandler(boolean endOfStream) {
            this.endOfStream = endOfStream;
        }

        //method where the thread execution will start
        public void run() {
            //logic to execute in a thread
            mVideoDecoderRunning = true;
            MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();
            ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();
            ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();
            long startNs = System.nanoTime();
            int generateIndex = 0;

            while (mVideoDecoderRunning) {
                int index = mVideoDecoder.dequeueInputBuffer(1000);
                if (index >= 0) {
                    // fill inputBuffers[inputBufferIndex] with valid data
                    ByteBuffer inputBuffer = inputBuffers[index];
                    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);

                    if (mExtractor.advance() && sampleSize > 0) {
                        Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +
                                mExtractor.getSampleTime() + " PresentationTime " +
                                (startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));
                        // mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
                                computePresentationTime(generateIndex, 30), 0);
                        generateIndex++;
                    } else {
                        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
                        mVideoDecoder.queueInputBuffer(
                                index,
                                0,
                                0,
                                0,
                                MediaCodec.BUFFER_FLAG_END_OF_STREAM
                        );
                    }
                }
                int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);

                switch (outIndex) {
                    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                        Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
                        outputBuffers = mVideoDecoder.getOutputBuffers();
                        break;
                    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                        Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +
                                mVideoDecoder.getOutputFormat());
                        break;
                    case MediaCodec.INFO_TRY_AGAIN_LATER:
                        Log.d(TAG, "INFO_TRY_AGAIN_LATER");
                        break;
                    default:
                        Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +
                                newBufferInfo.presentationTimeUs);
                        boolean render = newBufferInfo.size != 0;
                        long currentNs = System.nanoTime();
                        if (!render) {
                            mVideoDecoder.releaseOutputBuffer(outIndex, false);
                        } else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {
                            mVideoDecoder.releaseOutputBuffer(outIndex, false);
                            mDropCount++; // drop if more than 30ms late
                        } else {
                            try {
                                if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {
                                    Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);
                                }
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            mVideoDecoder.releaseOutputBuffer(outIndex, true);
                            mRenderCount++;
                        }
                }

                // All decoded frames have been rendered, we can stop playing now
                if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                    Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
                    break;
                }

                if (endOfStream) {
                    break;
                }
            }
            mVideoDecoder.stop();
            mVideoDecoder.release();
            mExtractor.release();
        }
    }


    /**
     * Generates the presentation time for frame N, in microseconds.
     */
    private static long computePresentationTime(int frameIndex, int FRAME_RATE) {
        return (132 + Long.valueOf(frameIndex) * 1000000
                / Long.valueOf(FRAME_RATE));
    }

}

This class takes the mp4 video clipPath, and has start and stop functions which start and stop decoding the mp4 video to the passed surface.

Here are the logs

04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 362231984
04-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 361965317
04-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 362265317
04-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 362131984
04-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 362298650
04-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 362098650
04-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 362331984
04-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 362165317
04-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 362365317
04-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 362065317
04-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 362398650
04-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 362265317
04-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 362431984
04-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 362231984
04-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 362465317
04-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650



04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 232
04-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 192
04-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424

Here are the logs with

                         mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
//                        mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
//                                computePresentationTime(generateIndex, 30), 0);
2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 907904455
2021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 907735489
2021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 907937788
2021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 907902323
2021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 907971122
2021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 907802223
2021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 908004455
2021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 907935690
2021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 908037788
2021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 907868956
2021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 908071122
2021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 908002423
2021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 908104455
2021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 908069156
2021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 908137788
2021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 907969056
2021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 908171122
2021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 908135890
2021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 908204455
2021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 908035790
2021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 908237788
2021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 908202623
2021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 908271122
2021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 908102523
2021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 908304455
2021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 908269356
2021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 908337788
2021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 908169256
2021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 908371122
2021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 908336089
2021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 908404455
2021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 908235990
2021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 908437788
2021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 908402823
2021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface

2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 909
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 1044
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953

Here are the logs with

mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);
2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 15515500
2021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 15415400
2021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 15582233
2021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 15482133
2021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 15648966
2021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 15548866
2021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 15682333
2021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 15615600
2021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 15749066
2021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 15815800
2021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 15715700
2021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 15882533
2021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 15782433
2021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 15949266
2021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 15849166
2021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999

By default int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000); should output frames in display order (which is the timestamp order) as I have checked in the documentation but the timestamps are out of order.

For some reason, the video playback seems smooth even with out of order timestamps and I am utterly confused.

Advertisement

Answer

I figured out the issue. It was in dequeueInputBuffer part. I was doing mExtractor.advance() beforehand in if statement.

The correct implementation is as follows

int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
    // fill inputBuffers[inputBufferIndex] with valid data
    ByteBuffer inputBuffer = inputBuffers[index];
    int sampleSize = mExtractor.readSampleData(inputBuffer, 0);

    if (sampleSize >= 0) {
        mVideoDecoder.queueInputBuffer(index, 0, sampleSize,
                mExtractor.getSampleTime(), 0);
    } else {
        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
        mVideoDecoder.queueInputBuffer(index, 0, 0,
                0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
    }
    mExtractor.advance();
}
Advertisement