Native Decoder for raw h264 video file and render frames to a surface

native-player-jni.cpp

/*
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* This is a JNI example where we use native methods to play video
 * using the native AMedia* APIs.
 * See the corresponding Java source file located at:
 *
 *   src/com/example/nativecodec/NativeMedia.java
 *
 * In this example we use assert() for "impossible" error conditions,
 * and explicit handling and recovery for more likely error conditions.
 */

#include <assert.h>
#include <jni.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

#include "Functions.h"

// for __android_log_print(ANDROID_LOG_INFO, "YourApp", "formatted message");
#include <android/log.h>

#define TAG "NativePlayer"
#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)

// for native window JNI
#include <android/native_window_jni.h>
#include <malloc.h>


workerdata data = {-1,-1,NULL, NULL, NULL, false, false, false, false, -1, -1, -1,-1,NULL,-1,false,false};
static mylooper *mlooper = NULL;

bool prepared = false;
clock_t start, end;
double cpu_time_used_by_frame;
double avg_cpu_time_used;
uint32_t *pu32SCCodeValue;
bool formatChanged = false;

int counter = 0;
int64_t startWhenUsec = systemnanotime() / 1000;
int64_t size;
uint8_t *pBuf;
uint8_t *b;
int64_t remainingSize = 0;

uint8_t *bTmp;

int64_t remainingSizeTmp;

int currentSizeTmp;
int64_t  lastTime = 0;

int64_t systemnanotime() {
    timespec now;
    clock_gettime(CLOCK_MONOTONIC, &now);
    return now.tv_sec * 1000000000LL + now.tv_nsec;
}

int64_t previoustimePTSUs = 0;

int64_t offsetPTSUs;


static
void _SCSearch_Init(uint32_t *const pu32SCCodeValue) {
    *pu32SCCodeValue = 0x00FFFFFF;
}

/**
 return next nalu position the pointed byte contains NRI|NaluType
 */
uint8_t *_SCSearch_FindNext(uint8_t *pBuf, const int BufferLength, uint32_t *const pu32SCCodeValue,
                            workerdata *d
) {
    uint8_t *pNextElt = pBuf;
    //if(BufferLength>0) {
    uint8_t *const pBufEnd = pBuf + BufferLength;
    d->currentSize = 0;
    uint32_t u32CodeValue = *pu32SCCodeValue;

    if (1 != (0x00FFFFFF &
              u32CodeValue)) /** this encode is perform in case the start code as been received earlier */
    {
        do {
            u32CodeValue = (u32CodeValue << 8) | *pNextElt;
            pNextElt++;
        } while (
                (1 != (0x00FFFFFF & u32CodeValue))
                && (pNextElt != pBufEnd)
                );
        if (pNextElt == pBufEnd) {
            d->currentSize = pNextElt - pBuf + 4;
            pNextElt = NULL;
            d->lastFrame = true;
        } else {
            d->currentSize = pNextElt - pBuf;
        }
        //  }
        /* update CodeValue */
        //LOGV("input buffer %d", currentSize);
        *pu32SCCodeValue = u32CodeValue;
    }
    return pNextElt;
}
/* swap colors */
//TODO : check if the color must be swaped (Semi-planar to planar) => mediaformat->color
uint8_t *swapSPtoP(uint8_t *yv12bytes, int width, int height) {
    width = width + width % 16;
    height = height + height % 16;
    uint8_t *i420bytes = (uint8_t *) malloc(width * height * 3 / 2);
    uint8_t *v = (uint8_t *) malloc(width / 2 * height / 2);
    for (int i = 0; i < width * height; i++) {
        i420bytes[i] = yv12bytes[i];
    }
    bool isU = true;
    int cpt = 0, cpt2;
    cpt2 = width * height;
    for (int i = width * height;
         i < width * height + 2 * (width / 2 * height / 2); i++) {
        if (isU) {
            i420bytes[cpt2] = yv12bytes[i];
            isU = false;
            cpt2++;
        } else {
            v[cpt] = yv12bytes[i];
            isU = true;
            cpt++;
        }
    }
    cpt = 0;
    for (int i = width * height + (width / 2 * height / 2);
         i < width * height + 2 * (width / 2 * height / 2); i++) {
        i420bytes[i] = v[cpt];
        cpt++;
    }
    if (v != NULL) {
        delete v;
        v = NULL;
    }
    return i420bytes;
}

// force multiple of 16
uint8_t *resize(uint8_t *notMultipleOf16, int width, int height) {
    int width16 = width + width % 16;
    int height16 = height + height % 16;
    uint8_t *multipleOf16 = (uint8_t *) malloc(width16 * height16 * 3 / 2);
    for (int i = 0; i < width * height; i++) {
        multipleOf16[i] = notMultipleOf16[i];
    }
    int cpt;
    for (cpt = width * height; cpt < width16 * height16; cpt++) {
        multipleOf16[cpt] = notMultipleOf16[cpt];
    }
    for (int i = width * height; i < width * height + (width / 2 * height / 2); i++) {
        multipleOf16[cpt] = notMultipleOf16[i];
        cpt++;
    }
    for (cpt = width * height + (width / 2 * height / 2);
         cpt < width16 * height16 + (width16 / 2 * height16 / 2); cpt++) {
        //cpt++;
    }
    for (int i = width * height + (width / 2 * height / 2);
         i < width * height + 2 * (width / 2 * height / 2); i++) {
        multipleOf16[cpt] = notMultipleOf16[i];
        cpt++;
    }
    return multipleOf16;
}


bool prepare(workerdata *d) {

    fseek(d->fd, 0L, SEEK_END);
    size = ftell(d->fd);
    pBuf = new u_int8_t[size];
    read(d->fd0, pBuf, size);

    startWhenUsec = systemnanotime() / 1000;
    pu32SCCodeValue = (uint32_t *) malloc(sizeof(uint32_t *));
    remainingSize = size;
    _SCSearch_Init(pu32SCCodeValue);
    b = _SCSearch_FindNext(pBuf, (int) remainingSize, pu32SCCodeValue, d);
    bTmp = pBuf;
    remainingSizeTmp =
            remainingSize;
    currentSizeTmp = d->currentSize;
    _SCSearch_Init(pu32SCCodeValue);
    _SCSearch_FindNext(b, (int) remainingSizeTmp, pu32SCCodeValue, d);
    remainingSize -= d->currentSize;
    _SCSearch_Init(pu32SCCodeValue);
    _SCSearch_FindNext(b, (int) remainingSize - d->currentSize, pu32SCCodeValue, d);
    uint8_t cptSps = 4, cptPps = 4;
    uint8_t *csd0 = (uint8_t *) malloc(d->currentSize);
    LOGV("LONG2 = %ld", d->currentSize);
    csd0[0] = 0;
    csd0[1] = 0;
    csd0[2] = 0;
    csd0[3] = 1;
    for (int i = 0; i < d->currentSize - 4; i++) {
        csd0[i + 4] = b[i];
        LOGV("%d", b[i]);
        cptSps++;
    }
    _SCSearch_Init(pu32SCCodeValue);
    b = _SCSearch_FindNext(b, (int) remainingSize, pu32SCCodeValue, d);
    remainingSize -= d->currentSize;
    _SCSearch_Init(pu32SCCodeValue);
    _SCSearch_FindNext(b, (int) remainingSize, pu32SCCodeValue, d);
    uint8_t *csd1 = (uint8_t *) malloc(d->currentSize);
    int ppsSize = d->currentSize;
    csd1[0] = 0;
    csd1[1] = 0;
    csd1[2] = 0;
    csd1[3] = 1;
    for (int i = 0; i < d->currentSize - 4; i++) {
        csd1[i + 4] = b[i];
        LOGV("%d", b[i]);
        cptPps++;
    }

    AMediaFormat_setString(d->format, AMEDIAFORMAT_KEY_MIME, "video/avc");
    AMediaFormat_setInt32(d->format, AMEDIAFORMAT_KEY_WIDTH, d->mWidth);
    AMediaFormat_setInt32(d->format, AMEDIAFORMAT_KEY_HEIGHT, d->mHeight);
    AMediaFormat_setBuffer(d->format, "csd-0", csd0, cptSps);
    AMediaFormat_setBuffer(d->format, "csd-1", csd1, cptPps);
    delete csd0;
    delete csd1;
    csd0 = NULL;
    csd1 = NULL;
    d->codec = AMediaCodec_createDecoderByType("video/avc");
    AMediaCodec_configure(d->codec, d->format, d->window, NULL, 0);
    if (AMEDIA_OK == AMediaCodec_start(d->codec))
        LOGV("********** codec %d ok", d->nb);
    AMediaFormat_delete(d->format);
    _SCSearch_Init(pu32SCCodeValue);
    bTmp = _SCSearch_FindNext(b, (int) remainingSize > 0 ? remainingSize : 0,
                                       pu32SCCodeValue,
                                       d);
    remainingSizeTmp =
            remainingSize > d->currentSize ? remainingSize - d->currentSize : remainingSize;
    _SCSearch_Init(pu32SCCodeValue);
    _SCSearch_FindNext(bTmp, (int) remainingSizeTmp, pu32SCCodeValue, d);
    currentSizeTmp = d->currentSize;
    LOGV("%d-------->",d->currentSize);
    uint8_t *tmp= _SCSearch_FindNext(bTmp, (int) remainingSize > 0 ? remainingSize : 0,
                              pu32SCCodeValue,
                              d);
    _SCSearch_Init(pu32SCCodeValue);
    tmp = _SCSearch_FindNext(tmp, (int) remainingSize > 0 ? remainingSize : 0,
                             pu32SCCodeValue,
                             d);
    currentSizeTmp = d->currentSize;
    LOGV("%d-------->",d->currentSize);
    do {
        LOGV("cc ->%d", b[0]);
        _SCSearch_Init(pu32SCCodeValue);
        b = _SCSearch_FindNext(b, (int) remainingSize > 0 ? remainingSize : 0, pu32SCCodeValue,
                               d);
        remainingSize =
                remainingSize > d->currentSize ? remainingSize - d->currentSize : remainingSize;
        _SCSearch_Init(pu32SCCodeValue);
        _SCSearch_FindNext(b, (int) remainingSize, pu32SCCodeValue, d);
        if (formatChanged) {
            AMediaCodec_flush(d->codec);
            d->renderstart = -1;
            d->sawInputEOS = false;
            d->sawOutputEOS = false;
            if (!d->isPlaying) {
                d->renderonce = true;
            }
            b = bTmp;
            remainingSize = remainingSizeTmp;
            d->currentSize = currentSizeTmp;
            return true;
        }
        if (b[0] == 33 || b[0] == 101 || b[0] == 65) {
            uint8_t *buff = (uint8_t *) malloc(d->currentSize);
            buff[0] = 0;
            buff[1] = 0;
            buff[2] = 0;
            buff[3] = 1;
            for (int i = 0; i < d->currentSize - 4; i++) {
                buff[i + 4] = b[i];
            }
            ssize_t bufidx = -1;
            if (!d->sawInputEOS) {
                bufidx = AMediaCodec_dequeueInputBuffer(d->codec, 2000);
                //LOGV("input buffer %zd", bufidx);
                if (bufidx >= 0) {
                    size_t bufsize;
                    auto buf = AMediaCodec_getInputBuffer(d->codec, bufidx, &bufsize);
                    if (d->currentSize == 0) {
                        d->sawInputEOS = true;
                        LOGV("EOS");
                    }
                    auto cc = memcpy(buf, buff, d->currentSize);
                    //LOGV("cc %x",buf[currentSize-1]&0xFF);
                    delete (buff);
                    buff = NULL;
//                    startWhenUsec = systemnanotime()/1000;
                    AMediaCodec_queueInputBuffer(d->codec, bufidx, 0, d->currentSize,
                                                 systemnanotime()/1000,
                                                 d->sawInputEOS
                                                 ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
                }
            }

            if (!d->sawOutputEOS) {
                AMediaCodecBufferInfo info;
                auto status = AMediaCodec_dequeueOutputBuffer(d->codec, &info, 0);
                if (status >= 0) {
                    if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                        LOGV("output EOS");
                        d->sawOutputEOS = true;
                    }

                    AMediaCodec_releaseOutputBuffer(d->codec, status, info.size != 0);
                } else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
                    LOGV("output buffers changed");
                } else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
                    auto format = AMediaCodec_getOutputFormat(d->codec);
                    LOGV("format changed to: %s", AMediaFormat_toString(format));
                    AMediaFormat_delete(format);
                    formatChanged = true;
                } else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
                    LOGV("no output buffer right now");
                } else {
                    LOGV("unexpected info code: %zd", status);
                }
            }
        }
    } while (!d->lastFrame);
    LOGV("Prepared");
    return false;
}


void doCodecWork(workerdata *d) {
    if (!d->sawInputEOS) {
//        if (b[0] == 33 || b[0] == 101 || b[0] == 65) {
            uint8_t *buff = (uint8_t *) malloc(d->currentSize);
            buff[0] = 0;
            buff[1] = 0;
            buff[2] = 0;
            buff[3] = 1;
            for (int i = 0; i < d->currentSize - 4; i++) {
                buff[i + 4] = b[i];
            }
            ssize_t bufidx = -1;
            bufidx = AMediaCodec_dequeueInputBuffer(d->codec, 1000);
            if (bufidx >= 0) {
                size_t bufsize;
                auto buf = AMediaCodec_getInputBuffer(d->codec, bufidx, &bufsize);
                if (d->currentSize == 0) {
                    d->sawInputEOS = true;
                    d->isPlaying = false;
                    LOGV("EOS");
                }
                auto cc = memcpy(buf, buff, d->currentSize);
                LOGV("Buff[4] =%x", buff[4] & 0xFF);
                LOGV("Buff[x] =%x", buff[d->currentSize]-1 & 0xFF);
                //LOGV("cc %x",buf[currentSize-1]&0xFF);
                delete (buff);
                buff = NULL;
                start = clock();
                LOGV("--------------------->%d",buf[4]);
                AMediaCodec_queueInputBuffer(d->codec, bufidx, 0, d->currentSize,
                                             systemnanotime()/1000,
                                             d->sawInputEOS
                                             ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
            }
//        }
        if (!d->sawInputEOS) {
            _SCSearch_Init(pu32SCCodeValue);
            b = _SCSearch_FindNext(b, (int) remainingSize > 0 ? remainingSize : 0, pu32SCCodeValue,
                                   d);
            remainingSize =
                    remainingSize > d->currentSize ? remainingSize - d->currentSize : remainingSize;
            _SCSearch_Init(pu32SCCodeValue);
            _SCSearch_FindNext(b, (int) remainingSize, pu32SCCodeValue, d);
        }
    }

    if (!d->sawOutputEOS) {
        AMediaCodecBufferInfo info;
        auto status = AMediaCodec_dequeueOutputBuffer(d->codec, &info, 0);
        if (status >= 0) {
            if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                LOGV("output EOS");
                d->sawOutputEOS = true;
                d->isPlaying = false;
            }
            int64_t delay = 333000;
            if (delay > 0) {
                usleep(delay/15/* 15 frame per second */);
            }
            previoustimePTSUs = info.presentationTimeUs;
            LOGV("FRAME num : %d",counter++);
            AMediaCodec_releaseOutputBuffer(d->codec, status, info.size != 0);
            if (d->renderonce) {
                d->renderonce = false;
                return;
            }
        } else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
            LOGV("output buffers changed");
        } else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
            auto format = AMediaCodec_getOutputFormat(d->codec);
            LOGV("format changed to: %s", AMediaFormat_toString(format));
            AMediaFormat_delete(format);
            formatChanged = true;
        } else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
            LOGV("no output buffer right now");
        } else {
            LOGV("unexpected info code: %zd", status);
        }
    }

    if (!d->sawInputEOS || !d->sawOutputEOS) {
        mlooper->
                post(kMsgCodecBuffer, d
        );
    }
}

void mylooper::handle(int what, void *obj) {
    switch (what) {
        case kMsgCodecBuffer:
            doCodecWork((workerdata *) obj);
            break;

        case kMsgDecodeDone: {
            workerdata *d = (workerdata *) obj;
            AMediaCodec_stop(d->codec);
            AMediaCodec_delete(d->codec);
            d->sawInputEOS = true;
            d->sawOutputEOS = true;
        }
            break;

        case kMsgSeek: {
            workerdata *d = (workerdata *) obj;
            AMediaCodec_flush(d->codec);
            d->renderstart = -1;
            d->sawInputEOS = false;
            d->sawOutputEOS = false;
            if (!d->isPlaying) {
                d->renderonce = true;
                //post(kMsgCodecBuffer, d);
            }
            b = bTmp;
            remainingSize = remainingSizeTmp;
            d->currentSize = currentSizeTmp;
            LOGV("seeked");
        }
            break;

        case kMsgPause: {
            workerdata *d = (workerdata *) obj;
            if (d->isPlaying) {
                // flush all outstanding codecbuffer messages with a no-op message
                d->isPlaying = false;
                lastTime = systemnanotime()/1000;
                post(kMsgPauseAck, NULL, true);
            }
        }
            break;

        case kMsgResume: {
            workerdata *d = (workerdata *) obj;
            if (!d->isPlaying) {
                d->renderstart = -1;
                d->isPlaying = true;
                if(lastTime!=0){
                    offsetPTSUs = systemnanotime()/1000 - lastTime;
                    lastTime = 0;
                }
                post(kMsgCodecBuffer, d);
            }
        }
            break;

        case kMsgPrepare: {
            workerdata *d = (workerdata *) obj;
            while (!prepared) {
                prepared = prepare(d);
            }
        }
            break;
    }
}

extern "C"
{

jboolean
Java_com_example_nativecodec_Player_PlayerActivity_createStreamingMediaPlayer(JNIEnv *env,
                                                                              jclass clazz,
                                                                              jstring filename) {
    LOGV("@@@ create");

    // convert Java string to UTF-8
    const char *utf8 = env->GetStringUTFChars(filename, NULL);
    LOGV("opening %s", utf8);
    int fd0 = open(utf8, O_RDWR, 0664);

    env->ReleaseStringUTFChars(filename, utf8);
    if (fd0 < 0) {
        LOGE("failed to open file: %s %d (%s)", utf8, fd0, strerror(errno));
        return JNI_FALSE;
    }

    workerdata *d = &data;

    FILE *file = fopen(utf8, "r+");

    //unsigned char buffer[mWidth * mHeight * 3 / 2];  // array of bytes, not pointers-to-bytes
    size_t bytesRead = 0;


    const char *mime = "video/avc";
    AMediaCodec *codec = NULL;
    d->format = AMediaFormat_new();
    if (!d->format)
        return JNI_FALSE;
    d->codec = codec;
    d->fd0 = fd0;
    d->encoder = false;
    d->fd = file;
    d->mHeight = 368;
    d->mWidth = 640;


    //close(d->fd0);

    d->renderstart = -1;
    d->sawInputEOS = false;
    d->sawOutputEOS = false;
    d->isPlaying = false;
    d->renderonce = true;

    mlooper = new mylooper();
    mlooper->post(kMsgPrepare,d);
    mlooper->post(kMsgCodecBuffer, d);

    return JNI_TRUE;
}

// set the playing state for the streaming media player
void
Java_com_example_nativecodec_Player_PlayerActivity_setPlayingStreamingMediaPlayer(JNIEnv *env,
                                                                                  jclass clazz,
                                                                                  jboolean isPlaying) {
    LOGV("@@@ playpause: %d", isPlaying);
    if (mlooper) {
        if (isPlaying) {
            mlooper->post(kMsgResume, &data);
        } else {
            mlooper->post(kMsgPause, &data);
        }
    }
}


// shut down the native media system
void Java_com_example_nativecodec_Player_PlayerActivity_shutdown(JNIEnv *env, jclass clazz) {
    LOGV("@@@ shutdown");
    if (mlooper) {
        mlooper->post(kMsgDecodeDone, &data, true /* flush */);
        mlooper->quit();
        delete mlooper;
        mlooper = NULL;
    }
    if (data.window) {
        ANativeWindow_release(data.window);
        data.window = NULL;
    }
    if(pBuf != NULL){
        delete pBuf;
        pBuf = NULL;
    }
}


// set the surface
void
Java_com_example_nativecodec_Player_PlayerActivity_setSurface(JNIEnv *env, jclass clazz,
                                                              jobject surface) {
    // obtain a native window from a Java surface
    if (data.window) {
        ANativeWindow_release(data.window);
        data.window = NULL;
    }
    data.window = ANativeWindow_fromSurface(env, surface);
    LOGV("@@@ setsurface %p", data.window);
}


// rewind the streaming media player
void
Java_com_example_nativecodec_Player_PlayerActivity_rewindStreamingMediaPlayer(JNIEnv *env,
                                                                              jclass clazz) {
    LOGV("@@@ rewind");
    if (mlooper) {
        data.isPlaying = false;
        mlooper->post(kMsgSeek, &data);
        counter = 0;
    }
}

}