decode a video with libav/ffmpeg

CSCubeBGRA.cc

#include "CSCubeBGRA.h""

#include <sstream>
#include <cmath>

int getNearestIndexForTime(const vector<double> &times, double rTime)
{
    // binary search
    // returns an iterator pointing to the first element in the range [first,last) which does not compare less than val.
    auto pFrameIndex = lower_bound(times.begin(),times.end(),rTime);
    int iframe = (pFrameIndex - times.begin());
    if (pFrameIndex != times.begin()) {
        auto pPrevIndex = pFrameIndex-1;
        // closer to previous value?
        if (fabs(*pPrevIndex - rTime) < fabs(*pFrameIndex - rTime) ) {
            iframe--;
        }
    }
    return iframe;

}


CSCubeBGRA::CSCubeBGRA(int width, int height, int nframes, float fps) : m_Width(0), m_Height(0), m_NFrames(0), m_FPS(24) 
{
    Set(width,height,nframes,fps);
}

CSCubeBGRA::CSCubeBGRA(int width, int height, int nframes, const unsigned char *data, float fps)  
    : m_Width(0), m_Height(0), m_NFrames(0), m_FPS(24) 
{   
    Set(width,height,nframes,data,fps);
}

void CSCubeBGRA::Set(int width, int height, int nframes, float fps) 
{
    m_FPS = fps;
    if ( (m_Width != width) || (m_Height != height) || (m_NFrames != nframes)) {
        m_Width = width;
        m_Height = height;
        // cout << "CSCubeBGRA::Set m_NFrames, nframes " << m_NFrames << "," << nframes << endl;
        if (m_NFrames != nframes) {
            m_FrameTimes.resize((unsigned long)nframes);
        }
        m_NFrames = nframes;
        // cout << "width height frames " << width << " " << height << " " << nframes << " m_Width,m_height,m_frames ";
        // cout << m_Width << "," << m_Height << "," << m_NFrames << " total bytes needed " << (unsigned long)m_Width*m_Height*m_NFrames*4 << endl;
        m_Data.resize((unsigned long)m_Width*m_Height*m_NFrames*4);
    }
}

void CSCubeBGRA::Set(int width, int height, int nframes, const unsigned char *data, float fps) 
{
    Set(width,height,nframes,fps);
    unsigned long nvals = (unsigned long)width*height*nframes*4;
    memcpy(pData(),data,nvals * sizeof(unsigned char));
}

void CSCubeBGRA::SetFrame(int iframe, const unsigned char *data) 
{
    long nvals = m_Width*m_Height*4;
    // cout << "CSCubeBGRA::SetFrame iframe " << iframe << endl;
    // cout << "CSCubeBGRA::SetFrame pFrameData(iframe) " << (void *)pFrameData(iframe) << endl;
    // cout << "CSCubeBGRA::SetFrame data " << (void *)data << endl;
    memcpy(pFrameData(iframe),data,nvals * sizeof(unsigned char));   
}

unsigned char *CSCubeBGRA::pData() 
{
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pData.ccode"); 
    
    if (!m_Width || !m_Height || !m_NFrames) {
        stringstream emsg;
        emsg << "attempting to get null ptr from pData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }

    return &(m_Data[0]); 
};

const unsigned char *CSCubeBGRA::pDataConst()  
{ 
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pDataConst.ccode"); 
    
    if (!m_Width || !m_Height || !m_NFrames) {
        stringstream emsg;
        emsg << "attempting to get null ptr from pData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    return &(m_Data[0]); 
};

unsigned char *CSCubeBGRA::pFrameData(int iframe) 
{
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pFrameData.ccode"); 
    
    if ( !m_Width || !m_Height || !m_NFrames || iframe >= m_NFrames ) {
        stringstream emsg;
        emsg << "attempting to get null ptr, or frame beyond nframes " << iframe << " from pFrameData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    // cout << "CSCubeBGRA::pFrameData sanity check original mem " << (void *) &(m_Data[(unsigned long)m_Width*m_Height*4*iframe]) << " new mem " << (void *) ( pData()+m_Width*m_Height*4*iframe ) << endl;
    return pData() + (unsigned long)m_Width*m_Height*4*iframe; 
};

const unsigned char *CSCubeBGRA::pFrameDataConst(int iframe) 
{
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pFrameDataConst.ccode"); 
    if ( !m_Width || !m_Height || !m_NFrames || iframe >= m_NFrames ) {
        stringstream emsg;
        emsg << "attempting to get null ptr, or frame beyond nframes " << iframe << " from pFrameData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    return pData() + (unsigned long)m_Width*m_Height*4*iframe; 
};

int CSCubeBGRA::getFrameForTime(double frameTime)
{
    return getNearestIndexForTime(m_FrameTimes,frameTime);
}

unsigned char *CSCubeBGRA::pFrameDataForTime(double frameTime) 
{
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pFrameDataForTime.ccode"); 
    int iframe = getFrameForTime(frameTime);
    
    if ( !m_Width || !m_Height || !m_NFrames || iframe >= m_NFrames ) {
        stringstream emsg;
        emsg << "attempting to get null ptr, or frame beyond nframes " << iframe << " from pFrameData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    // cout << "CSCubeBGRA::pFrameData sanity check original mem " << (void *) &(m_Data[(unsigned long)m_Width*m_Height*4*iframe]) << " new mem " << (void *) ( pData()+m_Width*m_Height*4*iframe ) << endl;
    return pData() + (unsigned long)m_Width*m_Height*4*iframe; 
};

const unsigned char *CSCubeBGRA::pFrameDataForTimeConst(double frameTime) 
{
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSCubeBGRA.pFrameDataForTimeConst.ccode");
    int iframe = getFrameForTime(frameTime);

    if ( !m_Width || !m_Height || !m_NFrames || iframe >= m_NFrames ) {
        stringstream emsg;
        emsg << "attempting to get null ptr, or frame beyond nframes " << iframe << " from pFrameData width,height,nframes " << m_Width << " , " << m_Height << "," << m_NFrames;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    return pData() + (unsigned long)m_Width*m_Height*4*iframe; 
};


void CSCubeBGRA::convertToFrames(vector<CSFrameBGRA> &frames)
{
    frames.resize(0);

    for (int i=0;i < m_NFrames;i++) {
        frames.push_back( CSFrameBGRA(m_Width,m_Height,pFrameData(i))  );
    }
}

CSCubeBGRA.h

#ifndef CS_CUBE_BGRA_H
#define CS_CUBE_BGRA_H

#include <vector>
#include <sstream>
#include "CSFrameBGRA.h"

using namespace std;

int getNearestIndexForTime(const vector<double> &times, double frameTime);


class CSCubeBGRA {
    public:
        CSCubeBGRA() :  m_Width(0), m_Height(0), m_NFrames(0), m_FPS(24) {};
        CSCubeBGRA(int width, int height, int nframes, float fps);
        CSCubeBGRA(int width, int height, int nframes, const unsigned char *data, float fps);
        virtual ~CSCubeBGRA() {};
        virtual void Set(int width, int height, int nframes, float fps);
        virtual void Set(int width, int height, int nframes, const unsigned char *data, float fps);
        virtual void SetFrame(int iframe, const unsigned char *data);

        virtual unsigned char *pData();
        virtual const unsigned char *pDataConst();
        virtual unsigned char *pFrameData(int iframe);
        virtual const unsigned char *pFrameDataConst(int iframe);
        virtual unsigned char *pFrameDataForTime(double iFrameTime);
        virtual const unsigned char *pFrameDataForTimeConst(double iFrameTime);

        int getFrameForTime(double frameTime);
        virtual void convertToFrames(vector<CSFrameBGRA> &frames);

        int m_Width;
        int m_Height;
        int m_NFrames;
        vector<unsigned char> m_Data;
        vector<double> m_FrameTimes;
        float m_FPS;
};


#endif

CSFrameBGRA.cc

#include <stdio.h>
#include <sstream>
#include "CSFrameBGRA.h"


CSFrameBGRA::CSFrameBGRA(int width, int height) : m_Width(0), m_Height(0) {
    Set(width,height);
}

CSFrameBGRA::CSFrameBGRA(int width, int height, const unsigned char *data) : m_Width(0), m_Height(0) {
    Set(width,height,data);
}


void CSFrameBGRA::Set(int width, int height) {
    if ( (m_Width != width) || (m_Height != height) ) {
        m_Width = width;
        m_Height = height;
        try
        {
            m_Data.resize(m_Width*m_Height*4);
        }
        catch (std::bad_alloc &ba)
        {
            cout << "caught std:bad_alloc for allocation of " << width << "x" << height << "x4 frame (" << width*height*4 << " bytes)" << endl;
            throw;
        }
    }
}

void CSFrameBGRA::Set(int width, int height, const unsigned char *data) {
    Set(width,height);
    long nvals = width*height*4;
    memcpy(pData(),data,(size_t) nvals * sizeof(unsigned char));
}

unsigned char *CSFrameBGRA::pData() { 
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSFrameBGRA.pData.ccode"); 
    
    if (!m_Width || !m_Height) {
        stringstream emsg;
        emsg << "attempting to get null ptr from pData width,height " << m_Width << " , " << m_Height;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    return &(m_Data[0]); 
};

const unsigned char *CSFrameBGRA::pDataConst() const { 
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSFrameBGRA.pData.ccode");     
    if (!m_Width || !m_Height) {
        stringstream emsg;
        emsg << "attempting to get null ptr from pData width,height " << m_Width << " , " << m_Height;
        cout << emsg.str() << endl;
        throw emsg.str();    
    }
    return &(m_Data[0]); 
};




void CSFrameBGRA::extract(int top, int left, int width, int height, CSFrameBGRA &oFrame) const {
    LogStream mout(LOG_DEBUG,"cameo-schneider.CSFrameBGRA.extract.ccode"); 


    oFrame.Set(width,height);

    unsigned int *pOut = (unsigned int *)oFrame.pData();
    unsigned int *pIn = (unsigned int *)pDataConst();

    int k,l, kend = top + height, lend = left + width;

    for (k=top;k < kend;k++) {

        int rfo = k * m_Width;
        int rso = (k - top) * width;

        for (l=left;l < lend;l++) {

            int cfo = l;
            int cso = l - left;

            // mout << "rfo+cfo " << rfo+cfo << " big frame pixel size 0- " << m_Data.size()/4 << LogStream::endl;
            // mout << "rso+cso " << rso+cso << " ext frame pixel size 0- " << oFrame.m_Data.size()/4 << LogStream::endl;
            pOut[rso + cso] = pIn[rfo + cfo];
        }
    }
}

CSFrameBGRA.h

#ifndef CS_FRAME_BGRA_H
#define CS_FRAME_BGRA_H

#include <vector>
#include <cstring>

using namespace std;

class CSFrameBGRA {
    public:
        CSFrameBGRA() :  m_Width(0), m_Height(0) {};
        CSFrameBGRA(int width, int height);
        CSFrameBGRA(int width, int height, const unsigned char *data);

        virtual ~CSFrameBGRA() {};

        virtual void Set(int width, int height);
        virtual void Set(int width, int height, const unsigned char *data);
        virtual unsigned char *pData();
        virtual const unsigned char *pDataConst() const;

        virtual void extract(int top, int left, int width, int height, CSFrameBGRA &oFrame) const;

        int m_Width;
        int m_Height;
        vector<unsigned char> m_Data;
};


#endif

SchneiderVideo.cc

SchneiderVideoAV::SchneiderVideoAV()
{
    m_Good          = 0;

    m_pFormatCtx    = NULL;
    m_pCodecCtx     = NULL;
    m_pCodec        = NULL;
    m_pFrame        = NULL; 
    m_pFrameBGRA    = NULL;
    m_pBuffer       = NULL;
    m_pOptionsDict  = NULL;
    m_pSWSCtx       = NULL;
    m_VideoStream   = -1;

    m_Width = m_Height = m_MaxFrames = m_numBytes = 0;
}


SchneiderVideoAV::SchneiderVideoAV(const string &fileName)
{
    initialize(fileName);
}


SchneiderVideoAV::~SchneiderVideoAV()
{
    cleanUp();
}

void SchneiderVideoAV::cleanUp()
{
    // Register all formats and codecs
    av_register_all();     

    // Free the RGB image
    if (m_pBuffer) av_free(m_pBuffer);
    if (m_pFrameBGRA) av_free(m_pFrameBGRA);
    
    // Free the YUV? frame
    if (m_pFrame) av_free(m_pFrame);
    
    // Close the codec
    if (m_pCodecCtx) avcodec_close(m_pCodecCtx);
    
    // Close the video file
    if (m_pFormatCtx) avformat_close_input(&m_pFormatCtx);

    // free dictionary
    if (m_pOptionsDict) av_dict_free(&m_pOptionsDict);

    m_pBuffer       = NULL;
    m_pFrameBGRA    = NULL;
    m_pFrame        = NULL; 
    m_pCodecCtx     = NULL;
    m_pFormatCtx    = NULL;
    m_pOptionsDict  = NULL;

    m_Width = m_Height = m_MaxFrames = m_numBytes = 0;    
    
}

int SchneiderVideoAV::initialize(const string &fileName)
{
    LogStream mout(LOG_DEBUG,"SchneiderVideoAV.initialize.ccode");

    m_FileName          = fileName;
    m_Good              = 1;

    m_pFormatCtx        = NULL;
    m_pCodecCtx         = NULL;
    m_pCodec            = NULL;
    m_pFrame            = NULL; 
    m_pFrameBGRA        = NULL;
    m_pBuffer           = NULL;
    m_pOptionsDict      = NULL;
    m_pSWSCtx           = NULL;
    m_VideoStream       = -1;
    m_RotationDegrees   = 0.0;

    m_Width = m_Height = m_MaxFrames = m_numBytes = 0;


    // Register all formats and codecs
    av_register_all();
    
    // Open video file
    if(avformat_open_input(&m_pFormatCtx, m_FileName.c_str(), NULL, NULL)!=0) {
        m_Good = 0;
        return VID_PROCESS_FAIL; // Couldn't open file
    }
    
    // Retrieve stream information
    if(avformat_find_stream_info(m_pFormatCtx, NULL)<0) {
        m_Good = 0;
        return VID_PROCESS_FAIL; // Couldn't find stream information
    }
    
    // Dump information about file onto standard error
    int64_t durationInt = m_pFormatCtx->duration;
    double durationSeconds = (double)durationInt / AV_TIME_BASE;

    // av_dump_format(m_pFormatCtx, 0, m_FileName.c_str(), 0);
    
    int i = 0;

    // Find the first video stream
    for(i=0; i< (int) m_pFormatCtx->nb_streams; i++) {
        // cout << "checking streams for video " << m_pFormatCtx->streams[i]->codec->codec_type;
        // cout << " av media type " << AVMEDIA_TYPE_VIDEO << endl;
        if(m_pFormatCtx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO) {
            if (m_VideoStream == -1) {
                m_VideoStream=i;
                // av_dump_format(m_pFormatCtx, 0, m_FileName.c_str(), 0);
                AVDictionaryEntry *pDictEntry = av_dict_get(m_pFormatCtx->streams[i]->metadata, "rotate", NULL, 0);
                if (pDictEntry) m_RotationDegrees = atof(pDictEntry->value);
                // cout << "Dict_entry for rotate from metatdata in stream " << pDictEntry->value << " rotation degrees " << m_RotationDegrees << endl;
            }
            if (VERBOSE_LOGGING) mout << "found a video stream " << i << LogStream::endl;
            // break;
        }
    }
    if(m_VideoStream==-1) {
        m_Good = 0;
        return VID_PROCESS_FAIL; // Didn't find a video stream
    }

    // Get a pointer to the codec context for the video stream
    m_pCodecCtx=m_pFormatCtx->streams[m_VideoStream]->codec;
    m_FPS = av_q2d(m_pFormatCtx->streams[m_VideoStream]->avg_frame_rate);
    // cout << "duration seconds " << durationSeconds << " frame # ";
    
    int frameCount = 0;
    if (m_pFormatCtx->streams[m_VideoStream]->nb_frames > 0) {
        frameCount = m_pFormatCtx->streams[m_VideoStream]->nb_frames;
    }
    else {
        frameCount = floor(durationSeconds * m_FPS);
    } 
    // cout << "frameCount " << frameCount << endl;

    m_TimeBase = (int64_t(m_pCodecCtx->time_base.num) * AV_TIME_BASE) / int64_t(m_pCodecCtx->time_base.den);

    m_Width     = m_pCodecCtx->width;
    m_Height    = m_pCodecCtx->height;
    m_MaxFrames = frameCount;

    // cout << " avg fps " << m_FPS << endl;
    
    // Find the decoder for the video stream
    AVPixelFormat usedPixelFormat;
    if (m_pCodecCtx->codec_id == CODEC_ID_H264) {
        m_pCodec=avcodec_find_decoder_by_name("h264_vda");
        usedPixelFormat = AV_PIX_FMT_VDA;
        m_pCodecCtx->pix_fmt = usedPixelFormat;
    }
    else {
        m_pCodec=avcodec_find_decoder(m_pCodecCtx->codec_id);
        usedPixelFormat = m_pCodecCtx->pix_fmt;
    }

    // ignore this if you are not messing with hwaccel decoding
    cout << "about to call ff_find_hwaccel codec, id, pix_fmt name, value " << m_pCodec->name << "," << m_pCodecCtx->codec_id;
    cout << "," << pixFmt2String(usedPixelFormat) << "," << usedPixelFormat << endl;
    m_pCodecCtx->hwaccel = ff_find_hwaccel(m_pCodecCtx->codec_id,usedPixelFormat);

    if(m_pCodec==NULL) {
        mout.setLog(LOG_ERR);
        mout << "SchneiderVideoAV::initialize Unsupported codec!" << LogStream::endl;
        mout.setLog(LOG_DEBUG);
        m_Good = 0;
        return VID_PROCESS_FAIL; // Codec not found
    }


    // Open codec
    if(avcodec_open2(m_pCodecCtx, m_pCodec, &m_pOptionsDict)<0) {
        mout.setLog(LOG_ERR);
        mout << "SchneiderVideoAV::initialize Unable to open codec" << LogStream::endl;
        mout.setLog(LOG_DEBUG);
        m_Good = 0;        
        return VID_PROCESS_FAIL; // Could not open codec
    }
    
    // Allocate video frame
    m_pFrame=av_frame_alloc();
    
    // Allocate an AVFrame structure
    m_pFrameBGRA=av_frame_alloc();
    if(m_pFrameBGRA==NULL) {
        m_Good = 0;        
        return VID_PROCESS_FAIL;
    }
    

    // Determine required buffer size and allocate buffer
    m_numBytes=avpicture_get_size(PIX_FMT_BGRA, m_pCodecCtx->width,
                                    m_pCodecCtx->height);

    m_pBuffer=(uint8_t *)av_malloc(m_numBytes*sizeof(uint8_t));


    if (VERBOSE_LOGGING) mout << "ZOMG right before m_pSWSCtx, and sws_getContext m_pCodecCtx width, height, FrameCount " << m_pCodecCtx->width << " , " << m_pCodecCtx->height << " pix_fmt " << m_pCodecCtx->pix_fmt << LogStream::endl;
    if (m_pCodecCtx->hwaccel == NULL) {
        m_pSWSCtx = sws_getContext
            (
                m_pCodecCtx->width,
                m_pCodecCtx->height,
                m_pCodecCtx->pix_fmt,
                // usedPixelFormat,
                m_pCodecCtx->width,
                m_pCodecCtx->height,
                PIX_FMT_BGRA,
                SWS_BILINEAR,
                NULL,
                NULL,
                NULL
            );
    }

    // cout << "m_pCodecCtx width,height pix_fmt " << m_pCodecCtx->width;
    // cout << "," << m_pCodecCtx->height << " pix_fmt " << m_pCodecCtx->pix_fmt;
    // cout << " PIX_FMT_BGRA " << PIX_FMT_BGRA << endl; 


    VIDEO_IO_MUTEX.unlock();

    return VID_PROCESS_OK;
}

int SchneiderVideoAV::loadCube(CSCubeBGRA *pCube, int frameStart, int frameEnd)
{
    // cout << "SchneiderVideoAV::loadCube frameStart,frameEnd " << frameStart << "," << frameEnd << endl;

    cleanUp();
    initialize(m_FileName);

    LogStream mout(LOG_DEBUG,"SchneiderVideoAV.loadCube.ccode");
    if (m_Good) {
        // cout << "max frames " << m_MaxFrames << endl;
        if (frameEnd >= m_MaxFrames) frameEnd = m_MaxFrames - 1;
        if (frameEnd < 0) frameEnd = m_MaxFrames - 1;
        if (frameStart > frameEnd) frameStart = frameEnd;
        if (frameStart < 0) frameStart = 0;
        int frameCount = frameEnd - frameStart + 1;

        // cout << "SchneiderVideoAV::loadCube width,height,frameCount " << m_Width << "," << m_Height << "," << frameCount << endl;
        pCube->Set(m_Width,m_Height,frameCount,m_FPS);

        // Assign appropriate parts of buffer to image planes in m_pFrameRGB
        // Note that m_pFrameRGB is an AVFrame, but AVFrame is a superset
        // of AVPicture
        avpicture_fill((AVPicture *)m_pFrameBGRA, m_pBuffer, PIX_FMT_BGRA,
                     m_pCodecCtx->width, m_pCodecCtx->height);

        // couldn't get seek to work, need to keep track of key/iFrames and read up to desired frame
        // if (frameStart > 0) {
        //     int64_t seekTarget = int64_t(frameStart) * m_TimeBase;

        //     if(av_seek_frame(m_pFormatCtx, -1, seekTarget, AVSEEK_FLAG_ANY) < 0) {
        //         mout << "averror av_seek_frame failed for file " << m_FileName.c_str() << LogStream::endl;            
        //         return -1;
        //     }
        // }

        int iFrame = 0;
        int iAbsoluteFrame = 0;
        
        int count_errs=0,skip_packets=0;
        const int max_number_of_attempts = 200;
        const int max_skipped_packets=10000;
        int frameFinished;

        bool FinishedReading = false;
        while(!FinishedReading) {
            AVPacket packet;
            int ret = av_read_frame(m_pFormatCtx, &packet);

            if (ret == AVERROR(EAGAIN) ) {
                skip_packets++;
                if (skip_packets > max_skipped_packets) {
                    mout << "averror exceeded max number of for file " << m_FileName.c_str() << LogStream::endl;
                    av_dump_format(m_pFormatCtx,0,m_FileName.c_str(),0); 
                    break;
                }
                continue;
            }

            // if (packet.stream_index == m_VideoStream) cout << "m_VideoStream " << m_VideoStream << " packet.stream_index " << packet.stream_index << " skip packets " << skip_packets << endl;

            if( packet.stream_index != m_VideoStream )
            {
                av_free_packet (&packet);
                // cout << "this packet isn't a m_VideoStream " << packet.stream_index << " skipped packets " << skip_packets;
                // cout << " frames " << frameCount << " nb_frames " << (unsigned long)(m_pFormatCtx->streams[m_VideoStream]->nb_frames) << endl;
                skip_packets++;
                if (skip_packets > max_skipped_packets) {
                    mout << "skipped packets (non Video Stream packets) exceeded max number of skipped packets for file " << m_FileName.c_str() << LogStream::endl;
                    av_dump_format(m_pFormatCtx,0,m_FileName.c_str(),0); 
                    break;
                }
                continue;
            }

            // Decode video frame
            avcodec_decode_video2(m_pCodecCtx, m_pFrame, &frameFinished, 
                         &packet);
            
            // Did we get a video frame?
            if(frameFinished) {

                iAbsoluteFrame++;
                if (iAbsoluteFrame > frameStart && iFrame < frameCount) { // skip frames until frameStart

                    // Convert the image from its native format to BGRA
                    if (m_pCodecCtx->hwaccel) {
                        m_pSWSCtx = sws_getCachedContext
                            (
                                m_pSWSCtx,
                                m_pCodecCtx->width,
                                m_pCodecCtx->height,
                                m_pCodecCtx->pix_fmt,
                                m_pCodecCtx->width,
                                m_pCodecCtx->height,
                                PIX_FMT_BGRA,
                                SWS_BILINEAR,
                                NULL,
                                NULL,
                                NULL                                
                            );
                    }
                    if (!m_pFrame->data[0]) {
                        cout << "skip this frame its not ready to be scaled" << endl;
                    }
                    else {

                        sws_scale
                        (
                                m_pSWSCtx,
                                (uint8_t const * const *)m_pFrame->data,
                                m_pFrame->linesize,
                                0,
                                m_pCodecCtx->height,
                                m_pFrameBGRA->data,
                                m_pFrameBGRA->linesize
                        );


                        if (!m_pFrameBGRA->data[0]) {
                            mout << "averror unable to get libav frame data = NULL done reading file " << m_FileName.c_str() << LogStream::endl;
                            FinishedReading = true;
                        }
                        else {
                            // Dump the frame to the console rgba order
                            // DumpFrame(m_pFrameBGRA, m_pCodecCtx->width, m_pCodecCtx->height);

                            // cout << "about to set frame " << iFrame << endl;
                            // bool usedPacket = false;
                            int64_t pts = packet.pts;
                            double frameTime = 0.0;
                            if (iFrame) frameTime = pCube->m_FrameTimes[iFrame-1] + 1.0/m_FPS;
                            if (pts != AV_NOPTS_VALUE) {
                                if (packet.stream_index == m_VideoStream) {
                                    AVRational *pTimeBase = &m_pFormatCtx->streams[packet.stream_index]->time_base;
                                    if (!iFrame) m_StartTime = av_q2d(*pTimeBase) * packet.pts;
                                    frameTime = av_q2d(*pTimeBase) * packet.pts - m_StartTime;
                                    // usedPacket = true;
                                }
                            }
                            // m_pFrameBGRA->data[0] is a pointer to a BGRA frame
                            pCube->m_FrameTimes[iFrame] = frameTime;
                            pCube->SetFrame(iFrame++,m_pFrameBGRA->data[0]);

                            bool lastFrame = false;
                            if (frameEnd >= 0 && iAbsoluteFrame > frameEnd) lastFrame = true;

                            if( iFrame >= frameCount || lastFrame ) {
                                FinishedReading = true;
                            }
                        }
                    }
                }
                    
            }
            else {
                count_errs++;
                if (count_errs > max_number_of_attempts) {
                    mout << "unable to read video file " << m_FileName.c_str() << " dumping metadata to std out from av function" << LogStream::endl;
                    av_dump_format(m_pFormatCtx,0,m_FileName.c_str(),0); 
                    break;
                }
            }
            
            // Free the packet that was allocated by av_read_frame
            av_free_packet(&packet);
        }
        

        if (count_errs > max_number_of_attempts) {
            return -1;
        }

        if (skip_packets > max_skipped_packets) {
            return -1;
        }

    }
    else { // no good return failure
        return VID_PROCESS_FAIL;
    }
    return VID_PROCESS_OK;
}

int SchneiderVideoAV::loadFrame(CSFrameBGRA *pFrame, int frameIndex)
{   
   
    CSCubeBGRA temp;
    int status = loadCube(&temp,frameIndex,frameIndex);
    
    if (status != VID_PROCESS_OK) {
        return VID_PROCESS_FAIL;
    }

    pFrame->Set(temp.m_Width,temp.m_Height,temp.pFrameData(0));
    return VID_PROCESS_OK;
}

SchneiderVideo.h

#ifndef SCHNEIDER_H
#define SCHNEIDER_H

class SchneiderVideoAV {
public:
    SchneiderVideoAV();
    SchneiderVideoAV(const string &fileName);
    
    ~SchneiderVideoAV();
    void cleanUp();

    int initialize(const string &fileName);
    // loads cube from framestart->end, defaults -1,-1 load entire video into cube
    int loadCube(CSCubeBGRA *pCube, int frameStart=-1, int frameEnd=-1);
    int loadFrame(CSFrameBGRA *pFrame, int frameIndex);

    AVFormatContext    *m_pFormatCtx;
    AVCodecContext     *m_pCodecCtx;
    AVCodec            *m_pCodec;
    AVFrame            *m_pFrame; 
    AVFrame            *m_pFrameBGRA;
    int                 m_numBytes;
    uint8_t            *m_pBuffer;
    int                 m_VideoStream;
    int64_t             m_TimeBase;


    AVDictionary       *m_pOptionsDict;
    struct SwsContext  *m_pSWSCtx;
    string              m_FileName;
    int                 m_Width;
    int                 m_Height;
    int                 m_MaxFrames;
    int                 m_Good;
    float               m_FPS;
    double              m_StartTime;
    double              m_RotationDegrees;
};

#endif