Convert from OpenCV image and write movie with FFmpeg (OpenCV 4.5, FFmpeg 4.4)

cv2ff.cpp

/*
 * Convert from OpenCV image and write movie with FFmpeg
 *
 * Copyright (c) 2021 yohhoy
 */
#include <iostream>
#include <vector>
// FFmpeg
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
// OpenCV
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>


int main(int argc, char* argv[])
{
    if (argc < 2) {
        std::cout << "Usage: cv2ff <outfile>" << std::endl;
        return 1;
    }
    const char* outfile = argv[1];

 // av_log_set_level(AV_LOG_DEBUG);
    int ret;

    const int dst_width = 640;
    const int dst_height = 480;
    const AVRational dst_fps = {30, 1};

    // initialize OpenCV capture as input frame generator
    cv::VideoCapture cvcap(0);
    if (!cvcap.isOpened()) {
        std::cerr << "fail to open cv::VideoCapture";
        return 2;
    }
    cvcap.set(cv::CAP_PROP_FRAME_WIDTH, dst_width);
    cvcap.set(cv::CAP_PROP_FRAME_HEIGHT, dst_height);
    cvcap.set(cv::CAP_PROP_FPS, dst_fps.num);
    // some device ignore above parameters for capturing image,
    // so we query actual parameters for image rescaler.
    const int cv_width = cvcap.get(cv::CAP_PROP_FRAME_WIDTH);
    const int cv_height = cvcap.get(cv::CAP_PROP_FRAME_HEIGHT);
    const int cv_fps = cvcap.get(cv::CAP_PROP_FPS);

    // open output format context
    AVFormatContext* outctx = nullptr;
    ret = avformat_alloc_output_context2(&outctx, nullptr, nullptr, outfile);
    if (ret < 0) {
        std::cerr << "fail to avformat_alloc_output_context2(" << outfile << "): ret=" << ret;
        return 2;
    }

    // create new video stream
    AVCodec* vcodec = avcodec_find_encoder(outctx->oformat->video_codec);
    AVStream* vstrm = avformat_new_stream(outctx, vcodec);
    if (!vstrm) {
        std::cerr << "fail to avformat_new_stream";
        return 2;
    }

    // open video encoder
    AVCodecContext* cctx = avcodec_alloc_context3(vcodec);
    if (!vstrm) {
        std::cerr << "fail to avcodec_alloc_context3";
        return 2;
    }
    cctx->width = dst_width;
    cctx->height = dst_height;
    cctx->pix_fmt = vcodec->pix_fmts[0];
    cctx->time_base = av_inv_q(dst_fps);
    cctx->framerate = dst_fps;
    if (outctx->oformat->flags & AVFMT_GLOBALHEADER)
        cctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    ret = avcodec_open2(cctx, vcodec, nullptr);
    if (ret < 0) {
        std::cerr << "fail to avcodec_open2: ret=" << ret;
        return 2;
    }
    avcodec_parameters_from_context(vstrm->codecpar, cctx);

    // initialize sample scaler
    SwsContext* swsctx = sws_getContext(
        cv_width, cv_height, AV_PIX_FMT_BGR24,
        dst_width, dst_height, cctx->pix_fmt,
        SWS_BILINEAR, nullptr, nullptr, nullptr);
    if (!swsctx) {
        std::cerr << "fail to sws_getContext";
        return 2;
    }

    // allocate frame buffer for encoding
    AVFrame* frame = av_frame_alloc();
    frame->width = dst_width;
    frame->height = dst_height;
    frame->format = static_cast<int>(cctx->pix_fmt);
    ret = av_frame_get_buffer(frame, 32);
    if (ret < 0) {
        std::cerr << "fail to av_frame_get_buffer: ret=" << ret;
        return 2;
    }

    // allocate packet to retrive encoded frame
    AVPacket* pkt = av_packet_alloc();

    // open output IO context
    ret = avio_open2(&outctx->pb, outfile, AVIO_FLAG_WRITE, nullptr, nullptr);
    if (ret < 0) {
        std::cerr << "fail to avio_open2: ret=" << ret;
        return 2;
    }

    std::cout
        << "camera:  " << cv_width << 'x' << cv_height << '@' << cv_fps << "\n"
        << "outfile: " << outfile << "\n"
        << "format:  " << outctx->oformat->name << "\n"
        << "vcodec:  " << vcodec->name << "\n"
        << "size:    " << dst_width << 'x' << dst_height << "\n"
        << "fps:     " << av_q2d(cctx->framerate) << "\n"
        << "pixfmt:  " << av_get_pix_fmt_name(cctx->pix_fmt) << "\n"
        << std::flush;

    // write media container header (if any)
    ret = avformat_write_header(outctx, nullptr);
    if (ret < 0) {
        std::cerr << "fail to avformat_write_header: ret=" << ret;
        return 2;
    }

    cv::Mat image;

    // encoding loop
    int64_t frame_pts = 0;
    unsigned nb_frames = 0;
    bool end_of_stream = false;
    for (;;) {
        if (!end_of_stream) {
            // retrieve source image
            cvcap >> image;
            cv::imshow("press ESC to exit", image);
            if (cv::waitKey(33) == 0x1b) {
                // flush encoder
                avcodec_send_frame(cctx, nullptr);
                end_of_stream = true;
            }
        }
        if (!end_of_stream) {
            // convert cv::Mat(OpenCV) to AVFrame(FFmpeg)
            const int stride[4] = { static_cast<int>(image.step[0]) };
            sws_scale(swsctx, &image.data, stride, 0, image.rows, frame->data, frame->linesize);
            frame->pts = frame_pts++;
            // encode video frame
            ret = avcodec_send_frame(cctx, frame);
            if (ret < 0) {
                std::cerr << "fail to avcodec_send_frame: ret=" << ret << "\n";
                break;
            }
        }
        while ((ret = avcodec_receive_packet(cctx, pkt)) >= 0) {
            // rescale packet timestamp
            pkt->duration = 1;
            av_packet_rescale_ts(pkt, cctx->time_base, vstrm->time_base);
            // write encoded packet
            av_write_frame(outctx, pkt);
            av_packet_unref(pkt);
            std::cout << nb_frames << '\r' << std::flush;  // dump progress
            ++nb_frames;
        }
        if (ret == AVERROR_EOF)
            break;
    };
    std::cout << nb_frames << " frames encoded" << std::endl;

    // write trailer and close file
    av_write_trailer(outctx);
    avio_close(outctx->pb);

    av_packet_free(&pkt);
    av_frame_free(&frame);
    sws_freeContext(swsctx);
    avcodec_free_context(&cctx);
    avformat_free_context(outctx);
    return 0;
}

Would you please provide command to compile? Thank you!

I got very confuesd trouble:

/*
 * Convert from OpenCV image and write movie with FFmpeg
 *
 * Copyright (c) 2021 yohhoy
 */
#include <iostream>
#include <vector>
// FFmpeg
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
// OpenCV
#include <opencv2/opencv.hpp>
// #include <opencv2/highgui.hpp>

using namespace cv;
using namespace std;

int main()
{

    std::cout << "Video opening" << std::endl;
    VideoCapture cap("ace_clip.mkv");
    if (!cap.isOpened()) {
        std::cerr << "fail to open cv::VideoCapture";
        return 2;
    }
    std::cout << "Video opened" << std::endl;

    const int cv_width = cap.get(cv::CAP_PROP_FRAME_WIDTH);
    const int cv_height = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
    const int cv_fps = cap.get(cv::CAP_PROP_FPS);


    const char* outfile = "out.avi";

    int ret;

    const int dst_width = 1440;
    const int dst_height = 1080;
    const AVRational dst_fps = {24, 1};

//   open output format context
    AVFormatContext* outctx = nullptr;
    ret = avformat_alloc_output_context2(&outctx, nullptr, nullptr, outfile);
    if (ret < 0) {
        std::cerr << "fail to avformat_alloc_output_context2(" << outfile << "): ret=" << ret;
        return 2;
    }

    // create new video stream
    AVCodec* vcodec = avcodec_find_encoder(outctx->oformat->video_codec);
    AVStream* vstrm = avformat_new_stream(outctx, vcodec);
    if (!vstrm) {
        std::cerr << "fail to avformat_new_stream";
        return 2;
    }

    // open video encoder
    AVCodecContext* cctx = avcodec_alloc_context3(vcodec);
    if (!vstrm) {
        std::cerr << "fail to avcodec_alloc_context3";
        return 2;
    }
    cctx->width = dst_width;
    cctx->height = dst_height;
    cctx->pix_fmt = vcodec->pix_fmts[0];
    cctx->time_base = av_inv_q(dst_fps);
    cctx->framerate = dst_fps;
    if (outctx->oformat->flags & AVFMT_GLOBALHEADER)
        cctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    ret = avcodec_open2(cctx, vcodec, nullptr);
    if (ret < 0) {
        std::cerr << "fail to avcodec_open2: ret=" << ret;
        return 2;
    }
    avcodec_parameters_from_context(vstrm->codecpar, cctx);

//     // initialize sample scaler
//     SwsContext* swsctx = sws_getContext(
//         cv_width, cv_height, AV_PIX_FMT_BGR24,
//         dst_width, dst_height, cctx->pix_fmt,
//         SWS_BILINEAR, nullptr, nullptr, nullptr);
//     if (!swsctx) {
//         std::cerr << "fail to sws_getContext";
//         return 2;
//     }

//     // allocate frame buffer for encoding
//     AVFrame* frame = av_frame_alloc();
//     frame->width = dst_width;
//     frame->height = dst_height;
//     frame->format = static_cast<int>(cctx->pix_fmt);
//     ret = av_frame_get_buffer(frame, 32);
//     if (ret < 0) {
//         std::cerr << "fail to av_frame_get_buffer: ret=" << ret;
//         return 2;
//     }

//     // allocate packet to retrive encoded frame
//     AVPacket* pkt = av_packet_alloc();

//     // open output IO context
//     ret = avio_open2(&outctx->pb, outfile, AVIO_FLAG_WRITE, nullptr, nullptr);
//     if (ret < 0) {
//         std::cerr << "fail to avio_open2: ret=" << ret;
//         return 2;
//     }

//     std::cout
//         << "camera:  " << cv_width << 'x' << cv_height << '@' << cv_fps << "\n"
//         << "outfile: " << outfile << "\n"
//         << "format:  " << outctx->oformat->name << "\n"
//         << "vcodec:  " << vcodec->name << "\n"
//         << "size:    " << dst_width << 'x' << dst_height << "\n"
//         << "fps:     " << av_q2d(cctx->framerate) << "\n"
//         << "pixfmt:  " << av_get_pix_fmt_name(cctx->pix_fmt) << "\n"
//         << std::flush;

//     // write media container header (if any)
//     ret = avformat_write_header(outctx, nullptr);
//     if (ret < 0) {
//         std::cerr << "fail to avformat_write_header: ret=" << ret;
//         return 2;
//     }

//     cv::Mat image;

//     // encoding loop
//     int64_t frame_pts = 0;
//     unsigned nb_frames = 0;
//     bool end_of_stream = false;
//     for (;;) {
//         if (!end_of_stream) {
//             // retrieve source image
//             cap >> image;
//             cv::imshow("press ESC to exit", image);
//             if (cv::waitKey(33) == 0x1b) {
//                 // flush encoder
//                 avcodec_send_frame(cctx, nullptr);
//                 end_of_stream = true;
//             }
//         }
//         if (!end_of_stream) {
//             // convert cv::Mat(OpenCV) to AVFrame(FFmpeg)
//             const int stride[4] = { static_cast<int>(image.step[0]) };
//             sws_scale(swsctx, &image.data, stride, 0, image.rows, frame->data, frame->linesize);
//             frame->pts = frame_pts++;
//             // encode video frame
//             ret = avcodec_send_frame(cctx, frame);
//             if (ret < 0) {
//                 std::cerr << "fail to avcodec_send_frame: ret=" << ret << "\n";
//                 break;
//             }
//         }
//         while ((ret = avcodec_receive_packet(cctx, pkt)) >= 0) {
//             // rescale packet timestamp
//             pkt->duration = 1;
//             av_packet_rescale_ts(pkt, cctx->time_base, vstrm->time_base);
//             // write encoded packet
//             av_write_frame(outctx, pkt);
//             av_packet_unref(pkt);
//             std::cout << nb_frames << '\r' << std::flush;  // dump progress
//             ++nb_frames;
//         }
//         if (ret == AVERROR_EOF)
//             break;
//     };
//     std::cout << nb_frames << " frames encoded" << std::endl;

//     // write trailer and close file
//     av_write_trailer(outctx);
//     avio_close(outctx->pb);

//     av_packet_free(&pkt);
//     av_frame_free(&frame);
//     sws_freeContext(swsctx);
//     avcodec_free_context(&cctx);
//     avformat_free_context(outctx);
    return 0;
}

I can open input video with opencv, only when I commened after:
ret = avformat_alloc_output_context2(&outctx, nullptr, nullptr, outfile);
output:

Video opening
Video opened

An if those lines has not be commened, even call VideoCapture cap("ace_clip.mkv"); at first,it still will cause:

Video opening
segment fault(core dumped)

My compile command is:
g++ cv2ffmpeg.cpp -o cv2ffmpeg -lstdc++ -lpthread -lopencv_core -lopencv_highgui -lopencv_videoio -lavformat -lavcodec -lavutil -lswscale $(pkg-config --cflags --libs opencv4) $(pkg-config --cflags --libs libavformat libavcodec)

Solved by compiling opencv without ffmpeg

Outstanding! Have been looking for the updated version as I'm pretty new to ffmpeg. Thanks!

Hi, any idea why the first 27 frames are not encoded with this? If I use codecs like h265, vp9 or hevc, in the first 27 frames,

        while ((ret = avcodec_receive_packet(cctx, pkt)) >= 0) {

This returns -11 code and doesn't go inside the while loop.

I tested on a rtsp video stream input file. It always stop to encode after some number of frames. The last message I can see is:

[```
swscaler @ 0x55557d48a0] bad src image pointers
send pkts of frame:77
77

then it just stop, but not exit. and the cpu load is very low.  What is happening?  Is it because of swscaler hang up?