I'm using an QOpenGL widget to draw frames. However, I'm struggling to get frames by using avcodec_receive_frame. It ended within the block else if (ret == AVERROR(EAGAIN)) and returned -11. I have no idea what made this happen. Also I checked that codecs were fine, so I guess the problem wasn't caused by codecs.
MyDemux.cpp
AVPacket* MyDemux::allocatePacket()
{
AVPacket* packet = av_packet_alloc();
return packet;
}
AVFrame* MyDemux::allocateFrame()
{
AVFrame* frame = av_frame_alloc();
return frame;
}
int MyDemux::readFrame(AVPacket* packet)
{
mux.lock();
if (!formatCtx) {
std::cout << "formaetCtx is null" << std::endl;
mux.unlock();
return -1;
}
int ret = av_read_frame(formatCtx, packet);
if (ret == AVERROR_EOF) {
return -2;
}
if (ret != 0) {
mux.unlock();
av_packet_free(&packet);
return -1;
}
media_type = packet->stream_index;
mux.unlock();
return 0;
}
MyDecode.cpp
void MyDecode::decode(AVFrame* frame, AVPacket* packet)
{
int ret;
ret = avcodec_send_packet(codecCtx, packet); // this returned 0
while (ret == 0) {
ret = avcodec_receive_frame(codecCtx, frame); // this returned -11
if (ret == AVERROR(EINVAL)) {
std::cout << "codec issue" << std::endl;
av_frame_free(&frame);
return;
}
else if (ret == AVERROR(EAGAIN)) { // program ends here
std::cout << "output is not available this state" << std::endl;
av_frame_free(&frame);
return;
}
else if (ret == AVERROR(EINVAL)) {
std::cout << "no more frames" << std::endl;
av_frame_free(&frame);
return;
}
}
}
main.cpp
class MyThread : public QThread
{
public:
MyDemux demux;
MyDecode video_decode;
myDecode audio_decode;
MyVideoWidget* videoWidget;
AVPacket* packet;
AVFrame* frame;
void initThread()
{
char* url = "demo.mp4";
demux.openFile(url);
video_decode.openCodec(demux.copy_video_codec_par());
audio_decode.openCodec(demux.copy_audio_codec_par());
packet = demux.allocatePacket();
frame = demux.allocateFrame();
}
void run()
{
while (demux.readFrame(packet) != -2) {
if (demux.get_media_type() == 0) {
video_decode.decode(frame, packet);
videoWidget->paintFrame(frame);
}
else if (demux.get_media_type() == 1) {
}
}
video_decode.decode(frame, nullptr);
demux.clear();
demux.close();
}
};
When you call avcodec_receive_frame and get EAGAIN error that means your decoder does not get enough data to decode (e.x you send a B-frame in video). So each time you get that error you should ignore it and go to next avcodec_send_packet
There is a delay between sending and receiving a frame. When receive returns EAGAIN, you should call send with the next encoded frame, then call receive again.
Related
First I run the following code:
IDXGIFactory* pFactory;
HRESULT hr = CreateDXGIFactory(__uuidof(IDXGIFactory1), (void**)(&pFactory));
if (FAILED(hr))
{
return -1;
}
then I tried to use ffmpeg's nvenc_h264 function:
AVCodec* m_encoder = avcodec_find_encoder_by_name("h264_nvenc");
if (!m_encoder) {
m_encoder = avcodec_find_encoder_by_name("nvenc_h264");
}
if (!m_encoder) {
err = -1;
std::cout << "find 264 encoder failed" << std::endl;
return 0;
}
AVCodecContext* m_encoder_ctx = avcodec_alloc_context3(m_encoder);
if (!m_encoder_ctx) {
err = -1;
std::cout << "avcodec_alloc_context3 failed" << std::endl;
return 0;
}
m_encoder_ctx->width = 1280;
m_encoder_ctx->height = 720;
m_encoder_ctx->time_base = { 1, (int)25 };
m_encoder_ctx->codec_id = m_encoder->id;
m_encoder_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
m_encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
ret = avcodec_open2(m_encoder_ctx, m_encoder, nullptr);
it's failed. the avcode_open2 would fail, it shows that
"[h264_nvenc # 0000016AE3F06F00] dl_fn->cuda_dl->cuInit(0) failed -> CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected"
but if I don't call the CreateDXGIFactory, then the avcodec_open2 would success.
The vcodec_receive_frame function didn't receive the rest of the frames. I tested that there were totally 132 frames of the video and it only received 125 frames losing 7 frames at the END of the video. How can I get the lost frames back?
But something weird happened. As you can see the output from my MyDecode::receiveFrame() function. The code inside the block if (ret != 0){} executed first, but the lost frames are at the end of the video. So how could they come out first? What caused this happen?
MyDecode.cpp
AVFrame* MyDecode::receiveFrame()
{
mux.lock();
if (!codecCtx) {
mux.unlock();
return 0;
}
AVFrame* frame = av_frame_alloc();
int ret = avcodec_receive_frame(codecCtx, frame);
mux.unlock();
if (ret != 0)
{
static int lost_frames = 1;
std::cout << "Lost frames: " << lost_frames << std::endl;
lost_frames += 1;
av_frame_free(&frame);
return nullptr;
}
std::cout << "Received frames: " << received_frame_num << std::endl;
received_frame_num += 1;
return frame;
}
bool MyDecode::sendPacket(AVPacket* packet)
{
if (!packet || !packet->data || packet->size == 0)
return false;
mux.lock();
if (!codecCtx) {
mux.unlock();
return false;
}
int ret = avcodec_send_packet(codecCtx, packet);
mux.unlock();
av_packet_free(&packet);
if (ret != 0) {
return false;
}
return true;
}
Console output
Total frames: 132
Lost frames: 1
Lost frames: 2
Lost frames: 3
Lost frames: 4
Lost frames: 5
Lost frames: 6
Lost frames: 7
Received frames: 1
Received frames: 2
Received frames: 3
................
Received frames: 125
UPDATE:
MyDemux.cpp
AVPacket* MyDemux::readFrame()
{
mux.lock();
if (!formatCtx) {
std::cout << "formaetCtx is null" << std::endl;
mux.unlock();
return nullptr;
}
AVPacket* packet = av_packet_alloc();
if (!packet) {
std::cout << "packet is null" << std::endl;
mux.unlock();
return nullptr;
}
int ret = av_read_frame(formatCtx, packet);
if (ret != 0) {
while (true) {
av_read_frame(formatCtx, nullptr);
}
mux.unlock();
av_packet_free(&packet);
av_packet_unref(packet);
return nullptr;
}
media_type = packet->stream_index;
mux.unlock();
return packet;
}
main.cpp
while (true) {
AVPacket* pkt = demux.readFrame();
if (demux.get_media_type() == 0) {
AVFrame* frame = video_decode.receiveFrame();
videoWidget->paintFrame(frame);
}
else if (demux.get_media_type() == 1) {
}
if (!pkt) {
std::cout << "to break" << std::endl;
break;
}
}
You have to send NULL pkts to the decoder to drain all pending frames.
From avcodec.h
End of stream situations. These require "flushing" (aka draining)
the codec, as the codec might buffer multiple frames or packets
internally for performance or out of necessity (consider B-frames).
This is handled as follows:
- Instead of valid input, send NULL to the avcodec_send_packet() (decoding)
or avcodec_send_frame() (encoding) functions. This will enter draining
mode.
- Call avcodec_receive_frame() (decoding) or avcodec_receive_packet()
(encoding) in a loop until AVERROR_EOF is returned. The functions will
not return AVERROR(EAGAIN), unless you forgot to enter draining mode.
- Before decoding can be resumed again, the codec has to be reset with
avcodec_flush_buffers().
I wrote 2 threads to decode RTSP stream from IP camera as below:
RTSP_read_paket function used to read Packets from RTSP link, packets stored in a queue named Packet_buf.
std::queue<AVPacket> Packet_buf;
bool pkt_pending_k = false;
int RTSP_read_packet (string url)
{
rtsp_init(url);
int ret;
AVPacket packet;
av_init_packet(&packet);
while(1)
{
ret = av_read_frame(pFormatCtx,&packet);
if(ret==0)
{
if (packet.stream_index == video_stream_index)
{
Packet_buf.push(packet);
if((ready1 == false))
{
ready1 = true;
conv1.notify_one();
}
}
av_packet_unref(&packet);
cout<<"number of RTSP packet: "<<Packet_buf.size()<<endl;
}
}
return 0;
}
ffmpeg_decode read packets from Packet_buf to decode frames
AVFrame ffmpeg_decode( void )
{
AVPacket avpkt;
av_init_packet(&avpkt);
int ret;
conv1.wait(lk1,[]{return ready1;});
while(1)
{
while(1)
{
ret = avcodec_receive_frame(pCodecCtx,pFrame);
if(ret == AVERROR(EAGAIN)||ret==AVERROR_EOF){
break;
}
return pFrame;
}
if(!Packet_buf.empty())
{
if(pkt_pending_k == false)
{
avpkt = Packet_buf.front();
Packet_buf.pop();
}else{
pkt_pending_k = false;
}
}
ret = avcodec_send_packet(pCodecCtx, &avpkt); //program halting here
cout<<"-------------> ret = "<<ret<<endl;
if(ret==AVERROR(EAGAIN))
{
pkt_pending_k = true;
}
if(ret<0||ret==AVERROR_EOF)
{
cout<<"avcodec_send_packet: "<<ret<<endl;
break;
}
}
}
int main () {
thread Camera2_readPackets(RTSP_read_packet,url);
thread Camera2_decode(ffmpeg_decode,url);
Camera2_decode.join();
return 0;
}
My program halt at line:
ret = avcodec_send_packet(pCodecCtx, &avpkt);
Anyone can help me find the problem, thanks !
P/s:
rtsp_init function:
int rtsp_init (string url)
{
av_register_all();
avdevice_register_all();
avcodec_register_all();
avformat_network_init();
const char *filenameSrc = url.c_str();
pFormatCtx = avformat_alloc_context();
if ( pFormatCtx == NULL )
return -8;
AVDictionary *options = NULL;
av_dict_set(&options,"rtsp_flags","prefer_tcp",0);
av_dict_set(&options,"stimeout","1000000",0);
int avret = avformat_open_input( &pFormatCtx, filenameSrc, NULL, &options );
av_dict_free(&options);
if ( avret != 0 ) {
std::cout << "Open File Error 12" << std::endl;
return -12;
}
avret = avformat_find_stream_info( pFormatCtx, NULL );
if ( avret < 0 ) {
std::cout << "Get Stream Information Error 13" << std::endl;
avformat_close_input( &pFormatCtx );
pFormatCtx = NULL;
return -13;
}
av_dump_format( pFormatCtx, 0, filenameSrc, 0 );
video_stream_index = av_find_best_stream(pFormatCtx,AVMEDIA_TYPE_VIDEO,-1,-1,NULL,0);
if ( video_stream_index < 0 ) {
std::cout << "Video stream was not found Error 14" << std::endl;
avformat_close_input( &pFormatCtx );
pFormatCtx = NULL;
return -14;
}
pCodecCtx = avcodec_alloc_context3(NULL);
avret = avcodec_parameters_to_context(pCodecCtx,pFormatCtx->streams[video_stream_index]->codecpar);
if(avret<0)
{
std::cout << "codec not found Error 15" << std::endl;
return -15;
}
pCodec = avcodec_find_decoder( pCodecCtx->codec_id );
avret = avcodec_open2( pCodecCtx, pCodec, NULL );
if ( avret < 0) {
std::cout << "Open Codec Error 16" << std::endl;
return -16;
}
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
pFrame->width = pCodecCtx->width;
pFrame->height = pCodecCtx->height;
pFrame->format = pCodecCtx->pix_fmt;
avret = av_frame_get_buffer(pFrame,0);
if (avret < 0)
{
return -17;
}
pFrameRGB->width = pCodecCtx->width;
pFrameRGB->height = pCodecCtx->height;
pFrameRGB->format = AV_PIX_FMT_BGR24;
avret = av_frame_get_buffer(pFrameRGB, 0);
if (avret < 0)
{
return -18;
}
return ( EXIT_SUCCESS );
}
I'm developing server-client application which manages req/res topics and does fan-out of some messages. Messages sends out with WSASend in overlapped mode with provided Completion routine. Noticed that very next send with WSASend after WSARead triggers sender's completion routine. While the sends originated in the server w/o preceding WSARead doesn't trigger completion routine. Why does that happen?
bool CCommunicationServer::ManageReadMessage(UCHAR index)
{
OVERLAPPED_EX *over = m_readov[index];
DWORD bytes = 0, flags = 0;
if (WSARecv(over->socket, &(over->wsabuf), 1, &bytes, &flags, over, &CCommunicationServer::WorkerReadRoutine) == SOCKET_ERROR) {
bytes = WSAGetLastError();
if (bytes != WSA_IO_PENDING) {
LOG(ERROR) << std::endl << "WSARecv() failed w/err " << bytes;
shutdown(m_readov[index]->socket, SD_BOTH);
return FALSE;
}
}
return TRUE;
}
bool CCommunicationServer::ManageSendMessage(UCHAR index, char* msg, OVERLAPPED_EX *moreover)
{
OVERLAPPED_EX *over = moreover;
DWORD bytes = 0;
SOCKET socket = m_readov[index] != NULL ? m_readov[index]->socket : NULL;
if (!over) {
ScopedLock lock(&m_lock);
if (!msg) {
LOG(WARNING) << "Empty send! Ignoring!";
return FALSE;
} else if (m_readov[index] != NULL) {
LOG(DEBUG) << ++created << "\t:\t" << deleted << "\t\t\t\r";
over = new OVERLAPPED_EX(this, m_readov[index]->socket, index, msg);
} else {
return FALSE;
}
}
if (WSASend(socket, &(over->wsabuf), 1, &bytes, 0, (LPWSAOVERLAPPED)over, &CCommunicationServer::WorkerSendRoutine) == SOCKET_ERROR) {
bytes = bytes = WSAGetLastError();
if (bytes != WSA_IO_PENDING) {
LOG(ERROR) << std::endl << "WSASend() failed w/err " << bytes;
shutdown(socket, SD_BOTH);
LOG(DEBUG) << created << "\t:\t" << ++deleted << "\t\t\t\r";
delete over;
return FALSE;
}
}
return TRUE;
}
And here are Completion routines:
void CALLBACK CCommunicationServer::WorkerReadRoutine(DWORD Error, DWORD BytesTransferred, LPWSAOVERLAPPED Overlapped, DWORD InFlags)
{
OVERLAPPED_EX *over = (OVERLAPPED_EX*)Overlapped;
if (Error)
return;
if (BytesTransferred > 0) {
over->server->GetRequest(over);
}
over->server->ManageReadMessage(over->index);
}
void CALLBACK CCommunicationServer::WorkerSendRoutine(DWORD Error, DWORD BytesTransferred, LPWSAOVERLAPPED Overlapped, DWORD InFlags)
{
OVERLAPPED_EX *over = (OVERLAPPED_EX*)Overlapped;
if (Error || over->wsabuf.len == BytesTransferred) {
LOG(DEBUG) << over->server->created << "\t:\t" << ++over->server->deleted << "\t\t\t\r";
delete over;
} else {
over->wsabuf.buf += BytesTransferred;
over->wsabuf.len -= BytesTransferred;
over->server->ManageSendMessage(over->index, NULL, over);
}
}
The variables created and deleted shows how many OVERLAPPED_EX structures were created and then released to avoid memory leakage.
I'm getting memory leaks in avcodec_find_encoder. Although I'm cleaning the resources properly
still I'm not able to get rid of the leak. By successive commenting the code I found that memory leaks happen only after the call of avcodec_find_encoder(). I've tried my code with different video files and I found that memory leaks blocks are always same. Also if I open only audio or video then I get just one memory leaks block.
Below is the part of Init and Clean-up code from the application.
Note that this is just part of code which contains initialization and resource release.
AVFormatContext *m_informat;
AVFormatContext *m_outformat;
AVStream *m_in_vid_strm, *m_out_vid_strm;
AVStream *m_in_aud_strm, *m_out_aud_strm;
int VideoClipper::Init(const wxString& filename)
{
int ret = 0;
char errbuf[64];
av_register_all();
if ((ret = avformat_open_input( &m_informat, filename.mb_str(), 0, 0)) != 0 )
{
av_strerror(ret,errbuf,sizeof(errbuf));
PRINT_VAL("Not able to Open file;; ", errbuf)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Opened File ")
}
if ((ret = avformat_find_stream_info(m_informat, 0))< 0 )
{
av_strerror(ret,errbuf,sizeof(errbuf));
PRINT_VAL("Not Able to find stream info:: ", errbuf)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Got stream Info ")
}
for(unsigned int i = 0; i<m_informat->nb_streams; i++)
{
if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
PRINT_MSG("Found Video Stream ")
m_in_vid_strm_idx = i;
m_in_vid_strm = m_informat->streams[i];
}
if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
PRINT_MSG("Found Audio Stream ")
m_in_aud_strm_idx = i;
m_in_aud_strm = m_informat->streams[i];
}
}
AVOutputFormat *outfmt = NULL;
std::string outfile = std::string(filename) + "clip_out.avi";
outfmt = av_guess_format(NULL,outfile.c_str(),NULL);
if(outfmt == NULL)
{
ret = -1;
return ret;
}
else
{
m_outformat = avformat_alloc_context();
if(m_outformat)
{
m_outformat->oformat = outfmt;
_snprintf(m_outformat->filename, sizeof(m_outformat->filename), "%s", outfile.c_str());
}
else
{
ret = -1;
return ret;
}
}
AVCodec *out_vid_codec,*out_aud_codec;
out_vid_codec = out_aud_codec = NULL;
if(outfmt->video_codec != AV_CODEC_ID_NONE && m_in_vid_strm != NULL)
{
out_vid_codec = avcodec_find_encoder(outfmt->video_codec);
if(NULL == out_vid_codec)
{
PRINT_MSG("Could Not Find Vid Encoder")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Found Out Vid Encoder ")
m_out_vid_strm = avformat_new_stream(m_outformat, out_vid_codec);
if(NULL == m_out_vid_strm)
{
PRINT_MSG("Failed to Allocate Output Vid Strm ")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Allocated Video Stream ")
if(avcodec_copy_context(m_out_vid_strm->codec, m_informat->streams[m_in_vid_strm_idx]->codec) != 0)
{
PRINT_MSG("Failed to Copy Context ")
ret = -1;
return ret;
}
}
}
}
if(outfmt->audio_codec != AV_CODEC_ID_NONE && m_in_aud_strm != NULL)
{
out_aud_codec = avcodec_find_encoder(outfmt->audio_codec);
if(NULL == out_aud_codec)
{
PRINT_MSG("Could Not Find Out Aud Encoder ")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Found Out Aud Encoder ")
m_out_aud_strm = avformat_new_stream(m_outformat, out_aud_codec);
if(NULL == m_out_aud_strm)
{
PRINT_MSG("Failed to Allocate Out Vid Strm ")
ret = -1;
return ret;
}
else
{
if(avcodec_copy_context(m_out_aud_strm->codec, m_informat->streams[m_in_aud_strm_idx]->codec) != 0)
{
PRINT_MSG("Failed to Copy Context ")
ret = -1;
return ret;
}
}
}
}
if (!(outfmt->flags & AVFMT_NOFILE))
{
if (avio_open2(&m_outformat->pb, outfile.c_str(), AVIO_FLAG_WRITE,NULL, NULL) < 0)
{
PRINT_VAL("Could Not Open File ", outfile)
ret = -1;
return ret;
}
}
/* Write the stream header, if any. */
if (avformat_write_header(m_outformat, NULL) < 0)
{
PRINT_VAL("Error Occurred While Writing Header ", outfile)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Written Output header ")
m_init_done = true;
}
return ret;
}
Here is the Clean-up part
void VideoClipper::ReleaseResource(void)
{
if(m_in_aud_strm && m_in_aud_strm->codec)
{
avcodec_close(m_in_aud_strm->codec);
PRINT_MSG("Closed Input Audio Codec ")
}
if(m_in_vid_strm && m_in_vid_strm->codec)
{
avcodec_close(m_in_vid_strm->codec);
PRINT_MSG("Closed Input Video Codec ")
}
if(m_informat)
{
avformat_close_input(&m_informat);
PRINT_MSG("Freed Input Format Contex ")
}
if(m_out_aud_strm && m_out_aud_strm->codec)
{
avcodec_close(m_out_aud_strm->codec);
PRINT_MSG("Closed Output Audio Codec ")
}
if(m_out_vid_strm && m_out_vid_strm->codec)
{
avcodec_close(m_out_vid_strm->codec);
PRINT_MSG("Closed Output Audio Codec ")
}
if(m_outformat)
{
avformat_close_input(&m_outformat);
m_outformat = NULL;
PRINT_MSG("Closed Output Format ")
}
}
Memory Leaks message
Detected memory leaks!
Dumping objects ->
{13691} normal block at 0x01046A60, 4479 bytes long.
Data: < > CD CD CD CD CD CD CD CD CD CD CD CD CD CD CD CD
{13685} normal block at 0x01043FD0, 10831 bytes long.
Data: < ? > CD CD CD CD CD CD CD CD D0 3F 04 01 ED ED ED ED
Object dump complete.
I'm using latest version of ffmpeg on Visual Studio 2012.
Please suggest where I'm missing.
Thanks
Pradeep
There are lots of thing that matters here
first you need to close io
if (!(fmt->flags & AVFMT_NOFILE))
{
/* Close the output file. */
avio_close(ctx->oc->pb);
}
you should also call
avformat_free_context(ctx->oc);
There is always 24 bytes memory leakage at my system due to allocation pthread_mutex in libavcodec/utils.c of ffmpeg, and there is no way to free that memory.atleast till I or someone fix the code.