record and play file using wrapper ffmpeg - ffmpeg

I am using C# wrapper for ffmpeg from ffmpeg
I want record rtsp stream and play it but I can not decoder frame from file
using this code I write file test.avi
unsafe
{
AVFormatContext* context = FFmpegInvoke.avformat_alloc_context();
int video_stream_index=0;
FFmpegInvoke.av_register_all();
FFmpegInvoke.avcodec_register_all();
FFmpegInvoke.avformat_network_init();
//open rtsp
if (FFmpegInvoke.avformat_open_input(&context, "rtsp://admin:admin#192.168.0.71:554", null, null) != 0)
{
return ;
}
if (FFmpegInvoke.avformat_find_stream_info(context, null) < 0)
{
return ;
}
//search video stream
for (int i = 0; i < context->nb_streams; i++)
{
if (context->streams[i]->codec->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO)
video_stream_index = i;
}
AVPacket packet;
FFmpegInvoke.av_init_packet(&packet);
//open output file
AVOutputFormat* fmt = FFmpegInvoke.av_guess_format("h264", null, null);
AVFormatContext* oc = FFmpegInvoke.avformat_alloc_context();
oc->oformat = fmt;
FFmpegInvoke.avio_open2(&oc->pb, "test.mkv", FFmpegInvoke.AVIO_FLAG_WRITE, null, null);
AVStream* stream = null;
int cnt = 0;
//start reading packets from stream and write them to file
/// FFmpegInvoke.av_read_play(context);//play RTSP
while (FFmpegInvoke.av_read_frame(context, &packet) >= 0 && cnt < 1000)
{//read 100 frames
if (packet.stream_index == video_stream_index)
{//packet is video
if (stream == null)
{//create stream in file
stream = FFmpegInvoke.avformat_new_stream(oc, context->streams[video_stream_index]->codec->codec);
FFmpegInvoke.avcodec_copy_context(stream->codec, context->streams[video_stream_index]->codec);
stream->sample_aspect_ratio = context->streams[video_stream_index]->codec->sample_aspect_ratio;
FFmpegInvoke.avformat_write_header(oc, null);
}
packet.stream_index = stream->id;
var p1 = new FileInfo("test.mkv").Length;
FFmpegInvoke.av_write_frame(oc, &packet);
cnt++;
}
FFmpegInvoke.av_free_packet(&packet);
FFmpegInvoke.av_init_packet(&packet);
}
FFmpegInvoke.av_read_pause(context);
FFmpegInvoke.av_write_trailer(oc);
FFmpegInvoke.avio_close(oc->pb);
FFmpegInvoke.avformat_free_context(oc);
}
and using this code I want to play me file
unsafe{
string url = "test.mkv";
FFmpegInvoke.av_register_all();
FFmpegInvoke.avcodec_register_all();
FFmpegInvoke.avformat_network_init();
AVFormatContext* pFormatContext = FFmpegInvoke.avformat_alloc_context();
if (FFmpegInvoke.avformat_open_input(&pFormatContext, url, null, null) != 0)
throw new Exception("Could not open file");
if (FFmpegInvoke.avformat_find_stream_info(pFormatContext, null) != 0)
throw new Exception("Could not find stream info");
AVStream* pStream = null;
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
if (pFormatContext->streams[i]->codec->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO)
{
pStream = pFormatContext->streams[i];
break;
}
}
var packet = new AVPacket();
AVPacket* pPacket = &packet;
FFmpegInvoke.av_init_packet(pPacket);
AVCodecContext codecContext = *(pStream->codec);
int width = codecContext.width;
int height = codecContext.height;
AVPixelFormat sourcePixFmt = codecContext.pix_fmt;
AVCodecID codecId = codecContext.codec_id;
var convertToPixFmt = AVPixelFormat.PIX_FMT_BGR24;
SwsContext* pConvertContext = FFmpegInvoke.sws_getContext(width, height, sourcePixFmt,
width, height, convertToPixFmt,
FFmpegInvoke.SWS_FAST_BILINEAR, null, null, null);
if (pConvertContext == null)
throw new Exception("Could not initialize the conversion context");
var pConvertedFrame = (AVPicture*)FFmpegInvoke.avcodec_alloc_frame();
int convertedFrameBufferSize = FFmpegInvoke.avpicture_get_size(convertToPixFmt, width, height);
var pConvertedFrameBuffer = (byte*)FFmpegInvoke.av_malloc((uint)convertedFrameBufferSize);
FFmpegInvoke.avpicture_fill(pConvertedFrame, pConvertedFrameBuffer, convertToPixFmt, width, height);
AVCodec* pCodec = FFmpegInvoke.avcodec_find_decoder(codecId);
if (pCodec == null)
throw new Exception("Unsupported codec");
// Reusing codec context from stream info,
// as an alternative way it could look like this: (but it works not for all kind of codecs)
// AVCodecContext* pCodecContext = FFmpegInvoke.avcodec_alloc_context3(pCodec);
AVCodecContext* pCodecContext = &codecContext;
if ((pCodec->capabilities & FFmpegInvoke.CODEC_CAP_TRUNCATED) == FFmpegInvoke.CODEC_CAP_TRUNCATED)
pCodecContext->flags |= FFmpegInvoke.CODEC_FLAG_TRUNCATED;
AVFrame* pDecodedFrame = FFmpegInvoke.avcodec_alloc_frame();
if (FFmpegInvoke.av_read_frame(pFormatContext, pPacket) < 0)
throw new System.IO.EndOfStreamException();
int gotPicture = 0;
int size = FFmpegInvoke.avcodec_decode_video2(pCodecContext, pDecodedFrame, &gotPicture, pPacket);
if (size < 0)
throw new Exception(string.Format("Error while decoding frame "));
if (gotPicture == 1)
{
}
size =-22.Why? what is wrong?How play my file using ffmpeg?

Related

libx264 Input picture width (640) is greater than stride (0)

I'm trying to encode a series of Cairo surfaces by using libav. Here I initialize AV stuff:
AVStream* video_stream;
AVCodec* vcodec;
gint ret;
/* Setup video container */
avformat_alloc_output_context2(&img->video_format_context, NULL, NULL, filename);
if (img->video_format_context == NULL)
{
img_message(img, TRUE, _("Failed to find a suitable container for %s\n"),filename);
return FALSE;
}
ret = avio_open(&img->video_format_context->pb, filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
img_message(img, TRUE, _("Couldn't write output file %s\n"),filename);
return FALSE;
}
/* Setup video codec */
vcodec = avcodec_find_encoder(codec_id);
if (!vcodec)
{
img_message(img, TRUE, _("Couldn't find any encoder for %s\n"),filename);
return FALSE;
}
/* Create video stream */
video_stream = avformat_new_stream(img->video_format_context, vcodec);
video_stream->id = 0;
if (! video_stream)
{
img_message(img, TRUE, _("Couldn't not allocate video stream\n"));
return FALSE;
}
/* Allocate video encoding context */
img->codec_context = avcodec_alloc_context3(vcodec);
if (! img->codec_context)
{
img_message(img, TRUE, _("Couldn't allocate video enconding context\n"));
return FALSE;
}
/* Setup video enconding context parameters */
img->codec_context->codec_id = codec_id;
img->codec_context->codec_type = AVMEDIA_TYPE_VIDEO;
img->codec_context->width = img->video_size[0];
img->codec_context->height = img->video_size[1];
img->codec_context->sample_aspect_ratio = (struct AVRational) {1, 1};
img->codec_context->pix_fmt = vcodec->pix_fmts[0];
img->codec_context->framerate = av_d2q(frame_rate, INT_MAX);
if (codec_id == AV_CODEC_ID_VP8 || codec_id == AV_CODEC_ID_VP9 || codec_id == AV_CODEC_ID_THEORA || codec_id == AV_CODEC_ID_FLV1 ||
AV_CODEC_ID_MPEG1VIDEO || codec_id == AV_CODEC_ID_MPEG2VIDEO)
img->codec_context->bit_rate = round(bitrate_crf * 1000000);
img->codec_context->time_base = av_inv_q(img->codec_context->framerate);
video_stream->time_base = img->codec_context->time_base;
if (img->video_format_context->oformat->flags & AVFMT_GLOBALHEADER)
img->codec_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
/* Some codecs require the CRF value */
if (codec_id == AV_CODEC_ID_H264 || codec_id == AV_CODEC_ID_H265)
{
gchar *crf = g_strdup_printf("%i", bitrate_crf);
av_opt_set(img->codec_context->priv_data, "crf", crf, AV_OPT_SEARCH_CHILDREN);
g_free(crf);
}
/* Set exporting stage to be multithreaded */
AVDictionary* opts = NULL;
av_dict_set(&opts, "threads", "auto", 0);
/* Open video encoder */
ret = avcodec_open2(img->codec_context, vcodec, &opts);
if (ret < 0)
{
img_message(img, TRUE, _("Failed to open the video encoder\n"));
return FALSE;
}
/* Copy video encoder parameters to output stream */
ret = avcodec_parameters_from_context(video_stream->codecpar, img->codec_context);
if (ret < 0)
{
img_message(img, TRUE, _("Failed to copy video encoder parameters to output stream\n"));
return FALSE;
}
/* AVFRAME stuff */
img->video_frame = av_frame_alloc();
img->video_frame->format = AV_PIX_FMT_RGBA;
img->video_frame->width = img->video_size[0];
img->video_frame->height = img->video_size[1];
av_frame_make_writable(img->video_frame);
ret = av_frame_get_buffer(img->video_frame, 1);
if (ret < 0)
img_message(img,TRUE, _("Could not allocate the video frame data\n"));
img->video_packet = av_packet_alloc();
And here I called repeatedly (the function is called somewehere else) av_send_frame() but it throws the error in the subject:
gint width, height, stride, row, col, offset;
uint8_t *pix;
/* Image info and pixel data */
width = cairo_image_surface_get_width( surface );
height = cairo_image_surface_get_height( surface );
stride = cairo_image_surface_get_stride( surface );
pix = cairo_image_surface_get_data( surface );
for( row = 0; row < height; row++ )
{
for( col = 0; col < width; col++ )
{
offset = 3 * col + row * img->video_frame->linesize[0];
img->video_frame->data[0][offset + 0] = pix[0];
img->video_frame->data[0][offset + 1] = pix[1];
img->video_frame->data[0][offset + 2] = pix[2];
}
}
img_export_encode_av_frame(img->video_frame, img->video_format_context, img->codec_context, img->video_packet);
return TRUE;
}
void img_export_encode_av_frame(AVFrame *frame, AVFormatContext *fmt, AVCodecContext *ctx, AVPacket *pkt)
{
gint ret;
/* send the frame to the encoder */
ret = avcodec_send_frame(ctx, frame);
if (ret < 0)
g_print("Error sending a frame for encoding\n");
while (ret >= 0)
{
ret = avcodec_receive_packet(ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
else if (ret < 0)
g_print("Error during encoding\n");
av_interleaved_write_frame(fmt, pkt);
av_packet_unref(pkt);
}
}
I googled here also but with no luck. It seems I'm the only one to encode a cairo surface. Grepping the error message in ffmpeg sources didn't help. How do I set the stride? I read ffmpeg does it for me once I allocate the buffer for the frame but in my case it seems it doesn't. Where am I wrong?

Request for ffmpeg raw data to mp4 container example

I have a binary file with raw h264 data which is arranged like that
NAL(SPS), NAL(PPS), NAL(Frame), NAL(SPS), NAL(PPS)....
and i want to mux it (without encode) into a mp4 container.
The muxing.c in the ffmpeg example do the encoding of yuv data, but it is different from my case, and i have no ideas how to change the example to do what i want to do...
I knew the commaned ffmpeg -i h264file -c copy h264.mp4 can do what i want to do, but i have to do it in my program, so i need to know how to use the ffmpeg api to do the same thing, but so far, i cannot find any simple example to do it. Is there anyone has hint on how to do it?? Thanks
Updated, i have write the code as below from the reference, it seems can create the mp4 but the time is not correct, it lost the frame rate information and the time information, it play very fast.
av_register_all();
int ret;
AVDictionary *opt = NULL;
//bool is264 = true;
const char * inputFileName = "input.264";
const char * outputFileName = "output.mp4";
AVFormatContext *ic = avformat_alloc_context();
if((ret = avformat_open_input(&ic, inputFileName, NULL, NULL)) < 0)
return -1;//
// Get format info (retrieve stream information)
if ((ret = avformat_find_stream_info(ic, NULL)) < 0)
return ret; // Couldn't find stream information
for (int i = 0; i < ic->nb_streams; i++)
{
AVStream *stream;
AVCodecContext *codec_ctx;
stream = ic->streams[i];
codec_ctx = stream->codec;
/* Reencode video & audio and remux subtitles etc. */
if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO
|| codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
/* Open decoder */
ret = avcodec_open2(codec_ctx, avcodec_find_decoder(codec_ctx->codec_id), NULL);
if (ret < 0) {
//av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i);
return ret;
}
}
}
// Dump information about file onto standard error
av_dump_format(ic, 0, inputFileName, 0);
AVFormatContext *oc;
avformat_alloc_output_context2(&oc, NULL, NULL, outputFileName);
if (!oc) {
//printf("Could not deduce output format from file extension: using MPEG.\n");
//avformat_alloc_output_context2(&oc, NULL, "mpeg", outputFileName);
return -1;
}
AVStream *ist = ic->streams[0];
AVCodec *out_vid_codec = avcodec_find_encoder(oc->oformat->video_codec);
if (NULL == out_vid_codec)
return -1; // Couldn't find video encoder
AVStream *out_vid_strm = avformat_new_stream(oc, out_vid_codec);
if (NULL == out_vid_strm)
return -1; // Couldn't output video stream
ret = avcodec_copy_context(out_vid_strm->codec, ist->codec);
if (ret < 0)
return ret; // Failed to copy context
ret = avio_open(&oc->pb, outputFileName, AVIO_FLAG_WRITE);
ret = avformat_write_header(oc, NULL);
AVPacket pkt;
while(1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(ic, &pkt);
if (ret < 0)
break;
pkt.stream_index = 0;
in_stream = ic->streams[pkt.stream_index];
out_stream = oc->streams[pkt.stream_index];
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
//log_packet(ofmt_ctx, &pkt, "out");
ret = av_interleaved_write_frame(oc, &pkt);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&pkt);
}
av_write_trailer(oc);

Output black when I decode h264 720p with ffmpeg

First, sorry for my english. When I decode h264 720p in ardrone2.0 my output is black and I cant see anything.
I have try to change the value of pCodecCtx->pix_fmt = AV_PIX_FMT_BGR24; to pCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P; and the value of pCodecCtxH264->pix_fmt = AV_PIX_FMT_BGR24; to pCodecCtxH264->pix_fmt = AV_PIX_FMT_YUV420P; but my program crash. What am I doing wrong?. Thank you, see part of my code:
av_register_all();
avcodec_register_all();
avformat_network_init();
// 1.2. Open video file
if(avformat_open_input(&pFormatCtx, drone_addr, NULL, NULL) != 0) {
mexPrintf("No conecct with Drone");
EndVideo();
return;
}
pCodec = avcodec_find_decoder(AV_CODEC_ID_H264);
pCodecCtx = avcodec_alloc_context3(pCodec);
pCodecCtx->pix_fmt = AV_PIX_FMT_BGR24;
pCodecCtx->skip_frame = AVDISCARD_DEFAULT;
pCodecCtx->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
pCodecCtx->err_recognition = AV_EF_CAREFUL;
pCodecCtx->skip_loop_filter = AVDISCARD_DEFAULT;
pCodecCtx->workaround_bugs = FF_BUG_AUTODETECT;
pCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
pCodecCtx->codec_id = AV_CODEC_ID_H264;
pCodecCtx->skip_idct = AVDISCARD_DEFAULT;
pCodecCtx->width = 1280;
pCodecCtx->height = 720;
pCodecH264 = avcodec_find_decoder(AV_CODEC_ID_H264);
pCodecCtxH264 = avcodec_alloc_context3(pCodecH264);
pCodecCtxH264->pix_fmt = AV_PIX_FMT_BGR24;
pCodecCtxH264->skip_frame = AVDISCARD_DEFAULT;
pCodecCtxH264->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
pCodecCtxH264->err_recognition = AV_EF_CAREFUL;
pCodecCtxH264->skip_loop_filter = AVDISCARD_DEFAULT;
pCodecCtxH264->workaround_bugs = FF_BUG_AUTODETECT;
pCodecCtxH264->codec_type = AVMEDIA_TYPE_VIDEO;
pCodecCtxH264->codec_id = AV_CODEC_ID_H264;
pCodecCtxH264->skip_idct = AVDISCARD_DEFAULT;
if(avcodec_open2(pCodecCtxH264, pCodecH264, &optionsDict) < 0)
{
mexPrintf("Error opening H264 codec");
return ;
}
pFrame_BGR24 = av_frame_alloc();
if(pFrame_BGR24 == NULL) {
mexPrintf("Could not allocate pFrame_BGR24\n");
return ;
}
// Determine required buffer size and allocate buffer
buffer_BGR24 =
(uint8_t *)av_mallocz(av_image_get_buffer_size(AV_PIX_FMT_BGR24,
pCodecCtx->width, ((pCodecCtx->height == 720) ? 720 : pCodecCtx->height) *
sizeof(uint8_t)*3,1));
// Assign buffer to image planes
av_image_fill_arrays(pFrame_BGR24->data, pFrame_BGR24->linesize,
buffer_BGR24,AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height,1);
// format conversion context
pConvertCtx_BGR24 = sws_getContext(pCodecCtx->width, pCodecCtx->height,
pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_BGR24,
SWS_BILINEAR | SWS_ACCURATE_RND, 0, 0, 0);
// 1.6. get video frames
pFrame = av_frame_alloc();
av_init_packet(&packet);
packet.data = NULL;
packet.size = 0;
}
//Captura un frame
void video::capture(mxArray *plhs[]) {
if(av_read_frame(pFormatCtx, &packet) < 0){
mexPrintf("Error al leer frame");
return;
}
do {
do {
rest = avcodec_send_packet(pCodecCtxH264, &packet);
} while(rest == AVERROR(EAGAIN));
if(rest == AVERROR_EOF || rest == AVERROR(EINVAL)) {
printf("AVERROR(EAGAIN): %d, AVERROR_EOF: %d,
AVERROR(EINVAL): %d\n", AVERROR(EAGAIN), AVERROR_EOF,
AVERROR(EINVAL));
printf("fe_read_frame: Frame getting error (%d)!\n", rest);
return;
}
rest = avcodec_receive_frame(pCodecCtxH264, pFrame);
} while(rest == AVERROR(EAGAIN));
if(rest == AVERROR_EOF || rest == AVERROR(EINVAL)) {
// An error or EOF occured,index break out and return what
// we have so far.
printf("AVERROR(EAGAIN): %d, AVERROR_EOF: %d, AVERROR(EINVAL): %d\n",
AVERROR(EAGAIN), AVERROR_EOF, AVERROR(EINVAL));
printf("fe_read_frame: EOF or some othere decoding error (%d)!\n",
rest);
return;
}
// 2.1.1. convert frame to GRAYSCALE [or BGR] for OpenCV
sws_scale(pConvertCtx_BGR24, (const uint8_t* const*)pFrame->data,
pFrame->linesize, 0,pCodecCtx->height, pFrame_BGR24->data,
pFrame_BGR24->linesize);
//}
av_packet_unref(&packet);
av_init_packet(&packet);
mwSize dims[] = {(pCodecCtx->width)*((pCodecCtx->height == 720) ? 720 :
pCodecCtx->height)*sizeof(uint8_t)*3};
plhs[0] = mxCreateNumericArray(1,dims,mxUINT8_CLASS, mxREAL);
//plhs[0]=mxCreateDoubleMatrix(pCodecCtx->height,pCodecCtx-
>width,mxREAL);
point=mxGetPr(plhs[0]);
memcpy(point, pFrame_BGR24->data[0],(pCodecCtx->width)*(pCodecCtx-
>height)*sizeof(uint8_t)*3);
}
Go to debugger and see your memcpy. I am not sure if it works for all dimensions that you want. Also, there may be more memory problems. For example, try to see what is the value of buffer_BGR24 and pFrame. I bet that sometimes, they do not return right values. Check them out in the code.

FFMPEG AAC encoding causes audio to be lower in pitch

I built a sample application that encodes AAC (from PortAudio) into a MP4 container (no video stream).
The resulting audio is lower in pitch.
#include "stdafx.h"
#include "TestRecording.h"
#include "libffmpeg.h"
TestRecording::TestRecording()
{
}
TestRecording::~TestRecording()
{
}
struct RecordingContext
{
RecordingContext()
{
formatContext = NULL;
audioStream = NULL;
audioFrame = NULL;
audioFrameframeNumber = 0;
}
libffmpeg::AVFormatContext* formatContext;
libffmpeg::AVStream* audioStream;
libffmpeg::AVFrame* audioFrame;
int audioFrameframeNumber;
};
static int AudioRecordCallback(const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer,
const PaStreamCallbackTimeInfo* timeInfo,
PaStreamCallbackFlags statusFlags,
void *userData)
{
RecordingContext* recordingContext = (RecordingContext*)userData;
libffmpeg::avcodec_fill_audio_frame(recordingContext->audioFrame,
recordingContext->audioFrame->channels,
recordingContext->audioStream->codec->sample_fmt,
static_cast<const unsigned char*>(inputBuffer),
(framesPerBuffer * sizeof(float) * recordingContext->audioFrame->channels),
0);
libffmpeg::AVPacket pkt;
libffmpeg::av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
int gotpacket;
int result = avcodec_encode_audio2(recordingContext->audioStream->codec, &pkt, recordingContext->audioFrame, &gotpacket);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't encode the audio frame to acc");
return paContinue;
}
if (gotpacket)
{
pkt.stream_index = recordingContext->audioStream->index;
recordingContext->audioFrameframeNumber++;
// this codec requires no bitstream filter, just send it to the muxer!
result = libffmpeg::av_write_frame(recordingContext->formatContext, &pkt);
if (result < 0)
{
LOG(ERROR) << "Couldn't write the encoded audio frame";
libffmpeg::av_free_packet(&pkt);
return paContinue;
}
libffmpeg::av_free_packet(&pkt);
}
return paContinue;
}
static bool InitializeRecordingContext(RecordingContext* recordingContext)
{
int result = libffmpeg::avformat_alloc_output_context2(&recordingContext->formatContext, NULL, NULL, "C:\\Users\\Paul\\Desktop\\test.mp4");
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't create output format context");
return false;
}
libffmpeg::AVCodec *audioCodec;
audioCodec = libffmpeg::avcodec_find_encoder(libffmpeg::AV_CODEC_ID_AAC);
if (audioCodec == NULL)
{
LOG(ERROR) << "Couldn't find the encoder for AAC";
}
recordingContext->audioStream = libffmpeg::avformat_new_stream(recordingContext->formatContext, audioCodec);
if (!recordingContext->audioStream)
{
LOG(ERROR) << "Couldn't create the audio stream";
return false;
}
recordingContext->audioStream->codec->bit_rate = 64000;
recordingContext->audioStream->codec->sample_fmt = libffmpeg::AV_SAMPLE_FMT_FLTP;
recordingContext->audioStream->codec->sample_rate = 48000;
recordingContext->audioStream->codec->channel_layout = AV_CH_LAYOUT_STEREO;
recordingContext->audioStream->codec->channels = libffmpeg::av_get_channel_layout_nb_channels(recordingContext->audioStream->codec->channel_layout);
recordingContext->audioStream->codecpar->bit_rate = recordingContext->audioStream->codec->bit_rate;
recordingContext->audioStream->codecpar->format = recordingContext->audioStream->codec->sample_fmt;
recordingContext->audioStream->codecpar->sample_rate = recordingContext->audioStream->codec->sample_rate;
recordingContext->audioStream->codecpar->channel_layout = recordingContext->audioStream->codec->channel_layout;
recordingContext->audioStream->codecpar->channels = recordingContext->audioStream->codec->channels;
result = libffmpeg::avcodec_open2(recordingContext->audioStream->codec, audioCodec, NULL);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't open the audio codec");
return false;
}
// create a new frame to store the audio samples
recordingContext->audioFrame = libffmpeg::av_frame_alloc();
if (!recordingContext->audioFrame)
{
LOG(ERROR) << "Couldn't alloce the output audio frame";
return false;
}
recordingContext->audioFrame->nb_samples = recordingContext->audioStream->codec->frame_size;
recordingContext->audioFrame->channel_layout = recordingContext->audioStream->codec->channel_layout;
recordingContext->audioFrame->channels = recordingContext->audioStream->codec->channels;
recordingContext->audioFrame->format = recordingContext->audioStream->codec->sample_fmt;
recordingContext->audioFrame->sample_rate = recordingContext->audioStream->codec->sample_rate;
result = libffmpeg::av_frame_get_buffer(recordingContext->audioFrame, 0);
if (result < 0)
{
LOG(ERROR) << "Coudln't initialize the output audio frame buffer";
return false;
}
// some formats want video_stream headers to be separate
if (!strcmp(recordingContext->formatContext->oformat->name, "mp4") || !strcmp(recordingContext->formatContext->oformat->name, "mov") || !strcmp(recordingContext->formatContext->oformat->name, "3gp"))
{
recordingContext->audioStream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
// open the ouput file
if (!(recordingContext->formatContext->oformat->flags & AVFMT_NOFILE))
{
result = libffmpeg::avio_open(&recordingContext->formatContext->pb, recordingContext->formatContext->filename, AVIO_FLAG_WRITE);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't open the output file");
return false;
}
}
// write the stream headers
result = libffmpeg::avformat_write_header(recordingContext->formatContext, NULL);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't write the headers to the file");
return false;
}
return true;
}
static bool FinalizeRecordingContext(RecordingContext* recordingContext)
{
int result = 0;
// write the trailing information
if (recordingContext->formatContext->pb)
{
result = libffmpeg::av_write_trailer(recordingContext->formatContext);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't write the trailer information");
return false;
}
}
// close all the codes
for (int i = 0; i < (int)recordingContext->formatContext->nb_streams; i++)
{
result = libffmpeg::avcodec_close(recordingContext->formatContext->streams[i]->codec);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't close the codec");
return false;
}
}
// close the output file
if (recordingContext->formatContext->pb)
{
if (!(recordingContext->formatContext->oformat->flags & AVFMT_NOFILE))
{
result = libffmpeg::avio_close(recordingContext->formatContext->pb);
if (result < 0)
{
LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't close the output file");
return false;
}
}
}
// free the format context and all of its data
libffmpeg::avformat_free_context(recordingContext->formatContext);
recordingContext->formatContext = NULL;
recordingContext->audioStream = NULL;
if (recordingContext->audioFrame)
{
libffmpeg::av_frame_free(&recordingContext->audioFrame);
recordingContext->audioFrame = NULL;
}
return true;
}
int TestRecording::Test()
{
PaError result = paNoError;
result = Pa_Initialize();
if (result != paNoError) LOGINT_WITH_MESSAGE(ERROR, result, "Error initializing audio device framework");
RecordingContext recordingContext;
if (!InitializeRecordingContext(&recordingContext))
{
LOG(ERROR) << "Couldn't start recording file";
return 0;
}
auto defaultDevice = Pa_GetDefaultInputDevice();
auto deviceInfo = Pa_GetDeviceInfo(defaultDevice);
PaStreamParameters inputParameters;
inputParameters.device = defaultDevice;
inputParameters.channelCount = 2;
inputParameters.sampleFormat = paFloat32;
inputParameters.suggestedLatency = deviceInfo->defaultLowInputLatency;
inputParameters.hostApiSpecificStreamInfo = NULL;
PaStream* stream = NULL;
result = Pa_OpenStream(
&stream,
&inputParameters,
NULL,
48000,
1024,
paClipOff,
AudioRecordCallback,
&recordingContext);
if (result != paNoError)LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't open the audio stream");
result = Pa_StartStream(stream);
if (result != paNoError)LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't start the audio stream");
Sleep(1000 * 5);
result = Pa_StopStream(stream);
if (result != paNoError)LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't stop the audio stream");
if (!FinalizeRecordingContext(&recordingContext)) LOG(ERROR) << "Couldn't stop recording file";
result = Pa_CloseStream(stream);
if (result != paNoError)LOGINT_WITH_MESSAGE(ERROR, result, "Couldn't stop the audio stream");
return 0;
}
Here is the stdout, in case it helps.
https://gist.github.com/pauldotknopf/9f24a604ce1f8a081aa68da1bf169e98
Why is the audio lower in pitch? I assume I am overlooking a parameter that needs to be configured between PortAudio and FFMPEG. Is there something super obvious that I am missing?

Audio encoding using avcodec_fill_audio_frame() and memory leaks

As a part of encoding decoded audio packets, I'm using avcodec_fill_audio_frame(). I'm passing allocated AVFrame pointer to along with buffer containing the decoded samples and other parameters number of channels, sample format, buffer size. Though the encoding is working fine I'm not able to completely eliminate the memory leaks. I've taken care of most of things but still I'm not able detect the leakage.
Below is the function which I'm using for encoding. Please suggest something.
AudioSample contains decoded data and it is completely managed in different class(free in class destructor). I'm freeing the AVFrame in FFmpegEncoder destructor and AVPacket is freed every time using av_free_packet() with av_packet_destruct enabled. What more do I need to free?
void FfmpegEncoder::WriteAudioSample(AudioSample *audS)
{
int num_audio_frame = 0;
AVCodecContext *c = NULL;
// AVFrame *frame;
AVPacket pkt;
av_init_packet(&pkt);
pkt.destruct = av_destruct_packet;
pkt.data = NULL;
pkt.size = 0;
int ret = 0, got_packet = 0;
c = m_out_aud_strm->codec;
static int64_t aud_pts_in = -1;
if((audS != NULL) && (audS->GetSampleLength() > 0) )
{
int byte_per_sample = av_get_bytes_per_sample(c->sample_fmt);
PRINT_VAL("Byte Per Sample ", byte_per_sample)
m_frame->nb_samples = (audS->GetSampleLength())/(c->channels*av_get_bytes_per_sample(c->sample_fmt));
if(m_frame->nb_samples == c->frame_size)
{
#if 1
if(m_need_resample && (c->channels >= 2))
{
uint8_t * t_buff1 = new uint8_t[audS->GetSampleLength()];
if(t_buff1 != NULL)
{
for(int64_t i = 0; i< m_frame->nb_samples; i++)
{
memcpy(t_buff1 + i*byte_per_sample, (uint8_t*)((uint8_t*)audS->GetAudioSampleData() + i*byte_per_sample*c->channels), byte_per_sample);
memcpy(t_buff1 + (audS->GetSampleLength())/2 + i*byte_per_sample, (uint8_t*)((uint8_t*)audS->GetAudioSampleData() + i*byte_per_sample*c->channels+ byte_per_sample), byte_per_sample);
}
audS->FillAudioSample(t_buff1, audS->GetSampleLength());
delete[] t_buff1;
}
}
#endif
ret = avcodec_fill_audio_frame(m_frame, c->channels, c->sample_fmt, (uint8_t*)audS->GetAudioSampleData(),m_frame->nb_samples*byte_per_sample*c->channels, 0);
//ret = avcodec_fill_audio_frame(&frame, c->channels, c->sample_fmt, t_buff,frame.nb_samples*byte_per_sample*c->channels, 0);
if(ret != 0)
{
PRINT_MSG("Avcodec Fill Audio Failed ")
}
else
{
got_packet = 0;
ret = avcodec_encode_audio2(c, &pkt, m_frame, &got_packet);
if(ret < 0 || got_packet == 0)
{
PRINT_MSG("failed to encode audio ")
}
else
{
PRINT_MSG("Audio Packet Encoded ");
aud_pts_in++;
pkt.pts = aud_pts_in;
pkt.dts = pkt.pts;
pkt.stream_index = m_out_aud_strm->index;
ret = av_interleaved_write_frame(oc, &pkt);
if(ret != 0)
{
PRINT_MSG("Error Write Audio PKT ")
}
else
{
PRINT_MSG("Audio PKT Writen ")
}
}
}
}
avcodec_flush_buffers(c);
// avcodec_free_frame(&frame);
}
av_free_packet(&pkt);
}
Thanks,
Pradeep
//================== SEND AUDIO OUTPUT =======================
void AVOutputStream::sendAudioOutput (AVFrame* inputFrame)
{
AVCodecContext *codecCtx = pOutputAudioStream->codec;
// set source data variables
sourceNumberOfChannels = inputFrame->channels;
sourceChannelLayout = inputFrame->channel_layout;
sourceSampleRate = inputFrame->sample_rate;
_sourceSampleFormat = (AVSampleFormat)inputFrame->format;
sourceNumberOfSamples = inputFrame->nb_samples;
// set destination data variables
destinationNumberOfChannels = codecCtx->channels;
destinationChannelLayout = codecCtx->channel_layout;
destinationSampleRate = codecCtx->sample_rate;
destinationSampleFormat = codecCtx->sample_fmt;//AV_SAMPLE_FMT_FLTP;//EncodecCtx->sample_fmt;
destinationLineSize = 0;
destinationData = NULL;
int returnVal = 0;
if (startDecode == false)
{
startDecode = true;
resamplerCtx = swr_alloc_set_opts(NULL,
destinationChannelLayout,
destinationSampleFormat,
destinationSampleRate,
sourceChannelLayout,
_sourceSampleFormat,
sourceSampleRate,
0,
NULL);
if (resamplerCtx == NULL)
{
std::cout << "Unable to create the resampler context for the audio frame";
isConnected = false;
}
// initialize the resampling context
returnVal = swr_init(resamplerCtx);
if (returnVal < 0)
{
std::cout << "Unable to init the resampler context, error:";
isConnected = false;
}
} //if (startDecode == false)
if (sourceSampleRate != 0)
destinationNumberOfSamples = destinationSampleRate/sourceSampleRate * sourceNumberOfSamples;
// allocate the destination samples buffer
returnVal = av_samples_alloc_array_and_samples(&destinationData,
&destinationLineSize,
destinationNumberOfChannels,
destinationNumberOfSamples,
destinationSampleFormat,
0);
if (returnVal < 0)
{
std::cout << "Unable to allocate destination samples, error";
isConnected = false;
}
// convert to destination format
returnVal = swr_convert(resamplerCtx,
destinationData,
destinationNumberOfSamples,
(const uint8_t **)inputFrame->data, //sourceData,
sourceNumberOfSamples);
if (returnVal < 0)
{
std::cout << "Resampling failed, error \n";
isConnected = false;
}
int bufferSize = av_samples_get_buffer_size(&destinationLineSize,
destinationNumberOfChannels,
destinationNumberOfSamples,
destinationSampleFormat,
0);
//whithout fifo
pOutputAudioFrame = av_frame_alloc();
pOutputAudioFrame->nb_samples = codecCtx->frame_size;//frameNumberOfSamples;
pOutputAudioFrame->format = codecCtx->sample_fmt;
pOutputAudioFrame->channel_layout = codecCtx->channel_layout;
pOutputAudioFrame->channels = codecCtx->channels;
pOutputAudioFrame->sample_rate = codecCtx->sample_rate;
returnVal = avcodec_fill_audio_frame(pOutputAudioFrame,
pOutputAudioFrame->channels,
(AVSampleFormat)pOutputAudioFrame->format,
(const uint8_t *)destinationData[0],
bufferSize,0);
pOutputAudioFrame->pts = inputFrame->pts;
if (returnVal < 0)
{
std::cout << "Unable to fill the audio frame wsampleIndexith captured audio data,error";
isConnected = false;
}
// encode the audio frame, fill a packet for streaming
av_init_packet(&outAudioPacket);
outAudioPacket.data = NULL;
outAudioPacket.size = 0;
outAudioPacket.dts = outAudioPacket.pts = 0;
int gotPacket;
// encoding
returnVal = avcodec_encode_audio2(codecCtx, &outAudioPacket, pOutputAudioFrame, &gotPacket);
// free buffers
av_freep(&destinationData[0]);
av_freep(&destinationData);
av_frame_free(&pOutputAudioFrame);
if (gotPacket)
{
outAudioPacket.stream_index = pOutputAudioStream->index;
outAudioPacket.flags |= AV_PKT_FLAG_KEY;
returnVal = av_interleaved_write_frame(pOutputFormatCtx, &outAudioPacket);
//returnVal = av_write_frame(pOutputFormatCtx, &outAudioPacket);
if (returnVal != 0)
{
std::cout << "Cannot write audio packet \n";
isConnected = false;
}
av_free_packet(&outAudioPacket);
} // if (gotPacket)
}
You can see after resample i free used buffers.
// free buffers
av_freep(&destinationData[0]);
av_freep(&destinationData);

Resources