FFmpeg remuxing parts of an audio file - ffmpeg

I'm trying to remux individual tracks from a FLAC file using the FFmpeg libraries.
I get the starting timestamps from a Cue sheet, I seek to the timestamps using avformat_seek_file. However after writing the packets to output files, they only have data from the beginning of the input file.
This is the code snippet which opens the input FLAC and also creates an output AVFormatContext for each track. I'm guessing the issue is avformat_seek_file, it doesn't seem to do anything, since even though I seek to the beginning of a track, the output file contains data from the beginning of the input.
for(int i = 0; i <= sheet.ntracks; i++) {
sheet.avfmtctx = avformat_alloc_context();
if(avformat_open_input(&sheet.avfmtctx, sheet.file, NULL, NULL) < 0) {
fprintf(stderr,
"avformat_open_input(): failed to open %s\n",
sheet.file);
return 1;
}
int audio_stream_idx = -1;
for(int i = 0; i < sheet.avfmtctx->nb_streams; i++) {
if(sheet.avfmtctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audio_stream_idx = i;
break;
}
}
avformat_find_stream_info(sheet.avfmtctx, NULL);
AVFormatContext *output;
char *filepath = title_to_filepath(&sheet.tracks[i], sheet.file);
avformat_alloc_output_context2(&output, NULL, NULL, filepath);
AVStream *out_audio_stream = avformat_new_stream(output, NULL);
avcodec_parameters_copy(out_audio_stream->codecpar,
sheet.avfmtctx->streams[audio_stream_idx]->codecpar);
if(avio_open(&output->pb, filepath, AVIO_FLAG_WRITE) < 0) {
fprintf(stderr, "Failed to open %s for writing\n", filepath);
return 1;
}
if(avformat_write_header(output, NULL) < 0) {
fprintf(stderr, "avformat_write_header() failed\n");
return 1;
}
int64_t current_frame = sheet.tracks[i].index;
int64_t next_track_index = (i < sheet.ntracks) ?
sheet.tracks[i + 1].index :
INT64_MAX;
if(avformat_seek_file(sheet.avfmtctx,
-1,
INT64_MIN,
current_frame,
current_frame,
0) < 0) {
fprintf(stderr, "Failed to seek to the index of track %d\n", i);
avformat_free_context(sheet.avfmtctx);
sheet.avfmtctx = NULL;
av_write_trailer(output);
avio_closep(&output->pb);
avformat_free_context(output);
free(filepath);
continue;
}
AVPacket *pkt = av_packet_alloc();
int64_t pts_diff = AV_NOPTS_VALUE, dts_diff = AV_NOPTS_VALUE;
while(current_frame < next_track_index && !avio_feof(output->pb)) {
int ret;
if((ret = av_read_frame(sheet.avfmtctx, pkt)) < 0) {
if(ret != AVERROR_EOF)
fprintf(stderr, "av_read_frame() failed: %s\n", av_err2str(ret));
break;
}
if(pkt->stream_index != audio_stream_idx)
continue;
// runs only once
if(pts_diff == AV_NOPTS_VALUE && dts_diff == AV_NOPTS_VALUE) {
pts_diff = pkt->pts;
dts_diff = pkt->dts;
}
pkt->stream_index = 0; // first and only stream
pkt->pts -= pts_diff;
pkt->dts -= dts_diff;
pkt->pos = -1;
av_interleaved_write_frame(output, pkt);
current_frame++;
}
avformat_free_context(sheet.avfmtctx);
sheet.avfmtctx = NULL;
av_write_trailer(output);
av_packet_free(&pkt);
avio_closep(&output->pb);
avformat_free_context(output);
free(filepath);
}
current_frame and next_track_index are calculated from the INDEX lines in the Cue sheet: MM * 60 * 75 + SS * 75 + FF.
Can someone tell me what I'm doing wrong, and how to get the data I need from the input?

Related

Saving frames as JPG with FFMPEG (Visual Studio / C++)

I am trying to save all frames from a mp4 video in separate JPG files, I have a code that runs and actually saves something to JPG files but files are not recognized as images and nothing is showing.
Below my full code, I am using Visual Studio 2022 in Windows 11 and FFMPEG 5.1. The function that saves the images is save_frame_as_jpeg which is actually an adaption from the code provided here but changing the use of avcodec_encode_video2 for avcodec_send_frame/avcodec_receive_packet as indicated in the documentation.
I am obiously doing something wrong but cannot quite find it, BTW, I know that a simple command (ffmpeg -i input.mp4 -vf fps=1 vid_%d.png) will do this but I am requiring to do it by code.
Any help is appreciated, thanks in advance!
// FfmpegTests.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#pragma warning(disable : 4996)
extern "C"
{
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavutil/avutil.h"
#include "libavutil/error.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
}
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "swscale.lib")
#include <cstdio>
#include <iostream>
#include <chrono>
#include <thread>
static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
AVFilterGraph* filter_graph;
AVFilterContext* buffersrc_ctx;
AVFilterContext* buffersink_ctx;
static int video_stream_index = -1;
const char* filter_descr = "scale=78:24,transpose=cclock";
static int64_t last_pts = AV_NOPTS_VALUE;
static int open_input_file(const char* filename)
{
const AVCodec* dec;
int ret;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char* filters_descr)
{
char args[512];
int ret = 0;
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame* frame, AVRational time_base)
{
int x, y;
uint8_t* p0, * p;
int64_t delay;
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
AVRational timeBaseQ;
timeBaseQ.num = 1;
timeBaseQ.den = AV_TIME_BASE;
delay = av_rescale_q(frame->pts - last_pts, time_base, timeBaseQ);
if (delay > 0 && delay < 1000000)
std::this_thread::sleep_for(std::chrono::microseconds(delay));
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
}
int save_frame_as_jpeg(AVCodecContext* pCodecCtx, AVFrame* pFrame, int FrameNo) {
int ret = 0;
const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_JPEG2000);
if (!jpegCodec) {
return -1;
}
AVCodecContext* jpegContext = avcodec_alloc_context3(jpegCodec);
if (!jpegContext) {
return -1;
}
jpegContext->pix_fmt = pCodecCtx->pix_fmt;
jpegContext->height = pFrame->height;
jpegContext->width = pFrame->width;
jpegContext->time_base = AVRational{ 1,10 };
ret = avcodec_open2(jpegContext, jpegCodec, NULL);
if (ret < 0) {
return ret;
}
FILE* JPEGFile;
char JPEGFName[256];
AVPacket packet;
packet.data = NULL;
packet.size = 0;
av_init_packet(&packet);
int gotFrame;
ret = avcodec_send_frame(jpegContext, pFrame);
if (ret < 0) {
return ret;
}
ret = avcodec_receive_packet(jpegContext, &packet);
if (ret < 0) {
return ret;
}
sprintf(JPEGFName, "c:\\folder\\dvr-%06d.jpg", FrameNo);
JPEGFile = fopen(JPEGFName, "wb");
fwrite(packet.data, 1, packet.size, JPEGFile);
fclose(JPEGFile);
av_packet_unref(&packet);
avcodec_close(jpegContext);
return 0;
}
int main(int argc, char** argv)
{
AVFrame* frame;
AVFrame* filt_frame;
AVPacket* packet;
int ret;
if (argc != 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
packet = av_packet_alloc();
if (!frame || !filt_frame || !packet) {
fprintf(stderr, "Could not allocate frame or packet\n");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
while (true)
{
if ((ret = av_read_frame(fmt_ctx, packet)) < 0)
break;
if (packet->stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0)
{
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
ret = save_frame_as_jpeg(dec_ctx, frame, dec_ctx->frame_number);
if (ret < 0)
goto end;
}
av_frame_unref(frame);
}
}
av_packet_unref(packet);
}
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
av_packet_free(&packet);
if (ret < 0 && ret != AVERROR_EOF) {
char errBuf[AV_ERROR_MAX_STRING_SIZE]{0};
int res = av_strerror(ret, errBuf, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error: %s\n", errBuf);
exit(1);
}
exit(0);
}
Well nevermind, I just realized I had an error with the specified codec for JPEG decoding, if somenone's facing this issue you have to use:
*const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);*
instead of:
*const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_JPEG2000);*
and also add this line:
*jpegContext->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;*
before the call to avcodec_open2

Use ffmpeg muxer to MPEG2TS , audio stream can't play in vlc palayer

I use libavformat to encapsulate an h264 stream and an aac stream into an mp4 file which is playable. However, when encapsulated into a ts file, it works fine in the Win10 player, but no audio in the vlc player. When encapsulating, the audio stream is printed, but with fprobe, the audio stream is printed with channel=0. What could be the reason for this?
And h264 source file is no pts.So I caculate it by myself.
ffprobe print
ffmpeg print
Here is my code.
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
static void log_packet(const AVFormatContext* fmt_ctx, const AVPacket* pkt, const char* tag)
{
AVRational* time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
printf("%s num=%d den=%d\n", tag, time_base->num, time_base->den);
printf("%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
tag,
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
pkt->stream_index);
}
int main()
{
const char* in_filename_v = "test.h264";
const char* in_filename_a = "aoutput.aac";
const char* out_filename = "lol.ts";
//Video Input AVFormatContext
AVFormatContext* ifmt_ctx_v = NULL;
int ret = avformat_open_input(&ifmt_ctx_v, in_filename_v, 0, 0);
if (ret < 0)
{
fprintf(stderr, "Could not open input_v %s", in_filename_v);
return -1;
}
//Find Video Stream Info
ret = avformat_find_stream_info(ifmt_ctx_v, 0);
if (ret < 0)
{
fprintf(stderr, "Could not find input_v stream info");
return -1;
}
//Audio Input AVFormatContext
AVFormatContext* ifmt_ctx_a = NULL;
ret = avformat_open_input(&ifmt_ctx_a, in_filename_a, 0, 0);
if (ret < 0)
{
fprintf(stderr, "Could not open input_a %s", in_filename_a);
return -1;
}
//Find Audio Stream Info
ret = avformat_find_stream_info(ifmt_ctx_a, 0);
if (ret < 0)
{
fprintf(stderr, "Could not find input_a stream info");
return -1;
}
printf("===========Input Information==========\n");
av_dump_format(ifmt_ctx_v, 0, in_filename_v, 0);
av_dump_format(ifmt_ctx_a, 0, in_filename_a, 0);
printf("======================================\n");
//Output AVFormatContext
AVFormatContext* ofmt_ctx = NULL;
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
{
fprintf(stderr, "cannot alloc OutputFromat context!");
ret = AVERROR_UNKNOWN;
return -1;
}
AVOutputFormat* ofmt = ofmt_ctx->oformat;
//Alloc AVSTREAM
int istream_index_v = 0, istream_index_a = 0, ostream_index_v = 0, ostream_index_a = 0;
for (int i = 0; i < ifmt_ctx_v->nb_streams; i++)
{
AVStream* outstream;
AVStream* in_stream = ifmt_ctx_v->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
continue;
outstream = avformat_new_stream(ofmt_ctx, NULL);
if (!outstream)
{
fprintf(stderr, "Failed allocating output stream\n");
return -1;
}
ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
if (ret < 0)
{
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
outstream->codecpar->codec_tag = 0;
// Remeber video stream id
istream_index_v = i;
ostream_index_v = 0;
break;
}
for (int i = 0; i < ifmt_ctx_a->nb_streams; i++)
{
AVStream* outstream;
AVStream* in_stream = ifmt_ctx_a->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO)
continue;
outstream = avformat_new_stream(ofmt_ctx, NULL);
if (!outstream)
{
fprintf(stderr, "Failed allocating output stream\n");
return -1;
}
ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
if (ret < 0)
{
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
outstream->codecpar->codec_tag = 0;
// Remeber audio stream id
istream_index_a = i;
ostream_index_a = 1;
break;
}
printf("===========Output Information==========\n");
av_dump_format(ofmt_ctx, 0, out_filename, 1);
printf("======================================\n");
if (!(ofmt->flags & AVFMT_NOFILE))
{
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
fprintf(stderr, "Could not open output file '%s'", out_filename);
return -1;
}
}
//Write file header
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
return -1;
}
//read and write packet
AVPacket* pkt = av_packet_alloc();
if (!pkt)
{
fprintf(stderr, "Could not allocate AVPacket\n");
return -1;
}
while (1)
{
AVStream* in_stream, * outstream;
ret = av_read_frame(ifmt_ctx_v, pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx_v->streams[pkt->stream_index];
if (pkt->stream_index != istream_index_v)
{
av_packet_unref(pkt);
continue;
}
pkt->stream_index = ostream_index_v;
outstream = ofmt_ctx->streams[pkt->stream_index];
// in log info
log_packet(ifmt_ctx_v, pkt, "in");
if (pkt->pts == AV_NOPTS_VALUE)
{
AVRational time_base1 = in_stream->time_base;
//
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
static int frame_index = 0;
pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
pkt->dts = pkt->pts;
pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
frame_index++;
}
// duration between two frames(us)
av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
pkt->pos = -1;
// out log info
log_packet(ofmt_ctx, pkt, "out");
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
}
while (1)
{
AVStream* in_stream, * outstream;
ret = av_read_frame(ifmt_ctx_a, pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx_a->streams[pkt->stream_index];
if (pkt->stream_index != istream_index_a)
{
av_packet_unref(pkt);
continue;
}
if (pkt->pts == AV_NOPTS_VALUE)
{
AVRational time_base1 = in_stream->time_base;
// duration between two frames(us)
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
static int frame_index = 0;
pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
pkt->dts = pkt->pts;
pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
frame_index++;
}
// in log info
log_packet(ifmt_ctx_a, pkt, "in");
pkt->stream_index = ostream_index_a;
outstream = ofmt_ctx->streams[pkt->stream_index];
//change timestamp
av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
pkt->pos = -1;
// out log info
log_packet(ofmt_ctx, pkt, "out");
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
}
//write file trailer
av_write_trailer(ofmt_ctx);
printf("===========Output Information==========\n");
av_dump_format(ofmt_ctx, 0, out_filename, 1);
printf("======================================\n");
}
DVB Insepctor video
DVB Insepctor audio
Thanks for #aergistal.
The Reason is that av_interleaved_write_frame has buffer limit.I
hadn't thought about this before so I write all video packages
firstly and then write all audio packages.In ts files, at front are
lots of video packages, followed by lots of audio packages, and
finally both packages interleaved.
Because of MPEG-TS is a stream consists of packages , so in a long time the player can't find audio packages resulting no voice.
Here is my new code that can work.
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#define EXTRAL 1
static void log_packet(const AVFormatContext* fmt_ctx, const AVPacket* pkt, const char* tag)
{
AVRational* time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
FILE* fp = fopen("fflog.log", "a+");
char buf[200];
sprintf(buf, "%s num=%d den=%d\n", tag, time_base->num, time_base->den);
for (int i = 0; *(buf + i) != '\0'; i++)
{
fwrite(buf + i, 1, 1, fp);
}
sprintf(buf, "%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
tag,
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
pkt->stream_index);
for (int i = 0; *(buf + i) != '\0'; i++)
{
fwrite(buf + i, 1, 1, fp);
}
fclose(fp);
}
int main()
{
const char* in_filename_v = "test.h264";
const char* in_filename_a = "aoutput.aac";
const char* out_filename = "lol.ts";
AVFormatContext* ifmt_ctx_v = NULL;
int ret = avformat_open_input(&ifmt_ctx_v, in_filename_v, 0, 0);
if (ret < 0)
{
fprintf(stderr, "Could not open input_v %s", in_filename_v);
return -1;
}
ret = avformat_find_stream_info(ifmt_ctx_v, 0);
if (ret < 0)
{
fprintf(stderr, "Could not find input_v stream info");
return -1;
}
AVFormatContext* ifmt_ctx_a = NULL;
ret = avformat_open_input(&ifmt_ctx_a, in_filename_a, 0, 0);
if (ret < 0)
{
fprintf(stderr, "Could not open input_a %s", in_filename_a);
return -1;
}
ret = avformat_find_stream_info(ifmt_ctx_a, 0);
if (ret < 0)
{
fprintf(stderr, "Could not find input_a stream info");
return -1;
}
#if EXTRAL
printf("===========Input Information==========\n");
av_dump_format(ifmt_ctx_v, 0, in_filename_v, 0);
av_dump_format(ifmt_ctx_a, 0, in_filename_a, 0);
printf("======================================\n");
#endif
AVFormatContext* ofmt_ctx = NULL;
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
{
fprintf(stderr, "cannot alloc OutputFromat context!");
ret = AVERROR_UNKNOWN;
return -1;
}
AVOutputFormat* ofmt = ofmt_ctx->oformat;
int istream_index_v = 0, istream_index_a = 0, ostream_index_v = 0, ostream_index_a = 0;
for (int i = 0; i < ifmt_ctx_v->nb_streams; i++)
{
AVStream* outstream;
AVStream* in_stream = ifmt_ctx_v->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
continue;
outstream = avformat_new_stream(ofmt_ctx, NULL);
if (!outstream)
{
fprintf(stderr, "Failed allocating output stream\n");
return -1;
}
ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
if (ret < 0)
{
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
outstream->codecpar->codec_tag = 0;
istream_index_v = i;
ostream_index_v = 0;
break;
}
for (int i = 0; i < ifmt_ctx_a->nb_streams; i++)
{
AVStream* outstream;
AVStream* in_stream = ifmt_ctx_a->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO)
continue;
outstream = avformat_new_stream(ofmt_ctx, NULL);
if (!outstream)
{
fprintf(stderr, "Failed allocating output stream\n");
return -1;
}
ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
if (ret < 0)
{
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
outstream->codecpar->codec_tag = 0;
istream_index_a = i;
ostream_index_a = 1;
break;
}
#if EXTRAL
printf("===========Output Information==========\n");
av_dump_format(ofmt_ctx, 0, out_filename, 1);
printf("======================================\n");
#endif
if (!(ofmt->flags & AVFMT_NOFILE))
{
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
fprintf(stderr, "Could not open output file '%s'", out_filename);
return -1;
}
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
return -1;
}
AVPacket* pkt = av_packet_alloc();
if (!pkt)
{
fprintf(stderr, "Could not allocate AVPacket\n");
return -1;
}
int64_t pts_v = 0, pts_a = 0;
while (1)
{
if (av_compare_ts(pts_a, ifmt_ctx_a->streams[istream_index_a]->time_base, pts_v, ifmt_ctx_v->streams[istream_index_v]->time_base) <= 0)
{
AVStream* in_stream, * outstream;
ret = av_read_frame(ifmt_ctx_a, pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx_a->streams[pkt->stream_index];
if (pkt->stream_index != istream_index_a)
{
av_packet_unref(pkt);
continue;
}
if (pkt->pts == AV_NOPTS_VALUE)
{
AVRational time_base1 = in_stream->time_base;
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
static int frame_index = 0;
pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
pkt->dts = pkt->pts;
pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
frame_index++;
}
pts_a = pkt->pts;
// in log info
log_packet(ifmt_ctx_a, pkt, "in audio");
pkt->stream_index = ostream_index_a;
outstream = ofmt_ctx->streams[pkt->stream_index];
av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
pkt->pos = -1;
// out log info
log_packet(ofmt_ctx, pkt, "out audio");
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
return -1;
}
}
else
{
AVStream* in_stream, * outstream;
ret = av_read_frame(ifmt_ctx_v, pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx_v->streams[pkt->stream_index];
if (pkt->stream_index != istream_index_v)
{
av_packet_unref(pkt);
continue;
}
pkt->stream_index = ostream_index_v;
outstream = ofmt_ctx->streams[pkt->stream_index];
if (pkt->pts == AV_NOPTS_VALUE)
{
AVRational time_base1 = in_stream->time_base;
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
static int frame_index = 0;
pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
pkt->dts = pkt->pts;
pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
frame_index++;
}
pts_v = pkt->pts;
// in log info
log_packet(ifmt_ctx_v, pkt, "in video");
av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
pkt->pos = -1;
// out log info
log_packet(ofmt_ctx, pkt, "out video");
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
return -1;
}
}
}
ret = av_write_trailer(ofmt_ctx);
if (ret < 0)
{
fprintf(stderr, "Error av_write_trailer\n");
}
#if EXTRAL
printf("===========Output Information==========\n");
av_dump_format(ofmt_ctx, 0, out_filename, 1);
printf("======================================\n");
#endif
av_packet_free(&pkt);
avformat_close_input(&ifmt_ctx_v);
avformat_close_input(&ifmt_ctx_a);
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
avio_closep(&ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
return 0;
}

ffmpeg record change color

Im try record video from screen, for write to video file im use ffmpeg(libavcodec). But on result i see other colors, my example:
AVCodec *codec;
AVCodecContext *c= NULL;
AVStream *video_stream;
AVOutputFormat *out;
AVFormatContext *out_context;
int i, ret, x, y, got_output;
AVFrame *frame;
AVPacket pkt;
uint8_t endcode[] = { 0, 0, 1, 0xb7 };
printf("Encode video file %s\n", filename);
out = av_guess_format(NULL, filename, NULL);
if (!out) {
std::cout << "Could not deduce output format from file extension: using MPEG.\n" << filename << std::endl;
out = av_guess_format("mpeg", filename, NULL);
}
if (!out) {
std::cout << "Could not find suitable output format\n" << std::endl;
return;
}
out->video_codec = (AVCodecID)codec_id;
out_context = avformat_alloc_context();
if (!out_context) {
std::cout << "Memory error\n";
return;
}
out_context->oformat = out;
codec = avcodec_find_encoder((AVCodecID)codec_id);
if (!codec) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
video_stream = avformat_new_stream(out_context, codec);
if (!video_stream) {
std::cout << "Could not alloc stream\n";
return;
}
c = video_stream->codec;
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
c->bit_rate = 20000000;
c->width = rect_width;
c->height = rect_height;
c->time_base = (AVRational){1,25};
c->gop_size = 10;
c->max_b_frames = 1;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if (codec_id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
avio_open2(&out_context->pb, filename, AVIO_FLAG_WRITE, NULL, NULL );
avformat_write_header(out_context, 0);
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = c->pix_fmt;
frame->width = c->width;
frame->height = c->height;
ret = av_image_alloc(frame->data, frame->linesize, c->width, c->height,
c->pix_fmt, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate raw picture buffer\n");
exit(1);
}
fprintf(stderr, "*0\n");
SwsContext *m_imageConvertContext = 0;
fprintf(stderr, "*0.1\n");
printf("size %3d %5d\n", rect_width, rect_height);
if ((rect_width % 4 != 0 && rect_width % 8 != 0 && rect_width % 16 != 0)
|| (rect_height % 4 != 0 && rect_height % 8 != 0 && rect_height % 16 != 0)) {
fprintf(stderr, "Video size dimensions must be multiple of 4,8 or 16.");
return;
}
m_imageConvertContext = sws_getCachedContext(m_imageConvertContext, frame->width, frame->height,AV_PIX_FMT_0RGB32, frame->width, frame->height, c->pix_fmt, SWS_BILINEAR, NULL, NULL, NULL);
fprintf(stderr, "*0.2\n");
int pts = 0;
for (i = 0; i < 25 * 20; i++) {
QImage image = QGuiApplication::primaryScreen()->grabWindow(0, rect_x, rect_y, rect_width, rect_height).toImage().convertToFormat(QImage::Format_RGB32);
pts ++;
av_init_packet(&pkt);
pkt.data = NULL; // packet data will be allocated by the encoder
pkt.size = 0;
fflush(stdout);
uint8_t *srcplanes[AV_NUM_DATA_POINTERS];
srcplanes[0]=(uint8_t*)image.bits();
srcplanes[1]=0;
srcplanes[2]=0;
srcplanes[3]=0;
srcplanes[4]=0;
srcplanes[5]=0;
srcplanes[6]=0;
srcplanes[7]=0;
int srcstride[AV_NUM_DATA_POINTERS];
srcstride[0]=image.bytesPerLine();
srcstride[1]=0;
srcstride[2]=0;
srcstride[3]=0;
srcstride[4]=0;
srcstride[5]=0;
srcstride[6]=0;
srcstride[7]=0;
int res = sws_scale(m_imageConvertContext, srcplanes, srcstride, 0,frame->height, frame->data, frame->linesize);
frame->pts = pts;
ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
printf("Write frame %3d (size=%5d)\n", i, pkt.size);
av_write_frame(out_context,&pkt);
av_free_packet(&pkt);
}
}
for (got_output = 1; got_output; i++) {
fflush(stdout);
ret = avcodec_encode_video2(c, &pkt, NULL, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
printf("Write frame %3d (size=%5d)\n", i, pkt.size);
av_interleaved_write_frame(out_context, &pkt);
av_free_packet(&pkt);
}
}
av_write_trailer(out_context);
avio_close(out_context->pb);
avcodec_close(c);
av_free(c);
av_freep(&frame->data[0]);
av_frame_free(&frame);
resolution 1920x1080. On original display look this http://joxi.ru/p27bbn6u093elm
on save video colors is a dimmer http://joxi.ru/YmE99W1fZvbdWm
It seems, white background color changes to gray. Or may be it is brightness. What im do is wrong? How i can save to color ?
UDP
im try use
int *inv_table, srcRange, *table, dstRange , brightness, contrast, saturation;
int ret = sws_getColorspaceDetails(m_imageConvertContext, &inv_table, &srcRange, &table, &dstRange, &brightness, &contrast, &saturation);
sws_setColorspaceDetails(m_imageConvertContext, sws_getCoefficients(SWS_CS_DEFAULT), srcRange, sws_getCoefficients(SWS_CS_ITU709), dstRange, brightness, contrast, saturation);
but this is never change

Why am I getting blips when encoding a sound file using Java JNA?

I have implemented a hello world libavcodec using JNA to generate a wav file containing a pure 440Hz sine wave. But when I actually run the program the wav file contains annoying clicks and blips (compare to pure sin wav created from the C program). How am I calling avcodec_encode_audio2 wrong?
Here is my Java code. All the sources are also at github in case you want to try to compile it.
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.IntBuffer;
import java.util.Objects;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.LineUnavailableException;
import javax.sound.sampled.TargetDataLine;
public class Sin {
/**
* Abstract class that allows you to put the initialization and cleanup
* code at the same place instead of separated by the big try block.
*/
public static abstract class SharedPtr<T> implements AutoCloseable {
public T ptr;
public SharedPtr(T ptr) {
this.ptr = ptr;
}
/**
* Abstract override forces method to throw no checked exceptions.
* Subclasses will call a C function that throws no exceptions.
*/
#Override public abstract void close();
}
/**
* #param args
* #throws IOException
* #throws LineUnavailableException
*/
public static void main(String[] args) throws IOException, LineUnavailableException {
final AvcodecLibrary avcodec = AvcodecLibrary.INSTANCE;
final AvformatLibrary avformat = AvformatLibrary.INSTANCE;
final AvutilLibrary avutil = AvutilLibrary.INSTANCE;
avcodec.avcodec_register_all();
avformat.av_register_all();
AVOutputFormat.ByReference format = null;
String format_name = "wav", file_url = "file:sinjava.wav";
for (AVOutputFormat.ByReference formatIter = avformat.av_oformat_next(null); formatIter != null; formatIter = avformat.av_oformat_next(formatIter)) {
formatIter.setAutoWrite(false);
String iterName = formatIter.name;
if (format_name.equals(iterName)) {
format = formatIter;
break;
}
}
Objects.requireNonNull(format);
System.out.format("Found format %s%n", format_name);
AVCodec codec = avcodec.avcodec_find_encoder(format.audio_codec); // one of AvcodecLibrary.CodecID
Objects.requireNonNull(codec);
codec.setAutoWrite(false);
try (
SharedPtr<AVFormatContext> fmtCtxPtr = new SharedPtr<AVFormatContext>(avformat.avformat_alloc_context()) {#Override public void close(){if (null!=ptr) avformat.avformat_free_context(ptr);}};
) {
AVFormatContext fmtCtx = Objects.requireNonNull(fmtCtxPtr.ptr);
fmtCtx.setAutoWrite(false);
fmtCtx.setAutoRead(false);
fmtCtx.oformat = format; fmtCtx.writeField("oformat");
AVStream st = avformat.avformat_new_stream(fmtCtx, codec);
if (null == st)
throw new IllegalStateException();
AVCodecContext c = st.codec;
if (null == c)
throw new IllegalStateException();
st.setAutoWrite(false);
fmtCtx.readField("nb_streams");
st.id = fmtCtx.nb_streams - 1; st.writeField("id");
assert st.id >= 0;
System.out.format("New stream: id=%d%n", st.id);
if (0 != (format.flags & AvformatLibrary.AVFMT_GLOBALHEADER)) {
c.flags |= AvcodecLibrary.CODEC_FLAG_GLOBAL_HEADER;
}
c.writeField("flags");
c.bit_rate = 64000; c.writeField("bit_rate");
int bestSampleRate;
if (null == codec.supported_samplerates) {
bestSampleRate = 44100;
} else {
bestSampleRate = 0;
for (int offset = 0, sample_rate = codec.supported_samplerates.getInt(offset); sample_rate != 0; codec.supported_samplerates.getInt(++offset)) {
bestSampleRate = Math.max(bestSampleRate, sample_rate);
}
assert bestSampleRate > 0;
}
c.sample_rate = bestSampleRate; c.writeField("sample_rate");
c.channel_layout = AvutilLibrary.AV_CH_LAYOUT_STEREO; c.writeField("channel_layout");
c.channels = avutil.av_get_channel_layout_nb_channels(c.channel_layout); c.writeField("channels");
assert 2 == c.channels;
c.sample_fmt = AvutilLibrary.AVSampleFormat.AV_SAMPLE_FMT_S16; c.writeField("sample_fmt");
c.time_base.num = 1;
c.time_base.den = bestSampleRate;
c.writeField("time_base");
c.setAutoWrite(false);
AudioFormat javaSoundFormat = new AudioFormat(bestSampleRate, Short.SIZE, c.channels, true, ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN);
DataLine.Info javaDataLineInfo = new DataLine.Info(TargetDataLine.class, javaSoundFormat);
if (! AudioSystem.isLineSupported(javaDataLineInfo))
throw new IllegalStateException();
int err;
if ((err = avcodec.avcodec_open(c, codec)) < 0) {
throw new IllegalStateException();
}
assert c.channels != 0;
AVIOContext.ByReference[] ioCtxReference = new AVIOContext.ByReference[1];
if (0 != (err = avformat.avio_open(ioCtxReference, file_url, AvformatLibrary.AVIO_FLAG_WRITE))) {
throw new IllegalStateException("averror " + err);
}
try (
SharedPtr<AVIOContext.ByReference> ioCtxPtr = new SharedPtr<AVIOContext.ByReference>(ioCtxReference[0]) {#Override public void close(){if (null!=ptr) avutil.av_free(ptr.getPointer());}}
) {
AVIOContext.ByReference ioCtx = Objects.requireNonNull(ioCtxPtr.ptr);
fmtCtx.pb = ioCtx; fmtCtx.writeField("pb");
int averr = avformat.avformat_write_header(fmtCtx, null);
if (averr < 0) {
throw new IllegalStateException("" + averr);
}
st.read(); // it is modified by avformat_write_header
System.out.format("Wrote header. fmtCtx->nb_streams=%d, st->time_base=%d/%d; st->avg_frame_rate=%d/%d%n", fmtCtx.nb_streams, st.time_base.num, st.time_base.den, st.avg_frame_rate.num, st.avg_frame_rate.den);
avformat.avio_flush(ioCtx);
int frame_size = c.frame_size != 0 ? c.frame_size : 4096;
int expectedBufferSize = frame_size * c.channels * (Short.SIZE/8);
boolean supports_small_last_frame = c.frame_size == 0 ? true : 0 != (codec.capabilities & AvcodecLibrary.CODEC_CAP_SMALL_LAST_FRAME);
int bufferSize = avutil.av_samples_get_buffer_size((IntBuffer)null, c.channels, frame_size, c.sample_fmt, 1);
assert bufferSize == expectedBufferSize: String.format("expected %d; got %d", expectedBufferSize, bufferSize);
ByteBuffer samples = ByteBuffer.allocate(expectedBufferSize);
samples.order(ByteOrder.nativeOrder());
int audio_time = 0; // unit: (c.time_base) s = (1/c.sample_rate) s
int audio_sample_count = supports_small_last_frame ?
3 * c.sample_rate :
3 * c.sample_rate / frame_size * frame_size;
while (audio_time < audio_sample_count) {
int frame_audio_time = audio_time;
samples.clear();
int nb_samples_in_frame = 0;
// encode a single tone sound
for (; samples.hasRemaining() && audio_time < audio_sample_count; nb_samples_in_frame++, audio_time++) {
double x = 2*Math.PI*440/c.sample_rate * audio_time;
double y = 10000 * Math.sin(x);
samples.putShort((short) y);
samples.putShort((short) y);
}
samples.flip();
try (
SharedPtr<AVFrame> framePtr = new SharedPtr<AVFrame>(avcodec.avcodec_alloc_frame()) {#Override public void close() {if (null!=ptr) avutil.av_free(ptr.getPointer());}};
) {
AVFrame frame = Objects.requireNonNull(framePtr.ptr);
frame.setAutoRead(false); // will be an in param
frame.setAutoWrite(false);
frame.nb_samples = nb_samples_in_frame; frame.writeField("nb_samples"); // actually unused during encoding
// Presentation time, in AVStream.time_base units.
frame.pts = avutil.av_rescale_q(frame_audio_time, c.time_base, st.time_base); // i * codec_time_base / st_time_base
frame.writeField("pts");
assert c.channels > 0;
int bytesPerSample = avutil.av_get_bytes_per_sample(c.sample_fmt);
assert bytesPerSample > 0;
if (0 != (err = avcodec.avcodec_fill_audio_frame(frame, c.channels, c.sample_fmt, samples, samples.capacity(), 1))) {
throw new IllegalStateException(""+err);
}
AVPacket packet = new AVPacket(); // one of the few structs from ffmpeg with guaranteed size
avcodec.av_init_packet(packet);
packet.size = 0;
packet.data = null;
packet.stream_index = st.index; packet.writeField("stream_index");
// encode the samples
IntBuffer gotPacket = IntBuffer.allocate(1);
if (0 != (err = avcodec.avcodec_encode_audio2(c, packet, frame, gotPacket))) {
throw new IllegalStateException("" + err);
} else if (0 != gotPacket.get()) {
packet.read();
averr = avformat.av_write_frame(fmtCtx, packet);
if (averr < 0)
throw new IllegalStateException("" + averr);
}
System.out.format("encoded frame: codec time = %d; pts=%d = av_rescale_q(%d,%d/%d,%d/%d) (%.02fs) contains %d samples (%.02fs); got_packet=%d; packet.size=%d%n",
frame_audio_time,
frame.pts,
frame_audio_time, st.codec.time_base.num,st.codec.time_base.den,st.time_base.num,st.time_base.den,
1.*frame_audio_time/c.sample_rate, frame.nb_samples, 1.*frame.nb_samples/c.sample_rate, gotPacket.array()[0], packet.size);
}
}
if (0 != (err = avformat.av_write_trailer(fmtCtx))) {
throw new IllegalStateException();
}
avformat.avio_flush(ioCtx);
}
}
System.out.println("Done writing");
}
}
I also rewrote it in C, and the C version works fine without any blips. But I can’t figure out how I am using the library differently; all the library function calls should be identical!
//! gcc --std=c99 sin.c $(pkg-config --cflags --libs libavutil libavformat libavcodec) -o sin
// sudo apt-get install libswscale-dev
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include <libavutil/opt.h>
#include <libavutil/mathematics.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavcodec/avcodec.h>
int main(int argc, char *argv[]) {
const char *format_name = "wav", *file_url = "file:sin.wav";
avcodec_register_all();
av_register_all();
AVOutputFormat *format = NULL;
for (AVOutputFormat *formatIter = av_oformat_next(NULL); formatIter != NULL; formatIter = av_oformat_next(formatIter)) {
int hasEncoder = NULL != avcodec_find_encoder(formatIter->audio_codec);
if (0 == strcmp(format_name, formatIter->name)) {
format = formatIter;
break;
}
}
printf("Found format %s\n", format->name);
AVCodec *codec = avcodec_find_encoder(format->audio_codec);
if (! codec) {
fprintf(stderr, "Could not find codec %d\n", format->audio_codec);
exit(1);
}
AVFormatContext *fmtCtx = avformat_alloc_context();
if (! fmtCtx) {
fprintf(stderr, "error allocating AVFormatContext\n");
exit(1);
}
fmtCtx->oformat = format;
AVStream *st = avformat_new_stream(fmtCtx, codec);
if (! st) {
fprintf(stderr, "error allocating AVStream\n");
exit(1);
}
if (fmtCtx->nb_streams != 1) {
fprintf(stderr, "avformat_new_stream should have incremented nb_streams, but it's still %d\n", fmtCtx->nb_streams);
exit(1);
}
AVCodecContext *c = st->codec;
if (! c) {
fprintf(stderr, "avformat_new_stream should have allocated a AVCodecContext for my stream\n");
exit(1);
}
st->id = fmtCtx->nb_streams - 1;
printf("Created stream %d\n", st->id);
if (0 != (format->flags & AVFMT_GLOBALHEADER)) {
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
c->bit_rate = 64000;
int bestSampleRate;
if (NULL == codec->supported_samplerates) {
bestSampleRate = 44100;
printf("Setting sample rate: %d\n", bestSampleRate);
} else {
bestSampleRate = 0;
for (const int *sample_rate_iter = codec->supported_samplerates; *sample_rate_iter != 0; sample_rate_iter++) {
if (*sample_rate_iter >= bestSampleRate)
bestSampleRate = *sample_rate_iter;
}
printf("Using best supported sample rate: %d\n", bestSampleRate);
}
c->sample_rate = bestSampleRate;
c->channel_layout = AV_CH_LAYOUT_STEREO;
c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
c->time_base.num = 1;
c->time_base.den = c->sample_rate;
if (c->channels != 2) {
fprintf(stderr, "av_get_channel_layout_nb_channels returned %d instead of 2\n", c->channels);
exit(1);
}
c->sample_fmt = AV_SAMPLE_FMT_S16;
int averr;
if ((averr = avcodec_open2(c, codec, NULL)) < 0) {
fprintf(stderr, "avcodec_open2 returned error %d\n", averr);
exit(1);
}
AVIOContext *ioCtx = NULL;
if (0 != (averr = avio_open(&ioCtx, file_url, AVIO_FLAG_WRITE))) {
fprintf(stderr, "avio_open returned error %d\n", averr);
exit(1);
}
if (ioCtx == NULL) {
fprintf(stderr, "AVIOContext should have been set by avio_open\n");
exit(1);
}
fmtCtx->pb = ioCtx;
if (0 != (averr = avformat_write_header(fmtCtx, NULL))) {
fprintf(stderr, "avformat_write_header returned error %d\n", averr);
exit(1);
}
printf("Wrote header. fmtCtx->nb_streams=%d, st->time_base=%d/%d; st->avg_frame_rate=%d/%d\n", fmtCtx->nb_streams, st->time_base.num, st->time_base.den, st->avg_frame_rate.num, st->avg_frame_rate.den);
int align = 1;
int sample_size = av_get_bytes_per_sample(c->sample_fmt);
if (sample_size != sizeof(int16_t)) {
fprintf(stderr, "expected sample size=%zu but got %d\n", sizeof(int16_t), sample_size);
exit(1);
}
int frame_size = c->frame_size != 0 ? c->frame_size : 4096;
int bufferSize = av_samples_get_buffer_size(NULL, c->channels, frame_size, c->sample_fmt, align);
int expectedBufferSize = frame_size * c->channels * sample_size;
int supports_small_last_frame = c->frame_size == 0 ? 1 : 0 != (codec->capabilities & CODEC_CAP_SMALL_LAST_FRAME);
if (bufferSize != expectedBufferSize) {
fprintf(stderr, "expected buffer size=%d but got %d\n", expectedBufferSize, bufferSize);
exit(1);
}
int16_t *samples = (int16_t*)malloc(bufferSize);
uint32_t audio_time = 0; // unit: (1/c->sample_rate) s
uint32_t audio_sample_count = supports_small_last_frame ?
3 * c->sample_rate :
3 * c->sample_rate / frame_size * frame_size;
while (audio_time < audio_sample_count) {
uint32_t frame_audio_time = audio_time; // unit: (1/c->sample_rate) s
AVFrame *frame = avcodec_alloc_frame();
if (frame == NULL) {
fprintf(stderr, "avcodec_alloc_frame failed\n");
exit(1);
}
for (uint32_t i = 0; i != frame_size && audio_time < audio_sample_count; i++, audio_time++) {
samples[2*i] = samples[2*i + 1] = 10000 * sin(2*M_PI*440/c->sample_rate * audio_time);
frame->nb_samples = i+1; // actually unused during encoding
}
// frame->format = c->sample_fmt; // unused during encoding
frame->pts = av_rescale_q(frame_audio_time, c->time_base, st->time_base);
if (0 != (averr = avcodec_fill_audio_frame(frame, c->channels, c->sample_fmt, (const uint8_t*)samples, bufferSize, align))) {
fprintf(stderr, "avcodec_fill_audio_frame returned error %d\n", averr);
exit(1);
}
AVPacket packet;
av_init_packet(&packet);
packet.data = NULL;
packet.size = 0;
int got_packet;
if (0 != (averr = avcodec_encode_audio2(c, &packet, frame, &got_packet))) {
fprintf(stderr, "avcodec_encode_audio2 returned error %d\n", averr);
exit(1);
}
if (got_packet) {
packet.stream_index = st->index;
if (0 < (averr = av_write_frame(fmtCtx, &packet))) {
fprintf(stderr, "av_write_frame returned error %d\n", averr);
exit(1);
} else if (averr == 1) {
// end of stream wanted.
}
}
printf("encoded frame: codec time = %u; format pts=%ld = av_rescale_q(%u,%d/%d,%d/%d) (%.02fs) contains %d samples (%.02fs); got_packet=%d; packet.size=%d\n",
frame_audio_time,
frame->pts,
frame_audio_time, c->time_base.num, c->time_base.den, st->time_base.num, st->time_base.den,
1.*frame_audio_time/c->sample_rate, frame->nb_samples, 1.*frame->nb_samples/c->sample_rate, got_packet, packet.size);
av_free(frame);
}
free(samples);
cleanupFile:
if (0 != (averr = av_write_trailer(fmtCtx))) {
fprintf(stderr, "av_write_trailer returned error %d\n", averr);
exit(1);
}
avio_flush(ioCtx);
avio_close(ioCtx);
avformat_free_context(fmtCtx);
}
The problem was that ByteBuffer.allocate(int) creates a buffer whose address is not stable across JNA function calls. Every time you call a native function, it copies the bytes into a temporary array just for that invocation. By contrast, ByteBuffer.allocateDirect(int) creates a buffer whose native pointers are stable. This is apparently a well-known pitfall of using ByteBuffer in JNA, but I didn’t notice it in the fine print of Using Pointers and Arrays.
So I just had to fix the samples creation to ByteBuffer samples = ByteBuffer.allocateDirect(expectedBufferSize);. The subsequent avcodec_fill_audio_frame call does not copy samples; it simply points the frame->data[0] to the uint8_t* address, so the samples array needs to have a stable address.
Without having done what you are doing, I suspect the garbage collector.
See How can I disable Java garbage collector? - it says you can't, so increase the memory.

Audio encoding using avcodec_fill_audio_frame() and memory leaks

As a part of encoding decoded audio packets, I'm using avcodec_fill_audio_frame(). I'm passing allocated AVFrame pointer to along with buffer containing the decoded samples and other parameters number of channels, sample format, buffer size. Though the encoding is working fine I'm not able to completely eliminate the memory leaks. I've taken care of most of things but still I'm not able detect the leakage.
Below is the function which I'm using for encoding. Please suggest something.
AudioSample contains decoded data and it is completely managed in different class(free in class destructor). I'm freeing the AVFrame in FFmpegEncoder destructor and AVPacket is freed every time using av_free_packet() with av_packet_destruct enabled. What more do I need to free?
void FfmpegEncoder::WriteAudioSample(AudioSample *audS)
{
int num_audio_frame = 0;
AVCodecContext *c = NULL;
// AVFrame *frame;
AVPacket pkt;
av_init_packet(&pkt);
pkt.destruct = av_destruct_packet;
pkt.data = NULL;
pkt.size = 0;
int ret = 0, got_packet = 0;
c = m_out_aud_strm->codec;
static int64_t aud_pts_in = -1;
if((audS != NULL) && (audS->GetSampleLength() > 0) )
{
int byte_per_sample = av_get_bytes_per_sample(c->sample_fmt);
PRINT_VAL("Byte Per Sample ", byte_per_sample)
m_frame->nb_samples = (audS->GetSampleLength())/(c->channels*av_get_bytes_per_sample(c->sample_fmt));
if(m_frame->nb_samples == c->frame_size)
{
#if 1
if(m_need_resample && (c->channels >= 2))
{
uint8_t * t_buff1 = new uint8_t[audS->GetSampleLength()];
if(t_buff1 != NULL)
{
for(int64_t i = 0; i< m_frame->nb_samples; i++)
{
memcpy(t_buff1 + i*byte_per_sample, (uint8_t*)((uint8_t*)audS->GetAudioSampleData() + i*byte_per_sample*c->channels), byte_per_sample);
memcpy(t_buff1 + (audS->GetSampleLength())/2 + i*byte_per_sample, (uint8_t*)((uint8_t*)audS->GetAudioSampleData() + i*byte_per_sample*c->channels+ byte_per_sample), byte_per_sample);
}
audS->FillAudioSample(t_buff1, audS->GetSampleLength());
delete[] t_buff1;
}
}
#endif
ret = avcodec_fill_audio_frame(m_frame, c->channels, c->sample_fmt, (uint8_t*)audS->GetAudioSampleData(),m_frame->nb_samples*byte_per_sample*c->channels, 0);
//ret = avcodec_fill_audio_frame(&frame, c->channels, c->sample_fmt, t_buff,frame.nb_samples*byte_per_sample*c->channels, 0);
if(ret != 0)
{
PRINT_MSG("Avcodec Fill Audio Failed ")
}
else
{
got_packet = 0;
ret = avcodec_encode_audio2(c, &pkt, m_frame, &got_packet);
if(ret < 0 || got_packet == 0)
{
PRINT_MSG("failed to encode audio ")
}
else
{
PRINT_MSG("Audio Packet Encoded ");
aud_pts_in++;
pkt.pts = aud_pts_in;
pkt.dts = pkt.pts;
pkt.stream_index = m_out_aud_strm->index;
ret = av_interleaved_write_frame(oc, &pkt);
if(ret != 0)
{
PRINT_MSG("Error Write Audio PKT ")
}
else
{
PRINT_MSG("Audio PKT Writen ")
}
}
}
}
avcodec_flush_buffers(c);
// avcodec_free_frame(&frame);
}
av_free_packet(&pkt);
}
Thanks,
Pradeep
//================== SEND AUDIO OUTPUT =======================
void AVOutputStream::sendAudioOutput (AVFrame* inputFrame)
{
AVCodecContext *codecCtx = pOutputAudioStream->codec;
// set source data variables
sourceNumberOfChannels = inputFrame->channels;
sourceChannelLayout = inputFrame->channel_layout;
sourceSampleRate = inputFrame->sample_rate;
_sourceSampleFormat = (AVSampleFormat)inputFrame->format;
sourceNumberOfSamples = inputFrame->nb_samples;
// set destination data variables
destinationNumberOfChannels = codecCtx->channels;
destinationChannelLayout = codecCtx->channel_layout;
destinationSampleRate = codecCtx->sample_rate;
destinationSampleFormat = codecCtx->sample_fmt;//AV_SAMPLE_FMT_FLTP;//EncodecCtx->sample_fmt;
destinationLineSize = 0;
destinationData = NULL;
int returnVal = 0;
if (startDecode == false)
{
startDecode = true;
resamplerCtx = swr_alloc_set_opts(NULL,
destinationChannelLayout,
destinationSampleFormat,
destinationSampleRate,
sourceChannelLayout,
_sourceSampleFormat,
sourceSampleRate,
0,
NULL);
if (resamplerCtx == NULL)
{
std::cout << "Unable to create the resampler context for the audio frame";
isConnected = false;
}
// initialize the resampling context
returnVal = swr_init(resamplerCtx);
if (returnVal < 0)
{
std::cout << "Unable to init the resampler context, error:";
isConnected = false;
}
} //if (startDecode == false)
if (sourceSampleRate != 0)
destinationNumberOfSamples = destinationSampleRate/sourceSampleRate * sourceNumberOfSamples;
// allocate the destination samples buffer
returnVal = av_samples_alloc_array_and_samples(&destinationData,
&destinationLineSize,
destinationNumberOfChannels,
destinationNumberOfSamples,
destinationSampleFormat,
0);
if (returnVal < 0)
{
std::cout << "Unable to allocate destination samples, error";
isConnected = false;
}
// convert to destination format
returnVal = swr_convert(resamplerCtx,
destinationData,
destinationNumberOfSamples,
(const uint8_t **)inputFrame->data, //sourceData,
sourceNumberOfSamples);
if (returnVal < 0)
{
std::cout << "Resampling failed, error \n";
isConnected = false;
}
int bufferSize = av_samples_get_buffer_size(&destinationLineSize,
destinationNumberOfChannels,
destinationNumberOfSamples,
destinationSampleFormat,
0);
//whithout fifo
pOutputAudioFrame = av_frame_alloc();
pOutputAudioFrame->nb_samples = codecCtx->frame_size;//frameNumberOfSamples;
pOutputAudioFrame->format = codecCtx->sample_fmt;
pOutputAudioFrame->channel_layout = codecCtx->channel_layout;
pOutputAudioFrame->channels = codecCtx->channels;
pOutputAudioFrame->sample_rate = codecCtx->sample_rate;
returnVal = avcodec_fill_audio_frame(pOutputAudioFrame,
pOutputAudioFrame->channels,
(AVSampleFormat)pOutputAudioFrame->format,
(const uint8_t *)destinationData[0],
bufferSize,0);
pOutputAudioFrame->pts = inputFrame->pts;
if (returnVal < 0)
{
std::cout << "Unable to fill the audio frame wsampleIndexith captured audio data,error";
isConnected = false;
}
// encode the audio frame, fill a packet for streaming
av_init_packet(&outAudioPacket);
outAudioPacket.data = NULL;
outAudioPacket.size = 0;
outAudioPacket.dts = outAudioPacket.pts = 0;
int gotPacket;
// encoding
returnVal = avcodec_encode_audio2(codecCtx, &outAudioPacket, pOutputAudioFrame, &gotPacket);
// free buffers
av_freep(&destinationData[0]);
av_freep(&destinationData);
av_frame_free(&pOutputAudioFrame);
if (gotPacket)
{
outAudioPacket.stream_index = pOutputAudioStream->index;
outAudioPacket.flags |= AV_PKT_FLAG_KEY;
returnVal = av_interleaved_write_frame(pOutputFormatCtx, &outAudioPacket);
//returnVal = av_write_frame(pOutputFormatCtx, &outAudioPacket);
if (returnVal != 0)
{
std::cout << "Cannot write audio packet \n";
isConnected = false;
}
av_free_packet(&outAudioPacket);
} // if (gotPacket)
}
You can see after resample i free used buffers.
// free buffers
av_freep(&destinationData[0]);
av_freep(&destinationData);

Resources